def _update_user(self, record):
email = record.get('author_email')
user_name = record.get('author_name')
user_e = user_processor.load_user(self.runtime_storage_inst,
email=email) or {}
gerrit_id = record.get('gerrit_id')
if gerrit_id:
user_g = user_processor.load_user(self.runtime_storage_inst,
gerrit_id=gerrit_id) or {}
else:
user_g = {}
if user_processor.are_users_same([user_e, user_g]):
# If sequence numbers are set and the same, merge is not needed
return user_e
user = user_processor.create_user(self.domains_index, email, gerrit_id,
user_name)
if user_e or user_g:
# merge between existing profiles and a new one
user, users_to_delete = user_processor.merge_user_profiles(
self.domains_index, [user_e, user_g, user])
# delete all unneeded profiles
user_processor.delete_users(self.runtime_storage_inst,
users_to_delete)
user_processor.store_user(self.runtime_storage_inst, user)
LOG.debug('Stored user: %s', user)
return user
def update_user(self, record):
email = record.get('author_email')
user_e = user_processor.load_user(
self.runtime_storage_inst, email=email) or {}
user_name = record.get('author_name')
launchpad_id = record.get('launchpad_id')
if (email and (not user_e) and (not launchpad_id) and
(not user_e.get('launchpad_id'))):
# query LP
launchpad_id, lp_user_name = self._get_lp_info(email)
if lp_user_name:
user_name = lp_user_name
gerrit_id = record.get('gerrit_id')
if gerrit_id:
user_g = user_processor.load_user(
self.runtime_storage_inst, gerrit_id=gerrit_id) or {}
if ((not user_g) and (not launchpad_id) and
(not user_e.get('launchpad_id'))):
# query LP
guessed_lp_id = gerrit_id
lp_user_name = self._get_lp_user_name(guessed_lp_id)
if lp_user_name == user_name:
launchpad_id = guessed_lp_id
else:
user_g = {}
user_l = user_processor.load_user(
self.runtime_storage_inst, launchpad_id=launchpad_id) or {}
if ((user_e.get('seq') == user_l.get('seq') == user_g.get('seq')) and
user_e.get('seq')):
# sequence numbers are set and the same, merge is not needed
user = user_e
else:
user = self._create_user(launchpad_id, email, gerrit_id, user_name)
if user_e or user_l or user_g:
user = self._merge_user_profiles(
[user_e, user_l, user_g, user])
else:
# create new
if not user_name:
user_name = self._get_lp_user_name(launchpad_id)
if user_name:
user['user_name'] = user_name
LOG.debug('Created new user: %s', user)
user_processor.store_user(self.runtime_storage_inst, user)
LOG.debug('Stored user: %s', user)
return user
def update_user(self, record):
email = record.get('author_email')
user_e = user_processor.load_user(self.runtime_storage_inst,
email=email) or {}
user_name = record.get('author_name')
launchpad_id = record.get('launchpad_id')
if (email and (not user_e) and (not launchpad_id)
and (not user_e.get('launchpad_id'))):
# query LP
launchpad_id, lp_user_name = self._get_lp_info(email)
if lp_user_name:
user_name = lp_user_name
gerrit_id = record.get('gerrit_id')
if gerrit_id:
user_g = user_processor.load_user(self.runtime_storage_inst,
gerrit_id=gerrit_id) or {}
if ((not user_g) and (not launchpad_id)
and (not user_e.get('launchpad_id'))):
# query LP
guessed_lp_id = gerrit_id
lp_user_name = self._get_lp_user_name(guessed_lp_id)
if lp_user_name == user_name:
launchpad_id = guessed_lp_id
else:
user_g = {}
user_l = user_processor.load_user(self.runtime_storage_inst,
launchpad_id=launchpad_id) or {}
if ((user_e.get('seq') == user_l.get('seq') == user_g.get('seq'))
and user_e.get('seq')):
# sequence numbers are set and the same, merge is not needed
user = user_e
else:
user = self._create_user(launchpad_id, email, gerrit_id, user_name)
if user_e or user_l or user_g:
user = self._merge_user_profiles(
[user_e, user_l, user_g, user])
else:
# create new
if not user_name:
user_name = self._get_lp_user_name(launchpad_id)
if user_name:
user['user_name'] = user_name
LOG.debug('Created new user: %s', user)
user_processor.store_user(self.runtime_storage_inst, user)
LOG.debug('Stored user: %s', user)
return user
def apply_user_corrections(self, user_corrections_iterator):
for user_correction in user_corrections_iterator:
stored_user = user_processor.load_user(
self, user_id=user_correction['user_id'])
updated_user = user_processor.update_user_profile(
stored_user, user_correction, is_correction=True)
user_processor.store_user(self, updated_user)
def _update_members_company_name(self):
LOG.debug('Update members with company names')
for record in self.runtime_storage_inst.get_all_records():
if record['record_type'] != 'member':
continue
company_draft = record['company_draft']
company_name = self.domains_index.get(
utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft))
if company_name == record['company_name']:
continue
LOG.debug('Update record %s, company name changed to %s',
record, company_name)
record['company_name'] = company_name
yield record
user = user_processor.load_user(self.runtime_storage_inst,
record['user_id'])
LOG.debug('Update user %s, company name changed to %s',
user, company_name)
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user_processor.store_user(self.runtime_storage_inst, user)
def get_user_from_runtime_storage(user_id):
runtime_storage_inst = get_runtime_storage()
user_index = get_vault()['user_index']
if user_id not in user_index:
user_index[user_id] = user_processor.load_user(runtime_storage_inst,
user_id=user_id)
return user_index[user_id]
def _update_members_company_name(self):
LOG.debug('Update members with company names')
for record in self.runtime_storage_inst.get_all_records():
if record['record_type'] != 'member':
continue
company_draft = record['company_draft']
company_name = self.domains_index.get(
utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft))
if company_name == record['company_name']:
continue
LOG.debug('Update record %s, company name changed to %s', record,
company_name)
record['company_name'] = company_name
yield record
user = user_processor.load_user(self.runtime_storage_inst,
record['user_id'])
LOG.debug('Update user %s, company name changed to %s', user,
company_name)
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user_processor.store_user(self.runtime_storage_inst, user)
def _process_member(self, record):
user_id = user_processor.make_user_id(member_id=record["member_id"])
record["primary_key"] = user_id
record["date"] = utils.member_date_to_timestamp(record["date_joined"])
record["author_name"] = record["member_name"]
record["module"] = "unknown"
company_draft = record["company_draft"]
company_name = self.domains_index.get(utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft)
)
# author_email is a key to create new user
record["author_email"] = user_id
record["company_name"] = company_name
# _update_record_and_user function will create new user if needed
self._update_record_and_user(record)
record["company_name"] = company_name
user = user_processor.load_user(self.runtime_storage_inst, user_id)
user["user_name"] = record["author_name"]
user["companies"] = [{"company_name": company_name, "end_date": 0}]
user["company_name"] = company_name
user_processor.store_user(self.runtime_storage_inst, user)
record["company_name"] = company_name
yield record
def get_user_from_runtime_storage(user_id):
runtime_storage_inst = get_runtime_storage()
user_index = get_vault()['user_index']
if user_id not in user_index:
user_index[user_id] = user_processor.load_user(
runtime_storage_inst, user_id=user_id)
return user_index[user_id]
def _process_member(self, record):
user_id = user_processor.make_user_id(member_id=record['member_id'])
record['primary_key'] = user_id
record['date'] = utils.member_date_to_timestamp(record['date_joined'])
record['author_name'] = record['member_name']
record['module'] = 'unknown'
company_draft = record['company_draft']
company_name = self.domains_index.get(
utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft))
# author_email is a key to create new user
record['author_email'] = user_id
record['company_name'] = company_name
# _update_record_and_user function will create new user if needed
self._update_record_and_user(record)
record['company_name'] = company_name
user = user_processor.load_user(self.runtime_storage_inst, user_id)
user['user_name'] = record['author_name']
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user['company_name'] = company_name
user_processor.store_user(self.runtime_storage_inst, user)
record['company_name'] = company_name
yield record
def record_handler(record):
if record['record_type'] != 'member':
return
company_draft = record['company_draft']
company_name = self.domains_index.get(
utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft))
if company_name == record['company_name']:
return
LOG.debug('Update record %s, company name changed to %s',
record, company_name)
record['company_name'] = company_name
yield record
user = user_processor.load_user(self.runtime_storage_inst,
record['user_id'])
LOG.debug('Update user %s, company name changed to %s',
user, company_name)
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user_processor.store_user(self.runtime_storage_inst, user)
def record_handler(record):
if record['record_type'] != 'member':
return
company_draft = record['company_draft']
company_name = self.domains_index.get(
utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft))
if company_name == record['company_name']:
return
LOG.debug('Update record %s, company name changed to %s', record,
company_name)
record['company_name'] = company_name
yield record
user = user_processor.load_user(self.runtime_storage_inst,
record['user_id'])
LOG.debug('Update user %s, company name changed to %s', user,
company_name)
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user_processor.store_user(self.runtime_storage_inst, user)
def _process_member(self, record):
user_id = user_processor.make_user_id(member_id=record['member_id'])
record['primary_key'] = user_id
record['date'] = utils.member_date_to_timestamp(record['date_joined'])
record['author_name'] = record['member_name']
record['module'] = 'unknown'
company_draft = record['company_draft']
company_name = self.domains_index.get(utils.normalize_company_name(
company_draft)) or (utils.normalize_company_draft(company_draft))
# author_email is a key to create new user
record['author_email'] = user_id
record['company_name'] = company_name
# _update_record_and_user function will create new user if needed
self._update_record_and_user(record)
record['company_name'] = company_name
user = user_processor.load_user(self.runtime_storage_inst, user_id)
user['user_name'] = record['author_name']
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user['company_name'] = company_name
user_processor.store_user(self.runtime_storage_inst, user)
record['company_name'] = company_name
yield record
def _store_users(runtime_storage_inst, users):
for user in users:
stored_user = user_processor.load_user(runtime_storage_inst,
user_id=user['user_id'])
if stored_user:
stored_user.update(user)
user = stored_user
user['static'] = True
user_processor.store_user(runtime_storage_inst, user)
def _store_users(runtime_storage_inst, users):
for user in users:
stored_user = user_processor.load_user(runtime_storage_inst,
user_id=user['user_id'])
if stored_user:
stored_user.update(user)
user = stored_user
user['static'] = True
user_processor.store_user(runtime_storage_inst, user)
def record_handler(record):
if record["record_type"] != "member":
return
company_draft = record["company_draft"]
company_name = self.domains_index.get(utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft)
)
if company_name == record["company_name"]:
return
LOG.debug("Update record %s, company name changed to %s", record, company_name)
record["company_name"] = company_name
yield record
user = user_processor.load_user(self.runtime_storage_inst, record["user_id"])
LOG.debug("Update user %s, company name changed to %s", user, company_name)
user["companies"] = [{"company_name": company_name, "end_date": 0}]
user_processor.store_user(self.runtime_storage_inst, user)
def test_process_commit_new_user_unknown(self):
record_processor_inst = self.make_record_processor(
companies=[{
'company_name': 'IBM',
'domains': ['ibm.com']
}])
processed_commit = list(
record_processor_inst.process(
generate_commits(author_email='*****@*****.**',
author_name='John Doe')))[0]
expected_commit = {
'author_email': '*****@*****.**',
'author_name': 'John Doe',
'company_name': 'IBM',
}
self.assertRecordsMatch(expected_commit, processed_commit)
user = user_processor.load_user(
record_processor_inst.runtime_storage_inst,
user_id='*****@*****.**')
self.assertIn('*****@*****.**', user['emails'])
self.assertEqual('IBM', user['companies'][0]['company_name'])
def _store_users(runtime_storage_inst, users):
for user in users:
stored_user = user_processor.load_user(runtime_storage_inst,
user_id=user['user_id'])
updated_user = user_processor.update_user_profile(stored_user, user)
user_processor.store_user(runtime_storage_inst, updated_user)
def _update_user(self, record):
email = record.get('author_email')
user_e = user_processor.load_user(self.runtime_storage_inst,
email=email) or {}
user_name = record.get('author_name')
launchpad_id = record.get('launchpad_id')
if (self._need_to_fetch_launchpad() and email and (not user_e)
and (not launchpad_id) and (not user_e.get('launchpad_id'))):
# query LP
launchpad_id, lp_user_name = launchpad_utils.query_lp_info(email)
if lp_user_name:
user_name = lp_user_name
gerrit_id = record.get('gerrit_id')
if gerrit_id:
user_g = user_processor.load_user(self.runtime_storage_inst,
gerrit_id=gerrit_id) or {}
if (self._need_to_fetch_launchpad() and (not user_g)
and (not launchpad_id)
and (not user_e.get('launchpad_id'))):
# query LP
guessed_lp_id = gerrit_id
lp_user_name = launchpad_utils.query_lp_user_name(
guessed_lp_id)
if lp_user_name == user_name:
launchpad_id = guessed_lp_id
else:
user_g = {}
zanata_id = record.get('zanata_id')
if zanata_id:
user_z = user_processor.load_user(self.runtime_storage_inst,
zanata_id=zanata_id) or {}
if (self._need_to_fetch_launchpad() and (not user_z)
and (not launchpad_id)
and (not user_e.get('launchpad_id'))):
# query LP
guessed_lp_id = zanata_id
user_name = launchpad_utils.query_lp_user_name(guessed_lp_id)
if user_name != guessed_lp_id:
launchpad_id = guessed_lp_id
else:
user_z = {}
user_l = user_processor.load_user(self.runtime_storage_inst,
launchpad_id=launchpad_id) or {}
if user_processor.are_users_same([user_e, user_l, user_g, user_z]):
# If sequence numbers are set and the same, merge is not needed
return user_e
user = user_processor.create_user(self.domains_index, launchpad_id,
email, gerrit_id, zanata_id,
user_name)
if user_e or user_l or user_g or user_z:
# merge between existing profiles and a new one
user, users_to_delete = user_processor.merge_user_profiles(
self.domains_index, [user_e, user_l, user_g, user_z, user])
# delete all unneeded profiles
user_processor.delete_users(self.runtime_storage_inst,
users_to_delete)
else:
# create new profile
if (self._need_to_fetch_launchpad() and not user_name):
user_name = launchpad_utils.query_lp_user_name(launchpad_id)
if user_name:
user['user_name'] = user_name
LOG.debug('Created new user: %s', user)
user_processor.store_user(self.runtime_storage_inst, user)
LOG.debug('Stored user: %s', user)
return user
