def _process_member(self, record):
user_id = "member:" + record['member_id']
record['primary_key'] = user_id
record['date'] = utils.member_date_to_timestamp(record['date_joined'])
record['author_name'] = record['member_name']
record['module'] = 'unknown'
company_draft = record['company_draft']
company_name = self.domains_index.get(utils.normalize_company_name(
company_draft)) or (utils.normalize_company_draft(company_draft))
# author_email is a key to create new user
record['author_email'] = user_id
record['company_name'] = company_name
# _update_record_and_user function will create new user if needed
self._update_record_and_user(record)
record['company_name'] = company_name
user = utils.load_user(self.runtime_storage_inst, user_id)
user['user_name'] = record['author_name']
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user['company_name'] = company_name
utils.store_user(self.runtime_storage_inst, user)
record['company_name'] = company_name
yield record
def update_user(self, record):
email = record.get('author_email')
user = utils.load_user(self.runtime_storage_inst, email)
if not user:
if record.get('launchpad_id'):
launchpad_id = record.get('launchpad_id')
user_name = record.get('author_name')
else:
launchpad_id, user_name = self._get_lp_info(email)
if launchpad_id:
user = utils.load_user(self.runtime_storage_inst, launchpad_id)
if user:
# merge emails
if email:
self._update_user_profile(user, email)
else:
# create new
if not user_name:
user_name = record.get('author_name')
if not user_name:
user_name = self._get_lp_user_name(launchpad_id)
user = self._create_user(launchpad_id, email, user_name)
utils.store_user(self.runtime_storage_inst, user)
return user
def _store_users(runtime_storage_inst, users):
for user in users:
stored_user = utils.load_user(runtime_storage_inst, user['user_id'])
if stored_user:
stored_user.update(user)
user = stored_user
utils.store_user(runtime_storage_inst, user)
def update_user(self, record):
email = record.get('author_email')
user_e = utils.load_user(self.runtime_storage_inst, email) or {}
user_name = record.get('author_name')
launchpad_id = record.get('launchpad_id')
if email and (not user_e) and (not launchpad_id):
# query LP
launchpad_id, lp_user_name = self._get_lp_info(email)
if lp_user_name:
user_name = lp_user_name
user_l = utils.load_user(self.runtime_storage_inst, launchpad_id) or {}
user = self._create_user(launchpad_id, email, user_name)
if (user_e.get('seq') == user_l.get('seq')) and user_e.get('seq'):
# sequence numbers are set and the same, merge is not needed
user = user_e
else:
if user_e or user_l:
user = self._merge_user_profiles([user_e, user_l, user])
else:
# create new
if not user_name:
user_name = self._get_lp_user_name(launchpad_id)
if user_name:
user['user_name'] = user_name
LOG.debug('Created new user: %s', user)
utils.store_user(self.runtime_storage_inst, user)
return user
def _store_users(runtime_storage_inst, users):
for user in users:
stored_user = utils.load_user(runtime_storage_inst, user['user_id'])
if stored_user:
stored_user.update(user)
user = stored_user
utils.store_user(runtime_storage_inst, user)
def _update_members_company_name(runtime_storage_inst):
LOG.debug('Update company names for members')
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
member_iterator = _get_changed_member_records(runtime_storage_inst,
record_processor_inst)
for record in member_iterator:
company_name = record['company_name']
user = utils.load_user(runtime_storage_inst, record['user_id'])
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user['company_name'] = company_name
utils.store_user(runtime_storage_inst, user)
LOG.debug('Company name changed for user %s', user)
record_id = record['record_id']
runtime_storage_inst.memcached.set(
runtime_storage_inst._get_record_name(record_id), record)
runtime_storage_inst._commit_update(record_id)
def update_user(self, record):
email = record['author_email']
if email in self.users_index:
user = self.users_index[email]
else:
if ('launchpad_id' in record) and (record['launchpad_id']):
launchpad_id = record['launchpad_id']
user_name = record['author_name']
else:
launchpad_id, user_name = self._get_lp_info(email)
if (launchpad_id) and (launchpad_id in self.users_index):
# merge emails
user = self.users_index[launchpad_id]
self._update_user_profile(user, email)
else:
# create new
if not user_name:
user_name = record['author_name']
user = self._create_user(launchpad_id, email, user_name)
utils.store_user(self.runtime_storage_inst, user)
self.users_index[email] = user
if user['launchpad_id']:
self.users_index[user['launchpad_id']] = user
return user
def update_user(self, record):
email = record.get('author_email')
if email in self.users_index:
user = self.users_index[email]
else:
if record.get('launchpad_id'):
launchpad_id = record.get('launchpad_id')
user_name = record.get('author_name')
else:
launchpad_id, user_name = self._get_lp_info(email)
if (launchpad_id) and (launchpad_id in self.users_index):
# merge emails
user = self.users_index[launchpad_id]
if email:
self._update_user_profile(user, email)
else:
# create new
if not user_name:
user_name = record.get('author_name')
if not user_name:
user_name = self._get_lp_user_name(launchpad_id)
user = self._create_user(launchpad_id, email, user_name)
utils.store_user(self.runtime_storage_inst, user)
if email:
self.users_index[email] = user
if user['launchpad_id']:
self.users_index[user['launchpad_id']] = user
return user
def _process_member(self, record):
user_id = "member:" + record["member_id"]
record["primary_key"] = user_id
record["date"] = utils.member_date_to_timestamp(record["date_joined"])
record["author_name"] = record["member_name"]
record["module"] = "unknown"
company_draft = record["company_draft"]
company_name = self.domains_index.get(company_draft) or company_draft
# author_email is a key to create new user
record["author_email"] = user_id
record["company_name"] = company_name
# _update_record_and_user function will create new user if needed
self._update_record_and_user(record)
record["company_name"] = company_name
user = utils.load_user(self.runtime_storage_inst, user_id)
user["user_name"] = record["author_name"]
user["companies"] = [{"company_name": company_name, "end_date": 0}]
user["company_name"] = company_name
utils.store_user(self.runtime_storage_inst, user)
record["company_name"] = company_name
yield record
def update_user(self, record):
email = record.get('author_email')
user_e = utils.load_user(self.runtime_storage_inst, email) or {}
user_name = record.get('author_name')
launchpad_id = record.get('launchpad_id')
if email and (not user_e) and (not launchpad_id):
# query LP
launchpad_id, lp_user_name = self._get_lp_info(email)
if lp_user_name:
user_name = lp_user_name
user_l = utils.load_user(self.runtime_storage_inst, launchpad_id) or {}
user = self._create_user(launchpad_id, email, user_name)
if (user_e.get('seq') == user_l.get('seq')) and user_e.get('seq'):
# sequence numbers are set and the same, merge is not needed
user = user_e
else:
if user_e or user_l:
user = self._merge_user_profiles(user_e, user_l, user)
else:
# create new
if not user_name:
user_name = self._get_lp_user_name(launchpad_id)
if user_name:
user['user_name'] = user_name
LOG.debug('Created new user: %s', user)
utils.store_user(self.runtime_storage_inst, user)
return user
def _update_members_company_name(self):
LOG.debug('Update members with company names')
for record in self.runtime_storage_inst.get_all_records():
if record['record_type'] != 'member':
continue
company_draft = record['company_draft']
company_name = self.domains_index.get(
utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft))
if company_name == record['company_name']:
continue
LOG.debug('Update record %s, company name changed to %s',
record, company_name)
record['company_name'] = company_name
yield record
user = utils.load_user(self.runtime_storage_inst,
record['user_id'])
LOG.debug('Update user %s, company name changed to %s',
user, company_name)
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
utils.store_user(self.runtime_storage_inst, user)
def _update_members_company_name(runtime_storage_inst):
LOG.debug('Update company names for members')
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
member_iterator = _get_changed_member_records(runtime_storage_inst,
record_processor_inst)
for record in member_iterator:
company_name = record['company_name']
user = utils.load_user(runtime_storage_inst, record['user_id'])
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user['company_name'] = company_name
utils.store_user(runtime_storage_inst, user)
LOG.debug('Company name changed for user %s', user)
record_id = record['record_id']
runtime_storage_inst.memcached.set(
runtime_storage_inst._get_record_name(record_id), record)
runtime_storage_inst._commit_update(record_id)
def _process_users(runtime_storage_inst, users):
users_index = {}
for user in users:
utils.store_user(runtime_storage_inst, user)
if 'user_id' in user:
users_index[user['user_id']] = user
if 'launchpad_id' in user:
users_index[user['launchpad_id']] = user
for email in user['emails']:
users_index[email] = user
runtime_storage_inst.set_by_key('users', users_index)
def _process_users(runtime_storage_inst, users):
users_index = {}
for user in users:
utils.store_user(runtime_storage_inst, user)
if 'user_id' in user:
users_index[user['user_id']] = user
if 'launchpad_id' in user:
users_index[user['launchpad_id']] = user
for email in user['emails']:
users_index[email] = user
runtime_storage_inst.set_by_key('users', users_index)
def _update_record_and_user(self, record):
email = record['author_email'].lower()
record['author_email'] = email
if email in self.users_index:
user = self.users_index[email]
record['launchpad_id'] = user['launchpad_id']
else:
if ('launchpad_id' in record) and (record['launchpad_id']):
launchpad_id = record['launchpad_id']
user_name = record['author_name']
else:
launchpad_id, user_name = self._get_lp_info(email)
record['launchpad_id'] = launchpad_id
if (launchpad_id) and (launchpad_id in self.users_index):
# merge emails
user = self.users_index[launchpad_id]
self._update_user(user, email)
else:
# create new
if not user_name:
user_name = record['author_name']
user = self._create_user(launchpad_id, email, user_name)
utils.store_user(self.runtime_storage_inst, user)
self.users_index[email] = user
if user['launchpad_id']:
self.users_index[user['launchpad_id']] = user
record['user_id'] = user['user_id']
company_by_user = self._find_company(user['companies'], record['date'])
if company_by_user == '*robots':
# don't map robots by email
company = company_by_user
else:
company = self._get_company_by_email(email)
if not company:
company = company_by_user
record['company_name'] = company
if ('user_name' in user) and (user['user_name']):
record['author_name'] = user['user_name']
def _update_record_and_user(self, record):
email = record['author_email'].lower()
record['author_email'] = email
if email in self.users_index:
user = self.users_index[email]
record['launchpad_id'] = user['launchpad_id']
else:
if ('launchpad_id' in record) and (record['launchpad_id']):
launchpad_id = record['launchpad_id']
user_name = record['author_name']
else:
launchpad_id, user_name = self._get_lp_info(email)
record['launchpad_id'] = launchpad_id
if (launchpad_id) and (launchpad_id in self.users_index):
# merge emails
user = self.users_index[launchpad_id]
self._update_user(user, email)
else:
# create new
if not user_name:
user_name = record['author_name']
user = self._create_user(launchpad_id, email, user_name)
utils.store_user(self.runtime_storage_inst, user)
self.users_index[email] = user
if user['launchpad_id']:
self.users_index[user['launchpad_id']] = user
record['user_id'] = user['user_id']
company_by_user = self._find_company(user['companies'], record['date'])
if company_by_user == '*robots':
# don't map robots by email
company = company_by_user
else:
company = self._get_company_by_email(email)
if not company:
company = company_by_user
record['company_name'] = company
if ('user_name' in user) and (user['user_name']):
record['author_name'] = user['user_name']
def _determine_core_contributors(self):
LOG.debug("Determine core contributors")
core_engineers = {}
quarter_ago = int(time.time()) - 60 * 60 * 24 * 30 * 3 # a quarter ago
for record in self.runtime_storage_inst.get_all_records():
if record["record_type"] == "mark" and record["date"] > quarter_ago and record["value"] in [2, -2]:
module_branch = (record["module"], record["branch"])
user_id = record["user_id"]
if user_id in core_engineers:
core_engineers[user_id].add(module_branch)
else:
core_engineers[user_id] = set([module_branch])
for user in self.runtime_storage_inst.get_all_users():
core_old = user.get("core")
user["core"] = list(core_engineers.get(user["user_id"], []))
if user["core"] != core_old:
utils.store_user(self.runtime_storage_inst, user)
def _update_members_company_name(runtime_storage_inst):
LOG.debug("Update company names for members")
record_processor_inst = record_processor.RecordProcessor(runtime_storage_inst)
member_iterator = _get_changed_member_records(runtime_storage_inst, record_processor_inst)
for record in member_iterator:
company_name = record["company_name"]
user = utils.load_user(runtime_storage_inst, record["user_id"])
user["companies"] = [{"company_name": company_name, "end_date": 0}]
user["company_name"] = company_name
utils.store_user(runtime_storage_inst, user)
LOG.debug("Company name changed for user %s", user)
record_id = record["record_id"]
runtime_storage_inst.memcached.set(runtime_storage_inst._get_record_name(record_id), record)
runtime_storage_inst._commit_update(record_id)
def _determine_core_contributors(self):
LOG.debug('Determine core contributors')
core_engineers = {}
quarter_ago = int(time.time()) - 60 * 60 * 24 * 30 * 3 # a quarter ago
for record in self.runtime_storage_inst.get_all_records():
if (record['record_type'] == 'mark' and
record['date'] > quarter_ago and
record['value'] in [2, -2]):
module_branch = (record['module'], record['branch'])
user_id = record['user_id']
if user_id in core_engineers:
core_engineers[user_id].add(module_branch)
else:
core_engineers[user_id] = set([module_branch])
for user in self.runtime_storage_inst.get_all_users():
core_old = user.get('core')
user['core'] = list(core_engineers.get(user['user_id'], []))
if user['core'] != core_old:
utils.store_user(self.runtime_storage_inst, user)
def _determine_core_contributors(self):
LOG.debug('Determine core contributors')
core_engineers = {}
quarter_ago = int(time.time()) - 60 * 60 * 24 * 30 * 3 # a quarter ago
for record in self.runtime_storage_inst.get_all_records():
if (record['record_type'] == 'mark' and
record['date'] > quarter_ago and
record['value'] in [2, -2]):
module_branch = (record['module'], record['branch'])
user_id = record['user_id']
if user_id in core_engineers:
core_engineers[user_id].add(module_branch)
else:
core_engineers[user_id] = set([module_branch])
for user in self.runtime_storage_inst.get_all_users():
core_old = user.get('core')
user['core'] = list(core_engineers.get(user['user_id'], []))
if user['core'] != core_old:
utils.store_user(self.runtime_storage_inst, user)