def _process_member(self, record): user_id = user_processor.make_user_id(member_id=record['member_id']) record['primary_key'] = user_id record['date'] = utils.member_date_to_timestamp(record['date_joined']) record['author_name'] = record['member_name'] record['module'] = 'unknown' company_draft = record['company_draft'] company_name = self.domains_index.get(utils.normalize_company_name( company_draft)) or (utils.normalize_company_draft(company_draft)) # author_email is a key to create new user record['author_email'] = user_id record['company_name'] = company_name # _update_record_and_user function will create new user if needed self._update_record_and_user(record) record['company_name'] = company_name user = user_processor.load_user(self.runtime_storage_inst, user_id) user['user_name'] = record['author_name'] user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] user['company_name'] = company_name user_processor.store_user(self.runtime_storage_inst, user) record['company_name'] = company_name yield record
def _update_members_company_name(self): LOG.debug('Update members with company names') for record in self.runtime_storage_inst.get_all_records(): if record['record_type'] != 'member': continue company_draft = record['company_draft'] company_name = self.domains_index.get( utils.normalize_company_name(company_draft)) or ( utils.normalize_company_draft(company_draft)) if company_name == record['company_name']: continue LOG.debug('Update record %s, company name changed to %s', record, company_name) record['company_name'] = company_name yield record user = user_processor.load_user(self.runtime_storage_inst, record['user_id']) LOG.debug('Update user %s, company name changed to %s', user, company_name) user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] user_processor.store_user(self.runtime_storage_inst, user)
def _determine_core_contributors(self): LOG.info('Determine core contributors') module_branches = collections.defaultdict(set) quarter_ago = int(time.time()) - 60 * 60 * 24 * 30 * 3 # a quarter ago def record_handler(record): if (record['record_type'] == 'mark' and record['date'] > quarter_ago and record['value'] in [2, -2]): module_branch = (record['module'], record['branch']) user_id = record['user_id'] module_branches[user_id].add(module_branch) yield record_handler for user in self.runtime_storage_inst.get_all_users(): core_old = user.get('core') user_module_branch = module_branches.get(user['user_id']) if user_module_branch: user['core'] = list(user_module_branch) elif user.get('core'): del user['core'] if user.get('core') != core_old: user_processor.store_user(self.runtime_storage_inst, user)
def _process_member(self, record): user_id = user_processor.make_user_id(member_id=record["member_id"]) record["primary_key"] = user_id record["date"] = utils.member_date_to_timestamp(record["date_joined"]) record["author_name"] = record["member_name"] record["module"] = "unknown" company_draft = record["company_draft"] company_name = self.domains_index.get(utils.normalize_company_name(company_draft)) or ( utils.normalize_company_draft(company_draft) ) # author_email is a key to create new user record["author_email"] = user_id record["company_name"] = company_name # _update_record_and_user function will create new user if needed self._update_record_and_user(record) record["company_name"] = company_name user = user_processor.load_user(self.runtime_storage_inst, user_id) user["user_name"] = record["author_name"] user["companies"] = [{"company_name": company_name, "end_date": 0}] user["company_name"] = company_name user_processor.store_user(self.runtime_storage_inst, user) record["company_name"] = company_name yield record
def _process_member(self, record): user_id = user_processor.make_user_id(member_id=record['member_id']) record['primary_key'] = user_id record['date'] = utils.member_date_to_timestamp(record['date_joined']) record['author_name'] = record['member_name'] record['module'] = 'unknown' company_draft = record['company_draft'] company_name = self.domains_index.get( utils.normalize_company_name(company_draft)) or ( utils.normalize_company_draft(company_draft)) # author_email is a key to create new user record['author_email'] = user_id record['company_name'] = company_name # _update_record_and_user function will create new user if needed self._update_record_and_user(record) record['company_name'] = company_name user = user_processor.load_user(self.runtime_storage_inst, user_id) user['user_name'] = record['author_name'] user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] user['company_name'] = company_name user_processor.store_user(self.runtime_storage_inst, user) record['company_name'] = company_name yield record
def record_handler(record): if record['record_type'] != 'member': return company_draft = record['company_draft'] company_name = self.domains_index.get( utils.normalize_company_name(company_draft)) or ( utils.normalize_company_draft(company_draft)) if company_name == record['company_name']: return LOG.debug('Update record %s, company name changed to %s', record, company_name) record['company_name'] = company_name yield record user = user_processor.load_user(self.runtime_storage_inst, record['user_id']) LOG.debug('Update user %s, company name changed to %s', user, company_name) user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] user_processor.store_user(self.runtime_storage_inst, user)
def _update_user(self, record): email = record.get('author_email') user_name = record.get('author_name') user_e = user_processor.load_user(self.runtime_storage_inst, email=email) or {} gerrit_id = record.get('gerrit_id') if gerrit_id: user_g = user_processor.load_user(self.runtime_storage_inst, gerrit_id=gerrit_id) or {} else: user_g = {} if user_processor.are_users_same([user_e, user_g]): # If sequence numbers are set and the same, merge is not needed return user_e user = user_processor.create_user(self.domains_index, email, gerrit_id, user_name) if user_e or user_g: # merge between existing profiles and a new one user, users_to_delete = user_processor.merge_user_profiles( self.domains_index, [user_e, user_g, user]) # delete all unneeded profiles user_processor.delete_users(self.runtime_storage_inst, users_to_delete) user_processor.store_user(self.runtime_storage_inst, user) LOG.debug('Stored user: %s', user) return user
def apply_user_corrections(self, user_corrections_iterator): for user_correction in user_corrections_iterator: stored_user = user_processor.load_user( self, user_id=user_correction['user_id']) updated_user = user_processor.update_user_profile( stored_user, user_correction, is_correction=True) user_processor.store_user(self, updated_user)
def _store_users(runtime_storage_inst, users): for user in users: stored_user = user_processor.load_user(runtime_storage_inst, user_id=user['user_id']) if stored_user: stored_user.update(user) user = stored_user user['static'] = True user_processor.store_user(runtime_storage_inst, user)
def update_user(self, record): email = record.get('author_email') user_e = user_processor.load_user(self.runtime_storage_inst, email=email) or {} user_name = record.get('author_name') launchpad_id = record.get('launchpad_id') if (email and (not user_e) and (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP launchpad_id, lp_user_name = self._get_lp_info(email) if lp_user_name: user_name = lp_user_name gerrit_id = record.get('gerrit_id') if gerrit_id: user_g = user_processor.load_user(self.runtime_storage_inst, gerrit_id=gerrit_id) or {} if ((not user_g) and (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP guessed_lp_id = gerrit_id lp_user_name = self._get_lp_user_name(guessed_lp_id) if lp_user_name == user_name: launchpad_id = guessed_lp_id else: user_g = {} user_l = user_processor.load_user(self.runtime_storage_inst, launchpad_id=launchpad_id) or {} if ((user_e.get('seq') == user_l.get('seq') == user_g.get('seq')) and user_e.get('seq')): # sequence numbers are set and the same, merge is not needed user = user_e else: user = self._create_user(launchpad_id, email, gerrit_id, user_name) if user_e or user_l or user_g: user = self._merge_user_profiles( [user_e, user_l, user_g, user]) else: # create new if not user_name: user_name = self._get_lp_user_name(launchpad_id) if user_name: user['user_name'] = user_name LOG.debug('Created new user: %s', user) user_processor.store_user(self.runtime_storage_inst, user) LOG.debug('Stored user: %s', user) return user
def update_user(self, record): email = record.get('author_email') user_e = user_processor.load_user( self.runtime_storage_inst, email=email) or {} user_name = record.get('author_name') launchpad_id = record.get('launchpad_id') if (email and (not user_e) and (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP launchpad_id, lp_user_name = self._get_lp_info(email) if lp_user_name: user_name = lp_user_name gerrit_id = record.get('gerrit_id') if gerrit_id: user_g = user_processor.load_user( self.runtime_storage_inst, gerrit_id=gerrit_id) or {} if ((not user_g) and (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP guessed_lp_id = gerrit_id lp_user_name = self._get_lp_user_name(guessed_lp_id) if lp_user_name == user_name: launchpad_id = guessed_lp_id else: user_g = {} user_l = user_processor.load_user( self.runtime_storage_inst, launchpad_id=launchpad_id) or {} if ((user_e.get('seq') == user_l.get('seq') == user_g.get('seq')) and user_e.get('seq')): # sequence numbers are set and the same, merge is not needed user = user_e else: user = self._create_user(launchpad_id, email, gerrit_id, user_name) if user_e or user_l or user_g: user = self._merge_user_profiles( [user_e, user_l, user_g, user]) else: # create new if not user_name: user_name = self._get_lp_user_name(launchpad_id) if user_name: user['user_name'] = user_name LOG.debug('Created new user: %s', user) user_processor.store_user(self.runtime_storage_inst, user) LOG.debug('Stored user: %s', user) return user
def record_handler(record): if record["record_type"] != "member": return company_draft = record["company_draft"] company_name = self.domains_index.get(utils.normalize_company_name(company_draft)) or ( utils.normalize_company_draft(company_draft) ) if company_name == record["company_name"]: return LOG.debug("Update record %s, company name changed to %s", record, company_name) record["company_name"] = company_name yield record user = user_processor.load_user(self.runtime_storage_inst, record["user_id"]) LOG.debug("Update user %s, company name changed to %s", user, company_name) user["companies"] = [{"company_name": company_name, "end_date": 0}] user_processor.store_user(self.runtime_storage_inst, user)
def _determine_core_contributors(self): LOG.info("Determine core contributors") module_branches = collections.defaultdict(set) quarter_ago = int(time.time()) - 60 * 60 * 24 * 30 * 3 # a quarter ago def record_handler(record): if record["record_type"] == "mark" and record["date"] > quarter_ago and record["value"] in [2, -2]: module_branch = (record["module"], record["branch"]) user_id = record["user_id"] module_branches[user_id].add(module_branch) yield record_handler for user in self.runtime_storage_inst.get_all_users(): core_old = user.get("core") user_module_branch = module_branches.get(user["user_id"]) if user_module_branch: user["core"] = list(user_module_branch) elif user.get("core"): del user["core"] if user.get("core") != core_old: user_processor.store_user(self.runtime_storage_inst, user)
def _store_users(runtime_storage_inst, users): for user in users: stored_user = user_processor.load_user(runtime_storage_inst, user_id=user['user_id']) updated_user = user_processor.update_user_profile(stored_user, user) user_processor.store_user(runtime_storage_inst, updated_user)
def _update_user(self, record): email = record.get('author_email') user_e = user_processor.load_user(self.runtime_storage_inst, email=email) or {} user_name = record.get('author_name') launchpad_id = record.get('launchpad_id') if (self._need_to_fetch_launchpad() and email and (not user_e) and (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP launchpad_id, lp_user_name = launchpad_utils.query_lp_info(email) if lp_user_name: user_name = lp_user_name gerrit_id = record.get('gerrit_id') if gerrit_id: user_g = user_processor.load_user(self.runtime_storage_inst, gerrit_id=gerrit_id) or {} if (self._need_to_fetch_launchpad() and (not user_g) and (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP guessed_lp_id = gerrit_id lp_user_name = launchpad_utils.query_lp_user_name( guessed_lp_id) if lp_user_name == user_name: launchpad_id = guessed_lp_id else: user_g = {} zanata_id = record.get('zanata_id') if zanata_id: user_z = user_processor.load_user(self.runtime_storage_inst, zanata_id=zanata_id) or {} if (self._need_to_fetch_launchpad() and (not user_z) and (not launchpad_id) and (not user_e.get('launchpad_id'))): # query LP guessed_lp_id = zanata_id user_name = launchpad_utils.query_lp_user_name(guessed_lp_id) if user_name != guessed_lp_id: launchpad_id = guessed_lp_id else: user_z = {} user_l = user_processor.load_user(self.runtime_storage_inst, launchpad_id=launchpad_id) or {} if user_processor.are_users_same([user_e, user_l, user_g, user_z]): # If sequence numbers are set and the same, merge is not needed return user_e user = user_processor.create_user(self.domains_index, launchpad_id, email, gerrit_id, zanata_id, user_name) if user_e or user_l or user_g or user_z: # merge between existing profiles and a new one user, users_to_delete = user_processor.merge_user_profiles( self.domains_index, [user_e, user_l, user_g, user_z, user]) # delete all unneeded profiles user_processor.delete_users(self.runtime_storage_inst, users_to_delete) else: # create new profile if (self._need_to_fetch_launchpad() and not user_name): user_name = launchpad_utils.query_lp_user_name(launchpad_id) if user_name: user['user_name'] = user_name LOG.debug('Created new user: %s', user) user_processor.store_user(self.runtime_storage_inst, user) LOG.debug('Stored user: %s', user) return user