def _process_member(self, record): user_id = "member:" + record['member_id'] record['primary_key'] = user_id record['date'] = utils.member_date_to_timestamp(record['date_joined']) record['author_name'] = record['member_name'] record['module'] = 'unknown' company_draft = record['company_draft'] company_name = self.domains_index.get(utils.normalize_company_name( company_draft)) or (utils.normalize_company_draft(company_draft)) # author_email is a key to create new user record['author_email'] = user_id record['company_name'] = company_name # _update_record_and_user function will create new user if needed self._update_record_and_user(record) record['company_name'] = company_name user = utils.load_user(self.runtime_storage_inst, user_id) user['user_name'] = record['author_name'] user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] user['company_name'] = company_name utils.store_user(self.runtime_storage_inst, user) record['company_name'] = company_name yield record
def update_user(self, record): email = record.get('author_email') user = utils.load_user(self.runtime_storage_inst, email) if not user: if record.get('launchpad_id'): launchpad_id = record.get('launchpad_id') user_name = record.get('author_name') else: launchpad_id, user_name = self._get_lp_info(email) if launchpad_id: user = utils.load_user(self.runtime_storage_inst, launchpad_id) if user: # merge emails if email: self._update_user_profile(user, email) else: # create new if not user_name: user_name = record.get('author_name') if not user_name: user_name = self._get_lp_user_name(launchpad_id) user = self._create_user(launchpad_id, email, user_name) utils.store_user(self.runtime_storage_inst, user) return user
def _store_users(runtime_storage_inst, users): for user in users: stored_user = utils.load_user(runtime_storage_inst, user['user_id']) if stored_user: stored_user.update(user) user = stored_user utils.store_user(runtime_storage_inst, user)
def update_user(self, record): email = record.get('author_email') user_e = utils.load_user(self.runtime_storage_inst, email) or {} user_name = record.get('author_name') launchpad_id = record.get('launchpad_id') if email and (not user_e) and (not launchpad_id): # query LP launchpad_id, lp_user_name = self._get_lp_info(email) if lp_user_name: user_name = lp_user_name user_l = utils.load_user(self.runtime_storage_inst, launchpad_id) or {} user = self._create_user(launchpad_id, email, user_name) if (user_e.get('seq') == user_l.get('seq')) and user_e.get('seq'): # sequence numbers are set and the same, merge is not needed user = user_e else: if user_e or user_l: user = self._merge_user_profiles([user_e, user_l, user]) else: # create new if not user_name: user_name = self._get_lp_user_name(launchpad_id) if user_name: user['user_name'] = user_name LOG.debug('Created new user: %s', user) utils.store_user(self.runtime_storage_inst, user) return user
def _update_members_company_name(runtime_storage_inst): LOG.debug('Update company names for members') record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) member_iterator = _get_changed_member_records(runtime_storage_inst, record_processor_inst) for record in member_iterator: company_name = record['company_name'] user = utils.load_user(runtime_storage_inst, record['user_id']) user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] user['company_name'] = company_name utils.store_user(runtime_storage_inst, user) LOG.debug('Company name changed for user %s', user) record_id = record['record_id'] runtime_storage_inst.memcached.set( runtime_storage_inst._get_record_name(record_id), record) runtime_storage_inst._commit_update(record_id)
def update_user(self, record): email = record['author_email'] if email in self.users_index: user = self.users_index[email] else: if ('launchpad_id' in record) and (record['launchpad_id']): launchpad_id = record['launchpad_id'] user_name = record['author_name'] else: launchpad_id, user_name = self._get_lp_info(email) if (launchpad_id) and (launchpad_id in self.users_index): # merge emails user = self.users_index[launchpad_id] self._update_user_profile(user, email) else: # create new if not user_name: user_name = record['author_name'] user = self._create_user(launchpad_id, email, user_name) utils.store_user(self.runtime_storage_inst, user) self.users_index[email] = user if user['launchpad_id']: self.users_index[user['launchpad_id']] = user return user
def update_user(self, record): email = record.get('author_email') if email in self.users_index: user = self.users_index[email] else: if record.get('launchpad_id'): launchpad_id = record.get('launchpad_id') user_name = record.get('author_name') else: launchpad_id, user_name = self._get_lp_info(email) if (launchpad_id) and (launchpad_id in self.users_index): # merge emails user = self.users_index[launchpad_id] if email: self._update_user_profile(user, email) else: # create new if not user_name: user_name = record.get('author_name') if not user_name: user_name = self._get_lp_user_name(launchpad_id) user = self._create_user(launchpad_id, email, user_name) utils.store_user(self.runtime_storage_inst, user) if email: self.users_index[email] = user if user['launchpad_id']: self.users_index[user['launchpad_id']] = user return user
def _process_member(self, record): user_id = "member:" + record["member_id"] record["primary_key"] = user_id record["date"] = utils.member_date_to_timestamp(record["date_joined"]) record["author_name"] = record["member_name"] record["module"] = "unknown" company_draft = record["company_draft"] company_name = self.domains_index.get(company_draft) or company_draft # author_email is a key to create new user record["author_email"] = user_id record["company_name"] = company_name # _update_record_and_user function will create new user if needed self._update_record_and_user(record) record["company_name"] = company_name user = utils.load_user(self.runtime_storage_inst, user_id) user["user_name"] = record["author_name"] user["companies"] = [{"company_name": company_name, "end_date": 0}] user["company_name"] = company_name utils.store_user(self.runtime_storage_inst, user) record["company_name"] = company_name yield record
def update_user(self, record): email = record.get('author_email') user_e = utils.load_user(self.runtime_storage_inst, email) or {} user_name = record.get('author_name') launchpad_id = record.get('launchpad_id') if email and (not user_e) and (not launchpad_id): # query LP launchpad_id, lp_user_name = self._get_lp_info(email) if lp_user_name: user_name = lp_user_name user_l = utils.load_user(self.runtime_storage_inst, launchpad_id) or {} user = self._create_user(launchpad_id, email, user_name) if (user_e.get('seq') == user_l.get('seq')) and user_e.get('seq'): # sequence numbers are set and the same, merge is not needed user = user_e else: if user_e or user_l: user = self._merge_user_profiles(user_e, user_l, user) else: # create new if not user_name: user_name = self._get_lp_user_name(launchpad_id) if user_name: user['user_name'] = user_name LOG.debug('Created new user: %s', user) utils.store_user(self.runtime_storage_inst, user) return user
def _update_members_company_name(self): LOG.debug('Update members with company names') for record in self.runtime_storage_inst.get_all_records(): if record['record_type'] != 'member': continue company_draft = record['company_draft'] company_name = self.domains_index.get( utils.normalize_company_name(company_draft)) or ( utils.normalize_company_draft(company_draft)) if company_name == record['company_name']: continue LOG.debug('Update record %s, company name changed to %s', record, company_name) record['company_name'] = company_name yield record user = utils.load_user(self.runtime_storage_inst, record['user_id']) LOG.debug('Update user %s, company name changed to %s', user, company_name) user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] utils.store_user(self.runtime_storage_inst, user)
def _process_users(runtime_storage_inst, users): users_index = {} for user in users: utils.store_user(runtime_storage_inst, user) if 'user_id' in user: users_index[user['user_id']] = user if 'launchpad_id' in user: users_index[user['launchpad_id']] = user for email in user['emails']: users_index[email] = user runtime_storage_inst.set_by_key('users', users_index)
def _update_record_and_user(self, record): email = record['author_email'].lower() record['author_email'] = email if email in self.users_index: user = self.users_index[email] record['launchpad_id'] = user['launchpad_id'] else: if ('launchpad_id' in record) and (record['launchpad_id']): launchpad_id = record['launchpad_id'] user_name = record['author_name'] else: launchpad_id, user_name = self._get_lp_info(email) record['launchpad_id'] = launchpad_id if (launchpad_id) and (launchpad_id in self.users_index): # merge emails user = self.users_index[launchpad_id] self._update_user(user, email) else: # create new if not user_name: user_name = record['author_name'] user = self._create_user(launchpad_id, email, user_name) utils.store_user(self.runtime_storage_inst, user) self.users_index[email] = user if user['launchpad_id']: self.users_index[user['launchpad_id']] = user record['user_id'] = user['user_id'] company_by_user = self._find_company(user['companies'], record['date']) if company_by_user == '*robots': # don't map robots by email company = company_by_user else: company = self._get_company_by_email(email) if not company: company = company_by_user record['company_name'] = company if ('user_name' in user) and (user['user_name']): record['author_name'] = user['user_name']
def _determine_core_contributors(self): LOG.debug("Determine core contributors") core_engineers = {} quarter_ago = int(time.time()) - 60 * 60 * 24 * 30 * 3 # a quarter ago for record in self.runtime_storage_inst.get_all_records(): if record["record_type"] == "mark" and record["date"] > quarter_ago and record["value"] in [2, -2]: module_branch = (record["module"], record["branch"]) user_id = record["user_id"] if user_id in core_engineers: core_engineers[user_id].add(module_branch) else: core_engineers[user_id] = set([module_branch]) for user in self.runtime_storage_inst.get_all_users(): core_old = user.get("core") user["core"] = list(core_engineers.get(user["user_id"], [])) if user["core"] != core_old: utils.store_user(self.runtime_storage_inst, user)
def _update_members_company_name(runtime_storage_inst): LOG.debug("Update company names for members") record_processor_inst = record_processor.RecordProcessor(runtime_storage_inst) member_iterator = _get_changed_member_records(runtime_storage_inst, record_processor_inst) for record in member_iterator: company_name = record["company_name"] user = utils.load_user(runtime_storage_inst, record["user_id"]) user["companies"] = [{"company_name": company_name, "end_date": 0}] user["company_name"] = company_name utils.store_user(runtime_storage_inst, user) LOG.debug("Company name changed for user %s", user) record_id = record["record_id"] runtime_storage_inst.memcached.set(runtime_storage_inst._get_record_name(record_id), record) runtime_storage_inst._commit_update(record_id)
def _determine_core_contributors(self): LOG.debug('Determine core contributors') core_engineers = {} quarter_ago = int(time.time()) - 60 * 60 * 24 * 30 * 3 # a quarter ago for record in self.runtime_storage_inst.get_all_records(): if (record['record_type'] == 'mark' and record['date'] > quarter_ago and record['value'] in [2, -2]): module_branch = (record['module'], record['branch']) user_id = record['user_id'] if user_id in core_engineers: core_engineers[user_id].add(module_branch) else: core_engineers[user_id] = set([module_branch]) for user in self.runtime_storage_inst.get_all_users(): core_old = user.get('core') user['core'] = list(core_engineers.get(user['user_id'], [])) if user['core'] != core_old: utils.store_user(self.runtime_storage_inst, user)