def _normalize_user(user): for c in user['companies']: c['end_date'] = utils.date_to_timestamp(c['end_date']) # sort companies by end_date def end_date_comparator(x, y): if x["end_date"] == 0: return 1 elif y["end_date"] == 0: return -1 else: return x["end_date"] - y["end_date"] user['companies'].sort(key=utils.cmp_to_key(end_date_comparator)) if user['companies']: if user['companies'][-1]['end_date'] != 0: user['companies'].append(dict(company_name='*independent', end_date=0)) user['user_id'] = user_processor.make_user_id( launchpad_id=user.get('launchpad_id'), emails=user.get('emails'), gerrit_id=user.get('gerrit_id'), github_id=user.get('github_id'), zanata_id=user.get('zanata_id'), ldap_id=user.get('ldap_id')) or user.get('user_id')
def _update_with_driverlog_data(default_data, driverlog_data_uri): LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri) driverlog_data = utils.read_json_from_uri(driverlog_data_uri) if not driverlog_data: LOG.warning('DriverLog data is not available') return module_cis = collections.defaultdict(list) for driver in driverlog_data['drivers']: if 'ci' not in driver: continue module = (driver.get('repo') or driver['project_id']).split('/')[1] module_cis[module].append(driver) default_data['users'].append({ 'user_id': user_processor.make_user_id(ci_id=driver['name']), 'user_name': driver['name'], 'static': True, 'companies': [ {'company_name': driver['vendor'], 'end_date': None}], }) for repo in default_data['repos']: if repo['module'] in module_cis: repo['drivers'] = module_cis[repo['module']]
def _create_user(self, launchpad_id, email, gerrit_id, user_name): company = (self._get_company_by_email(email) or self._get_independent()) emails = [] if email: emails = [email] user = { 'user_id': user_processor.make_user_id(emails=emails, launchpad_id=launchpad_id, gerrit_id=gerrit_id), 'launchpad_id': launchpad_id, 'user_name': user_name or '', 'companies': [{ 'company_name': company, 'end_date': 0, }], 'emails': emails, } if gerrit_id: user['gerrit_id'] = gerrit_id return user
def _process_member(self, record): user_id = user_processor.make_user_id(member_id=record['member_id']) record['primary_key'] = user_id record['date'] = utils.member_date_to_timestamp(record['date_joined']) record['author_name'] = record['member_name'] record['module'] = 'unknown' company_draft = record['company_draft'] company_name = self.domains_index.get( utils.normalize_company_name(company_draft)) or ( utils.normalize_company_draft(company_draft)) # author_email is a key to create new user record['author_email'] = user_id record['company_name'] = company_name # _update_record_and_user function will create new user if needed self._update_record_and_user(record) record['company_name'] = company_name user = user_processor.load_user(self.runtime_storage_inst, user_id) user['user_name'] = record['author_name'] user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] user['company_name'] = company_name user_processor.store_user(self.runtime_storage_inst, user) record['company_name'] = company_name yield record
def _update_with_driverlog_data(default_data, driverlog_data_uri): LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri) driverlog_data = utils.read_json_from_uri(driverlog_data_uri) module_ci_ids = {} ci_ids = set() for driver in driverlog_data['drivers']: if 'ci' in driver: module = driver['project_id'].split('/')[1] if module not in module_ci_ids: module_ci_ids[module] = {} ci_id = driver['ci']['id'] module_ci_ids[module][ci_id] = driver if ci_id not in ci_ids: ci_ids.add(ci_id) default_data['users'].append({ 'user_id': user_processor.make_user_id(gerrit_id=ci_id), 'gerrit_id': ci_id, 'user_name': ci_id, 'static': True, 'companies': [ {'company_name': driver['vendor'], 'end_date': None}], }) for repo in default_data['repos']: if repo['module'] in module_ci_ids: repo['ci'] = module_ci_ids[repo['module']]
def _update_with_driverlog_data(default_data, driverlog_data_uri): LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri) driverlog_data = utils.read_json_from_uri(driverlog_data_uri) if not driverlog_data: LOG.warning('DriverLog data is not available') return module_cis = collections.defaultdict(list) for driver in driverlog_data['drivers']: if 'ci' not in driver: continue module = (driver.get('repo') or driver['project_id']).split('/')[1] module_cis[module].append(driver) default_data['users'].append({ 'user_id': user_processor.make_user_id(ci_id=driver['name']), 'user_name': driver['name'], 'static': True, 'companies': [{ 'company_name': driver['vendor'], 'end_date': None }], }) for repo in default_data['repos']: if repo['module'] in module_cis: repo['drivers'] = module_cis[repo['module']]
def _process_member(self, record): user_id = user_processor.make_user_id(member_id=record['member_id']) record['primary_key'] = user_id record['date'] = utils.member_date_to_timestamp(record['date_joined']) record['author_name'] = record['member_name'] record['module'] = 'unknown' company_draft = record['company_draft'] company_name = self.domains_index.get(utils.normalize_company_name( company_draft)) or (utils.normalize_company_draft(company_draft)) # author_email is a key to create new user record['author_email'] = user_id record['company_name'] = company_name # _update_record_and_user function will create new user if needed self._update_record_and_user(record) record['company_name'] = company_name user = user_processor.load_user(self.runtime_storage_inst, user_id) user['user_name'] = record['author_name'] user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] user['company_name'] = company_name user_processor.store_user(self.runtime_storage_inst, user) record['company_name'] = company_name yield record
def _update_with_driverlog_data(default_data, driverlog_data_uri): LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri) driverlog_data = utils.read_json_from_uri(driverlog_data_uri) module_ci_ids = {} ci_ids = set() for driver in driverlog_data['drivers']: if 'ci' in driver: module = driver['project_id'].split('/')[1] if module not in module_ci_ids: module_ci_ids[module] = {} ci_id = driver['ci']['id'] module_ci_ids[module][ci_id] = driver if ci_id not in ci_ids: ci_ids.add(ci_id) default_data['users'].append({ 'user_id': user_processor.make_user_id(gerrit_id=ci_id), 'gerrit_id': ci_id, 'user_name': ci_id, 'static': True, 'companies': [ {'company_name': driver['vendor'], 'end_date': None}], }) for repo in default_data['repos']: if repo['module'] in module_ci_ids: repo['ci'] = module_ci_ids[repo['module']]
def _process_member(self, record): user_id = user_processor.make_user_id(member_id=record["member_id"]) record["primary_key"] = user_id record["date"] = utils.member_date_to_timestamp(record["date_joined"]) record["author_name"] = record["member_name"] record["module"] = "unknown" company_draft = record["company_draft"] company_name = self.domains_index.get(utils.normalize_company_name(company_draft)) or ( utils.normalize_company_draft(company_draft) ) # author_email is a key to create new user record["author_email"] = user_id record["company_name"] = company_name # _update_record_and_user function will create new user if needed self._update_record_and_user(record) record["company_name"] = company_name user = user_processor.load_user(self.runtime_storage_inst, user_id) user["user_name"] = record["author_name"] user["companies"] = [{"company_name": company_name, "end_date": 0}] user["company_name"] = company_name user_processor.store_user(self.runtime_storage_inst, user) record["company_name"] = company_name yield record
def _normalize_user(user): for c in user['companies']: c['end_date'] = utils.date_to_timestamp(c['end_date']) # sort companies by end_date def end_date_comparator(x, y): if x["end_date"] == 0: return 1 elif y["end_date"] == 0: return -1 else: return x["end_date"] - y["end_date"] user['companies'].sort(key=utils.cmp_to_key(end_date_comparator)) if user['companies']: if user['companies'][-1]['end_date'] != 0: user['companies'].append( dict(company_name='*independent', end_date=0)) user['user_id'] = user_processor.make_user_id( launchpad_id=user.get('launchpad_id'), emails=user.get('emails'), gerrit_id=user.get('gerrit_id'), github_id=user.get('github_id'), zanata_id=user.get('zanata_id'), ldap_id=user.get('ldap_id')) or user.get('user_id')
def _create_user(self, launchpad_id, email, gerrit_id, user_name): company = self._get_company_by_email(email) or self._get_independent() emails = [] if email: emails = [email] user = { "user_id": user_processor.make_user_id(emails=emails, launchpad_id=launchpad_id, gerrit_id=gerrit_id), "launchpad_id": launchpad_id, "user_name": user_name or "", "companies": [{"company_name": company, "end_date": 0}], "emails": emails, } if gerrit_id: user["gerrit_id"] = gerrit_id return user
def _normalize_user(user): for c in user['companies']: c['end_date'] = utils.date_to_timestamp(c['end_date']) # sort companies by end_date def end_date_comparator(x, y): if x["end_date"] == 0: return 1 elif y["end_date"] == 0: return -1 else: return x["end_date"] - y["end_date"] user['companies'].sort(key=utils.cmp_to_key(end_date_comparator)) user['user_id'] = user_processor.make_user_id( launchpad_id=user.get('launchpad_id'), email=user.get('email'))
def _process_translation(self, record): # todo split translation and approval translation = record.copy() user_id = user_processor.make_user_id(zanata_id=record['zanata_id']) translation['record_type'] = 'tr' translation['primary_key'] = '%s:%s:%s:%s' % ( user_id, record['module'], record['date'], record['branch']) translation['author_name'] = user_id # following fields are put into standard fields stored in dashboard mem translation['loc'] = record['translated'] translation['value'] = record['language'] self._update_record_and_user(translation) yield translation
def _process_translation(self, record): # todo split translation and approval translation = record.copy() user_id = user_processor.make_user_id(zanata_id=record['zanata_id']) translation['record_type'] = 'tr' translation['primary_key'] = '%s:%s:%s:%s' % ( user_id, record['module'], record['date'], record['branch']) translation['author_name'] = user_id # following fields are put into standard fields stored in dashboard mem translation['loc'] = record['translated'] translation['value'] = record['language'] self._update_record_and_user(translation) yield translation
def _normalize_user(user): for c in user['companies']: c['end_date'] = utils.date_to_timestamp(c['end_date']) # sort companies by end_date def end_date_comparator(x, y): if x["end_date"] == 0: return 1 elif y["end_date"] == 0: return -1 else: return x["end_date"] - y["end_date"] user['companies'].sort(key=utils.cmp_to_key(end_date_comparator)) user['user_id'] = user_processor.make_user_id( launchpad_id=user.get('launchpad_id'), email=user.get('email'))
def _create_user(self, launchpad_id, email, gerrit_id, user_name): company = (self._get_company_by_email(email) or self._get_independent()) emails = [] if email: emails = [email] user = { 'user_id': user_processor.make_user_id( emails=emails, launchpad_id=launchpad_id, gerrit_id=gerrit_id), 'launchpad_id': launchpad_id, 'user_name': user_name or '', 'companies': [{ 'company_name': company, 'end_date': 0, }], 'emails': emails, } if gerrit_id: user['gerrit_id'] = gerrit_id return user
def _find_ci_result(review, drivers): """For a given stream of reviews yields results produced by CIs.""" review_id = review['id'] review_number = review['number'] ci_id_set = set(d['ci']['id'] for d in drivers) candidate_drivers = [d for d in drivers] last_patch_set_number = review['patchSets'][-1]['number'] for comment in reversed(review.get('comments') or []): comment_author = comment['reviewer'].get('username') if comment_author not in ci_id_set: continue # not any of registered CIs message = comment['message'] prefix = 'Patch Set' if comment['message'].find(prefix) != 0: continue # look for special messages only prefix = 'Patch Set %s:' % last_patch_set_number if comment['message'].find(prefix) != 0: break # all comments from the latest patch set already parsed message = message[len(prefix):].strip() result = None matched_drivers = set() for driver in candidate_drivers: ci = driver['ci'] if ci['id'] != comment_author: continue # try to get result by parsing comment message success_pattern = ci.get('success_pattern') failure_pattern = ci.get('failure_pattern') message_lines = (l for l in message.split('\n') if l.strip()) line = '' for line in message_lines: if success_pattern and re.search(success_pattern, line): result = True break elif failure_pattern and re.search(failure_pattern, line): result = False break if result is not None: matched_drivers.add(driver['name']) record = { 'user_id': user_processor.make_user_id(ci_id=driver['name']), 'value': result, 'message': line, 'date': comment['timestamp'], 'branch': review['branch'], 'review_id': review_id, 'review_number': review_number, 'driver_name': driver['name'], 'driver_vendor': driver['vendor'], 'module': review['module'] } if review['branch'].find('/') > 0: record['release'] = review['branch'].split('/')[1] yield record candidate_drivers = [ d for d in candidate_drivers if d['name'] not in matched_drivers ] if not candidate_drivers: break # found results from all drivers
def _find_ci_result(review, drivers): """For a given stream of reviews yields results produced by CIs.""" review_id = review['id'] review_number = review['number'] ci_id_set = set(d['ci']['id'] for d in drivers) candidate_drivers = [d for d in drivers] last_patch_set_number = review['patchSets'][-1]['number'] for comment in reversed(review.get('comments') or []): comment_author = comment['reviewer'].get('username') if comment_author not in ci_id_set: continue # not any of registered CIs message = comment['message'] prefix = 'Patch Set' if comment['message'].find(prefix) != 0: continue # look for special messages only prefix = 'Patch Set %s:' % last_patch_set_number if comment['message'].find(prefix) != 0: break # all comments from the latest patch set already parsed message = message[len(prefix):].strip() result = None matched_drivers = set() for driver in candidate_drivers: ci = driver['ci'] if ci['id'] != comment_author: continue # try to get result by parsing comment message success_pattern = ci.get('success_pattern') failure_pattern = ci.get('failure_pattern') message_lines = (l for l in message.split('\n') if l.strip()) line = '' for line in message_lines: if success_pattern and re.search(success_pattern, line): result = True break elif failure_pattern and re.search(failure_pattern, line): result = False break if result is not None: matched_drivers.add(driver['name']) record = { 'user_id': user_processor.make_user_id( ci_id=driver['name']), 'value': result, 'message': line, 'date': comment['timestamp'], 'branch': review['branch'], 'review_id': review_id, 'review_number': review_number, 'driver_name': driver['name'], 'driver_vendor': driver['vendor'], 'module': review['module'] } if review['branch'].find('/') > 0: record['release'] = review['branch'].split('/')[1] yield record candidate_drivers = [d for d in candidate_drivers if d['name'] not in matched_drivers] if not candidate_drivers: break # found results from all drivers