def _normalize_user(user):
for c in user['companies']:
c['end_date'] = utils.date_to_timestamp(c['end_date'])
# sort companies by end_date
def end_date_comparator(x, y):
if x["end_date"] == 0:
return 1
elif y["end_date"] == 0:
return -1
else:
return x["end_date"] - y["end_date"]
user['companies'].sort(key=utils.cmp_to_key(end_date_comparator))
if user['companies']:
if user['companies'][-1]['end_date'] != 0:
user['companies'].append(dict(company_name='*independent',
end_date=0))
user['user_id'] = user_processor.make_user_id(
launchpad_id=user.get('launchpad_id'),
emails=user.get('emails'),
gerrit_id=user.get('gerrit_id'),
github_id=user.get('github_id'),
zanata_id=user.get('zanata_id'),
ldap_id=user.get('ldap_id')) or user.get('user_id')
def _update_with_driverlog_data(default_data, driverlog_data_uri):
LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri)
driverlog_data = utils.read_json_from_uri(driverlog_data_uri)
if not driverlog_data:
LOG.warning('DriverLog data is not available')
return
module_cis = collections.defaultdict(list)
for driver in driverlog_data['drivers']:
if 'ci' not in driver:
continue
module = (driver.get('repo') or driver['project_id']).split('/')[1]
module_cis[module].append(driver)
default_data['users'].append({
'user_id': user_processor.make_user_id(ci_id=driver['name']),
'user_name': driver['name'],
'static': True,
'companies': [
{'company_name': driver['vendor'], 'end_date': None}],
})
for repo in default_data['repos']:
if repo['module'] in module_cis:
repo['drivers'] = module_cis[repo['module']]
def _create_user(self, launchpad_id, email, gerrit_id, user_name):
company = (self._get_company_by_email(email)
or self._get_independent())
emails = []
if email:
emails = [email]
user = {
'user_id':
user_processor.make_user_id(emails=emails,
launchpad_id=launchpad_id,
gerrit_id=gerrit_id),
'launchpad_id':
launchpad_id,
'user_name':
user_name or '',
'companies': [{
'company_name': company,
'end_date': 0,
}],
'emails':
emails,
}
if gerrit_id:
user['gerrit_id'] = gerrit_id
return user
def _process_member(self, record):
user_id = user_processor.make_user_id(member_id=record['member_id'])
record['primary_key'] = user_id
record['date'] = utils.member_date_to_timestamp(record['date_joined'])
record['author_name'] = record['member_name']
record['module'] = 'unknown'
company_draft = record['company_draft']
company_name = self.domains_index.get(
utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft))
# author_email is a key to create new user
record['author_email'] = user_id
record['company_name'] = company_name
# _update_record_and_user function will create new user if needed
self._update_record_and_user(record)
record['company_name'] = company_name
user = user_processor.load_user(self.runtime_storage_inst, user_id)
user['user_name'] = record['author_name']
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user['company_name'] = company_name
user_processor.store_user(self.runtime_storage_inst, user)
record['company_name'] = company_name
yield record
def _update_with_driverlog_data(default_data, driverlog_data_uri):
LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri)
driverlog_data = utils.read_json_from_uri(driverlog_data_uri)
module_ci_ids = {}
ci_ids = set()
for driver in driverlog_data['drivers']:
if 'ci' in driver:
module = driver['project_id'].split('/')[1]
if module not in module_ci_ids:
module_ci_ids[module] = {}
ci_id = driver['ci']['id']
module_ci_ids[module][ci_id] = driver
if ci_id not in ci_ids:
ci_ids.add(ci_id)
default_data['users'].append({
'user_id': user_processor.make_user_id(gerrit_id=ci_id),
'gerrit_id': ci_id,
'user_name': ci_id,
'static': True,
'companies': [
{'company_name': driver['vendor'], 'end_date': None}],
})
for repo in default_data['repos']:
if repo['module'] in module_ci_ids:
repo['ci'] = module_ci_ids[repo['module']]
def _update_with_driverlog_data(default_data, driverlog_data_uri):
LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri)
driverlog_data = utils.read_json_from_uri(driverlog_data_uri)
if not driverlog_data:
LOG.warning('DriverLog data is not available')
return
module_cis = collections.defaultdict(list)
for driver in driverlog_data['drivers']:
if 'ci' not in driver:
continue
module = (driver.get('repo') or driver['project_id']).split('/')[1]
module_cis[module].append(driver)
default_data['users'].append({
'user_id':
user_processor.make_user_id(ci_id=driver['name']),
'user_name':
driver['name'],
'static':
True,
'companies': [{
'company_name': driver['vendor'],
'end_date': None
}],
})
for repo in default_data['repos']:
if repo['module'] in module_cis:
repo['drivers'] = module_cis[repo['module']]
def _process_member(self, record):
user_id = user_processor.make_user_id(member_id=record['member_id'])
record['primary_key'] = user_id
record['date'] = utils.member_date_to_timestamp(record['date_joined'])
record['author_name'] = record['member_name']
record['module'] = 'unknown'
company_draft = record['company_draft']
company_name = self.domains_index.get(utils.normalize_company_name(
company_draft)) or (utils.normalize_company_draft(company_draft))
# author_email is a key to create new user
record['author_email'] = user_id
record['company_name'] = company_name
# _update_record_and_user function will create new user if needed
self._update_record_and_user(record)
record['company_name'] = company_name
user = user_processor.load_user(self.runtime_storage_inst, user_id)
user['user_name'] = record['author_name']
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user['company_name'] = company_name
user_processor.store_user(self.runtime_storage_inst, user)
record['company_name'] = company_name
yield record
def _update_with_driverlog_data(default_data, driverlog_data_uri):
LOG.info('Reading DriverLog data from uri: %s', driverlog_data_uri)
driverlog_data = utils.read_json_from_uri(driverlog_data_uri)
module_ci_ids = {}
ci_ids = set()
for driver in driverlog_data['drivers']:
if 'ci' in driver:
module = driver['project_id'].split('/')[1]
if module not in module_ci_ids:
module_ci_ids[module] = {}
ci_id = driver['ci']['id']
module_ci_ids[module][ci_id] = driver
if ci_id not in ci_ids:
ci_ids.add(ci_id)
default_data['users'].append({
'user_id': user_processor.make_user_id(gerrit_id=ci_id),
'gerrit_id': ci_id,
'user_name': ci_id,
'static': True,
'companies': [
{'company_name': driver['vendor'], 'end_date': None}],
})
for repo in default_data['repos']:
if repo['module'] in module_ci_ids:
repo['ci'] = module_ci_ids[repo['module']]
def _process_member(self, record):
user_id = user_processor.make_user_id(member_id=record["member_id"])
record["primary_key"] = user_id
record["date"] = utils.member_date_to_timestamp(record["date_joined"])
record["author_name"] = record["member_name"]
record["module"] = "unknown"
company_draft = record["company_draft"]
company_name = self.domains_index.get(utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft)
)
# author_email is a key to create new user
record["author_email"] = user_id
record["company_name"] = company_name
# _update_record_and_user function will create new user if needed
self._update_record_and_user(record)
record["company_name"] = company_name
user = user_processor.load_user(self.runtime_storage_inst, user_id)
user["user_name"] = record["author_name"]
user["companies"] = [{"company_name": company_name, "end_date": 0}]
user["company_name"] = company_name
user_processor.store_user(self.runtime_storage_inst, user)
record["company_name"] = company_name
yield record
def _normalize_user(user):
for c in user['companies']:
c['end_date'] = utils.date_to_timestamp(c['end_date'])
# sort companies by end_date
def end_date_comparator(x, y):
if x["end_date"] == 0:
return 1
elif y["end_date"] == 0:
return -1
else:
return x["end_date"] - y["end_date"]
user['companies'].sort(key=utils.cmp_to_key(end_date_comparator))
if user['companies']:
if user['companies'][-1]['end_date'] != 0:
user['companies'].append(
dict(company_name='*independent', end_date=0))
user['user_id'] = user_processor.make_user_id(
launchpad_id=user.get('launchpad_id'),
emails=user.get('emails'),
gerrit_id=user.get('gerrit_id'),
github_id=user.get('github_id'),
zanata_id=user.get('zanata_id'),
ldap_id=user.get('ldap_id')) or user.get('user_id')
def _create_user(self, launchpad_id, email, gerrit_id, user_name):
company = self._get_company_by_email(email) or self._get_independent()
emails = []
if email:
emails = [email]
user = {
"user_id": user_processor.make_user_id(emails=emails, launchpad_id=launchpad_id, gerrit_id=gerrit_id),
"launchpad_id": launchpad_id,
"user_name": user_name or "",
"companies": [{"company_name": company, "end_date": 0}],
"emails": emails,
}
if gerrit_id:
user["gerrit_id"] = gerrit_id
return user
def _normalize_user(user):
for c in user['companies']:
c['end_date'] = utils.date_to_timestamp(c['end_date'])
# sort companies by end_date
def end_date_comparator(x, y):
if x["end_date"] == 0:
return 1
elif y["end_date"] == 0:
return -1
else:
return x["end_date"] - y["end_date"]
user['companies'].sort(key=utils.cmp_to_key(end_date_comparator))
user['user_id'] = user_processor.make_user_id(
launchpad_id=user.get('launchpad_id'), email=user.get('email'))
def _process_translation(self, record):
# todo split translation and approval
translation = record.copy()
user_id = user_processor.make_user_id(zanata_id=record['zanata_id'])
translation['record_type'] = 'tr'
translation['primary_key'] = '%s:%s:%s:%s' % (
user_id, record['module'], record['date'], record['branch'])
translation['author_name'] = user_id
# following fields are put into standard fields stored in dashboard mem
translation['loc'] = record['translated']
translation['value'] = record['language']
self._update_record_and_user(translation)
yield translation
def _process_translation(self, record):
# todo split translation and approval
translation = record.copy()
user_id = user_processor.make_user_id(zanata_id=record['zanata_id'])
translation['record_type'] = 'tr'
translation['primary_key'] = '%s:%s:%s:%s' % (
user_id, record['module'], record['date'], record['branch'])
translation['author_name'] = user_id
# following fields are put into standard fields stored in dashboard mem
translation['loc'] = record['translated']
translation['value'] = record['language']
self._update_record_and_user(translation)
yield translation
def _normalize_user(user):
for c in user['companies']:
c['end_date'] = utils.date_to_timestamp(c['end_date'])
# sort companies by end_date
def end_date_comparator(x, y):
if x["end_date"] == 0:
return 1
elif y["end_date"] == 0:
return -1
else:
return x["end_date"] - y["end_date"]
user['companies'].sort(key=utils.cmp_to_key(end_date_comparator))
user['user_id'] = user_processor.make_user_id(
launchpad_id=user.get('launchpad_id'),
email=user.get('email'))
def _create_user(self, launchpad_id, email, gerrit_id, user_name):
company = (self._get_company_by_email(email) or
self._get_independent())
emails = []
if email:
emails = [email]
user = {
'user_id': user_processor.make_user_id(
emails=emails, launchpad_id=launchpad_id, gerrit_id=gerrit_id),
'launchpad_id': launchpad_id,
'user_name': user_name or '',
'companies': [{
'company_name': company,
'end_date': 0,
}],
'emails': emails,
}
if gerrit_id:
user['gerrit_id'] = gerrit_id
return user
def _find_ci_result(review, drivers):
"""For a given stream of reviews yields results produced by CIs."""
review_id = review['id']
review_number = review['number']
ci_id_set = set(d['ci']['id'] for d in drivers)
candidate_drivers = [d for d in drivers]
last_patch_set_number = review['patchSets'][-1]['number']
for comment in reversed(review.get('comments') or []):
comment_author = comment['reviewer'].get('username')
if comment_author not in ci_id_set:
continue # not any of registered CIs
message = comment['message']
prefix = 'Patch Set'
if comment['message'].find(prefix) != 0:
continue # look for special messages only
prefix = 'Patch Set %s:' % last_patch_set_number
if comment['message'].find(prefix) != 0:
break # all comments from the latest patch set already parsed
message = message[len(prefix):].strip()
result = None
matched_drivers = set()
for driver in candidate_drivers:
ci = driver['ci']
if ci['id'] != comment_author:
continue
# try to get result by parsing comment message
success_pattern = ci.get('success_pattern')
failure_pattern = ci.get('failure_pattern')
message_lines = (l for l in message.split('\n') if l.strip())
line = ''
for line in message_lines:
if success_pattern and re.search(success_pattern, line):
result = True
break
elif failure_pattern and re.search(failure_pattern, line):
result = False
break
if result is not None:
matched_drivers.add(driver['name'])
record = {
'user_id':
user_processor.make_user_id(ci_id=driver['name']),
'value': result,
'message': line,
'date': comment['timestamp'],
'branch': review['branch'],
'review_id': review_id,
'review_number': review_number,
'driver_name': driver['name'],
'driver_vendor': driver['vendor'],
'module': review['module']
}
if review['branch'].find('/') > 0:
record['release'] = review['branch'].split('/')[1]
yield record
candidate_drivers = [
d for d in candidate_drivers if d['name'] not in matched_drivers
]
if not candidate_drivers:
break # found results from all drivers
def _find_ci_result(review, drivers):
"""For a given stream of reviews yields results produced by CIs."""
review_id = review['id']
review_number = review['number']
ci_id_set = set(d['ci']['id'] for d in drivers)
candidate_drivers = [d for d in drivers]
last_patch_set_number = review['patchSets'][-1]['number']
for comment in reversed(review.get('comments') or []):
comment_author = comment['reviewer'].get('username')
if comment_author not in ci_id_set:
continue # not any of registered CIs
message = comment['message']
prefix = 'Patch Set'
if comment['message'].find(prefix) != 0:
continue # look for special messages only
prefix = 'Patch Set %s:' % last_patch_set_number
if comment['message'].find(prefix) != 0:
break # all comments from the latest patch set already parsed
message = message[len(prefix):].strip()
result = None
matched_drivers = set()
for driver in candidate_drivers:
ci = driver['ci']
if ci['id'] != comment_author:
continue
# try to get result by parsing comment message
success_pattern = ci.get('success_pattern')
failure_pattern = ci.get('failure_pattern')
message_lines = (l for l in message.split('\n') if l.strip())
line = ''
for line in message_lines:
if success_pattern and re.search(success_pattern, line):
result = True
break
elif failure_pattern and re.search(failure_pattern, line):
result = False
break
if result is not None:
matched_drivers.add(driver['name'])
record = {
'user_id': user_processor.make_user_id(
ci_id=driver['name']),
'value': result,
'message': line,
'date': comment['timestamp'],
'branch': review['branch'],
'review_id': review_id,
'review_number': review_number,
'driver_name': driver['name'],
'driver_vendor': driver['vendor'],
'module': review['module']
}
if review['branch'].find('/') > 0:
record['release'] = review['branch'].split('/')[1]
yield record
candidate_drivers = [d for d in candidate_drivers
if d['name'] not in matched_drivers]
if not candidate_drivers:
break # found results from all drivers
