def extend_record(record):
record = vault.extend_record(record)
_extend_record_common_fields(record)
if record['record_type'] == 'commit':
record['branches'] = ','.join(record['branches'])
if 'correction_comment' not in record:
record['correction_comment'] = ''
record['message'] = make_commit_message(record)
if record['commit_date']:
record['commit_date_str'] = format_datetime(record['commit_date'])
elif record['record_type'] == 'mark':
review = vault.get_memory_storage().get_record_by_primary_key(
record['review_id'])
patch = vault.get_memory_storage().get_record_by_primary_key(
utils.get_patch_id(record['review_id'], record['patch']))
if not review or not patch:
return None
_extend_by_parent_info(record, review, 'parent_')
_extend_by_parent_info(record, patch, 'patch_')
elif record['record_type'] == 'patch':
review = vault.get_memory_storage().get_record_by_primary_key(
record['review_id'])
_extend_by_parent_info(record, review, 'parent_')
return record
def extend_record(record):
record = vault.extend_record(record)
_extend_record_common_fields(record)
if record['record_type'] == 'commit':
record['branches'] = ','.join(record['branches'])
if 'correction_comment' not in record:
record['correction_comment'] = ''
record['message'] = make_commit_message(record)
if record['commit_date']:
record['commit_date_str'] = format_datetime(record['commit_date'])
elif record['record_type'] == 'mark':
review = vault.get_memory_storage().get_record_by_primary_key(
record['review_id'])
patch = vault.get_memory_storage().get_record_by_primary_key(
utils.get_patch_id(record['review_id'], record['patch']))
if not review or not patch:
return None
_extend_by_parent_info(record, review, 'parent_')
_extend_by_parent_info(record, patch, 'patch_')
elif record['record_type'] == 'patch':
review = vault.get_memory_storage().get_record_by_primary_key(
record['review_id'])
_extend_by_parent_info(record, review, 'parent_')
elif record['record_type'] == 'email':
record['email_link'] = record.get('email_link') or ''
record['blueprint_links'] = []
for bp_id in record.get('blueprint_id', []):
bp_module, bp_name = bp_id.split(':')
record['blueprint_links'].append(
make_blueprint_link(bp_module, bp_name))
elif record['record_type'] in ['bpd', 'bpc']:
record['summary'] = utils.format_text(record['summary'])
if record.get('mention_count'):
record['mention_date_str'] = format_datetime(
record['mention_date'])
record['blueprint_link'] = make_blueprint_link(record['module'],
record['name'])
elif record['record_type'] in ['bugr', 'bugf']:
record['number'] = record['web_link'].split('/')[-1]
record['title'] = filter_bug_title(record['title'])
record['status_class'] = re.sub('\s+', '', record['status'])
elif record['record_type'] == 'tr':
record['date_str'] = format_date(record['date']) # no need for hours
return record
def extend_record(record):
record = vault.extend_record(record)
_extend_record_common_fields(record)
if record['record_type'] == 'commit':
record['branches'] = ','.join(record['branches'])
if 'correction_comment' not in record:
record['correction_comment'] = ''
record['message'] = make_commit_message(record)
if record['commit_date']:
record['commit_date_str'] = format_datetime(record['commit_date'])
elif record['record_type'] == 'mark':
review = vault.get_memory_storage().get_record_by_primary_key(
record['review_id'])
patch = vault.get_memory_storage().get_record_by_primary_key(
utils.get_patch_id(record['review_id'], record['patch']))
if not review or not patch:
return None
_extend_by_parent_info(record, review, 'parent_')
_extend_by_parent_info(record, patch, 'patch_')
elif record['record_type'] == 'patch':
review = vault.get_memory_storage().get_record_by_primary_key(
record['review_id'])
_extend_by_parent_info(record, review, 'parent_')
elif record['record_type'] == 'email':
record['email_link'] = record.get('email_link') or ''
record['blueprint_links'] = []
for bp_id in record.get('blueprint_id', []):
bp_module, bp_name = bp_id.split(':')
record['blueprint_links'].append(
make_blueprint_link(bp_module, bp_name))
elif record['record_type'] in ['bpd', 'bpc']:
record['summary'] = utils.format_text(record['summary'])
if record.get('mention_count'):
record['mention_date_str'] = format_datetime(
record['mention_date'])
record['blueprint_link'] = make_blueprint_link(record['module'],
record['name'])
elif record['record_type'] in ['bugr', 'bugf']:
record['number'] = record['web_link'].split('/')[-1]
record['title'] = filter_bug_title(record['title'])
record['status_class'] = re.sub('\s+', '', record['status'])
elif record['record_type'] == 'tr':
record['date_str'] = format_date(record['date']) # no need for hours
return record
def get_single_plus_two_reviews_report(records, **kwargs):
memory_storage_inst = vault.get_memory_storage()
plus_twos = collections.defaultdict(list)
for record in records:
if record['record_type'] != 'mark':
continue
if (record['branch'] == 'master' and record['type'] == 'Code-Review'
and record['value'] == +2):
review_id = record['review_id']
review = memory_storage_inst.get_record_by_primary_key(review_id)
if review and review['status'] == 'MERGED':
plus_twos[review_id].append(record)
response = []
for review_id in plus_twos.keys():
if len(plus_twos[review_id]) < 2:
mark = plus_twos[review_id][0]
review = memory_storage_inst.get_record_by_primary_key(
mark['review_id'])
response.append({
'review_by': review['user_id'],
'mark_by': mark['user_id'],
'subject': review['subject'],
'url': review['url'],
'project': review['project']
})
return response
def get_users_json(record_ids, **kwargs):
core_in = parameters.get_single_parameter(kwargs, 'core_in') or None
valid_modules = set()
if core_in:
core_in = set(core_in.split(','))
valid_modules = vault.resolve_project_types(
kwargs['_params']['project_type'])
valid_modules = set(m[0] for m in vault.resolve_modules(
valid_modules, kwargs['_params']['release']))
user_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
'user_id', record_ids)
if kwargs['_params']['user_id']:
user_ids.add(kwargs['_params']['user_id'][0])
result = []
for user_id in user_ids:
user = vault.get_user_from_runtime_storage(user_id)
r = {'id': user_id, 'text': user['user_name']}
add_flag = not core_in
if core_in and user.get('core'):
core_modules = [module_branch[0] for module_branch in user['core']
if (module_branch[1] in core_in and
module_branch[0] in valid_modules)]
if core_modules:
r['core'] = core_modules
if user['companies']:
r['company_name'] = user['companies'][-1]['company_name']
add_flag = True
if add_flag:
result.append(r)
result.sort(key=lambda x: x['text'])
return result
def open_reviews(module):
memory_storage_inst = vault.get_memory_storage()
time_now = int(time.time())
module_id_index = vault.get_vault()['module_id_index']
module = module.lower()
if module not in module_id_index:
flask.abort(404)
modules = module_id_index[module]['modules']
review_ids = (memory_storage_inst.get_record_ids_by_modules(modules) &
memory_storage_inst.get_record_ids_by_types(['review']))
waiting_on_reviewer = []
total_open = 0
for review in memory_storage_inst.get_records(review_ids):
if review.status == 'NEW':
total_open += 1
if review.value in [1, 2]:
waiting_on_reviewer.append(vault.extend_record(review))
return {
'module': module,
'total_open': total_open,
'waiting_on_reviewer': len(waiting_on_reviewer),
'waiting_on_submitter': total_open - len(waiting_on_reviewer),
'latest_revision': _process_stat(
waiting_on_reviewer, 'updated_on', time_now),
'first_revision': _process_stat(waiting_on_reviewer, 'date', time_now),
}
def get_users_json(record_ids, **kwargs):
core_in = parameters.get_single_parameter(kwargs, 'core_in') or None
valid_modules = set()
if core_in:
core_in = set(core_in.split(','))
valid_modules = vault.resolve_project_types(
kwargs['_params']['project_type'])
valid_modules = set(m[0] for m in vault.resolve_modules(
valid_modules, kwargs['_params']['release']))
user_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
'user_id', record_ids)
if kwargs['_params']['user_id']:
user_ids.add(kwargs['_params']['user_id'][0])
result = []
for user_id in user_ids:
user = vault.get_user_from_runtime_storage(user_id)
r = {'id': user_id, 'text': user['user_name']}
add_flag = not core_in
if core_in and user.get('core'):
core_modules = [module_branch[0] for module_branch in user['core']
if (module_branch[1] in core_in and
module_branch[0] in valid_modules)]
if core_modules:
r['core'] = core_modules
if user['companies']:
r['company_name'] = user['companies'][-1]['company_name']
add_flag = True
if add_flag:
result.append(r)
result.sort(key=lambda x: x['text'])
return result
def get_modules_json(record_ids, **kwargs):
module_id_index = vault.get_vault()['module_id_index']
tags = parameters.get_parameter({}, 'tag', 'tags')
# all modules mentioned in records
module_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
'module', record_ids)
add_modules = set([])
for module in six.itervalues(module_id_index):
if set(module['modules']) & module_ids:
add_modules.add(module['id'])
module_ids |= add_modules
# keep only modules with specified tags
if tags:
module_ids = set(module_id for module_id in module_ids
if ((module_id in module_id_index) and
(module_id_index[module_id].get('tag') in tags)))
result = []
for module_id in module_ids:
module = module_id_index[module_id]
result.append({'id': module['id'],
'text': module['module_group_name'],
'tag': module['tag']})
return sorted(result, key=operator.itemgetter('text'))
def get_single_plus_two_reviews_report(records, **kwargs):
memory_storage_inst = vault.get_memory_storage()
plus_twos = collections.defaultdict(list)
for record in records:
if record['record_type'] != 'mark':
continue
if (record['branch'] == 'master' and
record['type'] == 'Code-Review' and record['value'] == +2):
review_id = record['review_id']
review = memory_storage_inst.get_record_by_primary_key(review_id)
if review and review['status'] == 'MERGED':
plus_twos[review_id].append(record)
response = []
for review_id in plus_twos.keys():
if len(plus_twos[review_id]) < 2:
mark = plus_twos[review_id][0]
review = memory_storage_inst.get_record_by_primary_key(
mark['review_id'])
response.append({'review_by': review['user_id'],
'mark_by': mark['user_id'],
'subject': review['subject'],
'url': review['url'],
'project': review['project']})
return response
def blueprint_summary(module, blueprint_name):
blueprint_id = utils.get_blueprint_id(module, blueprint_name)
bpd = vault.get_memory_storage().get_record_by_primary_key(
'bpd:' + blueprint_id)
if not bpd:
flask.abort(404)
return
bpd = helpers.extend_record(bpd)
record_ids = vault.get_memory_storage().get_record_ids_by_blueprint_ids(
[blueprint_id])
activity = [helpers.extend_record(record) for record in
vault.get_memory_storage().get_records(record_ids)]
activity.sort(key=lambda x: x['date'], reverse=True)
return {'blueprint': bpd, 'activity': activity}
def blueprint_summary(module, blueprint_name):
blueprint_id = utils.get_blueprint_id(module, blueprint_name)
bpd = vault.get_memory_storage().get_record_by_primary_key(
'bpd:' + blueprint_id)
if not bpd:
flask.abort(404)
return
bpd = helpers.extend_record(bpd)
record_ids = vault.get_memory_storage().get_record_ids_by_blueprint_ids(
[blueprint_id])
activity = [helpers.extend_record(record) for record in
vault.get_memory_storage().get_records(record_ids)]
activity.sort(key=lambda x: x['date'], reverse=True)
return {'blueprint': bpd, 'activity': activity}
def get_modules_json(record_ids, **kwargs):
module_id_index = vault.get_vault()['module_id_index']
tags = parameters.get_parameter({}, 'tag', 'tags')
# all modules mentioned in records
module_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
'module', record_ids)
add_modules = set([])
for module in six.itervalues(module_id_index):
if set(module['modules']) & module_ids:
add_modules.add(module['id'])
module_ids |= add_modules
# keep only modules with specified tags
if tags:
module_ids = set(module_id for module_id in module_ids
if ((module_id in module_id_index) and (
module_id_index[module_id].get('tag') in tags)))
result = []
for module_id in module_ids:
module = module_id_index[module_id]
result.append({
'id': module['id'],
'text': module['module_group_name'],
'tag': module['tag']
})
return sorted(result, key=operator.itemgetter('text'))
def get_languages_json(record_ids, **kwargs):
memory_storage = vault.get_memory_storage()
languages = set(r.value for r in memory_storage.get_records(record_ids))
return [{
'id': c.lower().replace('&', ''),
'text': c
} for c in sorted(languages)]
def record(record_id):
memory_storage_inst = vault.get_memory_storage()
record_obj = memory_storage_inst.get_record_by_primary_key(record_id)
if not record_obj:
flask.abort(404)
result = dict(record=helpers.get_activity([record_obj], 0, 1)[0])
return result
def record(record_id):
memory_storage_inst = vault.get_memory_storage()
record_obj = memory_storage_inst.get_record_by_primary_key(record_id)
if not record_obj:
flask.abort(404)
result = dict(record=helpers.get_activity([record_obj], 0, 1)[0])
return result
def company_activity(company):
memory_storage_inst = vault.get_memory_storage()
original_name = memory_storage_inst.get_original_company_name(company)
result = _get_activity_summary(
memory_storage_inst.get_record_ids_by_companies([original_name]))
result['company_name'] = original_name
return result
def company_activity(company):
memory_storage_inst = vault.get_memory_storage()
original_name = memory_storage_inst.get_original_company_name(company)
result = _get_activity_summary(
memory_storage_inst.get_record_ids_by_companies([original_name]))
result['company_name'] = original_name
return result
def generate_records_for_person_day(record_ids):
memory_storage_inst = vault.get_memory_storage()
id_dates = []
for record in memory_storage_inst.get_records(record_ids):
id_dates.append((record.date, record.record_id))
id_dates.sort(key=operator.itemgetter(0))
for record in memory_storage_inst.get_records(
record_id for date, record_id in id_dates):
yield record
def generate_records_for_person_day(record_ids):
memory_storage_inst = vault.get_memory_storage()
id_dates = []
for record in memory_storage_inst.get_records(record_ids):
id_dates.append((record.date, record.record_id))
id_dates.sort(key=operator.itemgetter(0))
for record in memory_storage_inst.get_records(
record_id for date, record_id in id_dates):
yield record
def get_company(company_name, **kwargs):
memory_storage_inst = vault.get_memory_storage()
for company in memory_storage_inst.get_companies():
if company.lower() == company_name.lower():
return {
'id': company_name,
'text': memory_storage_inst.get_original_company_name(
company_name)
}
flask.abort(404)
def get_company(company_name, **kwargs):
memory_storage_inst = vault.get_memory_storage()
for company in memory_storage_inst.get_companies():
if company.lower() == company_name.lower():
return {
'id': company_name,
'text':
memory_storage_inst.get_original_company_name(company_name)
}
flask.abort(404)
def _validate_params(params):
vault_inst = vault.get_vault()
memory_storage_inst = vault.get_memory_storage()
_check_param_in(params, 'release', vault_inst['releases'], True)
_check_param_in(params, 'project_type', vault_inst['project_types_index'])
_check_param_in(params, 'module', vault_inst['module_id_index'])
_check_param_in(params, 'company',
memory_storage_inst.get_companies_lower())
_check_param_in(params, 'user_id', memory_storage_inst.get_user_ids())
_check_param_in(params, 'metric', parameters.METRIC_TO_RECORD_TYPE, True)
def open_reviews(module):
memory_storage_inst = vault.get_memory_storage()
time_now = int(time.time())
module_id_index = vault.get_vault()['module_id_index']
module = module.lower()
if module not in module_id_index:
flask.abort(404)
modules = module_id_index[module]['modules']
review_ids = (memory_storage_inst.get_record_ids_by_modules(modules) &
memory_storage_inst.get_record_ids_by_types(['review']))
waiting_on_reviewer = []
waiting_on_submitter = []
total_open = 0
for review in memory_storage_inst.get_records(review_ids):
if review.status == 'NEW':
total_open += 1
# review.value is minimum from votes made for the latest patch
if review.value in [1, 2]:
# CI or engineer liked this change request, waiting for someone
# to merge or to put dislike
waiting_on_reviewer.append(helpers.extend_record(review))
elif review.value in [-1, -2]:
# CI or reviewer do not like this, waiting for submitter to fix
waiting_on_submitter.append(helpers.extend_record(review))
else:
# new requests without votes, waiting for CI
pass
limit = int(flask.request.args.get('limit') or CONF.report_default_limit)
if limit < 0:
limit = None
return {
'module': module,
'total_open': total_open,
'waiting_on_reviewer': len(waiting_on_reviewer),
'waiting_on_submitter': len(waiting_on_submitter),
'waiting_on_ci': (total_open - len(waiting_on_reviewer) -
len(waiting_on_submitter)),
'reviewer_latest_revision': _process_stat(
waiting_on_reviewer, 'updated_on', time_now, limit),
'reviewer_first_revision': _process_stat(
waiting_on_reviewer, 'date', time_now, limit),
'submitter_latest_revision': _process_stat(
waiting_on_submitter, 'updated_on', time_now, limit),
'submitter_first_revision': _process_stat(
waiting_on_submitter, 'date', time_now, limit),
}
def open_reviews(module):
memory_storage_inst = vault.get_memory_storage()
time_now = int(time.time())
module_id_index = vault.get_vault()['module_id_index']
module = module.lower()
if module not in module_id_index:
flask.abort(404)
modules = module_id_index[module]['modules']
review_ids = (memory_storage_inst.get_record_ids_by_modules(modules)
& memory_storage_inst.get_record_ids_by_types(['review']))
waiting_on_reviewer = []
waiting_on_submitter = []
total_open = 0
for review in memory_storage_inst.get_records(review_ids):
if review.status == 'NEW':
total_open += 1
# review.value is minimum from votes made for the latest patch
if review.value in [1, 2]:
# CI or engineer liked this change request, waiting for someone
# to merge or to put dislike
waiting_on_reviewer.append(helpers.extend_record(review))
elif review.value in [-1, -2]:
# CI or reviewer do not like this, waiting for submitter to fix
waiting_on_submitter.append(helpers.extend_record(review))
else:
# new requests without votes, waiting for CI
pass
return {
'module':
module,
'total_open':
total_open,
'waiting_on_reviewer':
len(waiting_on_reviewer),
'waiting_on_submitter':
len(waiting_on_submitter),
'waiting_on_ci':
(total_open - len(waiting_on_reviewer) - len(waiting_on_submitter)),
'reviewer_latest_revision':
_process_stat(waiting_on_reviewer, 'updated_on', time_now),
'reviewer_first_revision':
_process_stat(waiting_on_reviewer, 'date', time_now),
'submitter_latest_revision':
_process_stat(waiting_on_submitter, 'updated_on', time_now),
'submitter_first_revision':
_process_stat(waiting_on_submitter, 'date', time_now),
}
def _validate_params(params):
vault_inst = vault.get_vault()
memory_storage_inst = vault.get_memory_storage()
_check_param_in(params, 'release', vault_inst['releases'], True)
_check_param_in(params, 'project_type', vault_inst['project_types_index'])
_check_param_in(params, 'module', vault_inst['module_id_index'])
_check_param_in(params, 'company',
memory_storage_inst.get_companies_lower())
_check_param_in(params, 'user_id', memory_storage_inst.get_user_ids())
_check_param_in(params, 'metric', parameters.METRIC_TO_RECORD_TYPE, True)
def get_companies_json(record_ids, **kwargs):
memory_storage = vault.get_memory_storage()
companies = set(memory_storage.get_original_company_name(company)
for company in memory_storage.get_index_keys_by_record_ids(
'company_name', record_ids))
if kwargs['_params']['company']:
companies.add(memory_storage.get_original_company_name(
kwargs['_params']['company'][0]))
return [{'id': utils.safe_encode(c.lower()), 'text': c}
for c in sorted(companies)]
def user_activity(user_id):
user = vault.get_user_from_runtime_storage(user_id)
if not user:
flask.abort(404)
user = helpers.extend_user(user)
memory_storage_inst = vault.get_memory_storage()
result = _get_activity_summary(
memory_storage_inst.get_record_ids_by_user_ids([user_id]))
result['user'] = user
return result
def user_activity(user_id):
user = vault.get_user_from_runtime_storage(user_id)
if not user:
flask.abort(404)
user = helpers.extend_user(user)
memory_storage_inst = vault.get_memory_storage()
result = _get_activity_summary(
memory_storage_inst.get_record_ids_by_user_ids([user_id]))
result['user'] = user
return result
def get_companies_json(record_ids, **kwargs):
memory_storage = vault.get_memory_storage()
companies = memory_storage.get_index_keys_by_record_ids(
'company_name', record_ids)
if kwargs['_params']['company']:
companies.add(kwargs['_params']['company'][0])
result = [memory_storage.get_original_company_name(company)
for company in companies]
return [{'id': utils.safe_encode(c.lower()), 'text': c}
for c in sorted(result)]
def get_companies_json(record_ids, **kwargs):
memory_storage = vault.get_memory_storage()
companies = set(company
for company in memory_storage.get_index_keys_by_record_ids(
'company_name', record_ids))
if kwargs['_params']['company']:
companies.add(memory_storage.get_original_company_name(
kwargs['_params']['company'][0]))
return [{'id': c.lower().replace('&', ''), 'text': c}
for c in sorted(companies)]
def get_users_json(record_ids, **kwargs):
user_ids = vault.get_memory_storage().get_index_keys_by_record_ids(
'user_id', record_ids)
if kwargs['_params']['user_id']:
user_ids.add(kwargs['_params']['user_id'][0])
result = [{'id': user_id,
'text': (vault.get_user_from_runtime_storage(user_id)
['user_name'])}
for user_id in user_ids]
result.sort(key=lambda x: x['text'])
return result
def _get_activity_summary(record_ids):
memory_storage_inst = vault.get_memory_storage()
record_ids_by_type = memory_storage_inst.get_record_ids_by_types(
['mark', 'patch', 'email', 'bpd', 'bpc', 'ci'])
record_ids &= record_ids_by_type
punch_card_data = _get_punch_card_data(
memory_storage_inst.get_records(record_ids))
return {
'punch_card_data': punch_card_data,
}
def _get_activity_summary(record_ids):
memory_storage_inst = vault.get_memory_storage()
record_ids_by_type = memory_storage_inst.get_record_ids_by_types(
['mark', 'patch', 'email', 'bpd', 'bpc', 'ci'])
record_ids &= record_ids_by_type
punch_card_data = _get_punch_card_data(
memory_storage_inst.get_records(record_ids))
return {
'punch_card_data': punch_card_data,
}
def extend_record(record):
record = vault.extend_record(record)
_extend_record_common_fields(record)
if record['record_type'] == 'commit':
record['branches'] = ','.join(record['branches'])
if 'correction_comment' not in record:
record['correction_comment'] = ''
record['message'] = make_commit_message(record)
if record['commit_date']:
record['commit_date_str'] = format_datetime(record['commit_date'])
elif record['record_type'] == 'mark':
parent = vault.get_memory_storage().get_record_by_primary_key(
record['review_id'])
if not parent:
return None
_extend_by_parent_info(record, parent)
elif record['record_type'] == 'patch':
parent = vault.get_memory_storage().get_record_by_primary_key(
record['review_id'])
_extend_by_parent_info(record, parent)
elif record['record_type'] == 'email':
record['email_link'] = record.get('email_link') or ''
record['blueprint_links'] = []
for bp_id in record.get('blueprint_id', []):
bp_module, bp_name = bp_id.split(':')
record['blueprint_links'].append(
make_blueprint_link(bp_module, bp_name))
elif record['record_type'] in ['bpd', 'bpc']:
record['summary'] = utils.format_text(record['summary'])
if record.get('mention_count'):
record['mention_date_str'] = format_datetime(
record['mention_date'])
record['blueprint_link'] = make_blueprint_link(record['module'],
record['name'])
return record
def get_engineers(records, metric_filter, finalize_handler, **kwargs):
modules_names = parameters.get_parameter({}, 'module', 'modules')
modules = set([m for m, r in vault.resolve_modules(modules_names, [''])])
def postprocessing(record):
if finalize_handler:
record = finalize_handler(record)
user = vault.get_user_from_runtime_storage(record['id'])
record['core'] = get_core_engineer_branch(user, modules)
return record
return _get_aggregated_stats(records, metric_filter,
vault.get_memory_storage().get_user_ids(),
'user_id', 'author_name',
finalize_handler=postprocessing)
def get_engineers(records, metric_filter, finalize_handler, **kwargs):
modules_names = parameters.get_parameter(kwargs, 'module')
modules = set([m for m, r in vault.resolve_modules(modules_names, [''])])
def postprocessing(record):
if finalize_handler:
record = finalize_handler(record)
user = vault.get_user_from_runtime_storage(record['id'])
record['core'] = get_core_engineer_branch(user, modules)
return record
return _get_aggregated_stats(records, metric_filter,
vault.get_memory_storage().get_user_ids(),
'user_id', 'author_name',
finalize_handler=postprocessing)
def _get_activity_summary(record_ids):
memory_storage_inst = vault.get_memory_storage()
types = ['mark', 'patch', 'email', 'bpd', 'bpc']
record_ids_by_type = set()
for t in types:
record_ids_by_type |= memory_storage_inst.get_record_ids_by_type(t)
record_ids &= record_ids_by_type
punch_card_data = _get_punch_card_data(
memory_storage_inst.get_records(record_ids))
return {
'punch_card_data': punch_card_data,
}
def make_page_title(company, user_id, module, release):
if company:
memory_storage = vault.get_memory_storage()
company = memory_storage.get_original_company_name(company)
if company or user_id:
if user_id:
s = vault.get_user_from_runtime_storage(user_id)['user_name']
if company:
s += ' (%s)' % company
else:
s = company
else:
s = 'OpenStack community'
s += ' contribution'
if module:
s += ' to %s' % module
if release != 'all':
s += ' in %s release' % release.capitalize()
else:
s += ' in all releases'
return s
def make_page_title(company, user_id, module, release):
if company:
memory_storage = vault.get_memory_storage()
company = memory_storage.get_original_company_name(company)
if company or user_id:
if user_id:
s = vault.get_user_from_runtime_storage(user_id)['user_name']
if company:
s += ' (%s)' % company
else:
s = company
else:
s = 'OpenStack community'
s += ' contribution'
if module:
s += ' to %s' % module
if release != 'all':
s += ' in %s release' % release.capitalize()
else:
s += ' in all releases'
return s
def generate_records_for_person_day(record_ids):
memory_storage_inst = vault.get_memory_storage()
for values in memory_storage_inst.day_index.values():
for record in memory_storage_inst.get_records(record_ids & values):
yield record
def get_languages_json(record_ids, **kwargs):
memory_storage = vault.get_memory_storage()
languages = set(r.value for r in memory_storage.get_records(record_ids))
return [{'id': c.lower().replace('&', ''), 'text': c}
for c in sorted(languages)]
def record_filter_decorated_function(*args, **kwargs):
memory_storage_inst = vault.get_memory_storage()
record_ids = None
params = _prepare_params(kwargs, ignore)
release = params['release']
if release:
if 'all' not in release:
record_ids = (
memory_storage_inst.get_record_ids_by_releases(
c.lower() for c in release))
project_type = params['project_type']
mr = None
if project_type:
mr = set(vault.resolve_modules(vault.resolve_project_types(
project_type), release))
module = params['module']
if module:
mr = _intersect(mr, set(vault.resolve_modules(
module, release)))
if mr is not None:
record_ids = _intersect(
record_ids, _filter_records_by_modules(
memory_storage_inst, mr))
user_id = params['user_id']
user_id = [u for u in user_id
if vault.get_user_from_runtime_storage(u)]
if user_id:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_user_ids(user_id))
company = params['company']
if company:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_companies(company))
metric = params['metric']
if 'all' not in metric:
for metric in metric:
if metric in parameters.METRIC_TO_RECORD_TYPE:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_type(
parameters.METRIC_TO_RECORD_TYPE[metric]))
if 'tm_marks' in metric:
filtered_ids = []
review_nth = int(parameters.get_parameter(
kwargs, 'review_nth')[0])
for record in memory_storage_inst.get_records(record_ids):
parent = memory_storage_inst.get_record_by_primary_key(
record['review_id'])
if (parent and ('review_number' in parent) and
(parent['review_number'] <= review_nth)):
filtered_ids.append(record['record_id'])
record_ids = filtered_ids
blueprint_id = params['blueprint_id']
if blueprint_id:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_blueprint_ids(
blueprint_id))
start_date = params['start_date']
end_date = params['end_date']
if start_date or end_date:
record_ids = _intersect(
record_ids, _filter_records_by_days(start_date, end_date,
memory_storage_inst))
kwargs['record_ids'] = record_ids
kwargs['records'] = memory_storage_inst.get_records(record_ids)
return f(*args, **kwargs)
def templated_decorated_function(*args, **kwargs):
vault_inst = vault.get_vault()
template_name = template
if template_name is None:
template_name = (flask.request.endpoint.replace('.', '/') +
'.html')
ctx = f(*args, **kwargs)
if ctx is None:
ctx = {}
try:
_prepare_params(kwargs, [])
except Exception:
if return_code == 200:
raise # do not re-raise on error page
# put parameters into template
ctx['metric'] = parameters.get_single_parameter(
kwargs, 'metric', use_default=True)
ctx['metric_label'] = parameters.METRIC_LABELS.get(ctx['metric'])
project_type = parameters.get_single_parameter(
kwargs, 'project_type', use_default=True)
ctx['project_type'] = project_type
ctx['project_type_inst'] = vault.get_project_type(project_type)
ctx['release'] = parameters.get_single_parameter(
kwargs, 'release', use_default=True)
company = parameters.get_single_parameter(kwargs, 'company')
ctx['company'] = company
if company:
ctx['company_original'] = (
vault.get_memory_storage().get_original_company_name(
ctx['company']))
module = parameters.get_single_parameter(kwargs, 'module')
ctx['module'] = module
if module and module in vault_inst['module_id_index']:
ctx['module_inst'] = vault_inst['module_id_index'][module]
ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id')
if ctx['user_id']:
ctx['user_inst'] = vault.get_user_from_runtime_storage(
ctx['user_id'])
ctx['page_title'] = helpers.make_page_title(
ctx['project_type_inst'],
ctx.get('release'), ctx.get('module_inst'),
ctx.get('company_original'), ctx.get('user_inst'))
ctx['stackalytics_version'] = (
stackalytics_version.version_info.version_string())
ctx['stackalytics_release'] = (
stackalytics_version.version_info.release_string())
update_time = vault_inst['runtime_storage_update_time']
ctx['runtime_storage_update_time'] = update_time
ctx['runtime_storage_update_time_str'] = helpers.format_datetime(
update_time) if update_time else None
# deprecated -- top mentor report
ctx['review_nth'] = parameters.get_single_parameter(
kwargs, 'review_nth')
return flask.render_template(template_name, **ctx), return_code
def templated_decorated_function(*args, **kwargs):
vault_inst = vault.get_vault()
template_name = template
if template_name is None:
template_name = (flask.request.endpoint.replace('.', '/') +
'.html')
ctx = f(*args, **kwargs)
if ctx is None:
ctx = {}
# put parameters into template
metric = flask.request.args.get('metric')
if metric not in parameters.METRIC_LABELS:
metric = None
ctx['metric'] = metric or parameters.get_default('metric')
ctx['metric_label'] = parameters.METRIC_LABELS[ctx['metric']]
project_type = flask.request.args.get('project_type')
if not vault.is_project_type_valid(project_type):
project_type = parameters.get_default('project_type')
ctx['project_type'] = project_type
release = flask.request.args.get('release')
releases = vault_inst['releases']
if release:
release = release.lower()
if release != 'all':
if release not in releases:
release = None
else:
release = releases[release]['release_name']
ctx['release'] = (release or
parameters.get_default('release')).lower()
ctx['review_nth'] = (flask.request.args.get('review_nth') or
parameters.get_default('review_nth'))
ctx['company'] = parameters.get_single_parameter(kwargs, 'company')
ctx['company_original'] = (
vault.get_memory_storage().get_original_company_name(
ctx['company']))
module = parameters.get_single_parameter(kwargs, 'module')
ctx['module'] = module
module_name = None
if module and module in vault_inst['module_id_index']:
ctx['module_inst'] = vault_inst['module_id_index'][module]
module_name = ctx['module_inst']['module_group_name']
ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id')
if ctx['user_id']:
ctx['user_inst'] = vault.get_user_from_runtime_storage(
ctx['user_id'])
ctx['page_title'] = helpers.make_page_title(
ctx['company'], ctx['user_id'], module_name, ctx['release'])
ctx['stackalytics_version'] = (
stackalytics_version.version_info.version_string())
ctx['stackalytics_release'] = (
stackalytics_version.version_info.release_string())
ctx['runtime_storage_update_time'] = (
vault_inst['runtime_storage_update_time'])
return flask.render_template(template_name, **ctx), return_code
def record_filter_decorated_function(*args, **kwargs):
memory_storage_inst = vault.get_memory_storage()
record_ids = None
params = _prepare_params(kwargs, ignore)
release = params['release']
if release:
if 'all' not in release:
record_ids = (
memory_storage_inst.get_record_ids_by_releases(
c.lower() for c in release))
project_type = params['project_type']
mr = None
if project_type:
mr = set(
vault.resolve_modules(
vault.resolve_project_types(project_type), release))
module = params['module']
if module:
mr = _intersect(mr, set(vault.resolve_modules(module,
release)))
if mr is not None:
record_ids = _intersect(
record_ids,
_filter_records_by_modules(memory_storage_inst, mr))
user_id = params['user_id']
user_id = [
u for u in user_id if vault.get_user_from_runtime_storage(u)
]
if user_id:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_user_ids(user_id))
company = params['company']
if company:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_companies(company))
metric = params['metric']
if 'all' not in metric:
for m in metric:
if m in parameters.METRIC_TO_RECORD_TYPE:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_types(
parameters.METRIC_TO_RECORD_TYPE[m]))
start_date = params['start_date']
end_date = params['end_date']
if start_date or end_date:
record_ids = _intersect(
record_ids,
_filter_records_by_days(start_date, end_date,
memory_storage_inst))
kwargs['record_ids'] = record_ids
kwargs['records'] = memory_storage_inst.get_records(record_ids)
return f(*args, **kwargs)
def get_modules(records, metric_filter, finalize_handler, **kwargs):
return _get_aggregated_stats(records,
metric_filter,
vault.get_memory_storage().get_modules(),
'module',
finalize_handler=finalize_handler)
def record_filter_decorated_function(*args, **kwargs):
memory_storage_inst = vault.get_memory_storage()
record_ids = None
params = _prepare_params(kwargs, ignore)
release = params['release']
if release:
if 'all' not in release:
record_ids = (
memory_storage_inst.get_record_ids_by_releases(
c.lower() for c in release))
project_type = params['project_type']
mr = None
if project_type:
mr = set(
vault.resolve_modules(
vault.resolve_project_types(project_type), release))
module = params['module']
if module:
mr = _intersect(mr, set(vault.resolve_modules(module,
release)))
if mr is not None:
record_ids = _intersect(
record_ids,
_filter_records_by_modules(memory_storage_inst, mr))
user_id = params['user_id']
user_id = [
u for u in user_id if vault.get_user_from_runtime_storage(u)
]
if user_id:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_user_ids(user_id))
company = params['company']
if company:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_companies(company))
metric = params['metric']
if 'all' not in metric:
for metric in metric:
if metric in parameters.METRIC_TO_RECORD_TYPE:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_types(
parameters.METRIC_TO_RECORD_TYPE[metric]))
if 'tm_marks' in metric:
filtered_ids = []
review_nth = int(parameters.get_parameter('review_nth')[0])
for record in memory_storage_inst.get_records(record_ids):
parent = memory_storage_inst.get_record_by_primary_key(
record['review_id'])
if (parent and ('review_number' in parent)
and (parent['review_number'] <= review_nth)):
filtered_ids.append(record['record_id'])
record_ids = filtered_ids
blueprint_id = params['blueprint_id']
if blueprint_id:
record_ids = _intersect(
record_ids,
memory_storage_inst.get_record_ids_by_blueprint_ids(
blueprint_id))
start_date = params['start_date']
end_date = params['end_date']
if start_date or end_date:
record_ids = _intersect(
record_ids,
_filter_records_by_days(start_date, end_date,
memory_storage_inst))
kwargs['record_ids'] = record_ids
kwargs['records'] = memory_storage_inst.get_records(record_ids)
return f(*args, **kwargs)
def templated_decorated_function(*args, **kwargs):
vault_inst = vault.get_vault()
template_name = template
if template_name is None:
template_name = (flask.request.endpoint.replace('.', '/') +
'.html')
ctx = f(*args, **kwargs)
if ctx is None:
ctx = {}
try:
_prepare_params(kwargs, [])
except Exception:
if return_code == 200:
raise # do not re-raise on error page
# put parameters into template
ctx['metric'] = parameters.get_single_parameter(kwargs,
'metric',
use_default=True)
ctx['metric_label'] = parameters.METRIC_LABELS.get(ctx['metric'])
project_type = parameters.get_single_parameter(kwargs,
'project_type',
use_default=True)
ctx['project_type'] = project_type
ctx['release'] = parameters.get_single_parameter(kwargs,
'release',
use_default=True)
company = parameters.get_single_parameter(kwargs, 'company')
ctx['company'] = company
if company:
ctx['company_original'] = (
vault.get_memory_storage().get_original_company_name(
ctx['company']))
module = parameters.get_single_parameter(kwargs, 'module')
ctx['module'] = module
if module and module in vault_inst['module_id_index']:
ctx['module_inst'] = vault_inst['module_id_index'][module]
ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id')
if ctx['user_id']:
ctx['user_inst'] = vault.get_user_from_runtime_storage(
ctx['user_id'])
ctx['page_title'] = helpers.make_page_title(
ctx.get('release'), ctx.get('module_inst'),
ctx.get('company_original'), ctx.get('user_inst'))
ctx['stackalytics_version'] = (
stackalytics_version.version_info.version_string())
ctx['stackalytics_release'] = (
stackalytics_version.version_info.release_string())
ctx['runtime_storage_update_time'] = (
vault_inst['runtime_storage_update_time'])
# deprecated -- top mentor report
ctx['review_nth'] = parameters.get_single_parameter(
kwargs, 'review_nth')
return flask.render_template(template_name, **ctx), return_code
def get_modules(records, metric_filter, finalize_handler, **kwargs):
return _get_aggregated_stats(records, metric_filter,
vault.get_memory_storage().get_modules(),
'module', finalize_handler=finalize_handler)
def templated_decorated_function(*args, **kwargs):
vault_inst = vault.get_vault()
template_name = template
if template_name is None:
template_name = (flask.request.endpoint.replace('.', '/') +
'.html')
ctx = f(*args, **kwargs)
if ctx is None:
ctx = {}
# put parameters into template
metric = flask.request.args.get('metric')
if metric not in parameters.METRIC_LABELS:
metric = None
ctx['metric'] = metric or parameters.get_default('metric')
ctx['metric_label'] = parameters.METRIC_LABELS[ctx['metric']]
project_type = flask.request.args.get('project_type')
if not vault.is_project_type_valid(project_type):
project_type = parameters.get_default('project_type')
ctx['project_type'] = project_type
release = flask.request.args.get('release')
releases = vault_inst['releases']
if release:
release = release.lower()
if release != 'all':
if release not in releases:
release = None
else:
release = releases[release]['release_name']
ctx['release'] = (release or
parameters.get_default('release')).lower()
ctx['review_nth'] = (flask.request.args.get('review_nth') or
parameters.get_default('review_nth'))
ctx['company'] = parameters.get_single_parameter(kwargs, 'company')
ctx['company_original'] = (
vault.get_memory_storage().get_original_company_name(
ctx['company']))
module = parameters.get_single_parameter(kwargs, 'module')
ctx['module'] = module
module_name = None
if module and module in vault_inst['module_id_index']:
ctx['module_inst'] = vault_inst['module_id_index'][module]
module_name = ctx['module_inst']['module_group_name']
ctx['user_id'] = parameters.get_single_parameter(kwargs, 'user_id')
if ctx['user_id']:
ctx['user_inst'] = vault.get_user_from_runtime_storage(
ctx['user_id'])
ctx['page_title'] = helpers.make_page_title(
ctx['company'], ctx['user_id'], module_name, ctx['release'])
ctx['stackalytics_version'] = (
stackalytics_version.version_info.version_string())
ctx['stackalytics_release'] = (
stackalytics_version.version_info.release_string())
ctx['runtime_storage_update_time'] = (
vault_inst['runtime_storage_update_time'])
return flask.render_template(template_name, **ctx), return_code
