def get_ecosystem_summary(self, ecosystem, total_stack_requests, all_deps,
all_unknown_deps,
unique_stacks_with_recurrence_count,
unique_stacks_with_deps_count, avg_response_time,
unknown_deps_ingestion_report):
"""Generate ecosystem specific stack summary."""
unique_dep_frequency = self.populate_key_count(
self.flatten_list(all_deps[ecosystem]))
rectify_latest_version(unique_dep_frequency, ecosystem, True)
return {
'stack_requests_count':
total_stack_requests[ecosystem],
'unique_dependencies_with_frequency':
self.populate_key_count(self.flatten_list(all_deps[ecosystem])),
'unique_unknown_dependencies_with_frequency':
unique_dep_frequency,
'unique_stacks_with_frequency':
unique_stacks_with_recurrence_count[ecosystem],
'unique_stacks_with_deps_count':
unique_stacks_with_deps_count[ecosystem],
'average_response_time':
'{} ms'.format(avg_response_time[ecosystem]),
'trending': {
'top_stacks':
self.get_trending(
unique_stacks_with_recurrence_count[ecosystem], 3),
'top_deps':
self.get_trending(
self.populate_key_count(
self.flatten_list(all_deps[ecosystem])), 5),
},
'previously_unknown_dependencies':
unknown_deps_ingestion_report[ecosystem]
}
def generate_results(self, epvs, template, pkg_output, ver_output):
"""Get package information from graph."""
template['ingestion_summary']['incorrect_latest_version'] = {}
template['ingestion_summary']['unknown_deps'] = {}
count = {}
latest_epvs = []
checked_pkgs = []
for epv in epvs:
eco = epv['ecosystem']
pkg = epv['name']
ver = epv['version']
# Add parameters to count the different params
if eco not in count:
count[eco] = {
'incorrect_latest_versions': 0,
'correct_latest_versions': 0,
'ingested_in_graph': 0,
'not_ingested_in_graph': 0
}
if eco not in template['ingestion_summary']['incorrect_latest_version']:
template['ingestion_summary']['incorrect_latest_version'][eco] = []
template['ingestion_summary']['unknown_deps'][eco] = []
pkg_data = pkg_output[eco + "@DELIM@" + pkg]
ver_data = ver_output[eco + "@DELIM@" + pkg + "@DELIM@" + ver]
actual_latest_ver = pkg_data['actual_latest_version']
# check if the package is publicly available
if actual_latest_ver:
known_latest_ver = pkg_data['known_latest_version']
if actual_latest_ver != known_latest_ver and (eco + "@DELIM@" + pkg
not in checked_pkgs):
checked_pkgs.append(eco + "@DELIM@" + pkg)
tmp = {
"package": pkg,
"actual_latest_version": actual_latest_ver,
"known_latest_version": known_latest_ver
}
template['ingestion_summary']['incorrect_latest_version'][eco].append(tmp)
count[eco]['incorrect_latest_versions'] += 1
template['ingestion_details'][eco][pkg]['known_latest_version'] \
= known_latest_ver
template['ingestion_details'][eco][pkg]['actual_latest_version'] \
= actual_latest_ver
non_cve_version = pkg_data.get('non_cve_version', '')
if non_cve_version:
template['ingestion_details'][eco][pkg]['non_cve_version'] \
= non_cve_version
latest_json = {
"ecosystem": eco,
"name": pkg,
"version": actual_latest_ver
}
latest_epvs.append(latest_json)
# Count the correct latest version EPVs
if actual_latest_ver == known_latest_ver:
count[eco]['correct_latest_versions'] += 1
# Add to report if the EPV exist in the graph or not
template['ingestion_details'][eco][pkg][ver]['synced_to_graph'] = ver_data
if ver_data == "false":
template['ingestion_summary']['unknown_deps'][eco].append(epv)
count[eco]['not_ingested_in_graph'] += 1
else:
count[eco]['ingested_in_graph'] += 1
else:
# Mark the package as private as the information is not present publicly
template['ingestion_details'][eco][pkg]['private_pkg'] = "true"
# For each ecosystem, calculate the %age accuracy
for eco in count:
correct = count[eco]['correct_latest_versions']
incorrect = count[eco]['incorrect_latest_versions']
# Calculate the %age of latest version accuracy
if correct != 0 or incorrect != 0:
count[eco]['latest_version_accuracy'] = round(((correct * 100) /
(correct + incorrect)), 2)
correct = count[eco]['ingested_in_graph']
incorrect = count[eco]['not_ingested_in_graph']
# Calculate the %age of successful ingestion
if correct != 0 or incorrect != 0:
count[eco]['ingestion_accuracy'] = round(((correct * 100) /
(correct + incorrect)), 2)
# Rectify the latest versions only if present
if count[eco]['incorrect_latest_versions'] != 0:
summary = template['ingestion_summary']
logger.info("Information related to incorrect latest version --")
logger.info(summary['incorrect_latest_version'][eco])
rectify_latest_version(summary['incorrect_latest_version'][eco], eco)
template['ingestion_summary']['stats'] = count
return template, latest_epvs