def get_ecosystem_summary(self, ecosystem, total_stack_requests, all_deps, all_unknown_deps, unique_stacks_with_recurrence_count, unique_stacks_with_deps_count, avg_response_time, unknown_deps_ingestion_report): """Generate ecosystem specific stack summary.""" unique_dep_frequency = self.populate_key_count( self.flatten_list(all_deps[ecosystem])) rectify_latest_version(unique_dep_frequency, ecosystem, True) return { 'stack_requests_count': total_stack_requests[ecosystem], 'unique_dependencies_with_frequency': self.populate_key_count(self.flatten_list(all_deps[ecosystem])), 'unique_unknown_dependencies_with_frequency': unique_dep_frequency, 'unique_stacks_with_frequency': unique_stacks_with_recurrence_count[ecosystem], 'unique_stacks_with_deps_count': unique_stacks_with_deps_count[ecosystem], 'average_response_time': '{} ms'.format(avg_response_time[ecosystem]), 'trending': { 'top_stacks': self.get_trending( unique_stacks_with_recurrence_count[ecosystem], 3), 'top_deps': self.get_trending( self.populate_key_count( self.flatten_list(all_deps[ecosystem])), 5), }, 'previously_unknown_dependencies': unknown_deps_ingestion_report[ecosystem] }
def generate_results(self, epvs, template, pkg_output, ver_output): """Get package information from graph.""" template['ingestion_summary']['incorrect_latest_version'] = {} template['ingestion_summary']['unknown_deps'] = {} count = {} latest_epvs = [] checked_pkgs = [] for epv in epvs: eco = epv['ecosystem'] pkg = epv['name'] ver = epv['version'] # Add parameters to count the different params if eco not in count: count[eco] = { 'incorrect_latest_versions': 0, 'correct_latest_versions': 0, 'ingested_in_graph': 0, 'not_ingested_in_graph': 0 } if eco not in template['ingestion_summary']['incorrect_latest_version']: template['ingestion_summary']['incorrect_latest_version'][eco] = [] template['ingestion_summary']['unknown_deps'][eco] = [] pkg_data = pkg_output[eco + "@DELIM@" + pkg] ver_data = ver_output[eco + "@DELIM@" + pkg + "@DELIM@" + ver] actual_latest_ver = pkg_data['actual_latest_version'] # check if the package is publicly available if actual_latest_ver: known_latest_ver = pkg_data['known_latest_version'] if actual_latest_ver != known_latest_ver and (eco + "@DELIM@" + pkg not in checked_pkgs): checked_pkgs.append(eco + "@DELIM@" + pkg) tmp = { "package": pkg, "actual_latest_version": actual_latest_ver, "known_latest_version": known_latest_ver } template['ingestion_summary']['incorrect_latest_version'][eco].append(tmp) count[eco]['incorrect_latest_versions'] += 1 template['ingestion_details'][eco][pkg]['known_latest_version'] \ = known_latest_ver template['ingestion_details'][eco][pkg]['actual_latest_version'] \ = actual_latest_ver non_cve_version = pkg_data.get('non_cve_version', '') if non_cve_version: template['ingestion_details'][eco][pkg]['non_cve_version'] \ = non_cve_version latest_json = { "ecosystem": eco, "name": pkg, "version": actual_latest_ver } latest_epvs.append(latest_json) # Count the correct latest version EPVs if actual_latest_ver == known_latest_ver: count[eco]['correct_latest_versions'] += 1 # Add to report if the EPV exist in the graph or not template['ingestion_details'][eco][pkg][ver]['synced_to_graph'] = ver_data if ver_data == "false": template['ingestion_summary']['unknown_deps'][eco].append(epv) count[eco]['not_ingested_in_graph'] += 1 else: count[eco]['ingested_in_graph'] += 1 else: # Mark the package as private as the information is not present publicly template['ingestion_details'][eco][pkg]['private_pkg'] = "true" # For each ecosystem, calculate the %age accuracy for eco in count: correct = count[eco]['correct_latest_versions'] incorrect = count[eco]['incorrect_latest_versions'] # Calculate the %age of latest version accuracy if correct != 0 or incorrect != 0: count[eco]['latest_version_accuracy'] = round(((correct * 100) / (correct + incorrect)), 2) correct = count[eco]['ingested_in_graph'] incorrect = count[eco]['not_ingested_in_graph'] # Calculate the %age of successful ingestion if correct != 0 or incorrect != 0: count[eco]['ingestion_accuracy'] = round(((correct * 100) / (correct + incorrect)), 2) # Rectify the latest versions only if present if count[eco]['incorrect_latest_versions'] != 0: summary = template['ingestion_summary'] logger.info("Information related to incorrect latest version --") logger.info(summary['incorrect_latest_version'][eco]) rectify_latest_version(summary['incorrect_latest_version'][eco], eco) template['ingestion_summary']['stats'] = count return template, latest_epvs