def normalize_ingestion_data(self, start_date, end_date, ingestion_data, frequency='daily'): """Normalize worker data for reporting.""" logger.info("Normalize Ingestion Data started") report_type = 'ingestion-data' if frequency == 'monthly': report_name = dt.strptime(end_date, '%Y-%m-%d').strftime('%Y-%m') else: report_name = dt.strptime(end_date, '%Y-%m-%d').strftime('%Y-%m-%d') template = { 'report': { 'from': start_date, 'to': end_date, 'generated_on': dt.now().isoformat('T') }, 'ingestion_summary': {}, 'ingestion_details': {} } epv_data = ingestion_data['EPV_DATA'] epv_data = json.loads(epv_data) # Populate the default template with EPV info template, epvs = self.populate_default_information(epv_data, template) logger.info("Fetching details of the latest version for the epvs") today = dt.today() pkg_output = generate_report_for_latest_version(epvs, today) logger.info("Fetching details of the unknown packages for the epvs") ver_output = generate_report_for_unknown_epvs(epvs) # Call the function to add the package information to the template template, latest_epvs = self.generate_results(epvs, template, pkg_output, ver_output) # Call the function to get the availability of latest node logger.info("Checking if latest node exists in graph") template = self.check_latest_node(latest_epvs, template) # Saving the final report in the relevant S3 bucket try: obj_key = '{type}/epv/{report_name}.json'.format( type=report_type, report_name=report_name) self.s3.store_json_content(content=template, obj_key=obj_key, bucket_name=self.s3.report_bucket_name) except Exception as e: logger.exception('Unable to store the report on S3. Reason: %r' % e) return template
def check_latest_node(self, latest_epvs, template): """Get if latest node is present in graph.""" graph_output = generate_report_for_unknown_epvs(latest_epvs) missing_latest = {} for epv in latest_epvs: eco = epv['ecosystem'] pkg = epv['name'] ver = epv['version'] output = graph_output[eco + "@DELIM@" + pkg + "@DELIM@" + ver] template['ingestion_details'][eco][pkg][ 'latest_node_in_graph'] = output # If the EPV is missing in graph, add it to the summary if output == "false": if eco not in missing_latest: missing_latest[eco] = [] tmp = {"package": pkg, "version": ver} missing_latest[eco].append(tmp) template['ingestion_summary']['missing_latest_node'] = missing_latest return template
def normalize_ingestion_data(self, start_date, end_date, ingestion_data, frequency='daily'): """Normalize worker data for reporting.""" report_type = 'ingestion-data' if frequency == 'monthly': report_name = dt.strptime(end_date, '%Y-%m-%d').strftime('%Y-%m') else: report_name = dt.strptime(end_date, '%Y-%m-%d').strftime('%Y-%m-%d') template = { 'report': { 'from': start_date, 'to': end_date, 'generated_on': dt.now().isoformat('T') }, 'ingestion_summary': {}, 'ingestion_details_v2': {} } all_deps_count = {'all': 0} failed_deps_count = {'all': 0} all_epv_list_v2 = {} # Graph Availability validation starts success_epv_data = ingestion_data['EPV_GRAPH_SUCCESS_DATA'] success_epv_data = json.loads(success_epv_data) graph_input = [] for data in success_epv_data: all_deps_count['all'] = all_deps_count['all'] + 1 if all_deps_count.get(data[0]) is None: all_deps_count[data[0]] = 0 all_deps_count[data[0]] = all_deps_count[data[0]] + 1 graph_template = { 'ecosystem': data[0], 'name': data[1], 'version': data[2] } graph_input.append(graph_template) # The below graph call determines the eistence of ingested epvs in the graph graph_output = generate_report_for_unknown_epvs(graph_input) for attributes, values in graph_output.items(): versn_template = {} epv_arr = attributes.split('@') if all_epv_list_v2.get(epv_arr[0]) is None: all_epv_list_v2[epv_arr[0]] = {} all_epv_list_v2[epv_arr[0]][epv_arr[1]] = {} all_epv_list_v2[epv_arr[0]][epv_arr[1]]['package_known'] = values all_epv_list_v2[epv_arr[0]][epv_arr[1]]['versions'] = [] versn_template['version'] = epv_arr[2] versn_template['ingested_in_graph'] = values if values == 'false': failed_deps_count['all'] = failed_deps_count['all'] + 1 if failed_deps_count.get(epv_arr[0]) is None: failed_deps_count[epv_arr[0]] = 0 failed_deps_count[ epv_arr[0]] = failed_deps_count[epv_arr[0]] + 1 all_epv_list_v2[epv_arr[0]][epv_arr[1]]['versions'].append( versn_template) failed_epv_data = ingestion_data['EPV_GRAPH_FAILED_DATA'] failed_epv_data = json.loads(failed_epv_data) for data in failed_epv_data: versn_template = {} all_deps_count['all'] = all_deps_count['all'] + 1 if all_deps_count.get(data[0]) is None: all_deps_count[data[0]] = 0 all_deps_count[data[0]] = all_deps_count[data[0]] + 1 failed_deps_count['all'] = failed_deps_count['all'] + 1 if failed_deps_count.get(data[0]) is None: failed_deps_count[data[0]] = 0 failed_deps_count[data[0]] = failed_deps_count[data[0]] + 1 if all_epv_list_v2.get(data[0]) is None: all_epv_list_v2[data[0]] = {} all_epv_list_v2[data[0]][data[1]] = {} all_epv_list_v2[data[0]][data[1]]['versions'] = [] versn_template['version'] = data[2] versn_template['ingested_in_graph'] = 'false' all_epv_list_v2[data[0]][data[1]]['versions'].append( versn_template) graph_template = { 'ecosystem': data[0], 'name': data[1], 'version': data[2] } graph_input.append(graph_template) # The below graph call determines the latest version information for # ingested epvs in the graph graph_output = generate_report_for_latest_version(graph_input) for attributes, values in graph_output.items(): epv_arr = attributes.split('@') if all_epv_list_v2.get(epv_arr[0]) is None: all_epv_list_v2[epv_arr[0]] = {} all_epv_list_v2[epv_arr[0]][epv_arr[1]]['known_latest_version'] = \ values['known_latest_version'] all_epv_list_v2[epv_arr[0]][epv_arr[1]]['actual_latest_version'] = \ values['actual_latest_version'] if values['known_latest_version'] == '': all_epv_list_v2[epv_arr[0]][ epv_arr[1]]['package_known'] = 'false' template['ingestion_details_v2'] = all_epv_list_v2 # creating the epv ingestion statistics info according to the ecosystems template['ingestion_summary'][ 'total_epv_ingestion_count'] = all_deps_count['all'] for data in all_deps_count: if failed_deps_count.get(data) is None: failed_deps_count[data] = 0 stats_template = { 'epv_ingestion_count': all_deps_count[data], 'epv_successfully_ingested_count': all_deps_count[data] - failed_deps_count[data], 'failed_epv_ingestion_count': failed_deps_count[data] } template['ingestion_summary'][data] = stats_template # Saving the final report in the relevant S3 bucket try: obj_key = '{depl_prefix}/{type}/epv/{report_name}.json'.format( depl_prefix=self.s3.deployment_prefix, type=report_type, report_name=report_name) self.s3.store_json_content(content=template, obj_key=obj_key, bucket_name=self.s3.report_bucket_name) except Exception as e: logger.exception('Unable to store the report on S3. Reason: %r' % e) return template