def _merge_infer_report_files(root_paths, merged_out_path): potential_report_files = [ os.path.join(p, config.JSON_REPORT_FILENAME) for p in root_paths ] report_files = filter(os.path.exists, potential_report_files) all_results = issues.merge_reports_from_paths(report_files) utils.dump_json_to_path(all_results, merged_out_path)
def capture_with_flavors(self): ret = self._run_buck_with_flavors() if not ret == os.EX_OK: return ret result_files = self._get_analysis_result_files() all_results = issues.merge_reports_from_paths(result_files) merged_results_path = os.path.join(self.args.infer_out, config.JSON_REPORT_FILENAME) utils.dump_json_to_path(all_results, merged_results_path) print('Results saved in {results_path}'.format( results_path=merged_results_path)) return os.EX_OK
def save_report(reports, filename): # sorting to avoid spurious differences between two lists of reports reports.sort() def filter_report(report): return dict((k, v) for (k, v) in report.items() if k in REPORT_FIELDS) def should_report(report): return len(report) > 0 filtered = filter(should_report, map(filter_report, reports)) utils.dump_json_to_path(filtered, filename, separators=(',', ': '), sort_keys=True)
def _merge_infer_report_files(root_paths, merged_out_path): potential_report_files = [os.path.join(p, config.JSON_REPORT_FILENAME) for p in root_paths] report_files = filter(os.path.exists, potential_report_files) all_results = issues.merge_reports_from_paths(report_files) utils.dump_json_to_path(all_results, merged_out_path)
def collect_results(args, start_time): """Walks through buck-gen, collects results for the different buck targets and stores them in in args.infer_out/results.csv. """ buck_stats = get_buck_stats() logging.info(buck_stats) with open(os.path.join(args.infer_out, ANALYSIS_SUMMARY_OUTPUT), 'w') as f: f.write(buck_stats) all_csv_rows = set() all_json_rows = set() headers = [] stats = init_stats(args, start_time) accumulation_whitelist = list( map(re.compile, [ '^cores$', '^time$', '^start_time$', '.*_pc', ])) expected_analyzer = stats['normal']['analyzer'] expected_version = stats['normal']['infer_version'] for root, _, files in os.walk(DEFAULT_BUCK_OUT_GEN): for f in [f for f in files if f.endswith('.jar')]: path = os.path.join(root, f) try: with zipfile.ZipFile(path) as jar: # Accumulate integers and float values target_stats = load_stats(jar) found_analyzer = target_stats['normal']['analyzer'] found_version = target_stats['normal']['infer_version'] if (found_analyzer != expected_analyzer or found_version != expected_version): continue else: for type_k in ['int', 'float']: items = target_stats.get(type_k, {}).items() for key, value in items: if not any( map(lambda r: r.match(key), accumulation_whitelist)): old_value = stats[type_k].get(key, 0) stats[type_k][key] = old_value + value csv_rows = load_csv_report(jar) if len(csv_rows) > 0: headers.append(csv_rows[0]) for row in csv_rows[1:]: all_csv_rows.add(tuple(row)) json_rows = load_json_report(jar) for row in json_rows: all_json_rows.add(json.dumps(row)) # Override normals stats['normal'].update(target_stats.get('normal', {})) except NotFoundInJar: pass except zipfile.BadZipfile: logging.warn('Bad zip file %s', path) csv_report = os.path.join(args.infer_out, config.CSV_REPORT_FILENAME) json_report = os.path.join(args.infer_out, config.JSON_REPORT_FILENAME) bugs_out = os.path.join(args.infer_out, config.BUGS_FILENAME) if len(headers) == 0: with open(csv_report, 'w'): pass logging.info('No reports found') return elif len(headers) > 1: if any(map(lambda x: x != headers[0], headers)): raise Exception('Inconsistent reports found') # Convert all float values to integer values for key, value in stats.get('float', {}).items(): stats['int'][key] = int(round(value)) # Delete the float entries before exporting the results del (stats['float']) with open(csv_report, 'w') as report: writer = csv.writer(report) all_csv_rows = [list(row) for row in all_csv_rows] writer.writerows([headers[0]] + all_csv_rows) report.flush() with open(json_report, 'w') as report: json_string = '[' json_string += ','.join(all_json_rows) json_string += ']' report.write(json_string) report.flush() print('\n') issues.print_and_save_errors(json_report, bugs_out) stats['int']['total_time'] = int(round(utils.elapsed_time(start_time))) store_performances_csv(args.infer_out, stats) stats_filename = os.path.join(args.infer_out, config.STATS_FILENAME) utils.dump_json_to_path(stats, stats_filename) basic_stats = get_basic_stats(stats) if args.print_harness: harness_code = get_harness_code() basic_stats += harness_code logging.info(basic_stats) with open(os.path.join(args.infer_out, ANALYSIS_SUMMARY_OUTPUT), 'a') as f: f.write(basic_stats)
def collect_results(args, start_time, targets): """Walks through buck-gen, collects results for the different buck targets and stores them in in args.infer_out/results.csv. """ buck_stats = get_buck_stats() logging.info(buck_stats) with open(os.path.join(args.infer_out, ANALYSIS_SUMMARY_OUTPUT), 'w') as f: f.write(buck_stats) all_csv_rows = set() all_json_rows = set() headers = [] stats = init_stats(args, start_time) accumulation_whitelist = list(map(re.compile, [ '^cores$', '^time$', '^start_time$', '.*_pc', ])) expected_analyzer = stats['normal']['analyzer'] expected_version = stats['normal']['infer_version'] for path in get_output_jars(targets): try: with zipfile.ZipFile(path) as jar: # Accumulate integers and float values target_stats = load_stats(jar) found_analyzer = target_stats['normal']['analyzer'] found_version = target_stats['normal']['infer_version'] if found_analyzer != expected_analyzer \ or found_version != expected_version: continue else: for type_k in ['int', 'float']: items = target_stats.get(type_k, {}).items() for key, value in items: if not any(map(lambda r: r.match(key), accumulation_whitelist)): old_value = stats[type_k].get(key, 0) stats[type_k][key] = old_value + value csv_rows = load_csv_report(jar) if len(csv_rows) > 0: headers.append(csv_rows[0]) for row in csv_rows[1:]: all_csv_rows.add(tuple(row)) json_rows = load_json_report(jar) for row in json_rows: all_json_rows.add(json.dumps(row)) # Override normals stats['normal'].update(target_stats.get('normal', {})) except NotFoundInJar: pass except zipfile.BadZipfile: logging.warn('Bad zip file %s', path) csv_report = os.path.join(args.infer_out, config.CSV_REPORT_FILENAME) json_report = os.path.join(args.infer_out, config.JSON_REPORT_FILENAME) bugs_out = os.path.join(args.infer_out, config.BUGS_FILENAME) if len(headers) > 1: if any(map(lambda x: x != headers[0], headers)): raise Exception('Inconsistent reports found') # Convert all float values to integer values for key, value in stats.get('float', {}).items(): stats['int'][key] = int(round(value)) # Delete the float entries before exporting the results del(stats['float']) with open(csv_report, 'w') as report: if len(headers) > 0: writer = csv.writer(report) all_csv_rows = [list(row) for row in all_csv_rows] writer.writerows([headers[0]] + all_csv_rows) report.flush() with open(json_report, 'w') as report: json_string = '[' json_string += ','.join(all_json_rows) json_string += ']' report.write(json_string) report.flush() print('\n') xml_out = None if args.pmd_xml: xml_out = os.path.join(args.infer_out, config.PMD_XML_FILENAME) issues.print_and_save_errors(json_report, bugs_out, xml_out) stats['int']['total_time'] = int(round(utils.elapsed_time(start_time))) store_performances_csv(args.infer_out, stats) stats_filename = os.path.join(args.infer_out, config.STATS_FILENAME) utils.dump_json_to_path(stats, stats_filename) basic_stats = get_basic_stats(stats) if args.print_harness: harness_code = get_harness_code() basic_stats += harness_code logging.info(basic_stats) with open(os.path.join(args.infer_out, ANALYSIS_SUMMARY_OUTPUT), 'a') as f: f.write(basic_stats)
def collect_results(args, start_time): """Walks through buck-gen, collects results for the different buck targets and stores them in in args.infer_out/results.csv. """ buck_stats = get_buck_stats() logging.info(buck_stats) with open(os.path.join(args.infer_out, ANALYSIS_SUMMARY_OUTPUT), "w") as f: f.write(buck_stats) all_csv_rows = set() all_json_rows = set() headers = [] stats = init_stats(args, start_time) accumulation_whitelist = list(map(re.compile, ["^cores$", "^time$", "^start_time$", ".*_pc"])) expected_analyzer = stats["normal"]["analyzer"] expected_version = stats["normal"]["infer_version"] for root, _, files in os.walk(DEFAULT_BUCK_OUT_GEN): for f in [f for f in files if f.endswith(".jar")]: path = os.path.join(root, f) try: with zipfile.ZipFile(path) as jar: # Accumulate integers and float values target_stats = load_stats(jar) found_analyzer = target_stats["normal"]["analyzer"] found_version = target_stats["normal"]["infer_version"] if found_analyzer != expected_analyzer or found_version != expected_version: continue else: for type_k in ["int", "float"]: items = target_stats.get(type_k, {}).items() for key, value in items: if not any(map(lambda r: r.match(key), accumulation_whitelist)): old_value = stats[type_k].get(key, 0) stats[type_k][key] = old_value + value csv_rows = load_csv_report(jar) if len(csv_rows) > 0: headers.append(csv_rows[0]) for row in csv_rows[1:]: all_csv_rows.add(tuple(row)) json_rows = load_json_report(jar) for row in json_rows: all_json_rows.add(json.dumps(row)) # Override normals stats["normal"].update(target_stats.get("normal", {})) except NotFoundInJar: pass except zipfile.BadZipfile: logging.warn("Bad zip file %s", path) csv_report = os.path.join(args.infer_out, config.CSV_REPORT_FILENAME) json_report = os.path.join(args.infer_out, config.JSON_REPORT_FILENAME) bugs_out = os.path.join(args.infer_out, config.BUGS_FILENAME) if len(headers) > 1: if any(map(lambda x: x != headers[0], headers)): raise Exception("Inconsistent reports found") # Convert all float values to integer values for key, value in stats.get("float", {}).items(): stats["int"][key] = int(round(value)) # Delete the float entries before exporting the results del (stats["float"]) with open(csv_report, "w") as report: if len(headers) > 0: writer = csv.writer(report) all_csv_rows = [list(row) for row in all_csv_rows] writer.writerows([headers[0]] + all_csv_rows) report.flush() with open(json_report, "w") as report: json_string = "[" json_string += ",".join(all_json_rows) json_string += "]" report.write(json_string) report.flush() print("\n") xml_out = None if args.pmd_xml: xml_out = os.path.join(args.infer_out, config.PMD_XML_FILENAME) issues.print_and_save_errors(json_report, bugs_out, xml_out) stats["int"]["total_time"] = int(round(utils.elapsed_time(start_time))) store_performances_csv(args.infer_out, stats) stats_filename = os.path.join(args.infer_out, config.STATS_FILENAME) utils.dump_json_to_path(stats, stats_filename) basic_stats = get_basic_stats(stats) if args.print_harness: harness_code = get_harness_code() basic_stats += harness_code logging.info(basic_stats) with open(os.path.join(args.infer_out, ANALYSIS_SUMMARY_OUTPUT), "a") as f: f.write(basic_stats)
def collect_results(args, start_time, targets): """Walks through buck-gen, collects results for the different buck targets and stores them in in args.infer_out/results.csv. """ buck_stats = get_buck_stats() logging.info(buck_stats) with open(os.path.join(args.infer_out, ANALYSIS_SUMMARY_OUTPUT), 'w') as f: f.write(buck_stats) all_json_rows = set() stats = init_stats(args, start_time) accumulation_whitelist = list(map(re.compile, [ '^cores$', '^time$', '^start_time$', '.*_pc', ])) expected_analyzer = stats['normal']['analyzer'] expected_version = stats['normal']['infer_version'] for path in get_output_jars(targets): try: with zipfile.ZipFile(path) as jar: # Accumulate integers and float values target_stats = load_stats(jar) found_analyzer = target_stats['normal']['analyzer'] found_version = target_stats['normal']['infer_version'] if found_analyzer != expected_analyzer \ or found_version != expected_version: continue else: for type_k in ['int', 'float']: items = target_stats.get(type_k, {}).items() for key, value in items: if not any(map(lambda r: r.match(key), accumulation_whitelist)): old_value = stats[type_k].get(key, 0) stats[type_k][key] = old_value + value json_rows = load_json_report(jar) for row in json_rows: all_json_rows.add(json.dumps(row)) # Override normals stats['normal'].update(target_stats.get('normal', {})) except NotFoundInJar: pass except zipfile.BadZipfile: logging.warn('Bad zip file %s', path) json_report = os.path.join(args.infer_out, config.JSON_REPORT_FILENAME) # Convert all float values to integer values for key, value in stats.get('float', {}).items(): stats['int'][key] = int(round(value)) # Delete the float entries before exporting the results del(stats['float']) with open(json_report, 'w') as report: json_string = '[' json_string += ','.join(all_json_rows) json_string += ']' report.write(json_string) report.flush() print('\n') json_report = os.path.join(args.infer_out, config.JSON_REPORT_FILENAME) bugs_out = os.path.join(args.infer_out, config.BUGS_FILENAME) issues.print_and_save_errors(args.infer_out, args.project_root, json_report, bugs_out, args.pmd_xml) stats['int']['total_time'] = int(round(utils.elapsed_time(start_time))) store_performances_csv(args.infer_out, stats) stats_filename = os.path.join(args.infer_out, config.STATS_FILENAME) utils.dump_json_to_path(stats, stats_filename) basic_stats = get_basic_stats(stats) if args.print_harness: harness_code = get_harness_code() basic_stats += harness_code logging.info(basic_stats) with open(os.path.join(args.infer_out, ANALYSIS_SUMMARY_OUTPUT), 'a') as f: f.write(basic_stats)