def skip_html_report_data_handler(report_hash, source_file, report_line, checker_name, diag, files): """ Report handler which skips bugs which were suppressed by source code comments. This function will return a tuple. The first element will decide whether the report should be skipped or not and the second element will be a list of source code comments related to the actual report. """ files_dict = {k: v for k, v in enumerate(files)} report = Report({'check_name': checker_name}, diag['path'], files_dict, metadata=None) path_hash = get_report_path_hash(report) if path_hash in processed_path_hashes: LOG.debug("Skip report because it is a deduplication of an " "already processed report!") LOG.debug("Path hash: %s", path_hash) LOG.debug(diag) return True, [] skip, source_code_comments = skip_report(report_hash, source_file, report_line, checker_name, suppr_handler, src_comment_status_filter) if skip_handler: skip |= skip_handler.should_skip(source_file) if not skip: processed_path_hashes.add(path_hash) return skip, source_code_comments
def test_report_path_hash_generation(self): """ Test report path hash generation. """ clang50_trunk_plist = os.path.join( self.__plist_test_files, 'clang-5.0-trunk.plist') files, reports = plist_parser.parse_plist_file(clang50_trunk_plist, False) self.assertEqual(len(reports), 3) # Generate dummy file_ids which should come from the database. file_ids = {} for i, file_name in enumerate(files, 1): file_ids[file_name] = i msg = "This test is prepared to handle 3 reports." self.assertEqual(len(reports), 3, msg) report_hash_to_path_hash = { '79e31a6ba028f0b7d9779faf4a6cb9cf': 'acb1d3dc1459f681bd3c743e6c015b37', '8714f42d8328bc78d5d7bff6ced918cc': 'dcaaf2905d607a16e3fa330edb8e9f89', 'a6d3464f8aab9eb31a8ea7e167e84322': 'd089a50f34051c68c7bb4c5ac2c4c5d5' } for report in reports: path_hash = get_report_path_hash(report) bug_hash = report.main['issue_hash_content_of_line_in_context'] self.assertEqual(path_hash, report_hash_to_path_hash[bug_hash])
def test_report_path_hash_generation(self): """ Test report path hash generation. """ clang50_trunk_plist = os.path.join(self.__plist_test_files, 'clang-5.0-trunk.plist') files, reports = plist_parser.parse_plist_file(clang50_trunk_plist, None, False) self.assertEqual(len(reports), 3) # Generate dummy file_ids which should come from the database. file_ids = {} for i, file_name in enumerate(files, 1): file_ids[file_name] = i msg = "This test is prepared to handle 3 reports." self.assertEqual(len(reports), 3, msg) report_hash_to_path_hash = { '79e31a6ba028f0b7d9779faf4a6cb9cf': 'c473c1a55df72ea4c6e055e18370ac65', '8714f42d8328bc78d5d7bff6ced918cc': '94f2a6eee8af6462a810218dff35056a', 'a6d3464f8aab9eb31a8ea7e167e84322': '11f410136724cf43c63526841007897e' } for report in reports: path_hash = get_report_path_hash(report.bug_path, files) bug_hash = report.main['issue_hash_content_of_line_in_context'] self.assertEqual(path_hash, report_hash_to_path_hash[bug_hash])
def test_gen_report_path_hash(self): """ Test path hash generation for multiple errors. """ test_plist = os.path.join(self.test_file_dir, 'cpp', 'multi_error.plist') plist = plistlib.readPlist(test_plist) expected_path_hash = { 'f48840093ef89e291fb68a95a5181612': 'a92b18ee6ee267cd82d0f056ef2bbe4b', 'e4907182b363faf2ec905fc32cc5a4ab': '2b010e31810074749d485f1db8e419e9' } files = plist['files'] for diag in plist['diagnostics']: file_path = files[diag['location']['file']] path_hash = get_report_path_hash(diag['path'], file_path) actual_report_hash = diag['issue_hash_content_of_line_in_context'] self.assertEqual(path_hash, expected_path_hash[actual_report_hash])
def test_gen_report_path_hash(self): """ Test path hash generation for multiple errors. """ test_plist = os.path.join(self.test_file_dir, 'cpp', 'multi_error.plist') plist = plistlib.readPlist(test_plist) expected_path_hash = { 'f48840093ef89e291fb68a95a5181612': '93cb93bdcee10434f9cf9f486947c88e', 'e4907182b363faf2ec905fc32cc5a4ab': '71a4dc24bf88af2b13be83d8d15bd6f0' } for diag in plist['diagnostics']: diag['bug_path'] = diag['path'] diag['files'] = plist['files'] path_hash = get_report_path_hash( namedtuple('Report', diag.keys())(*diag.values())) actual_report_hash = diag['issue_hash_content_of_line_in_context'] self.assertEqual(path_hash, expected_path_hash[actual_report_hash])
def skip_html_report_data_handler(report_hash, source_file, report_line, checker_name, diag, files): """ Report handler which skips bugs which were suppressed by source code comments. """ report = Report(None, diag['path'], files) path_hash = get_report_path_hash(report.bug_path, files) if path_hash in processed_path_hashes: LOG.debug("Skip report because it is a deduplication of an " "already processed report!") LOG.debug("Path hash: %s", path_hash) LOG.debug(diag) return True skip = skip_report(report_hash, source_file, report_line, checker_name, suppr_handler) if skip_handler: skip |= skip_handler.should_skip(source_file) if not skip: processed_path_hashes.add(path_hash) return skip
def write(self, file_report_map: Dict[str, List[Report]], output=sys.stdout): """ Format an already parsed plist report file to a more human readable format. The formatted text is written to the output. During writing the output statistics are collected. Write out the bugs to the output and collect report statistics. """ severity_stats = defaultdict(int) file_stats = defaultdict(int) report_count = defaultdict(int) for file_path in sorted(file_report_map, key=lambda key: len(file_report_map[key])): non_suppressed = 0 sorted_reports = sorted(file_report_map[file_path], key=lambda r: r.main['location']['line']) for report in sorted_reports: path_hash = get_report_path_hash(report) if path_hash in self._processed_path_hashes: LOG.debug("Not showing report because it is a " "deduplication of an already processed report!") LOG.debug("Path hash: %s", path_hash) LOG.debug(report) continue self._processed_path_hashes.add(path_hash) events = [ i for i in report.bug_path if i.get('kind') == 'event' ] f_path = report.files[events[-1]['location']['file']] if self.skiplist_handler and \ self.skiplist_handler.should_skip(f_path): LOG.debug("Skipped report in '%s'", f_path) LOG.debug(report) continue last_report_event = report.bug_path[-1] source_file = \ report.files[last_report_event['location']['file']] report_line = last_report_event['location']['line'] report_hash = \ report.main['issue_hash_content_of_line_in_context'] checker_name = report.main['check_name'] skip, source_code_comments = \ skip_report(report_hash, source_file, report_line, checker_name, self.src_comment_handler, self.src_comment_status_filter) if skip: continue if self._trim_path_prefixes: report.trim_path_prefixes(self._trim_path_prefixes) trimmed_source_file = \ report.files[last_report_event['location']['file']] file_stats[f_path] += 1 severity = self.__severity_map.get(checker_name) severity_stats[severity] += 1 report_count["report_count"] += 1 review_status = None if len(source_code_comments) == 1: review_status = source_code_comments[0]['status'] output.write( self.__format_bug_event(checker_name, severity, last_report_event, trimmed_source_file, review_status)) output.write('\n') # Print source code comments. for source_code_comment in source_code_comments: output.write(source_code_comment['line'].rstrip()) output.write('\n') output.write( self.__format_location(last_report_event, source_file)) output.write('\n') if self.print_steps: output.write(' Report hash: ' + report_hash + '\n') # Print out macros. macros = report.macro_expansions if macros: output.write(' Macro expansions:\n') index_format = ' %%%dd, ' % \ int(math.floor( math.log10(len(macros))) + 1) for index, macro in enumerate(macros): output.write(index_format % (index + 1)) source = report.files[macro['location']['file']] output.write( self.__format_macro_expansion(macro, source)) output.write('\n') # Print out notes. notes = report.notes if notes: output.write(' Notes:\n') index_format = ' %%%dd, ' % \ int(math.floor( math.log10(len(notes))) + 1) for index, note in enumerate(notes): output.write(index_format % (index + 1)) source_file = report.files[note['location'] ['file']] output.write( self.__format_bug_note(note, source_file)) output.write('\n') output.write(' Steps:\n') index_format = ' %%%dd, ' % \ int(math.floor(math.log10(len(events))) + 1) for index, event in enumerate(events): output.write(index_format % (index + 1)) source_file = report.files[event['location']['file']] output.write( self.__format_bug_event(None, None, event, source_file)) output.write('\n') output.write('\n') non_suppressed += 1 base_file = os.path.basename(file_path) if non_suppressed == 0: output.write('Found no defects in %s\n' % base_file) else: output.write('Found %d defect(s) in %s\n\n' % (non_suppressed, base_file)) return { "severity": severity_stats, "files": file_stats, "reports": report_count }
def __process_report_file(self, report_file_path: str, session: DBSession, source_root: str, run_id: int, file_path_to_id: Dict[str, int], run_history_time: datetime, skip_handler: skiplist_handler.SkipListHandler, hash_map_reports: Dict[str, List[Any]]) -> bool: """ Process and save reports from the given report file to the database. """ try: files, reports = plist_parser.parse_plist_file(report_file_path) except Exception as ex: LOG.warning('Parsing the plist failed: %s', str(ex)) return False if not reports: return True trimmed_files = {} file_ids = {} missing_ids_for_files = [] for k, v in files.items(): trimmed_files[k] = \ util.trim_path_prefixes(v, self.__trim_path_prefixes) for file_name in trimmed_files.values(): file_id = file_path_to_id.get(file_name, -1) if file_id == -1: missing_ids_for_files.append(file_name) continue file_ids[file_name] = file_id if missing_ids_for_files: LOG.warning("Failed to get file path id for '%s'!", ' '.join(missing_ids_for_files)) return False def set_review_status(report: ReportType): """ Set review status for the given report if there is any source code comment. """ checker_name = report.main['check_name'] last_report_event = report.bug_path[-1] # The original file path is needed here not the trimmed # because the source files are extracted as the original # file path. file_name = files[last_report_event['location']['file']] source_file_name = os.path.realpath( os.path.join(source_root, file_name.strip("/"))) # Check and store source code comments. if not os.path.isfile(source_file_name): return report_line = last_report_event['location']['line'] source_file = os.path.basename(file_name) src_comment_data = parse_codechecker_review_comment( source_file_name, report_line, checker_name) if len(src_comment_data) == 1: status = src_comment_data[0]['status'] rw_status = ttypes.ReviewStatus.FALSE_POSITIVE if status == 'confirmed': rw_status = ttypes.ReviewStatus.CONFIRMED elif status == 'intentional': rw_status = ttypes.ReviewStatus.INTENTIONAL self.__report_server._setReviewStatus( session, report.report_hash, rw_status, src_comment_data[0]['message'], run_history_time) elif len(src_comment_data) > 1: LOG.warning( "Multiple source code comment can be found " "for '%s' checker in '%s' at line %s. " "This bug will not be suppressed!", checker_name, source_file, report_line) self.__wrong_src_code_comments.append( f"{source_file}|{report_line}|{checker_name}") root_dir_path = os.path.dirname(report_file_path) mip = self.__mips[root_dir_path] analysis_info = self.__analysis_info.get(root_dir_path) for report in reports: self.__all_report_checkers.add(report.check_name) if skip_handler.should_skip(report.file_path): continue report.trim_path_prefixes(self.__trim_path_prefixes) report_path_hash = get_report_path_hash(report) if report_path_hash in self.__already_added_report_hashes: LOG.debug('Not storing report. Already added: %s', report) continue LOG.debug("Storing report to the database...") bug_id = report.report_hash detection_status = 'new' detected_at = run_history_time if bug_id in hash_map_reports: old_report = hash_map_reports[bug_id][0] old_status = old_report.detection_status detection_status = 'reopened' \ if old_status == 'resolved' else 'unresolved' detected_at = old_report.detected_at analyzer_name = get_analyzer_name(report.check_name, mip.checker_to_analyzer, report.metadata) path_events = collect_paths_events(report, file_ids, trimmed_files) report_id = self.__add_report(session, run_id, file_ids[report.file_path], report.main, path_events, detection_status, detected_at, analysis_info, analyzer_name) self.__new_report_hashes.add(bug_id) self.__already_added_report_hashes.add(report_path_hash) set_review_status(report) LOG.debug("Storing report done. ID=%d", report_id) return True