def skip_html_report_data_handler(report_hash, source_file, report_line,
checker_name, diag, files):
"""
Report handler which skips bugs which were suppressed by source code
comments. This function will return a tuple. The first element
will decide whether the report should be skipped or not and the second
element will be a list of source code comments related to the actual
report.
"""
files_dict = {k: v for k, v in enumerate(files)}
report = Report({'check_name': checker_name},
diag['path'],
files_dict,
metadata=None)
path_hash = get_report_path_hash(report)
if path_hash in processed_path_hashes:
LOG.debug("Skip report because it is a deduplication of an "
"already processed report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(diag)
return True, []
skip, source_code_comments = skip_report(report_hash, source_file,
report_line, checker_name,
suppr_handler,
src_comment_status_filter)
if skip_handler:
skip |= skip_handler.should_skip(source_file)
if not skip:
processed_path_hashes.add(path_hash)
return skip, source_code_comments
def test_report_path_hash_generation(self):
"""
Test report path hash generation.
"""
clang50_trunk_plist = os.path.join(
self.__plist_test_files, 'clang-5.0-trunk.plist')
files, reports = plist_parser.parse_plist_file(clang50_trunk_plist,
False)
self.assertEqual(len(reports), 3)
# Generate dummy file_ids which should come from the database.
file_ids = {}
for i, file_name in enumerate(files, 1):
file_ids[file_name] = i
msg = "This test is prepared to handle 3 reports."
self.assertEqual(len(reports), 3, msg)
report_hash_to_path_hash = {
'79e31a6ba028f0b7d9779faf4a6cb9cf':
'acb1d3dc1459f681bd3c743e6c015b37',
'8714f42d8328bc78d5d7bff6ced918cc':
'dcaaf2905d607a16e3fa330edb8e9f89',
'a6d3464f8aab9eb31a8ea7e167e84322':
'd089a50f34051c68c7bb4c5ac2c4c5d5'
}
for report in reports:
path_hash = get_report_path_hash(report)
bug_hash = report.main['issue_hash_content_of_line_in_context']
self.assertEqual(path_hash, report_hash_to_path_hash[bug_hash])
def test_report_path_hash_generation(self):
"""
Test report path hash generation.
"""
clang50_trunk_plist = os.path.join(self.__plist_test_files,
'clang-5.0-trunk.plist')
files, reports = plist_parser.parse_plist_file(clang50_trunk_plist,
None, False)
self.assertEqual(len(reports), 3)
# Generate dummy file_ids which should come from the database.
file_ids = {}
for i, file_name in enumerate(files, 1):
file_ids[file_name] = i
msg = "This test is prepared to handle 3 reports."
self.assertEqual(len(reports), 3, msg)
report_hash_to_path_hash = {
'79e31a6ba028f0b7d9779faf4a6cb9cf':
'c473c1a55df72ea4c6e055e18370ac65',
'8714f42d8328bc78d5d7bff6ced918cc':
'94f2a6eee8af6462a810218dff35056a',
'a6d3464f8aab9eb31a8ea7e167e84322':
'11f410136724cf43c63526841007897e'
}
for report in reports:
path_hash = get_report_path_hash(report.bug_path, files)
bug_hash = report.main['issue_hash_content_of_line_in_context']
self.assertEqual(path_hash, report_hash_to_path_hash[bug_hash])
def test_gen_report_path_hash(self):
""" Test path hash generation for multiple errors. """
test_plist = os.path.join(self.test_file_dir, 'cpp',
'multi_error.plist')
plist = plistlib.readPlist(test_plist)
expected_path_hash = {
'f48840093ef89e291fb68a95a5181612':
'a92b18ee6ee267cd82d0f056ef2bbe4b',
'e4907182b363faf2ec905fc32cc5a4ab':
'2b010e31810074749d485f1db8e419e9'
}
files = plist['files']
for diag in plist['diagnostics']:
file_path = files[diag['location']['file']]
path_hash = get_report_path_hash(diag['path'], file_path)
actual_report_hash = diag['issue_hash_content_of_line_in_context']
self.assertEqual(path_hash, expected_path_hash[actual_report_hash])
def test_gen_report_path_hash(self):
""" Test path hash generation for multiple errors. """
test_plist = os.path.join(self.test_file_dir, 'cpp',
'multi_error.plist')
plist = plistlib.readPlist(test_plist)
expected_path_hash = {
'f48840093ef89e291fb68a95a5181612':
'93cb93bdcee10434f9cf9f486947c88e',
'e4907182b363faf2ec905fc32cc5a4ab':
'71a4dc24bf88af2b13be83d8d15bd6f0'
}
for diag in plist['diagnostics']:
diag['bug_path'] = diag['path']
diag['files'] = plist['files']
path_hash = get_report_path_hash(
namedtuple('Report', diag.keys())(*diag.values()))
actual_report_hash = diag['issue_hash_content_of_line_in_context']
self.assertEqual(path_hash, expected_path_hash[actual_report_hash])
def skip_html_report_data_handler(report_hash, source_file, report_line,
checker_name, diag, files):
"""
Report handler which skips bugs which were suppressed by source code
comments.
"""
report = Report(None, diag['path'], files)
path_hash = get_report_path_hash(report.bug_path, files)
if path_hash in processed_path_hashes:
LOG.debug("Skip report because it is a deduplication of an "
"already processed report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(diag)
return True
skip = skip_report(report_hash, source_file, report_line, checker_name,
suppr_handler)
if skip_handler:
skip |= skip_handler.should_skip(source_file)
if not skip:
processed_path_hashes.add(path_hash)
return skip
def write(self,
file_report_map: Dict[str, List[Report]],
output=sys.stdout):
"""
Format an already parsed plist report file to a more
human readable format.
The formatted text is written to the output.
During writing the output statistics are collected.
Write out the bugs to the output and collect report statistics.
"""
severity_stats = defaultdict(int)
file_stats = defaultdict(int)
report_count = defaultdict(int)
for file_path in sorted(file_report_map,
key=lambda key: len(file_report_map[key])):
non_suppressed = 0
sorted_reports = sorted(file_report_map[file_path],
key=lambda r: r.main['location']['line'])
for report in sorted_reports:
path_hash = get_report_path_hash(report)
if path_hash in self._processed_path_hashes:
LOG.debug("Not showing report because it is a "
"deduplication of an already processed report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(report)
continue
self._processed_path_hashes.add(path_hash)
events = [
i for i in report.bug_path if i.get('kind') == 'event'
]
f_path = report.files[events[-1]['location']['file']]
if self.skiplist_handler and \
self.skiplist_handler.should_skip(f_path):
LOG.debug("Skipped report in '%s'", f_path)
LOG.debug(report)
continue
last_report_event = report.bug_path[-1]
source_file = \
report.files[last_report_event['location']['file']]
report_line = last_report_event['location']['line']
report_hash = \
report.main['issue_hash_content_of_line_in_context']
checker_name = report.main['check_name']
skip, source_code_comments = \
skip_report(report_hash,
source_file,
report_line,
checker_name,
self.src_comment_handler,
self.src_comment_status_filter)
if skip:
continue
if self._trim_path_prefixes:
report.trim_path_prefixes(self._trim_path_prefixes)
trimmed_source_file = \
report.files[last_report_event['location']['file']]
file_stats[f_path] += 1
severity = self.__severity_map.get(checker_name)
severity_stats[severity] += 1
report_count["report_count"] += 1
review_status = None
if len(source_code_comments) == 1:
review_status = source_code_comments[0]['status']
output.write(
self.__format_bug_event(checker_name, severity,
last_report_event,
trimmed_source_file,
review_status))
output.write('\n')
# Print source code comments.
for source_code_comment in source_code_comments:
output.write(source_code_comment['line'].rstrip())
output.write('\n')
output.write(
self.__format_location(last_report_event, source_file))
output.write('\n')
if self.print_steps:
output.write(' Report hash: ' + report_hash + '\n')
# Print out macros.
macros = report.macro_expansions
if macros:
output.write(' Macro expansions:\n')
index_format = ' %%%dd, ' % \
int(math.floor(
math.log10(len(macros))) + 1)
for index, macro in enumerate(macros):
output.write(index_format % (index + 1))
source = report.files[macro['location']['file']]
output.write(
self.__format_macro_expansion(macro, source))
output.write('\n')
# Print out notes.
notes = report.notes
if notes:
output.write(' Notes:\n')
index_format = ' %%%dd, ' % \
int(math.floor(
math.log10(len(notes))) + 1)
for index, note in enumerate(notes):
output.write(index_format % (index + 1))
source_file = report.files[note['location']
['file']]
output.write(
self.__format_bug_note(note, source_file))
output.write('\n')
output.write(' Steps:\n')
index_format = ' %%%dd, ' % \
int(math.floor(math.log10(len(events))) + 1)
for index, event in enumerate(events):
output.write(index_format % (index + 1))
source_file = report.files[event['location']['file']]
output.write(
self.__format_bug_event(None, None, event,
source_file))
output.write('\n')
output.write('\n')
non_suppressed += 1
base_file = os.path.basename(file_path)
if non_suppressed == 0:
output.write('Found no defects in %s\n' % base_file)
else:
output.write('Found %d defect(s) in %s\n\n' %
(non_suppressed, base_file))
return {
"severity": severity_stats,
"files": file_stats,
"reports": report_count
}
def __process_report_file(self, report_file_path: str, session: DBSession,
source_root: str, run_id: int,
file_path_to_id: Dict[str, int],
run_history_time: datetime,
skip_handler: skiplist_handler.SkipListHandler,
hash_map_reports: Dict[str, List[Any]]) -> bool:
"""
Process and save reports from the given report file to the database.
"""
try:
files, reports = plist_parser.parse_plist_file(report_file_path)
except Exception as ex:
LOG.warning('Parsing the plist failed: %s', str(ex))
return False
if not reports:
return True
trimmed_files = {}
file_ids = {}
missing_ids_for_files = []
for k, v in files.items():
trimmed_files[k] = \
util.trim_path_prefixes(v, self.__trim_path_prefixes)
for file_name in trimmed_files.values():
file_id = file_path_to_id.get(file_name, -1)
if file_id == -1:
missing_ids_for_files.append(file_name)
continue
file_ids[file_name] = file_id
if missing_ids_for_files:
LOG.warning("Failed to get file path id for '%s'!",
' '.join(missing_ids_for_files))
return False
def set_review_status(report: ReportType):
"""
Set review status for the given report if there is any source code
comment.
"""
checker_name = report.main['check_name']
last_report_event = report.bug_path[-1]
# The original file path is needed here not the trimmed
# because the source files are extracted as the original
# file path.
file_name = files[last_report_event['location']['file']]
source_file_name = os.path.realpath(
os.path.join(source_root, file_name.strip("/")))
# Check and store source code comments.
if not os.path.isfile(source_file_name):
return
report_line = last_report_event['location']['line']
source_file = os.path.basename(file_name)
src_comment_data = parse_codechecker_review_comment(
source_file_name, report_line, checker_name)
if len(src_comment_data) == 1:
status = src_comment_data[0]['status']
rw_status = ttypes.ReviewStatus.FALSE_POSITIVE
if status == 'confirmed':
rw_status = ttypes.ReviewStatus.CONFIRMED
elif status == 'intentional':
rw_status = ttypes.ReviewStatus.INTENTIONAL
self.__report_server._setReviewStatus(
session, report.report_hash, rw_status,
src_comment_data[0]['message'], run_history_time)
elif len(src_comment_data) > 1:
LOG.warning(
"Multiple source code comment can be found "
"for '%s' checker in '%s' at line %s. "
"This bug will not be suppressed!", checker_name,
source_file, report_line)
self.__wrong_src_code_comments.append(
f"{source_file}|{report_line}|{checker_name}")
root_dir_path = os.path.dirname(report_file_path)
mip = self.__mips[root_dir_path]
analysis_info = self.__analysis_info.get(root_dir_path)
for report in reports:
self.__all_report_checkers.add(report.check_name)
if skip_handler.should_skip(report.file_path):
continue
report.trim_path_prefixes(self.__trim_path_prefixes)
report_path_hash = get_report_path_hash(report)
if report_path_hash in self.__already_added_report_hashes:
LOG.debug('Not storing report. Already added: %s', report)
continue
LOG.debug("Storing report to the database...")
bug_id = report.report_hash
detection_status = 'new'
detected_at = run_history_time
if bug_id in hash_map_reports:
old_report = hash_map_reports[bug_id][0]
old_status = old_report.detection_status
detection_status = 'reopened' \
if old_status == 'resolved' else 'unresolved'
detected_at = old_report.detected_at
analyzer_name = get_analyzer_name(report.check_name,
mip.checker_to_analyzer,
report.metadata)
path_events = collect_paths_events(report, file_ids, trimmed_files)
report_id = self.__add_report(session, run_id,
file_ids[report.file_path],
report.main, path_events,
detection_status, detected_at,
analysis_info, analyzer_name)
self.__new_report_hashes.add(bug_id)
self.__already_added_report_hashes.add(report_path_hash)
set_review_status(report)
LOG.debug("Storing report done. ID=%d", report_id)
return True
