def test_skip_no_pre_from_parse(self):
"""Keep everything pre analysis needs it in ctu or statistics mode."""
cmp_cmd_json = [{
"directory": "/tmp/lib1",
"command": "g++ /tmp/lib1/a.cpp",
"file": "a.cpp"
}, {
"directory": "/tmp/lib1",
"command": "g++ /tmp/lib1/b.cpp",
"file": "b.cpp"
}, {
"directory": "/tmp/lib2",
"command": "g++ /tmp/lib2/a.cpp",
"file": "a.cpp"
}]
skip_list = """
-*/lib1/*
"""
analysis_skip = SkipListHandlers([SkipListHandler(skip_list)])
pre_analysis_skip = SkipListHandlers([SkipListHandler("")])
build_actions, _ = log_parser.\
parse_unique_log(cmp_cmd_json, self.__this_dir,
analysis_skip_handlers=analysis_skip,
ctu_or_stats_enabled=True,
pre_analysis_skip_handlers=pre_analysis_skip)
self.assertEqual(len(build_actions), 3)
def test_no_skip_from_parse(self):
"""Keep everything for analysis, no skipping there."""
cmp_cmd_json = [{
"directory": "/tmp/lib1",
"command": "g++ /tmp/lib1/a.cpp",
"file": "a.cpp"
}, {
"directory": "/tmp/lib1",
"command": "g++ /tmp/lib1/b.cpp",
"file": "b.cpp"
}, {
"directory": "/tmp/lib2",
"command": "g++ /tmp/lib2/a.cpp",
"file": "a.cpp"
}]
skip_list = """
-*/lib1/*
"""
analysis_skip = SkipListHandlers([SkipListHandler("")])
pre_analysis_skip = SkipListHandlers([SkipListHandler(skip_list)])
build_actions, _ = log_parser.\
parse_unique_log(cmp_cmd_json, self.__this_dir,
analysis_skip_handlers=analysis_skip,
pre_analysis_skip_handlers=pre_analysis_skip)
self.assertEqual(len(build_actions), 3)
def test_skip_everything_from_parse(self):
"""Same skip file for pre analysis and analysis. Skip everything."""
cmp_cmd_json = [{
"directory": "/tmp/lib1",
"command": "g++ /tmp/lib1/a.cpp",
"file": "a.cpp"
}, {
"directory": "/tmp/lib1",
"command": "g++ /tmp/lib1/b.cpp",
"file": "b.cpp"
}, {
"directory": "/tmp/lib2",
"command": "g++ /tmp/lib2/a.cpp",
"file": "a.cpp"
}]
skip_list = """
-*/lib1/*
-*/lib2/*
"""
analysis_skip = SkipListHandlers([SkipListHandler(skip_list)])
pre_analysis_skip = SkipListHandlers([SkipListHandler(skip_list)])
build_actions, _ = log_parser.\
parse_unique_log(cmp_cmd_json, self.__this_dir,
analysis_skip_handlers=analysis_skip,
pre_analysis_skip_handlers=pre_analysis_skip)
self.assertEqual(len(build_actions), 0)
def test_skip_x_header(self):
""" Test skipping a header file. """
with open('skip_x_header.txt', encoding="utf-8", errors="ignore") as f:
skip_handlers = SkipListHandlers([SkipListHandler(f.read())])
self.__test_skip_reports('x.plist', 'skip_x_header.expected.plist',
skip_handlers)
def test_keep_only_empty(self):
""" Test skipping all files except empty. """
with open('keep_only_empty.txt',
encoding="utf-8", errors="ignore") as skip_file:
skip_handler = SkipListHandler(skip_file.read())
self.__test_skip_reports(
'x.plist', 'keep_only_empty.expected.plist', skip_handler)
def test_skip_all_header(self):
""" Test skipping all header files. """
with open('skip_all_header.txt',
encoding="utf-8", errors="ignore") as skip_file:
skip_handler = SkipListHandler(skip_file.read())
self.__test_skip_reports(
'x.plist', 'skip_all_header.expected.plist', skip_handler)
def test_skip_everything_from_parse_relative_path(self):
"""
Same skip file for pre analysis and analysis. Skip everything.
Source file contains relative path.
"""
cmp_cmd_json = [{
"directory": "/tmp/lib1/Debug",
"command": "g++ ../a.cpp",
"file": "../a.cpp"
}, {
"directory": "/tmp/lib1/Debug/rel",
"command": "g++ ../../b.cpp",
"file": "../../b.cpp"
}, {
"directory": "/tmp/lib1/Debug",
"command": "g++ ../d.cpp",
"file": "../d.cpp"
}, {
"directory": "/tmp/lib2/Debug",
"command": "g++ ../a.cpp",
"file": "../a.cpp"
}]
skip_list = """
+/tmp/lib1/d.cpp
-*/lib1/Debug/rel/../../*
-*/lib1/a.cpp
-/tmp/lib2/a.cpp
"""
analysis_skip = SkipListHandlers([SkipListHandler(skip_list)])
pre_analysis_skip = SkipListHandlers([SkipListHandler(skip_list)])
build_actions, _ = log_parser.\
parse_unique_log(cmp_cmd_json, self.__this_dir,
analysis_skip_handlers=analysis_skip,
pre_analysis_skip_handlers=pre_analysis_skip)
self.assertEqual(len(build_actions), 1)
self.assertEqual(build_actions[0].source, '/tmp/lib1/d.cpp')
def test_skip_all_in_pre_from_parse(self):
"""Pre analysis skips everything but keep build action for analysis."""
cmp_cmd_json = [{
"directory": "/tmp/lib1",
"command": "g++ /tmp/lib1/a.cpp",
"file": "a.cpp"
}, {
"directory": "/tmp/lib1",
"command": "g++ /tmp/lib1/b.cpp",
"file": "b.cpp"
}, {
"directory": "/tmp/lib2",
"command": "g++ /tmp/lib2/a.cpp",
"file": "a.cpp"
}]
keep = cmp_cmd_json[2]
skip_list = """
-*/lib1/*
"""
pre_skip_list = """
-*
"""
analysis_skip = SkipListHandlers([SkipListHandler(skip_list)])
pre_analysis_skip = SkipListHandlers([SkipListHandler(pre_skip_list)])
build_actions, _ = log_parser.\
parse_unique_log(cmp_cmd_json, self.__this_dir,
analysis_skip_handlers=analysis_skip,
pre_analysis_skip_handlers=pre_analysis_skip)
self.assertEqual(len(build_actions), 1)
source_file = os.path.join(keep['directory'], keep['file'])
self.assertEqual(build_actions[0].source, source_file)
self.assertEqual(build_actions[0].original_command, keep['command'])
def main(args):
"""
Entry point for parsing some analysis results and printing them to the
stdout in a human-readable format.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
try:
cmd_config.check_config_file(args)
except FileNotFoundError as fnerr:
LOG.error(fnerr)
sys.exit(1)
export = args.export if 'export' in args else None
if export == 'html' and 'output_path' not in args:
LOG.error("Argument --export not allowed without argument --output "
"when exporting to HTML.")
sys.exit(1)
if export == 'gerrit' and not gerrit.mandatory_env_var_is_set():
sys.exit(1)
context = analyzer_context.get_context()
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
original_cwd = os.getcwd()
src_comment_status_filter = args.review_status
suppr_handler = None
if 'suppress' in args:
__make_handler = False
if not os.path.isfile(args.suppress):
if 'create_suppress' in args:
with open(args.suppress,
'w',
encoding='utf-8',
errors='ignore') as _:
# Just create the file.
__make_handler = True
LOG.info(
"Will write source-code suppressions to "
"suppress file: %s", args.suppress)
else:
LOG.warning(
"Suppress file '%s' given, but it does not exist"
" -- will not suppress anything.", args.suppress)
else:
__make_handler = True
if __make_handler:
suppr_handler = suppress_handler.\
GenericSuppressHandler(args.suppress,
'create_suppress' in args,
src_comment_status_filter)
elif 'create_suppress' in args:
LOG.error("Can't use '--export-source-suppress' unless '--suppress "
"SUPPRESS_FILE' is also given.")
sys.exit(2)
processed_path_hashes = set()
skip_handler = None
if 'skipfile' in args:
with open(args.skipfile, 'r', encoding='utf-8',
errors='ignore') as skip_file:
skip_handler = SkipListHandler(skip_file.read())
trim_path_prefixes = args.trim_path_prefix if \
'trim_path_prefix' in args else None
if export:
if export not in EXPORT_TYPES:
LOG.error(f"Unknown export format: {export}")
return
# The HTML part will be handled separately below.
if export != 'html':
try:
res = parse_convert_reports(args.input, export,
context.severity_map,
trim_path_prefixes)
if 'output_path' in args:
output_path = os.path.abspath(args.output_path)
if not os.path.exists(output_path):
os.mkdir(output_path)
reports_json = os.path.join(output_path, 'reports.json')
with open(reports_json,
mode='w',
encoding='utf-8',
errors="ignore") as output_f:
output_f.write(json.dumps(res))
return print(json.dumps(res))
except Exception as ex:
LOG.error(ex)
sys.exit(1)
def trim_path_prefixes_handler(source_file):
"""
Callback to util.trim_path_prefixes to prevent module dependency
of plist_to_html
"""
return util.trim_path_prefixes(source_file, trim_path_prefixes)
html_builder = None
def skip_html_report_data_handler(report_hash, source_file, report_line,
checker_name, diag, files):
"""
Report handler which skips bugs which were suppressed by source code
comments. This function will return a tuple. The first element
will decide whether the report should be skipped or not and the second
element will be a list of source code comments related to the actual
report.
"""
files_dict = {k: v for k, v in enumerate(files)}
report = Report({'check_name': checker_name},
diag['path'],
files_dict,
metadata=None)
path_hash = get_report_path_hash(report)
if path_hash in processed_path_hashes:
LOG.debug("Skip report because it is a deduplication of an "
"already processed report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(diag)
return True, []
skip, source_code_comments = skip_report(report_hash, source_file,
report_line, checker_name,
suppr_handler,
src_comment_status_filter)
if skip_handler:
skip |= skip_handler.should_skip(source_file)
if not skip:
processed_path_hashes.add(path_hash)
return skip, source_code_comments
file_change = set()
severity_stats = defaultdict(int)
file_stats = defaultdict(int)
report_count = 0
for input_path in args.input:
input_path = os.path.abspath(input_path)
os.chdir(original_cwd)
LOG.debug("Parsing input argument: '%s'", input_path)
if export == 'html':
output_path = os.path.abspath(args.output_path)
if not html_builder:
html_builder = \
PlistToHtml.HtmlBuilder(context.path_plist_to_html_dist,
context.severity_map)
LOG.info("Generating html output files:")
PlistToHtml.parse(input_path, output_path,
context.path_plist_to_html_dist,
skip_html_report_data_handler, html_builder,
trim_path_prefixes_handler)
continue
files = []
metadata_dict = {}
if os.path.isfile(input_path):
files.append(input_path)
elif os.path.isdir(input_path):
metadata_file = os.path.join(input_path, "metadata.json")
if os.path.exists(metadata_file):
metadata_dict = util.load_json_or_empty(metadata_file)
LOG.debug(metadata_dict)
if 'working_directory' in metadata_dict:
working_dir = metadata_dict['working_directory']
try:
os.chdir(working_dir)
except OSError as oerr:
LOG.debug(oerr)
LOG.error(
"Working directory %s is missing.\n"
"Can not parse reports safely.", working_dir)
sys.exit(1)
_, _, file_names = next(os.walk(input_path), ([], [], []))
files = [
os.path.join(input_path, file_name) for file_name in file_names
]
file_report_map = defaultdict(list)
plist_pltf = PlistToPlaintextFormatter(suppr_handler, skip_handler,
context.severity_map,
processed_path_hashes,
trim_path_prefixes,
src_comment_status_filter)
plist_pltf.print_steps = 'print_steps' in args
for file_path in files:
f_change = parse_with_plt_formatter(file_path, metadata_dict,
plist_pltf, file_report_map)
file_change = file_change.union(f_change)
report_stats = plist_pltf.write(file_report_map)
sev_stats = report_stats.get('severity')
for severity in sev_stats:
severity_stats[severity] += sev_stats[severity]
f_stats = report_stats.get('files')
for file_path in f_stats:
file_stats[file_path] += f_stats[file_path]
rep_stats = report_stats.get('reports')
report_count += rep_stats.get("report_count", 0)
# Create index.html and statistics.html for the generated html files.
if html_builder:
html_builder.create_index_html(args.output_path)
html_builder.create_statistics_html(args.output_path)
print('\nTo view statistics in a browser run:\n> firefox {0}'.format(
os.path.join(args.output_path, 'statistics.html')))
print('\nTo view the results in a browser run:\n> firefox {0}'.format(
os.path.join(args.output_path, 'index.html')))
else:
print("\n----==== Summary ====----")
if file_stats:
vals = [[os.path.basename(k), v]
for k, v in dict(file_stats).items()]
vals.sort(key=itemgetter(0))
keys = ['Filename', 'Report count']
table = twodim.to_str('table', keys, vals, 1, True)
print(table)
if severity_stats:
vals = [[k, v] for k, v in dict(severity_stats).items()]
vals.sort(key=itemgetter(0))
keys = ['Severity', 'Report count']
table = twodim.to_str('table', keys, vals, 1, True)
print(table)
print("----=================----")
print("Total number of reports: {}".format(report_count))
print("----=================----")
if file_change:
changed_files = '\n'.join([' - ' + f for f in file_change])
LOG.warning(
"The following source file contents changed since the "
"latest analysis:\n%s\nMultiple reports were not "
"shown and skipped from the statistics. Please "
"analyze your project again to update the "
"reports!", changed_files)
os.chdir(original_cwd)
if report_count != 0:
sys.exit(2)
def main(args):
"""
Entry point for parsing some analysis results and printing them to the
stdout in a human-readable format.
"""
logger.setup_logger(args.verbose if 'verbose' in args else None)
context = analyzer_context.get_context()
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
original_cwd = os.getcwd()
suppr_handler = None
if 'suppress' in args:
__make_handler = False
if not os.path.isfile(args.suppress):
if 'create_suppress' in args:
with open(args.suppress, 'w') as _:
# Just create the file.
__make_handler = True
LOG.info("Will write source-code suppressions to "
"suppress file: %s", args.suppress)
else:
LOG.warning("Suppress file '%s' given, but it does not exist"
" -- will not suppress anything.", args.suppress)
else:
__make_handler = True
if __make_handler:
suppr_handler = suppress_handler.\
GenericSuppressHandler(args.suppress,
'create_suppress' in args)
elif 'create_suppress' in args:
LOG.error("Can't use '--export-source-suppress' unless '--suppress "
"SUPPRESS_FILE' is also given.")
sys.exit(2)
processed_path_hashes = set()
skip_handler = None
if 'skipfile' in args:
with open(args.skipfile, 'r') as skip_file:
skip_handler = SkipListHandler(skip_file.read())
trim_path_prefixes = args.trim_path_prefix if \
'trim_path_prefix' in args else None
def trim_path_prefixes_handler(source_file):
"""
Callback to util.trim_path_prefixes to prevent module dependency
of plist_to_html
"""
return util.trim_path_prefixes(source_file, trim_path_prefixes)
html_builder = None
def skip_html_report_data_handler(report_hash, source_file, report_line,
checker_name, diag, files):
"""
Report handler which skips bugs which were suppressed by source code
comments.
"""
report = Report(None, diag['path'], files)
path_hash = get_report_path_hash(report, files)
if path_hash in processed_path_hashes:
LOG.debug("Skip report because it is a deduplication of an "
"already processed report!")
LOG.debug("Path hash: %s", path_hash)
LOG.debug(diag)
return True
skip = skip_report(report_hash,
source_file,
report_line,
checker_name,
suppr_handler)
if skip_handler:
skip |= skip_handler.should_skip(source_file)
if not skip:
processed_path_hashes.add(path_hash)
return skip
file_change = set()
severity_stats = defaultdict(int)
file_stats = defaultdict(int)
report_count = 0
for input_path in args.input:
input_path = os.path.abspath(input_path)
os.chdir(original_cwd)
LOG.debug("Parsing input argument: '%s'", input_path)
export = args.export if 'export' in args else None
if export is not None and export == 'html':
output_path = os.path.abspath(args.output_path)
if not html_builder:
html_builder = \
PlistToHtml.HtmlBuilder(context.path_plist_to_html_dist,
context.severity_map)
LOG.info("Generating html output files:")
PlistToHtml.parse(input_path,
output_path,
context.path_plist_to_html_dist,
skip_html_report_data_handler,
html_builder,
trim_path_prefixes_handler)
continue
files = []
metadata_dict = {}
if os.path.isfile(input_path):
files.append(input_path)
elif os.path.isdir(input_path):
metadata_file = os.path.join(input_path, "metadata.json")
if os.path.exists(metadata_file):
metadata_dict = util.load_json_or_empty(metadata_file)
LOG.debug(metadata_dict)
if 'working_directory' in metadata_dict:
working_dir = metadata_dict['working_directory']
try:
os.chdir(working_dir)
except OSError as oerr:
LOG.debug(oerr)
LOG.error("Working directory %s is missing.\n"
"Can not parse reports safely.", working_dir)
sys.exit(1)
_, _, file_names = next(os.walk(input_path), ([], [], []))
files = [os.path.join(input_path, file_name) for file_name
in file_names]
file_report_map = defaultdict(list)
rh = PlistToPlaintextFormatter(suppr_handler,
skip_handler,
context.severity_map,
processed_path_hashes,
trim_path_prefixes)
rh.print_steps = 'print_steps' in args
for file_path in files:
f_change = parse(file_path, metadata_dict, rh, file_report_map)
file_change = file_change.union(f_change)
report_stats = rh.write(file_report_map)
sev_stats = report_stats.get('severity')
for severity in sev_stats:
severity_stats[severity] += sev_stats[severity]
f_stats = report_stats.get('files')
for file_path in f_stats:
file_stats[file_path] += f_stats[file_path]
rep_stats = report_stats.get('reports')
report_count += rep_stats.get("report_count", 0)
print("\n----==== Summary ====----")
if file_stats:
vals = [[os.path.basename(k), v] for k, v in
dict(file_stats).items()]
keys = ['Filename', 'Report count']
table = twodim_to_str('table', keys, vals, 1, True)
print(table)
if severity_stats:
vals = [[k, v] for k, v in dict(severity_stats).items()]
keys = ['Severity', 'Report count']
table = twodim_to_str('table', keys, vals, 1, True)
print(table)
print("----=================----")
print("Total number of reports: {}".format(report_count))
print("----=================----")
if file_change:
changed_files = '\n'.join([' - ' + f for f in file_change])
LOG.warning("The following source file contents changed since the "
"latest analysis:\n%s\nMultiple reports were not "
"shown and skipped from the statistics. Please "
"analyze your project again to update the "
"reports!", changed_files)
os.chdir(original_cwd)
# Create index.html and statistics.html for the generated html files.
if html_builder:
html_builder.create_index_html(args.output_path)
html_builder.create_statistics_html(args.output_path)
print('\nTo view statistics in a browser run:\n> firefox {0}'.format(
os.path.join(args.output_path, 'statistics.html')))
print('\nTo view the results in a browser run:\n> firefox {0}'.format(
os.path.join(args.output_path, 'index.html')))
def __process_report_file(self, report_file_path: str, session: DBSession,
source_root: str, run_id: int,
file_path_to_id: Dict[str, int],
run_history_time: datetime,
skip_handler: skiplist_handler.SkipListHandler,
hash_map_reports: Dict[str, List[Any]]) -> bool:
"""
Process and save reports from the given report file to the database.
"""
try:
files, reports = plist_parser.parse_plist_file(report_file_path)
except Exception as ex:
LOG.warning('Parsing the plist failed: %s', str(ex))
return False
if not reports:
return True
trimmed_files = {}
file_ids = {}
missing_ids_for_files = []
for k, v in files.items():
trimmed_files[k] = \
util.trim_path_prefixes(v, self.__trim_path_prefixes)
for file_name in trimmed_files.values():
file_id = file_path_to_id.get(file_name, -1)
if file_id == -1:
missing_ids_for_files.append(file_name)
continue
file_ids[file_name] = file_id
if missing_ids_for_files:
LOG.warning("Failed to get file path id for '%s'!",
' '.join(missing_ids_for_files))
return False
def set_review_status(report: ReportType):
"""
Set review status for the given report if there is any source code
comment.
"""
checker_name = report.main['check_name']
last_report_event = report.bug_path[-1]
# The original file path is needed here not the trimmed
# because the source files are extracted as the original
# file path.
file_name = files[last_report_event['location']['file']]
source_file_name = os.path.realpath(
os.path.join(source_root, file_name.strip("/")))
# Check and store source code comments.
if not os.path.isfile(source_file_name):
return
report_line = last_report_event['location']['line']
source_file = os.path.basename(file_name)
src_comment_data = parse_codechecker_review_comment(
source_file_name, report_line, checker_name)
if len(src_comment_data) == 1:
status = src_comment_data[0]['status']
rw_status = ttypes.ReviewStatus.FALSE_POSITIVE
if status == 'confirmed':
rw_status = ttypes.ReviewStatus.CONFIRMED
elif status == 'intentional':
rw_status = ttypes.ReviewStatus.INTENTIONAL
self.__report_server._setReviewStatus(
session, report.report_hash, rw_status,
src_comment_data[0]['message'], run_history_time)
elif len(src_comment_data) > 1:
LOG.warning(
"Multiple source code comment can be found "
"for '%s' checker in '%s' at line %s. "
"This bug will not be suppressed!", checker_name,
source_file, report_line)
self.__wrong_src_code_comments.append(
f"{source_file}|{report_line}|{checker_name}")
root_dir_path = os.path.dirname(report_file_path)
mip = self.__mips[root_dir_path]
analysis_info = self.__analysis_info.get(root_dir_path)
for report in reports:
self.__all_report_checkers.add(report.check_name)
if skip_handler.should_skip(report.file_path):
continue
report.trim_path_prefixes(self.__trim_path_prefixes)
report_path_hash = get_report_path_hash(report)
if report_path_hash in self.__already_added_report_hashes:
LOG.debug('Not storing report. Already added: %s', report)
continue
LOG.debug("Storing report to the database...")
bug_id = report.report_hash
detection_status = 'new'
detected_at = run_history_time
if bug_id in hash_map_reports:
old_report = hash_map_reports[bug_id][0]
old_status = old_report.detection_status
detection_status = 'reopened' \
if old_status == 'resolved' else 'unresolved'
detected_at = old_report.detected_at
analyzer_name = get_analyzer_name(report.check_name,
mip.checker_to_analyzer,
report.metadata)
path_events = collect_paths_events(report, file_ids, trimmed_files)
report_id = self.__add_report(session, run_id,
file_ids[report.file_path],
report.main, path_events,
detection_status, detected_at,
analysis_info, analyzer_name)
self.__new_report_hashes.add(bug_id)
self.__already_added_report_hashes.add(report_path_hash)
set_review_status(report)
LOG.debug("Storing report done. ID=%d", report_id)
return True
def __process_report_file(
self,
report_file_path: str,
session: DBSession,
source_root: str,
run_id: int,
file_path_to_id: Dict[str, int],
run_history_time: datetime,
skip_handler: skiplist_handler.SkipListHandler,
hash_map_reports: Dict[str, List[Any]]
) -> bool:
"""
Process and save reports from the given report file to the database.
"""
reports = report_file.get_reports(report_file_path)
if not reports:
return True
def set_review_status(report: Report):
"""
Set review status for the given report if there is any source code
comment.
"""
checker_name = report.checker_name
last_report_event = report.bug_path_events[-1]
# The original file path is needed here, not the trimmed, because
# the source files are extracted as the original file path.
file_name = report.file.original_path
source_file_name = os.path.realpath(
os.path.join(source_root, file_name.strip("/")))
# Check and store source code comments.
if not os.path.isfile(source_file_name):
return
report_line = last_report_event.range.end_line
source_file = os.path.basename(file_name)
src_comment_data = parse_codechecker_review_comment(
source_file_name, report_line, checker_name)
if len(src_comment_data) == 1:
status = src_comment_data[0].status
rw_status = ttypes.ReviewStatus.FALSE_POSITIVE
if status == 'confirmed':
rw_status = ttypes.ReviewStatus.CONFIRMED
elif status == 'intentional':
rw_status = ttypes.ReviewStatus.INTENTIONAL
self.__report_server._setReviewStatus(
session, report.report_hash, rw_status,
src_comment_data[0].message, run_history_time)
elif len(src_comment_data) > 1:
LOG.warning(
"Multiple source code comment can be found "
"for '%s' checker in '%s' at line %s. "
"This bug will not be suppressed!",
checker_name, source_file, report_line)
self.__wrong_src_code_comments.append(
f"{source_file}|{report_line}|{checker_name}")
def get_missing_file_ids(report: Report) -> List[str]:
""" Returns file paths which database file id is missing. """
missing_ids_for_files = []
for file_path in report.trimmed_files:
file_id = file_path_to_id.get(file_path, -1)
if file_id == -1:
missing_ids_for_files.append(file_path)
return missing_ids_for_files
root_dir_path = os.path.dirname(report_file_path)
mip = self.__mips[root_dir_path]
analysis_info = self.__analysis_info.get(root_dir_path)
for report in reports:
report.trim_path_prefixes(self.__trim_path_prefixes)
missing_ids_for_files = get_missing_file_ids(report)
if missing_ids_for_files:
LOG.warning("Failed to get database id for file path '%s'! "
"Skip adding report: %s:%d:%d [%s]",
' '.join(missing_ids_for_files), report.file.path,
report.line, report.column, report.checker_name)
continue
self.__all_report_checkers.add(report.checker_name)
if skip_handler.should_skip(report.file.original_path):
continue
report_path_hash = get_report_path_hash(report)
if report_path_hash in self.__already_added_report_hashes:
LOG.debug('Not storing report. Already added: %s', report)
continue
LOG.debug("Storing report to the database...")
detection_status = 'new'
detected_at = run_history_time
if report.report_hash in hash_map_reports:
old_report = hash_map_reports[report.report_hash][0]
old_status = old_report.detection_status
detection_status = 'reopened' \
if old_status == 'resolved' else 'unresolved'
detected_at = old_report.detected_at
analyzer_name = mip.checker_to_analyzer.get(
report.checker_name, report.analyzer_name)
report_id = self.__add_report(
session, run_id, report, file_path_to_id,
detection_status, detected_at, analysis_info, analyzer_name)
self.__new_report_hashes.add(report.report_hash)
self.__already_added_report_hashes.add(report_path_hash)
set_review_status(report)
LOG.debug("Storing report done. ID=%d", report_id)
return True
def main(args):
"""
Entry point for parsing some analysis results and printing them to the
stdout in a human-readable format.
"""
# If the given output format is not 'table', redirect logger's output to
# the stderr.
stream = None
if 'export' in args and args.export not in [None, 'table', 'html']:
stream = 'stderr'
init_logger(args.verbose if 'verbose' in args else None, stream)
try:
cmd_config.check_config_file(args)
except FileNotFoundError as fnerr:
LOG.error(fnerr)
sys.exit(1)
export = args.export if 'export' in args else None
if export == 'html' and 'output_path' not in args:
LOG.error("Argument --export not allowed without argument --output "
"when exporting to HTML.")
sys.exit(1)
if export == 'gerrit' and not gerrit.mandatory_env_var_is_set():
sys.exit(1)
if export and export not in EXPORT_TYPES:
LOG.error("Unknown export format: %s", export)
sys.exit(1)
context = analyzer_context.get_context()
# To ensure the help message prints the default folder properly,
# the 'default' for 'args.input' is a string, not a list.
# But we need lists for the foreach here to work.
if isinstance(args.input, str):
args.input = [args.input]
src_comment_status_filter = args.review_status
suppr_handler = None
if 'suppress' in args:
__make_handler = False
if not os.path.isfile(args.suppress):
if 'create_suppress' in args:
with open(args.suppress,
'w',
encoding='utf-8',
errors='ignore') as _:
# Just create the file.
__make_handler = True
LOG.info(
"Will write source-code suppressions to "
"suppress file: %s", args.suppress)
else:
LOG.warning(
"Suppress file '%s' given, but it does not exist"
" -- will not suppress anything.", args.suppress)
else:
__make_handler = True
if __make_handler:
suppr_handler = suppress_handler.\
GenericSuppressHandler(args.suppress,
'create_suppress' in args,
src_comment_status_filter)
elif 'create_suppress' in args:
LOG.error("Can't use '--export-source-suppress' unless '--suppress "
"SUPPRESS_FILE' is also given.")
sys.exit(1)
output_dir_path = None
output_file_path = None
if 'output_path' in args:
output_path = os.path.abspath(args.output_path)
if export == 'html':
output_dir_path = output_path
else:
if os.path.exists(output_path) and os.path.isdir(output_path):
# For backward compatibility reason we handle the use case
# when directory is provided to this command.
LOG.error(
"Please provide a file path instead of a directory "
"for '%s' export type!", export)
sys.exit(1)
if export == 'baseline' and not baseline.check(output_path):
LOG.error("Baseline files must have '.baseline' extensions.")
sys.exit(1)
output_file_path = output_path
output_dir_path = os.path.dirname(output_file_path)
if not os.path.exists(output_dir_path):
os.makedirs(output_dir_path)
def get_output_file_path(default_file_name: str) -> Optional[str]:
""" Return an output file path. """
if output_file_path:
return output_file_path
if output_dir_path:
return os.path.join(output_dir_path, default_file_name)
skip_handlers = SkipListHandlers()
if 'files' in args:
items = [f"+{file_path}" for file_path in args.files]
items.append("-*")
skip_handlers.append(SkipListHandler("\n".join(items)))
if 'skipfile' in args:
with open(args.skipfile, 'r', encoding='utf-8', errors='ignore') as f:
skip_handlers.append(SkipListHandler(f.read()))
trim_path_prefixes = args.trim_path_prefix if \
'trim_path_prefix' in args else None
all_reports = []
statistics = Statistics()
file_cache = {} # For memory effiency.
changed_files: Set[str] = set()
processed_path_hashes = set()
processed_file_paths = set()
print_steps = 'print_steps' in args
html_builder: Optional[report_to_html.HtmlBuilder] = None
if export == 'html':
html_builder = report_to_html.HtmlBuilder(
context.path_plist_to_html_dist, context.checker_labels)
for dir_path, file_paths in report_file.analyzer_result_files(args.input):
metadata = get_metadata(dir_path)
for file_path in file_paths:
reports = report_file.get_reports(file_path,
context.checker_labels,
file_cache)
reports = reports_helper.skip(reports, processed_path_hashes,
skip_handlers, suppr_handler,
src_comment_status_filter)
statistics.num_of_analyzer_result_files += 1
for report in reports:
if report.changed_files:
changed_files.update(report.changed_files)
statistics.add_report(report)
if trim_path_prefixes:
report.trim_path_prefixes(trim_path_prefixes)
all_reports.extend(reports)
# Print reports continously.
if not export:
file_report_map = plaintext.get_file_report_map(
reports, file_path, metadata)
plaintext.convert(file_report_map, processed_file_paths,
print_steps)
elif export == 'html':
print(f"Parsing input file '{file_path}'.")
report_to_html.convert(file_path, reports, output_dir_path,
html_builder)
if export is None: # Plain text output
statistics.write()
elif export == 'html':
html_builder.finish(output_dir_path, statistics)
elif export == 'json':
data = report_to_json.convert(all_reports)
dump_json_output(data, get_output_file_path("reports.json"))
elif export == 'codeclimate':
data = codeclimate.convert(all_reports)
dump_json_output(data, get_output_file_path("reports.json"))
elif export == 'gerrit':
data = gerrit.convert(all_reports)
dump_json_output(data, get_output_file_path("reports.json"))
elif export == 'baseline':
data = baseline.convert(all_reports)
output_path = get_output_file_path("reports.baseline")
if output_path:
baseline.write(output_path, data)
reports_helper.dump_changed_files(changed_files)
if statistics.num_of_reports:
sys.exit(2)