def compare(parser, args):
dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
CmpRuns.dump_scan_build_results_diff(dir_old,
dir_new,
show_stats=args.show_stats,
stats_only=args.stats_only,
histogram=args.histogram,
verbose_log=args.verbose_log)
def compare(parser, args):
import CmpRuns
choices = [
CmpRuns.HistogramType.RELATIVE.value,
CmpRuns.HistogramType.LOG_RELATIVE.value,
CmpRuns.HistogramType.ABSOLUTE.value
]
if args.histogram is not None and args.histogram not in choices:
parser.error(
"Incorrect histogram type, available choices are {}".format(
choices))
dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
CmpRuns.dump_scan_build_results_diff(dir_old,
dir_new,
show_stats=args.show_stats,
stats_only=args.stats_only,
histogram=args.histogram,
verbose_log=args.verbose_log)
def run_cmp_results(directory: str, strictness: int = 0) -> bool:
"""
Compare the warnings produced by scan-build.
strictness defines the success criteria for the test:
0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the strictness
criteria.
"""
tests_passed = True
start_time = time.time()
ref_dir = os.path.join(directory, REF_PREFIX + OUTPUT_DIR_NAME)
new_dir = os.path.join(directory, OUTPUT_DIR_NAME)
# We have to go one level down the directory tree.
ref_list = glob.glob(ref_dir + "/*")
new_list = glob.glob(new_dir + "/*")
# Log folders are also located in the results dir, so ignore them.
ref_log_dir = os.path.join(ref_dir, LOG_DIR_NAME)
if ref_log_dir in ref_list:
ref_list.remove(ref_log_dir)
new_list.remove(os.path.join(new_dir, LOG_DIR_NAME))
if len(ref_list) != len(new_list):
stderr(f"Mismatch in number of results folders: "
f"{ref_list} vs {new_list}")
sys.exit(1)
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if len(ref_list) > 1:
# Assume that the corresponding folders have the same names.
ref_list.sort()
new_list.sort()
# Iterate and find the differences.
num_diffs = 0
for ref_dir, new_dir in zip(ref_list, new_list):
assert(ref_dir != new_dir)
if VERBOSE >= 1:
stdout(f" Comparing Results: {ref_dir} {new_dir}\n")
patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
ref_results = CmpRuns.ResultsDirectory(ref_dir)
new_results = CmpRuns.ResultsDirectory(new_dir, patched_source)
# Scan the results, delete empty plist files.
num_diffs, reports_in_ref, reports_in_new = \
CmpRuns.dump_scan_build_results_diff(ref_results, new_results,
delete_empty=False,
out=LOCAL.stdout)
if num_diffs > 0:
stdout(f"Warning: {num_diffs} differences in diagnostics.\n")
if strictness >= 2 and num_diffs > 0:
stdout("Error: Diffs found in strict mode (2).\n")
tests_passed = False
elif strictness >= 1 and reports_in_ref != reports_in_new:
stdout("Error: The number of results are different "
" strict mode (1).\n")
tests_passed = False
stdout(f"Diagnostic comparison complete "
f"(time: {time.time() - start_time:.2f}).\n")
return tests_passed
