def runCmpResults(Dir, Strictness=0):
"""
Compare the warnings produced by scan-build.
Strictness defines the success criteria for the test:
0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the Strictness
criteria.
"""
TestsPassed = True
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
# Log folders are also located in the results dir, so ignore them.
RefLogDir = os.path.join(RefDir, LogFolderName)
if RefLogDir in RefList:
RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName))
if len(RefList) != len(NewList):
print "Mismatch in number of results folders: %s vs %s" % (
RefList, NewList)
sys.exit(1)
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
NumDiffs = 0
PairList = zip(RefList, NewList)
for P in PairList:
RefDir = P[0]
NewDir = P[1]
assert(RefDir != NewDir)
if Verbose == 1:
Local.stdout.write(" Comparing Results: %s %s\n" % (
RefDir, NewDir))
PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
Opts, Args = CmpRuns.generate_option_parser().parse_args(
["--rootA", "", "--rootB", PatchedSourceDirPath])
# Scan the results, delete empty plist files.
NumDiffs, ReportsInRef, ReportsInNew = \
CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts,
deleteEmpty=False,
Stdout=Local.stdout)
if (NumDiffs > 0):
Local.stdout.write("Warning: %s differences in diagnostics.\n"
% NumDiffs)
if Strictness >= 2 and NumDiffs > 0:
Local.stdout.write("Error: Diffs found in strict mode (2).\n")
TestsPassed = False
elif Strictness >= 1 and ReportsInRef != ReportsInNew:
Local.stdout.write("Error: The number of results are different " +
" strict mode (1).\n")
TestsPassed = False
Local.stdout.write("Diagnostic comparison complete (time: %.2f).\n" % (
time.time() - TBegin))
return TestsPassed
def runCmpResults(Dir, Strictness=0):
"""
Compare the warnings produced by scan-build.
Strictness defines the success criteria for the test:
0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the Strictness
criteria.
"""
TestsPassed = True
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
# Log folders are also located in the results dir, so ignore them.
RefLogDir = os.path.join(RefDir, LogFolderName)
if RefLogDir in RefList:
RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName))
if len(RefList) != len(NewList):
print("Mismatch in number of results folders: %s vs %s" %
(RefList, NewList))
sys.exit(1)
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
NumDiffs = 0
for P in zip(RefList, NewList):
RefDir = P[0]
NewDir = P[1]
assert (RefDir != NewDir)
if Verbose == 1:
Local.stdout.write(" Comparing Results: %s %s\n" %
(RefDir, NewDir))
PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
Opts, Args = CmpRuns.generate_option_parser().parse_args(
["--rootA", "", "--rootB", PatchedSourceDirPath])
# Scan the results, delete empty plist files.
NumDiffs, ReportsInRef, ReportsInNew = \
CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts,
deleteEmpty=False,
Stdout=Local.stdout)
if (NumDiffs > 0):
Local.stdout.write("Warning: %s differences in diagnostics.\n" %
NumDiffs)
if Strictness >= 2 and NumDiffs > 0:
Local.stdout.write("Error: Diffs found in strict mode (2).\n")
TestsPassed = False
elif Strictness >= 1 and ReportsInRef != ReportsInNew:
Local.stdout.write("Error: The number of results are different " +
" strict mode (1).\n")
TestsPassed = False
Local.stdout.write("Diagnostic comparison complete (time: %.2f).\n" %
(time.time() - TBegin))
return TestsPassed
def run_cmp_results(directory: str, strictness: int = 0) -> bool:
"""
Compare the warnings produced by scan-build.
strictness defines the success criteria for the test:
0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the strictness
criteria.
"""
tests_passed = True
start_time = time.time()
ref_dir = os.path.join(directory, REF_PREFIX + OUTPUT_DIR_NAME)
new_dir = os.path.join(directory, OUTPUT_DIR_NAME)
# We have to go one level down the directory tree.
ref_list = glob.glob(ref_dir + "/*")
new_list = glob.glob(new_dir + "/*")
# Log folders are also located in the results dir, so ignore them.
ref_log_dir = os.path.join(ref_dir, LOG_DIR_NAME)
if ref_log_dir in ref_list:
ref_list.remove(ref_log_dir)
new_list.remove(os.path.join(new_dir, LOG_DIR_NAME))
if len(ref_list) != len(new_list):
stderr(f"Mismatch in number of results folders: "
f"{ref_list} vs {new_list}")
sys.exit(1)
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if len(ref_list) > 1:
# Assume that the corresponding folders have the same names.
ref_list.sort()
new_list.sort()
# Iterate and find the differences.
num_diffs = 0
for ref_dir, new_dir in zip(ref_list, new_list):
assert(ref_dir != new_dir)
if VERBOSE == 1:
stdout(f" Comparing Results: {ref_dir} {new_dir}\n")
patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
# TODO: get rid of option parser invocation here
opts, args = CmpRuns.generate_option_parser().parse_args(
["--rootA", "", "--rootB", patched_source])
# Scan the results, delete empty plist files.
num_diffs, reports_in_ref, reports_in_new = \
CmpRuns.dumpScanBuildResultsDiff(ref_dir, new_dir, opts,
deleteEmpty=False,
Stdout=LOCAL.stdout)
if num_diffs > 0:
stdout(f"Warning: {num_diffs} differences in diagnostics.\n")
if strictness >= 2 and num_diffs > 0:
stdout("Error: Diffs found in strict mode (2).\n")
tests_passed = False
elif strictness >= 1 and reports_in_ref != reports_in_new:
stdout("Error: The number of results are different "
" strict mode (1).\n")
tests_passed = False
stdout(f"Diagnostic comparison complete "
f"(time: {time.time() - start_time:.2f}).\n")
return tests_passed
