def runCmpResults(Dir, Strictness=0):
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
# Log folders are also located in the results dir, so ignore them.
RefLogDir = os.path.join(RefDir, LogFolderName)
if RefLogDir in RefList:
RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName))
if len(RefList) == 0 or len(NewList) == 0:
return False
assert (len(RefList) == len(NewList))
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
NumDiffs = 0
PairList = zip(RefList, NewList)
for P in PairList:
RefDir = P[0]
NewDir = P[1]
assert (RefDir != NewDir)
if Verbose == 1:
print " Comparing Results: %s %s" % (RefDir, NewDir)
DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
Opts = CmpRuns.CmpOptions(DiffsPath, "", Dir)
# Discard everything coming out of stdout (CmpRun produces a lot of them).
OLD_STDOUT = sys.stdout
sys.stdout = Discarder()
# Scan the results, delete empty plist files.
NumDiffs, ReportsInRef, ReportsInNew = \
CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
sys.stdout = OLD_STDOUT
if (NumDiffs > 0):
print "Warning: %r differences in diagnostics. See %s" % \
(NumDiffs, DiffsPath,)
if Strictness >= 2 and NumDiffs > 0:
print "Error: Diffs found in strict mode (2)."
sys.exit(-1)
elif Strictness >= 1 and ReportsInRef != ReportsInNew:
print "Error: The number of results are different in strict mode (1)."
sys.exit(-1)
print "Diagnostic comparison complete (time: %.2f)." % (time.time() -
TBegin)
return (NumDiffs > 0)
def compare(parser, args):
dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
CmpRuns.dump_scan_build_results_diff(dir_old,
dir_new,
show_stats=args.show_stats,
stats_only=args.stats_only,
histogram=args.histogram,
verbose_log=args.verbose_log)
def runCmpResults(Dir, Strictness = 0):
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
# Log folders are also located in the results dir, so ignore them.
RefLogDir = os.path.join(RefDir, LogFolderName)
if RefLogDir in RefList:
RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName))
if len(RefList) == 0 or len(NewList) == 0:
return False
assert(len(RefList) == len(NewList))
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
NumDiffs = 0
PairList = zip(RefList, NewList)
for P in PairList:
RefDir = P[0]
NewDir = P[1]
assert(RefDir != NewDir)
if Verbose == 1:
print " Comparing Results: %s %s" % (RefDir, NewDir)
DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
Opts = CmpRuns.CmpOptions(DiffsPath, "", PatchedSourceDirPath)
# Discard everything coming out of stdout (CmpRun produces a lot of them).
OLD_STDOUT = sys.stdout
sys.stdout = Discarder()
# Scan the results, delete empty plist files.
NumDiffs, ReportsInRef, ReportsInNew = \
CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
sys.stdout = OLD_STDOUT
if (NumDiffs > 0) :
print "Warning: %r differences in diagnostics. See %s" % \
(NumDiffs, DiffsPath,)
if Strictness >= 2 and NumDiffs > 0:
print "Error: Diffs found in strict mode (2)."
sys.exit(-1)
elif Strictness >= 1 and ReportsInRef != ReportsInNew:
print "Error: The number of results are different in strict mode (1)."
sys.exit(-1)
print "Diagnostic comparison complete (time: %.2f)." % (time.time()-TBegin)
return (NumDiffs > 0)
def runCmpResults(Dir):
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
if len(RefList) == 0 or len(NewList) == 0:
return False
assert (len(RefList) == len(NewList))
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
HaveDiffs = False
PairList = zip(RefList, NewList)
for P in PairList:
RefDir = P[0]
NewDir = P[1]
assert (RefDir != NewDir)
if Verbose == 1:
print " Comparing Results: %s %s" % (RefDir, NewDir)
DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
Opts = CmpRuns.CmpOptions(DiffsPath)
# Discard everything coming out of stdout (CmpRun produces a lot of them).
OLD_STDOUT = sys.stdout
sys.stdout = Discarder()
# Scan the results, delete empty plist files.
HaveDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False)
sys.stdout = OLD_STDOUT
if HaveDiffs:
print "Warning: difference in diagnostics. See %s" % (DiffsPath, )
HaveDiffs = True
print "Diagnostic comparison complete (time: %.2f)." % (time.time() -
TBegin)
return HaveDiffs
def compare(parser, args):
import CmpRuns
choices = [
CmpRuns.HistogramType.RELATIVE.value,
CmpRuns.HistogramType.LOG_RELATIVE.value,
CmpRuns.HistogramType.ABSOLUTE.value
]
if args.histogram is not None and args.histogram not in choices:
parser.error(
"Incorrect histogram type, available choices are {}".format(
choices))
dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old)
dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new)
CmpRuns.dump_scan_build_results_diff(dir_old,
dir_new,
show_stats=args.show_stats,
stats_only=args.stats_only,
histogram=args.histogram,
verbose_log=args.verbose_log)
def runCmpResults(Dir):
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
if len(RefList) == 0 or len(NewList) == 0:
return False
assert len(RefList) == len(NewList)
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if len(RefList) > 1:
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
HaveDiffs = False
PairList = zip(RefList, NewList)
for P in PairList:
RefDir = P[0]
NewDir = P[1]
assert RefDir != NewDir
if Verbose == 1:
print " Comparing Results: %s %s" % (RefDir, NewDir)
DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
Opts = CmpRuns.CmpOptions(DiffsPath)
# Discard everything coming out of stdout (CmpRun produces a lot of them).
OLD_STDOUT = sys.stdout
sys.stdout = Discarder()
# Scan the results, delete empty plist files.
HaveDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False)
sys.stdout = OLD_STDOUT
if HaveDiffs:
print "Warning: difference in diagnostics. See %s" % (DiffsPath,)
HaveDiffs = True
print "Diagnostic comparison complete (time: %.2f)." % (time.time() - TBegin)
return HaveDiffs
def runCmpResults(Dir, Strictness=0):
"""
Compare the warnings produced by scan-build.
Strictness defines the success criteria for the test:
0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the Strictness
criteria.
"""
TestsPassed = True
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
# Log folders are also located in the results dir, so ignore them.
RefLogDir = os.path.join(RefDir, LogFolderName)
if RefLogDir in RefList:
RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName))
if len(RefList) != len(NewList):
print("Mismatch in number of results folders: %s vs %s" %
(RefList, NewList))
sys.exit(1)
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
NumDiffs = 0
for P in zip(RefList, NewList):
RefDir = P[0]
NewDir = P[1]
assert (RefDir != NewDir)
if Verbose == 1:
Local.stdout.write(" Comparing Results: %s %s\n" %
(RefDir, NewDir))
PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
Opts, Args = CmpRuns.generate_option_parser().parse_args(
["--rootA", "", "--rootB", PatchedSourceDirPath])
# Scan the results, delete empty plist files.
NumDiffs, ReportsInRef, ReportsInNew = \
CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts,
deleteEmpty=False,
Stdout=Local.stdout)
if (NumDiffs > 0):
Local.stdout.write("Warning: %s differences in diagnostics.\n" %
NumDiffs)
if Strictness >= 2 and NumDiffs > 0:
Local.stdout.write("Error: Diffs found in strict mode (2).\n")
TestsPassed = False
elif Strictness >= 1 and ReportsInRef != ReportsInNew:
Local.stdout.write("Error: The number of results are different " +
" strict mode (1).\n")
TestsPassed = False
Local.stdout.write("Diagnostic comparison complete (time: %.2f).\n" %
(time.time() - TBegin))
return TestsPassed
def runCmpResults(Dir, Strictness=0):
"""
Compare the warnings produced by scan-build.
Strictness defines the success criteria for the test:
0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the Strictness
criteria.
"""
TestsPassed = True
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
# Log folders are also located in the results dir, so ignore them.
RefLogDir = os.path.join(RefDir, LogFolderName)
if RefLogDir in RefList:
RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName))
if len(RefList) != len(NewList):
print "Mismatch in number of results folders: %s vs %s" % (
RefList, NewList)
sys.exit(1)
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
NumDiffs = 0
PairList = zip(RefList, NewList)
for P in PairList:
RefDir = P[0]
NewDir = P[1]
assert(RefDir != NewDir)
if Verbose == 1:
print " Comparing Results: %s %s" % (RefDir, NewDir)
PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
Opts = CmpRuns.CmpOptions(rootA="", rootB=PatchedSourceDirPath)
# Scan the results, delete empty plist files.
NumDiffs, ReportsInRef, ReportsInNew = \
CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
if (NumDiffs > 0):
print "Warning: %s differences in diagnostics." % NumDiffs
if Strictness >= 2 and NumDiffs > 0:
print "Error: Diffs found in strict mode (2)."
TestsPassed = False
elif Strictness >= 1 and ReportsInRef != ReportsInNew:
print "Error: The number of results are different in "\
"strict mode (1)."
TestsPassed = False
print "Diagnostic comparison complete (time: %.2f)." % (
time.time() - TBegin)
return TestsPassed
def run_cmp_results(directory: str, strictness: int = 0) -> bool:
"""
Compare the warnings produced by scan-build.
strictness defines the success criteria for the test:
0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the strictness
criteria.
"""
tests_passed = True
start_time = time.time()
ref_dir = os.path.join(directory, REF_PREFIX + OUTPUT_DIR_NAME)
new_dir = os.path.join(directory, OUTPUT_DIR_NAME)
# We have to go one level down the directory tree.
ref_list = glob.glob(ref_dir + "/*")
new_list = glob.glob(new_dir + "/*")
# Log folders are also located in the results dir, so ignore them.
ref_log_dir = os.path.join(ref_dir, LOG_DIR_NAME)
if ref_log_dir in ref_list:
ref_list.remove(ref_log_dir)
new_list.remove(os.path.join(new_dir, LOG_DIR_NAME))
if len(ref_list) != len(new_list):
stderr(f"Mismatch in number of results folders: "
f"{ref_list} vs {new_list}")
sys.exit(1)
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if len(ref_list) > 1:
# Assume that the corresponding folders have the same names.
ref_list.sort()
new_list.sort()
# Iterate and find the differences.
num_diffs = 0
for ref_dir, new_dir in zip(ref_list, new_list):
assert(ref_dir != new_dir)
if VERBOSE >= 1:
stdout(f" Comparing Results: {ref_dir} {new_dir}\n")
patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)
ref_results = CmpRuns.ResultsDirectory(ref_dir)
new_results = CmpRuns.ResultsDirectory(new_dir, patched_source)
# Scan the results, delete empty plist files.
num_diffs, reports_in_ref, reports_in_new = \
CmpRuns.dump_scan_build_results_diff(ref_results, new_results,
delete_empty=False,
out=LOCAL.stdout)
if num_diffs > 0:
stdout(f"Warning: {num_diffs} differences in diagnostics.\n")
if strictness >= 2 and num_diffs > 0:
stdout("Error: Diffs found in strict mode (2).\n")
tests_passed = False
elif strictness >= 1 and reports_in_ref != reports_in_new:
stdout("Error: The number of results are different "
" strict mode (1).\n")
tests_passed = False
stdout(f"Diagnostic comparison complete "
f"(time: {time.time() - start_time:.2f}).\n")
return tests_passed
def runCmpResults(Dir, Strictness=0):
"""
Compare the warnings produced by scan-build.
Strictness defines the success criteria for the test:
0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the Strictness
criteria.
"""
TestsPassed = True
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
# Log folders are also located in the results dir, so ignore them.
RefLogDir = os.path.join(RefDir, LogFolderName)
if RefLogDir in RefList:
RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName))
assert (len(RefList) == len(NewList))
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
NumDiffs = 0
PairList = zip(RefList, NewList)
for P in PairList:
RefDir = P[0]
NewDir = P[1]
assert (RefDir != NewDir)
if Verbose == 1:
print " Comparing Results: %s %s" % (RefDir, NewDir)
DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
Opts = CmpRuns.CmpOptions(DiffsPath, "", PatchedSourceDirPath)
# Discard everything coming out of stdout
# (CmpRun produces a lot of them).
OLD_STDOUT = sys.stdout
sys.stdout = SATestUtils.Discarder()
# Scan the results, delete empty plist files.
NumDiffs, ReportsInRef, ReportsInNew = \
CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
sys.stdout = OLD_STDOUT
if (NumDiffs > 0):
print "Warning: %r differences in diagnostics. See %s" % \
(NumDiffs, DiffsPath,)
if Strictness >= 2 and NumDiffs > 0:
print "Error: Diffs found in strict mode (2)."
TestsPassed = False
elif Strictness >= 1 and ReportsInRef != ReportsInNew:
print "Error: The number of results are different in "\
"strict mode (1)."
TestsPassed = False
print "Diagnostic comparison complete (time: %.2f)." % (time.time() -
TBegin)
return TestsPassed
