def runCmpResults(Dir, Strictness=0):
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
# Log folders are also located in the results dir, so ignore them.
RefLogDir = os.path.join(RefDir, LogFolderName)
if RefLogDir in RefList:
RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName))
if len(RefList) == 0 or len(NewList) == 0:
return False
assert (len(RefList) == len(NewList))
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
NumDiffs = 0
PairList = zip(RefList, NewList)
for P in PairList:
RefDir = P[0]
NewDir = P[1]
assert (RefDir != NewDir)
if Verbose == 1:
print " Comparing Results: %s %s" % (RefDir, NewDir)
DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
Opts = CmpRuns.CmpOptions(DiffsPath, "", Dir)
# Discard everything coming out of stdout (CmpRun produces a lot of them).
OLD_STDOUT = sys.stdout
sys.stdout = Discarder()
# Scan the results, delete empty plist files.
NumDiffs, ReportsInRef, ReportsInNew = \
CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
sys.stdout = OLD_STDOUT
if (NumDiffs > 0):
print "Warning: %r differences in diagnostics. See %s" % \
(NumDiffs, DiffsPath,)
if Strictness >= 2 and NumDiffs > 0:
print "Error: Diffs found in strict mode (2)."
sys.exit(-1)
elif Strictness >= 1 and ReportsInRef != ReportsInNew:
print "Error: The number of results are different in strict mode (1)."
sys.exit(-1)
print "Diagnostic comparison complete (time: %.2f)." % (time.time() -
TBegin)
return (NumDiffs > 0)
def runCmpResults(Dir):
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
if len(RefList) == 0 or len(NewList) == 0:
return False
assert (len(RefList) == len(NewList))
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
HaveDiffs = False
PairList = zip(RefList, NewList)
for P in PairList:
RefDir = P[0]
NewDir = P[1]
assert (RefDir != NewDir)
if Verbose == 1:
print " Comparing Results: %s %s" % (RefDir, NewDir)
DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
Opts = CmpRuns.CmpOptions(DiffsPath)
# Discard everything coming out of stdout (CmpRun produces a lot of them).
OLD_STDOUT = sys.stdout
sys.stdout = Discarder()
# Scan the results, delete empty plist files.
HaveDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False)
sys.stdout = OLD_STDOUT
if HaveDiffs:
print "Warning: difference in diagnostics. See %s" % (DiffsPath, )
HaveDiffs = True
print "Diagnostic comparison complete (time: %.2f)." % (time.time() -
TBegin)
return HaveDiffs
def runCmpResults(Dir, Strictness=0):
"""
Compare the warnings produced by scan-build.
Strictness defines the success criteria for the test:
0 - success if there are no crashes or analyzer failure.
1 - success if there are no difference in the number of reported bugs.
2 - success if all the bug reports are identical.
:return success: Whether tests pass according to the Strictness
criteria.
"""
TestsPassed = True
TBegin = time.time()
RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
NewDir = os.path.join(Dir, SBOutputDirName)
# We have to go one level down the directory tree.
RefList = glob.glob(RefDir + "/*")
NewList = glob.glob(NewDir + "/*")
# Log folders are also located in the results dir, so ignore them.
RefLogDir = os.path.join(RefDir, LogFolderName)
if RefLogDir in RefList:
RefList.remove(RefLogDir)
NewList.remove(os.path.join(NewDir, LogFolderName))
if len(RefList) != len(NewList):
print "Mismatch in number of results folders: %s vs %s" % (
RefList, NewList)
sys.exit(1)
# There might be more then one folder underneath - one per each scan-build
# command (Ex: one for configure and one for make).
if (len(RefList) > 1):
# Assume that the corresponding folders have the same names.
RefList.sort()
NewList.sort()
# Iterate and find the differences.
NumDiffs = 0
PairList = zip(RefList, NewList)
for P in PairList:
RefDir = P[0]
NewDir = P[1]
assert(RefDir != NewDir)
if Verbose == 1:
print " Comparing Results: %s %s" % (RefDir, NewDir)
PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
Opts = CmpRuns.CmpOptions(rootA="", rootB=PatchedSourceDirPath)
# Scan the results, delete empty plist files.
NumDiffs, ReportsInRef, ReportsInNew = \
CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
if (NumDiffs > 0):
print "Warning: %s differences in diagnostics." % NumDiffs
if Strictness >= 2 and NumDiffs > 0:
print "Error: Diffs found in strict mode (2)."
TestsPassed = False
elif Strictness >= 1 and ReportsInRef != ReportsInNew:
print "Error: The number of results are different in "\
"strict mode (1)."
TestsPassed = False
print "Diagnostic comparison complete (time: %.2f)." % (
time.time() - TBegin)
return TestsPassed