Esempio n. 1
0
def runCmpResults(Dir, Strictness=0):
    TBegin = time.time()

    RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
    NewDir = os.path.join(Dir, SBOutputDirName)

    # We have to go one level down the directory tree.
    RefList = glob.glob(RefDir + "/*")
    NewList = glob.glob(NewDir + "/*")

    # Log folders are also located in the results dir, so ignore them.
    RefLogDir = os.path.join(RefDir, LogFolderName)
    if RefLogDir in RefList:
        RefList.remove(RefLogDir)
    NewList.remove(os.path.join(NewDir, LogFolderName))

    if len(RefList) == 0 or len(NewList) == 0:
        return False
    assert (len(RefList) == len(NewList))

    # There might be more then one folder underneath - one per each scan-build
    # command (Ex: one for configure and one for make).
    if (len(RefList) > 1):
        # Assume that the corresponding folders have the same names.
        RefList.sort()
        NewList.sort()

    # Iterate and find the differences.
    NumDiffs = 0
    PairList = zip(RefList, NewList)
    for P in PairList:
        RefDir = P[0]
        NewDir = P[1]

        assert (RefDir != NewDir)
        if Verbose == 1:
            print "  Comparing Results: %s %s" % (RefDir, NewDir)

        DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
        Opts = CmpRuns.CmpOptions(DiffsPath, "", Dir)
        # Discard everything coming out of stdout (CmpRun produces a lot of them).
        OLD_STDOUT = sys.stdout
        sys.stdout = Discarder()
        # Scan the results, delete empty plist files.
        NumDiffs, ReportsInRef, ReportsInNew = \
            CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
        sys.stdout = OLD_STDOUT
        if (NumDiffs > 0):
            print "Warning: %r differences in diagnostics. See %s" % \
                  (NumDiffs, DiffsPath,)
        if Strictness >= 2 and NumDiffs > 0:
            print "Error: Diffs found in strict mode (2)."
            sys.exit(-1)
        elif Strictness >= 1 and ReportsInRef != ReportsInNew:
            print "Error: The number of results are different in strict mode (1)."
            sys.exit(-1)

    print "Diagnostic comparison complete (time: %.2f)." % (time.time() -
                                                            TBegin)
    return (NumDiffs > 0)
Esempio n. 2
0
def runCmpResults(Dir):
    TBegin = time.time()

    RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
    NewDir = os.path.join(Dir, SBOutputDirName)

    # We have to go one level down the directory tree.
    RefList = glob.glob(RefDir + "/*")
    NewList = glob.glob(NewDir + "/*")
    if len(RefList) == 0 or len(NewList) == 0:
        return False
    assert (len(RefList) == len(NewList))

    # There might be more then one folder underneath - one per each scan-build
    # command (Ex: one for configure and one for make).
    if (len(RefList) > 1):
        # Assume that the corresponding folders have the same names.
        RefList.sort()
        NewList.sort()

    # Iterate and find the differences.
    HaveDiffs = False
    PairList = zip(RefList, NewList)
    for P in PairList:
        RefDir = P[0]
        NewDir = P[1]

        assert (RefDir != NewDir)
        if Verbose == 1:
            print "  Comparing Results: %s %s" % (RefDir, NewDir)

        DiffsPath = os.path.join(NewDir, DiffsSummaryFileName)
        Opts = CmpRuns.CmpOptions(DiffsPath)
        # Discard everything coming out of stdout (CmpRun produces a lot of them).
        OLD_STDOUT = sys.stdout
        sys.stdout = Discarder()
        # Scan the results, delete empty plist files.
        HaveDiffs = CmpRuns.cmpScanBuildResults(RefDir, NewDir, Opts, False)
        sys.stdout = OLD_STDOUT
        if HaveDiffs:
            print "Warning: difference in diagnostics. See %s" % (DiffsPath, )
            HaveDiffs = True

    print "Diagnostic comparison complete (time: %.2f)." % (time.time() -
                                                            TBegin)
    return HaveDiffs
Esempio n. 3
0
def runCmpResults(Dir, Strictness=0):
    """
    Compare the warnings produced by scan-build.
    Strictness defines the success criteria for the test:
      0 - success if there are no crashes or analyzer failure.
      1 - success if there are no difference in the number of reported bugs.
      2 - success if all the bug reports are identical.

    :return success: Whether tests pass according to the Strictness
    criteria.
    """
    TestsPassed = True
    TBegin = time.time()

    RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
    NewDir = os.path.join(Dir, SBOutputDirName)

    # We have to go one level down the directory tree.
    RefList = glob.glob(RefDir + "/*")
    NewList = glob.glob(NewDir + "/*")

    # Log folders are also located in the results dir, so ignore them.
    RefLogDir = os.path.join(RefDir, LogFolderName)
    if RefLogDir in RefList:
        RefList.remove(RefLogDir)
    NewList.remove(os.path.join(NewDir, LogFolderName))

    if len(RefList) != len(NewList):
        print "Mismatch in number of results folders: %s vs %s" % (
            RefList, NewList)
        sys.exit(1)

    # There might be more then one folder underneath - one per each scan-build
    # command (Ex: one for configure and one for make).
    if (len(RefList) > 1):
        # Assume that the corresponding folders have the same names.
        RefList.sort()
        NewList.sort()

    # Iterate and find the differences.
    NumDiffs = 0
    PairList = zip(RefList, NewList)
    for P in PairList:
        RefDir = P[0]
        NewDir = P[1]

        assert(RefDir != NewDir)
        if Verbose == 1:
            print "  Comparing Results: %s %s" % (RefDir, NewDir)

        PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
        Opts = CmpRuns.CmpOptions(rootA="", rootB=PatchedSourceDirPath)
        # Scan the results, delete empty plist files.
        NumDiffs, ReportsInRef, ReportsInNew = \
            CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts, False)
        if (NumDiffs > 0):
            print "Warning: %s differences in diagnostics." % NumDiffs
        if Strictness >= 2 and NumDiffs > 0:
            print "Error: Diffs found in strict mode (2)."
            TestsPassed = False
        elif Strictness >= 1 and ReportsInRef != ReportsInNew:
            print "Error: The number of results are different in "\
                  "strict mode (1)."
            TestsPassed = False

    print "Diagnostic comparison complete (time: %.2f)." % (
          time.time() - TBegin)
    return TestsPassed