Esempio n. 1
0
def runCmpResults(Dir, Strictness=0):
    """
    Compare the warnings produced by scan-build.
    Strictness defines the success criteria for the test:
      0 - success if there are no crashes or analyzer failure.
      1 - success if there are no difference in the number of reported bugs.
      2 - success if all the bug reports are identical.

    :return success: Whether tests pass according to the Strictness
    criteria.
    """
    TestsPassed = True
    TBegin = time.time()

    RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
    NewDir = os.path.join(Dir, SBOutputDirName)

    # We have to go one level down the directory tree.
    RefList = glob.glob(RefDir + "/*")
    NewList = glob.glob(NewDir + "/*")

    # Log folders are also located in the results dir, so ignore them.
    RefLogDir = os.path.join(RefDir, LogFolderName)
    if RefLogDir in RefList:
        RefList.remove(RefLogDir)
    NewList.remove(os.path.join(NewDir, LogFolderName))

    if len(RefList) != len(NewList):
        print "Mismatch in number of results folders: %s vs %s" % (
            RefList, NewList)
        sys.exit(1)

    # There might be more then one folder underneath - one per each scan-build
    # command (Ex: one for configure and one for make).
    if (len(RefList) > 1):
        # Assume that the corresponding folders have the same names.
        RefList.sort()
        NewList.sort()

    # Iterate and find the differences.
    NumDiffs = 0
    PairList = zip(RefList, NewList)
    for P in PairList:
        RefDir = P[0]
        NewDir = P[1]

        assert(RefDir != NewDir)
        if Verbose == 1:
            Local.stdout.write("  Comparing Results: %s %s\n" % (
                               RefDir, NewDir))

        PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
        Opts, Args = CmpRuns.generate_option_parser().parse_args(
            ["--rootA", "", "--rootB", PatchedSourceDirPath])
        # Scan the results, delete empty plist files.
        NumDiffs, ReportsInRef, ReportsInNew = \
            CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts,
                                             deleteEmpty=False,
                                             Stdout=Local.stdout)
        if (NumDiffs > 0):
            Local.stdout.write("Warning: %s differences in diagnostics.\n"
                               % NumDiffs)
        if Strictness >= 2 and NumDiffs > 0:
            Local.stdout.write("Error: Diffs found in strict mode (2).\n")
            TestsPassed = False
        elif Strictness >= 1 and ReportsInRef != ReportsInNew:
            Local.stdout.write("Error: The number of results are different " +
                               " strict mode (1).\n")
            TestsPassed = False

    Local.stdout.write("Diagnostic comparison complete (time: %.2f).\n" % (
                       time.time() - TBegin))
    return TestsPassed
Esempio n. 2
0
def runCmpResults(Dir, Strictness=0):
    """
    Compare the warnings produced by scan-build.
    Strictness defines the success criteria for the test:
      0 - success if there are no crashes or analyzer failure.
      1 - success if there are no difference in the number of reported bugs.
      2 - success if all the bug reports are identical.

    :return success: Whether tests pass according to the Strictness
    criteria.
    """
    TestsPassed = True
    TBegin = time.time()

    RefDir = os.path.join(Dir, SBOutputDirReferencePrefix + SBOutputDirName)
    NewDir = os.path.join(Dir, SBOutputDirName)

    # We have to go one level down the directory tree.
    RefList = glob.glob(RefDir + "/*")
    NewList = glob.glob(NewDir + "/*")

    # Log folders are also located in the results dir, so ignore them.
    RefLogDir = os.path.join(RefDir, LogFolderName)
    if RefLogDir in RefList:
        RefList.remove(RefLogDir)
    NewList.remove(os.path.join(NewDir, LogFolderName))

    if len(RefList) != len(NewList):
        print("Mismatch in number of results folders: %s vs %s" %
              (RefList, NewList))
        sys.exit(1)

    # There might be more then one folder underneath - one per each scan-build
    # command (Ex: one for configure and one for make).
    if (len(RefList) > 1):
        # Assume that the corresponding folders have the same names.
        RefList.sort()
        NewList.sort()

    # Iterate and find the differences.
    NumDiffs = 0
    for P in zip(RefList, NewList):
        RefDir = P[0]
        NewDir = P[1]

        assert (RefDir != NewDir)
        if Verbose == 1:
            Local.stdout.write("  Comparing Results: %s %s\n" %
                               (RefDir, NewDir))

        PatchedSourceDirPath = os.path.join(Dir, PatchedSourceDirName)
        Opts, Args = CmpRuns.generate_option_parser().parse_args(
            ["--rootA", "", "--rootB", PatchedSourceDirPath])
        # Scan the results, delete empty plist files.
        NumDiffs, ReportsInRef, ReportsInNew = \
            CmpRuns.dumpScanBuildResultsDiff(RefDir, NewDir, Opts,
                                             deleteEmpty=False,
                                             Stdout=Local.stdout)
        if (NumDiffs > 0):
            Local.stdout.write("Warning: %s differences in diagnostics.\n" %
                               NumDiffs)
        if Strictness >= 2 and NumDiffs > 0:
            Local.stdout.write("Error: Diffs found in strict mode (2).\n")
            TestsPassed = False
        elif Strictness >= 1 and ReportsInRef != ReportsInNew:
            Local.stdout.write("Error: The number of results are different " +
                               " strict mode (1).\n")
            TestsPassed = False

    Local.stdout.write("Diagnostic comparison complete (time: %.2f).\n" %
                       (time.time() - TBegin))
    return TestsPassed
Esempio n. 3
0
def run_cmp_results(directory: str, strictness: int = 0) -> bool:
    """
    Compare the warnings produced by scan-build.
    strictness defines the success criteria for the test:
      0 - success if there are no crashes or analyzer failure.
      1 - success if there are no difference in the number of reported bugs.
      2 - success if all the bug reports are identical.

    :return success: Whether tests pass according to the strictness
    criteria.
    """
    tests_passed = True
    start_time = time.time()

    ref_dir = os.path.join(directory, REF_PREFIX + OUTPUT_DIR_NAME)
    new_dir = os.path.join(directory, OUTPUT_DIR_NAME)

    # We have to go one level down the directory tree.
    ref_list = glob.glob(ref_dir + "/*")
    new_list = glob.glob(new_dir + "/*")

    # Log folders are also located in the results dir, so ignore them.
    ref_log_dir = os.path.join(ref_dir, LOG_DIR_NAME)
    if ref_log_dir in ref_list:
        ref_list.remove(ref_log_dir)
    new_list.remove(os.path.join(new_dir, LOG_DIR_NAME))

    if len(ref_list) != len(new_list):
        stderr(f"Mismatch in number of results folders: "
               f"{ref_list} vs {new_list}")
        sys.exit(1)

    # There might be more then one folder underneath - one per each scan-build
    # command (Ex: one for configure and one for make).
    if len(ref_list) > 1:
        # Assume that the corresponding folders have the same names.
        ref_list.sort()
        new_list.sort()

    # Iterate and find the differences.
    num_diffs = 0
    for ref_dir, new_dir in zip(ref_list, new_list):
        assert(ref_dir != new_dir)

        if VERBOSE == 1:
            stdout(f"  Comparing Results: {ref_dir} {new_dir}\n")

        patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME)

        # TODO: get rid of option parser invocation here
        opts, args = CmpRuns.generate_option_parser().parse_args(
            ["--rootA", "", "--rootB", patched_source])
        # Scan the results, delete empty plist files.
        num_diffs, reports_in_ref, reports_in_new = \
            CmpRuns.dumpScanBuildResultsDiff(ref_dir, new_dir, opts,
                                             deleteEmpty=False,
                                             Stdout=LOCAL.stdout)

        if num_diffs > 0:
            stdout(f"Warning: {num_diffs} differences in diagnostics.\n")

        if strictness >= 2 and num_diffs > 0:
            stdout("Error: Diffs found in strict mode (2).\n")
            tests_passed = False

        elif strictness >= 1 and reports_in_ref != reports_in_new:
            stdout("Error: The number of results are different "
                   " strict mode (1).\n")
            tests_passed = False

    stdout(f"Diagnostic comparison complete "
           f"(time: {time.time() - start_time:.2f}).\n")

    return tests_passed