Beispiel #1
0
def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False,
         exclude=False, single=False, randomize=False, fromfile=None,
         findleaks=False, use_resources=None, trace=False, coverdir='coverage',
         runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
         random_seed=None):
    """Execute a test suite.

    This also parses command-line options and modifies its behavior
    accordingly.

    tests -- a list of strings containing test names (optional)
    testdir -- the directory in which to look for tests (optional)

    Users other than the Python test suite will certainly want to
    specify testdir; if it's omitted, the directory containing the
    Python test suite is searched for.

    If the tests argument is omitted, the tests listed on the
    command-line will be used.  If that's empty, too, then all *.py
    files beginning with test_ will be used.

    The other default arguments (verbose, quiet, generate, exclude,
    single, randomize, findleaks, use_resources, trace, coverdir,
    print_slow, and random_seed) allow programmers calling main()
    directly to set the values that would normally be set by flags
    on the command line.
    """

    support.record_original_stdout(sys.stdout)
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'hvgqxsSrf:lu:t:TD:NLR:wM:n',
                                   ['help', 'verbose', 'quiet', 'exclude',
                                    'single', 'slow', 'random', 'fromfile',
                                    'findleaks', 'use=', 'threshold=', 'trace',
                                    'coverdir=', 'nocoverdir', 'runleaks',
                                    'huntrleaks=', 'verbose2', 'memlimit=',
                                    'debug', 'start=', 'nowindows',
                                    'randseed=',
                                    ])
    except getopt.error as msg:
        usage(msg)

    # Defaults
    if random_seed is None:
        random_seed = random.randrange(10000000)
    if use_resources is None:
        use_resources = []
    debug = False
    start = None
    for o, a in opts:
        if o in ('-h', '--help'):
            print(__doc__)
            return
        elif o in ('-v', '--verbose'):
            verbose += 1
        elif o in ('-w', '--verbose2'):
            verbose2 = True
        elif o in ('-d', '--debug'):
            debug = True
        elif o in ('-q', '--quiet'):
            quiet = True;
            verbose = 0
        elif o in ('-x', '--exclude'):
            exclude = True
        elif o in ('-S', '--start'):
            start = a
        elif o in ('-s', '--single'):
            single = True
        elif o in ('-S', '--slow'):
            print_slow = True
        elif o in ('-r', '--randomize'):
            randomize = True
        elif o == '--randseed':
            random_seed = int(a)
        elif o in ('-f', '--fromfile'):
            fromfile = a
        elif o in ('-l', '--findleaks'):
            findleaks = True
        elif o in ('-L', '--runleaks'):
            runleaks = True
        elif o in ('-t', '--threshold'):
            import gc
            gc.set_threshold(int(a))
        elif o in ('-T', '--coverage'):
            trace = True
        elif o in ('-D', '--coverdir'):
            coverdir = os.path.join(os.getcwd(), a)
        elif o in ('-N', '--nocoverdir'):
            coverdir = None
        elif o in ('-R', '--huntrleaks'):
            huntrleaks = a.split(':')
            if len(huntrleaks) not in (2, 3):
                print(a, huntrleaks)
                usage('-R takes 2 or 3 colon-separated arguments')
            if not huntrleaks[0]:
                huntrleaks[0] = 5
            else:
                huntrleaks[0] = int(huntrleaks[0])
            if not huntrleaks[1]:
                huntrleaks[1] = 4
            else:
                huntrleaks[1] = int(huntrleaks[1])
            if len(huntrleaks) == 2 or not huntrleaks[2]:
                huntrleaks[2:] = ["reflog.txt"]
            # Avoid false positives due to the character cache in
            # stringobject.c filling slowly with random data
            warm_char_cache()
        elif o in ('-M', '--memlimit'):
            support.set_memlimit(a)
        elif o in ('-u', '--use'):
            u = [x.lower() for x in a.split(',')]
            for r in u:
                if r == 'all':
                    use_resources[:] = RESOURCE_NAMES
                    continue
                remove = False
                if r[0] == '-':
                    remove = True
                    r = r[1:]
                if r not in RESOURCE_NAMES:
                    usage('Invalid -u/--use option: ' + a)
                if remove:
                    if r in use_resources:
                        use_resources.remove(r)
                elif r not in use_resources:
                    use_resources.append(r)
        elif o in ('-n', '--nowindows'):
            import msvcrt
            msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS|
                    msvcrt.SEM_NOALIGNMENTFAULTEXCEPT|
                    msvcrt.SEM_NOGPFAULTERRORBOX|
                    msvcrt.SEM_NOOPENFILEERRORBOX)
            try:
                msvcrt.CrtSetReportMode
            except AttributeError:
                # release build
                pass
            else:
                for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
                    msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
                    msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
    if generate and verbose:
        usage("-g and -v don't go together!")
    if single and fromfile:
        usage("-s and -f don't go together!")

    good = []
    bad = []
    skipped = []
    resource_denieds = []

    if findleaks:
        try:
            import gc
        except ImportError:
            print('No GC available, disabling findleaks.')
            findleaks = False
        else:
            # Uncomment the line below to report garbage that is not
            # freeable by reference counting alone.  By default only
            # garbage that is not collectable by the GC is reported.
            #gc.set_debug(gc.DEBUG_SAVEALL)
            found_garbage = []

    if single:
        from tempfile import gettempdir
        filename = os.path.join(gettempdir(), 'pynexttest')
        try:
            fp = open(filename, 'r')
            next = fp.read().strip()
            tests = [next]
            fp.close()
        except IOError:
            pass

    if fromfile:
        tests = []
        fp = open(fromfile)
        for line in fp:
            guts = line.split() # assuming no test has whitespace in its name
            if guts and not guts[0].startswith('#'):
                tests.extend(guts)
        fp.close()

    # Strip .py extensions.
    if args:
        args = list(map(removepy, args))
    if tests:
        tests = list(map(removepy, tests))

    stdtests = STDTESTS[:]
    nottests = NOTTESTS.copy()
    if exclude:
        for arg in args:
            if arg in stdtests:
                stdtests.remove(arg)
            nottests.add(arg)
        args = []
    tests = tests or args or findtests(testdir, stdtests, nottests)
    if single:
        tests = tests[:1]
    # Remove all the tests that precede start if it's set.
    if start:
        try:
            del tests[:tests.index(start)]
        except ValueError:
            print("Couldn't find starting test (%s), using all tests" % start)
    if randomize:
        random.seed(random_seed)
        print("Using random seed", random_seed)
        random.shuffle(tests)
    if trace:
        import trace, tempfile
        tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix,
                                         tempfile.gettempdir()],
                             trace=False, count=True)
    test_times = []
    support.verbose = verbose      # Tell tests to be moderately quiet
    support.use_resources = use_resources
    save_modules = sys.modules.keys()
    for test in tests:
        if not quiet:
            print(test)
            sys.stdout.flush()
        if trace:
            # If we're tracing code coverage, then we don't exit with status
            # if on a false return value from main.
            tracer.runctx('runtest(test, generate, verbose, quiet,'
                          '        test_times, testdir)',
                          globals=globals(), locals=vars())
        else:
            try:
                ok = runtest(test, generate, verbose, quiet, test_times,
                             testdir, huntrleaks)
            except KeyboardInterrupt:
                # print a newline separate from the ^C
                print()
                break
            except:
                raise
            if ok > 0:
                good.append(test)
            elif ok == 0:
                bad.append(test)
            else:
                skipped.append(test)
                if ok == -2:
                    resource_denieds.append(test)
        if findleaks:
            gc.collect()
            if gc.garbage:
                print("Warning: test created", len(gc.garbage), end=' ')
                print("uncollectable object(s).")
                # move the uncollectable objects somewhere so we don't see
                # them again
                found_garbage.extend(gc.garbage)
                del gc.garbage[:]
        # Unload the newly imported modules (best effort finalization)
        for module in sys.modules.keys():
            if module not in save_modules and module.startswith("test."):
                support.unload(module)

    # The lists won't be sorted if running with -r
    good.sort()
    bad.sort()
    skipped.sort()

    if good and not quiet:
        if not bad and not skipped and len(good) > 1:
            print("All", end=' ')
        print(count(len(good), "test"), "OK.")
        if verbose:
            print("CAUTION:  stdout isn't compared in verbose mode:")
            print("a test that passes in verbose mode may fail without it.")
    if print_slow:
        test_times.sort(reverse=True)
        print("10 slowest tests:")
        for time, test in test_times[:10]:
            print("%s: %.1fs" % (test, time))
    if bad:
        print(count(len(bad), "test"), "failed:")
        printlist(bad)
    if skipped and not quiet:
        print(count(len(skipped), "test"), "skipped:")
        printlist(skipped)

        e = _ExpectedSkips()
        plat = sys.platform
        if e.isvalid():
            surprise = set(skipped) - e.getexpected() - set(resource_denieds)
            if surprise:
                print(count(len(surprise), "skip"), \
                      "unexpected on", plat + ":")
                printlist(surprise)
            else:
                print("Those skips are all expected on", plat + ".")
        else:
            print("Ask someone to teach regrtest.py about which tests are")
            print("expected to get skipped on", plat + ".")

    if verbose2 and bad:
        print("Re-running failed tests in verbose mode")
        for test in bad:
            print("Re-running test %r in verbose mode" % test)
            sys.stdout.flush()
            try:
                support.verbose = True
                ok = runtest(test, generate, True, quiet, test_times, testdir,
                             huntrleaks, debug)
            except KeyboardInterrupt:
                # print a newline separate from the ^C
                print()
                break
            except:
                raise

    if single:
        alltests = findtests(testdir, stdtests, nottests)
        for i in range(len(alltests)):
            if tests[0] == alltests[i]:
                if i == len(alltests) - 1:
                    os.unlink(filename)
                else:
                    fp = open(filename, 'w')
                    fp.write(alltests[i+1] + '\n')
                    fp.close()
                break
        else:
            os.unlink(filename)

    if trace:
        r = tracer.results()
        r.write_results(show_missing=True, summary=True, coverdir=coverdir)

    if runleaks:
        os.system("leaks %d" % os.getpid())

    sys.exit(len(bad) > 0)
Beispiel #2
0
def main(tests=None, **kwargs):
    """Execute a test suite.

    This also parses command-line options and modifies its behavior
    accordingly.

    tests -- a list of strings containing test names (optional)
    testdir -- the directory in which to look for tests (optional)

    Users other than the Python test suite will certainly want to
    specify testdir; if it's omitted, the directory containing the
    Python test suite is searched for.

    If the tests argument is omitted, the tests listed on the
    command-line will be used.  If that's empty, too, then all *.py
    files beginning with test_ will be used.

    The other default arguments (verbose, quiet, exclude,
    single, randomize, findleaks, use_resources, trace, coverdir,
    print_slow, and random_seed) allow programmers calling main()
    directly to set the values that would normally be set by flags
    on the command line.
    """
    # Display the Python traceback on fatal errors (e.g. segfault)
    faulthandler.enable(all_threads=True)

    # Display the Python traceback on SIGALRM or SIGUSR1 signal
    signals = []
    if hasattr(signal, 'SIGALRM'):
        signals.append(signal.SIGALRM)
    if hasattr(signal, 'SIGUSR1'):
        signals.append(signal.SIGUSR1)
    for signum in signals:
        faulthandler.register(signum, chain=True)

    replace_stdout()

    support.record_original_stdout(sys.stdout)

    ns = _parse_args(sys.argv[1:], **kwargs)

    if ns.huntrleaks:
        # Avoid false positives due to various caches
        # filling slowly with random data:
        warm_caches()
    if ns.memlimit is not None:
        support.set_memlimit(ns.memlimit)
    if ns.threshold is not None:
        import gc
        gc.set_threshold(ns.threshold)
    if ns.nowindows:
        import msvcrt
        msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS|
                            msvcrt.SEM_NOALIGNMENTFAULTEXCEPT|
                            msvcrt.SEM_NOGPFAULTERRORBOX|
                            msvcrt.SEM_NOOPENFILEERRORBOX)
        try:
            msvcrt.CrtSetReportMode
        except AttributeError:
            # release build
            pass
        else:
            for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
                msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
                msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)
    if ns.wait:
        input("Press any key to continue...")

    if ns.slaveargs is not None:
        args, kwargs = json.loads(ns.slaveargs)
        if kwargs.get('huntrleaks'):
            unittest.BaseTestSuite._cleanup = False
        try:
            result = runtest(*args, **kwargs)
        except KeyboardInterrupt:
            result = INTERRUPTED, ''
        except BaseException as e:
            traceback.print_exc()
            result = CHILD_ERROR, str(e)
        sys.stdout.flush()
        print()   # Force a newline (just in case)
        print(json.dumps(result))
        sys.exit(0)

    good = []
    bad = []
    skipped = []
    resource_denieds = []
    environment_changed = []
    interrupted = False

    if ns.findleaks:
        try:
            import gc
        except ImportError:
            print('No GC available, disabling findleaks.')
            ns.findleaks = False
        else:
            # Uncomment the line below to report garbage that is not
            # freeable by reference counting alone.  By default only
            # garbage that is not collectable by the GC is reported.
            #gc.set_debug(gc.DEBUG_SAVEALL)
            found_garbage = []

    if ns.huntrleaks:
        unittest.BaseTestSuite._cleanup = False

    if ns.single:
        filename = os.path.join(TEMPDIR, 'pynexttest')
        try:
            with open(filename, 'r') as fp:
                next_test = fp.read().strip()
                tests = [next_test]
        except OSError:
            pass

    if ns.fromfile:
        tests = []
        with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp:
            count_pat = re.compile(r'\[\s*\d+/\s*\d+\]')
            for line in fp:
                line = count_pat.sub('', line)
                guts = line.split() # assuming no test has whitespace in its name
                if guts and not guts[0].startswith('#'):
                    tests.extend(guts)

    # Strip .py extensions.
    removepy(ns.args)
    removepy(tests)

    stdtests = STDTESTS[:]
    nottests = NOTTESTS.copy()
    if ns.exclude:
        for arg in ns.args:
            if arg in stdtests:
                stdtests.remove(arg)
            nottests.add(arg)
        ns.args = []

    # For a partial run, we do not need to clutter the output.
    if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args):
        # Print basic platform information
        print("==", platform.python_implementation(), *sys.version.split())
        print("==  ", platform.platform(aliased=True),
                      "%s-endian" % sys.byteorder)
        print("==  ", "hash algorithm:", sys.hash_info.algorithm,
              "64bit" if sys.maxsize > 2**32 else "32bit")
        print("==  ", os.getcwd())
        print("Testing with flags:", sys.flags)

    # if testdir is set, then we are not running the python tests suite, so
    # don't add default tests to be executed or skipped (pass empty values)
    if ns.testdir:
        alltests = findtests(ns.testdir, list(), set())
    else:
        alltests = findtests(ns.testdir, stdtests, nottests)

    selected = tests or ns.args or alltests
    if ns.single:
        selected = selected[:1]
        try:
            next_single_test = alltests[alltests.index(selected[0])+1]
        except IndexError:
            next_single_test = None
    # Remove all the selected tests that precede start if it's set.
    if ns.start:
        try:
            del selected[:selected.index(ns.start)]
        except ValueError:
            print("Couldn't find starting test (%s), using all tests" % ns.start)
    if ns.randomize:
        if ns.random_seed is None:
            ns.random_seed = random.randrange(10000000)
        random.seed(ns.random_seed)
        print("Using random seed", ns.random_seed)
        random.shuffle(selected)
    if ns.trace:
        import trace, tempfile
        tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,
                                         tempfile.gettempdir()],
                             trace=False, count=True)

    test_times = []
    support.verbose = ns.verbose      # Tell tests to be moderately quiet
    support.use_resources = ns.use_resources
    save_modules = sys.modules.keys()

    def accumulate_result(test, result):
        ok, test_time = result
        test_times.append((test_time, test))
        if ok == PASSED:
            good.append(test)
        elif ok == FAILED:
            bad.append(test)
        elif ok == ENV_CHANGED:
            environment_changed.append(test)
        elif ok == SKIPPED:
            skipped.append(test)
        elif ok == RESOURCE_DENIED:
            skipped.append(test)
            resource_denieds.append(test)

    if ns.forever:
        def test_forever(tests=list(selected)):
            while True:
                for test in tests:
                    yield test
                    if bad:
                        return
        tests = test_forever()
        test_count = ''
        test_count_width = 3
    else:
        tests = iter(selected)
        test_count = '/{}'.format(len(selected))
        test_count_width = len(test_count) - 1

    if ns.use_mp:
        try:
            from threading import Thread
        except ImportError:
            print("Multiprocess option requires thread support")
            sys.exit(2)
        from queue import Queue
        debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$")
        output = Queue()
        pending = MultiprocessTests(tests)
        def work():
            # A worker thread.
            try:
                while True:
                    try:
                        test = next(pending)
                    except StopIteration:
                        output.put((None, None, None, None))
                        return
                    retcode, stdout, stderr = run_test_in_subprocess(test, ns)
                    # Strip last refcount output line if it exists, since it
                    # comes from the shutdown of the interpreter in the subcommand.
                    stderr = debug_output_pat.sub("", stderr)
                    stdout, _, result = stdout.strip().rpartition("\n")
                    if retcode != 0:
                        result = (CHILD_ERROR, "Exit code %s" % retcode)
                        output.put((test, stdout.rstrip(), stderr.rstrip(), result))
                        return
                    if not result:
                        output.put((None, None, None, None))
                        return
                    result = json.loads(result)
                    output.put((test, stdout.rstrip(), stderr.rstrip(), result))
            except BaseException:
                output.put((None, None, None, None))
                raise
        workers = [Thread(target=work) for i in range(ns.use_mp)]
        for worker in workers:
            worker.start()
        finished = 0
        test_index = 1
        try:
            while finished < ns.use_mp:
                test, stdout, stderr, result = output.get()
                if test is None:
                    finished += 1
                    continue
                accumulate_result(test, result)
                if not ns.quiet:
                    fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
                    print(fmt.format(
                        test_count_width, test_index, test_count,
                        len(bad), test))
                if stdout:
                    print(stdout)
                if stderr:
                    print(stderr, file=sys.stderr)
                sys.stdout.flush()
                sys.stderr.flush()
                if result[0] == INTERRUPTED:
                    raise KeyboardInterrupt
                if result[0] == CHILD_ERROR:
                    raise Exception("Child error on {}: {}".format(test, result[1]))
                test_index += 1
        except KeyboardInterrupt:
            interrupted = True
            pending.interrupted = True
        for worker in workers:
            worker.join()
    else:
        for test_index, test in enumerate(tests, 1):
            if not ns.quiet:
                fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}"
                print(fmt.format(
                    test_count_width, test_index, test_count, len(bad), test))
                sys.stdout.flush()
            if ns.trace:
                # If we're tracing code coverage, then we don't exit with status
                # if on a false return value from main.
                tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)',
                              globals=globals(), locals=vars())
            else:
                try:
                    result = runtest(test, ns.verbose, ns.quiet,
                                     ns.huntrleaks,
                                     output_on_failure=ns.verbose3,
                                     timeout=ns.timeout, failfast=ns.failfast,
                                     match_tests=ns.match_tests)
                    accumulate_result(test, result)
                except KeyboardInterrupt:
                    interrupted = True
                    break
            if ns.findleaks:
                gc.collect()
                if gc.garbage:
                    print("Warning: test created", len(gc.garbage), end=' ')
                    print("uncollectable object(s).")
                    # move the uncollectable objects somewhere so we don't see
                    # them again
                    found_garbage.extend(gc.garbage)
                    del gc.garbage[:]
            # Unload the newly imported modules (best effort finalization)
            for module in sys.modules.keys():
                if module not in save_modules and module.startswith("test."):
                    support.unload(module)

    if interrupted:
        # print a newline after ^C
        print()
        print("Test suite interrupted by signal SIGINT.")
        omitted = set(selected) - set(good) - set(bad) - set(skipped)
        print(count(len(omitted), "test"), "omitted:")
        printlist(omitted)
    if good and not ns.quiet:
        if not bad and not skipped and not interrupted and len(good) > 1:
            print("All", end=' ')
        print(count(len(good), "test"), "OK.")
    if ns.print_slow:
        test_times.sort(reverse=True)
        print("10 slowest tests:")
        for time, test in test_times[:10]:
            print("%s: %.1fs" % (test, time))
    if bad:
        print(count(len(bad), "test"), "failed:")
        printlist(bad)
    if environment_changed:
        print("{} altered the execution environment:".format(
                 count(len(environment_changed), "test")))
        printlist(environment_changed)
    if skipped and not ns.quiet:
        print(count(len(skipped), "test"), "skipped:")
        printlist(skipped)

    if ns.verbose2 and bad:
        print("Re-running failed tests in verbose mode")
        for test in bad[:]:
            print("Re-running test %r in verbose mode" % test)
            sys.stdout.flush()
            try:
                ns.verbose = True
                ok = runtest(test, True, ns.quiet, ns.huntrleaks,
                             timeout=ns.timeout)
            except KeyboardInterrupt:
                # print a newline separate from the ^C
                print()
                break
            else:
                if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}:
                    bad.remove(test)
        else:
            if bad:
                print(count(len(bad), 'test'), "failed again:")
                printlist(bad)

    if ns.single:
        if next_single_test:
            with open(filename, 'w') as fp:
                fp.write(next_single_test + '\n')
        else:
            os.unlink(filename)

    if ns.trace:
        r = tracer.results()
        r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir)

    if ns.runleaks:
        os.system("leaks %d" % os.getpid())

    sys.exit(len(bad) > 0 or interrupted)
Beispiel #3
0
def setup_tests(ns):
    # Display the Python traceback on fatal errors (e.g. segfault)
    faulthandler.enable(all_threads=True)

    # Display the Python traceback on SIGALRM or SIGUSR1 signal
    signals = []
    if hasattr(signal, 'SIGALRM'):
        signals.append(signal.SIGALRM)
    if hasattr(signal, 'SIGUSR1'):
        signals.append(signal.SIGUSR1)
    for signum in signals:
        faulthandler.register(signum, chain=True)

    replace_stdout()
    support.record_original_stdout(sys.stdout)

    # Some times __path__ and __file__ are not absolute (e.g. while running from
    # Lib/) and, if we change the CWD to run the tests in a temporary dir, some
    # imports might fail.  This affects only the modules imported before os.chdir().
    # These modules are searched first in sys.path[0] (so '' -- the CWD) and if
    # they are found in the CWD their __file__ and __path__ will be relative (this
    # happens before the chdir).  All the modules imported after the chdir, are
    # not found in the CWD, and since the other paths in sys.path[1:] are absolute
    # (site.py absolutize them), the __file__ and __path__ will be absolute too.
    # Therefore it is necessary to absolutize manually the __file__ and __path__ of
    # the packages to prevent later imports to fail when the CWD is different.
    for module in sys.modules.values():
        if hasattr(module, '__path__'):
            module.__path__ = [os.path.abspath(path)
                               for path in module.__path__]
        if hasattr(module, '__file__'):
            module.__file__ = os.path.abspath(module.__file__)

    # MacOSX (a.k.a. Darwin) has a default stack size that is too small
    # for deeply recursive regular expressions.  We see this as crashes in
    # the Python test suite when running test_re.py and test_sre.py.  The
    # fix is to set the stack limit to 2048.
    # This approach may also be useful for other Unixy platforms that
    # suffer from small default stack limits.
    if sys.platform == 'darwin':
        try:
            import resource
        except ImportError:
            pass
        else:
            soft, hard = resource.getrlimit(resource.RLIMIT_STACK)
            newsoft = min(hard, max(soft, 1024*2048))
            resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard))

    if ns.huntrleaks:
        unittest.BaseTestSuite._cleanup = False

        # Avoid false positives due to various caches
        # filling slowly with random data:
        warm_caches()

    if ns.memlimit is not None:
        support.set_memlimit(ns.memlimit)

    if ns.threshold is not None:
        gc.set_threshold(ns.threshold)

    if ns.nowindows:
        import msvcrt
        msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS|
                            msvcrt.SEM_NOALIGNMENTFAULTEXCEPT|
                            msvcrt.SEM_NOGPFAULTERRORBOX|
                            msvcrt.SEM_NOOPENFILEERRORBOX)
        try:
            msvcrt.CrtSetReportMode
        except AttributeError:
            # release build
            pass
        else:
            for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]:
                msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE)
                msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR)

    support.use_resources = ns.use_resources
Beispiel #4
0
def main(tests=None, testdir=None, verbose=0, quiet=False,
         exclude=False, single=False, randomize=False, fromfile=None,
         findleaks=False, use_resources=None, trace=False, coverdir='coverage',
         runleaks=False, huntrleaks=False, verbose2=False, print_slow=False,
         expected=False, memo=None, junit_xml=None):
    """Execute a test suite.

    This also parses command-line options and modifies its behavior
    accordingly.

    tests -- a list of strings containing test names (optional)
    testdir -- the directory in which to look for tests (optional)

    Users other than the Python test suite will certainly want to
    specify testdir; if it's omitted, the directory containing the
    Python test suite is searched for.

    If the tests argument is omitted, the tests listed on the
    command-line will be used.  If that's empty, too, then all *.py
    files beginning with test_ will be used.

    The other default arguments (verbose, quiet, exclude,
    single, randomize, findleaks, use_resources, trace, coverdir, and
    print_slow) allow programmers calling main() directly to set the
    values that would normally be set by flags on the command line.
    """

    support.record_original_stdout(sys.stdout)
    try:
        opts, args = getopt.getopt(sys.argv[1:], 'hvqxsSrf:lu:t:TD:NLR:wM:em:j:',
                                   ['help', 'verbose', 'quiet', 'exclude',
                                    'single', 'slow', 'random', 'fromfile',
                                    'findleaks', 'use=', 'threshold=', 'trace',
                                    'coverdir=', 'nocoverdir', 'runleaks',
                                    'huntrleaks=', 'verbose2', 'memlimit=',
                                    'expected', 'memo'
                                    ])
    except getopt.error as msg:
        usage(2, msg)

    # Defaults
    allran = True
    if use_resources is None:
        use_resources = []
    for o, a in opts:
        if o in ('-h', '--help'):
            usage(0)
        elif o in ('-v', '--verbose'):
            verbose += 1
        elif o in ('-w', '--verbose2'):
            verbose2 = True
        elif o in ('-q', '--quiet'):
            quiet = True;
            verbose = 0
        elif o in ('-x', '--exclude'):
            exclude = True
            allran = False
        elif o in ('-e', '--expected'):
            expected = True
            allran = False
        elif o in ('-s', '--single'):
            single = True
        elif o in ('-S', '--slow'):
            print_slow = True
        elif o in ('-r', '--randomize'):
            randomize = True
        elif o in ('-f', '--fromfile'):
            fromfile = a
        elif o in ('-l', '--findleaks'):
            findleaks = True
        elif o in ('-L', '--runleaks'):
            runleaks = True
        elif o in ('-m', '--memo'):
            memo = a
        elif o in ('-j', '--junit-xml'):
            junit_xml = a
        elif o in ('-t', '--threshold'):
            import gc
            gc.set_threshold(int(a))
        elif o in ('-T', '--coverage'):
            trace = True
        elif o in ('-D', '--coverdir'):
            coverdir = os.path.join(os.getcwd(), a)
        elif o in ('-N', '--nocoverdir'):
            coverdir = None
        elif o in ('-R', '--huntrleaks'):
            huntrleaks = a.split(':')
            if len(huntrleaks) != 3:
                print(a, huntrleaks)
                usage(2, '-R takes three colon-separated arguments')
            if len(huntrleaks[0]) == 0:
                huntrleaks[0] = 5
            else:
                huntrleaks[0] = int(huntrleaks[0])
            if len(huntrleaks[1]) == 0:
                huntrleaks[1] = 4
            else:
                huntrleaks[1] = int(huntrleaks[1])
            if len(huntrleaks[2]) == 0:
                huntrleaks[2] = "reflog.txt"
        elif o in ('-M', '--memlimit'):
            support.set_memlimit(a)
        elif o in ('-u', '--use'):
            u = [x.lower() for x in a.split(',')]
            for r in u:
                if r == 'all':
                    use_resources[:] = RESOURCE_NAMES
                    continue
                remove = False
                if r[0] == '-':
                    remove = True
                    r = r[1:]
                if r not in RESOURCE_NAMES:
                    usage(1, 'Invalid -u/--use option: ' + a)
                if remove:
                    if r in use_resources:
                        use_resources.remove(r)
                elif r not in use_resources:
                    use_resources.append(r)
        else:
            print(("No handler for option {0}.  Please "
                "report this as a bug at http://bugs.python.org.").format(o), file=sys.stderr)
            sys.exit(1)
    if single and fromfile:
        usage(2, "-s and -f don't go together!")

    good = []
    bad = []
    skipped = []
    resource_denieds = []

    if findleaks:
        try:
            if support.is_jython:
                raise ImportError()
            import gc
        except ImportError:
            print('No GC available, disabling findleaks.')
            findleaks = False
        else:
            # Uncomment the line below to report garbage that is not
            # freeable by reference counting alone.  By default only
            # garbage that is not collectable by the GC is reported.
            #gc.set_debug(gc.DEBUG_SAVEALL)
            found_garbage = []

    if single:
        from tempfile import gettempdir
        filename = os.path.join(gettempdir(), 'pynexttest')
        try:
            fp = open(filename, 'r')
            next = fp.read().strip()
            tests = [next]
            fp.close()
        except IOError:
            pass

    if fromfile:
        tests = []
        fp = open(fromfile)
        for line in fp:
            guts = line.split() # assuming no test has whitespace in its name
            if guts and not guts[0].startswith('#'):
                tests.extend(guts)
        fp.close()

    # Strip .py extensions.
    if args:
        args = list(map(removepy, args))
        allran = False
    if tests:
        tests = list(map(removepy, tests))

    stdtests = STDTESTS[:]
    nottests = list(NOTTESTS)
    if exclude:
        for arg in args:
            if arg in stdtests:
                stdtests.remove(arg)
        nottests[:0] = args
        args = []
    tests = tests or args or findtests(testdir, stdtests, nottests)
    if single:
        tests = tests[:1]
    if randomize:
        random.shuffle(tests)
    if trace:
        import trace
        tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix],
                             trace=False, count=True)

    test_times = []
    support.verbose = verbose      # Tell tests to be moderately quiet
    support.use_resources = use_resources
    support.junit_xml_dir = junit_xml
    save_modules = list(sys.modules.keys())

    skips = _ExpectedSkips()
    failures = _ExpectedFailures()

    if expected:
        # Suppress expected failure from the list of tests.
        for t in failures.getexpected():
            if t in tests:
                tests.remove(t)
        # Suppress expected skips from the list of tests.
        for t in skips.getexpected():
            if t in tests:
                tests.remove(t)

    # Prevent reporting unexpected success in things we failed to try
    failures.keep_only(tests)
    skips.keep_only(tests)

    for test in tests:
        if not quiet:
            print(test)
            sys.stdout.flush()
        if trace:
            # If we're tracing code coverage, then we don't exit with status
            # if on a false return value from main.
            tracer.runctx('runtest(test, verbose, quiet,'
                          '        test_times, testdir)',
                          globals=globals(), locals=vars())
        else:
            try:
                ok = runtest(test, verbose, quiet, test_times,
                             testdir, huntrleaks, junit_xml)
            except KeyboardInterrupt:
                # print a newline separate from the ^C
                print()
                break
            except:
                raise
            if ok > 0:
                good.append(test)
            elif ok == 0:
                bad.append(test)
            else:
                skipped.append(test)
                if ok == -2:
                    resource_denieds.append(test)
        if findleaks:
            gc.collect()
            if gc.garbage:
                print("Warning: test created", len(gc.garbage), end=' ')
                print("uncollectable object(s).")
                # move the uncollectable objects somewhere so we don't see
                # them again
                found_garbage.extend(gc.garbage)
                del gc.garbage[:]
        # Unload the newly imported modules (best effort finalization)
        for module in list(sys.modules.keys()):
            if module not in save_modules and module.startswith("test."):
                support.unload(module)
                module = module[5:]
                if hasattr(_test, module):
                    delattr(_test, module)

    if good and not quiet:
        if not bad and not skipped and len(good) > 1:
            print("All", end=' ')
        print(count(len(good), "test"), "OK.")
    if print_slow:
        test_times.sort(reverse=True)
        print("10 slowest tests:")
        for time, test in test_times[:10]:
            print("%s: %.1fs" % (test, time))
    surprises = 0
    if skipped and not quiet:
        print(count(len(skipped), "test"), "skipped:")
        surprises += countsurprises(skips, skipped, 'skip', 'ran', allran, resource_denieds)
    if bad:
         print(count(len(bad), "test"), "failed:")
         surprises += countsurprises(failures, bad, 'fail', 'passed', allran, resource_denieds)

    if verbose2 and bad:
        print("Re-running failed tests in verbose mode")
        for test in bad:
            print("Re-running test %r in verbose mode" % test)
            sys.stdout.flush()
            try:
                support.verbose = True
                ok = runtest(test, True, quiet, test_times, testdir,
                             huntrleaks)
            except KeyboardInterrupt:
                # print a newline separate from the ^C
                print()
                break
            except:
                raise

    if single:
        alltests = findtests(testdir, stdtests, nottests)
        for i in range(len(alltests)):
            if tests[0] == alltests[i]:
                if i == len(alltests) - 1:
                    os.unlink(filename)
                else:
                    fp = open(filename, 'w')
                    fp.write(alltests[i+1] + '\n')
                    fp.close()
                break
        else:
            os.unlink(filename)

    if trace:
        r = tracer.results()
        r.write_results(show_missing=True, summary=True, coverdir=coverdir)

    if runleaks:
        os.system("leaks %d" % os.getpid())

    if memo:
        savememo(memo,good,bad,skipped)

    sys.exit(surprises > 0)