def setup_tests(ns): try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): # Catch ValueError to catch io.UnsupportedOperation on TextIOBase # and ValueError on a closed stream. # # Catch AttributeError for stderr being None. stderr_fd = None else: # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True, file=stderr_fd) # Display the Python traceback on SIGALRM or SIGUSR1 signal signals = [] if hasattr(signal, 'SIGALRM'): signals.append(signal.SIGALRM) if hasattr(signal, 'SIGUSR1'): signals.append(signal.SIGUSR1) for signum in signals: faulthandler.register(signum, chain=True, file=stderr_fd) replace_stdout() support.record_original_stdout(sys.stdout) if ns.testdir: # Prepend test directory to sys.path, so runtest() will be able # to locate tests sys.path.insert(0, os.path.abspath(ns.testdir)) # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). # These modules are searched first in sys.path[0] (so '' -- the CWD) and if # they are found in the CWD their __file__ and __path__ will be relative (this # happens before the chdir). All the modules imported after the chdir, are # not found in the CWD, and since the other paths in sys.path[1:] are absolute # (site.py absolutize them), the __file__ and __path__ will be absolute too. # Therefore it is necessary to absolutize manually the __file__ and __path__ of # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) if getattr(module, '__file__', None): module.__file__ = os.path.abspath(module.__file__) if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False if ns.memlimit is not None: support.set_memlimit(ns.memlimit) if ns.threshold is not None: gc.set_threshold(ns.threshold) suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2) support.use_resources = ns.use_resources
def setup_tests(ns): faulthandler.enable(all_threads=True) signals = [] if hasattr(signal, 'SIGALRM'): signals.append(signal.SIGALRM) if hasattr(signal, 'SIGUSR1'): signals.append(signal.SIGUSR1) for signum in signals: faulthandler.register(signum, chain=True) replace_stdout() support.record_original_stdout(sys.stdout) if ns.testdir: sys.path.insert(0, os.path.abspath(ns.testdir)) for module in sys.modules.values(): if hasattr(module, '__path__'): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) if hasattr(module, '__file__'): module.__file__ = os.path.abspath(module.__file__) if sys.platform == 'darwin': try: import resource except ImportError: pass else: soft, hard = resource.getrlimit(resource.RLIMIT_STACK) newsoft = min(hard, max(soft, 1024 * 2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False warm_caches() if ns.memlimit is not None: support.set_memlimit(ns.memlimit) if ns.threshold is not None: gc.set_threshold(ns.threshold) try: import msvcrt except ImportError: pass else: msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS | msvcrt.SEM_NOALIGNMENTFAULTEXCEPT | msvcrt.SEM_NOGPFAULTERRORBOX | msvcrt.SEM_NOOPENFILEERRORBOX) try: msvcrt.CrtSetReportMode except AttributeError: pass else: for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: if ns.verbose and ns.verbose >= 2: msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) else: msvcrt.CrtSetReportMode(m, 0) support.use_resources = ns.use_resources
self.assertEqual(len(l), size) self.assertEqual(l[-2:], [10, 10]) @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_reverse(self, size): l = [1, 2, 3, 4, 5] * size l.reverse() self.assertEqual(len(l), size * 5) self.assertEqual(l[-5:], [5, 4, 3, 2, 1]) self.assertEqual(l[:5], [5, 4, 3, 2, 1]) @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_sort(self, size): l = [1, 2, 3, 4, 5] * size l.sort() self.assertEqual(len(l), size * 5) self.assertEqual(l.count(1), size) self.assertEqual(l[:10], [1] * 10) self.assertEqual(l[-10:], [5] * 10) def test_main(): support.run_unittest(StrTest, BytesTest, BytearrayTest, TupleTest, ListTest) if __name__ == '__main__': if len(sys.argv) > 1: support.set_memlimit(sys.argv[1]) test_main()
l.remove(5) size -= 1 self.assertEquals(len(l), size) self.assertEquals(l[-2:], [10, 10]) @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5) def test_reverse(self, size): l = [1, 2, 3, 4, 5] * size l.reverse() self.assertEquals(len(l), size * 5) self.assertEquals(l[-5:], [5, 4, 3, 2, 1]) self.assertEquals(l[:5], [5, 4, 3, 2, 1]) @bigmemtest(minsize=_2G // 5 + 2, memuse=8 * 5) def test_sort(self, size): l = [1, 2, 3, 4, 5] * size l.sort() self.assertEquals(len(l), size * 5) self.assertEquals(l.count(1), size) self.assertEquals(l[:10], [1] * 10) self.assertEquals(l[-10:], [5] * 10) def test_main(): support.run_unittest(StrTest, BytesTest, BytearrayTest, TupleTest, ListTest) if __name__ == '__main__': if len(sys.argv) > 1: support.set_memlimit(sys.argv[1]) test_main()
def main(tests=None, testdir=None, verbose=0, quiet=False, exclude=False, single=False, randomize=False, fromfile=None, findleaks=False, use_resources=None, trace=False, coverdir='coverage', runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, expected=False, memo=None, junit_xml=None): """Execute a test suite. This also parses command-line options and modifies its behavior accordingly. tests -- a list of strings containing test names (optional) testdir -- the directory in which to look for tests (optional) Users other than the Python test suite will certainly want to specify testdir; if it's omitted, the directory containing the Python test suite is searched for. If the tests argument is omitted, the tests listed on the command-line will be used. If that's empty, too, then all *.py files beginning with test_ will be used. The other default arguments (verbose, quiet, exclude, single, randomize, findleaks, use_resources, trace, coverdir, and print_slow) allow programmers calling main() directly to set the values that would normally be set by flags on the command line. """ support.record_original_stdout(sys.stdout) try: opts, args = getopt.getopt( sys.argv[1:], 'hvqxsSrf:lu:t:TD:NLR:wM:em:j:', [ 'help', 'verbose', 'quiet', 'exclude', 'single', 'slow', 'random', 'fromfile', 'findleaks', 'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir', 'runleaks', 'huntrleaks=', 'verbose2', 'memlimit=', 'expected', 'memo' ]) except getopt.error as msg: usage(2, msg) # Defaults allran = True if use_resources is None: use_resources = [] for o, a in opts: if o in ('-h', '--help'): usage(0) elif o in ('-v', '--verbose'): verbose += 1 elif o in ('-w', '--verbose2'): verbose2 = True elif o in ('-q', '--quiet'): quiet = True verbose = 0 elif o in ('-x', '--exclude'): exclude = True allran = False elif o in ('-e', '--expected'): expected = True allran = False elif o in ('-s', '--single'): single = True elif o in ('-S', '--slow'): print_slow = True elif o in ('-r', '--randomize'): randomize = True elif o in ('-f', '--fromfile'): fromfile = a elif o in ('-l', '--findleaks'): findleaks = True elif o in ('-L', '--runleaks'): runleaks = True elif o in ('-m', '--memo'): memo = a elif o in ('-j', '--junit-xml'): junit_xml = a elif o in ('-t', '--threshold'): import gc gc.set_threshold(int(a)) elif o in ('-T', '--coverage'): trace = True elif o in ('-D', '--coverdir'): coverdir = os.path.join(os.getcwd(), a) elif o in ('-N', '--nocoverdir'): coverdir = None elif o in ('-R', '--huntrleaks'): huntrleaks = a.split(':') if len(huntrleaks) != 3: print(a, huntrleaks) usage(2, '-R takes three colon-separated arguments') if len(huntrleaks[0]) == 0: huntrleaks[0] = 5 else: huntrleaks[0] = int(huntrleaks[0]) if len(huntrleaks[1]) == 0: huntrleaks[1] = 4 else: huntrleaks[1] = int(huntrleaks[1]) if len(huntrleaks[2]) == 0: huntrleaks[2] = "reflog.txt" elif o in ('-M', '--memlimit'): support.set_memlimit(a) elif o in ('-u', '--use'): u = [x.lower() for x in a.split(',')] for r in u: if r == 'all': use_resources[:] = RESOURCE_NAMES continue remove = False if r[0] == '-': remove = True r = r[1:] if r not in RESOURCE_NAMES: usage(1, 'Invalid -u/--use option: ' + a) if remove: if r in use_resources: use_resources.remove(r) elif r not in use_resources: use_resources.append(r) else: print( ("No handler for option {0}. Please " "report this as a bug at http://bugs.python.org.").format(o), file=sys.stderr) sys.exit(1) if single and fromfile: usage(2, "-s and -f don't go together!") good = [] bad = [] skipped = [] resource_denieds = [] if findleaks: try: if support.is_jython: raise ImportError() import gc except ImportError: print('No GC available, disabling findleaks.') findleaks = False else: # Uncomment the line below to report garbage that is not # freeable by reference counting alone. By default only # garbage that is not collectable by the GC is reported. #gc.set_debug(gc.DEBUG_SAVEALL) found_garbage = [] if single: from tempfile import gettempdir filename = os.path.join(gettempdir(), 'pynexttest') try: fp = open(filename, 'r') next = fp.read().strip() tests = [next] fp.close() except IOError: pass if fromfile: tests = [] fp = open(fromfile) for line in fp: guts = line.split() # assuming no test has whitespace in its name if guts and not guts[0].startswith('#'): tests.extend(guts) fp.close() # Strip .py extensions. if args: args = list(map(removepy, args)) allran = False if tests: tests = list(map(removepy, tests)) stdtests = STDTESTS[:] nottests = list(NOTTESTS) if exclude: for arg in args: if arg in stdtests: stdtests.remove(arg) nottests[:0] = args args = [] tests = tests or args or findtests(testdir, stdtests, nottests) if single: tests = tests[:1] if randomize: random.shuffle(tests) if trace: import trace tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix], trace=False, count=True) test_times = [] support.verbose = verbose # Tell tests to be moderately quiet support.use_resources = use_resources support.junit_xml_dir = junit_xml save_modules = list(sys.modules.keys()) skips = _ExpectedSkips() failures = _ExpectedFailures() if expected: # Suppress expected failure from the list of tests. for t in failures.getexpected(): if t in tests: tests.remove(t) # Suppress expected skips from the list of tests. for t in skips.getexpected(): if t in tests: tests.remove(t) # Prevent reporting unexpected success in things we failed to try failures.keep_only(tests) skips.keep_only(tests) for test in tests: if not quiet: print(test) sys.stdout.flush() if trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. tracer.runctx( 'runtest(test, verbose, quiet,' ' test_times, testdir)', globals=globals(), locals=vars()) else: try: ok = runtest(test, verbose, quiet, test_times, testdir, huntrleaks, junit_xml) except KeyboardInterrupt: # print a newline separate from the ^C print() break except: raise if ok > 0: good.append(test) elif ok == 0: bad.append(test) else: skipped.append(test) if ok == -2: resource_denieds.append(test) if findleaks: gc.collect() if gc.garbage: print("Warning: test created", len(gc.garbage), end=' ') print("uncollectable object(s).") # move the uncollectable objects somewhere so we don't see # them again found_garbage.extend(gc.garbage) del gc.garbage[:] # Unload the newly imported modules (best effort finalization) for module in list(sys.modules.keys()): if module not in save_modules and module.startswith("test."): support.unload(module) module = module[5:] if hasattr(_test, module): delattr(_test, module) if good and not quiet: if not bad and not skipped and len(good) > 1: print("All", end=' ') print(count(len(good), "test"), "OK.") if print_slow: test_times.sort(reverse=True) print("10 slowest tests:") for time, test in test_times[:10]: print("%s: %.1fs" % (test, time)) surprises = 0 if skipped and not quiet: print(count(len(skipped), "test"), "skipped:") surprises += countsurprises(skips, skipped, 'skip', 'ran', allran, resource_denieds) if bad: print(count(len(bad), "test"), "failed:") surprises += countsurprises(failures, bad, 'fail', 'passed', allran, resource_denieds) if verbose2 and bad: print("Re-running failed tests in verbose mode") for test in bad: print("Re-running test %r in verbose mode" % test) sys.stdout.flush() try: support.verbose = True ok = runtest(test, True, quiet, test_times, testdir, huntrleaks) except KeyboardInterrupt: # print a newline separate from the ^C print() break except: raise if single: alltests = findtests(testdir, stdtests, nottests) for i in range(len(alltests)): if tests[0] == alltests[i]: if i == len(alltests) - 1: os.unlink(filename) else: fp = open(filename, 'w') fp.write(alltests[i + 1] + '\n') fp.close() break else: os.unlink(filename) if trace: r = tracer.results() r.write_results(show_missing=True, summary=True, coverdir=coverdir) if runleaks: os.system("leaks %d" % os.getpid()) if memo: savememo(memo, good, bad, skipped) sys.exit(surprises > 0)
def setup_tests(ns): try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): # Catch ValueError to catch io.UnsupportedOperation on TextIOBase # and ValueError on a closed stream. # # Catch AttributeError for stderr being None. stderr_fd = None else: # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True, file=stderr_fd) # Display the Python traceback on SIGALRM or SIGUSR1 signal signals = [] if hasattr(signal, 'SIGALRM'): signals.append(signal.SIGALRM) if hasattr(signal, 'SIGUSR1'): signals.append(signal.SIGUSR1) for signum in signals: faulthandler.register(signum, chain=True, file=stderr_fd) _adjust_resource_limits() replace_stdout() support.record_original_stdout(sys.stdout) if ns.testdir: # Prepend test directory to sys.path, so runtest() will be able # to locate tests sys.path.insert(0, os.path.abspath(ns.testdir)) # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). # These modules are searched first in sys.path[0] (so '' -- the CWD) and if # they are found in the CWD their __file__ and __path__ will be relative (this # happens before the chdir). All the modules imported after the chdir, are # not found in the CWD, and since the other paths in sys.path[1:] are absolute # (site.py absolutize them), the __file__ and __path__ will be absolute too. # Therefore it is necessary to absolutize manually the __file__ and __path__ of # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) if getattr(module, '__file__', None): module.__file__ = os.path.abspath(module.__file__) if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False if ns.memlimit is not None: support.set_memlimit(ns.memlimit) if ns.threshold is not None: gc.set_threshold(ns.threshold) support.suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2) support.use_resources = ns.use_resources if hasattr(sys, 'addaudithook'): # Add an auditing hook for all tests to ensure PySys_Audit is tested def _test_audit_hook(name, args): pass sys.addaudithook(_test_audit_hook) setup_unraisable_hook() setup_threading_excepthook() if ns.timeout is not None: # For a slow buildbot worker, increase SHORT_TIMEOUT and LONG_TIMEOUT support.SHORT_TIMEOUT = max(support.SHORT_TIMEOUT, ns.timeout / 40) support.LONG_TIMEOUT = max(support.LONG_TIMEOUT, ns.timeout / 4) # If --timeout is short: reduce timeouts support.LOOPBACK_TIMEOUT = min(support.LOOPBACK_TIMEOUT, ns.timeout) support.INTERNET_TIMEOUT = min(support.INTERNET_TIMEOUT, ns.timeout) support.SHORT_TIMEOUT = min(support.SHORT_TIMEOUT, ns.timeout) support.LONG_TIMEOUT = min(support.LONG_TIMEOUT, ns.timeout) if ns.xmlpath: from test.support.testresult import RegressionTestResult RegressionTestResult.USE_XML = True # Ensure there's a non-ASCII character in env vars at all times to force # tests consider this case. See BPO-44647 for details. os.environ.setdefault( UNICODE_GUARD_ENV, "\N{SMILING FACE WITH SUNGLASSES}", )
def setup_tests(ns): try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): # Catch ValueError to catch io.UnsupportedOperation on TextIOBase # and ValueError on a closed stream. # # Catch AttributeError for stderr being None. stderr_fd = None else: # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True, file=stderr_fd) # Display the Python traceback on SIGALRM or SIGUSR1 signal signals = [] if hasattr(signal, 'SIGALRM'): signals.append(signal.SIGALRM) if hasattr(signal, 'SIGUSR1'): signals.append(signal.SIGUSR1) for signum in signals: faulthandler.register(signum, chain=True, file=stderr_fd) replace_stdout() support.record_original_stdout(sys.stdout) if ns.testdir: # Prepend test directory to sys.path, so runtest() will be able # to locate tests sys.path.insert(0, os.path.abspath(ns.testdir)) # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). # These modules are searched first in sys.path[0] (so '' -- the CWD) and if # they are found in the CWD their __file__ and __path__ will be relative (this # happens before the chdir). All the modules imported after the chdir, are # not found in the CWD, and since the other paths in sys.path[1:] are absolute # (site.py absolutize them), the __file__ and __path__ will be absolute too. # Therefore it is necessary to absolutize manually the __file__ and __path__ of # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) if getattr(module, '__file__', None): module.__file__ = os.path.abspath(module.__file__) if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False # Avoid false positives due to various caches # filling slowly with random data: warm_caches() if ns.memlimit is not None: support.set_memlimit(ns.memlimit) if ns.threshold is not None: gc.set_threshold(ns.threshold) try: import msvcrt except ImportError: pass else: msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| msvcrt.SEM_NOGPFAULTERRORBOX| msvcrt.SEM_NOOPENFILEERRORBOX) try: msvcrt.CrtSetReportMode except AttributeError: # release build pass else: for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: if ns.verbose and ns.verbose >= 2: msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) else: msvcrt.CrtSetReportMode(m, 0) support.use_resources = ns.use_resources
def setup_tests(ns): try: stderr_fd = sys.__stderr__.fileno() except (ValueError, AttributeError): # Catch ValueError to catch io.UnsupportedOperation on TextIOBase # and ValueError on a closed stream. # # Catch AttributeError for stderr being None. stderr_fd = None else: # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True, file=stderr_fd) # Display the Python traceback on SIGALRM or SIGUSR1 signal signals = [] if hasattr(signal, 'SIGALRM'): signals.append(signal.SIGALRM) if hasattr(signal, 'SIGUSR1'): signals.append(signal.SIGUSR1) for signum in signals: faulthandler.register(signum, chain=True, file=stderr_fd) replace_stdout() support.record_original_stdout(sys.stdout) if ns.testdir: # Prepend test directory to sys.path, so runtest() will be able # to locate tests sys.path.insert(0, os.path.abspath(ns.testdir)) # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). # These modules are searched first in sys.path[0] (so '' -- the CWD) and if # they are found in the CWD their __file__ and __path__ will be relative (this # happens before the chdir). All the modules imported after the chdir, are # not found in the CWD, and since the other paths in sys.path[1:] are absolute # (site.py absolutize them), the __file__ and __path__ will be absolute too. # Therefore it is necessary to absolutize manually the __file__ and __path__ of # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): for index, path in enumerate(module.__path__): module.__path__[index] = os.path.abspath(path) if getattr(module, '__file__', None): module.__file__ = os.path.abspath(module.__file__) # MacOSX (a.k.a. Darwin) has a default stack size that is too small # for deeply recursive regular expressions. We see this as crashes in # the Python test suite when running test_re.py and test_sre.py. The # fix is to set the stack limit to 2048. # This approach may also be useful for other Unixy platforms that # suffer from small default stack limits. if sys.platform == 'darwin': try: import resource except ImportError: pass else: soft, hard = resource.getrlimit(resource.RLIMIT_STACK) newsoft = min(hard, max(soft, 1024 * 2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False if ns.memlimit is not None: support.set_memlimit(ns.memlimit) if ns.threshold is not None: gc.set_threshold(ns.threshold) suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2) support.use_resources = ns.use_resources if hasattr(sys, 'addaudithook'): # Add an auditing hook for all tests to ensure PySys_Audit is tested def _test_audit_hook(name, args): pass sys.addaudithook(_test_audit_hook)
def test_reverse(self, size): l = [1, 2, 3, 4, 5] * size l.reverse() self.assertEqual(len(l), size * 5) self.assertEqual(l[-5:], [5, 4, 3, 2, 1]) self.assertEqual(l[:5], [5, 4, 3, 2, 1]) @bigmemtest(size=_2G // 5 + 2, memuse=8 * 5) def test_sort(self, size): l = [1, 2, 3, 4, 5] * size l.sort() self.assertEqual(len(l), size * 5) self.assertEqual(l.count(1), size) self.assertEqual(l[:10], [1] * 10) self.assertEqual(l[-10:], [5] * 10) def test_main(): support.run_unittest(StrTest, BytesTest, BytearrayTest, TupleTest, ListTest) if __name__ == '__main__': if len(sys.argv) > 1: arg = sys.argv[1] if arg not in ('-v', '-vv'): support.set_memlimit(arg) elif len(sys.argv) > 2: support.set_memlimit(sys.argv[2]) test_main()
def main(tests=None, testdir=None, verbose=0, quiet=False, generate=False, exclude=False, single=False, randomize=False, fromfile=None, findleaks=False, use_resources=None, trace=False, coverdir='coverage', runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, random_seed=None): """Execute a test suite. This also parses command-line options and modifies its behavior accordingly. tests -- a list of strings containing test names (optional) testdir -- the directory in which to look for tests (optional) Users other than the Python test suite will certainly want to specify testdir; if it's omitted, the directory containing the Python test suite is searched for. If the tests argument is omitted, the tests listed on the command-line will be used. If that's empty, too, then all *.py files beginning with test_ will be used. The other default arguments (verbose, quiet, generate, exclude, single, randomize, findleaks, use_resources, trace, coverdir, print_slow, and random_seed) allow programmers calling main() directly to set the values that would normally be set by flags on the command line. """ support.record_original_stdout(sys.stdout) try: opts, args = getopt.getopt(sys.argv[1:], 'hvgqxsSrf:lu:t:TD:NLR:wM:n', ['help', 'verbose', 'quiet', 'exclude', 'single', 'slow', 'random', 'fromfile', 'findleaks', 'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir', 'runleaks', 'huntrleaks=', 'verbose2', 'memlimit=', 'debug', 'start=', 'nowindows', 'randseed=', ]) except getopt.error as msg: usage(msg) # Defaults if random_seed is None: random_seed = random.randrange(10000000) if use_resources is None: use_resources = [] debug = False start = None for o, a in opts: if o in ('-h', '--help'): print(__doc__) return elif o in ('-v', '--verbose'): verbose += 1 elif o in ('-w', '--verbose2'): verbose2 = True elif o in ('-d', '--debug'): debug = True elif o in ('-q', '--quiet'): quiet = True; verbose = 0 elif o in ('-x', '--exclude'): exclude = True elif o in ('-S', '--start'): start = a elif o in ('-s', '--single'): single = True elif o in ('-S', '--slow'): print_slow = True elif o in ('-r', '--randomize'): randomize = True elif o == '--randseed': random_seed = int(a) elif o in ('-f', '--fromfile'): fromfile = a elif o in ('-l', '--findleaks'): findleaks = True elif o in ('-L', '--runleaks'): runleaks = True elif o in ('-t', '--threshold'): import gc gc.set_threshold(int(a)) elif o in ('-T', '--coverage'): trace = True elif o in ('-D', '--coverdir'): coverdir = os.path.join(os.getcwd(), a) elif o in ('-N', '--nocoverdir'): coverdir = None elif o in ('-R', '--huntrleaks'): huntrleaks = a.split(':') if len(huntrleaks) not in (2, 3): print(a, huntrleaks) usage('-R takes 2 or 3 colon-separated arguments') if not huntrleaks[0]: huntrleaks[0] = 5 else: huntrleaks[0] = int(huntrleaks[0]) if not huntrleaks[1]: huntrleaks[1] = 4 else: huntrleaks[1] = int(huntrleaks[1]) if len(huntrleaks) == 2 or not huntrleaks[2]: huntrleaks[2:] = ["reflog.txt"] # Avoid false positives due to the character cache in # stringobject.c filling slowly with random data warm_char_cache() elif o in ('-M', '--memlimit'): support.set_memlimit(a) elif o in ('-u', '--use'): u = [x.lower() for x in a.split(',')] for r in u: if r == 'all': use_resources[:] = RESOURCE_NAMES continue remove = False if r[0] == '-': remove = True r = r[1:] if r not in RESOURCE_NAMES: usage('Invalid -u/--use option: ' + a) if remove: if r in use_resources: use_resources.remove(r) elif r not in use_resources: use_resources.append(r) elif o in ('-n', '--nowindows'): import msvcrt msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| msvcrt.SEM_NOGPFAULTERRORBOX| msvcrt.SEM_NOOPENFILEERRORBOX) try: msvcrt.CrtSetReportMode except AttributeError: # release build pass else: for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) if generate and verbose: usage("-g and -v don't go together!") if single and fromfile: usage("-s and -f don't go together!") good = [] bad = [] skipped = [] resource_denieds = [] if findleaks: try: import gc except ImportError: print('No GC available, disabling findleaks.') findleaks = False else: # Uncomment the line below to report garbage that is not # freeable by reference counting alone. By default only # garbage that is not collectable by the GC is reported. #gc.set_debug(gc.DEBUG_SAVEALL) found_garbage = [] if single: from tempfile import gettempdir filename = os.path.join(gettempdir(), 'pynexttest') try: fp = open(filename, 'r') next = fp.read().strip() tests = [next] fp.close() except IOError: pass if fromfile: tests = [] fp = open(fromfile) for line in fp: guts = line.split() # assuming no test has whitespace in its name if guts and not guts[0].startswith('#'): tests.extend(guts) fp.close() # Strip .py extensions. if args: args = list(map(removepy, args)) if tests: tests = list(map(removepy, tests)) stdtests = STDTESTS[:] nottests = NOTTESTS.copy() if exclude: for arg in args: if arg in stdtests: stdtests.remove(arg) nottests.add(arg) args = [] tests = tests or args or findtests(testdir, stdtests, nottests) if single: tests = tests[:1] # Remove all the tests that precede start if it's set. if start: try: del tests[:tests.index(start)] except ValueError: print("Couldn't find starting test (%s), using all tests" % start) if randomize: random.seed(random_seed) print("Using random seed", random_seed) random.shuffle(tests) if trace: import trace, tempfile tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix, tempfile.gettempdir()], trace=False, count=True) test_times = [] support.verbose = verbose # Tell tests to be moderately quiet support.use_resources = use_resources save_modules = sys.modules.keys() for test in tests: if not quiet: print(test) sys.stdout.flush() if trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. tracer.runctx('runtest(test, generate, verbose, quiet,' ' test_times, testdir)', globals=globals(), locals=vars()) else: try: ok = runtest(test, generate, verbose, quiet, test_times, testdir, huntrleaks) except KeyboardInterrupt: # print a newline separate from the ^C print() break except: raise if ok > 0: good.append(test) elif ok == 0: bad.append(test) else: skipped.append(test) if ok == -2: resource_denieds.append(test) if findleaks: gc.collect() if gc.garbage: print("Warning: test created", len(gc.garbage), end=' ') print("uncollectable object(s).") # move the uncollectable objects somewhere so we don't see # them again found_garbage.extend(gc.garbage) del gc.garbage[:] # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) # The lists won't be sorted if running with -r good.sort() bad.sort() skipped.sort() if good and not quiet: if not bad and not skipped and len(good) > 1: print("All", end=' ') print(count(len(good), "test"), "OK.") if verbose: print("CAUTION: stdout isn't compared in verbose mode:") print("a test that passes in verbose mode may fail without it.") if print_slow: test_times.sort(reverse=True) print("10 slowest tests:") for time, test in test_times[:10]: print("%s: %.1fs" % (test, time)) if bad: print(count(len(bad), "test"), "failed:") printlist(bad) if skipped and not quiet: print(count(len(skipped), "test"), "skipped:") printlist(skipped) e = _ExpectedSkips() plat = sys.platform if e.isvalid(): surprise = set(skipped) - e.getexpected() - set(resource_denieds) if surprise: print(count(len(surprise), "skip"), \ "unexpected on", plat + ":") printlist(surprise) else: print("Those skips are all expected on", plat + ".") else: print("Ask someone to teach regrtest.py about which tests are") print("expected to get skipped on", plat + ".") if verbose2 and bad: print("Re-running failed tests in verbose mode") for test in bad: print("Re-running test %r in verbose mode" % test) sys.stdout.flush() try: support.verbose = True ok = runtest(test, generate, True, quiet, test_times, testdir, huntrleaks, debug) except KeyboardInterrupt: # print a newline separate from the ^C print() break except: raise if single: alltests = findtests(testdir, stdtests, nottests) for i in range(len(alltests)): if tests[0] == alltests[i]: if i == len(alltests) - 1: os.unlink(filename) else: fp = open(filename, 'w') fp.write(alltests[i+1] + '\n') fp.close() break else: os.unlink(filename) if trace: r = tracer.results() r.write_results(show_missing=True, summary=True, coverdir=coverdir) if runleaks: os.system("leaks %d" % os.getpid()) sys.exit(len(bad) > 0)
def setup_tests(ns): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) # Display the Python traceback on SIGALRM or SIGUSR1 signal signals = [] if hasattr(signal, 'SIGALRM'): signals.append(signal.SIGALRM) if hasattr(signal, 'SIGUSR1'): signals.append(signal.SIGUSR1) for signum in signals: faulthandler.register(signum, chain=True) replace_stdout() support.record_original_stdout(sys.stdout) # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). # These modules are searched first in sys.path[0] (so '' -- the CWD) and if # they are found in the CWD their __file__ and __path__ will be relative (this # happens before the chdir). All the modules imported after the chdir, are # not found in the CWD, and since the other paths in sys.path[1:] are absolute # (site.py absolutize them), the __file__ and __path__ will be absolute too. # Therefore it is necessary to absolutize manually the __file__ and __path__ of # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): module.__path__ = [os.path.abspath(path) for path in module.__path__] if hasattr(module, '__file__'): module.__file__ = os.path.abspath(module.__file__) # MacOSX (a.k.a. Darwin) has a default stack size that is too small # for deeply recursive regular expressions. We see this as crashes in # the Python test suite when running test_re.py and test_sre.py. The # fix is to set the stack limit to 2048. # This approach may also be useful for other Unixy platforms that # suffer from small default stack limits. if sys.platform == 'darwin': try: import resource except ImportError: pass else: soft, hard = resource.getrlimit(resource.RLIMIT_STACK) newsoft = min(hard, max(soft, 1024*2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False # Avoid false positives due to various caches # filling slowly with random data: warm_caches() if ns.memlimit is not None: support.set_memlimit(ns.memlimit) if ns.threshold is not None: gc.set_threshold(ns.threshold) if ns.nowindows: import msvcrt msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| msvcrt.SEM_NOGPFAULTERRORBOX| msvcrt.SEM_NOOPENFILEERRORBOX) try: msvcrt.CrtSetReportMode except AttributeError: # release build pass else: for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) support.use_resources = ns.use_resources
TestFailed, TESTFN, run_with_locale, _2G, _4G, bigmemtest, set_memlimit ) from pickle import bytes_types # Tests that try a number of pickle protocols should have a # for proto in protocols: # kind of outer loop. protocols = [3] character_size = 4 if sys.maxunicode > 0xFFFF else 2 import platform if platform.architecture()[0] == '64bit': set_memlimit("4G") class UnseekableIO(io.BytesIO): def peek(self, *args): raise NotImplementedError def seekable(self): return False def seek(self, *args): raise io.UnsupportedOperation def tell(self): raise io.UnsupportedOperation
def setup_tests(ns): # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) # Display the Python traceback on SIGALRM or SIGUSR1 signal signals = [] if hasattr(signal, 'SIGALRM'): signals.append(signal.SIGALRM) if hasattr(signal, 'SIGUSR1'): signals.append(signal.SIGUSR1) for signum in signals: faulthandler.register(signum, chain=True) replace_stdout() support.record_original_stdout(sys.stdout) # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). # These modules are searched first in sys.path[0] (so '' -- the CWD) and if # they are found in the CWD their __file__ and __path__ will be relative (this # happens before the chdir). All the modules imported after the chdir, are # not found in the CWD, and since the other paths in sys.path[1:] are absolute # (site.py absolutize them), the __file__ and __path__ will be absolute too. # Therefore it is necessary to absolutize manually the __file__ and __path__ of # the packages to prevent later imports to fail when the CWD is different. for module in sys.modules.values(): if hasattr(module, '__path__'): module.__path__ = [os.path.abspath(path) for path in module.__path__] if hasattr(module, '__file__'): module.__file__ = os.path.abspath(module.__file__) # MacOSX (a.k.a. Darwin) has a default stack size that is too small # for deeply recursive regular expressions. We see this as crashes in # the Python test suite when running test_re.py and test_sre.py. The # fix is to set the stack limit to 2048. # This approach may also be useful for other Unixy platforms that # suffer from small default stack limits. if sys.platform == 'darwin': try: import resource except ImportError: pass else: soft, hard = resource.getrlimit(resource.RLIMIT_STACK) newsoft = min(hard, max(soft, 1024*2048)) resource.setrlimit(resource.RLIMIT_STACK, (newsoft, hard)) if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False # Avoid false positives due to various caches # filling slowly with random data: warm_caches() if ns.memlimit is not None: support.set_memlimit(ns.memlimit) if ns.threshold is not None: gc.set_threshold(ns.threshold) try: import msvcrt except ImportError: pass else: msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| msvcrt.SEM_NOGPFAULTERRORBOX| msvcrt.SEM_NOOPENFILEERRORBOX) try: msvcrt.CrtSetReportMode except AttributeError: # release build pass else: for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: if ns.verbose and ns.verbose >= 2: msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) else: msvcrt.CrtSetReportMode(m, 0) support.use_resources = ns.use_resources
def main(tests=None, **kwargs): """Execute a test suite. This also parses command-line options and modifies its behavior accordingly. tests -- a list of strings containing test names (optional) testdir -- the directory in which to look for tests (optional) Users other than the Python test suite will certainly want to specify testdir; if it's omitted, the directory containing the Python test suite is searched for. If the tests argument is omitted, the tests listed on the command-line will be used. If that's empty, too, then all *.py files beginning with test_ will be used. The other default arguments (verbose, quiet, exclude, single, randomize, findleaks, use_resources, trace, coverdir, print_slow, and random_seed) allow programmers calling main() directly to set the values that would normally be set by flags on the command line. """ # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) # Display the Python traceback on SIGALRM or SIGUSR1 signal signals = [] if hasattr(signal, 'SIGALRM'): signals.append(signal.SIGALRM) if hasattr(signal, 'SIGUSR1'): signals.append(signal.SIGUSR1) for signum in signals: faulthandler.register(signum, chain=True) replace_stdout() support.record_original_stdout(sys.stdout) ns = _parse_args(sys.argv[1:], **kwargs) if ns.huntrleaks: # Avoid false positives due to various caches # filling slowly with random data: warm_caches() if ns.memlimit is not None: support.set_memlimit(ns.memlimit) if ns.threshold is not None: import gc gc.set_threshold(ns.threshold) if ns.nowindows: import msvcrt msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| msvcrt.SEM_NOGPFAULTERRORBOX| msvcrt.SEM_NOOPENFILEERRORBOX) try: msvcrt.CrtSetReportMode except AttributeError: # release build pass else: for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) if ns.wait: input("Press any key to continue...") if ns.slaveargs is not None: args, kwargs = json.loads(ns.slaveargs) if kwargs.get('huntrleaks'): unittest.BaseTestSuite._cleanup = False try: result = runtest(*args, **kwargs) except KeyboardInterrupt: result = INTERRUPTED, '' except BaseException as e: traceback.print_exc() result = CHILD_ERROR, str(e) sys.stdout.flush() print() # Force a newline (just in case) print(json.dumps(result)) sys.exit(0) good = [] bad = [] skipped = [] resource_denieds = [] environment_changed = [] interrupted = False if ns.findleaks: try: import gc except ImportError: print('No GC available, disabling findleaks.') ns.findleaks = False else: # Uncomment the line below to report garbage that is not # freeable by reference counting alone. By default only # garbage that is not collectable by the GC is reported. #gc.set_debug(gc.DEBUG_SAVEALL) found_garbage = [] if ns.huntrleaks: unittest.BaseTestSuite._cleanup = False if ns.single: filename = os.path.join(TEMPDIR, 'pynexttest') try: with open(filename, 'r') as fp: next_test = fp.read().strip() tests = [next_test] except OSError: pass if ns.fromfile: tests = [] with open(os.path.join(support.SAVEDCWD, ns.fromfile)) as fp: count_pat = re.compile(r'\[\s*\d+/\s*\d+\]') for line in fp: line = count_pat.sub('', line) guts = line.split() # assuming no test has whitespace in its name if guts and not guts[0].startswith('#'): tests.extend(guts) # Strip .py extensions. removepy(ns.args) removepy(tests) stdtests = STDTESTS[:] nottests = NOTTESTS.copy() if ns.exclude: for arg in ns.args: if arg in stdtests: stdtests.remove(arg) nottests.add(arg) ns.args = [] # For a partial run, we do not need to clutter the output. if ns.verbose or ns.header or not (ns.quiet or ns.single or tests or ns.args): # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) print("== ", platform.platform(aliased=True), "%s-endian" % sys.byteorder) print("== ", "hash algorithm:", sys.hash_info.algorithm, "64bit" if sys.maxsize > 2**32 else "32bit") print("== ", os.getcwd()) print("Testing with flags:", sys.flags) # if testdir is set, then we are not running the python tests suite, so # don't add default tests to be executed or skipped (pass empty values) if ns.testdir: alltests = findtests(ns.testdir, list(), set()) else: alltests = findtests(ns.testdir, stdtests, nottests) selected = tests or ns.args or alltests if ns.single: selected = selected[:1] try: next_single_test = alltests[alltests.index(selected[0])+1] except IndexError: next_single_test = None # Remove all the selected tests that precede start if it's set. if ns.start: try: del selected[:selected.index(ns.start)] except ValueError: print("Couldn't find starting test (%s), using all tests" % ns.start) if ns.randomize: if ns.random_seed is None: ns.random_seed = random.randrange(10000000) random.seed(ns.random_seed) print("Using random seed", ns.random_seed) random.shuffle(selected) if ns.trace: import trace, tempfile tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, tempfile.gettempdir()], trace=False, count=True) test_times = [] support.verbose = ns.verbose # Tell tests to be moderately quiet support.use_resources = ns.use_resources save_modules = sys.modules.keys() def accumulate_result(test, result): ok, test_time = result test_times.append((test_time, test)) if ok == PASSED: good.append(test) elif ok == FAILED: bad.append(test) elif ok == ENV_CHANGED: environment_changed.append(test) elif ok == SKIPPED: skipped.append(test) elif ok == RESOURCE_DENIED: skipped.append(test) resource_denieds.append(test) if ns.forever: def test_forever(tests=list(selected)): while True: for test in tests: yield test if bad: return tests = test_forever() test_count = '' test_count_width = 3 else: tests = iter(selected) test_count = '/{}'.format(len(selected)) test_count_width = len(test_count) - 1 if ns.use_mp: try: from threading import Thread except ImportError: print("Multiprocess option requires thread support") sys.exit(2) from queue import Queue debug_output_pat = re.compile(r"\[\d+ refs, \d+ blocks\]$") output = Queue() pending = MultiprocessTests(tests) def work(): # A worker thread. try: while True: try: test = next(pending) except StopIteration: output.put((None, None, None, None)) return retcode, stdout, stderr = run_test_in_subprocess(test, ns) # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") if retcode != 0: result = (CHILD_ERROR, "Exit code %s" % retcode) output.put((test, stdout.rstrip(), stderr.rstrip(), result)) return if not result: output.put((None, None, None, None)) return result = json.loads(result) output.put((test, stdout.rstrip(), stderr.rstrip(), result)) except BaseException: output.put((None, None, None, None)) raise workers = [Thread(target=work) for i in range(ns.use_mp)] for worker in workers: worker.start() finished = 0 test_index = 1 try: while finished < ns.use_mp: test, stdout, stderr, result = output.get() if test is None: finished += 1 continue accumulate_result(test, result) if not ns.quiet: fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" print(fmt.format( test_count_width, test_index, test_count, len(bad), test)) if stdout: print(stdout) if stderr: print(stderr, file=sys.stderr) sys.stdout.flush() sys.stderr.flush() if result[0] == INTERRUPTED: raise KeyboardInterrupt if result[0] == CHILD_ERROR: raise Exception("Child error on {}: {}".format(test, result[1])) test_index += 1 except KeyboardInterrupt: interrupted = True pending.interrupted = True for worker in workers: worker.join() else: for test_index, test in enumerate(tests, 1): if not ns.quiet: fmt = "[{1:{0}}{2}/{3}] {4}" if bad else "[{1:{0}}{2}] {4}" print(fmt.format( test_count_width, test_index, test_count, len(bad), test)) sys.stdout.flush() if ns.trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. tracer.runctx('runtest(test, ns.verbose, ns.quiet, timeout=ns.timeout)', globals=globals(), locals=vars()) else: try: result = runtest(test, ns.verbose, ns.quiet, ns.huntrleaks, output_on_failure=ns.verbose3, timeout=ns.timeout, failfast=ns.failfast, match_tests=ns.match_tests) accumulate_result(test, result) except KeyboardInterrupt: interrupted = True break if ns.findleaks: gc.collect() if gc.garbage: print("Warning: test created", len(gc.garbage), end=' ') print("uncollectable object(s).") # move the uncollectable objects somewhere so we don't see # them again found_garbage.extend(gc.garbage) del gc.garbage[:] # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) if interrupted: # print a newline after ^C print() print("Test suite interrupted by signal SIGINT.") omitted = set(selected) - set(good) - set(bad) - set(skipped) print(count(len(omitted), "test"), "omitted:") printlist(omitted) if good and not ns.quiet: if not bad and not skipped and not interrupted and len(good) > 1: print("All", end=' ') print(count(len(good), "test"), "OK.") if ns.print_slow: test_times.sort(reverse=True) print("10 slowest tests:") for time, test in test_times[:10]: print("%s: %.1fs" % (test, time)) if bad: print(count(len(bad), "test"), "failed:") printlist(bad) if environment_changed: print("{} altered the execution environment:".format( count(len(environment_changed), "test"))) printlist(environment_changed) if skipped and not ns.quiet: print(count(len(skipped), "test"), "skipped:") printlist(skipped) if ns.verbose2 and bad: print("Re-running failed tests in verbose mode") for test in bad[:]: print("Re-running test %r in verbose mode" % test) sys.stdout.flush() try: ns.verbose = True ok = runtest(test, True, ns.quiet, ns.huntrleaks, timeout=ns.timeout) except KeyboardInterrupt: # print a newline separate from the ^C print() break else: if ok[0] in {PASSED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED}: bad.remove(test) else: if bad: print(count(len(bad), 'test'), "failed again:") printlist(bad) if ns.single: if next_single_test: with open(filename, 'w') as fp: fp.write(next_single_test + '\n') else: os.unlink(filename) if ns.trace: r = tracer.results() r.write_results(show_missing=True, summary=True, coverdir=ns.coverdir) if ns.runleaks: os.system("leaks %d" % os.getpid()) sys.exit(len(bad) > 0 or interrupted)
def main(tests=None, testdir=None, verbose=0, quiet=False, exclude=False, single=False, randomize=False, fromfile=None, findleaks=False, use_resources=None, trace=False, coverdir='coverage', runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, expected=False, memo=None, junit_xml=None): """Execute a test suite. This also parses command-line options and modifies its behavior accordingly. tests -- a list of strings containing test names (optional) testdir -- the directory in which to look for tests (optional) Users other than the Python test suite will certainly want to specify testdir; if it's omitted, the directory containing the Python test suite is searched for. If the tests argument is omitted, the tests listed on the command-line will be used. If that's empty, too, then all *.py files beginning with test_ will be used. The other default arguments (verbose, quiet, exclude, single, randomize, findleaks, use_resources, trace, coverdir, and print_slow) allow programmers calling main() directly to set the values that would normally be set by flags on the command line. """ support.record_original_stdout(sys.stdout) try: opts, args = getopt.getopt(sys.argv[1:], 'hvqxsSrf:lu:t:TD:NLR:wM:em:j:', ['help', 'verbose', 'quiet', 'exclude', 'single', 'slow', 'random', 'fromfile', 'findleaks', 'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir', 'runleaks', 'huntrleaks=', 'verbose2', 'memlimit=', 'expected', 'memo' ]) except getopt.error as msg: usage(2, msg) # Defaults allran = True if use_resources is None: use_resources = [] for o, a in opts: if o in ('-h', '--help'): usage(0) elif o in ('-v', '--verbose'): verbose += 1 elif o in ('-w', '--verbose2'): verbose2 = True elif o in ('-q', '--quiet'): quiet = True; verbose = 0 elif o in ('-x', '--exclude'): exclude = True allran = False elif o in ('-e', '--expected'): expected = True allran = False elif o in ('-s', '--single'): single = True elif o in ('-S', '--slow'): print_slow = True elif o in ('-r', '--randomize'): randomize = True elif o in ('-f', '--fromfile'): fromfile = a elif o in ('-l', '--findleaks'): findleaks = True elif o in ('-L', '--runleaks'): runleaks = True elif o in ('-m', '--memo'): memo = a elif o in ('-j', '--junit-xml'): junit_xml = a elif o in ('-t', '--threshold'): import gc gc.set_threshold(int(a)) elif o in ('-T', '--coverage'): trace = True elif o in ('-D', '--coverdir'): coverdir = os.path.join(os.getcwd(), a) elif o in ('-N', '--nocoverdir'): coverdir = None elif o in ('-R', '--huntrleaks'): huntrleaks = a.split(':') if len(huntrleaks) != 3: print(a, huntrleaks) usage(2, '-R takes three colon-separated arguments') if len(huntrleaks[0]) == 0: huntrleaks[0] = 5 else: huntrleaks[0] = int(huntrleaks[0]) if len(huntrleaks[1]) == 0: huntrleaks[1] = 4 else: huntrleaks[1] = int(huntrleaks[1]) if len(huntrleaks[2]) == 0: huntrleaks[2] = "reflog.txt" elif o in ('-M', '--memlimit'): support.set_memlimit(a) elif o in ('-u', '--use'): u = [x.lower() for x in a.split(',')] for r in u: if r == 'all': use_resources[:] = RESOURCE_NAMES continue remove = False if r[0] == '-': remove = True r = r[1:] if r not in RESOURCE_NAMES: usage(1, 'Invalid -u/--use option: ' + a) if remove: if r in use_resources: use_resources.remove(r) elif r not in use_resources: use_resources.append(r) else: print(("No handler for option {0}. Please " "report this as a bug at http://bugs.python.org.").format(o), file=sys.stderr) sys.exit(1) if single and fromfile: usage(2, "-s and -f don't go together!") good = [] bad = [] skipped = [] resource_denieds = [] if findleaks: try: if support.is_jython: raise ImportError() import gc except ImportError: print('No GC available, disabling findleaks.') findleaks = False else: # Uncomment the line below to report garbage that is not # freeable by reference counting alone. By default only # garbage that is not collectable by the GC is reported. #gc.set_debug(gc.DEBUG_SAVEALL) found_garbage = [] if single: from tempfile import gettempdir filename = os.path.join(gettempdir(), 'pynexttest') try: fp = open(filename, 'r') next = fp.read().strip() tests = [next] fp.close() except IOError: pass if fromfile: tests = [] fp = open(fromfile) for line in fp: guts = line.split() # assuming no test has whitespace in its name if guts and not guts[0].startswith('#'): tests.extend(guts) fp.close() # Strip .py extensions. if args: args = list(map(removepy, args)) allran = False if tests: tests = list(map(removepy, tests)) stdtests = STDTESTS[:] nottests = list(NOTTESTS) if exclude: for arg in args: if arg in stdtests: stdtests.remove(arg) nottests[:0] = args args = [] tests = tests or args or findtests(testdir, stdtests, nottests) if single: tests = tests[:1] if randomize: random.shuffle(tests) if trace: import trace tracer = trace.Trace(ignoredirs=[sys.prefix, sys.exec_prefix], trace=False, count=True) test_times = [] support.verbose = verbose # Tell tests to be moderately quiet support.use_resources = use_resources support.junit_xml_dir = junit_xml save_modules = list(sys.modules.keys()) skips = _ExpectedSkips() failures = _ExpectedFailures() if expected: # Suppress expected failure from the list of tests. for t in failures.getexpected(): if t in tests: tests.remove(t) # Suppress expected skips from the list of tests. for t in skips.getexpected(): if t in tests: tests.remove(t) # Prevent reporting unexpected success in things we failed to try failures.keep_only(tests) skips.keep_only(tests) for test in tests: if not quiet: print(test) sys.stdout.flush() if trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. tracer.runctx('runtest(test, verbose, quiet,' ' test_times, testdir)', globals=globals(), locals=vars()) else: try: ok = runtest(test, verbose, quiet, test_times, testdir, huntrleaks, junit_xml) except KeyboardInterrupt: # print a newline separate from the ^C print() break except: raise if ok > 0: good.append(test) elif ok == 0: bad.append(test) else: skipped.append(test) if ok == -2: resource_denieds.append(test) if findleaks: gc.collect() if gc.garbage: print("Warning: test created", len(gc.garbage), end=' ') print("uncollectable object(s).") # move the uncollectable objects somewhere so we don't see # them again found_garbage.extend(gc.garbage) del gc.garbage[:] # Unload the newly imported modules (best effort finalization) for module in list(sys.modules.keys()): if module not in save_modules and module.startswith("test."): support.unload(module) module = module[5:] if hasattr(_test, module): delattr(_test, module) if good and not quiet: if not bad and not skipped and len(good) > 1: print("All", end=' ') print(count(len(good), "test"), "OK.") if print_slow: test_times.sort(reverse=True) print("10 slowest tests:") for time, test in test_times[:10]: print("%s: %.1fs" % (test, time)) surprises = 0 if skipped and not quiet: print(count(len(skipped), "test"), "skipped:") surprises += countsurprises(skips, skipped, 'skip', 'ran', allran, resource_denieds) if bad: print(count(len(bad), "test"), "failed:") surprises += countsurprises(failures, bad, 'fail', 'passed', allran, resource_denieds) if verbose2 and bad: print("Re-running failed tests in verbose mode") for test in bad: print("Re-running test %r in verbose mode" % test) sys.stdout.flush() try: support.verbose = True ok = runtest(test, True, quiet, test_times, testdir, huntrleaks) except KeyboardInterrupt: # print a newline separate from the ^C print() break except: raise if single: alltests = findtests(testdir, stdtests, nottests) for i in range(len(alltests)): if tests[0] == alltests[i]: if i == len(alltests) - 1: os.unlink(filename) else: fp = open(filename, 'w') fp.write(alltests[i+1] + '\n') fp.close() break else: os.unlink(filename) if trace: r = tracer.results() r.write_results(show_missing=True, summary=True, coverdir=coverdir) if runleaks: os.system("leaks %d" % os.getpid()) if memo: savememo(memo,good,bad,skipped) sys.exit(surprises > 0)