def display_result(self): # If running the test suite for PGO then no one cares about results. if self.ns.pgo: return print() print("== Tests result: %s ==" % self.get_tests_result()) if self.interrupted: print() # print a newline after ^C print("Test suite interrupted by signal SIGINT.") executed = set(self.good) | set(self.bad) | set(self.skipped) omitted = set(self.selected) - executed print(count(len(omitted), "test"), "omitted:") printlist(omitted) if self.good and not self.ns.quiet: print() if (not self.bad and not self.skipped and not self.interrupted and len(self.good) > 1): print("All", end=' ') print(count(len(self.good), "test"), "OK.") if self.ns.print_slow: self.test_times.sort(reverse=True) print() print("10 slowest tests:") for time, test in self.test_times[:10]: print("- %s: %s" % (test, format_duration(time))) if self.bad: print() print(count(len(self.bad), "test"), "failed:") printlist(self.bad) if self.environment_changed: print() print("{} altered the execution environment:".format( count(len(self.environment_changed), "test"))) printlist(self.environment_changed) if self.skipped and not self.ns.quiet: print() print(count(len(self.skipped), "test"), "skipped:") printlist(self.skipped) if self.rerun: print() print("%s:" % count(len(self.rerun), "re-run test")) printlist(self.rerun) if self.run_no_tests: print() print(count(len(self.run_no_tests), "test"), "run no tests:") printlist(self.run_no_tests)
def get_running(workers): running = [] for worker in workers: current_test = worker.current_test if not current_test: continue dt = time.monotonic() - worker.start_time if dt >= PROGRESS_MIN_TIME: text = '%s (%s)' % (current_test, format_duration(dt)) running.append(text) return running
def test_format_duration(self): self.assertEqual(utils.format_duration(0), '0 ms') self.assertEqual(utils.format_duration(1e-9), '1 ms') self.assertEqual(utils.format_duration(10e-3), '10 ms') self.assertEqual(utils.format_duration(1.5), '1 sec 500 ms') self.assertEqual(utils.format_duration(1), '1 sec') self.assertEqual(utils.format_duration(2 * 60), '2 min') self.assertEqual(utils.format_duration(2 * 60 + 1), '2 min 1 sec') self.assertEqual(utils.format_duration(3 * 3600), '3 hour') self.assertEqual(utils.format_duration(3 * 3600 + 2 * 60 + 1), '3 hour 2 min') self.assertEqual(utils.format_duration(3 * 3600 + 1), '3 hour 1 sec')
def test_format_duration(self): self.assertEqual(utils.format_duration(0), '0 ms') self.assertEqual(utils.format_duration(1e-9), '1 ms') self.assertEqual(utils.format_duration(10e-3), '10 ms') self.assertEqual(utils.format_duration(1.5), '1 sec 500 ms') self.assertEqual(utils.format_duration(1), '1 sec') self.assertEqual(utils.format_duration(2 * 60), '2 min') self.assertEqual(utils.format_duration(2 * 60 + 1), '2 min 1 sec') self.assertEqual(utils.format_duration(3 * 3600), '3 hour') self.assertEqual(utils.format_duration(3 * 3600 + 2 * 60 + 1), '3 hour 2 min') self.assertEqual(utils.format_duration(3 * 3600 + 1), '3 hour 1 sec')
def display_result(self): # If running the test suite for PGO then no one cares about results. if self.ns.pgo: return print() print("== Tests result: %s ==" % self.get_tests_result()) if self.interrupted: print() # print a newline after ^C print("Test suite interrupted by signal SIGINT.") executed = set(self.good) | set(self.bad) | set(self.skipped) omitted = set(self.selected) - executed print(count(len(omitted), "test"), "omitted:") printlist(omitted) if self.good and not self.ns.quiet: print() if (not self.bad and not self.skipped and not self.interrupted and len(self.good) > 1): print("All", end=' ') print(count(len(self.good), "test"), "OK.") if self.ns.print_slow: self.test_times.sort(reverse=True) print() print("10 slowest tests:") for time, test in self.test_times[:10]: print("- %s: %s" % (test, format_duration(time))) if self.bad: print() print(count(len(self.bad), "test"), "failed:") printlist(self.bad) if self.environment_changed: print() print("{} altered the execution environment:".format( count(len(self.environment_changed), "test"))) printlist(self.environment_changed) if self.skipped and not self.ns.quiet: print() print(count(len(self.skipped), "test"), "skipped:") printlist(self.skipped) if self.rerun: print() print("%s:" % count(len(self.rerun), "re-run test")) printlist(self.rerun)
def run_tests_sequential(self): if self.ns.trace: import trace self.tracer = trace.Trace(trace=False, count=True) save_modules = sys.modules.keys() print("Run tests sequentially") previous_test = None for test_index, test_name in enumerate(self.tests, 1): start_time = time.monotonic() text = test_name if previous_test: text = '%s -- %s' % (text, previous_test) self.display_progress(test_index, text) if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. cmd = ('result = runtest(self.ns, test_name); ' 'self.accumulate_result(result)') ns = dict(locals()) self.tracer.runctx(cmd, globals=globals(), locals=ns) result = ns['result'] else: result = runtest(self.ns, test_name) self.accumulate_result(result) if result.result == INTERRUPTED: break previous_test = format_test_result(result) test_time = time.monotonic() - start_time if test_time >= PROGRESS_MIN_TIME: previous_test = "%s in %s" % (previous_test, format_duration(test_time)) elif result.result == PASSED: # be quiet: say nothing if the test passed shortly previous_test = None # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) if self.ns.failfast and is_failed(result, self.ns): break if previous_test: print(previous_test)
def display_result(self, mp_result): result = mp_result.result text = format_test_result(result) if mp_result.error_msg is not None: # CHILD_ERROR text += ' (%s)' % mp_result.error_msg elif (result.test_time >= PROGRESS_MIN_TIME and not self.ns.pgo): text += ' (%s)' % format_duration(result.test_time) running = get_running(self.workers) if running and not self.ns.pgo: text += ' -- running: %s' % ', '.join(running) self.regrtest.display_progress(self.test_index, text)
def __repr__(self): info = [f'TestWorkerProcess #{self.worker_id}'] if self.is_alive(): dt = time.monotonic() - self.start_time info.append("running for %s" % format_duration(dt)) else: info.append('stopped') test = self.current_test_name if test: info.append(f'test={test}') popen = self._popen if popen: info.append(f'pid={popen.pid}') return '<%s>' % ' '.join(info)
def finalize(self): if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: fp.write(self.next_single_test + '\n') else: os.unlink(self.next_single_filename) if self.tracer: r = self.tracer.results() r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) print() duration = time.monotonic() - self.start_time print("Total duration: %s" % format_duration(duration)) print("Tests result: %s" % self.get_tests_result()) if self.ns.runleaks: os.system("leaks %d" % os.getpid())
def finalize(self): if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: fp.write(self.next_single_test + '\n') else: os.unlink(self.next_single_filename) if self.tracer: r = self.tracer.results() r.write_results(show_missing=True, summary=True, coverdir=self.ns.coverdir) print() duration = time.monotonic() - self.start_time print("Total duration: %s" % format_duration(duration)) print("Tests result: %s" % self.get_tests_result()) if self.ns.runleaks: os.system("leaks %d" % os.getpid())
def run_tests_multiprocess(regrtest): output = queue.Queue() pending = MultiprocessIterator(regrtest.tests) test_timeout = regrtest.ns.timeout use_timeout = (test_timeout is not None) workers = [ MultiprocessThread(pending, output, regrtest.ns) for i in range(regrtest.ns.use_mp) ] print("Run tests in parallel using %s child processes" % len(workers)) for worker in workers: worker.start() def get_running(workers): running = [] for worker in workers: current_test = worker.current_test if not current_test: continue dt = time.monotonic() - worker.start_time if dt >= PROGRESS_MIN_TIME: text = '%s (%s)' % (current_test, format_duration(dt)) running.append(text) return running finished = 0 test_index = 1 get_timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) try: while finished < regrtest.ns.use_mp: if use_timeout: faulthandler.dump_traceback_later(test_timeout, exit=True) try: item = output.get(timeout=get_timeout) except queue.Empty: running = get_running(workers) if running and not regrtest.ns.pgo: print('running: %s' % ', '.join(running), flush=True) continue test, stdout, stderr, result = item if test is None: finished += 1 continue regrtest.accumulate_result(test, result) # Display progress ok, test_time, xml_data = result text = format_test_result(test, ok) if (ok not in (CHILD_ERROR, INTERRUPTED) and test_time >= PROGRESS_MIN_TIME and not regrtest.ns.pgo): text += ' (%s)' % format_duration(test_time) elif ok == CHILD_ERROR: text = '%s (%s)' % (text, test_time) running = get_running(workers) if running and not regrtest.ns.pgo: text += ' -- running: %s' % ', '.join(running) regrtest.display_progress(test_index, text) # Copy stdout and stderr from the child process if stdout: print(stdout, flush=True) if stderr and not regrtest.ns.pgo: print(stderr, file=sys.stderr, flush=True) if result[0] == INTERRUPTED: raise KeyboardInterrupt test_index += 1 except KeyboardInterrupt: regrtest.interrupted = True pending.interrupted = True print() finally: if use_timeout: faulthandler.cancel_dump_traceback_later() # If tests are interrupted, wait until tests complete wait_start = time.monotonic() while True: running = [worker.current_test for worker in workers] running = list(filter(bool, running)) if not running: break dt = time.monotonic() - wait_start line = "Waiting for %s (%s tests)" % (', '.join(running), len(running)) if dt >= WAIT_PROGRESS: line = "%s since %.0f sec" % (line, dt) print(line, flush=True) for worker in workers: worker.join(WAIT_PROGRESS)
def run_tests_sequential(self): if self.ns.trace: import trace self.tracer = trace.Trace(trace=False, count=True) save_modules = sys.modules.keys() print("Run tests sequentially") previous_test = None for test_index, test in enumerate(self.tests, 1): start_time = time.monotonic() text = test if previous_test: text = '%s -- %s' % (text, previous_test) self.display_progress(test_index, text) if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. cmd = ('result = runtest(self.ns, test); ' 'self.accumulate_result(test, result)') ns = dict(locals()) self.tracer.runctx(cmd, globals=globals(), locals=ns) result = ns['result'] else: try: result = runtest(self.ns, test) except KeyboardInterrupt: self.interrupted = True self.accumulate_result(test, (INTERRUPTED, None, None)) break else: self.accumulate_result(test, result) previous_test = format_test_result(test, result[0]) test_time = time.monotonic() - start_time if test_time >= PROGRESS_MIN_TIME: previous_test = "%s in %s" % (previous_test, format_duration(test_time)) elif result[0] == PASSED: # be quiet: say nothing if the test passed shortly previous_test = None if self.ns.findleaks: gc.collect() if gc.garbage: print("Warning: test created", len(gc.garbage), end=' ') print("uncollectable object(s).") # move the uncollectable objects somewhere so we don't see # them again self.found_garbage.extend(gc.garbage) del gc.garbage[:] # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) if previous_test: print(previous_test)
def run_tests_sequential(self): if self.ns.trace: import trace self.tracer = trace.Trace(trace=False, count=True) save_modules = sys.modules.keys() print("Run tests sequentially") previous_test = None for test_index, test in enumerate(self.tests, 1): start_time = time.monotonic() text = test if previous_test: text = '%s -- %s' % (text, previous_test) self.display_progress(test_index, text) if self.tracer: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. cmd = ('result = runtest(self.ns, test); ' 'self.accumulate_result(test, result)') ns = dict(locals()) self.tracer.runctx(cmd, globals=globals(), locals=ns) result = ns['result'] else: try: result = runtest(self.ns, test) except KeyboardInterrupt: self.interrupted = True self.accumulate_result(test, (INTERRUPTED, None, None)) break else: self.accumulate_result(test, result) previous_test = format_test_result(test, result[0]) test_time = time.monotonic() - start_time if test_time >= PROGRESS_MIN_TIME: previous_test = "%s in %s" % (previous_test, format_duration(test_time)) elif result[0] == PASSED: # be quiet: say nothing if the test passed shortly previous_test = None if self.ns.findleaks: gc.collect() if gc.garbage: print("Warning: test created", len(gc.garbage), end=' ') print("uncollectable object(s).") # move the uncollectable objects somewhere so we don't see # them again self.found_garbage.extend(gc.garbage) del gc.garbage[:] # Unload the newly imported modules (best effort finalization) for module in sys.modules.keys(): if module not in save_modules and module.startswith("test."): support.unload(module) if previous_test: print(previous_test)
def format_test_result(result): fmt = _FORMAT_TEST_RESULT.get(result.result, "%s") text = fmt % result.test_name if result.result == TIMEOUT: text = '%s (%s)' % (text, format_duration(result.test_time)) return text
def run_tests_multiprocess(regrtest): output = queue.Queue() pending = MultiprocessIterator(regrtest.tests) test_timeout = regrtest.ns.timeout use_timeout = (test_timeout is not None) workers = [MultiprocessThread(pending, output, regrtest.ns) for i in range(regrtest.ns.use_mp)] print("Run tests in parallel using %s child processes" % len(workers)) for worker in workers: worker.start() def get_running(workers): running = [] for worker in workers: current_test = worker.current_test if not current_test: continue dt = time.monotonic() - worker.start_time if dt >= PROGRESS_MIN_TIME: text = '%s (%s)' % (current_test, format_duration(dt)) running.append(text) return running finished = 0 test_index = 1 get_timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) try: while finished < regrtest.ns.use_mp: if use_timeout: faulthandler.dump_traceback_later(test_timeout, exit=True) try: item = output.get(timeout=get_timeout) except queue.Empty: running = get_running(workers) if running and not regrtest.ns.pgo: print('running: %s' % ', '.join(running), flush=True) continue test, stdout, stderr, result = item if test is None: finished += 1 continue regrtest.accumulate_result(test, result) # Display progress ok, test_time = result text = format_test_result(test, ok) if (ok not in (CHILD_ERROR, INTERRUPTED) and test_time >= PROGRESS_MIN_TIME and not regrtest.ns.pgo): text += ' (%s)' % format_duration(test_time) elif ok == CHILD_ERROR: text = '%s (%s)' % (text, test_time) running = get_running(workers) if running and not regrtest.ns.pgo: text += ' -- running: %s' % ', '.join(running) regrtest.display_progress(test_index, text) # Copy stdout and stderr from the child process if stdout: print(stdout, flush=True) if stderr and not regrtest.ns.pgo: print(stderr, file=sys.stderr, flush=True) if result[0] == INTERRUPTED: raise KeyboardInterrupt test_index += 1 except KeyboardInterrupt: regrtest.interrupted = True pending.interrupted = True print() finally: if use_timeout: faulthandler.cancel_dump_traceback_later() # If tests are interrupted, wait until tests complete wait_start = time.monotonic() while True: running = [worker.current_test for worker in workers] running = list(filter(bool, running)) if not running: break dt = time.monotonic() - wait_start line = "Waiting for %s (%s tests)" % (', '.join(running), len(running)) if dt >= WAIT_PROGRESS: line = "%s since %.0f sec" % (line, dt) print(line, flush=True) for worker in workers: worker.join(WAIT_PROGRESS)