def run_tests_sequential(self):
if self.ns.trace:
import trace
self.tracer = trace.Trace(trace=False, count=True)
save_modules = sys.modules.keys()
self.log("Run tests sequentially")
#show used memory
if sys.platform == 'OpenVMS':
print('Used memory: %s' % format_mem(get_mem()))
previous_test = None
for test_index, test_name in enumerate(self.tests, 1):
start_time = time.monotonic()
text = test_name
if previous_test:
text = '%s -- %s' % (text, previous_test)
self.display_progress(test_index, text)
if self.tracer:
# If we're tracing code coverage, then we don't exit with status
# if on a false return value from main.
cmd = ('result = runtest(self.ns, test_name); '
'self.accumulate_result(result)')
ns = dict(locals())
self.tracer.runctx(cmd, globals=globals(), locals=ns)
result = ns['result']
else:
result = runtest(self.ns, test_name)
self.accumulate_result(result)
if result.result == INTERRUPTED:
break
previous_test = format_test_result(result)
test_time = time.monotonic() - start_time
if test_time >= PROGRESS_MIN_TIME:
previous_test = "%s in %s" % (previous_test,
format_duration(test_time))
elif result.result == PASSED:
# be quiet: say nothing if the test passed shortly
previous_test = None
# Unload the newly imported modules (best effort finalization)
for module in sys.modules.keys():
if module not in save_modules and module.startswith("test."):
support.unload(module)
#show used memory
if sys.platform == 'OpenVMS':
print('Used memory: %s' % format_mem(get_mem()))
if self.ns.failfast and is_failed(result, self.ns):
break
if previous_test:
print(previous_test)
def run_tests_sequential(self):
if self.ns.trace:
import trace
self.tracer = trace.Trace(trace=False, count=True)
save_modules = sys.modules.keys()
print("Run tests sequentially")
previous_test = None
for test_index, test_name in enumerate(self.tests, 1):
start_time = time.monotonic()
text = test_name
if previous_test:
text = '%s -- %s' % (text, previous_test)
self.display_progress(test_index, text)
if self.tracer:
# If we're tracing code coverage, then we don't exit with status
# if on a false return value from main.
cmd = ('result = runtest(self.ns, test_name); '
'self.accumulate_result(result)')
ns = dict(locals())
self.tracer.runctx(cmd, globals=globals(), locals=ns)
result = ns['result']
else:
result = runtest(self.ns, test_name)
self.accumulate_result(result)
if result.result == INTERRUPTED:
self.interrupted = True
break
previous_test = format_test_result(result)
test_time = time.monotonic() - start_time
if test_time >= PROGRESS_MIN_TIME:
previous_test = "%s in %s" % (previous_test,
format_duration(test_time))
elif result[0] == PASSED:
# be quiet: say nothing if the test passed shortly
previous_test = None
if self.ns.findleaks:
gc.collect()
if gc.garbage:
print("Warning: test created", len(gc.garbage), end=' ')
print("uncollectable object(s).")
# move the uncollectable objects somewhere so we don't see
# them again
self.found_garbage.extend(gc.garbage)
del gc.garbage[:]
# Unload the newly imported modules (best effort finalization)
for module in sys.modules.keys():
if module not in save_modules and module.startswith("test."):
support.unload(module)
if previous_test:
print(previous_test)
def display_result(self, mp_result):
result = mp_result.result
text = format_test_result(result)
if mp_result.error_msg is not None:
# CHILD_ERROR
text += ' (%s)' % mp_result.error_msg
elif (result.test_time >= PROGRESS_MIN_TIME and not self.ns.pgo):
text += ' (%s)' % format_duration(result.test_time)
running = get_running(self.workers)
if running and not self.ns.pgo:
text += ' -- running: %s' % ', '.join(running)
self.regrtest.display_progress(self.test_index, text)
def run_tests_sequential(self):
if self.ns.trace:
import trace
self.tracer = trace.Trace(trace=False, count=True)
save_modules = sys.modules.keys()
print('Run tests sequentially')
previous_test = None
for test_index, test in enumerate(self.tests, 1):
start_time = time.monotonic()
text = test
if previous_test:
text = '%s -- %s' % (text, previous_test)
self.display_progress(test_index, text)
if self.tracer:
cmd = (
'result = runtest(self.ns, test); self.accumulate_result(test, result)'
)
ns = dict(locals())
self.tracer.runctx(cmd, globals=globals(), locals=ns)
result = ns['result']
else:
try:
result = runtest(self.ns, test)
except KeyboardInterrupt:
self.interrupted = True
self.accumulate_result(test, (INTERRUPTED, None))
break
else:
self.accumulate_result(test, result)
previous_test = format_test_result(test, result[0])
test_time = time.monotonic() - start_time
if test_time >= PROGRESS_MIN_TIME:
previous_test = '%s in %s' % (previous_test,
format_duration(test_time))
elif result[0] == PASSED:
previous_test = None
if self.ns.findleaks:
gc.collect()
if gc.garbage:
print('Warning: test created', len(gc.garbage), end=' ')
print('uncollectable object(s).')
self.found_garbage.extend(gc.garbage)
del gc.garbage[:]
for module in sys.modules.keys():
if module not in save_modules and module.startswith('test.'):
support.unload(module)
if previous_test:
print(previous_test)
def run_tests_sequential(self):
if self.ns.trace:
import trace
self.tracer = trace.Trace(trace=False, count=True)
save_modules = sys.modules.keys()
print("Run tests sequentially")
previous_test = None
for test_index, test in enumerate(self.tests, 1):
start_time = time.monotonic()
text = test
if previous_test:
text = '%s -- %s' % (text, previous_test)
self.display_progress(test_index, text)
if self.tracer:
# If we're tracing code coverage, then we don't exit with status
# if on a false return value from main.
cmd = ('result = runtest(self.ns, test); '
'self.accumulate_result(test, result)')
ns = dict(locals())
self.tracer.runctx(cmd, globals=globals(), locals=ns)
result = ns['result']
else:
try:
result = runtest(self.ns, test)
except KeyboardInterrupt:
self.interrupted = True
self.accumulate_result(test, (INTERRUPTED, None))
break
else:
self.accumulate_result(test, result)
previous_test = format_test_result(test, result[0])
test_time = time.monotonic() - start_time
if test_time >= PROGRESS_MIN_TIME:
previous_test = "%s in %s" % (previous_test, format_duration(test_time))
elif result[0] == PASSED:
# be quiet: say nothing if the test passed shortly
previous_test = None
if self.ns.findleaks:
gc.collect()
if gc.garbage:
print("Warning: test created", len(gc.garbage), end=' ')
print("uncollectable object(s).")
# move the uncollectable objects somewhere so we don't see
# them again
self.found_garbage.extend(gc.garbage)
del gc.garbage[:]
# Unload the newly imported modules (best effort finalization)
for module in sys.modules.keys():
if module not in save_modules and module.startswith("test."):
support.unload(module)
if previous_test:
print(previous_test)
def run_tests_multiprocess(regrtest):
output = queue.Queue()
pending = MultiprocessIterator(regrtest.tests)
test_timeout = regrtest.ns.timeout
use_timeout = (test_timeout is not None)
workers = [
MultiprocessThread(pending, output, regrtest.ns)
for i in range(regrtest.ns.use_mp)
]
print("Run tests in parallel using %s child processes" % len(workers))
for worker in workers:
worker.start()
def get_running(workers):
running = []
for worker in workers:
current_test = worker.current_test
if not current_test:
continue
dt = time.monotonic() - worker.start_time
if dt >= PROGRESS_MIN_TIME:
text = '%s (%s)' % (current_test, format_duration(dt))
running.append(text)
return running
finished = 0
test_index = 1
get_timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME)
try:
while finished < regrtest.ns.use_mp:
if use_timeout:
faulthandler.dump_traceback_later(test_timeout, exit=True)
try:
item = output.get(timeout=get_timeout)
except queue.Empty:
running = get_running(workers)
if running and not regrtest.ns.pgo:
print('running: %s' % ', '.join(running), flush=True)
continue
test, stdout, stderr, result = item
if test is None:
finished += 1
continue
regrtest.accumulate_result(test, result)
# Display progress
ok, test_time = result
text = format_test_result(test, ok)
if (ok not in (CHILD_ERROR, INTERRUPTED)
and test_time >= PROGRESS_MIN_TIME
and not regrtest.ns.pgo):
text += ' (%.0f sec)' % test_time
elif ok == CHILD_ERROR:
text = '%s (%s)' % (text, test_time)
running = get_running(workers)
if running and not regrtest.ns.pgo:
text += ' -- running: %s' % ', '.join(running)
regrtest.display_progress(test_index, text)
# Copy stdout and stderr from the child process
if stdout:
print(stdout, flush=True)
if stderr and not regrtest.ns.pgo:
print(stderr, file=sys.stderr, flush=True)
if result[0] == INTERRUPTED:
raise KeyboardInterrupt
test_index += 1
except KeyboardInterrupt:
regrtest.interrupted = True
pending.interrupted = True
print()
finally:
if use_timeout:
faulthandler.cancel_dump_traceback_later()
# If tests are interrupted, wait until tests complete
wait_start = time.monotonic()
while True:
running = [worker.current_test for worker in workers]
running = list(filter(bool, running))
if not running:
break
dt = time.monotonic() - wait_start
line = "Waiting for %s (%s tests)" % (', '.join(running), len(running))
if dt >= WAIT_PROGRESS:
line = "%s since %.0f sec" % (line, dt)
print(line, flush=True)
for worker in workers:
worker.join(WAIT_PROGRESS)
def _run_tests_with_n_workers(self, tests: Iterable[str], n_workers: int,
replay_infos: List[ReplayInfo]) -> None:
resultq = queue.Queue()
testq = queue.Queue()
ntests_remaining = 0
for test in tests:
ntests_remaining += 1
testq.put(RunTest(test))
threads = []
for i in range(n_workers):
testq.put(ShutdownWorker())
t = threading.Thread(
target=manage_worker,
args=(self.ns, testq, resultq, self._worker_timeout,
self._use_rr),
)
t.start()
threads.append(t)
try:
active_tests: Dict[str, ActiveTest] = {}
worker_infos: Dict[int, ReplayInfo] = {}
while ntests_remaining:
try:
msg = resultq.get(timeout=10)
except queue.Empty:
print_running_tests(active_tests)
continue
if isinstance(msg, TestStarted):
active_tests[msg.test_name] = ActiveTest(
msg.worker_pid, time.time(), msg.test_log,
msg.rr_trace_dir)
print(
f"Running test '{msg.test_name}' on worker "
f"{msg.worker_pid}",
file=self._jit_regr_runner_logfile,
)
self._jit_regr_runner_logfile.flush()
elif isinstance(msg, TestComplete):
ntests_remaining -= 1
self._ntests_done += 1
result_status = msg.result.result
if result_status not in (PASSED, RESOURCE_DENIED, SKIPPED):
worker_pid = active_tests[msg.test_name].worker_pid
rr_trace_dir = active_tests[msg.test_name].rr_trace_dir
status_str = format_test_result(msg.result)
err = f"TEST ERROR: {status_str} in pid {worker_pid}"
if rr_trace_dir:
# TODO: Add link to fdb documentation
err += (
f" Replay recording with: fdb replay debug {rr_trace_dir}"
)
log_err(f"{err}\n")
if worker_pid not in worker_infos:
log = active_tests[msg.test_name].worker_test_log
worker_infos[worker_pid] = (ReplayInfo(
worker_pid, log, rr_trace_dir))
replay_info = worker_infos[worker_pid]
if result_status == CHILD_ERROR:
replay_info.crashed = msg.test_name
else:
replay_info.failed.append(msg.test_name)
self.accumulate_result(msg.result)
self.display_progress(self._ntests_done, msg.test_name)
del active_tests[msg.test_name]
except KeyboardInterrupt:
replay_infos.extend(worker_infos.values())
self._show_replay_infos(replay_infos)
self._save_recording_metadata(replay_infos)
# Kill the whole process group, getting rid of this process and all
# workers, including those which have gone a bit rogue.
os.killpg(os.getpgid(os.getpid()), signal.SIGTERM)
replay_infos.extend(worker_infos.values())
for t in threads:
t.join()
def run_tests_multiprocess(regrtest):
output = queue.Queue()
pending = MultiprocessIterator(regrtest.tests)
test_timeout = regrtest.ns.timeout
use_timeout = (test_timeout is not None)
workers = [MultiprocessThread(pending, output, regrtest.ns)
for i in range(regrtest.ns.use_mp)]
print("Run tests in parallel using %s child processes"
% len(workers))
for worker in workers:
worker.start()
def get_running(workers):
running = []
for worker in workers:
current_test = worker.current_test
if not current_test:
continue
dt = time.monotonic() - worker.start_time
if dt >= PROGRESS_MIN_TIME:
text = '%s (%s)' % (current_test, format_duration(dt))
running.append(text)
return running
finished = 0
test_index = 1
get_timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME)
try:
while finished < regrtest.ns.use_mp:
if use_timeout:
faulthandler.dump_traceback_later(test_timeout, exit=True)
try:
item = output.get(timeout=get_timeout)
except queue.Empty:
running = get_running(workers)
if running and not regrtest.ns.pgo:
print('running: %s' % ', '.join(running), flush=True)
continue
test, stdout, stderr, result = item
if test is None:
finished += 1
continue
regrtest.accumulate_result(test, result)
# Display progress
ok, test_time = result
text = format_test_result(test, ok)
if (ok not in (CHILD_ERROR, INTERRUPTED)
and test_time >= PROGRESS_MIN_TIME
and not regrtest.ns.pgo):
text += ' (%s)' % format_duration(test_time)
elif ok == CHILD_ERROR:
text = '%s (%s)' % (text, test_time)
running = get_running(workers)
if running and not regrtest.ns.pgo:
text += ' -- running: %s' % ', '.join(running)
regrtest.display_progress(test_index, text)
# Copy stdout and stderr from the child process
if stdout:
print(stdout, flush=True)
if stderr and not regrtest.ns.pgo:
print(stderr, file=sys.stderr, flush=True)
if result[0] == INTERRUPTED:
raise KeyboardInterrupt
test_index += 1
except KeyboardInterrupt:
regrtest.interrupted = True
pending.interrupted = True
print()
finally:
if use_timeout:
faulthandler.cancel_dump_traceback_later()
# If tests are interrupted, wait until tests complete
wait_start = time.monotonic()
while True:
running = [worker.current_test for worker in workers]
running = list(filter(bool, running))
if not running:
break
dt = time.monotonic() - wait_start
line = "Waiting for %s (%s tests)" % (', '.join(running), len(running))
if dt >= WAIT_PROGRESS:
line = "%s since %.0f sec" % (line, dt)
print(line, flush=True)
for worker in workers:
worker.join(WAIT_PROGRESS)
