def init_listeners(self):
args = lib.Options().args
watch_hang = args.no_output_timeout >= 0 and \
not args.gdb and \
not args.gdbserver and \
not args.lldb and \
not args.valgrind and \
not args.long
watch_fail = not lib.Options().args.is_force
log_output_watcher = listeners.LogOutputWatcher()
self.statistics = listeners.StatisticsWatcher(
log_output_watcher.get_logfile)
output_watcher = listeners.OutputWatcher()
self.listeners = [self.statistics, log_output_watcher, output_watcher]
if watch_fail:
self.fail_watcher = listeners.FailWatcher(
self.terminate_all_workers)
self.listeners.append(self.fail_watcher)
if watch_hang:
warn_timeout = 10.0
no_output_timeout = float(args.no_output_timeout or 120)
hang_watcher = listeners.HangWatcher(
output_watcher.not_done_worker_ids, self.kill_all_workers,
warn_timeout, no_output_timeout)
self.listeners.append(hang_watcher)
def main_loop_parallel():
color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text')
jobs = lib.Options().args.jobs
if jobs < 1:
# faster result I got was with 2 * cpu_count
jobs = 2 * multiprocessing.cpu_count()
if jobs > 0:
color_stdout("Running in parallel with %d workers\n\n" % jobs,
schema='tr_text')
randomize = True
task_groups = lib.worker.get_task_groups()
if lib.Options().args.reproduce:
task_groups = lib.worker.reproduce_task_groups(task_groups)
jobs = 1
randomize = False
dispatcher = Dispatcher(task_groups, jobs, randomize)
dispatcher.start()
lib.worker.print_greetings()
color_stdout("\n", '=' * 86, "\n", schema='separator')
color_stdout("WORKR".ljust(6), schema='t_name')
color_stdout("TEST".ljust(48), schema='t_name')
color_stdout("PARAMS".ljust(16), schema='test_var')
color_stdout("RESULT\n", schema='test_pass')
color_stdout('-' * 81, "\n", schema='separator')
try:
is_force = lib.Options().args.is_force
dispatcher.wait()
dispatcher.wait_processes()
color_stdout('-' * 81, "\n", schema='separator')
has_failed = dispatcher.statistics.print_statistics()
has_undone = dispatcher.report_undone(
verbose=bool(is_force or not has_failed))
if has_failed:
return EXIT_FAILED_TEST
if has_undone:
return EXIT_NOTDONE_TEST
except KeyboardInterrupt:
color_stdout('-' * 81, "\n", schema='separator')
dispatcher.statistics.print_statistics()
dispatcher.report_undone(verbose=False)
raise
except HangError:
color_stdout('-' * 81, "\n", schema='separator')
dispatcher.statistics.print_statistics()
dispatcher.report_undone(verbose=False)
return EXIT_HANG
return EXIT_SUCCESS
def find_suites():
suite_names = lib.Options().args.suites
if suite_names == []:
for root, dirs, names in os.walk(os.getcwd(), followlinks=True):
if "suite.ini" in names:
suite_names.append(os.path.basename(root))
suites = [
TestSuite(suite_name,
lib.Options().args) for suite_name in sorted(suite_names)
]
return suites
def __init__(self):
self.fds = dict()
self.logdir = os.path.join(lib.Options().args.vardir, 'log')
try:
os.makedirs(self.logdir)
except OSError:
pass
def run_test(self, test, server, inspector):
""" Returns short status of the test as a string: 'skip', 'pass',
'new', 'fail', or 'disabled'.
"""
test.inspector = inspector
color_stdout(os.path.join(self.ini['suite'],
os.path.basename(test.name)).ljust(48),
schema='t_name')
# for better diagnostics in case of a long-running test
conf = ''
if test.run_params:
conf = test.conf_name
color_stdout(conf.ljust(16), schema='test_var')
test_name = os.path.basename(test.name)
if self.is_test_enabled(test, conf, server):
short_status = test.run(server)
else:
color_stdout("[ disabled ]\n", schema='t_name')
short_status = 'disabled'
# cleanup only if test passed or if --force mode enabled
if lib.Options().args.is_force or short_status == 'pass':
inspector.cleanup_nondefault()
return short_status
def main_loop_consistent(failed_test_ids):
# find and prepare all tasks/groups, print information
task_groups = lib.worker.get_task_groups().items()
lib.worker.print_greetings()
for name, task_group in task_groups:
# print information about current test suite
color_stdout("\n", '=' * 80, "\n", schema='separator')
color_stdout("TEST".ljust(48), schema='t_name')
color_stdout("PARAMS".ljust(16), schema='test_var')
color_stdout("RESULT\n", schema='test_pass')
color_stdout('-' * 75, "\n", schema='separator')
task_ids = task_group['task_ids']
if not task_ids:
continue
worker_id = 1
worker = task_group['gen_worker'](worker_id)
for task_id in task_ids:
short_status = worker.run_task(task_id)
if short_status == 'fail':
failed_test_ids.append(task_id)
if not lib.Options().args.is_force:
worker.stop_server(cleanup=False)
return
color_stdout('-' * 75, "\n", schema='separator')
worker.stop_server(silent=False)
color_stdout()
def reproduce_task_groups(task_groups):
"""Filter provided task_groups down to the one certain group. Sort tests in
this group as in the reproduce file.
"""
found_keys = []
reproduce = parse_reproduce_file(lib.Options().args.reproduce)
if not reproduce:
raise ValueError('[reproduce] Tests list cannot be empty')
for i, task_id in enumerate(reproduce):
for key, task_group in task_groups.items():
if task_id in task_group['task_ids']:
found_keys.append(key)
break
if len(found_keys) != i + 1:
raise ValueError('[reproduce] Cannot find test "%s"' %
str(task_id))
found_keys = list(set(found_keys))
if len(found_keys) < 1:
raise ValueError('[reproduce] Cannot find any suite for given tests')
elif len(found_keys) > 1:
raise ValueError(
'[reproduce] Given tests contained by different suites')
res_key = found_keys[0]
res_task_group = copy.deepcopy(task_groups[key])
res_task_group['task_ids'] = reproduce
return {res_key: res_task_group}
def run_loop(self, task_queue, result_queue):
""" called from 'run_all' """
while True:
task_id = self.task_get(task_queue)
# None is 'stop worker' marker
if task_id is None:
color_log('Worker "%s" exhausted task queue; '
'stopping the server...\n' % self.name,
schema='test_var')
self.stop_worker(task_queue, result_queue)
break
short_status = self.run_task(task_id)
result_queue.put(self.wrap_result(task_id, short_status))
if not lib.Options().args.is_force and short_status == 'fail':
color_stdout(
'Worker "%s" got failed test; stopping the server...\n' %
self.name,
schema='test_var')
raise VoluntaryStopException()
if self.sigterm_received:
color_stdout('Worker "%s" got signal to terminate; '
'stopping the server...\n' % self.name,
schema='test_var')
raise VoluntaryStopException()
self.task_done(task_queue)
def collect_tests(self):
if self.tests_are_collected:
return self.tests
if self.ini['core'] == 'tarantool':
TarantoolServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'app':
AppServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'unittest':
UnittestServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'stress':
# parallel tests are not supported and disabled for now
self.tests = []
self.tests_are_collected = True
return self.tests
else:
raise ValueError('Cannot collect tests of unknown type')
if not lib.Options().args.reproduce:
color_stdout("Collecting tests in ", schema='ts_text')
color_stdout('%s (Found %s tests)' % (repr(
self.suite_path).ljust(16), str(len(self.tests)).ljust(3)),
schema='path')
color_stdout(": ",
self.ini["description"],
".\n",
schema='ts_text')
self.tests_are_collected = True
return self.tests
def main_consistent():
color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text')
failed_test_ids = []
try:
main_loop_consistent(failed_test_ids)
except KeyboardInterrupt:
color_stdout('[Main loop] Caught keyboard interrupt\n',
schema='test_var')
except RuntimeError as e:
color_stdout("\nFatal error: %s. Execution aborted.\n" % e,
schema='error')
if lib.Options().args.gdb:
time.sleep(100)
return -1
if failed_test_ids and lib.Options().args.is_force:
color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids),
schema='error')
for test_id in failed_test_ids:
color_stdout("----- %s\n" % str(test_id), schema='info')
return (-1 if failed_test_ids else 0)
except RuntimeError as e:
color_stdout("\nFatal error: %s. Execution aborted.\n" % e,
schema='error')
if lib.Options().args.gdb:
time.sleep(100)
return -1
if failed_test_ids and lib.Options().args.is_force:
color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids),
schema='error')
for test_id in failed_test_ids:
color_stdout("----- %s\n" % str(test_id), schema='info')
return (-1 if failed_test_ids else 0)
if __name__ == "__main__":
# don't sure why, but it values 1 or 2 gives 1.5x speedup for parallel
# test-run (and almost doesn't affect consistent test-run)
os.environ['OMP_NUM_THREADS'] = '2'
status = 0
force_parallel = bool(lib.Options().args.reproduce)
if not force_parallel and lib.Options().args.jobs == -1:
status = main_consistent()
else:
status = main_parallel()
exit(status)
def run(self, server):
""" Execute the test assuming it's a python program. If the test
aborts, print its output to stdout, and raise an exception. Else,
comprare result and reject files. If there is a difference, print
it to stdout.
Returns short status of the test as a string: 'skip', 'pass',
'new', or 'fail'. There is also one possible value for
short_status, 'disabled', but it returned in the caller,
TestSuite.run_test().
"""
# Note: test was created before certain worker become known, so we need
# to update temporary result directory here as it depends on 'vardir'.
self.tmp_result = os.path.join(self.suite_ini['vardir'],
os.path.basename(self.result))
diagnostics = "unknown"
save_stdout = sys.stdout
try:
self.skip = False
if os.path.exists(self.skip_cond):
sys.stdout = FilteredStream(self.tmp_result)
stdout_fileno = sys.stdout.stream.fileno()
execfile(self.skip_cond, dict(locals(), **server.__dict__))
sys.stdout.close()
sys.stdout = save_stdout
if not self.skip:
sys.stdout = FilteredStream(self.tmp_result)
stdout_fileno = sys.stdout.stream.fileno()
self.execute(server)
sys.stdout.flush()
self.is_executed_ok = True
except TestExecutionError:
self.is_executed_ok = False
except Exception as e:
if e.__class__.__name__ == 'TarantoolStartError':
# worker should stop
raise
color_stdout('\nTest.run() received the following error:\n' +
traceback.format_exc() + '\n',
schema='error')
diagnostics = str(e)
finally:
if sys.stdout and sys.stdout != save_stdout:
sys.stdout.close()
sys.stdout = save_stdout
self.is_executed = True
sys.stdout.flush()
is_tap = False
if not self.skip:
if self.is_executed_ok and os.path.isfile(self.result):
self.is_equal_result = filecmp.cmp(self.result,
self.tmp_result)
elif self.is_executed_ok:
if lib.Options().args.is_verbose:
color_stdout('\n')
with open(self.tmp_result, 'r') as f:
color_stdout(f.read(), schema='log')
is_tap, is_ok = self.check_tap_output()
self.is_equal_result = is_ok
else:
self.is_equal_result = 1
if self.args.valgrind:
non_empty_logs = non_empty_valgrind_logs(
server.current_valgrind_logs(for_test=True))
self.is_valgrind_clean = not bool(non_empty_logs)
short_status = None
if self.skip:
short_status = 'skip'
color_stdout("[ skip ]\n", schema='test_skip')
if os.path.exists(self.tmp_result):
os.remove(self.tmp_result)
elif self.is_executed_ok and self.is_equal_result and self.is_valgrind_clean:
short_status = 'pass'
color_stdout("[ pass ]\n", schema='test_pass')
if os.path.exists(self.tmp_result):
os.remove(self.tmp_result)
elif (self.is_executed_ok and not self.is_equal_result
and not os.path.isfile(self.result)) and not is_tap:
shutil.copy(self.tmp_result, self.result)
short_status = 'new'
color_stdout("[ new ]\n", schema='test_new')
else:
shutil.copy(self.tmp_result, self.reject)
short_status = 'fail'
color_stdout("[ fail ]\n", schema='test_fail')
where = ""
if not self.is_crash_reported and not self.is_executed_ok:
self.print_diagnostics(
self.reject,
"Test failed! Last 15 lines of the result file:\n")
server.print_log(15)
where = ": test execution aborted, reason '{0}'".format(
diagnostics)
elif not self.is_crash_reported and not self.is_equal_result:
self.print_unidiff()
server.print_log(15)
where = ": wrong test output"
elif not self.is_crash_reported and not self.is_valgrind_clean:
os.remove(self.reject)
for log_file in non_empty_logs:
self.print_diagnostics(
log_file,
"Test failed! Last 10 lines of {}:\n".format(log_file))
where = ": there were warnings in the valgrind log file(s)"
return short_status
def get_reproduce_file(worker_name):
main_vardir = os.path.realpath(lib.Options().args.vardir)
reproduce_dir = os.path.join(main_vardir, 'reproduce')
return os.path.join(reproduce_dir, '%s.list.yaml' % worker_name)
