def init_listeners(self): args = lib.Options().args watch_hang = args.no_output_timeout >= 0 and \ not args.gdb and \ not args.gdbserver and \ not args.lldb and \ not args.valgrind and \ not args.long watch_fail = not lib.Options().args.is_force log_output_watcher = listeners.LogOutputWatcher() self.statistics = listeners.StatisticsWatcher( log_output_watcher.get_logfile) output_watcher = listeners.OutputWatcher() self.listeners = [self.statistics, log_output_watcher, output_watcher] if watch_fail: self.fail_watcher = listeners.FailWatcher( self.terminate_all_workers) self.listeners.append(self.fail_watcher) if watch_hang: warn_timeout = 10.0 no_output_timeout = float(args.no_output_timeout or 120) hang_watcher = listeners.HangWatcher( output_watcher.not_done_worker_ids, self.kill_all_workers, warn_timeout, no_output_timeout) self.listeners.append(hang_watcher)
def main_loop_parallel(): color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text') jobs = lib.Options().args.jobs if jobs < 1: # faster result I got was with 2 * cpu_count jobs = 2 * multiprocessing.cpu_count() if jobs > 0: color_stdout("Running in parallel with %d workers\n\n" % jobs, schema='tr_text') randomize = True task_groups = lib.worker.get_task_groups() if lib.Options().args.reproduce: task_groups = lib.worker.reproduce_task_groups(task_groups) jobs = 1 randomize = False dispatcher = Dispatcher(task_groups, jobs, randomize) dispatcher.start() lib.worker.print_greetings() color_stdout("\n", '=' * 86, "\n", schema='separator') color_stdout("WORKR".ljust(6), schema='t_name') color_stdout("TEST".ljust(48), schema='t_name') color_stdout("PARAMS".ljust(16), schema='test_var') color_stdout("RESULT\n", schema='test_pass') color_stdout('-' * 81, "\n", schema='separator') try: is_force = lib.Options().args.is_force dispatcher.wait() dispatcher.wait_processes() color_stdout('-' * 81, "\n", schema='separator') has_failed = dispatcher.statistics.print_statistics() has_undone = dispatcher.report_undone( verbose=bool(is_force or not has_failed)) if has_failed: return EXIT_FAILED_TEST if has_undone: return EXIT_NOTDONE_TEST except KeyboardInterrupt: color_stdout('-' * 81, "\n", schema='separator') dispatcher.statistics.print_statistics() dispatcher.report_undone(verbose=False) raise except HangError: color_stdout('-' * 81, "\n", schema='separator') dispatcher.statistics.print_statistics() dispatcher.report_undone(verbose=False) return EXIT_HANG return EXIT_SUCCESS
def find_suites(): suite_names = lib.Options().args.suites if suite_names == []: for root, dirs, names in os.walk(os.getcwd(), followlinks=True): if "suite.ini" in names: suite_names.append(os.path.basename(root)) suites = [ TestSuite(suite_name, lib.Options().args) for suite_name in sorted(suite_names) ] return suites
def __init__(self): self.fds = dict() self.logdir = os.path.join(lib.Options().args.vardir, 'log') try: os.makedirs(self.logdir) except OSError: pass
def run_test(self, test, server, inspector): """ Returns short status of the test as a string: 'skip', 'pass', 'new', 'fail', or 'disabled'. """ test.inspector = inspector color_stdout(os.path.join(self.ini['suite'], os.path.basename(test.name)).ljust(48), schema='t_name') # for better diagnostics in case of a long-running test conf = '' if test.run_params: conf = test.conf_name color_stdout(conf.ljust(16), schema='test_var') test_name = os.path.basename(test.name) if self.is_test_enabled(test, conf, server): short_status = test.run(server) else: color_stdout("[ disabled ]\n", schema='t_name') short_status = 'disabled' # cleanup only if test passed or if --force mode enabled if lib.Options().args.is_force or short_status == 'pass': inspector.cleanup_nondefault() return short_status
def main_loop_consistent(failed_test_ids): # find and prepare all tasks/groups, print information task_groups = lib.worker.get_task_groups().items() lib.worker.print_greetings() for name, task_group in task_groups: # print information about current test suite color_stdout("\n", '=' * 80, "\n", schema='separator') color_stdout("TEST".ljust(48), schema='t_name') color_stdout("PARAMS".ljust(16), schema='test_var') color_stdout("RESULT\n", schema='test_pass') color_stdout('-' * 75, "\n", schema='separator') task_ids = task_group['task_ids'] if not task_ids: continue worker_id = 1 worker = task_group['gen_worker'](worker_id) for task_id in task_ids: short_status = worker.run_task(task_id) if short_status == 'fail': failed_test_ids.append(task_id) if not lib.Options().args.is_force: worker.stop_server(cleanup=False) return color_stdout('-' * 75, "\n", schema='separator') worker.stop_server(silent=False) color_stdout()
def reproduce_task_groups(task_groups): """Filter provided task_groups down to the one certain group. Sort tests in this group as in the reproduce file. """ found_keys = [] reproduce = parse_reproduce_file(lib.Options().args.reproduce) if not reproduce: raise ValueError('[reproduce] Tests list cannot be empty') for i, task_id in enumerate(reproduce): for key, task_group in task_groups.items(): if task_id in task_group['task_ids']: found_keys.append(key) break if len(found_keys) != i + 1: raise ValueError('[reproduce] Cannot find test "%s"' % str(task_id)) found_keys = list(set(found_keys)) if len(found_keys) < 1: raise ValueError('[reproduce] Cannot find any suite for given tests') elif len(found_keys) > 1: raise ValueError( '[reproduce] Given tests contained by different suites') res_key = found_keys[0] res_task_group = copy.deepcopy(task_groups[key]) res_task_group['task_ids'] = reproduce return {res_key: res_task_group}
def run_loop(self, task_queue, result_queue): """ called from 'run_all' """ while True: task_id = self.task_get(task_queue) # None is 'stop worker' marker if task_id is None: color_log('Worker "%s" exhausted task queue; ' 'stopping the server...\n' % self.name, schema='test_var') self.stop_worker(task_queue, result_queue) break short_status = self.run_task(task_id) result_queue.put(self.wrap_result(task_id, short_status)) if not lib.Options().args.is_force and short_status == 'fail': color_stdout( 'Worker "%s" got failed test; stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() if self.sigterm_received: color_stdout('Worker "%s" got signal to terminate; ' 'stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() self.task_done(task_queue)
def collect_tests(self): if self.tests_are_collected: return self.tests if self.ini['core'] == 'tarantool': TarantoolServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'app': AppServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'unittest': UnittestServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'stress': # parallel tests are not supported and disabled for now self.tests = [] self.tests_are_collected = True return self.tests else: raise ValueError('Cannot collect tests of unknown type') if not lib.Options().args.reproduce: color_stdout("Collecting tests in ", schema='ts_text') color_stdout('%s (Found %s tests)' % (repr( self.suite_path).ljust(16), str(len(self.tests)).ljust(3)), schema='path') color_stdout(": ", self.ini["description"], ".\n", schema='ts_text') self.tests_are_collected = True return self.tests
def main_consistent(): color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text') failed_test_ids = [] try: main_loop_consistent(failed_test_ids) except KeyboardInterrupt: color_stdout('[Main loop] Caught keyboard interrupt\n', schema='test_var') except RuntimeError as e: color_stdout("\nFatal error: %s. Execution aborted.\n" % e, schema='error') if lib.Options().args.gdb: time.sleep(100) return -1 if failed_test_ids and lib.Options().args.is_force: color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids), schema='error') for test_id in failed_test_ids: color_stdout("----- %s\n" % str(test_id), schema='info') return (-1 if failed_test_ids else 0)
except RuntimeError as e: color_stdout("\nFatal error: %s. Execution aborted.\n" % e, schema='error') if lib.Options().args.gdb: time.sleep(100) return -1 if failed_test_ids and lib.Options().args.is_force: color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids), schema='error') for test_id in failed_test_ids: color_stdout("----- %s\n" % str(test_id), schema='info') return (-1 if failed_test_ids else 0) if __name__ == "__main__": # don't sure why, but it values 1 or 2 gives 1.5x speedup for parallel # test-run (and almost doesn't affect consistent test-run) os.environ['OMP_NUM_THREADS'] = '2' status = 0 force_parallel = bool(lib.Options().args.reproduce) if not force_parallel and lib.Options().args.jobs == -1: status = main_consistent() else: status = main_parallel() exit(status)
def run(self, server): """ Execute the test assuming it's a python program. If the test aborts, print its output to stdout, and raise an exception. Else, comprare result and reject files. If there is a difference, print it to stdout. Returns short status of the test as a string: 'skip', 'pass', 'new', or 'fail'. There is also one possible value for short_status, 'disabled', but it returned in the caller, TestSuite.run_test(). """ # Note: test was created before certain worker become known, so we need # to update temporary result directory here as it depends on 'vardir'. self.tmp_result = os.path.join(self.suite_ini['vardir'], os.path.basename(self.result)) diagnostics = "unknown" save_stdout = sys.stdout try: self.skip = False if os.path.exists(self.skip_cond): sys.stdout = FilteredStream(self.tmp_result) stdout_fileno = sys.stdout.stream.fileno() execfile(self.skip_cond, dict(locals(), **server.__dict__)) sys.stdout.close() sys.stdout = save_stdout if not self.skip: sys.stdout = FilteredStream(self.tmp_result) stdout_fileno = sys.stdout.stream.fileno() self.execute(server) sys.stdout.flush() self.is_executed_ok = True except TestExecutionError: self.is_executed_ok = False except Exception as e: if e.__class__.__name__ == 'TarantoolStartError': # worker should stop raise color_stdout('\nTest.run() received the following error:\n' + traceback.format_exc() + '\n', schema='error') diagnostics = str(e) finally: if sys.stdout and sys.stdout != save_stdout: sys.stdout.close() sys.stdout = save_stdout self.is_executed = True sys.stdout.flush() is_tap = False if not self.skip: if self.is_executed_ok and os.path.isfile(self.result): self.is_equal_result = filecmp.cmp(self.result, self.tmp_result) elif self.is_executed_ok: if lib.Options().args.is_verbose: color_stdout('\n') with open(self.tmp_result, 'r') as f: color_stdout(f.read(), schema='log') is_tap, is_ok = self.check_tap_output() self.is_equal_result = is_ok else: self.is_equal_result = 1 if self.args.valgrind: non_empty_logs = non_empty_valgrind_logs( server.current_valgrind_logs(for_test=True)) self.is_valgrind_clean = not bool(non_empty_logs) short_status = None if self.skip: short_status = 'skip' color_stdout("[ skip ]\n", schema='test_skip') if os.path.exists(self.tmp_result): os.remove(self.tmp_result) elif self.is_executed_ok and self.is_equal_result and self.is_valgrind_clean: short_status = 'pass' color_stdout("[ pass ]\n", schema='test_pass') if os.path.exists(self.tmp_result): os.remove(self.tmp_result) elif (self.is_executed_ok and not self.is_equal_result and not os.path.isfile(self.result)) and not is_tap: shutil.copy(self.tmp_result, self.result) short_status = 'new' color_stdout("[ new ]\n", schema='test_new') else: shutil.copy(self.tmp_result, self.reject) short_status = 'fail' color_stdout("[ fail ]\n", schema='test_fail') where = "" if not self.is_crash_reported and not self.is_executed_ok: self.print_diagnostics( self.reject, "Test failed! Last 15 lines of the result file:\n") server.print_log(15) where = ": test execution aborted, reason '{0}'".format( diagnostics) elif not self.is_crash_reported and not self.is_equal_result: self.print_unidiff() server.print_log(15) where = ": wrong test output" elif not self.is_crash_reported and not self.is_valgrind_clean: os.remove(self.reject) for log_file in non_empty_logs: self.print_diagnostics( log_file, "Test failed! Last 10 lines of {}:\n".format(log_file)) where = ": there were warnings in the valgrind log file(s)" return short_status
def get_reproduce_file(worker_name): main_vardir = os.path.realpath(lib.Options().args.vardir) reproduce_dir = os.path.join(main_vardir, 'reproduce') return os.path.join(reproduce_dir, '%s.list.yaml' % worker_name)