def init_listeners(self): args = Options().args watch_hang = args.no_output_timeout >= 0 and \ not args.gdb and \ not args.gdbserver and \ not args.lldb and \ not args.valgrind watch_fail = not Options().args.is_force log_output_watcher = LogOutputWatcher() self.statistics = StatisticsWatcher(log_output_watcher.get_logfile) self.artifacts = ArtifactsWatcher(log_output_watcher.get_logfile) output_watcher = OutputWatcher() self.listeners = [ self.statistics, log_output_watcher, output_watcher, self.artifacts ] if sampler.is_enabled: self.listeners.append(sampler.watcher) if watch_fail: self.fail_watcher = FailWatcher(self.terminate_all_workers) self.listeners.append(self.fail_watcher) if watch_hang: warn_timeout = 60.0 if args.long else 10.0 hang_watcher = HangWatcher(output_watcher.not_done_worker_ids, self.kill_all_workers, warn_timeout, float(args.no_output_timeout)) self.listeners.append(hang_watcher)
def find_suites(): suite_names = Options().args.suites if suite_names == []: for root, dirs, names in os.walk(os.getcwd(), followlinks=True): if "suite.ini" in names: suite_names.append(os.path.basename(root)) suites = [ TestSuite(suite_name, Options().args) for suite_name in sorted(suite_names) ] return suites
def process_result(self, obj): if isinstance(obj, WorkerDone): bufferized = self.buffer.get(obj.worker_id, '') if bufferized: OutputWatcher._write(bufferized, obj.worker_id) if obj.worker_id in self.buffer.keys(): del self.buffer[obj.worker_id] return # Skip irrelevant events. if not isinstance(obj, WorkerOutput): return # Skip color_log() events if --debug is not passed. if obj.log_only and not Options().args.debug: return # Prepend color_log() messages with a timestamp. if obj.log_only: prefix = '[{}] '.format(obj.timestamp) obj.output = prefix_each_line(prefix, obj.output) bufferized = self.buffer.get(obj.worker_id, '') if decolor(obj.output).endswith('\n'): OutputWatcher._write(bufferized + obj.output, obj.worker_id) self.buffer[obj.worker_id] = '' else: self.buffer[obj.worker_id] = bufferized + obj.output
def __init__(self): self.fds = dict() self.logdir = os.path.join(Options().args.vardir, 'log') try: os.makedirs(self.logdir) except OSError: pass
def save_artifacts(self): if not self.failed_workers: return vardir = Options().args.vardir artifacts_dir = os.path.join(vardir, 'artifacts') artifacts_log_dir = os.path.join(artifacts_dir, 'log') artifacts_reproduce_dir = os.path.join(artifacts_dir, 'reproduce') safe_makedirs(artifacts_dir) safe_makedirs(artifacts_log_dir) safe_makedirs(artifacts_reproduce_dir) for worker_name in self.failed_workers: logfile = self.get_logfile(worker_name) reproduce_file_path = get_reproduce_file(worker_name) shutil.copy( logfile, os.path.join(artifacts_log_dir, os.path.basename(logfile))) shutil.copy( reproduce_file_path, os.path.join(artifacts_reproduce_dir, os.path.basename(reproduce_file_path))) shutil.copytree(os.path.join(vardir, worker_name), os.path.join(artifacts_dir, worker_name), ignore=shutil.ignore_patterns( '*.socket-iproto', '*.socket-admin', '*.sock', '*.control')) shutil.copytree(os.path.join(vardir, 'statistics'), os.path.join(artifacts_dir, 'statistics'))
def print_statistics(self): """Print statistics and results of testing.""" # Prepare standalone subpath '<vardir>/statistics' for statistics files. stats_dir = os.path.join(Options().args.vardir, 'statistics') safe_makedirs(stats_dir) self.print_rss_summary(stats_dir) self.print_duration(stats_dir) if self.stats: color_stdout('Statistics:\n', schema='test_var') for short_status, cnt in self.stats.items(): color_stdout('* %s: %d\n' % (short_status, cnt), schema='test_var') if not self.failed_tasks: return False color_stdout('Failed tasks:\n', schema='test_var') for task_id, worker_name, result_checksum, show_reproduce_content in self.failed_tasks: logfile = self.get_logfile(worker_name) task_id_str = yaml.safe_dump(task_id, default_flow_style=True) color_stdout('- %s' % task_id_str, schema='test_var') color_stdout('# results file checksum: %s\n' % result_checksum) color_stdout('# logfile: %s\n' % logfile) reproduce_file_path = get_reproduce_file(worker_name) color_stdout('# reproduce file: %s\n' % reproduce_file_path) if show_reproduce_content: color_stdout("---\n", schema='separator') print_tail_n(reproduce_file_path) color_stdout("...\n", schema='separator') return True
def run_project(args): options = Options() options.parse(args[1:]) project = Project(options) print 'Archirtecture chosen:', project.print_arg()
def reproduce_task_groups(task_groups): """Filter provided task_groups down to the one certain group. Sort tests in this group as in the reproduce file. """ found_keys = [] reproduce = parse_reproduce_file(Options().args.reproduce) if not reproduce: raise ValueError('[reproduce] Tests list cannot be empty') for i, task_id in enumerate(reproduce): for key, task_group in task_groups.items(): if task_id in task_group['task_ids']: found_keys.append(key.rstrip('_fragile')) break if len(found_keys) != i + 1: raise ValueError('[reproduce] Cannot find test "%s"' % str(task_id)) found_keys = list(set(found_keys)) if len(found_keys) < 1: raise ValueError('[reproduce] Cannot find any suite for given tests') elif len(found_keys) > 1: raise ValueError( '[reproduce] Given tests contained by different suites') res_key = found_keys[0] res_task_group = copy.deepcopy(task_groups[key]) res_task_group['task_ids'] = reproduce return {res_key: res_task_group}
def run_test(self, test, server, inspector): """ Returns short status of the test as a string: 'skip', 'pass', 'new', 'fail', or 'disabled' and results file checksum on fail. """ test.inspector = inspector test_name = os.path.basename(test.name) full_test_name = os.path.join(self.ini['suite'], test_name) color_stdout(just_and_trim(full_test_name, 47) + ' ', schema='t_name') # for better diagnostics in case of a long-running test conf = '' if test.run_params: conf = test.conf_name color_stdout(just_and_trim(conf, 15) + ' ', schema='test_var') start_time = time.time() if self.is_test_enabled(test, conf, server): short_status, result_checksum = test.run(server) else: color_stdout("[ disabled ]\n", schema='t_name') short_status = 'disabled' result_checksum = None duration = time.time() - start_time # cleanup only if test passed or if --force mode enabled if Options().args.is_force or short_status == 'pass': inspector.cleanup_nondefault() return short_status, result_checksum, duration
def collect_tests(self): if self.tests_are_collected: return self.tests if self.ini['core'] == 'tarantool': TarantoolServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'app': AppServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'unittest': UnittestServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'stress': # parallel tests are not supported and disabled for now self.tests = [] self.tests_are_collected = True return self.tests else: raise ValueError('Cannot collect tests of unknown type') if not Options().args.reproduce: color_stdout("Collecting tests in ", schema='ts_text') color_stdout( '%s (Found %s tests)' % ( repr(self.suite_path).ljust(16), str(len(self.tests)).ljust(3) ), schema='path' ) color_stdout(": ", self.ini["description"], ".\n", schema='ts_text') self.tests_are_collected = True return self.tests
def start(args): options = Options() options.parse(args) project = Project(options) project.run()
def run(args): options = Options() options.parse(args[1:]) project = Project(options) print 'Architecture chosen:', project.print_arg() project.run_command()
def run_project(args): options = Options() options.parse(args[1:]) project = Project(options) print 'Printing date:', project.date() print 'Printing example arg:', project.print_example_arg()
def main_consistent(): color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text') failed_test_ids = [] try: main_loop_consistent(failed_test_ids) except KeyboardInterrupt: color_stdout('[Main loop] Caught keyboard interrupt\n', schema='test_var') except RuntimeError as e: color_stdout("\nFatal error: %s. Execution aborted.\n" % e, schema='error') if Options().args.gdb: time.sleep(100) return -1 if failed_test_ids and Options().args.is_force: color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids), schema='error') for test_id in failed_test_ids: color_stdout("----- %s\n" % str(test_id), schema='info') return (-1 if failed_test_ids else 0)
def process_result(self, obj): # Track tasks in progress. if isinstance(obj, WorkerCurrentTask): self.worker_current_task[obj.worker_id] = obj # Skip irrelevant events. if not isinstance(obj, WorkerOutput): return # Skip color_log() events if --debug is not passed. if obj.log_only and not Options().args.debug: return self.warned_seconds_ago = 0.0 self.inactivity = 0.0
def run_project(args): options = Options() options.parse(args[1:]) project = Survey(options) print '\n--------------------------------Printing Survey Summary --------------------------------------\n' print 'Printing date:', project.date() # print 'Printing arg:', project.print_example_arg() print '\n--------------------------------Printing Survey Question Details -----------------------------\n', project.parse_surveydata( ) print '\n--------------------------------Printing Survey Responces Details ----------------------------\n', project.parse_responces( ) print '\nSubmission Percentage: ', project.submission_percentage(), '%' print '\nSubmission Count Total: ', project.submission_count_total() print '\nAverage Rating Q1: ', project.avg_rating('ANS_Q1') print '\nAverage Rating Q2: ', project.avg_rating('ANS_Q2') print '\nAverage Rating Q3: ', project.avg_rating('ANS_Q3') print '\nAverage Rating Q4: ', project.avg_rating('ANS_Q4') print '\nAverage Rating Q5: ', project.avg_rating('ANS_Q5') print '\n--------------------------------End Survey Summary -------------------------------------------\n'
def main_loop_consistent(failed_test_ids): # find and prepare all tasks/groups, print information task_groups = get_task_groups().items() print_greetings() for name, task_group in task_groups: # print information about current test suite color_stdout("\n", '=' * 80, "\n", schema='separator') color_stdout("TEST".ljust(48), schema='t_name') color_stdout("PARAMS".ljust(16), schema='test_var') color_stdout("RESULT\n", schema='test_pass') color_stdout('-' * 75, "\n", schema='separator') task_ids = task_group['task_ids'] show_reproduce_content = task_group['show_reproduce_content'] if not task_ids: continue worker_id = 1 worker = task_group['gen_worker'](worker_id) for task_id in task_ids: short_status = worker.run_task(task_id) if short_status == 'fail': reproduce_file_path = \ get_reproduce_file(worker.name) color_stdout('Reproduce file %s\n' % reproduce_file_path, schema='error') if show_reproduce_content: color_stdout("---\n", schema='separator') print_tail_n(reproduce_file_path) color_stdout("...\n", schema='separator') failed_test_ids.append(task_id) if not Options().args.is_force: worker.stop_server(cleanup=False) return color_stdout('-' * 75, "\n", schema='separator') worker.stop_server(silent=False) color_stdout()
def main(args): options = Options() config = options.parse(args[1:]) repo = GitRepository(os.path.curdir) old_rev, new_rev, ref_name = sys.stdin.readline().split(' ') if repo.is_revision_empty(old_rev): old_rev = repo.get_default_branch_name() if repo.is_revision_empty(new_rev): # case of branch deletion exit(0) if config.php_cs_fixer_enabled: fixer = PhpCsFixer(repo, config.php_cs_fixer_executable, config.php_cs_fixer_config_path, config.php_cs_fixer_dirs_to_create) fixer.check_push(old_rev, new_rev) each_commit_checker = EachCommitChecker(config, repo) each_commit_checker.check(old_rev, new_rev)
def check_tap_output(self): """ Returns is_tap, is_ok """ try: with open(self.tmp_result, 'r') as f: content = f.read() tap = pytap13.TAP13() tap.parse(content) except (ValueError, UnicodeDecodeError) as e: color_stdout('\nTAP13 parse failed (%s).\n' % str(e), schema='error') color_stdout('\nNo result file (%s) found.\n' % self.result, schema='error') if not Options().args.update_result: msg = 'Run the test with --update-result option to write the new result file.\n' color_stdout(msg, schema='error') self.is_crash_reported = True return False, False is_ok = True for test_case in tap.tests: if test_case.result == 'ok': continue if is_ok: color_stdout('\n') color_stdout( '%s %s %s # %s %s\n' % (test_case.result, test_case.id or '', test_case.description or '-', test_case.directive or '', test_case.comment or ''), schema='error') if test_case.yaml: self.tap_parse_print_yaml(test_case.yaml) is_ok = False if not is_ok: color_stdout('Rejected result file: %s\n' % self.reject, schema='test_var') self.is_crash_reported = True return True, is_ok
import sys from lib import Options if __name__ == '__main__': options = Options() if len(sys.argv[1:]) == 0: options.help() sys.exit(0) opts = options.parse(sys.argv[1:]) if opts.input_file == None: print "Please supply an input file" sys.exit(1) else: print "At this point I'll process", opts.input_file # TODO Create an object for the processer and pass # the input file to it sys.exit(0)
def main_loop_parallel(): color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text') args = Options().args jobs = args.jobs if jobs < 1: # faster result I got was with 2 * cpu_count jobs = 2 * multiprocessing.cpu_count() if jobs > 0: color_stdout("Running in parallel with %d workers\n\n" % jobs, schema='tr_text') randomize = True color_stdout("Timeout options:\n", schema='tr_text') color_stdout('-' * 19, "\n", schema='separator') color_stdout("REPLICATION_SYNC_TIMEOUT:".ljust(26) + "{}\n".format(args.replication_sync_timeout), schema='tr_text') color_stdout("TEST_TIMEOUT:".ljust(26) + "{}\n".format(args.test_timeout), schema='tr_text') color_stdout("NO_OUTPUT_TIMEOUT:".ljust(26) + "{}\n".format(args.no_output_timeout), schema='tr_text') color_stdout("\n", schema='tr_text') task_groups = get_task_groups() if Options().args.reproduce: task_groups = reproduce_task_groups(task_groups) jobs = 1 randomize = False dispatcher = Dispatcher(task_groups, jobs, randomize) dispatcher.start() print_greetings() color_stdout("\n", '=' * 86, "\n", schema='separator') color_stdout("WORKR".ljust(6), schema='t_name') color_stdout("TEST".ljust(48), schema='t_name') color_stdout("PARAMS".ljust(16), schema='test_var') color_stdout("RESULT\n", schema='test_pass') color_stdout('-' * 81, "\n", schema='separator') try: is_force = Options().args.is_force dispatcher.wait() dispatcher.wait_processes() color_stdout('-' * 81, "\n", schema='separator') has_failed = dispatcher.statistics.print_statistics() has_undone = dispatcher.report_undone( verbose=bool(is_force or not has_failed)) if has_failed: dispatcher.artifacts.save_artifacts() return EXIT_FAILED_TEST if has_undone: return EXIT_NOTDONE_TEST except KeyboardInterrupt: color_stdout('-' * 81, "\n", schema='separator') dispatcher.statistics.print_statistics() dispatcher.report_undone(verbose=False) raise except HangError: color_stdout('-' * 81, "\n", schema='separator') dispatcher.statistics.print_statistics() dispatcher.report_undone(verbose=False) return EXIT_HANG return EXIT_SUCCESS
def run_loop(self, task_queue, result_queue): """ called from 'run_all' """ while True: task_id = self.task_get(task_queue) # None is 'stop worker' marker if task_id is None: color_log('Worker "%s" exhausted task queue; ' 'stopping the server...\n' % self.name, schema='test_var') self.stop_worker(task_queue, result_queue) break short_status = None result_checksum = None duration = 0.0 result_queue.put(self.current_task(task_id)) testname = os.path.basename(task_id[0]) fragile_checksums = self.suite.get_test_fragile_checksums(testname) retries_left = self.suite.fragile_retries() # let's run till short_status became 'pass' while short_status != 'pass' and retries_left >= 0: self.restart_server() # print message only after some fails occurred if short_status == 'fail': color_stdout('Test "%s", conf: "%s"\n' '\tfrom "fragile" list failed with results' ' file checksum: "%s", rerunning ...\n' % (task_id[0], task_id[1], result_checksum), schema='error') # run task and save the result to short_status short_status, result_checksum, duration = self.run_task( task_id) # check if the results file checksum set on fail and if # the newly created results file is known by checksum if not result_checksum or (result_checksum not in fragile_checksums): break retries_left = retries_left - 1 result_queue.put( self.wrap_result(task_id, short_status, result_checksum, duration)) if short_status == 'fail': if Options().args.is_force: self.restart_server() color_stdout( 'Worker "%s" got failed test; restarted the server\n' % self.name, schema='test_var') else: color_stdout( 'Worker "%s" got failed test; stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() if self.sigterm_received: color_stdout('Worker "%s" got signal to terminate; ' 'stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() self.task_done(task_queue)
# So we hack the open() builtin. # # https://stackoverflow.com/a/53347548/1598057 if PY3 and sys.version_info[0:2] < (3, 7): std_open = __builtins__.open def open_as_utf8(*args, **kwargs): if len(args) >= 2: mode = args[1] else: mode = kwargs.get('mode', '') if 'b' not in mode: kwargs.setdefault('encoding', 'utf-8') return std_open(*args, **kwargs) __builtins__.open = open_as_utf8 # don't sure why, but it values 1 or 2 gives 1.5x speedup for parallel # test-run (and almost doesn't affect consistent test-run) os.environ['OMP_NUM_THREADS'] = '2' status = 0 force_parallel = bool(Options().args.reproduce) if not force_parallel and Options().args.jobs == -1: status = main_consistent() else: status = main_parallel() exit(status)
def get_reproduce_file(worker_name): main_vardir = os.path.realpath(Options().args.vardir) reproduce_dir = os.path.join(main_vardir, 'reproduce') return os.path.join(reproduce_dir, '%s.list.yaml' % worker_name)
def run(args): opts = Options() opts.parse(args[1:]) w = Project(opts) print w.welcome()
def run(self, server): """ Execute the test assuming it's a python program. If the test aborts, print its output to stdout, and raise an exception. Else, comprare result and reject files. If there is a difference, print it to stdout. Returns short status of the test as a string: 'skip', 'pass', 'new', 'updated' or 'fail' and results file checksum on fail. There is also one possible value for short_status, 'disabled', but it returned in the caller, TestSuite.run_test(). """ # Note: test was created before certain worker become known, so we need # to update temporary result directory here as it depends on 'vardir'. self.tmp_result = os.path.join(self.suite_ini['vardir'], os.path.basename(self.result)) diagnostics = "unknown" save_stdout = sys.stdout try: self.skip = False if os.path.exists(self.skip_cond): sys.stdout = FilteredStream(self.tmp_result) stdout_fileno = sys.stdout.stream.fileno() new_globals = dict(locals(), **server.__dict__) with open(self.skip_cond, 'r') as f: code = compile(f.read(), self.skip_cond, 'exec') exec(code, new_globals) sys.stdout.close() sys.stdout = save_stdout if not self.skip: sys.stdout = FilteredStream(self.tmp_result) stdout_fileno = sys.stdout.stream.fileno() self.execute(server) sys.stdout.flush() self.is_executed_ok = True except TestExecutionError: self.is_executed_ok = False except Exception as e: if e.__class__.__name__ == 'TarantoolStartError': # worker should stop raise color_stdout('\nTest.run() received the following error:\n' '{0}\n'.format(traceback.format_exc()), schema='error') diagnostics = str(e) finally: if sys.stdout and sys.stdout != save_stdout: sys.stdout.close() sys.stdout = save_stdout self.is_executed = True sys.stdout.flush() is_tap = False if not self.skip: if not os.path.exists(self.tmp_result): self.is_executed_ok = False self.is_equal_result = False elif self.is_executed_ok and os.path.isfile(self.result): self.is_equal_result = filecmp.cmp(self.result, self.tmp_result) elif self.is_executed_ok: if Options().args.is_verbose: color_stdout('\n') with open(self.tmp_result, 'r') as f: color_stdout(f.read(), schema='log') is_tap, is_ok = self.check_tap_output() self.is_equal_result = is_ok else: self.is_equal_result = 1 if self.args.valgrind: non_empty_logs = non_empty_valgrind_logs( server.current_valgrind_logs(for_test=True)) self.is_valgrind_clean = not bool(non_empty_logs) short_status = None result_checksum = None if self.skip: short_status = 'skip' color_stdout("[ skip ]\n", schema='test_skip') if os.path.exists(self.tmp_result): os.remove(self.tmp_result) elif (self.is_executed_ok and self.is_equal_result and self.is_valgrind_clean): short_status = 'pass' color_stdout("[ pass ]\n", schema='test_pass') if os.path.exists(self.tmp_result): os.remove(self.tmp_result) elif (self.is_executed_ok and not self.is_equal_result and not os.path.isfile(self.result) and not is_tap and Options().args.update_result): shutil.copy(self.tmp_result, self.result) short_status = 'new' color_stdout("[ new ]\n", schema='test_new') elif (self.is_executed_ok and not self.is_equal_result and os.path.isfile(self.result) and not is_tap and Options().args.update_result): shutil.copy(self.tmp_result, self.result) short_status = 'updated' color_stdout("[ updated ]\n", schema='test_new') else: has_result = os.path.exists(self.tmp_result) if has_result: safe_makedirs(self.var_suite_path) shutil.copy(self.tmp_result, self.reject) with open(self.tmp_result, mode='rb') as result_file: result_checksum = md5(result_file.read()).hexdigest() short_status = 'fail' color_stdout("[ fail ]\n", schema='test_fail') where = "" if not self.is_crash_reported and not has_result: color_stdout('\nCannot open %s\n' % self.tmp_result, schema='error') elif not self.is_crash_reported and not self.is_executed_ok: self.print_diagnostics( self.reject, "Test failed! Output from reject file " "{0}:\n".format(self.reject)) server.print_log(15) where = ": test execution aborted, reason " \ "'{0}'".format(diagnostics) elif not self.is_crash_reported and not self.is_equal_result: self.print_unidiff() server.print_log(15) where = ": wrong test output" elif not self.is_crash_reported and not self.is_valgrind_clean: os.remove(self.reject) for log_file in non_empty_logs: self.print_diagnostics( log_file, "Test failed! Output from log file " "{0}:\n".format(log_file)) where = ": there were warnings in the valgrind log file(s)" return short_status, result_checksum
def setUp(self): self.options = Options()
def _default_options(self): options = Options() opts, args = options.parse() return opts