def run_loop(self, task_queue, result_queue): """ called from 'run_all' """ while True: task_id = self.task_get(task_queue) # None is 'stop worker' marker if task_id is None: color_log('Worker "%s" exhausted task queue; ' 'stopping the server...\n' % self.name, schema='test_var') self.stop_worker(task_queue, result_queue) break result_queue.put(self.current_task(task_id)) short_status = self.run_task(task_id) result_queue.put(self.wrap_result(task_id, short_status)) if not lib.Options().args.is_force and short_status == 'fail': color_stdout( 'Worker "%s" got failed test; stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() if self.sigterm_received: color_stdout('Worker "%s" got signal to terminate; ' 'stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() self.task_done(task_queue)
def print_unidiff(self): """Print a unified diff between .test and .result files. Used to establish the cause of a failure when .test differs from .result.""" color_stdout("\nTest failed! Result content mismatch:\n", schema='error') utils_print_unidiff(self.result, self.reject)
def print_log(self, lines=None): msg = '\n{prefix} of Tarantool Log file [Instance "{instance}"][{logfile}]:\n'.format( prefix="Last {0} lines".format(lines) if lines else "Output", instance=self.name, logfile=self.logfile or 'null') color_stdout(msg, schema='error') if os.path.exists(self.logfile): print_tail_n(self.logfile, lines) else: color_stdout(" Can't find log:\n", schema='error')
def main_parallel(): res = EXIT_UNKNOWN_ERROR try: res = main_loop_parallel() except KeyboardInterrupt: color_stdout('\n[Main process] Caught keyboard interrupt\n', schema='test_var') res = EXIT_INTERRUPTED return res
def process_result(self, obj): if not isinstance(obj, WorkerTaskResult): return if obj.short_status == 'fail': color_stdout('[Main process] Got failed test; ' 'gently terminate all workers...\n', schema='test_var') self.got_fail = True self.terminate_all_workers()
def warn_unix_socket(path): real_path = os.path.realpath(path) if len(real_path) <= UNIX_SOCKET_LEN_LIMIT or \ real_path in warn_unix_socket.warned: return color_stdout( '\nWARGING: unix socket\'s "%s" path has length %d symbols that is ' 'longer than %d. That likely will cause failing of tests.\n' % (real_path, len(real_path), UNIX_SOCKET_LEN_LIMIT), schema='error') warn_unix_socket.warned.add(real_path)
def print_log(self, lines=None): msg = ('\n{prefix} of Tarantool Log file [Instance "{instance}"]' + '[{logfile}]:\n').format(prefix="Last {0} lines".format(lines) if lines else "Output", instance=self.name, logfile=self.logfile or 'null') color_stdout(msg, schema='error') if os.path.exists(self.logfile): print_tail_n(self.logfile, lines) else: color_stdout(" Can't find log:\n", schema='error')
def process_result(self, obj): if not isinstance(obj, WorkerTaskResult): return if obj.short_status == 'fail': color_stdout( '[Main process] Got failed test; ' 'gently terminate all workers...\n', schema='test_var') self.got_fail = True self.terminate_all_workers()
def warn_unix_sockets_at_start(vardir): max_unix_socket_rel = '???_replication/autobootstrap_guest3.control' real_vardir = os.path.realpath(vardir) max_unix_socket_abs = os.path.join(real_vardir, max_unix_socket_rel) max_unix_socket_real = os.path.realpath(max_unix_socket_abs) if len(max_unix_socket_real) > UNIX_SOCKET_LEN_LIMIT: color_stdout( 'WARGING: unix sockets can become longer than %d symbols:\n' % UNIX_SOCKET_LEN_LIMIT, schema='error') color_stdout('WARNING: for example: "%s" has length %d\n' % (max_unix_socket_real, len(max_unix_socket_real)), schema='error')
def kill_old_server(self, silent=True): pid = self.read_pidfile() if pid == -1: return False if not silent: color_stdout( ' Found old server, pid {0}, killing ...'.format(pid), schema='info') try: os.kill(pid, signal.SIGTERM) except OSError: pass self.wait_until_stopped(pid) return True
def check(self): """Check the arguments for correctness.""" check_error = False conflict_options = ('valgrind', 'gdb', 'lldb', 'strace') for op1, op2 in product(conflict_options, repeat=2): if op1 != op2 and getattr(self, op1, '') and \ getattr(self, op2, ''): format_str = "Error: option --{} is not compatible \ with option --{}" color_stdout(format_str.format(op1, op2), schema='error') check_error = True if check_error: exit(-1)
def check_tap_output(self): """ Returns is_tap, is_ok """ if not os.path.isfile(self.tmp_result): color_strout('\nCannot find %s\n' % self.tmp_result, schema='error') self.is_crash_reported = True return False with open(self.tmp_result, 'r') as f: content = f.read() tap = pytap13.TAP13() try: tap.parse(content) except ValueError as e: color_stdout('\nTAP13 parse failed: %s\n' % str(e), schema='error') self.is_crash_reported = True return False, False is_ok = True for test_case in tap.tests: if test_case.result == 'ok': continue if is_ok: color_stdout('\n') color_stdout( '%s %s %s # %s %s\n' % (test_case.result, test_case.id or '', test_case.description or '-', test_case.directive or '', test_case.comment or ''), schema='error') if test_case.yaml: self.tap_parse_print_yaml(test_case.yaml) is_ok = False if not is_ok: color_stdout('Rejected result file: %s\n' % self.reject, schema='test_var') self.is_crash_reported = True return True, is_ok
def parse_reproduce_file(filepath): reproduce = [] if not filepath: return reproduce try: with open(filepath, 'r') as f: for task_id in yaml.safe_load(f): task_name, task_conf = task_id reproduce.append((task_name, task_conf)) except IOError: color_stdout('Cannot read "%s" passed as --reproduce argument\n' % filepath, schema='error') exit(1) return reproduce
def run_all(self, task_queue, result_queue): if not self.initialized: self.flush_all_tasks(task_queue, result_queue) result_queue.put(self.done_marker()) return try: self.run_loop(task_queue, result_queue) except (KeyboardInterrupt, Exception) as e: if not isinstance(e, KeyboardInterrupt) and \ not isinstance(e, VoluntaryStopException): color_stdout('Exception: %s\n' % e, schema='error') self.stop_worker(task_queue, result_queue, cleanup=False) result_queue.put(self.done_marker())
def parse_reproduce_file(filepath): reproduce = [] if not filepath: return reproduce try: with open(filepath, 'r') as f: for task_id in yaml.load(f): task_name, task_conf = task_id reproduce.append((task_name, task_conf)) except IOError: color_stdout('Cannot read "%s" passed as --reproduce argument\n' % filepath, schema='error') exit(1) return reproduce
def check_for_dead_processes(self): for pid in self.pids: exited = False try: os.waitpid(pid, os.WNOHANG) except OSError: exited = True if exited: worker_id = self.pid_to_worker_id[pid] color_stdout( "[Main process] Worker %d don't reported work " "done using results queue, but the corresponding " "process seems dead. Removing it from Dispatcher.\n" % worker_id, schema='test_var') self.del_worker(worker_id)
def process_file(filepath): fh = None try: fh = open(filepath, 'r') lines = fh.readlines() ctime = time.ctime(os.stat(filepath).st_mtime) except Exception: if not os.path.exists(filepath): color_stdout('[File does not exists: {}]'.format(filepath), schema='error') lines = [] ctime = time.ctime() if fh: fh.close() return lines, ctime
def print_exe(cls): color_stdout('Tarantool server information\n', schema='info') if cls.binary: color_stdout(' | Found executable at {}\n'.format(cls.binary)) if cls.ctl_path: color_stdout(' | Found tarantoolctl at {}\n'.format(cls.ctl_path)) color_stdout('\n' + prefix_each_line(' | ', cls.version()) + '\n', schema='version')
def save_join(green_obj, timeout=None): """ Gevent join wrapper for test-run stop-on-crash/stop-on-timeout feature """ try: green_obj.get(timeout=timeout) except Timeout: color_stdout("Test timeout of %d secs reached\t" % timeout, schema='error') # We should kill the greenlet that writes to a temporary # result file. If the same test is run several times (e.g. # on different configurations), this greenlet may wake up # and write to the temporary result file of the new run of # the test. green_obj.kill() except GreenletExit: pass
def print_unidiff(self): """Print a unified diff between .test and .result files. Used to establish the cause of a failure when .test differs from .result.""" color_stdout("\nTest failed! Result content mismatch:\n", schema='error') with open(self.result, "r") as result: with open(self.reject, "r") as reject: result_time = time.ctime(os.stat(self.result).st_mtime) reject_time = time.ctime(os.stat(self.reject).st_mtime) diff = difflib.unified_diff(result.readlines(), reject.readlines(), self.result, self.reject, result_time, reject_time) color_stdout.writeout_unidiff(diff)
def check_libs(): deps = [('msgpack', 'msgpack-python'), ('tarantool', 'tarantool-python')] base_path = os.path.dirname(os.path.abspath(__file__)) for (mod_name, mod_dir) in deps: mod_path = os.path.join(base_path, mod_dir) if mod_path not in sys.path: sys.path = [mod_path] + sys.path for (mod_name, _mod_dir) in deps: try: __import__(mod_name) except ImportError as e: color_stdout("\n\nNo %s library found\n" % mod_name, schema='error') print(e) sys.exit(1)
def run_task(self, task_id): if not self.initialized: return self.done_marker() try: task = self.find_task(task_id) with open(self.reproduce_file, 'a') as f: f.write('- ' + yaml.safe_dump(task.id)) short_status = self.suite.run_test( task, self.server, self.inspector) except KeyboardInterrupt: self.report_keyboard_interrupt() raise except Exception: color_stdout( '\nWorker "%s" received the following error; stopping...\n' % self.name + traceback.format_exc() + '\n', schema='error') raise return short_status
def kill_old_server(self, silent=True): pid = self.read_pidfile() if pid == -1: return False if not silent: color_stdout( ' Found old server, pid {0}, killing ...'.format(pid), schema='info' ) else: color_log(' Found old server, pid {0}, killing ...'.format(pid), schema='info') try: os.kill(pid, signal.SIGTERM) except OSError: pass self.wait_until_stopped(pid) return True
def run_task(self, task_id): if not self.initialized: return self.done_marker() try: task = self.find_task(task_id) with open(self.reproduce_file, 'a') as f: f.write('- ' + yaml.safe_dump(task.id)) short_status = self.suite.run_test(task, self.server, self.inspector) except KeyboardInterrupt: self.report_keyboard_interrupt() raise except Exception as e: color_stdout( '\nWorker "%s" received the following error; stopping...\n' % self.name + traceback.format_exc() + '\n', schema='error') raise return short_status
def check_libs(): deps = [ ('msgpack', 'msgpack-python'), ('tarantool', 'tarantool-python') ] base_path = os.path.dirname(os.path.abspath(__file__)) for (mod_name, mod_dir) in deps: mod_path = os.path.join(base_path, mod_dir) if mod_path not in sys.path: sys.path = [mod_path] + sys.path for (mod_name, _mod_dir) in deps: try: __import__(mod_name) except ImportError as e: color_stdout("\n\nNo %s library found\n" % mod_name, schema='error') print(e) sys.exit(1)
def process_timeout(self, delta_seconds): self.warned_seconds_ago += delta_seconds self.inactivity += delta_seconds worker_ids = self.get_not_done_worker_ids() if self.warned_seconds_ago < self.warn_timeout: return is_warning = self.inactivity < self.kill_timeout color_schema = 'test_var' if is_warning else 'error' color_stdout( "No output during {0.inactivity:.0f} seconds. " "Will abort after {0.kill_timeout:.0f} seconds without output. " "List of workers not reporting the status:\n".format(self), schema=color_schema) hung_tasks = [ task for worker_id, task in self.worker_current_task.items() if worker_id in worker_ids ] for task in hung_tasks: result_file = task.task_tmp_result result_file_summary = '(no result file {})'.format(result_file) if os.path.exists(result_file): with open(result_file, 'r') as f: lines = sum(1 for _ in f) result_file_summary = 'at {}:{}'.format(result_file, lines) color_stdout('- {} [{}, {}] {}\n'.format(task.worker_name, task.task_name, task.task_param, result_file_summary), schema=color_schema) self.warned_seconds_ago = 0.0 if is_warning: return for task in hung_tasks: color_stdout("Test hung! Result content mismatch:\n", schema='error') print_unidiff(task.task_result, task.task_tmp_result) color_stdout( '\n[Main process] No output from workers. ' 'It seems that we hang. Send SIGKILL to workers; ' 'exiting...\n', schema='error') self.kill_all_workers() raise HangError()
def stop(self, silent=True, signal=signal.SIGTERM): """ Kill tarantool server using specified signal (SIGTERM by default) signal - a number of a signal """ if self._start_against_running: color_log('Server [%s] start against running ...\n', schema='test_var') return if self.status != 'started': if not silent: raise Exception('Server is not started') else: color_log( 'Server [{0.name}] is not started ' '(status:{0.status}) ...\n'.format(self), schema='test_var' ) return if not silent: color_stdout('Stopping the server ...\n', schema='serv_text') else: color_log('Stopping the server ...\n', schema='serv_text') # kill only if process is alive if self.process is not None and self.process.returncode is None: color_log('TarantoolServer.stop(): stopping the {0}\n'.format( format_process(self.process.pid)), schema='test_var') try: color_log('Sending signal {0} ({1}) to process {2}\n'.format( signal, signame(signal), self.process.pid)) self.process.send_signal(signal) except OSError: pass if self.crash_detector is not None: save_join(self.crash_detector) self.wait_stop() self.status = None if re.search(r'^/', str(self._admin.port)): if os.path.exists(self._admin.port): os.unlink(self._admin.port)
def process_timeout(self, delta_seconds): self.warned_seconds_ago += delta_seconds self.inactivity += delta_seconds worker_ids = self.get_not_done_worker_ids() if self.warned_seconds_ago < self.warn_timeout: return color_stdout("No output during %d seconds. " "List of workers not reporting the status: %s; " "Will abort after %d seconds without output.\n" % (self.inactivity, worker_ids, self.kill_timeout), schema='test_var') self.warned_seconds_ago = 0.0 if self.inactivity < self.kill_timeout: return color_stdout( '\n[Main process] No output from workers. ' 'It seems that we hang. Send SIGKILL to workers; ' 'exiting...\n', schema='test_var') self.kill_all_workers() raise HangError()
def run_test(self, test, server, inspector): """ Returns short status of the test as a string: 'skip', 'pass', 'new', 'fail', or 'disabled' and results file checksum on fail. """ test.inspector = inspector test_name = os.path.basename(test.name) full_test_name = os.path.join(self.ini['suite'], test_name) color_stdout(just_and_trim(full_test_name, 47) + ' ', schema='t_name') # for better diagnostics in case of a long-running test conf = '' if test.run_params: conf = test.conf_name color_stdout(just_and_trim(conf, 15) + ' ', schema='test_var') start_time = time.time() if self.is_test_enabled(test, conf, server): short_status, result_checksum = test.run(server) else: color_stdout("[ disabled ]\n", schema='t_name') short_status = 'disabled' result_checksum = None duration = time.time() - start_time # cleanup only if test passed or if --force mode enabled if Options().args.is_force or short_status == 'pass': inspector.cleanup_nondefault() return short_status, result_checksum, duration
def check(self): """Check the arguments for correctness.""" check_error = False conflict_options = ('valgrind', 'gdb', 'lldb', 'strace') for op1, op2 in product(conflict_options, repeat=2): if op1 != op2 and getattr(self.args, op1, '') and \ getattr(self.args, op2, ''): format_str = "\nError: option --{} is not compatible with option --{}\n" color_stdout(format_str.format(op1, op2), schema='error') check_error = True break snapshot_path = self.args.snapshot_path if self.args.disable_schema_upgrade and not snapshot_path: color_stdout( "\nOption --disable-schema-upgrade requires --snapshot\n", schema='error') check_error = True if snapshot_path and not os.path.exists(snapshot_path): color_stdout("\nPath {} does not exist\n".format(snapshot_path), schema='error') check_error = True if check_error: exit(-1)
def run_test(self, test, server, inspector): """ Returns short status of the test as a string: 'skip', 'pass', 'new', 'fail', or 'disabled'. """ test.inspector = inspector test_name = os.path.basename(test.name) color_stdout(os.path.join(self.ini['suite'], test_name).ljust(48), schema='t_name') # for better diagnostics in case of a long-running test conf = '' if test.run_params: conf = test.conf_name color_stdout(conf.ljust(16), schema='test_var') if self.is_test_enabled(test, conf, server): short_status = test.run(server) else: color_stdout("[ disabled ]\n", schema='t_name') short_status = 'disabled' # cleanup only if test passed or if --force mode enabled if lib.Options().args.is_force or short_status == 'pass': inspector.cleanup_nondefault() return short_status
def run_test(self, test, server, inspector): """ Returns short status of the test as a string: 'skip', 'pass', 'new', 'fail', or 'disabled'. """ test.inspector = inspector color_stdout(os.path.join(self.ini['suite'], os.path.basename(test.name)).ljust(48), schema='t_name') # for better diagnostics in case of a long-running test conf = '' if test.run_params: conf = test.conf_name color_stdout(conf.ljust(16), schema='test_var') test_name = os.path.basename(test.name) if self.is_test_enabled(test, conf, server): short_status = test.run(server) else: color_stdout("[ disabled ]\n", schema='t_name') short_status = 'disabled' # cleanup only if test passed or if --force mode enabled if lib.Options().args.is_force or short_status == 'pass': inspector.cleanup_nondefault() return short_status
def collect_tests(self): if self.tests_are_collected: return self.tests if self.ini['core'] == 'tarantool': TarantoolServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'app': AppServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'unittest': UnittestServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'stress': # parallel tests are not supported and disabled for now self.tests = [] self.tests_are_collected = True return self.tests else: raise ValueError('Cannot collect tests of unknown type') if not Options().args.reproduce: color_stdout("Collecting tests in ", schema='ts_text') color_stdout( '%s (Found %s tests)' % ( repr(self.suite_path).ljust(16), str(len(self.tests)).ljust(3) ), schema='path' ) color_stdout(": ", self.ini["description"], ".\n", schema='ts_text') self.tests_are_collected = True return self.tests
def find_tests(self): if self.ini['core'] == 'tarantool': TarantoolServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'app': AppServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'unittest': UnittestServer.find_tests(self, self.suite_path) elif self.ini['core'] == 'stress': # parallel tests are not supported and disabled for now return [] else: raise ValueError('Cannot collect tests of unknown type') if not lib.Options().args.reproduce: color_stdout("Collecting tests in ", schema='ts_text') color_stdout( '%s (Found %s tests)' % ( repr(self.suite_path).ljust(16), str(len(self.tests)).ljust(3) ), schema='path' ) color_stdout(": ", self.ini["description"], ".\n", schema='ts_text') return self.tests
def stop(self, silent=True, signal=signal.SIGTERM): """ Kill tarantool server using specified signal (SIGTERM by default) signal - a number of a signal """ if self._start_against_running: color_log('Server [%s] start against running ...\n', schema='test_var') return if self.status != 'started': if not silent: raise Exception('Server is not started') else: color_log('Server [{0.name}] is not started ' '(status:{0.status}) ...\n'.format(self), schema='test_var') return if not silent: color_stdout('Stopping the server ...\n', schema='serv_text') else: color_log('Stopping the server ...\n', schema='serv_text') # kill only if process is alive if self.process is not None and self.process.returncode is None: color_log('TarantoolServer.stop(): stopping the {0}\n'.format( format_process(self.process.pid)), schema='test_var') try: color_log('Sending signal {0} ({1}) to process {2}\n'.format( signal, signame(signal), self.process.pid)) self.process.send_signal(signal) except OSError: pass if self.crash_detector is not None: save_join(self.crash_detector) self.wait_stop() self.status = None if re.search(r'^/', str(self._admin.port)): if os.path.exists(self._admin.port): os.unlink(self._admin.port)
def execute(self, server): super(LuaTest, self).execute(server) cls_name = server.__class__.__name__.lower() if 'gdb' in cls_name or 'lldb' in cls_name or 'strace' in cls_name: # don't propagate gdb/lldb/strace mixin to non-default servers, # it doesn't work properly for now # TODO: strace isn't interactive, so it's easy to make it works for # non-default server create_server = TarantoolServer else: # propagate valgrind mixin to non-default servers create_server = server.__class__ ts = TestState( self.suite_ini, server, create_server, self.run_params ) self.inspector.set_parser(ts) lua = TestRunGreenlet(self.exec_loop, ts) self.current_test_greenlet = lua lua.start() crash_occured = True try: crash_occured = save_join(lua, timeout=self.TIMEOUT) self.killall_servers(server, ts, crash_occured) except KeyboardInterrupt: # prevent tests greenlet from writing to the real stdout lua.kill() ts.stop_nondefault() raise except TarantoolStartError as e: if not self.is_crash_reported: self.is_crash_reported = True color_stdout('\n[Instance "{0}"] Failed to start tarantool ' 'server from a test\n'.format(e.name), schema='error') server.print_log(15) server.kill_current_test()
def main_consistent(): color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text') failed_test_ids = [] try: main_loop_consistent(failed_test_ids) except KeyboardInterrupt: color_stdout('[Main loop] Caught keyboard interrupt\n', schema='test_var') except RuntimeError as e: color_stdout("\nFatal error: %s. Execution aborted.\n" % e, schema='error') if lib.Options().args.gdb: time.sleep(100) return -1 if failed_test_ids and lib.Options().args.is_force: color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids), schema='error') for test_id in failed_test_ids: color_stdout("----- %s\n" % str(test_id), schema='info') return (-1 if failed_test_ids else 0)
def check_tap_output(self): """ Returns is_tap, is_ok """ if not os.path.isfile(self.tmp_result): color_stdout('\nCannot find %s\n' % self.tmp_result, schema='error') self.is_crash_reported = True return False with open(self.tmp_result, 'r') as f: content = f.read() tap = pytap13.TAP13() try: tap.parse(content) except ValueError as e: color_stdout('\nTAP13 parse failed: %s\n' % str(e), schema='error') self.is_crash_reported = True return False, False is_ok = True for test_case in tap.tests: if test_case.result == 'ok': continue if is_ok: color_stdout('\n') color_stdout('%s %s %s # %s %s\n' % ( test_case.result, test_case.id or '', test_case.description or '-', test_case.directive or '', test_case.comment or ''), schema='error') if test_case.yaml: self.tap_parse_print_yaml(test_case.yaml) is_ok = False if not is_ok: color_stdout('Rejected result file: %s\n' % self.reject, schema='test_var') self.is_crash_reported = True return True, is_ok
def handle(self, socket, addr): if self.parser is None: raise AttributeError('Parser is not defined') self.sem.acquire() for line in self.readline(socket): try: result = self.parser.parse_preprocessor(line) except (KeyboardInterrupt, TarantoolStartError): # propagate to the main greenlet raise except Exception as e: self.parser.kill_current_test() color_stdout('\nTarantoolInpector.handle() received the following error:\n' + traceback.format_exc() + '\n', schema='error') result = { "error": repr(e) } if result == None: result = True result = yaml.dump(result) if not result.endswith('...\n'): result = result + '...\n' socket.sendall(result) self.sem.release()
def print_log(self, lines): color_stdout( '\nLast {0} lines of Tarantool Log file [Instance "{1}"][{2}]:\n'. format(lines, self.name, self.logfile or 'null'), schema='error') if os.path.exists(self.logfile): with open(self.logfile, 'r') as log: color_stdout(''.join(log.readlines()[-lines:])) else: color_stdout(" Can't find log:\n", schema='error')
def process_timeout(self, delta_seconds): self.warned_seconds_ago += delta_seconds self.inactivity += delta_seconds worker_ids = self.get_not_done_worker_ids() if self.warned_seconds_ago < self.warn_timeout: return is_warning = self.inactivity < self.kill_timeout color_stdout( "No output during {0.inactivity:.0f} seconds. " "Will abort after {0.kill_timeout:.0f} seconds without output. " "List of workers not reporting the status:\n".format(self), schema=('test_var' if is_warning else 'error')) hung_tasks = [ task for worker_id, task in self.worker_current_task.iteritems() if worker_id in worker_ids ] for task in hung_tasks: with open(task.task_tmp_result, 'r') as f: lines = sum(1 for _ in f) color_stdout("- {0} [{1}, {2}] at {3}:{4}\n".format( task.worker_name, task.task_name, task.task_param, task.task_tmp_result, lines), schema=('test_var' if is_warning else 'error')) self.warned_seconds_ago = 0.0 if is_warning: return for task in hung_tasks: color_stdout("Test hung! Result content mismatch:\n", schema='error') lib.utils.print_unidiff(task.task_result, task.task_tmp_result) color_stdout( '\n[Main process] No output from workers. ' 'It seems that we hang. Send SIGKILL to workers; ' 'exiting...\n', schema='error') self.kill_all_workers() raise HangError()
def process_timeout(self, delta_seconds): self.warned_seconds_ago += delta_seconds self.inactivity += delta_seconds worker_ids = self.get_not_done_worker_ids() if self.warned_seconds_ago < self.warn_timeout: return is_warning = self.inactivity < self.kill_timeout color_stdout( "No output during {0.inactivity:.0f} seconds. " "Will abort after {0.kill_timeout:.0f} seconds without output. " "List of workers not reporting the status:\n".format(self), schema=('test_var' if is_warning else 'error')) hung_tasks = [task for worker_id, task in self.worker_current_task.iteritems() if worker_id in worker_ids] for task in hung_tasks: with open(task.task_tmp_result, 'r') as f: lines = sum(1 for _ in f) color_stdout("- {0} [{1}, {2}] at {3}:{4}\n".format( task.worker_name, task.task_name, task.task_param, task.task_tmp_result, lines), schema=('test_var' if is_warning else 'error')) self.warned_seconds_ago = 0.0 if is_warning: return for task in hung_tasks: color_stdout("Test hung! Result content mismatch:\n", schema='error') lib.utils.print_unidiff(task.task_result, task.task_tmp_result) color_stdout('\n[Main process] No output from workers. ' 'It seems that we hang. Send SIGKILL to workers; ' 'exiting...\n', schema='error') self.kill_all_workers() raise HangError()
def crash_grep(self): print_log_lines = 15 assert_fail_re = re.compile(r'^.*: Assertion .* failed\.$') # find and save backtrace or assertion fail assert_lines = list() bt = list() with open(self.logfile, 'r') as log: lines = log.readlines() for rpos, line in enumerate(reversed(lines)): if line.startswith('Segmentation fault'): bt = lines[-rpos - 1:] break if assert_fail_re.match(line): pos = len(lines) - rpos assert_lines = lines[max(0, pos - print_log_lines):pos] break else: bt = list() # print insident meat if self.process.returncode < 0: color_stdout('\n\n[Instance "%s" killed by signal: %d (%s)]\n' % (self.name, -self.process.returncode, signame(-self.process.returncode)), schema='error') else: color_stdout( '\n\n[Instance "%s" returns with non-zero exit code: %d]\n' % (self.name, self.process.returncode), schema='error') # print assert line if any and return if assert_lines: color_stdout('Found assertion fail in the results file [%s]:\n' % self.logfile, schema='error') sys.stderr.flush() for line in assert_lines: sys.stderr.write(line) sys.stderr.flush() return # print backtrace if any sys.stderr.flush() for trace in bt: sys.stderr.write(trace) # print log otherwise (if backtrace was not found) if not bt: self.print_log(print_log_lines) sys.stderr.flush()
def crash_grep(self): if self.process.returncode < 0 or \ not non_empty_valgrind_logs([self.valgrind_log]): super(ValgrindMixin, self).crash_grep() return lines_cnt = 50 color_stdout('\n\nValgrind for [Instance "%s"] returns non-zero exit code: %d\n' % ( self.name, self.process.returncode), schema='error') color_stdout("It's known that it can be valgrind's \"the 'impossible' happened\" error\n", schema='error') color_stdout('Last %d lines of valgring log file [%s]:\n' % ( lines_cnt, self.valgrind_log), schema='error') print_tail_n(self.valgrind_log, 50)
def crash_grep(self): if self.process.returncode < 0 or \ not non_empty_valgrind_logs([self.valgrind_log]): super(ValgrindMixin, self).crash_grep() return lines_cnt = 50 color_stdout('\n\nValgrind for [Instance "%s"] returns non-zero exit code: %d\n' % ( self.name, self.process.returncode), schema='error') color_stdout("It's known that it can be valgrind's \"the 'impossible' happened\" error\n", schema='error') color_stdout('Last %d lines of valgring log file [%s]:\n' % ( lines_cnt, self.valgrind_log), schema='error') print_tail_n(self.valgrind_log, lines_cnt)
def crash_grep(self): print_log_lines = 15 assert_fail_re = re.compile(r'^.*: Assertion .* failed\.$') # find and save backtrace or assertion fail assert_lines = list() bt = list() with open(self.logfile, 'r') as log: lines = log.readlines() for rpos, line in enumerate(reversed(lines)): if line.startswith('Segmentation fault'): bt = lines[-rpos - 1:] break if assert_fail_re.match(line): pos = len(lines) - rpos assert_lines = lines[max(0, pos - print_log_lines):pos] break else: bt = list() # print insident meat if self.process.returncode < 0: color_stdout('\n\n[Instance "%s" killed by signal: %d (%s)]\n' % ( self.name, -self.process.returncode, signame(-self.process.returncode)), schema='error') else: color_stdout('\n\n[Instance "%s" returns with non-zero exit code: ' '%d]\n' % (self.name, self.process.returncode), schema='error') # print assert line if any and return if assert_lines: color_stdout('Found assertion fail in the results file ' '[%s]:\n' % self.logfile, schema='error') sys.stderr.flush() for line in assert_lines: sys.stderr.write(line) sys.stderr.flush() return # print backtrace if any sys.stderr.flush() for trace in bt: sys.stderr.write(trace) # print log otherwise (if backtrace was not found) if not bt: self.print_log(print_log_lines) sys.stderr.flush()
def report_undone(self, verbose): undone = self.undone_tasks() if not bool(undone): return False if verbose: color_stdout( 'The following tasks were dispatched on some worker task ' 'queue, but were not reported as done (does not matters ' 'success or fail):\n', schema='test_var') for task_id in undone: color_stdout('- %s' % yaml.safe_dump(task_id)) else: color_stdout("Count of didn't processed tasks: %d\n" % len(undone), schema='test_var') return True
def report_undone(self, verbose): undone = self.undone_tasks() if not bool(undone): return False if verbose: color_stdout( '[Internal test-run error] ' 'The following tasks were dispatched to some worker task ' 'queue, but were not reported as done (does not matters ' 'success or fail):\n', schema='test_var') for task_id in undone: color_stdout('- %s' % yaml.safe_dump(task_id)) else: # Visually continue StatisticsWatcher.print_statistics() output. color_stdout('* undone: %d\n' % len(undone), schema='test_var') return True
def prepare_args(self, args=[]): screen_name = self.debugger_args['screen_name'] debugger = self.debugger_args['debugger'] gdbserver_port = self.debugger_args['gdbserver_port'] gdbserver_opts = self.debugger_args['gdbserver_opts'] sh_string = self.debugger_args['sh_string'] is_under_gdbserver = 'GdbServer' in self.__class__.__name__ if not is_under_gdbserver and not find_in_path('screen'): raise OSError('`screen` executables not found in PATH') if not find_in_path(debugger): raise OSError('`%s` executables not found in PATH' % debugger) is_tarantoolserver = 'TarantoolServer' in self.__class__.__name__ if is_tarantoolserver or is_under_gdbserver: color_stdout('\nYou started the server in %s mode.\n' % debugger, schema='info') if is_under_gdbserver: color_stdout("To attach, use `gdb -ex 'target remote :%s'`\n" % gdbserver_port, schema='info') else: color_stdout('To attach, use `screen -r %s`\n' % screen_name, schema='info') # detach only for TarantoolServer screen_opts = '-d' if is_tarantoolserver else '' orig_args = super(DebugMixin, self).prepare_args(args) res_args = shlex.split(sh_string.format( screen_name=screen_name, screen_opts=screen_opts, binary=self.binary, args=' '.join(orig_args), logfile=self.logfile, debugger=debugger, gdbserver_port=gdbserver_port, gdbserver_opts=gdbserver_opts)) color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var') return res_args
def __init__(self, suite, _id): self.sigterm_received = False signal.signal(signal.SIGTERM, lambda x, y, z=self: z.sigterm_handler(x, y)) self.initialized = False self.server = None self.inspector = None self.id = _id self.suite = suite self.name = '%03d_%s' % (self.id, self.suite.suite_path) main_vardir = self.suite.ini['vardir'] self.suite.ini['vardir'] = os.path.join(main_vardir, self.name) self.reproduce_file = get_reproduce_file(self.name) safe_makedirs(os.path.dirname(self.reproduce_file)) color_stdout.queue_msg_wrapper = self.wrap_output self.last_task_done = True self.last_task_id = -1 try: self.server = suite.gen_server() self.inspector = suite.start_server(self.server) self.initialized = True except KeyboardInterrupt: self.report_keyboard_interrupt() self.stop_server(cleanup=False) except Exception as e: color_stdout('Worker "%s" cannot start tarantool server; ' 'the tasks will be ignored...\n' % self.name, schema='error') color_stdout("The raised exception is '%s' of type '%s'.\n" % (str(e), str(type(e))), schema='error') color_stdout('Worker "%s" received the following error:\n' % self.name + traceback.format_exc() + '\n', schema='error') self.stop_server(cleanup=False)
def print_statistics(self): """Returns are there failed tasks.""" if self.stats: color_stdout('Statistics:\n', schema='test_var') for short_status, cnt in self.stats.items(): color_stdout('* %s: %d\n' % (short_status, cnt), schema='test_var') if not self.failed_tasks: return False color_stdout('Failed tasks:\n', schema='test_var') for task_id, worker_name, show_reproduce_content in self.failed_tasks: logfile = self.get_logfile(worker_name) color_stdout('- %s' % yaml.safe_dump(task_id), schema='test_var') color_stdout('# logfile: %s\n' % logfile) reproduce_file_path = get_reproduce_file(worker_name) color_stdout('# reproduce file: %s\n' % reproduce_file_path) if show_reproduce_content: color_stdout("---\n", schema='separator') lib.utils.print_tail_n(reproduce_file_path) color_stdout("...\n", schema='separator') return True
def start(self, silent=True, wait=True, wait_load=True, rais=True, args=[], **kwargs): if self._start_against_running: return if self.status == 'started': if not silent: color_stdout('The server is already started.\n', schema='lerror') return args = self.prepare_args(args) self.pidfile = '%s.pid' % self.name self.logfile = '%s.log' % self.name path = self.script_dst if self.script else \ os.path.basename(self.binary) color_log("Starting the server ...\n", schema='serv_text') color_log("Starting ", schema='serv_text') color_log(path + " \n", schema='path') color_log(self.version() + "\n", schema='version') os.putenv("LISTEN", self.iproto.uri) os.putenv("ADMIN", self.admin.uri) if self.rpl_master: os.putenv("MASTER", self.rpl_master.iproto.uri) self.logfile_pos = self.logfile # redirect stdout from tarantoolctl and tarantool os.putenv("TEST_WORKDIR", self.vardir) self.process = subprocess.Popen(args, cwd=self.vardir, stdout=self.log_des, stderr=self.log_des) del(self.log_des) # gh-19 crash detection self.crash_detector = TestRunGreenlet(self.crash_detect) self.crash_detector.info = "Crash detector: %s" % self.process self.crash_detector.start() if wait: try: self.wait_until_started(wait_load) except TarantoolStartError: # Raise exception when caller ask for it (e.g. in case of # non-default servers) if rais: raise # Python tests expect we raise an exception when non-default # server fails if self.crash_expected: raise if not (self.current_test and self.current_test.is_crash_reported): if self.current_test: self.current_test.is_crash_reported = True color_stdout('\n[Instance "{0.name}"] Tarantool server ' 'failed to start\n'.format(self), schema='error') self.print_log(15) # if the server fails before any test started, we should inform # a caller by the exception if not self.current_test: raise self.kill_current_test() port = self.admin.port self.admin.disconnect() self.admin = CON_SWITCH[self.tests_type]('localhost', port) self.status = 'started'
def print_exe(cls): color_stdout('Installing the server ...\n', schema='serv_text') if cls.binary: color_stdout(' Found executable at ', schema='serv_text') color_stdout(cls.binary + '\n', schema='path') if cls.ctl_path: color_stdout(' Found tarantoolctl at ', schema='serv_text') color_stdout(cls.ctl_path + '\n', schema='path') color_stdout("\n", cls.version(), "\n", schema='version')