def run_loop(self, task_queue, result_queue): """ called from 'run_all' """ while True: task_id = self.task_get(task_queue) # None is 'stop worker' marker if task_id is None: color_log('Worker "%s" exhausted task queue; ' 'stopping the server...\n' % self.name, schema='test_var') self.stop_worker(task_queue, result_queue) break result_queue.put(self.current_task(task_id)) short_status = self.run_task(task_id) result_queue.put(self.wrap_result(task_id, short_status)) if not lib.Options().args.is_force and short_status == 'fail': color_stdout( 'Worker "%s" got failed test; stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() if self.sigterm_received: color_stdout('Worker "%s" got signal to terminate; ' 'stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() self.task_done(task_queue)
def stop(self, silent=True): if self._start_against_running: return if self.status != 'started': if not silent: raise Exception('Server is not started') return if not silent: color_stdout('Stopping the server ...\n', schema='serv_text') # kill only if process is alive if self.process is not None and self.process.returncode is None: color_log('TarantoolServer.stop(): stopping the %s\n' % format_process(self.process.pid), schema='test_var') try: self.process.terminate() except OSError: pass if self.crash_detector is not None: save_join(self.crash_detector) self.wait_stop() self.status = None if re.search(r'^/', str(self._admin.port)): if os.path.exists(self._admin.port): os.unlink(self._admin.port)
def execute(self, server): super(AppTest, self).execute(server) ts = TestState(self.suite_ini, None, TarantoolServer, self.run_params, default_server_no_connect=server) self.inspector.set_parser(ts) execs = server.prepare_args() retval = dict() tarantool = TestRunGreenlet(run_server, execs, server.vardir, server, server.logfile, retval, self.id) self.current_test_greenlet = tarantool # Copy the snapshot right before starting the server. # Otherwise pretest_clean() would remove it. if server.snapshot_path: snapshot_dest = os.path.join(server.vardir, DEFAULT_SNAPSHOT_NAME) color_log("Copying snapshot {} to {}\n".format( server.snapshot_path, snapshot_dest)) shutil.copy(server.snapshot_path, snapshot_dest) try: tarantool.start() tarantool.join() except TarantoolStartError: # A non-default server failed to start. raise TestExecutionError finally: self.teardown(server, ts) if retval.get('returncode', None) != 0: raise TestExecutionError
def prepare_args(self, args=[]): if not find_in_path('valgrind'): raise OSError('`valgrind` executables not found in PATH') orig_args = super(ValgrindMixin, self).prepare_args(args) res_args = self.valgrind_cmd_args + orig_args color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var') return res_args
def handle(self, socket, addr): if self.parser is None: raise AttributeError('Parser is not defined') self.sem.acquire() for line in self.readline(socket): color_log('DEBUG: test-run received command: {}\n'.format(line), schema='test-run command') try: result = self.parser.parse_preprocessor(line) except (KeyboardInterrupt, TarantoolStartError): # propagate to the main greenlet raise except LuaPreprocessorException as e: qa_notice(str(e)) result = {'error': str(e)} except Exception as e: self.parser.kill_current_test() color_stdout('\nTarantoolInpector.handle() received the ' + 'following error:\n' + traceback.format_exc() + '\n', schema='error') result = {"error": repr(e)} if result is None: result = True result = yaml.dump(result) if not result.endswith('...\n'): result = result + '...\n' color_log("DEBUG: test-run's response for [{}]\n{}\n".format( line, prefix_each_line(' | ', result)), schema='test-run command') socket.sendall(str_to_bytes(result)) self.sem.release()
def find_exe(cls, builddir, silent=True): cls.builddir = os.path.abspath(builddir) builddir = os.path.join(builddir, "src") path = builddir + os.pathsep + os.environ["PATH"] color_log("Looking for server binary in ", schema='serv_text') color_log(path + ' ...\n', schema='path') for _dir in path.split(os.pathsep): exe = os.path.join(_dir, cls.default_tarantool["bin"]) ctl_dir = _dir # check local tarantoolctl source if _dir == builddir: ctl_dir = os.path.join(_dir, '../extra/dist') ctl = os.path.join(ctl_dir, cls.default_tarantool['ctl']) need_lua_path = False if os.path.isdir(ctl) or not os.access(ctl, os.X_OK): ctl_dir = os.path.join(_dir, '../extra/dist') ctl = os.path.join(ctl_dir, cls.default_tarantool['ctl']) need_lua_path = True if os.access(exe, os.X_OK) and os.access(ctl, os.X_OK): cls.binary = os.path.abspath(exe) cls.ctl_path = os.path.abspath(ctl) cls.ctl_plugins = os.path.abspath(os.path.join(ctl_dir, '..')) os.environ["PATH"] = os.pathsep.join([ os.path.abspath(ctl_dir), os.path.abspath(_dir), os.environ["PATH"] ]) os.environ["TARANTOOLCTL"] = ctl if need_lua_path: os.environ["LUA_PATH"] = \ ctl_dir + '/?.lua;' + \ ctl_dir + '/?/init.lua;' + \ os.environ.get("LUA_PATH", ";;") return exe raise RuntimeError("Can't find server executable in " + path)
def killall_servers(self, server, ts, crash_occured): """ kill all servers and crash detectors before stream swap """ color_log('Kill all servers ...\n', schema='info') server_list = ts.servers.values() + [ server, ] check_list = [s for s in server_list if 'process' in s.__dict__] # check that all servers stopped correctly for server in check_list: bad_returncode = server.process.returncode not in (None, 0, -signal.SIGKILL, -signal.SIGTERM) # if non-default server crashed but it was expected # don't kill the default server and crash detectors crash_occured = crash_occured or \ bad_returncode and not server.crash_expected for server in check_list: server.process.poll() if crash_occured: # kill all servers and crash detectors on crash if server.process.returncode is None: server.process.kill() if server.crash_detector is not None: gevent.kill(server.crash_detector) elif server.process.returncode is not None: # join crash detectors of stopped servers if server.crash_detector is not None: save_join(server.crash_detector)
def run_loop(self, task_queue, result_queue): """ called from 'run_all' """ while True: task_id = self.task_get(task_queue) # None is 'stop worker' marker if task_id is None: color_log('Worker "%s" exhausted task queue; ' 'stopping the server...\n' % self.name, schema='test_var') self.stop_worker(task_queue, result_queue) break short_status = self.run_task(task_id) result_queue.put(self.wrap_result(task_id, short_status)) if not lib.Options().args.is_force and short_status == 'fail': color_stdout( 'Worker "%s" got failed test; stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() if self.sigterm_received: color_stdout('Worker "%s" got signal to terminate; ' 'stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() self.task_done(task_queue)
def killall_servers(self, server, ts, crash_occured): """ kill all servers and crash detectors before stream swap """ color_log('Kill all servers ...\n', schema='info') check_list = ts.servers.values() + [server, ] # check that all servers stopped correctly for server in check_list: bad_returncode = server.process.returncode not in (None, 0, -signal.SIGKILL, -signal.SIGTERM) # if non-default server crashed but it was expected # don't kill the default server and crash detectors crash_occured = crash_occured or \ bad_returncode and not server.crash_expected for server in check_list: server.process.poll() if crash_occured: # kill all servers and crash detectors on crash if server.process.returncode is None: server.process.kill() if server.crash_detector is not None: gevent.kill(server.crash_detector) elif server.process.returncode is not None: # join crash detectors of stopped servers if server.crash_detector is not None: save_join(server.crash_detector)
def prepare_args(self, args=[]): if not find_in_path('strace'): raise OSError('`strace` executables not found in PATH') orig_args = super(StraceMixin, self).prepare_args(args) res_args = shlex.split("strace -o {log} -f -tt -T -x -I1 {bin}".format( bin=' '.join(orig_args), log=self.strace_log)) color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var') return res_args
def server_stop(self, ctype, sname, opts): color_log('\nDEBUG: TestState[%s].server_stop(%s, %s, %s)\n' % ( hex(id(self)), str(ctype), str(sname), str(opts)), schema='test_var') if sname not in self.servers: raise LuaPreprocessorException('Can\'t stop nonexistent server '+repr(sname)) self.connections[sname].disconnect() self.connections.pop(sname) self.servers[sname].stop()
def cleanup_nondefault(self): color_log('\nDEBUG: TestState[%s].cleanup()\n' % hex(id(self)), schema='test_var') for k, v in self.servers.iteritems(): # don't cleanup the default server if k == 'default': continue v.cleanup()
def cleanup_nondefault(self): color_log('\nDEBUG: TestState[%s].cleanup_nondefault()\n' % hex(id(self)), schema='test_var') for k, v in self.servers.iteritems(): # don't cleanup the default server if k == 'default': continue v.cleanup()
def stop(self, silent): if not self.process: return color_log('AppServer.stop(): stopping the %s\n' % format_process(self.process.pid), schema='test_var') try: self.process.terminate() except OSError: pass
def cleanup_nondefault(self): names = [k for k in self.servers.keys() if k != 'default'] color_log('DEBUG: Cleanup non-default servers: {}\n'.format(names), schema='info') for k, v in self.servers.items(): # don't cleanup the default server if k == 'default': continue v.cleanup()
def prepare_args(self, args=[]): if not find_in_path('strace'): raise OSError('`strace` executables not found in PATH') orig_args = super(StraceMixin, self).prepare_args(args) res_args = shlex.split("strace -o {log} -f -tt -T -x -I1 {bin}".format( bin=' '.join(orig_args), log=self.strace_log )) color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var') return res_args
def stop(self, silent=True, signal=signal.SIGTERM): # FIXME: Extract common parts of AppServer.stop() and # TarantoolServer.stop() to an utility function. color_log('DEBUG: [app server] Stopping the server...\n', schema='info') if not self.process: color_log(' | Nothing to do: the process does not exist\n', schema='info') return if self.process.returncode: if self.process.returncode < 0: signaled_by = -self.process.returncode color_log(' | Nothing to do: the process was terminated by ' 'signal {} ({})\n'.format(signaled_by, signame(signaled_by)), schema='info') else: color_log(' | Nothing to do: the process was exited with code ' '{}\n'.format(self.process.returncode), schema='info') return color_log(' | Sending signal {0} ({1}) to {2}\n'.format( signal, signame(signal), format_process(self.process.pid))) try: self.process.send_signal(signal) except OSError: pass # Waiting for stopping the server. If the timeout # reached, send SIGKILL. timeout = 5 def kill(): qa_notice('The app server does not stop during {} ' 'seconds after the {} ({}) signal.\n' 'Info: {}\n' 'Sending SIGKILL...'.format( timeout, signal, signame(signal), format_process(self.process.pid))) try: self.process.kill() except OSError: pass timer = Timer(timeout, kill) timer.start() self.process.wait() timer.cancel()
def stop_nondefault(self): color_log('\nDEBUG: TestState[%s].stop_nondefault()\n' % hex(id(self)), schema='test_var') if sys.stdout.__class__.__name__ == 'FilteredStream': sys.stdout.clear_all_filters() for k, v in self.servers.iteritems(): # don't stop the default server if k == 'default': continue v.stop(silent=True) if k in self.connections: self.connections[k].disconnect() self.connections.pop(k)
def server_create(self, ctype, sname, opts): color_log('\nDEBUG: TestState[%s].server_create(%s, %s, %s)\n' % ( hex(id(self)), str(ctype), str(sname), str(opts)), schema='test_var') if sname in self.servers: raise LuaPreprocessorException('Server {0} already exists'.format(repr(sname))) temp = self.create_server() temp.name = sname if 'need_init' in opts: temp.need_init = True if opts['need_init'] == 'True' else False if 'script' in opts: temp.script = opts['script'][1:-1] if 'lua_libs' in opts: temp.lua_libs = opts['lua_libs'][1:-1].split(' ') temp.rpl_master = None if 'rpl_master' in opts: temp.rpl_master = self.servers[opts['rpl_master']] temp.vardir = self.suite_ini['vardir'] temp.use_unix_sockets = self.suite_ini['use_unix_sockets'] temp.use_unix_sockets_iproto = self.suite_ini['use_unix_sockets_iproto'] temp.inspector_port = int(self.suite_ini.get( 'inspector_port', temp.DEFAULT_INSPECTOR )) if self.default_server_no_connect: temp.current_test = self.default_server_no_connect.current_test elif self.servers['default']: temp.current_test = self.servers['default'].current_test temp.install(silent=True) self.servers[sname] = temp if 'workdir' in opts: copy_from = opts['workdir'] copy_to = self.servers[sname].name os.system('rm -rf %s/%s' % ( self.servers[sname].vardir, copy_to )) os.system('cp -r %s %s/%s' % ( copy_from, self.servers[sname].vardir, copy_to )) nmsp = Namespace() setattr(nmsp, 'admin', temp.admin.port) setattr(nmsp, 'listen', temp.iproto.port) if temp.rpl_master: setattr(nmsp, 'master', temp.rpl_master.iproto.port) setattr(self.environ, sname, nmsp) if 'return_listen_uri' in opts and opts['return_listen_uri'] == 'True': return self.servers[sname].iproto.uri
def stop_nondefault(self, signal=signal.SIGTERM): names = [k for k in self.servers.keys() if k != 'default'] color_log('DEBUG: Stop non-default servers using ' 'signal {} ({}): {}\n'.format(signal, signame(signal), names), schema='info') if sys.stdout.__class__.__name__ == 'FilteredStream': sys.stdout.clear_all_filters() for k, v in self.servers.items(): # don't stop the default server if k == 'default': continue v.stop(silent=True, signal=signal) if k in self.connections: self.connections[k].disconnect() self.connections.pop(k)
def send_command_raw(self, command, ts): """ Send a command to tarantool and read a response. """ color_log('DEBUG: sending command: {}\n'.format(command.rstrip()), schema='tarantool command') # Evaluate the request on the first connection, save the # response. result = ts.curcon[0](command, silent=True) # Evaluate on other connections, ignore responses. for conn in ts.curcon[1:]: conn(command, silent=True) # gh-24 fix if result is None: result = '[Lost current connection]\n' color_log("DEBUG: tarantool's response for [{}]\n{}\n".format( command.rstrip(), prefix_each_line(' | ', result)), schema='tarantool command') return result
def kill_old_server(self, silent=True): pid = self.read_pidfile() if pid == -1: return False if not silent: color_stdout( ' Found old server, pid {0}, killing ...'.format(pid), schema='info') else: color_log(' Found old server, pid {0}, killing ...'.format(pid), schema='info') try: os.kill(pid, signal.SIGTERM) except OSError: pass self.wait_until_stopped(pid) return True
def server_create(self, ctype, sname, opts): color_log('\nDEBUG: TestState[%s].server_create(%s, %s, %s)\n' % ( hex(id(self)), str(ctype), str(sname), str(opts)), schema='test_var') if sname in self.servers: raise LuaPreprocessorException('Server {0} already exists'.format(repr(sname))) temp = self.create_server() temp.name = sname if 'need_init' in opts: temp.need_init = True if opts['need_init'] == 'True' else False if 'script' in opts: temp.script = opts['script'][1:-1] if 'lua_libs' in opts: temp.lua_libs = opts['lua_libs'][1:-1].split(' ') temp.rpl_master = None if 'rpl_master' in opts: temp.rpl_master = self.servers[opts['rpl_master']] temp.vardir = self.suite_ini['vardir'] temp.inspector_port = int(self.suite_ini.get( 'inspector_port', temp.DEFAULT_INSPECTOR )) if self.default_server_no_connect: temp.current_test = self.default_server_no_connect.current_test elif self.servers['default']: temp.current_test = self.servers['default'].current_test temp.install(silent=True) self.servers[sname] = temp if 'workdir' in opts: copy_from = opts['workdir'] copy_to = self.servers[sname].name os.system('rm -rf %s/%s' % ( self.servers[sname].vardir, copy_to )) os.system('cp -r %s %s/%s' % ( copy_from, self.servers[sname].vardir, copy_to )) nmsp = Namespace() setattr(nmsp, 'admin', temp.admin.port) setattr(nmsp, 'listen', temp.iproto.port) if temp.rpl_master: setattr(nmsp, 'master', temp.rpl_master.iproto.port) setattr(self.environ, sname, nmsp) if 'return_listen_uri' in opts and opts['return_listen_uri'] == 'True': return self.servers[sname].iproto.uri
def kill_old_server(self, silent=True): pid = self.read_pidfile() if pid == -1: return False if not silent: color_stdout( ' Found old server, pid {0}, killing ...'.format(pid), schema='info' ) else: color_log(' Found old server, pid {0}, killing ...'.format(pid), schema='info') try: os.kill(pid, signal.SIGTERM) except OSError: pass self.wait_until_stopped(pid) return True
def server_stop(self, ctype, sname, opts): color_log('\nDEBUG: TestState[%s].server_stop(%s, %s, %s)\n' % ( hex(id(self)), str(ctype), str(sname), str(opts)), schema='test_var') if sname not in self.servers: raise LuaPreprocessorException('Can\'t stop nonexistent server {0}'.format(repr(sname))) self.connections[sname].disconnect() self.connections.pop(sname) if 'signal' in opts: # convert to an integer if a number is passed, leave a string # otherwise try: signal = int(opts['signal']) except ValueError: signal = opts['signal'] self.servers[sname].stop(silent=True, signal=signum(signal)) else: # use default signal self.servers[sname].stop(silent=True)
def copy_files(self): if self.script: shutil.copy(self.script, self.script_dst) os.chmod(self.script_dst, 0o777) if self.lua_libs: for i in self.lua_libs: source = os.path.join(self.testdir, i) try: if os.path.isdir(source): shutil.copytree(source, os.path.join(self.vardir, os.path.basename(source))) else: shutil.copy(source, self.vardir) except IOError as e: if (e.errno == errno.ENOENT): continue raise # Previously tarantoolctl configuration file located in tarantool # repository at test/ directory. Currently it is located in root # path of test-run/ submodule repository. For backward compatibility # this file should be checked at the old place and only after at # the current. tntctl_file = '.tarantoolctl' if not os.path.exists(tntctl_file): tntctl_file = os.path.join(self.TEST_RUN_DIR, '.tarantoolctl') shutil.copy(tntctl_file, self.vardir) shutil.copy(os.path.join(self.TEST_RUN_DIR, 'test_run.lua'), self.vardir) if self.snapshot_path: # Copy snapshot to the workdir. # Usually Tarantool looking for snapshots on start in a current directory # or in a directories that specified in memtx_dir or vinyl_dir box settings. # Before running test current directory (workdir) passed to a new instance in # an environment variable TEST_WORKDIR and then tarantoolctl # adds to it instance_name and set to memtx_dir and vinyl_dir. (instance_name, _) = os.path.splitext(os.path.basename(self.script)) instance_dir = os.path.join(self.vardir, instance_name) safe_makedirs(instance_dir) snapshot_dest = os.path.join(instance_dir, DEFAULT_SNAPSHOT_NAME) color_log("Copying snapshot {} to {}\n".format( self.snapshot_path, snapshot_dest)) shutil.copy(self.snapshot_path, snapshot_dest)
def _log(self, event, pid): # Those logs are not written due to gh-247. process_def = self._processes[pid] task_id = process_def['task_id'] test_name = task_id[0] + ((':' + task_id[1]) if task_id[1] else '') worker_name = process_def['worker_name'] server_name = process_def['server_name'] color_log('DEBUG: sampler: {} {}\n'.format(event, format_process(pid)), schema='info') color_log(' | worker: {}\n'.format(worker_name)) color_log(' | test: {}\n'.format(test_name)) color_log(' | server: {}\n'.format(str(server_name)))
def prepare_args(self, args=[]): screen_name = self.debugger_args['screen_name'] debugger = self.debugger_args['debugger'] gdbserver_port = self.debugger_args['gdbserver_port'] gdbserver_opts = self.debugger_args['gdbserver_opts'] sh_string = self.debugger_args['sh_string'] is_under_gdbserver = 'GdbServer' in self.__class__.__name__ if not is_under_gdbserver and not find_in_path('screen'): raise OSError('`screen` executables not found in PATH') if not find_in_path(debugger): raise OSError('`%s` executables not found in PATH' % debugger) is_tarantoolserver = 'TarantoolServer' in self.__class__.__name__ if is_tarantoolserver or is_under_gdbserver: color_stdout('\nYou started the server in %s mode.\n' % debugger, schema='info') if is_under_gdbserver: color_stdout("To attach, use `gdb -ex 'target remote :%s'`\n" % gdbserver_port, schema='info') else: color_stdout('To attach, use `screen -r %s`\n' % screen_name, schema='info') # detach only for TarantoolServer screen_opts = '-d' if is_tarantoolserver else '' orig_args = super(DebugMixin, self).prepare_args(args) res_args = shlex.split( sh_string.format(screen_name=screen_name, screen_opts=screen_opts, binary=self.binary, args=' '.join(orig_args), logfile=self.logfile, debugger=debugger, gdbserver_port=gdbserver_port, gdbserver_opts=gdbserver_opts)) color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var') return res_args
def stop(self, silent=True, signal=signal.SIGTERM): """ Kill tarantool server using specified signal (SIGTERM by default) signal - a number of a signal """ if self._start_against_running: color_log('Server [%s] start against running ...\n', schema='test_var') return if self.status != 'started': if not silent: raise Exception('Server is not started') else: color_log( 'Server [{0.name}] is not started ' '(status:{0.status}) ...\n'.format(self), schema='test_var' ) return if not silent: color_stdout('Stopping the server ...\n', schema='serv_text') else: color_log('Stopping the server ...\n', schema='serv_text') # kill only if process is alive if self.process is not None and self.process.returncode is None: color_log('TarantoolServer.stop(): stopping the {0}\n'.format( format_process(self.process.pid)), schema='test_var') try: color_log('Sending signal {0} ({1}) to process {2}\n'.format( signal, signame(signal), self.process.pid)) self.process.send_signal(signal) except OSError: pass if self.crash_detector is not None: save_join(self.crash_detector) self.wait_stop() self.status = None if re.search(r'^/', str(self._admin.port)): if os.path.exists(self._admin.port): os.unlink(self._admin.port)
def prepare_args(self, args=[]): screen_name = self.debugger_args['screen_name'] debugger = self.debugger_args['debugger'] gdbserver_port = self.debugger_args['gdbserver_port'] gdbserver_opts = self.debugger_args['gdbserver_opts'] sh_string = self.debugger_args['sh_string'] is_under_gdbserver = 'GdbServer' in self.__class__.__name__ if not is_under_gdbserver and not find_in_path('screen'): raise OSError('`screen` executables not found in PATH') if not find_in_path(debugger): raise OSError('`%s` executables not found in PATH' % debugger) is_tarantoolserver = 'TarantoolServer' in self.__class__.__name__ if is_tarantoolserver or is_under_gdbserver: color_stdout('\nYou started the server in %s mode.\n' % debugger, schema='info') if is_under_gdbserver: color_stdout("To attach, use `gdb -ex 'target remote :%s'`\n" % gdbserver_port, schema='info') else: color_stdout('To attach, use `screen -r %s`\n' % screen_name, schema='info') # detach only for TarantoolServer screen_opts = '-d' if is_tarantoolserver else '' orig_args = super(DebugMixin, self).prepare_args(args) res_args = shlex.split(sh_string.format( screen_name=screen_name, screen_opts=screen_opts, binary=self.binary, args=' '.join(orig_args), logfile=self.logfile, debugger=debugger, gdbserver_port=gdbserver_port, gdbserver_opts=gdbserver_opts)) color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var') return res_args
def stop(self, silent=True, signal=signal.SIGTERM): """ Kill tarantool server using specified signal (SIGTERM by default) signal - a number of a signal """ if self._start_against_running: color_log('Server [%s] start against running ...\n', schema='test_var') return if self.status != 'started': if not silent: raise Exception('Server is not started') else: color_log('Server [{0.name}] is not started ' '(status:{0.status}) ...\n'.format(self), schema='test_var') return if not silent: color_stdout('Stopping the server ...\n', schema='serv_text') else: color_log('Stopping the server ...\n', schema='serv_text') # kill only if process is alive if self.process is not None and self.process.returncode is None: color_log('TarantoolServer.stop(): stopping the {0}\n'.format( format_process(self.process.pid)), schema='test_var') try: color_log('Sending signal {0} ({1}) to process {2}\n'.format( signal, signame(signal), self.process.pid)) self.process.send_signal(signal) except OSError: pass if self.crash_detector is not None: save_join(self.crash_detector) self.wait_stop() self.status = None if re.search(r'^/', str(self._admin.port)): if os.path.exists(self._admin.port): os.unlink(self._admin.port)
def find_exe(cls, builddir, silent=True): cls.builddir = os.path.abspath(builddir) builddir = os.path.join(builddir, "src") path = builddir + os.pathsep + os.environ["PATH"] color_log("Looking for server binary in ", schema='serv_text') color_log(path + ' ...\n', schema='path') for _dir in path.split(os.pathsep): exe = os.path.join(_dir, cls.default_tarantool["bin"]) ctl_dir = _dir # check local tarantoolctl source if _dir == builddir: ctl_dir = os.path.join(_dir, '../extra/dist') ctl = os.path.join(ctl_dir, cls.default_tarantool['ctl']) need_lua_path = False if os.path.isdir(ctl) or not os.access(ctl, os.X_OK): ctl_dir = os.path.join(_dir, '../extra/dist') ctl = os.path.join(ctl_dir, cls.default_tarantool['ctl']) need_lua_path = True if os.access(exe, os.X_OK) and os.access(ctl, os.X_OK): cls.binary = os.path.abspath(exe) cls.ctl_path = os.path.abspath(ctl) cls.ctl_plugins = os.path.abspath( os.path.join(ctl_dir, '..') ) os.environ["PATH"] = os.pathsep.join([ os.path.abspath(ctl_dir), os.path.abspath(_dir), os.environ["PATH"] ]) os.environ["TARANTOOLCTL"] = ctl if need_lua_path: os.environ["LUA_PATH"] = \ ctl_dir + '/?.lua;' + \ ctl_dir + '/?/init.lua;' + \ os.environ.get("LUA_PATH", ";;") return exe raise RuntimeError("Can't find server executable in " + path)
def server_start(self, ctype, sname, opts): color_log('\nDEBUG: TestState[%s].server_start(%s, %s, %s)\n' % ( hex(id(self)), str(ctype), str(sname), str(opts)), schema='test_var') if sname not in self.servers: raise LuaPreprocessorException('Can\'t start nonexistent server '+repr(sname)) wait = True if 'wait' in opts and opts['wait'] == 'False': wait = False wait_load = True if 'wait_load' in opts and opts['wait_load'] == 'False': wait_load = False args = [] if 'args' in opts: args = opts['args'][1:-1].split(' ') crash_expected = 'crash_expected' in opts and \ opts['crash_expected'] == 'True' crash_occured = False try: if crash_expected: # disable crash detector self.servers[sname].crash_expected = True self.servers[sname].start(silent=True, rais=True, wait=wait, wait_load=wait_load, args=args) except Exception as e: crash_occured = True if not (crash_expected and \ e.__class__.__name__ == 'TarantoolStartError'): raise if not crash_occured: self.connections[sname] = self.servers[sname].admin try: self.connections[sname]('return true', silent=True) except socket.error as e: LuaPreprocessorException('Can\'t start server '+repr(sname)) return not crash_occured
def server_start(self, ctype, sname, opts): color_log('\nDEBUG: TestState[%s].server_start(%s, %s, %s)\n' % ( hex(id(self)), str(ctype), str(sname), str(opts)), schema='test_var') if sname not in self.servers: raise LuaPreprocessorException('Can\'t start nonexistent server {0}'.format(repr(sname))) wait = True if 'wait' in opts and opts['wait'] == 'False': wait = False wait_load = True if 'wait_load' in opts and opts['wait_load'] == 'False': wait_load = False args = [] if 'args' in opts: args = opts['args'][1:-1].split(' ') crash_expected = 'crash_expected' in opts and \ opts['crash_expected'] == 'True' crash_occured = False try: if crash_expected: # disable crash detector self.servers[sname].crash_expected = True self.servers[sname].start(silent=True, rais=True, wait=wait, wait_load=wait_load, args=args) except Exception as e: crash_occured = True if not (crash_expected and e.__class__.__name__ == 'TarantoolStartError'): raise if not crash_occured: self.connections[sname] = self.servers[sname].admin try: self.connections[sname]('return true', silent=True) except socket.error as e: LuaPreprocessorException('Can\'t start server {0}'.format(repr(sname))) return not crash_occured
def install(self, silent=True): if self._start_against_running: self._iproto = self._start_against_running self._admin = int(self._start_against_running) + 1 return color_log('DEBUG: [Instance {}] Installing the server...\n'.format( self.name), schema='info') color_log(' | Found executable at {}\n'.format(self.binary)) color_log(' | Found tarantoolctl at {}\n'.format(self.ctl_path)) color_log(' | Creating and populating working directory in ' '{}...\n'.format(self.vardir)) if not os.path.exists(self.vardir): os.makedirs(self.vardir) else: color_log(' | Found old workdir, deleting...\n') self.kill_old_server() self.cleanup() self.copy_files() if self.use_unix_sockets: path = os.path.join(self.vardir, self.name + ".socket-admin") warn_unix_socket(path) self._admin = path else: self._admin = find_port() if self.use_unix_sockets_iproto: path = os.path.join(self.vardir, self.name + ".socket-iproto") warn_unix_socket(path) self._iproto = path else: self._iproto = find_port() # these sockets will be created by tarantool itself path = os.path.join(self.vardir, self.name + '.control') warn_unix_socket(path)
def wait_until_started(self, wait_load=True): """ Wait until server is started. Server consists of two parts: 1) wait until server is listening on sockets 2) wait until server tells us his status """ color_log('DEBUG: [Instance {}] Waiting until started ' '(wait_load={})\n'.format(self.name, str(wait_load)), schema='info') if wait_load: msg = 'entering the event loop|will retry binding|hot standby mode' p = self.process if not self.gdb and not self.lldb else None self.logfile_pos.seek_wait(msg, p, self.name) while True: try: temp = AdminConnection('localhost', self.admin.port) if not wait_load: ans = yaml.safe_load(temp.execute("2 + 2")) color_log(" | Successful connection check; don't wait for " "loading") return True ans = yaml.safe_load(temp.execute('box.info.status'))[0] if ans in ('running', 'hot_standby', 'orphan'): color_log(" | Started {} (box.info.status: '{}')\n".format( format_process(self.process.pid), ans)) return True elif ans in ('loading',): continue else: raise Exception( "Strange output for `box.info.status`: %s" % (ans) ) except socket.error as e: if e.errno == errno.ECONNREFUSED: color_log(' | Connection refused; will retry every 0.1 ' 'seconds...') time.sleep(0.1) continue raise
def start(self, silent=True, wait=True, wait_load=True, rais=True, args=[], **kwargs): if self._start_against_running: return if self.status == 'started': if not silent: color_stdout('The server is already started.\n', schema='lerror') return args = self.prepare_args(args) self.pidfile = '%s.pid' % self.name self.logfile = '%s.log' % self.name path = self.script_dst if self.script else \ os.path.basename(self.binary) color_log("Starting the server ...\n", schema='serv_text') color_log("Starting ", schema='serv_text') color_log(path + " \n", schema='path') color_log(self.version() + "\n", schema='version') os.putenv("LISTEN", self.iproto.uri) os.putenv("ADMIN", self.admin.uri) if self.rpl_master: os.putenv("MASTER", self.rpl_master.iproto.uri) self.logfile_pos = self.logfile # redirect stdout from tarantoolctl and tarantool os.putenv("TEST_WORKDIR", self.vardir) self.process = subprocess.Popen(args, cwd=self.vardir, stdout=self.log_des, stderr=self.log_des) del(self.log_des) # gh-19 crash detection self.crash_detector = TestRunGreenlet(self.crash_detect) self.crash_detector.info = "Crash detector: %s" % self.process self.crash_detector.start() if wait: try: self.wait_until_started(wait_load) except TarantoolStartError: # Raise exception when caller ask for it (e.g. in case of # non-default servers) if rais: raise # Python tests expect we raise an exception when non-default # server fails if self.crash_expected: raise if not (self.current_test and self.current_test.is_crash_reported): if self.current_test: self.current_test.is_crash_reported = True color_stdout('\n[Instance "{0.name}"] Tarantool server ' 'failed to start\n'.format(self), schema='error') self.print_log(15) # if the server fails before any test started, we should inform # a caller by the exception if not self.current_test: raise self.kill_current_test() port = self.admin.port self.admin.disconnect() self.admin = CON_SWITCH[self.tests_type]('localhost', port) self.status = 'started'
def install(self, silent=True): if self._start_against_running: self._iproto = self._start_against_running self._admin = int(self._start_against_running) + 1 return color_log('Installing the server ...\n', schema='serv_text') color_log(' Found executable at ', schema='serv_text') color_log(self.binary + '\n', schema='path') color_log(' Found tarantoolctl at ', schema='serv_text') color_log(self.ctl_path + '\n', schema='path') color_log(' Creating and populating working directory in ', schema='serv_text') color_log(self.vardir + ' ...\n', schema='path') if not os.path.exists(self.vardir): os.makedirs(self.vardir) else: color_log(' Found old workdir, deleting ...\n', schema='serv_text') self.kill_old_server() self.cleanup() self.copy_files() if self.use_unix_sockets: path = os.path.join(self.vardir, self.name + ".socket-admin") warn_unix_socket(path) self._admin = path else: self._admin = find_port() if self.use_unix_sockets_iproto: path = os.path.join(self.vardir, self.name + ".socket-iproto") warn_unix_socket(path) self._iproto = path else: self._iproto = find_port() # these sockets will be created by tarantool itself path = os.path.join(self.vardir, self.name + '.control') warn_unix_socket(path)
def run_loop(self, task_queue, result_queue): """ called from 'run_all' """ while True: task_id = self.task_get(task_queue) # None is 'stop worker' marker if task_id is None: color_log('Worker "%s" exhausted task queue; ' 'stopping the server...\n' % self.name, schema='test_var') self.stop_worker(task_queue, result_queue) break short_status = None result_checksum = None duration = 0.0 result_queue.put(self.current_task(task_id)) testname = os.path.basename(task_id[0]) fragile_checksums = self.suite.get_test_fragile_checksums(testname) retries_left = self.suite.fragile_retries() # let's run till short_status became 'pass' while short_status != 'pass' and retries_left >= 0: self.restart_server() # print message only after some fails occurred if short_status == 'fail': color_stdout('Test "%s", conf: "%s"\n' '\tfrom "fragile" list failed with results' ' file checksum: "%s", rerunning ...\n' % (task_id[0], task_id[1], result_checksum), schema='error') # run task and save the result to short_status short_status, result_checksum, duration = self.run_task( task_id) # check if the results file checksum set on fail and if # the newly created results file is known by checksum if not result_checksum or (result_checksum not in fragile_checksums): break retries_left = retries_left - 1 result_queue.put( self.wrap_result(task_id, short_status, result_checksum, duration)) if short_status == 'fail': if Options().args.is_force: self.restart_server() color_stdout( 'Worker "%s" got failed test; restarted the server\n' % self.name, schema='test_var') else: color_stdout( 'Worker "%s" got failed test; stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() if self.sigterm_received: color_stdout('Worker "%s" got signal to terminate; ' 'stopping the server...\n' % self.name, schema='test_var') raise VoluntaryStopException() self.task_done(task_queue)
def start(self, silent=True, wait=True, wait_load=True, rais=True, args=[], **kwargs): if self._start_against_running: return if self.status == 'started': if not silent: color_stdout('The server is already started.\n', schema='lerror') return args = self.prepare_args(args) self.pidfile = '%s.pid' % self.name self.logfile = '%s.log' % self.name path = self.script_dst if self.script else \ os.path.basename(self.binary) color_log("Starting the server ...\n", schema='serv_text') color_log("Starting ", schema='serv_text') color_log(path + " \n", schema='path') color_log(self.version() + "\n", schema='version') os.putenv("LISTEN", self.iproto.uri) os.putenv("ADMIN", self.admin.uri) if self.rpl_master: os.putenv("MASTER", self.rpl_master.iproto.uri) self.logfile_pos = self.logfile # redirect stdout from tarantoolctl and tarantool os.putenv("TEST_WORKDIR", self.vardir) self.process = subprocess.Popen(args, cwd=self.vardir, stdout=self.log_des, stderr=self.log_des) del (self.log_des) # gh-19 crash detection self.crash_detector = TestRunGreenlet(self.crash_detect) self.crash_detector.info = "Crash detector: %s" % self.process self.crash_detector.start() if wait: try: self.wait_until_started(wait_load) except TarantoolStartError: # Python tests expect we raise an exception when non-default # server fails if self.crash_expected: raise if not (self.current_test and self.current_test.is_crash_reported): if self.current_test: self.current_test.is_crash_reported = True color_stdout('\n[Instance "{0.name}"] Tarantool server ' 'failed to start\n'.format(self), schema='error') self.print_log(15) # Raise exception when caller ask for it (e.g. in case of # non-default servers) if rais: raise # if the server fails before any test started, we should inform # a caller by the exception if not self.current_test: raise self.kill_current_test() port = self.admin.port self.admin.disconnect() self.admin = CON_SWITCH[self.tests_type]('localhost', port) self.status = 'started'
def stop(self, silent=True, signal=signal.SIGTERM): """ Kill tarantool server using specified signal (SIGTERM by default) signal - a number of a signal """ if self._start_against_running: color_log('Server [%s] start against running ...\n', schema='test_var') return if self.status != 'started': if not silent: raise Exception('Server is not started') else: color_log( 'Server [{0.name}] is not started ' '(status:{0.status}) ...\n'.format(self), schema='test_var' ) return if not silent: color_stdout('[Instance {}] Stopping the server...\n'.format( self.name), schema='info') else: color_log('DEBUG: [Instance {}] Stopping the server...\n'.format( self.name), schema='info') # kill only if process is alive if self.process is not None and self.process.returncode is None: color_log(' | Sending signal {0} ({1}) to {2}\n'.format( signal, signame(signal), format_process(self.process.pid))) try: self.process.send_signal(signal) except OSError: pass # Waiting for stopping the server. If the timeout # reached, send SIGKILL. timeout = 5 def kill(): qa_notice('The server \'{}\' does not stop during {} ' 'seconds after the {} ({}) signal.\n' 'Info: {}\n' 'Sending SIGKILL...'.format( self.name, timeout, signal, signame(signal), format_process(self.process.pid))) try: self.process.kill() except OSError: pass timer = Timer(timeout, kill) timer.start() if self.crash_detector is not None: save_join(self.crash_detector) self.wait_stop() timer.cancel() self.status = None if re.search(r'^/', str(self._admin.port)): if os.path.exists(self._admin.port): os.unlink(self._admin.port)
def install(self, silent=True): if self._start_against_running: self._iproto = self._start_against_running self._admin = int(self._start_against_running) + 1 return color_log('Installing the server ...\n', schema='serv_text') color_log(' Found executable at ', schema='serv_text') color_log(self.binary + '\n', schema='path') color_log(' Found tarantoolctl at ', schema='serv_text') color_log(self.ctl_path + '\n', schema='path') color_log(' Creating and populating working directory in ', schema='serv_text') color_log(self.vardir + ' ...\n', schema='path') if not os.path.exists(self.vardir): os.makedirs(self.vardir) else: color_log(' Found old vardir, deleting ...\n', schema='serv_text') self.kill_old_server() self.cleanup() self.copy_files() if self.use_unix_sockets: self._admin = os.path.join(self.vardir, "socket-admin") else: self._admin = find_port() self._iproto = find_port() # these sockets will be created by tarantool itself path = os.path.join(self.vardir, self.name + '.control') warn_unix_socket(path)
def start(self, silent=True, wait=True, wait_load=True, rais=True, args=[], **kwargs): if self._start_against_running: return if self.status == 'started': if not silent: color_stdout('The server is already started.\n', schema='lerror') return args = self.prepare_args(args) self.pidfile = '%s.pid' % self.name self.logfile = '%s.log' % self.name path = self.script_dst if self.script else \ os.path.basename(self.binary) color_log('DEBUG: [Instance {}] Starting the server...\n'.format( self.name), schema='info') color_log(' | ' + path + '\n', schema='path') color_log(prefix_each_line(' | ', self.version()) + '\n', schema='version') os.putenv("LISTEN", self.iproto.uri) os.putenv("ADMIN", self.admin.uri) if self.rpl_master: os.putenv("MASTER", self.rpl_master.iproto.uri) self.logfile_pos = self.logfile # This is strange, but tarantooctl leans on the PWD # environment variable, not a real current working # directory, when it performs search for the # .tarantoolctl configuration file. os.environ['PWD'] = self.vardir # redirect stdout from tarantoolctl and tarantool os.putenv("TEST_WORKDIR", self.vardir) self.process = subprocess.Popen(args, cwd=self.vardir, stdout=self.log_des, stderr=self.log_des) del(self.log_des) # Restore the actual PWD value. os.environ['PWD'] = os.getcwd() # Track non-default server metrics as part of current # test. if self.current_test: sampler.register_process(self.process.pid, self.current_test.id, self.name) # gh-19 crash detection self.crash_detector = TestRunGreenlet(self.crash_detect) self.crash_detector.info = "Crash detector: %s" % self.process self.crash_detector.start() if wait: try: self.wait_until_started(wait_load) except TarantoolStartError: # Python tests expect we raise an exception when non-default # server fails if self.crash_expected: raise if not (self.current_test and self.current_test.is_crash_reported): if self.current_test: self.current_test.is_crash_reported = True color_stdout('\n[Instance "{0.name}"] Tarantool server ' 'failed to start\n'.format(self), schema='error') self.print_log(15) # Raise exception when caller ask for it (e.g. in case of # non-default servers) if rais: raise # if the server fails before any test started, we should inform # a caller by the exception if not self.current_test: raise self.kill_current_test() port = self.admin.port self.admin.disconnect() self.admin = CON_SWITCH[self.tests_type]('localhost', port) self.status = 'started' # Verify that the schema actually was not upgraded. if self.disable_schema_upgrade: expected_version = extract_schema_from_snapshot(self.snapshot_path) actual_version = tuple(yaml.safe_load(self.admin.execute( 'box.space._schema:get{"version"}'))[0][1:]) if expected_version != actual_version: color_stdout('Schema version check fails: expected ' '{}, got {}\n'.format(expected_version, actual_version), schema='error') raise TarantoolStartError(self.name)