def run_loop(self, task_queue, result_queue):
""" called from 'run_all' """
while True:
task_id = self.task_get(task_queue)
# None is 'stop worker' marker
if task_id is None:
color_log('Worker "%s" exhausted task queue; '
'stopping the server...\n' % self.name,
schema='test_var')
self.stop_worker(task_queue, result_queue)
break
result_queue.put(self.current_task(task_id))
short_status = self.run_task(task_id)
result_queue.put(self.wrap_result(task_id, short_status))
if not lib.Options().args.is_force and short_status == 'fail':
color_stdout(
'Worker "%s" got failed test; stopping the server...\n'
% self.name, schema='test_var')
raise VoluntaryStopException()
if self.sigterm_received:
color_stdout('Worker "%s" got signal to terminate; '
'stopping the server...\n' % self.name,
schema='test_var')
raise VoluntaryStopException()
self.task_done(task_queue)
def stop(self, silent=True):
if self._start_against_running:
return
if self.status != 'started':
if not silent:
raise Exception('Server is not started')
return
if not silent:
color_stdout('Stopping the server ...\n', schema='serv_text')
# kill only if process is alive
if self.process is not None and self.process.returncode is None:
color_log('TarantoolServer.stop(): stopping the %s\n' %
format_process(self.process.pid),
schema='test_var')
try:
self.process.terminate()
except OSError:
pass
if self.crash_detector is not None:
save_join(self.crash_detector)
self.wait_stop()
self.status = None
if re.search(r'^/', str(self._admin.port)):
if os.path.exists(self._admin.port):
os.unlink(self._admin.port)
def execute(self, server):
super(AppTest, self).execute(server)
ts = TestState(self.suite_ini,
None,
TarantoolServer,
self.run_params,
default_server_no_connect=server)
self.inspector.set_parser(ts)
execs = server.prepare_args()
retval = dict()
tarantool = TestRunGreenlet(run_server, execs, server.vardir, server,
server.logfile, retval, self.id)
self.current_test_greenlet = tarantool
# Copy the snapshot right before starting the server.
# Otherwise pretest_clean() would remove it.
if server.snapshot_path:
snapshot_dest = os.path.join(server.vardir, DEFAULT_SNAPSHOT_NAME)
color_log("Copying snapshot {} to {}\n".format(
server.snapshot_path, snapshot_dest))
shutil.copy(server.snapshot_path, snapshot_dest)
try:
tarantool.start()
tarantool.join()
except TarantoolStartError:
# A non-default server failed to start.
raise TestExecutionError
finally:
self.teardown(server, ts)
if retval.get('returncode', None) != 0:
raise TestExecutionError
def prepare_args(self, args=[]):
if not find_in_path('valgrind'):
raise OSError('`valgrind` executables not found in PATH')
orig_args = super(ValgrindMixin, self).prepare_args(args)
res_args = self.valgrind_cmd_args + orig_args
color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var')
return res_args
def handle(self, socket, addr):
if self.parser is None:
raise AttributeError('Parser is not defined')
self.sem.acquire()
for line in self.readline(socket):
color_log('DEBUG: test-run received command: {}\n'.format(line),
schema='test-run command')
try:
result = self.parser.parse_preprocessor(line)
except (KeyboardInterrupt, TarantoolStartError):
# propagate to the main greenlet
raise
except LuaPreprocessorException as e:
qa_notice(str(e))
result = {'error': str(e)}
except Exception as e:
self.parser.kill_current_test()
color_stdout('\nTarantoolInpector.handle() received the ' +
'following error:\n' + traceback.format_exc() +
'\n',
schema='error')
result = {"error": repr(e)}
if result is None:
result = True
result = yaml.dump(result)
if not result.endswith('...\n'):
result = result + '...\n'
color_log("DEBUG: test-run's response for [{}]\n{}\n".format(
line, prefix_each_line(' | ', result)),
schema='test-run command')
socket.sendall(str_to_bytes(result))
self.sem.release()
def find_exe(cls, builddir, silent=True):
cls.builddir = os.path.abspath(builddir)
builddir = os.path.join(builddir, "src")
path = builddir + os.pathsep + os.environ["PATH"]
color_log("Looking for server binary in ", schema='serv_text')
color_log(path + ' ...\n', schema='path')
for _dir in path.split(os.pathsep):
exe = os.path.join(_dir, cls.default_tarantool["bin"])
ctl_dir = _dir
# check local tarantoolctl source
if _dir == builddir:
ctl_dir = os.path.join(_dir, '../extra/dist')
ctl = os.path.join(ctl_dir, cls.default_tarantool['ctl'])
need_lua_path = False
if os.path.isdir(ctl) or not os.access(ctl, os.X_OK):
ctl_dir = os.path.join(_dir, '../extra/dist')
ctl = os.path.join(ctl_dir, cls.default_tarantool['ctl'])
need_lua_path = True
if os.access(exe, os.X_OK) and os.access(ctl, os.X_OK):
cls.binary = os.path.abspath(exe)
cls.ctl_path = os.path.abspath(ctl)
cls.ctl_plugins = os.path.abspath(os.path.join(ctl_dir, '..'))
os.environ["PATH"] = os.pathsep.join([
os.path.abspath(ctl_dir),
os.path.abspath(_dir), os.environ["PATH"]
])
os.environ["TARANTOOLCTL"] = ctl
if need_lua_path:
os.environ["LUA_PATH"] = \
ctl_dir + '/?.lua;' + \
ctl_dir + '/?/init.lua;' + \
os.environ.get("LUA_PATH", ";;")
return exe
raise RuntimeError("Can't find server executable in " + path)
def killall_servers(self, server, ts, crash_occured):
""" kill all servers and crash detectors before stream swap """
color_log('Kill all servers ...\n', schema='info')
server_list = ts.servers.values() + [
server,
]
check_list = [s for s in server_list if 'process' in s.__dict__]
# check that all servers stopped correctly
for server in check_list:
bad_returncode = server.process.returncode not in (None, 0,
-signal.SIGKILL,
-signal.SIGTERM)
# if non-default server crashed but it was expected
# don't kill the default server and crash detectors
crash_occured = crash_occured or \
bad_returncode and not server.crash_expected
for server in check_list:
server.process.poll()
if crash_occured:
# kill all servers and crash detectors on crash
if server.process.returncode is None:
server.process.kill()
if server.crash_detector is not None:
gevent.kill(server.crash_detector)
elif server.process.returncode is not None:
# join crash detectors of stopped servers
if server.crash_detector is not None:
save_join(server.crash_detector)
def prepare_args(self, args=[]):
if not find_in_path('valgrind'):
raise OSError('`valgrind` executables not found in PATH')
orig_args = super(ValgrindMixin, self).prepare_args(args)
res_args = self.valgrind_cmd_args + orig_args
color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var')
return res_args
def run_loop(self, task_queue, result_queue):
""" called from 'run_all' """
while True:
task_id = self.task_get(task_queue)
# None is 'stop worker' marker
if task_id is None:
color_log('Worker "%s" exhausted task queue; '
'stopping the server...\n' % self.name,
schema='test_var')
self.stop_worker(task_queue, result_queue)
break
short_status = self.run_task(task_id)
result_queue.put(self.wrap_result(task_id, short_status))
if not lib.Options().args.is_force and short_status == 'fail':
color_stdout(
'Worker "%s" got failed test; stopping the server...\n' %
self.name,
schema='test_var')
raise VoluntaryStopException()
if self.sigterm_received:
color_stdout('Worker "%s" got signal to terminate; '
'stopping the server...\n' % self.name,
schema='test_var')
raise VoluntaryStopException()
self.task_done(task_queue)
def killall_servers(self, server, ts, crash_occured):
""" kill all servers and crash detectors before stream swap """
color_log('Kill all servers ...\n', schema='info')
check_list = ts.servers.values() + [server, ]
# check that all servers stopped correctly
for server in check_list:
bad_returncode = server.process.returncode not in (None,
0,
-signal.SIGKILL,
-signal.SIGTERM)
# if non-default server crashed but it was expected
# don't kill the default server and crash detectors
crash_occured = crash_occured or \
bad_returncode and not server.crash_expected
for server in check_list:
server.process.poll()
if crash_occured:
# kill all servers and crash detectors on crash
if server.process.returncode is None:
server.process.kill()
if server.crash_detector is not None:
gevent.kill(server.crash_detector)
elif server.process.returncode is not None:
# join crash detectors of stopped servers
if server.crash_detector is not None:
save_join(server.crash_detector)
def prepare_args(self, args=[]):
if not find_in_path('strace'):
raise OSError('`strace` executables not found in PATH')
orig_args = super(StraceMixin, self).prepare_args(args)
res_args = shlex.split("strace -o {log} -f -tt -T -x -I1 {bin}".format(
bin=' '.join(orig_args), log=self.strace_log))
color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var')
return res_args
def server_stop(self, ctype, sname, opts):
color_log('\nDEBUG: TestState[%s].server_stop(%s, %s, %s)\n' % (
hex(id(self)), str(ctype), str(sname), str(opts)), schema='test_var')
if sname not in self.servers:
raise LuaPreprocessorException('Can\'t stop nonexistent server '+repr(sname))
self.connections[sname].disconnect()
self.connections.pop(sname)
self.servers[sname].stop()
def cleanup_nondefault(self):
color_log('\nDEBUG: TestState[%s].cleanup()\n' % hex(id(self)),
schema='test_var')
for k, v in self.servers.iteritems():
# don't cleanup the default server
if k == 'default':
continue
v.cleanup()
def cleanup_nondefault(self):
color_log('\nDEBUG: TestState[%s].cleanup_nondefault()\n' % hex(id(self)),
schema='test_var')
for k, v in self.servers.iteritems():
# don't cleanup the default server
if k == 'default':
continue
v.cleanup()
def stop(self, silent):
if not self.process:
return
color_log('AppServer.stop(): stopping the %s\n' %
format_process(self.process.pid), schema='test_var')
try:
self.process.terminate()
except OSError:
pass
def cleanup_nondefault(self):
names = [k for k in self.servers.keys() if k != 'default']
color_log('DEBUG: Cleanup non-default servers: {}\n'.format(names),
schema='info')
for k, v in self.servers.items():
# don't cleanup the default server
if k == 'default':
continue
v.cleanup()
def stop(self, silent):
if not self.process:
return
color_log('AppServer.stop(): stopping the %s\n' %
format_process(self.process.pid), schema='test_var')
try:
self.process.terminate()
except OSError:
pass
def prepare_args(self, args=[]):
if not find_in_path('strace'):
raise OSError('`strace` executables not found in PATH')
orig_args = super(StraceMixin, self).prepare_args(args)
res_args = shlex.split("strace -o {log} -f -tt -T -x -I1 {bin}".format(
bin=' '.join(orig_args),
log=self.strace_log
))
color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var')
return res_args
def stop(self, silent=True, signal=signal.SIGTERM):
# FIXME: Extract common parts of AppServer.stop() and
# TarantoolServer.stop() to an utility function.
color_log('DEBUG: [app server] Stopping the server...\n',
schema='info')
if not self.process:
color_log(' | Nothing to do: the process does not exist\n',
schema='info')
return
if self.process.returncode:
if self.process.returncode < 0:
signaled_by = -self.process.returncode
color_log(' | Nothing to do: the process was terminated by '
'signal {} ({})\n'.format(signaled_by,
signame(signaled_by)),
schema='info')
else:
color_log(' | Nothing to do: the process was exited with code '
'{}\n'.format(self.process.returncode),
schema='info')
return
color_log(' | Sending signal {0} ({1}) to {2}\n'.format(
signal, signame(signal), format_process(self.process.pid)))
try:
self.process.send_signal(signal)
except OSError:
pass
# Waiting for stopping the server. If the timeout
# reached, send SIGKILL.
timeout = 5
def kill():
qa_notice('The app server does not stop during {} '
'seconds after the {} ({}) signal.\n'
'Info: {}\n'
'Sending SIGKILL...'.format(
timeout, signal, signame(signal),
format_process(self.process.pid)))
try:
self.process.kill()
except OSError:
pass
timer = Timer(timeout, kill)
timer.start()
self.process.wait()
timer.cancel()
def stop_nondefault(self):
color_log('\nDEBUG: TestState[%s].stop_nondefault()\n'
% hex(id(self)), schema='test_var')
if sys.stdout.__class__.__name__ == 'FilteredStream':
sys.stdout.clear_all_filters()
for k, v in self.servers.iteritems():
# don't stop the default server
if k == 'default':
continue
v.stop(silent=True)
if k in self.connections:
self.connections[k].disconnect()
self.connections.pop(k)
def stop_nondefault(self):
color_log('\nDEBUG: TestState[%s].stop_nondefault()\n'
% hex(id(self)), schema='test_var')
if sys.stdout.__class__.__name__ == 'FilteredStream':
sys.stdout.clear_all_filters()
for k, v in self.servers.iteritems():
# don't stop the default server
if k == 'default':
continue
v.stop(silent=True)
if k in self.connections:
self.connections[k].disconnect()
self.connections.pop(k)
def server_create(self, ctype, sname, opts):
color_log('\nDEBUG: TestState[%s].server_create(%s, %s, %s)\n' % (
hex(id(self)), str(ctype), str(sname), str(opts)),
schema='test_var')
if sname in self.servers:
raise LuaPreprocessorException('Server {0} already exists'.format(repr(sname)))
temp = self.create_server()
temp.name = sname
if 'need_init' in opts:
temp.need_init = True if opts['need_init'] == 'True' else False
if 'script' in opts:
temp.script = opts['script'][1:-1]
if 'lua_libs' in opts:
temp.lua_libs = opts['lua_libs'][1:-1].split(' ')
temp.rpl_master = None
if 'rpl_master' in opts:
temp.rpl_master = self.servers[opts['rpl_master']]
temp.vardir = self.suite_ini['vardir']
temp.use_unix_sockets = self.suite_ini['use_unix_sockets']
temp.use_unix_sockets_iproto = self.suite_ini['use_unix_sockets_iproto']
temp.inspector_port = int(self.suite_ini.get(
'inspector_port', temp.DEFAULT_INSPECTOR
))
if self.default_server_no_connect:
temp.current_test = self.default_server_no_connect.current_test
elif self.servers['default']:
temp.current_test = self.servers['default'].current_test
temp.install(silent=True)
self.servers[sname] = temp
if 'workdir' in opts:
copy_from = opts['workdir']
copy_to = self.servers[sname].name
os.system('rm -rf %s/%s' % (
self.servers[sname].vardir, copy_to
))
os.system('cp -r %s %s/%s' % (
copy_from,
self.servers[sname].vardir,
copy_to
))
nmsp = Namespace()
setattr(nmsp, 'admin', temp.admin.port)
setattr(nmsp, 'listen', temp.iproto.port)
if temp.rpl_master:
setattr(nmsp, 'master', temp.rpl_master.iproto.port)
setattr(self.environ, sname, nmsp)
if 'return_listen_uri' in opts and opts['return_listen_uri'] == 'True':
return self.servers[sname].iproto.uri
def stop_nondefault(self, signal=signal.SIGTERM):
names = [k for k in self.servers.keys() if k != 'default']
color_log('DEBUG: Stop non-default servers using '
'signal {} ({}): {}\n'.format(signal, signame(signal),
names),
schema='info')
if sys.stdout.__class__.__name__ == 'FilteredStream':
sys.stdout.clear_all_filters()
for k, v in self.servers.items():
# don't stop the default server
if k == 'default':
continue
v.stop(silent=True, signal=signal)
if k in self.connections:
self.connections[k].disconnect()
self.connections.pop(k)
def send_command_raw(self, command, ts):
""" Send a command to tarantool and read a response. """
color_log('DEBUG: sending command: {}\n'.format(command.rstrip()),
schema='tarantool command')
# Evaluate the request on the first connection, save the
# response.
result = ts.curcon[0](command, silent=True)
# Evaluate on other connections, ignore responses.
for conn in ts.curcon[1:]:
conn(command, silent=True)
# gh-24 fix
if result is None:
result = '[Lost current connection]\n'
color_log("DEBUG: tarantool's response for [{}]\n{}\n".format(
command.rstrip(), prefix_each_line(' | ', result)),
schema='tarantool command')
return result
def kill_old_server(self, silent=True):
pid = self.read_pidfile()
if pid == -1:
return False
if not silent:
color_stdout(
' Found old server, pid {0}, killing ...'.format(pid),
schema='info')
else:
color_log(' Found old server, pid {0}, killing ...'.format(pid),
schema='info')
try:
os.kill(pid, signal.SIGTERM)
except OSError:
pass
self.wait_until_stopped(pid)
return True
def server_create(self, ctype, sname, opts):
color_log('\nDEBUG: TestState[%s].server_create(%s, %s, %s)\n' % (
hex(id(self)), str(ctype), str(sname), str(opts)),
schema='test_var')
if sname in self.servers:
raise LuaPreprocessorException('Server {0} already exists'.format(repr(sname)))
temp = self.create_server()
temp.name = sname
if 'need_init' in opts:
temp.need_init = True if opts['need_init'] == 'True' else False
if 'script' in opts:
temp.script = opts['script'][1:-1]
if 'lua_libs' in opts:
temp.lua_libs = opts['lua_libs'][1:-1].split(' ')
temp.rpl_master = None
if 'rpl_master' in opts:
temp.rpl_master = self.servers[opts['rpl_master']]
temp.vardir = self.suite_ini['vardir']
temp.inspector_port = int(self.suite_ini.get(
'inspector_port', temp.DEFAULT_INSPECTOR
))
if self.default_server_no_connect:
temp.current_test = self.default_server_no_connect.current_test
elif self.servers['default']:
temp.current_test = self.servers['default'].current_test
temp.install(silent=True)
self.servers[sname] = temp
if 'workdir' in opts:
copy_from = opts['workdir']
copy_to = self.servers[sname].name
os.system('rm -rf %s/%s' % (
self.servers[sname].vardir, copy_to
))
os.system('cp -r %s %s/%s' % (
copy_from,
self.servers[sname].vardir,
copy_to
))
nmsp = Namespace()
setattr(nmsp, 'admin', temp.admin.port)
setattr(nmsp, 'listen', temp.iproto.port)
if temp.rpl_master:
setattr(nmsp, 'master', temp.rpl_master.iproto.port)
setattr(self.environ, sname, nmsp)
if 'return_listen_uri' in opts and opts['return_listen_uri'] == 'True':
return self.servers[sname].iproto.uri
def kill_old_server(self, silent=True):
pid = self.read_pidfile()
if pid == -1:
return False
if not silent:
color_stdout(
' Found old server, pid {0}, killing ...'.format(pid),
schema='info'
)
else:
color_log(' Found old server, pid {0}, killing ...'.format(pid),
schema='info')
try:
os.kill(pid, signal.SIGTERM)
except OSError:
pass
self.wait_until_stopped(pid)
return True
def server_stop(self, ctype, sname, opts):
color_log('\nDEBUG: TestState[%s].server_stop(%s, %s, %s)\n' % (
hex(id(self)), str(ctype), str(sname), str(opts)), schema='test_var')
if sname not in self.servers:
raise LuaPreprocessorException('Can\'t stop nonexistent server {0}'.format(repr(sname)))
self.connections[sname].disconnect()
self.connections.pop(sname)
if 'signal' in opts:
# convert to an integer if a number is passed, leave a string
# otherwise
try:
signal = int(opts['signal'])
except ValueError:
signal = opts['signal']
self.servers[sname].stop(silent=True, signal=signum(signal))
else:
# use default signal
self.servers[sname].stop(silent=True)
def copy_files(self):
if self.script:
shutil.copy(self.script, self.script_dst)
os.chmod(self.script_dst, 0o777)
if self.lua_libs:
for i in self.lua_libs:
source = os.path.join(self.testdir, i)
try:
if os.path.isdir(source):
shutil.copytree(source,
os.path.join(self.vardir,
os.path.basename(source)))
else:
shutil.copy(source, self.vardir)
except IOError as e:
if (e.errno == errno.ENOENT):
continue
raise
# Previously tarantoolctl configuration file located in tarantool
# repository at test/ directory. Currently it is located in root
# path of test-run/ submodule repository. For backward compatibility
# this file should be checked at the old place and only after at
# the current.
tntctl_file = '.tarantoolctl'
if not os.path.exists(tntctl_file):
tntctl_file = os.path.join(self.TEST_RUN_DIR, '.tarantoolctl')
shutil.copy(tntctl_file, self.vardir)
shutil.copy(os.path.join(self.TEST_RUN_DIR, 'test_run.lua'),
self.vardir)
if self.snapshot_path:
# Copy snapshot to the workdir.
# Usually Tarantool looking for snapshots on start in a current directory
# or in a directories that specified in memtx_dir or vinyl_dir box settings.
# Before running test current directory (workdir) passed to a new instance in
# an environment variable TEST_WORKDIR and then tarantoolctl
# adds to it instance_name and set to memtx_dir and vinyl_dir.
(instance_name, _) = os.path.splitext(os.path.basename(self.script))
instance_dir = os.path.join(self.vardir, instance_name)
safe_makedirs(instance_dir)
snapshot_dest = os.path.join(instance_dir, DEFAULT_SNAPSHOT_NAME)
color_log("Copying snapshot {} to {}\n".format(
self.snapshot_path, snapshot_dest))
shutil.copy(self.snapshot_path, snapshot_dest)
def _log(self, event, pid):
# Those logs are not written due to gh-247.
process_def = self._processes[pid]
task_id = process_def['task_id']
test_name = task_id[0] + ((':' + task_id[1]) if task_id[1] else '')
worker_name = process_def['worker_name']
server_name = process_def['server_name']
color_log('DEBUG: sampler: {} {}\n'.format(event, format_process(pid)),
schema='info')
color_log(' | worker: {}\n'.format(worker_name))
color_log(' | test: {}\n'.format(test_name))
color_log(' | server: {}\n'.format(str(server_name)))
def prepare_args(self, args=[]):
screen_name = self.debugger_args['screen_name']
debugger = self.debugger_args['debugger']
gdbserver_port = self.debugger_args['gdbserver_port']
gdbserver_opts = self.debugger_args['gdbserver_opts']
sh_string = self.debugger_args['sh_string']
is_under_gdbserver = 'GdbServer' in self.__class__.__name__
if not is_under_gdbserver and not find_in_path('screen'):
raise OSError('`screen` executables not found in PATH')
if not find_in_path(debugger):
raise OSError('`%s` executables not found in PATH' % debugger)
is_tarantoolserver = 'TarantoolServer' in self.__class__.__name__
if is_tarantoolserver or is_under_gdbserver:
color_stdout('\nYou started the server in %s mode.\n' % debugger,
schema='info')
if is_under_gdbserver:
color_stdout("To attach, use `gdb -ex 'target remote :%s'`\n" %
gdbserver_port,
schema='info')
else:
color_stdout('To attach, use `screen -r %s`\n' % screen_name,
schema='info')
# detach only for TarantoolServer
screen_opts = '-d' if is_tarantoolserver else ''
orig_args = super(DebugMixin, self).prepare_args(args)
res_args = shlex.split(
sh_string.format(screen_name=screen_name,
screen_opts=screen_opts,
binary=self.binary,
args=' '.join(orig_args),
logfile=self.logfile,
debugger=debugger,
gdbserver_port=gdbserver_port,
gdbserver_opts=gdbserver_opts))
color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var')
return res_args
def stop(self, silent=True, signal=signal.SIGTERM):
""" Kill tarantool server using specified signal (SIGTERM by default)
signal - a number of a signal
"""
if self._start_against_running:
color_log('Server [%s] start against running ...\n',
schema='test_var')
return
if self.status != 'started':
if not silent:
raise Exception('Server is not started')
else:
color_log(
'Server [{0.name}] is not started '
'(status:{0.status}) ...\n'.format(self),
schema='test_var'
)
return
if not silent:
color_stdout('Stopping the server ...\n', schema='serv_text')
else:
color_log('Stopping the server ...\n', schema='serv_text')
# kill only if process is alive
if self.process is not None and self.process.returncode is None:
color_log('TarantoolServer.stop(): stopping the {0}\n'.format(
format_process(self.process.pid)), schema='test_var')
try:
color_log('Sending signal {0} ({1}) to process {2}\n'.format(
signal, signame(signal), self.process.pid))
self.process.send_signal(signal)
except OSError:
pass
if self.crash_detector is not None:
save_join(self.crash_detector)
self.wait_stop()
self.status = None
if re.search(r'^/', str(self._admin.port)):
if os.path.exists(self._admin.port):
os.unlink(self._admin.port)
def prepare_args(self, args=[]):
screen_name = self.debugger_args['screen_name']
debugger = self.debugger_args['debugger']
gdbserver_port = self.debugger_args['gdbserver_port']
gdbserver_opts = self.debugger_args['gdbserver_opts']
sh_string = self.debugger_args['sh_string']
is_under_gdbserver = 'GdbServer' in self.__class__.__name__
if not is_under_gdbserver and not find_in_path('screen'):
raise OSError('`screen` executables not found in PATH')
if not find_in_path(debugger):
raise OSError('`%s` executables not found in PATH' % debugger)
is_tarantoolserver = 'TarantoolServer' in self.__class__.__name__
if is_tarantoolserver or is_under_gdbserver:
color_stdout('\nYou started the server in %s mode.\n' % debugger,
schema='info')
if is_under_gdbserver:
color_stdout("To attach, use `gdb -ex 'target remote :%s'`\n" %
gdbserver_port, schema='info')
else:
color_stdout('To attach, use `screen -r %s`\n' % screen_name,
schema='info')
# detach only for TarantoolServer
screen_opts = '-d' if is_tarantoolserver else ''
orig_args = super(DebugMixin, self).prepare_args(args)
res_args = shlex.split(sh_string.format(
screen_name=screen_name,
screen_opts=screen_opts,
binary=self.binary,
args=' '.join(orig_args),
logfile=self.logfile,
debugger=debugger,
gdbserver_port=gdbserver_port,
gdbserver_opts=gdbserver_opts))
color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var')
return res_args
def stop(self, silent=True, signal=signal.SIGTERM):
""" Kill tarantool server using specified signal (SIGTERM by default)
signal - a number of a signal
"""
if self._start_against_running:
color_log('Server [%s] start against running ...\n',
schema='test_var')
return
if self.status != 'started':
if not silent:
raise Exception('Server is not started')
else:
color_log('Server [{0.name}] is not started '
'(status:{0.status}) ...\n'.format(self),
schema='test_var')
return
if not silent:
color_stdout('Stopping the server ...\n', schema='serv_text')
else:
color_log('Stopping the server ...\n', schema='serv_text')
# kill only if process is alive
if self.process is not None and self.process.returncode is None:
color_log('TarantoolServer.stop(): stopping the {0}\n'.format(
format_process(self.process.pid)),
schema='test_var')
try:
color_log('Sending signal {0} ({1}) to process {2}\n'.format(
signal, signame(signal), self.process.pid))
self.process.send_signal(signal)
except OSError:
pass
if self.crash_detector is not None:
save_join(self.crash_detector)
self.wait_stop()
self.status = None
if re.search(r'^/', str(self._admin.port)):
if os.path.exists(self._admin.port):
os.unlink(self._admin.port)
def find_exe(cls, builddir, silent=True):
cls.builddir = os.path.abspath(builddir)
builddir = os.path.join(builddir, "src")
path = builddir + os.pathsep + os.environ["PATH"]
color_log("Looking for server binary in ", schema='serv_text')
color_log(path + ' ...\n', schema='path')
for _dir in path.split(os.pathsep):
exe = os.path.join(_dir, cls.default_tarantool["bin"])
ctl_dir = _dir
# check local tarantoolctl source
if _dir == builddir:
ctl_dir = os.path.join(_dir, '../extra/dist')
ctl = os.path.join(ctl_dir, cls.default_tarantool['ctl'])
need_lua_path = False
if os.path.isdir(ctl) or not os.access(ctl, os.X_OK):
ctl_dir = os.path.join(_dir, '../extra/dist')
ctl = os.path.join(ctl_dir, cls.default_tarantool['ctl'])
need_lua_path = True
if os.access(exe, os.X_OK) and os.access(ctl, os.X_OK):
cls.binary = os.path.abspath(exe)
cls.ctl_path = os.path.abspath(ctl)
cls.ctl_plugins = os.path.abspath(
os.path.join(ctl_dir, '..')
)
os.environ["PATH"] = os.pathsep.join([
os.path.abspath(ctl_dir),
os.path.abspath(_dir),
os.environ["PATH"]
])
os.environ["TARANTOOLCTL"] = ctl
if need_lua_path:
os.environ["LUA_PATH"] = \
ctl_dir + '/?.lua;' + \
ctl_dir + '/?/init.lua;' + \
os.environ.get("LUA_PATH", ";;")
return exe
raise RuntimeError("Can't find server executable in " + path)
def server_start(self, ctype, sname, opts):
color_log('\nDEBUG: TestState[%s].server_start(%s, %s, %s)\n' % (
hex(id(self)), str(ctype), str(sname), str(opts)),
schema='test_var')
if sname not in self.servers:
raise LuaPreprocessorException('Can\'t start nonexistent server '+repr(sname))
wait = True
if 'wait' in opts and opts['wait'] == 'False':
wait = False
wait_load = True
if 'wait_load' in opts and opts['wait_load'] == 'False':
wait_load = False
args = []
if 'args' in opts:
args = opts['args'][1:-1].split(' ')
crash_expected = 'crash_expected' in opts and \
opts['crash_expected'] == 'True'
crash_occured = False
try:
if crash_expected:
# disable crash detector
self.servers[sname].crash_expected = True
self.servers[sname].start(silent=True, rais=True, wait=wait,
wait_load=wait_load, args=args)
except Exception as e:
crash_occured = True
if not (crash_expected and \
e.__class__.__name__ == 'TarantoolStartError'):
raise
if not crash_occured:
self.connections[sname] = self.servers[sname].admin
try:
self.connections[sname]('return true', silent=True)
except socket.error as e:
LuaPreprocessorException('Can\'t start server '+repr(sname))
return not crash_occured
def server_start(self, ctype, sname, opts):
color_log('\nDEBUG: TestState[%s].server_start(%s, %s, %s)\n' % (
hex(id(self)), str(ctype), str(sname), str(opts)),
schema='test_var')
if sname not in self.servers:
raise LuaPreprocessorException('Can\'t start nonexistent server {0}'.format(repr(sname)))
wait = True
if 'wait' in opts and opts['wait'] == 'False':
wait = False
wait_load = True
if 'wait_load' in opts and opts['wait_load'] == 'False':
wait_load = False
args = []
if 'args' in opts:
args = opts['args'][1:-1].split(' ')
crash_expected = 'crash_expected' in opts and \
opts['crash_expected'] == 'True'
crash_occured = False
try:
if crash_expected:
# disable crash detector
self.servers[sname].crash_expected = True
self.servers[sname].start(silent=True, rais=True, wait=wait,
wait_load=wait_load, args=args)
except Exception as e:
crash_occured = True
if not (crash_expected and
e.__class__.__name__ == 'TarantoolStartError'):
raise
if not crash_occured:
self.connections[sname] = self.servers[sname].admin
try:
self.connections[sname]('return true', silent=True)
except socket.error as e:
LuaPreprocessorException('Can\'t start server {0}'.format(repr(sname)))
return not crash_occured
def install(self, silent=True):
if self._start_against_running:
self._iproto = self._start_against_running
self._admin = int(self._start_against_running) + 1
return
color_log('DEBUG: [Instance {}] Installing the server...\n'.format(
self.name), schema='info')
color_log(' | Found executable at {}\n'.format(self.binary))
color_log(' | Found tarantoolctl at {}\n'.format(self.ctl_path))
color_log(' | Creating and populating working directory in '
'{}...\n'.format(self.vardir))
if not os.path.exists(self.vardir):
os.makedirs(self.vardir)
else:
color_log(' | Found old workdir, deleting...\n')
self.kill_old_server()
self.cleanup()
self.copy_files()
if self.use_unix_sockets:
path = os.path.join(self.vardir, self.name + ".socket-admin")
warn_unix_socket(path)
self._admin = path
else:
self._admin = find_port()
if self.use_unix_sockets_iproto:
path = os.path.join(self.vardir, self.name + ".socket-iproto")
warn_unix_socket(path)
self._iproto = path
else:
self._iproto = find_port()
# these sockets will be created by tarantool itself
path = os.path.join(self.vardir, self.name + '.control')
warn_unix_socket(path)
def wait_until_started(self, wait_load=True):
""" Wait until server is started.
Server consists of two parts:
1) wait until server is listening on sockets
2) wait until server tells us his status
"""
color_log('DEBUG: [Instance {}] Waiting until started '
'(wait_load={})\n'.format(self.name, str(wait_load)),
schema='info')
if wait_load:
msg = 'entering the event loop|will retry binding|hot standby mode'
p = self.process if not self.gdb and not self.lldb else None
self.logfile_pos.seek_wait(msg, p, self.name)
while True:
try:
temp = AdminConnection('localhost', self.admin.port)
if not wait_load:
ans = yaml.safe_load(temp.execute("2 + 2"))
color_log(" | Successful connection check; don't wait for "
"loading")
return True
ans = yaml.safe_load(temp.execute('box.info.status'))[0]
if ans in ('running', 'hot_standby', 'orphan'):
color_log(" | Started {} (box.info.status: '{}')\n".format(
format_process(self.process.pid), ans))
return True
elif ans in ('loading',):
continue
else:
raise Exception(
"Strange output for `box.info.status`: %s" % (ans)
)
except socket.error as e:
if e.errno == errno.ECONNREFUSED:
color_log(' | Connection refused; will retry every 0.1 '
'seconds...')
time.sleep(0.1)
continue
raise
def start(self, silent=True, wait=True, wait_load=True, rais=True, args=[],
**kwargs):
if self._start_against_running:
return
if self.status == 'started':
if not silent:
color_stdout('The server is already started.\n',
schema='lerror')
return
args = self.prepare_args(args)
self.pidfile = '%s.pid' % self.name
self.logfile = '%s.log' % self.name
path = self.script_dst if self.script else \
os.path.basename(self.binary)
color_log("Starting the server ...\n", schema='serv_text')
color_log("Starting ", schema='serv_text')
color_log(path + " \n", schema='path')
color_log(self.version() + "\n", schema='version')
os.putenv("LISTEN", self.iproto.uri)
os.putenv("ADMIN", self.admin.uri)
if self.rpl_master:
os.putenv("MASTER", self.rpl_master.iproto.uri)
self.logfile_pos = self.logfile
# redirect stdout from tarantoolctl and tarantool
os.putenv("TEST_WORKDIR", self.vardir)
self.process = subprocess.Popen(args,
cwd=self.vardir,
stdout=self.log_des,
stderr=self.log_des)
del(self.log_des)
# gh-19 crash detection
self.crash_detector = TestRunGreenlet(self.crash_detect)
self.crash_detector.info = "Crash detector: %s" % self.process
self.crash_detector.start()
if wait:
try:
self.wait_until_started(wait_load)
except TarantoolStartError:
# Raise exception when caller ask for it (e.g. in case of
# non-default servers)
if rais:
raise
# Python tests expect we raise an exception when non-default
# server fails
if self.crash_expected:
raise
if not (self.current_test and
self.current_test.is_crash_reported):
if self.current_test:
self.current_test.is_crash_reported = True
color_stdout('\n[Instance "{0.name}"] Tarantool server '
'failed to start\n'.format(self),
schema='error')
self.print_log(15)
# if the server fails before any test started, we should inform
# a caller by the exception
if not self.current_test:
raise
self.kill_current_test()
port = self.admin.port
self.admin.disconnect()
self.admin = CON_SWITCH[self.tests_type]('localhost', port)
self.status = 'started'
def install(self, silent=True):
if self._start_against_running:
self._iproto = self._start_against_running
self._admin = int(self._start_against_running) + 1
return
color_log('Installing the server ...\n', schema='serv_text')
color_log(' Found executable at ', schema='serv_text')
color_log(self.binary + '\n', schema='path')
color_log(' Found tarantoolctl at ', schema='serv_text')
color_log(self.ctl_path + '\n', schema='path')
color_log(' Creating and populating working directory in ',
schema='serv_text')
color_log(self.vardir + ' ...\n', schema='path')
if not os.path.exists(self.vardir):
os.makedirs(self.vardir)
else:
color_log(' Found old workdir, deleting ...\n',
schema='serv_text')
self.kill_old_server()
self.cleanup()
self.copy_files()
if self.use_unix_sockets:
path = os.path.join(self.vardir, self.name + ".socket-admin")
warn_unix_socket(path)
self._admin = path
else:
self._admin = find_port()
if self.use_unix_sockets_iproto:
path = os.path.join(self.vardir, self.name + ".socket-iproto")
warn_unix_socket(path)
self._iproto = path
else:
self._iproto = find_port()
# these sockets will be created by tarantool itself
path = os.path.join(self.vardir, self.name + '.control')
warn_unix_socket(path)
def run_loop(self, task_queue, result_queue):
""" called from 'run_all' """
while True:
task_id = self.task_get(task_queue)
# None is 'stop worker' marker
if task_id is None:
color_log('Worker "%s" exhausted task queue; '
'stopping the server...\n' % self.name,
schema='test_var')
self.stop_worker(task_queue, result_queue)
break
short_status = None
result_checksum = None
duration = 0.0
result_queue.put(self.current_task(task_id))
testname = os.path.basename(task_id[0])
fragile_checksums = self.suite.get_test_fragile_checksums(testname)
retries_left = self.suite.fragile_retries()
# let's run till short_status became 'pass'
while short_status != 'pass' and retries_left >= 0:
self.restart_server()
# print message only after some fails occurred
if short_status == 'fail':
color_stdout('Test "%s", conf: "%s"\n'
'\tfrom "fragile" list failed with results'
' file checksum: "%s", rerunning ...\n' %
(task_id[0], task_id[1], result_checksum),
schema='error')
# run task and save the result to short_status
short_status, result_checksum, duration = self.run_task(
task_id)
# check if the results file checksum set on fail and if
# the newly created results file is known by checksum
if not result_checksum or (result_checksum
not in fragile_checksums):
break
retries_left = retries_left - 1
result_queue.put(
self.wrap_result(task_id, short_status, result_checksum,
duration))
if short_status == 'fail':
if Options().args.is_force:
self.restart_server()
color_stdout(
'Worker "%s" got failed test; restarted the server\n' %
self.name,
schema='test_var')
else:
color_stdout(
'Worker "%s" got failed test; stopping the server...\n'
% self.name,
schema='test_var')
raise VoluntaryStopException()
if self.sigterm_received:
color_stdout('Worker "%s" got signal to terminate; '
'stopping the server...\n' % self.name,
schema='test_var')
raise VoluntaryStopException()
self.task_done(task_queue)
def start(self,
silent=True,
wait=True,
wait_load=True,
rais=True,
args=[],
**kwargs):
if self._start_against_running:
return
if self.status == 'started':
if not silent:
color_stdout('The server is already started.\n',
schema='lerror')
return
args = self.prepare_args(args)
self.pidfile = '%s.pid' % self.name
self.logfile = '%s.log' % self.name
path = self.script_dst if self.script else \
os.path.basename(self.binary)
color_log("Starting the server ...\n", schema='serv_text')
color_log("Starting ", schema='serv_text')
color_log(path + " \n", schema='path')
color_log(self.version() + "\n", schema='version')
os.putenv("LISTEN", self.iproto.uri)
os.putenv("ADMIN", self.admin.uri)
if self.rpl_master:
os.putenv("MASTER", self.rpl_master.iproto.uri)
self.logfile_pos = self.logfile
# redirect stdout from tarantoolctl and tarantool
os.putenv("TEST_WORKDIR", self.vardir)
self.process = subprocess.Popen(args,
cwd=self.vardir,
stdout=self.log_des,
stderr=self.log_des)
del (self.log_des)
# gh-19 crash detection
self.crash_detector = TestRunGreenlet(self.crash_detect)
self.crash_detector.info = "Crash detector: %s" % self.process
self.crash_detector.start()
if wait:
try:
self.wait_until_started(wait_load)
except TarantoolStartError:
# Python tests expect we raise an exception when non-default
# server fails
if self.crash_expected:
raise
if not (self.current_test
and self.current_test.is_crash_reported):
if self.current_test:
self.current_test.is_crash_reported = True
color_stdout('\n[Instance "{0.name}"] Tarantool server '
'failed to start\n'.format(self),
schema='error')
self.print_log(15)
# Raise exception when caller ask for it (e.g. in case of
# non-default servers)
if rais:
raise
# if the server fails before any test started, we should inform
# a caller by the exception
if not self.current_test:
raise
self.kill_current_test()
port = self.admin.port
self.admin.disconnect()
self.admin = CON_SWITCH[self.tests_type]('localhost', port)
self.status = 'started'
def stop(self, silent=True, signal=signal.SIGTERM):
""" Kill tarantool server using specified signal (SIGTERM by default)
signal - a number of a signal
"""
if self._start_against_running:
color_log('Server [%s] start against running ...\n',
schema='test_var')
return
if self.status != 'started':
if not silent:
raise Exception('Server is not started')
else:
color_log(
'Server [{0.name}] is not started '
'(status:{0.status}) ...\n'.format(self),
schema='test_var'
)
return
if not silent:
color_stdout('[Instance {}] Stopping the server...\n'.format(
self.name), schema='info')
else:
color_log('DEBUG: [Instance {}] Stopping the server...\n'.format(
self.name), schema='info')
# kill only if process is alive
if self.process is not None and self.process.returncode is None:
color_log(' | Sending signal {0} ({1}) to {2}\n'.format(
signal, signame(signal),
format_process(self.process.pid)))
try:
self.process.send_signal(signal)
except OSError:
pass
# Waiting for stopping the server. If the timeout
# reached, send SIGKILL.
timeout = 5
def kill():
qa_notice('The server \'{}\' does not stop during {} '
'seconds after the {} ({}) signal.\n'
'Info: {}\n'
'Sending SIGKILL...'.format(
self.name, timeout, signal, signame(signal),
format_process(self.process.pid)))
try:
self.process.kill()
except OSError:
pass
timer = Timer(timeout, kill)
timer.start()
if self.crash_detector is not None:
save_join(self.crash_detector)
self.wait_stop()
timer.cancel()
self.status = None
if re.search(r'^/', str(self._admin.port)):
if os.path.exists(self._admin.port):
os.unlink(self._admin.port)
def install(self, silent=True):
if self._start_against_running:
self._iproto = self._start_against_running
self._admin = int(self._start_against_running) + 1
return
color_log('Installing the server ...\n', schema='serv_text')
color_log(' Found executable at ', schema='serv_text')
color_log(self.binary + '\n', schema='path')
color_log(' Found tarantoolctl at ', schema='serv_text')
color_log(self.ctl_path + '\n', schema='path')
color_log(' Creating and populating working directory in ',
schema='serv_text')
color_log(self.vardir + ' ...\n', schema='path')
if not os.path.exists(self.vardir):
os.makedirs(self.vardir)
else:
color_log(' Found old vardir, deleting ...\n',
schema='serv_text')
self.kill_old_server()
self.cleanup()
self.copy_files()
if self.use_unix_sockets:
self._admin = os.path.join(self.vardir, "socket-admin")
else:
self._admin = find_port()
self._iproto = find_port()
# these sockets will be created by tarantool itself
path = os.path.join(self.vardir, self.name + '.control')
warn_unix_socket(path)
def start(self, silent=True, wait=True, wait_load=True, rais=True, args=[],
**kwargs):
if self._start_against_running:
return
if self.status == 'started':
if not silent:
color_stdout('The server is already started.\n',
schema='lerror')
return
args = self.prepare_args(args)
self.pidfile = '%s.pid' % self.name
self.logfile = '%s.log' % self.name
path = self.script_dst if self.script else \
os.path.basename(self.binary)
color_log('DEBUG: [Instance {}] Starting the server...\n'.format(
self.name), schema='info')
color_log(' | ' + path + '\n', schema='path')
color_log(prefix_each_line(' | ', self.version()) + '\n',
schema='version')
os.putenv("LISTEN", self.iproto.uri)
os.putenv("ADMIN", self.admin.uri)
if self.rpl_master:
os.putenv("MASTER", self.rpl_master.iproto.uri)
self.logfile_pos = self.logfile
# This is strange, but tarantooctl leans on the PWD
# environment variable, not a real current working
# directory, when it performs search for the
# .tarantoolctl configuration file.
os.environ['PWD'] = self.vardir
# redirect stdout from tarantoolctl and tarantool
os.putenv("TEST_WORKDIR", self.vardir)
self.process = subprocess.Popen(args,
cwd=self.vardir,
stdout=self.log_des,
stderr=self.log_des)
del(self.log_des)
# Restore the actual PWD value.
os.environ['PWD'] = os.getcwd()
# Track non-default server metrics as part of current
# test.
if self.current_test:
sampler.register_process(self.process.pid, self.current_test.id,
self.name)
# gh-19 crash detection
self.crash_detector = TestRunGreenlet(self.crash_detect)
self.crash_detector.info = "Crash detector: %s" % self.process
self.crash_detector.start()
if wait:
try:
self.wait_until_started(wait_load)
except TarantoolStartError:
# Python tests expect we raise an exception when non-default
# server fails
if self.crash_expected:
raise
if not (self.current_test and
self.current_test.is_crash_reported):
if self.current_test:
self.current_test.is_crash_reported = True
color_stdout('\n[Instance "{0.name}"] Tarantool server '
'failed to start\n'.format(self),
schema='error')
self.print_log(15)
# Raise exception when caller ask for it (e.g. in case of
# non-default servers)
if rais:
raise
# if the server fails before any test started, we should inform
# a caller by the exception
if not self.current_test:
raise
self.kill_current_test()
port = self.admin.port
self.admin.disconnect()
self.admin = CON_SWITCH[self.tests_type]('localhost', port)
self.status = 'started'
# Verify that the schema actually was not upgraded.
if self.disable_schema_upgrade:
expected_version = extract_schema_from_snapshot(self.snapshot_path)
actual_version = tuple(yaml.safe_load(self.admin.execute(
'box.space._schema:get{"version"}'))[0][1:])
if expected_version != actual_version:
color_stdout('Schema version check fails: expected '
'{}, got {}\n'.format(expected_version,
actual_version),
schema='error')
raise TarantoolStartError(self.name)
