def run_loop(self, task_queue, result_queue):
""" called from 'run_all' """
while True:
task_id = self.task_get(task_queue)
# None is 'stop worker' marker
if task_id is None:
color_log('Worker "%s" exhausted task queue; '
'stopping the server...\n' % self.name,
schema='test_var')
self.stop_worker(task_queue, result_queue)
break
result_queue.put(self.current_task(task_id))
short_status = self.run_task(task_id)
result_queue.put(self.wrap_result(task_id, short_status))
if not lib.Options().args.is_force and short_status == 'fail':
color_stdout(
'Worker "%s" got failed test; stopping the server...\n'
% self.name, schema='test_var')
raise VoluntaryStopException()
if self.sigterm_received:
color_stdout('Worker "%s" got signal to terminate; '
'stopping the server...\n' % self.name,
schema='test_var')
raise VoluntaryStopException()
self.task_done(task_queue)
def print_unidiff(self):
"""Print a unified diff between .test and .result files. Used
to establish the cause of a failure when .test differs
from .result."""
color_stdout("\nTest failed! Result content mismatch:\n",
schema='error')
utils_print_unidiff(self.result, self.reject)
def print_log(self, lines=None):
msg = '\n{prefix} of Tarantool Log file [Instance "{instance}"][{logfile}]:\n'.format(
prefix="Last {0} lines".format(lines) if lines else "Output",
instance=self.name,
logfile=self.logfile or 'null')
color_stdout(msg, schema='error')
if os.path.exists(self.logfile):
print_tail_n(self.logfile, lines)
else:
color_stdout(" Can't find log:\n", schema='error')
def main_parallel():
res = EXIT_UNKNOWN_ERROR
try:
res = main_loop_parallel()
except KeyboardInterrupt:
color_stdout('\n[Main process] Caught keyboard interrupt\n',
schema='test_var')
res = EXIT_INTERRUPTED
return res
def process_result(self, obj):
if not isinstance(obj, WorkerTaskResult):
return
if obj.short_status == 'fail':
color_stdout('[Main process] Got failed test; '
'gently terminate all workers...\n',
schema='test_var')
self.got_fail = True
self.terminate_all_workers()
def warn_unix_socket(path):
real_path = os.path.realpath(path)
if len(real_path) <= UNIX_SOCKET_LEN_LIMIT or \
real_path in warn_unix_socket.warned:
return
color_stdout(
'\nWARGING: unix socket\'s "%s" path has length %d symbols that is '
'longer than %d. That likely will cause failing of tests.\n' %
(real_path, len(real_path), UNIX_SOCKET_LEN_LIMIT), schema='error')
warn_unix_socket.warned.add(real_path)
def print_log(self, lines=None):
msg = ('\n{prefix} of Tarantool Log file [Instance "{instance}"]' +
'[{logfile}]:\n').format(prefix="Last {0} lines".format(lines)
if lines else "Output",
instance=self.name,
logfile=self.logfile or 'null')
color_stdout(msg, schema='error')
if os.path.exists(self.logfile):
print_tail_n(self.logfile, lines)
else:
color_stdout(" Can't find log:\n", schema='error')
def warn_unix_socket(path):
real_path = os.path.realpath(path)
if len(real_path) <= UNIX_SOCKET_LEN_LIMIT or \
real_path in warn_unix_socket.warned:
return
color_stdout(
'\nWARGING: unix socket\'s "%s" path has length %d symbols that is '
'longer than %d. That likely will cause failing of tests.\n' %
(real_path, len(real_path), UNIX_SOCKET_LEN_LIMIT),
schema='error')
warn_unix_socket.warned.add(real_path)
def process_result(self, obj):
if not isinstance(obj, WorkerTaskResult):
return
if obj.short_status == 'fail':
color_stdout(
'[Main process] Got failed test; '
'gently terminate all workers...\n',
schema='test_var')
self.got_fail = True
self.terminate_all_workers()
def warn_unix_sockets_at_start(vardir):
max_unix_socket_rel = '???_replication/autobootstrap_guest3.control'
real_vardir = os.path.realpath(vardir)
max_unix_socket_abs = os.path.join(real_vardir, max_unix_socket_rel)
max_unix_socket_real = os.path.realpath(max_unix_socket_abs)
if len(max_unix_socket_real) > UNIX_SOCKET_LEN_LIMIT:
color_stdout(
'WARGING: unix sockets can become longer than %d symbols:\n'
% UNIX_SOCKET_LEN_LIMIT,
schema='error')
color_stdout('WARNING: for example: "%s" has length %d\n' %
(max_unix_socket_real, len(max_unix_socket_real)),
schema='error')
def warn_unix_sockets_at_start(vardir):
max_unix_socket_rel = '???_replication/autobootstrap_guest3.control'
real_vardir = os.path.realpath(vardir)
max_unix_socket_abs = os.path.join(real_vardir, max_unix_socket_rel)
max_unix_socket_real = os.path.realpath(max_unix_socket_abs)
if len(max_unix_socket_real) > UNIX_SOCKET_LEN_LIMIT:
color_stdout(
'WARGING: unix sockets can become longer than %d symbols:\n' %
UNIX_SOCKET_LEN_LIMIT,
schema='error')
color_stdout('WARNING: for example: "%s" has length %d\n' %
(max_unix_socket_real, len(max_unix_socket_real)),
schema='error')
def kill_old_server(self, silent=True):
pid = self.read_pidfile()
if pid == -1:
return False
if not silent:
color_stdout(
' Found old server, pid {0}, killing ...'.format(pid),
schema='info')
try:
os.kill(pid, signal.SIGTERM)
except OSError:
pass
self.wait_until_stopped(pid)
return True
def check(self):
"""Check the arguments for correctness."""
check_error = False
conflict_options = ('valgrind', 'gdb', 'lldb', 'strace')
for op1, op2 in product(conflict_options, repeat=2):
if op1 != op2 and getattr(self, op1, '') and \
getattr(self, op2, ''):
format_str = "Error: option --{} is not compatible \
with option --{}"
color_stdout(format_str.format(op1, op2), schema='error')
check_error = True
if check_error:
exit(-1)
def check(self):
"""Check the arguments for correctness."""
check_error = False
conflict_options = ('valgrind', 'gdb', 'lldb', 'strace')
for op1, op2 in product(conflict_options, repeat=2):
if op1 != op2 and getattr(self, op1, '') and \
getattr(self, op2, ''):
format_str = "Error: option --{} is not compatible \
with option --{}"
color_stdout(format_str.format(op1, op2), schema='error')
check_error = True
if check_error:
exit(-1)
def check_tap_output(self):
""" Returns is_tap, is_ok """
if not os.path.isfile(self.tmp_result):
color_strout('\nCannot find %s\n' % self.tmp_result,
schema='error')
self.is_crash_reported = True
return False
with open(self.tmp_result, 'r') as f:
content = f.read()
tap = pytap13.TAP13()
try:
tap.parse(content)
except ValueError as e:
color_stdout('\nTAP13 parse failed: %s\n' % str(e), schema='error')
self.is_crash_reported = True
return False, False
is_ok = True
for test_case in tap.tests:
if test_case.result == 'ok':
continue
if is_ok:
color_stdout('\n')
color_stdout(
'%s %s %s # %s %s\n' %
(test_case.result, test_case.id or '', test_case.description
or '-', test_case.directive or '', test_case.comment or ''),
schema='error')
if test_case.yaml:
self.tap_parse_print_yaml(test_case.yaml)
is_ok = False
if not is_ok:
color_stdout('Rejected result file: %s\n' % self.reject,
schema='test_var')
self.is_crash_reported = True
return True, is_ok
def parse_reproduce_file(filepath):
reproduce = []
if not filepath:
return reproduce
try:
with open(filepath, 'r') as f:
for task_id in yaml.safe_load(f):
task_name, task_conf = task_id
reproduce.append((task_name, task_conf))
except IOError:
color_stdout('Cannot read "%s" passed as --reproduce argument\n' %
filepath, schema='error')
exit(1)
return reproduce
def run_all(self, task_queue, result_queue):
if not self.initialized:
self.flush_all_tasks(task_queue, result_queue)
result_queue.put(self.done_marker())
return
try:
self.run_loop(task_queue, result_queue)
except (KeyboardInterrupt, Exception) as e:
if not isinstance(e, KeyboardInterrupt) and \
not isinstance(e, VoluntaryStopException):
color_stdout('Exception: %s\n' % e, schema='error')
self.stop_worker(task_queue, result_queue, cleanup=False)
result_queue.put(self.done_marker())
def run_all(self, task_queue, result_queue):
if not self.initialized:
self.flush_all_tasks(task_queue, result_queue)
result_queue.put(self.done_marker())
return
try:
self.run_loop(task_queue, result_queue)
except (KeyboardInterrupt, Exception) as e:
if not isinstance(e, KeyboardInterrupt) and \
not isinstance(e, VoluntaryStopException):
color_stdout('Exception: %s\n' % e, schema='error')
self.stop_worker(task_queue, result_queue, cleanup=False)
result_queue.put(self.done_marker())
def parse_reproduce_file(filepath):
reproduce = []
if not filepath:
return reproduce
try:
with open(filepath, 'r') as f:
for task_id in yaml.load(f):
task_name, task_conf = task_id
reproduce.append((task_name, task_conf))
except IOError:
color_stdout('Cannot read "%s" passed as --reproduce argument\n' %
filepath,
schema='error')
exit(1)
return reproduce
def check_for_dead_processes(self):
for pid in self.pids:
exited = False
try:
os.waitpid(pid, os.WNOHANG)
except OSError:
exited = True
if exited:
worker_id = self.pid_to_worker_id[pid]
color_stdout(
"[Main process] Worker %d don't reported work "
"done using results queue, but the corresponding "
"process seems dead. Removing it from Dispatcher.\n"
% worker_id, schema='test_var')
self.del_worker(worker_id)
def process_file(filepath):
fh = None
try:
fh = open(filepath, 'r')
lines = fh.readlines()
ctime = time.ctime(os.stat(filepath).st_mtime)
except Exception:
if not os.path.exists(filepath):
color_stdout('[File does not exists: {}]'.format(filepath),
schema='error')
lines = []
ctime = time.ctime()
if fh:
fh.close()
return lines, ctime
def print_exe(cls):
color_stdout('Tarantool server information\n', schema='info')
if cls.binary:
color_stdout(' | Found executable at {}\n'.format(cls.binary))
if cls.ctl_path:
color_stdout(' | Found tarantoolctl at {}\n'.format(cls.ctl_path))
color_stdout('\n' + prefix_each_line(' | ', cls.version()) + '\n',
schema='version')
def check_for_dead_processes(self):
for pid in self.pids:
exited = False
try:
os.waitpid(pid, os.WNOHANG)
except OSError:
exited = True
if exited:
worker_id = self.pid_to_worker_id[pid]
color_stdout(
"[Main process] Worker %d don't reported work "
"done using results queue, but the corresponding "
"process seems dead. Removing it from Dispatcher.\n" %
worker_id,
schema='test_var')
self.del_worker(worker_id)
def save_join(green_obj, timeout=None):
"""
Gevent join wrapper for
test-run stop-on-crash/stop-on-timeout feature
"""
try:
green_obj.get(timeout=timeout)
except Timeout:
color_stdout("Test timeout of %d secs reached\t" % timeout, schema='error')
# We should kill the greenlet that writes to a temporary
# result file. If the same test is run several times (e.g.
# on different configurations), this greenlet may wake up
# and write to the temporary result file of the new run of
# the test.
green_obj.kill()
except GreenletExit:
pass
def print_unidiff(self):
"""Print a unified diff between .test and .result files. Used
to establish the cause of a failure when .test differs
from .result."""
color_stdout("\nTest failed! Result content mismatch:\n",
schema='error')
with open(self.result, "r") as result:
with open(self.reject, "r") as reject:
result_time = time.ctime(os.stat(self.result).st_mtime)
reject_time = time.ctime(os.stat(self.reject).st_mtime)
diff = difflib.unified_diff(result.readlines(),
reject.readlines(), self.result,
self.reject, result_time,
reject_time)
color_stdout.writeout_unidiff(diff)
def check_libs():
deps = [('msgpack', 'msgpack-python'), ('tarantool', 'tarantool-python')]
base_path = os.path.dirname(os.path.abspath(__file__))
for (mod_name, mod_dir) in deps:
mod_path = os.path.join(base_path, mod_dir)
if mod_path not in sys.path:
sys.path = [mod_path] + sys.path
for (mod_name, _mod_dir) in deps:
try:
__import__(mod_name)
except ImportError as e:
color_stdout("\n\nNo %s library found\n" % mod_name,
schema='error')
print(e)
sys.exit(1)
def run_task(self, task_id):
if not self.initialized:
return self.done_marker()
try:
task = self.find_task(task_id)
with open(self.reproduce_file, 'a') as f:
f.write('- ' + yaml.safe_dump(task.id))
short_status = self.suite.run_test(
task, self.server, self.inspector)
except KeyboardInterrupt:
self.report_keyboard_interrupt()
raise
except Exception:
color_stdout(
'\nWorker "%s" received the following error; stopping...\n'
% self.name + traceback.format_exc() + '\n', schema='error')
raise
return short_status
def kill_old_server(self, silent=True):
pid = self.read_pidfile()
if pid == -1:
return False
if not silent:
color_stdout(
' Found old server, pid {0}, killing ...'.format(pid),
schema='info'
)
else:
color_log(' Found old server, pid {0}, killing ...'.format(pid),
schema='info')
try:
os.kill(pid, signal.SIGTERM)
except OSError:
pass
self.wait_until_stopped(pid)
return True
def run_task(self, task_id):
if not self.initialized:
return self.done_marker()
try:
task = self.find_task(task_id)
with open(self.reproduce_file, 'a') as f:
f.write('- ' + yaml.safe_dump(task.id))
short_status = self.suite.run_test(task, self.server,
self.inspector)
except KeyboardInterrupt:
self.report_keyboard_interrupt()
raise
except Exception as e:
color_stdout(
'\nWorker "%s" received the following error; stopping...\n' %
self.name + traceback.format_exc() + '\n',
schema='error')
raise
return short_status
def check_libs():
deps = [
('msgpack', 'msgpack-python'),
('tarantool', 'tarantool-python')
]
base_path = os.path.dirname(os.path.abspath(__file__))
for (mod_name, mod_dir) in deps:
mod_path = os.path.join(base_path, mod_dir)
if mod_path not in sys.path:
sys.path = [mod_path] + sys.path
for (mod_name, _mod_dir) in deps:
try:
__import__(mod_name)
except ImportError as e:
color_stdout("\n\nNo %s library found\n" % mod_name, schema='error')
print(e)
sys.exit(1)
def process_timeout(self, delta_seconds):
self.warned_seconds_ago += delta_seconds
self.inactivity += delta_seconds
worker_ids = self.get_not_done_worker_ids()
if self.warned_seconds_ago < self.warn_timeout:
return
is_warning = self.inactivity < self.kill_timeout
color_schema = 'test_var' if is_warning else 'error'
color_stdout(
"No output during {0.inactivity:.0f} seconds. "
"Will abort after {0.kill_timeout:.0f} seconds without output. "
"List of workers not reporting the status:\n".format(self),
schema=color_schema)
hung_tasks = [
task for worker_id, task in self.worker_current_task.items()
if worker_id in worker_ids
]
for task in hung_tasks:
result_file = task.task_tmp_result
result_file_summary = '(no result file {})'.format(result_file)
if os.path.exists(result_file):
with open(result_file, 'r') as f:
lines = sum(1 for _ in f)
result_file_summary = 'at {}:{}'.format(result_file, lines)
color_stdout('- {} [{}, {}] {}\n'.format(task.worker_name,
task.task_name,
task.task_param,
result_file_summary),
schema=color_schema)
self.warned_seconds_ago = 0.0
if is_warning:
return
for task in hung_tasks:
color_stdout("Test hung! Result content mismatch:\n",
schema='error')
print_unidiff(task.task_result, task.task_tmp_result)
color_stdout(
'\n[Main process] No output from workers. '
'It seems that we hang. Send SIGKILL to workers; '
'exiting...\n',
schema='error')
self.kill_all_workers()
raise HangError()
def stop(self, silent=True, signal=signal.SIGTERM):
""" Kill tarantool server using specified signal (SIGTERM by default)
signal - a number of a signal
"""
if self._start_against_running:
color_log('Server [%s] start against running ...\n',
schema='test_var')
return
if self.status != 'started':
if not silent:
raise Exception('Server is not started')
else:
color_log(
'Server [{0.name}] is not started '
'(status:{0.status}) ...\n'.format(self),
schema='test_var'
)
return
if not silent:
color_stdout('Stopping the server ...\n', schema='serv_text')
else:
color_log('Stopping the server ...\n', schema='serv_text')
# kill only if process is alive
if self.process is not None and self.process.returncode is None:
color_log('TarantoolServer.stop(): stopping the {0}\n'.format(
format_process(self.process.pid)), schema='test_var')
try:
color_log('Sending signal {0} ({1}) to process {2}\n'.format(
signal, signame(signal), self.process.pid))
self.process.send_signal(signal)
except OSError:
pass
if self.crash_detector is not None:
save_join(self.crash_detector)
self.wait_stop()
self.status = None
if re.search(r'^/', str(self._admin.port)):
if os.path.exists(self._admin.port):
os.unlink(self._admin.port)
def process_timeout(self, delta_seconds):
self.warned_seconds_ago += delta_seconds
self.inactivity += delta_seconds
worker_ids = self.get_not_done_worker_ids()
if self.warned_seconds_ago < self.warn_timeout:
return
color_stdout("No output during %d seconds. "
"List of workers not reporting the status: %s; "
"Will abort after %d seconds without output.\n" %
(self.inactivity, worker_ids, self.kill_timeout),
schema='test_var')
self.warned_seconds_ago = 0.0
if self.inactivity < self.kill_timeout:
return
color_stdout(
'\n[Main process] No output from workers. '
'It seems that we hang. Send SIGKILL to workers; '
'exiting...\n',
schema='test_var')
self.kill_all_workers()
raise HangError()
def run_test(self, test, server, inspector):
""" Returns short status of the test as a string: 'skip', 'pass',
'new', 'fail', or 'disabled' and results file checksum on fail.
"""
test.inspector = inspector
test_name = os.path.basename(test.name)
full_test_name = os.path.join(self.ini['suite'], test_name)
color_stdout(just_and_trim(full_test_name, 47) + ' ', schema='t_name')
# for better diagnostics in case of a long-running test
conf = ''
if test.run_params:
conf = test.conf_name
color_stdout(just_and_trim(conf, 15) + ' ', schema='test_var')
start_time = time.time()
if self.is_test_enabled(test, conf, server):
short_status, result_checksum = test.run(server)
else:
color_stdout("[ disabled ]\n", schema='t_name')
short_status = 'disabled'
result_checksum = None
duration = time.time() - start_time
# cleanup only if test passed or if --force mode enabled
if Options().args.is_force or short_status == 'pass':
inspector.cleanup_nondefault()
return short_status, result_checksum, duration
def check(self):
"""Check the arguments for correctness."""
check_error = False
conflict_options = ('valgrind', 'gdb', 'lldb', 'strace')
for op1, op2 in product(conflict_options, repeat=2):
if op1 != op2 and getattr(self.args, op1, '') and \
getattr(self.args, op2, ''):
format_str = "\nError: option --{} is not compatible with option --{}\n"
color_stdout(format_str.format(op1, op2), schema='error')
check_error = True
break
snapshot_path = self.args.snapshot_path
if self.args.disable_schema_upgrade and not snapshot_path:
color_stdout(
"\nOption --disable-schema-upgrade requires --snapshot\n",
schema='error')
check_error = True
if snapshot_path and not os.path.exists(snapshot_path):
color_stdout("\nPath {} does not exist\n".format(snapshot_path),
schema='error')
check_error = True
if check_error:
exit(-1)
def run_test(self, test, server, inspector):
""" Returns short status of the test as a string: 'skip', 'pass',
'new', 'fail', or 'disabled'.
"""
test.inspector = inspector
test_name = os.path.basename(test.name)
color_stdout(os.path.join(self.ini['suite'], test_name).ljust(48),
schema='t_name')
# for better diagnostics in case of a long-running test
conf = ''
if test.run_params:
conf = test.conf_name
color_stdout(conf.ljust(16), schema='test_var')
if self.is_test_enabled(test, conf, server):
short_status = test.run(server)
else:
color_stdout("[ disabled ]\n", schema='t_name')
short_status = 'disabled'
# cleanup only if test passed or if --force mode enabled
if lib.Options().args.is_force or short_status == 'pass':
inspector.cleanup_nondefault()
return short_status
def run_test(self, test, server, inspector):
""" Returns short status of the test as a string: 'skip', 'pass',
'new', 'fail', or 'disabled'.
"""
test.inspector = inspector
color_stdout(os.path.join(self.ini['suite'],
os.path.basename(test.name)).ljust(48),
schema='t_name')
# for better diagnostics in case of a long-running test
conf = ''
if test.run_params:
conf = test.conf_name
color_stdout(conf.ljust(16), schema='test_var')
test_name = os.path.basename(test.name)
if self.is_test_enabled(test, conf, server):
short_status = test.run(server)
else:
color_stdout("[ disabled ]\n", schema='t_name')
short_status = 'disabled'
# cleanup only if test passed or if --force mode enabled
if lib.Options().args.is_force or short_status == 'pass':
inspector.cleanup_nondefault()
return short_status
def collect_tests(self):
if self.tests_are_collected:
return self.tests
if self.ini['core'] == 'tarantool':
TarantoolServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'app':
AppServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'unittest':
UnittestServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'stress':
# parallel tests are not supported and disabled for now
self.tests = []
self.tests_are_collected = True
return self.tests
else:
raise ValueError('Cannot collect tests of unknown type')
if not Options().args.reproduce:
color_stdout("Collecting tests in ", schema='ts_text')
color_stdout(
'%s (Found %s tests)' % (
repr(self.suite_path).ljust(16),
str(len(self.tests)).ljust(3)
),
schema='path'
)
color_stdout(": ", self.ini["description"], ".\n",
schema='ts_text')
self.tests_are_collected = True
return self.tests
def find_tests(self):
if self.ini['core'] == 'tarantool':
TarantoolServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'app':
AppServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'unittest':
UnittestServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'stress':
# parallel tests are not supported and disabled for now
return []
else:
raise ValueError('Cannot collect tests of unknown type')
if not lib.Options().args.reproduce:
color_stdout("Collecting tests in ", schema='ts_text')
color_stdout(
'%s (Found %s tests)' % (
repr(self.suite_path).ljust(16),
str(len(self.tests)).ljust(3)
),
schema='path'
)
color_stdout(": ", self.ini["description"], ".\n",
schema='ts_text')
return self.tests
def stop(self, silent=True, signal=signal.SIGTERM):
""" Kill tarantool server using specified signal (SIGTERM by default)
signal - a number of a signal
"""
if self._start_against_running:
color_log('Server [%s] start against running ...\n',
schema='test_var')
return
if self.status != 'started':
if not silent:
raise Exception('Server is not started')
else:
color_log('Server [{0.name}] is not started '
'(status:{0.status}) ...\n'.format(self),
schema='test_var')
return
if not silent:
color_stdout('Stopping the server ...\n', schema='serv_text')
else:
color_log('Stopping the server ...\n', schema='serv_text')
# kill only if process is alive
if self.process is not None and self.process.returncode is None:
color_log('TarantoolServer.stop(): stopping the {0}\n'.format(
format_process(self.process.pid)),
schema='test_var')
try:
color_log('Sending signal {0} ({1}) to process {2}\n'.format(
signal, signame(signal), self.process.pid))
self.process.send_signal(signal)
except OSError:
pass
if self.crash_detector is not None:
save_join(self.crash_detector)
self.wait_stop()
self.status = None
if re.search(r'^/', str(self._admin.port)):
if os.path.exists(self._admin.port):
os.unlink(self._admin.port)
def execute(self, server):
super(LuaTest, self).execute(server)
cls_name = server.__class__.__name__.lower()
if 'gdb' in cls_name or 'lldb' in cls_name or 'strace' in cls_name:
# don't propagate gdb/lldb/strace mixin to non-default servers,
# it doesn't work properly for now
# TODO: strace isn't interactive, so it's easy to make it works for
# non-default server
create_server = TarantoolServer
else:
# propagate valgrind mixin to non-default servers
create_server = server.__class__
ts = TestState(
self.suite_ini, server, create_server,
self.run_params
)
self.inspector.set_parser(ts)
lua = TestRunGreenlet(self.exec_loop, ts)
self.current_test_greenlet = lua
lua.start()
crash_occured = True
try:
crash_occured = save_join(lua, timeout=self.TIMEOUT)
self.killall_servers(server, ts, crash_occured)
except KeyboardInterrupt:
# prevent tests greenlet from writing to the real stdout
lua.kill()
ts.stop_nondefault()
raise
except TarantoolStartError as e:
if not self.is_crash_reported:
self.is_crash_reported = True
color_stdout('\n[Instance "{0}"] Failed to start tarantool '
'server from a test\n'.format(e.name),
schema='error')
server.print_log(15)
server.kill_current_test()
def main_consistent():
color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text')
failed_test_ids = []
try:
main_loop_consistent(failed_test_ids)
except KeyboardInterrupt:
color_stdout('[Main loop] Caught keyboard interrupt\n',
schema='test_var')
except RuntimeError as e:
color_stdout("\nFatal error: %s. Execution aborted.\n" % e,
schema='error')
if lib.Options().args.gdb:
time.sleep(100)
return -1
if failed_test_ids and lib.Options().args.is_force:
color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids),
schema='error')
for test_id in failed_test_ids:
color_stdout("----- %s\n" % str(test_id), schema='info')
return (-1 if failed_test_ids else 0)
def main_consistent():
color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text')
failed_test_ids = []
try:
main_loop_consistent(failed_test_ids)
except KeyboardInterrupt:
color_stdout('[Main loop] Caught keyboard interrupt\n',
schema='test_var')
except RuntimeError as e:
color_stdout("\nFatal error: %s. Execution aborted.\n" % e,
schema='error')
if lib.Options().args.gdb:
time.sleep(100)
return -1
if failed_test_ids and lib.Options().args.is_force:
color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids),
schema='error')
for test_id in failed_test_ids:
color_stdout("----- %s\n" % str(test_id), schema='info')
return (-1 if failed_test_ids else 0)
def check_tap_output(self):
""" Returns is_tap, is_ok """
if not os.path.isfile(self.tmp_result):
color_stdout('\nCannot find %s\n' % self.tmp_result,
schema='error')
self.is_crash_reported = True
return False
with open(self.tmp_result, 'r') as f:
content = f.read()
tap = pytap13.TAP13()
try:
tap.parse(content)
except ValueError as e:
color_stdout('\nTAP13 parse failed: %s\n' % str(e),
schema='error')
self.is_crash_reported = True
return False, False
is_ok = True
for test_case in tap.tests:
if test_case.result == 'ok':
continue
if is_ok:
color_stdout('\n')
color_stdout('%s %s %s # %s %s\n' % (
test_case.result,
test_case.id or '',
test_case.description or '-',
test_case.directive or '',
test_case.comment or ''), schema='error')
if test_case.yaml:
self.tap_parse_print_yaml(test_case.yaml)
is_ok = False
if not is_ok:
color_stdout('Rejected result file: %s\n' % self.reject,
schema='test_var')
self.is_crash_reported = True
return True, is_ok
def handle(self, socket, addr):
if self.parser is None:
raise AttributeError('Parser is not defined')
self.sem.acquire()
for line in self.readline(socket):
try:
result = self.parser.parse_preprocessor(line)
except (KeyboardInterrupt, TarantoolStartError):
# propagate to the main greenlet
raise
except Exception as e:
self.parser.kill_current_test()
color_stdout('\nTarantoolInpector.handle() received the following error:\n' +
traceback.format_exc() + '\n', schema='error')
result = { "error": repr(e) }
if result == None:
result = True
result = yaml.dump(result)
if not result.endswith('...\n'):
result = result + '...\n'
socket.sendall(result)
self.sem.release()
def print_log(self, lines):
color_stdout(
'\nLast {0} lines of Tarantool Log file [Instance "{1}"][{2}]:\n'.
format(lines, self.name, self.logfile or 'null'),
schema='error')
if os.path.exists(self.logfile):
with open(self.logfile, 'r') as log:
color_stdout(''.join(log.readlines()[-lines:]))
else:
color_stdout(" Can't find log:\n", schema='error')
def process_timeout(self, delta_seconds):
self.warned_seconds_ago += delta_seconds
self.inactivity += delta_seconds
worker_ids = self.get_not_done_worker_ids()
if self.warned_seconds_ago < self.warn_timeout:
return
is_warning = self.inactivity < self.kill_timeout
color_stdout(
"No output during {0.inactivity:.0f} seconds. "
"Will abort after {0.kill_timeout:.0f} seconds without output. "
"List of workers not reporting the status:\n".format(self),
schema=('test_var' if is_warning else 'error'))
hung_tasks = [
task for worker_id, task in self.worker_current_task.iteritems()
if worker_id in worker_ids
]
for task in hung_tasks:
with open(task.task_tmp_result, 'r') as f:
lines = sum(1 for _ in f)
color_stdout("- {0} [{1}, {2}] at {3}:{4}\n".format(
task.worker_name, task.task_name, task.task_param,
task.task_tmp_result, lines),
schema=('test_var' if is_warning else 'error'))
self.warned_seconds_ago = 0.0
if is_warning:
return
for task in hung_tasks:
color_stdout("Test hung! Result content mismatch:\n",
schema='error')
lib.utils.print_unidiff(task.task_result, task.task_tmp_result)
color_stdout(
'\n[Main process] No output from workers. '
'It seems that we hang. Send SIGKILL to workers; '
'exiting...\n',
schema='error')
self.kill_all_workers()
raise HangError()
def process_timeout(self, delta_seconds):
self.warned_seconds_ago += delta_seconds
self.inactivity += delta_seconds
worker_ids = self.get_not_done_worker_ids()
if self.warned_seconds_ago < self.warn_timeout:
return
is_warning = self.inactivity < self.kill_timeout
color_stdout(
"No output during {0.inactivity:.0f} seconds. "
"Will abort after {0.kill_timeout:.0f} seconds without output. "
"List of workers not reporting the status:\n".format(self),
schema=('test_var' if is_warning else 'error'))
hung_tasks = [task for worker_id, task
in self.worker_current_task.iteritems()
if worker_id in worker_ids]
for task in hung_tasks:
with open(task.task_tmp_result, 'r') as f:
lines = sum(1 for _ in f)
color_stdout("- {0} [{1}, {2}] at {3}:{4}\n".format(
task.worker_name, task.task_name, task.task_param,
task.task_tmp_result, lines),
schema=('test_var' if is_warning else 'error'))
self.warned_seconds_ago = 0.0
if is_warning:
return
for task in hung_tasks:
color_stdout("Test hung! Result content mismatch:\n",
schema='error')
lib.utils.print_unidiff(task.task_result, task.task_tmp_result)
color_stdout('\n[Main process] No output from workers. '
'It seems that we hang. Send SIGKILL to workers; '
'exiting...\n', schema='error')
self.kill_all_workers()
raise HangError()
def crash_grep(self):
print_log_lines = 15
assert_fail_re = re.compile(r'^.*: Assertion .* failed\.$')
# find and save backtrace or assertion fail
assert_lines = list()
bt = list()
with open(self.logfile, 'r') as log:
lines = log.readlines()
for rpos, line in enumerate(reversed(lines)):
if line.startswith('Segmentation fault'):
bt = lines[-rpos - 1:]
break
if assert_fail_re.match(line):
pos = len(lines) - rpos
assert_lines = lines[max(0, pos - print_log_lines):pos]
break
else:
bt = list()
# print insident meat
if self.process.returncode < 0:
color_stdout('\n\n[Instance "%s" killed by signal: %d (%s)]\n' %
(self.name, -self.process.returncode,
signame(-self.process.returncode)),
schema='error')
else:
color_stdout(
'\n\n[Instance "%s" returns with non-zero exit code: %d]\n' %
(self.name, self.process.returncode),
schema='error')
# print assert line if any and return
if assert_lines:
color_stdout('Found assertion fail in the results file [%s]:\n' %
self.logfile,
schema='error')
sys.stderr.flush()
for line in assert_lines:
sys.stderr.write(line)
sys.stderr.flush()
return
# print backtrace if any
sys.stderr.flush()
for trace in bt:
sys.stderr.write(trace)
# print log otherwise (if backtrace was not found)
if not bt:
self.print_log(print_log_lines)
sys.stderr.flush()
def crash_grep(self):
if self.process.returncode < 0 or \
not non_empty_valgrind_logs([self.valgrind_log]):
super(ValgrindMixin, self).crash_grep()
return
lines_cnt = 50
color_stdout('\n\nValgrind for [Instance "%s"] returns non-zero exit code: %d\n' % (
self.name, self.process.returncode), schema='error')
color_stdout("It's known that it can be valgrind's \"the 'impossible' happened\" error\n", schema='error')
color_stdout('Last %d lines of valgring log file [%s]:\n' % (
lines_cnt, self.valgrind_log), schema='error')
print_tail_n(self.valgrind_log, 50)
def crash_grep(self):
if self.process.returncode < 0 or \
not non_empty_valgrind_logs([self.valgrind_log]):
super(ValgrindMixin, self).crash_grep()
return
lines_cnt = 50
color_stdout('\n\nValgrind for [Instance "%s"] returns non-zero exit code: %d\n' % (
self.name, self.process.returncode), schema='error')
color_stdout("It's known that it can be valgrind's \"the 'impossible' happened\" error\n", schema='error')
color_stdout('Last %d lines of valgring log file [%s]:\n' % (
lines_cnt, self.valgrind_log), schema='error')
print_tail_n(self.valgrind_log, lines_cnt)
def crash_grep(self):
print_log_lines = 15
assert_fail_re = re.compile(r'^.*: Assertion .* failed\.$')
# find and save backtrace or assertion fail
assert_lines = list()
bt = list()
with open(self.logfile, 'r') as log:
lines = log.readlines()
for rpos, line in enumerate(reversed(lines)):
if line.startswith('Segmentation fault'):
bt = lines[-rpos - 1:]
break
if assert_fail_re.match(line):
pos = len(lines) - rpos
assert_lines = lines[max(0, pos - print_log_lines):pos]
break
else:
bt = list()
# print insident meat
if self.process.returncode < 0:
color_stdout('\n\n[Instance "%s" killed by signal: %d (%s)]\n' % (
self.name, -self.process.returncode,
signame(-self.process.returncode)), schema='error')
else:
color_stdout('\n\n[Instance "%s" returns with non-zero exit code: '
'%d]\n' % (self.name, self.process.returncode),
schema='error')
# print assert line if any and return
if assert_lines:
color_stdout('Found assertion fail in the results file '
'[%s]:\n' % self.logfile,
schema='error')
sys.stderr.flush()
for line in assert_lines:
sys.stderr.write(line)
sys.stderr.flush()
return
# print backtrace if any
sys.stderr.flush()
for trace in bt:
sys.stderr.write(trace)
# print log otherwise (if backtrace was not found)
if not bt:
self.print_log(print_log_lines)
sys.stderr.flush()
def report_undone(self, verbose):
undone = self.undone_tasks()
if not bool(undone):
return False
if verbose:
color_stdout(
'The following tasks were dispatched on some worker task '
'queue, but were not reported as done (does not matters '
'success or fail):\n',
schema='test_var')
for task_id in undone:
color_stdout('- %s' % yaml.safe_dump(task_id))
else:
color_stdout("Count of didn't processed tasks: %d\n" % len(undone),
schema='test_var')
return True
def report_undone(self, verbose):
undone = self.undone_tasks()
if not bool(undone):
return False
if verbose:
color_stdout(
'[Internal test-run error] '
'The following tasks were dispatched to some worker task '
'queue, but were not reported as done (does not matters '
'success or fail):\n', schema='test_var')
for task_id in undone:
color_stdout('- %s' % yaml.safe_dump(task_id))
else:
# Visually continue StatisticsWatcher.print_statistics() output.
color_stdout('* undone: %d\n' % len(undone), schema='test_var')
return True
def report_undone(self, verbose):
undone = self.undone_tasks()
if not bool(undone):
return False
if verbose:
color_stdout(
'[Internal test-run error] '
'The following tasks were dispatched to some worker task '
'queue, but were not reported as done (does not matters '
'success or fail):\n',
schema='test_var')
for task_id in undone:
color_stdout('- %s' % yaml.safe_dump(task_id))
else:
# Visually continue StatisticsWatcher.print_statistics() output.
color_stdout('* undone: %d\n' % len(undone), schema='test_var')
return True
def prepare_args(self, args=[]):
screen_name = self.debugger_args['screen_name']
debugger = self.debugger_args['debugger']
gdbserver_port = self.debugger_args['gdbserver_port']
gdbserver_opts = self.debugger_args['gdbserver_opts']
sh_string = self.debugger_args['sh_string']
is_under_gdbserver = 'GdbServer' in self.__class__.__name__
if not is_under_gdbserver and not find_in_path('screen'):
raise OSError('`screen` executables not found in PATH')
if not find_in_path(debugger):
raise OSError('`%s` executables not found in PATH' % debugger)
is_tarantoolserver = 'TarantoolServer' in self.__class__.__name__
if is_tarantoolserver or is_under_gdbserver:
color_stdout('\nYou started the server in %s mode.\n' % debugger,
schema='info')
if is_under_gdbserver:
color_stdout("To attach, use `gdb -ex 'target remote :%s'`\n" %
gdbserver_port, schema='info')
else:
color_stdout('To attach, use `screen -r %s`\n' % screen_name,
schema='info')
# detach only for TarantoolServer
screen_opts = '-d' if is_tarantoolserver else ''
orig_args = super(DebugMixin, self).prepare_args(args)
res_args = shlex.split(sh_string.format(
screen_name=screen_name,
screen_opts=screen_opts,
binary=self.binary,
args=' '.join(orig_args),
logfile=self.logfile,
debugger=debugger,
gdbserver_port=gdbserver_port,
gdbserver_opts=gdbserver_opts))
color_log('\nRUN: ' + shlex_join(res_args) + '\n', schema='test_var')
return res_args
def __init__(self, suite, _id):
self.sigterm_received = False
signal.signal(signal.SIGTERM, lambda x, y, z=self:
z.sigterm_handler(x, y))
self.initialized = False
self.server = None
self.inspector = None
self.id = _id
self.suite = suite
self.name = '%03d_%s' % (self.id, self.suite.suite_path)
main_vardir = self.suite.ini['vardir']
self.suite.ini['vardir'] = os.path.join(main_vardir, self.name)
self.reproduce_file = get_reproduce_file(self.name)
safe_makedirs(os.path.dirname(self.reproduce_file))
color_stdout.queue_msg_wrapper = self.wrap_output
self.last_task_done = True
self.last_task_id = -1
try:
self.server = suite.gen_server()
self.inspector = suite.start_server(self.server)
self.initialized = True
except KeyboardInterrupt:
self.report_keyboard_interrupt()
self.stop_server(cleanup=False)
except Exception as e:
color_stdout('Worker "%s" cannot start tarantool server; '
'the tasks will be ignored...\n' % self.name,
schema='error')
color_stdout("The raised exception is '%s' of type '%s'.\n"
% (str(e), str(type(e))), schema='error')
color_stdout('Worker "%s" received the following error:\n'
% self.name + traceback.format_exc() + '\n',
schema='error')
self.stop_server(cleanup=False)
def print_statistics(self):
"""Returns are there failed tasks."""
if self.stats:
color_stdout('Statistics:\n', schema='test_var')
for short_status, cnt in self.stats.items():
color_stdout('* %s: %d\n' % (short_status, cnt), schema='test_var')
if not self.failed_tasks:
return False
color_stdout('Failed tasks:\n', schema='test_var')
for task_id, worker_name, show_reproduce_content in self.failed_tasks:
logfile = self.get_logfile(worker_name)
color_stdout('- %s' % yaml.safe_dump(task_id), schema='test_var')
color_stdout('# logfile: %s\n' % logfile)
reproduce_file_path = get_reproduce_file(worker_name)
color_stdout('# reproduce file: %s\n' % reproduce_file_path)
if show_reproduce_content:
color_stdout("---\n", schema='separator')
lib.utils.print_tail_n(reproduce_file_path)
color_stdout("...\n", schema='separator')
return True
def start(self, silent=True, wait=True, wait_load=True, rais=True, args=[],
**kwargs):
if self._start_against_running:
return
if self.status == 'started':
if not silent:
color_stdout('The server is already started.\n',
schema='lerror')
return
args = self.prepare_args(args)
self.pidfile = '%s.pid' % self.name
self.logfile = '%s.log' % self.name
path = self.script_dst if self.script else \
os.path.basename(self.binary)
color_log("Starting the server ...\n", schema='serv_text')
color_log("Starting ", schema='serv_text')
color_log(path + " \n", schema='path')
color_log(self.version() + "\n", schema='version')
os.putenv("LISTEN", self.iproto.uri)
os.putenv("ADMIN", self.admin.uri)
if self.rpl_master:
os.putenv("MASTER", self.rpl_master.iproto.uri)
self.logfile_pos = self.logfile
# redirect stdout from tarantoolctl and tarantool
os.putenv("TEST_WORKDIR", self.vardir)
self.process = subprocess.Popen(args,
cwd=self.vardir,
stdout=self.log_des,
stderr=self.log_des)
del(self.log_des)
# gh-19 crash detection
self.crash_detector = TestRunGreenlet(self.crash_detect)
self.crash_detector.info = "Crash detector: %s" % self.process
self.crash_detector.start()
if wait:
try:
self.wait_until_started(wait_load)
except TarantoolStartError:
# Raise exception when caller ask for it (e.g. in case of
# non-default servers)
if rais:
raise
# Python tests expect we raise an exception when non-default
# server fails
if self.crash_expected:
raise
if not (self.current_test and
self.current_test.is_crash_reported):
if self.current_test:
self.current_test.is_crash_reported = True
color_stdout('\n[Instance "{0.name}"] Tarantool server '
'failed to start\n'.format(self),
schema='error')
self.print_log(15)
# if the server fails before any test started, we should inform
# a caller by the exception
if not self.current_test:
raise
self.kill_current_test()
port = self.admin.port
self.admin.disconnect()
self.admin = CON_SWITCH[self.tests_type]('localhost', port)
self.status = 'started'
def print_exe(cls):
color_stdout('Installing the server ...\n', schema='serv_text')
if cls.binary:
color_stdout(' Found executable at ', schema='serv_text')
color_stdout(cls.binary + '\n', schema='path')
if cls.ctl_path:
color_stdout(' Found tarantoolctl at ', schema='serv_text')
color_stdout(cls.ctl_path + '\n', schema='path')
color_stdout("\n", cls.version(), "\n", schema='version')
