def print_statistics(self):
"""Print statistics and results of testing."""
# Prepare standalone subpath '<vardir>/statistics' for statistics files.
stats_dir = os.path.join(Options().args.vardir, 'statistics')
safe_makedirs(stats_dir)
self.print_rss_summary(stats_dir)
self.print_duration(stats_dir)
if self.stats:
color_stdout('Statistics:\n', schema='test_var')
for short_status, cnt in self.stats.items():
color_stdout('* %s: %d\n' % (short_status, cnt), schema='test_var')
if not self.failed_tasks:
return False
color_stdout('Failed tasks:\n', schema='test_var')
for task_id, worker_name, result_checksum, show_reproduce_content in self.failed_tasks:
logfile = self.get_logfile(worker_name)
task_id_str = yaml.safe_dump(task_id, default_flow_style=True)
color_stdout('- %s' % task_id_str, schema='test_var')
color_stdout('# results file checksum: %s\n' % result_checksum)
color_stdout('# logfile: %s\n' % logfile)
reproduce_file_path = get_reproduce_file(worker_name)
color_stdout('# reproduce file: %s\n' % reproduce_file_path)
if show_reproduce_content:
color_stdout("---\n", schema='separator')
print_tail_n(reproduce_file_path)
color_stdout("...\n", schema='separator')
return True
def copy_files(self):
if self.script:
shutil.copy(self.script, self.script_dst)
os.chmod(self.script_dst, 0o777)
if self.lua_libs:
for i in self.lua_libs:
source = os.path.join(self.testdir, i)
try:
if os.path.isdir(source):
shutil.copytree(source,
os.path.join(self.vardir,
os.path.basename(source)))
else:
shutil.copy(source, self.vardir)
except IOError as e:
if (e.errno == errno.ENOENT):
continue
raise
# Previously tarantoolctl configuration file located in tarantool
# repository at test/ directory. Currently it is located in root
# path of test-run/ submodule repository. For backward compatibility
# this file should be checked at the old place and only after at
# the current.
tntctl_file = '.tarantoolctl'
if not os.path.exists(tntctl_file):
tntctl_file = os.path.join(self.TEST_RUN_DIR, '.tarantoolctl')
shutil.copy(tntctl_file, self.vardir)
shutil.copy(os.path.join(self.TEST_RUN_DIR, 'test_run.lua'),
self.vardir)
if self.snapshot_path:
# Copy snapshot to the workdir.
# Usually Tarantool looking for snapshots on start in a current directory
# or in a directories that specified in memtx_dir or vinyl_dir box settings.
# Before running test current directory (workdir) passed to a new instance in
# an environment variable TEST_WORKDIR and then tarantoolctl
# adds to it instance_name and set to memtx_dir and vinyl_dir.
(instance_name, _) = os.path.splitext(os.path.basename(self.script))
instance_dir = os.path.join(self.vardir, instance_name)
safe_makedirs(instance_dir)
snapshot_dest = os.path.join(instance_dir, DEFAULT_SNAPSHOT_NAME)
color_log("Copying snapshot {} to {}\n".format(
self.snapshot_path, snapshot_dest))
shutil.copy(self.snapshot_path, snapshot_dest)
def __init__(self, suite, _id):
self.sigterm_received = False
signal.signal(signal.SIGTERM,
lambda x, y, z=self: z.sigterm_handler(x, y))
self.initialized = False
self.server = None
self.inspector = None
self.id = _id
self.suite = suite
self.name = '%03d_%s' % (self.id, self.suite.suite_path)
main_vardir = self.suite.ini['vardir']
self.suite.ini['vardir'] = os.path.join(main_vardir, self.name)
self.reproduce_file = get_reproduce_file(self.name)
safe_makedirs(os.path.dirname(self.reproduce_file))
color_stdout.queue_msg_wrapper = self.wrap_output
self.last_task_done = True
self.last_task_id = -1
try:
self.server = suite.gen_server()
self.inspector = suite.start_server(self.server)
self.initialized = True
except KeyboardInterrupt:
self.report_keyboard_interrupt()
self.stop_server(cleanup=False)
except Exception as e:
color_stdout('Worker "%s" cannot start tarantool server; '
'the tasks will be ignored...\n' % self.name,
schema='error')
color_stdout("The raised exception is '%s' of type '%s'.\n" %
(str(e), str(type(e))),
schema='error')
color_stdout(
'Worker "%s" received the following error:\n' % self.name +
traceback.format_exc() + '\n',
schema='error')
self.stop_server(cleanup=False)
def __init__(self, suite, _id):
self.sigterm_received = False
signal.signal(signal.SIGTERM, lambda x, y, z=self:
z.sigterm_handler(x, y))
self.initialized = False
self.server = None
self.inspector = None
self.id = _id
self.suite = suite
self.name = '%03d_%s' % (self.id, self.suite.suite_path)
main_vardir = self.suite.ini['vardir']
self.suite.ini['vardir'] = os.path.join(main_vardir, self.name)
self.reproduce_file = get_reproduce_file(self.name)
safe_makedirs(os.path.dirname(self.reproduce_file))
color_stdout.queue_msg_wrapper = self.wrap_output
self.last_task_done = True
self.last_task_id = -1
try:
self.server = suite.gen_server()
self.inspector = suite.start_server(self.server)
self.initialized = True
except KeyboardInterrupt:
self.report_keyboard_interrupt()
self.stop_server(cleanup=False)
except Exception as e:
color_stdout('Worker "%s" cannot start tarantool server; '
'the tasks will be ignored...\n' % self.name,
schema='error')
color_stdout("The raised exception is '%s' of type '%s'.\n"
% (str(e), str(type(e))), schema='error')
color_stdout('Worker "%s" received the following error:\n'
% self.name + traceback.format_exc() + '\n',
schema='error')
self.stop_server(cleanup=False)
def save_artifacts(self):
if not self.failed_workers:
return
vardir = Options().args.vardir
artifacts_dir = os.path.join(vardir, 'artifacts')
artifacts_log_dir = os.path.join(artifacts_dir, 'log')
artifacts_reproduce_dir = os.path.join(artifacts_dir, 'reproduce')
safe_makedirs(artifacts_dir)
safe_makedirs(artifacts_log_dir)
safe_makedirs(artifacts_reproduce_dir)
for worker_name in self.failed_workers:
logfile = self.get_logfile(worker_name)
reproduce_file_path = get_reproduce_file(worker_name)
shutil.copy(
logfile,
os.path.join(artifacts_log_dir, os.path.basename(logfile)))
shutil.copy(
reproduce_file_path,
os.path.join(artifacts_reproduce_dir,
os.path.basename(reproduce_file_path)))
shutil.copytree(os.path.join(vardir, worker_name),
os.path.join(artifacts_dir, worker_name),
ignore=shutil.ignore_patterns(
'*.socket-iproto', '*.socket-admin', '*.sock',
'*.control'))
shutil.copytree(os.path.join(vardir, 'statistics'),
os.path.join(artifacts_dir, 'statistics'))
def run(self, server):
""" Execute the test assuming it's a python program. If the test
aborts, print its output to stdout, and raise an exception. Else,
comprare result and reject files. If there is a difference, print
it to stdout.
Returns short status of the test as a string: 'skip', 'pass',
'new', 'updated' or 'fail' and results file checksum on fail.
There is also one possible value for short_status, 'disabled',
but it returned in the caller, TestSuite.run_test().
"""
# Note: test was created before certain worker become known, so we need
# to update temporary result directory here as it depends on 'vardir'.
self.tmp_result = os.path.join(self.suite_ini['vardir'],
os.path.basename(self.result))
diagnostics = "unknown"
save_stdout = sys.stdout
try:
self.skip = False
if os.path.exists(self.skip_cond):
sys.stdout = FilteredStream(self.tmp_result)
stdout_fileno = sys.stdout.stream.fileno()
new_globals = dict(locals(), **server.__dict__)
with open(self.skip_cond, 'r') as f:
code = compile(f.read(), self.skip_cond, 'exec')
exec(code, new_globals)
sys.stdout.close()
sys.stdout = save_stdout
if not self.skip:
sys.stdout = FilteredStream(self.tmp_result)
stdout_fileno = sys.stdout.stream.fileno()
self.execute(server)
sys.stdout.flush()
self.is_executed_ok = True
except TestExecutionError:
self.is_executed_ok = False
except Exception as e:
if e.__class__.__name__ == 'TarantoolStartError':
# worker should stop
raise
color_stdout('\nTest.run() received the following error:\n'
'{0}\n'.format(traceback.format_exc()),
schema='error')
diagnostics = str(e)
finally:
if sys.stdout and sys.stdout != save_stdout:
sys.stdout.close()
sys.stdout = save_stdout
self.is_executed = True
sys.stdout.flush()
is_tap = False
if not self.skip:
if not os.path.exists(self.tmp_result):
self.is_executed_ok = False
self.is_equal_result = False
elif self.is_executed_ok and os.path.isfile(self.result):
self.is_equal_result = filecmp.cmp(self.result,
self.tmp_result)
elif self.is_executed_ok:
if Options().args.is_verbose:
color_stdout('\n')
with open(self.tmp_result, 'r') as f:
color_stdout(f.read(), schema='log')
is_tap, is_ok = self.check_tap_output()
self.is_equal_result = is_ok
else:
self.is_equal_result = 1
if self.args.valgrind:
non_empty_logs = non_empty_valgrind_logs(
server.current_valgrind_logs(for_test=True))
self.is_valgrind_clean = not bool(non_empty_logs)
short_status = None
result_checksum = None
if self.skip:
short_status = 'skip'
color_stdout("[ skip ]\n", schema='test_skip')
if os.path.exists(self.tmp_result):
os.remove(self.tmp_result)
elif (self.is_executed_ok and self.is_equal_result
and self.is_valgrind_clean):
short_status = 'pass'
color_stdout("[ pass ]\n", schema='test_pass')
if os.path.exists(self.tmp_result):
os.remove(self.tmp_result)
elif (self.is_executed_ok and not self.is_equal_result
and not os.path.isfile(self.result) and not is_tap
and Options().args.update_result):
shutil.copy(self.tmp_result, self.result)
short_status = 'new'
color_stdout("[ new ]\n", schema='test_new')
elif (self.is_executed_ok and not self.is_equal_result
and os.path.isfile(self.result) and not is_tap
and Options().args.update_result):
shutil.copy(self.tmp_result, self.result)
short_status = 'updated'
color_stdout("[ updated ]\n", schema='test_new')
else:
has_result = os.path.exists(self.tmp_result)
if has_result:
safe_makedirs(self.var_suite_path)
shutil.copy(self.tmp_result, self.reject)
with open(self.tmp_result, mode='rb') as result_file:
result_checksum = md5(result_file.read()).hexdigest()
short_status = 'fail'
color_stdout("[ fail ]\n", schema='test_fail')
where = ""
if not self.is_crash_reported and not has_result:
color_stdout('\nCannot open %s\n' % self.tmp_result,
schema='error')
elif not self.is_crash_reported and not self.is_executed_ok:
self.print_diagnostics(
self.reject, "Test failed! Output from reject file "
"{0}:\n".format(self.reject))
server.print_log(15)
where = ": test execution aborted, reason " \
"'{0}'".format(diagnostics)
elif not self.is_crash_reported and not self.is_equal_result:
self.print_unidiff()
server.print_log(15)
where = ": wrong test output"
elif not self.is_crash_reported and not self.is_valgrind_clean:
os.remove(self.reject)
for log_file in non_empty_logs:
self.print_diagnostics(
log_file, "Test failed! Output from log file "
"{0}:\n".format(log_file))
where = ": there were warnings in the valgrind log file(s)"
return short_status, result_checksum