def init_listeners(self):
args = Options().args
watch_hang = args.no_output_timeout >= 0 and \
not args.gdb and \
not args.gdbserver and \
not args.lldb and \
not args.valgrind
watch_fail = not Options().args.is_force
log_output_watcher = LogOutputWatcher()
self.statistics = StatisticsWatcher(log_output_watcher.get_logfile)
self.artifacts = ArtifactsWatcher(log_output_watcher.get_logfile)
output_watcher = OutputWatcher()
self.listeners = [
self.statistics, log_output_watcher, output_watcher, self.artifacts
]
if sampler.is_enabled:
self.listeners.append(sampler.watcher)
if watch_fail:
self.fail_watcher = FailWatcher(self.terminate_all_workers)
self.listeners.append(self.fail_watcher)
if watch_hang:
warn_timeout = 60.0 if args.long else 10.0
hang_watcher = HangWatcher(output_watcher.not_done_worker_ids,
self.kill_all_workers, warn_timeout,
float(args.no_output_timeout))
self.listeners.append(hang_watcher)
def find_suites():
suite_names = Options().args.suites
if suite_names == []:
for root, dirs, names in os.walk(os.getcwd(), followlinks=True):
if "suite.ini" in names:
suite_names.append(os.path.basename(root))
suites = [
TestSuite(suite_name,
Options().args) for suite_name in sorted(suite_names)
]
return suites
def process_result(self, obj):
if isinstance(obj, WorkerDone):
bufferized = self.buffer.get(obj.worker_id, '')
if bufferized:
OutputWatcher._write(bufferized, obj.worker_id)
if obj.worker_id in self.buffer.keys():
del self.buffer[obj.worker_id]
return
# Skip irrelevant events.
if not isinstance(obj, WorkerOutput):
return
# Skip color_log() events if --debug is not passed.
if obj.log_only and not Options().args.debug:
return
# Prepend color_log() messages with a timestamp.
if obj.log_only:
prefix = '[{}] '.format(obj.timestamp)
obj.output = prefix_each_line(prefix, obj.output)
bufferized = self.buffer.get(obj.worker_id, '')
if decolor(obj.output).endswith('\n'):
OutputWatcher._write(bufferized + obj.output, obj.worker_id)
self.buffer[obj.worker_id] = ''
else:
self.buffer[obj.worker_id] = bufferized + obj.output
def __init__(self):
self.fds = dict()
self.logdir = os.path.join(Options().args.vardir, 'log')
try:
os.makedirs(self.logdir)
except OSError:
pass
def save_artifacts(self):
if not self.failed_workers:
return
vardir = Options().args.vardir
artifacts_dir = os.path.join(vardir, 'artifacts')
artifacts_log_dir = os.path.join(artifacts_dir, 'log')
artifacts_reproduce_dir = os.path.join(artifacts_dir, 'reproduce')
safe_makedirs(artifacts_dir)
safe_makedirs(artifacts_log_dir)
safe_makedirs(artifacts_reproduce_dir)
for worker_name in self.failed_workers:
logfile = self.get_logfile(worker_name)
reproduce_file_path = get_reproduce_file(worker_name)
shutil.copy(
logfile,
os.path.join(artifacts_log_dir, os.path.basename(logfile)))
shutil.copy(
reproduce_file_path,
os.path.join(artifacts_reproduce_dir,
os.path.basename(reproduce_file_path)))
shutil.copytree(os.path.join(vardir, worker_name),
os.path.join(artifacts_dir, worker_name),
ignore=shutil.ignore_patterns(
'*.socket-iproto', '*.socket-admin', '*.sock',
'*.control'))
shutil.copytree(os.path.join(vardir, 'statistics'),
os.path.join(artifacts_dir, 'statistics'))
def print_statistics(self):
"""Print statistics and results of testing."""
# Prepare standalone subpath '<vardir>/statistics' for statistics files.
stats_dir = os.path.join(Options().args.vardir, 'statistics')
safe_makedirs(stats_dir)
self.print_rss_summary(stats_dir)
self.print_duration(stats_dir)
if self.stats:
color_stdout('Statistics:\n', schema='test_var')
for short_status, cnt in self.stats.items():
color_stdout('* %s: %d\n' % (short_status, cnt), schema='test_var')
if not self.failed_tasks:
return False
color_stdout('Failed tasks:\n', schema='test_var')
for task_id, worker_name, result_checksum, show_reproduce_content in self.failed_tasks:
logfile = self.get_logfile(worker_name)
task_id_str = yaml.safe_dump(task_id, default_flow_style=True)
color_stdout('- %s' % task_id_str, schema='test_var')
color_stdout('# results file checksum: %s\n' % result_checksum)
color_stdout('# logfile: %s\n' % logfile)
reproduce_file_path = get_reproduce_file(worker_name)
color_stdout('# reproduce file: %s\n' % reproduce_file_path)
if show_reproduce_content:
color_stdout("---\n", schema='separator')
print_tail_n(reproduce_file_path)
color_stdout("...\n", schema='separator')
return True
def run_project(args):
options = Options()
options.parse(args[1:])
project = Project(options)
print 'Archirtecture chosen:', project.print_arg()
def reproduce_task_groups(task_groups):
"""Filter provided task_groups down to the one certain group. Sort tests in
this group as in the reproduce file.
"""
found_keys = []
reproduce = parse_reproduce_file(Options().args.reproduce)
if not reproduce:
raise ValueError('[reproduce] Tests list cannot be empty')
for i, task_id in enumerate(reproduce):
for key, task_group in task_groups.items():
if task_id in task_group['task_ids']:
found_keys.append(key.rstrip('_fragile'))
break
if len(found_keys) != i + 1:
raise ValueError('[reproduce] Cannot find test "%s"' %
str(task_id))
found_keys = list(set(found_keys))
if len(found_keys) < 1:
raise ValueError('[reproduce] Cannot find any suite for given tests')
elif len(found_keys) > 1:
raise ValueError(
'[reproduce] Given tests contained by different suites')
res_key = found_keys[0]
res_task_group = copy.deepcopy(task_groups[key])
res_task_group['task_ids'] = reproduce
return {res_key: res_task_group}
def run_test(self, test, server, inspector):
""" Returns short status of the test as a string: 'skip', 'pass',
'new', 'fail', or 'disabled' and results file checksum on fail.
"""
test.inspector = inspector
test_name = os.path.basename(test.name)
full_test_name = os.path.join(self.ini['suite'], test_name)
color_stdout(just_and_trim(full_test_name, 47) + ' ', schema='t_name')
# for better diagnostics in case of a long-running test
conf = ''
if test.run_params:
conf = test.conf_name
color_stdout(just_and_trim(conf, 15) + ' ', schema='test_var')
start_time = time.time()
if self.is_test_enabled(test, conf, server):
short_status, result_checksum = test.run(server)
else:
color_stdout("[ disabled ]\n", schema='t_name')
short_status = 'disabled'
result_checksum = None
duration = time.time() - start_time
# cleanup only if test passed or if --force mode enabled
if Options().args.is_force or short_status == 'pass':
inspector.cleanup_nondefault()
return short_status, result_checksum, duration
def collect_tests(self):
if self.tests_are_collected:
return self.tests
if self.ini['core'] == 'tarantool':
TarantoolServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'app':
AppServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'unittest':
UnittestServer.find_tests(self, self.suite_path)
elif self.ini['core'] == 'stress':
# parallel tests are not supported and disabled for now
self.tests = []
self.tests_are_collected = True
return self.tests
else:
raise ValueError('Cannot collect tests of unknown type')
if not Options().args.reproduce:
color_stdout("Collecting tests in ", schema='ts_text')
color_stdout(
'%s (Found %s tests)' % (
repr(self.suite_path).ljust(16),
str(len(self.tests)).ljust(3)
),
schema='path'
)
color_stdout(": ", self.ini["description"], ".\n",
schema='ts_text')
self.tests_are_collected = True
return self.tests
def start(args):
options = Options()
options.parse(args)
project = Project(options)
project.run()
def run(args):
options = Options()
options.parse(args[1:])
project = Project(options)
print 'Architecture chosen:', project.print_arg()
project.run_command()
def run_project(args):
options = Options()
options.parse(args[1:])
project = Project(options)
print 'Printing date:', project.date()
print 'Printing example arg:', project.print_example_arg()
def main_consistent():
color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text')
failed_test_ids = []
try:
main_loop_consistent(failed_test_ids)
except KeyboardInterrupt:
color_stdout('[Main loop] Caught keyboard interrupt\n',
schema='test_var')
except RuntimeError as e:
color_stdout("\nFatal error: %s. Execution aborted.\n" % e,
schema='error')
if Options().args.gdb:
time.sleep(100)
return -1
if failed_test_ids and Options().args.is_force:
color_stdout("\n===== %d tests failed:\n" % len(failed_test_ids),
schema='error')
for test_id in failed_test_ids:
color_stdout("----- %s\n" % str(test_id), schema='info')
return (-1 if failed_test_ids else 0)
def process_result(self, obj):
# Track tasks in progress.
if isinstance(obj, WorkerCurrentTask):
self.worker_current_task[obj.worker_id] = obj
# Skip irrelevant events.
if not isinstance(obj, WorkerOutput):
return
# Skip color_log() events if --debug is not passed.
if obj.log_only and not Options().args.debug:
return
self.warned_seconds_ago = 0.0
self.inactivity = 0.0
def run_project(args):
options = Options()
options.parse(args[1:])
project = Survey(options)
print '\n--------------------------------Printing Survey Summary --------------------------------------\n'
print 'Printing date:', project.date()
# print 'Printing arg:', project.print_example_arg()
print '\n--------------------------------Printing Survey Question Details -----------------------------\n', project.parse_surveydata(
)
print '\n--------------------------------Printing Survey Responces Details ----------------------------\n', project.parse_responces(
)
print '\nSubmission Percentage: ', project.submission_percentage(), '%'
print '\nSubmission Count Total: ', project.submission_count_total()
print '\nAverage Rating Q1: ', project.avg_rating('ANS_Q1')
print '\nAverage Rating Q2: ', project.avg_rating('ANS_Q2')
print '\nAverage Rating Q3: ', project.avg_rating('ANS_Q3')
print '\nAverage Rating Q4: ', project.avg_rating('ANS_Q4')
print '\nAverage Rating Q5: ', project.avg_rating('ANS_Q5')
print '\n--------------------------------End Survey Summary -------------------------------------------\n'
def main_loop_consistent(failed_test_ids):
# find and prepare all tasks/groups, print information
task_groups = get_task_groups().items()
print_greetings()
for name, task_group in task_groups:
# print information about current test suite
color_stdout("\n", '=' * 80, "\n", schema='separator')
color_stdout("TEST".ljust(48), schema='t_name')
color_stdout("PARAMS".ljust(16), schema='test_var')
color_stdout("RESULT\n", schema='test_pass')
color_stdout('-' * 75, "\n", schema='separator')
task_ids = task_group['task_ids']
show_reproduce_content = task_group['show_reproduce_content']
if not task_ids:
continue
worker_id = 1
worker = task_group['gen_worker'](worker_id)
for task_id in task_ids:
short_status = worker.run_task(task_id)
if short_status == 'fail':
reproduce_file_path = \
get_reproduce_file(worker.name)
color_stdout('Reproduce file %s\n' % reproduce_file_path,
schema='error')
if show_reproduce_content:
color_stdout("---\n", schema='separator')
print_tail_n(reproduce_file_path)
color_stdout("...\n", schema='separator')
failed_test_ids.append(task_id)
if not Options().args.is_force:
worker.stop_server(cleanup=False)
return
color_stdout('-' * 75, "\n", schema='separator')
worker.stop_server(silent=False)
color_stdout()
def main(args):
options = Options()
config = options.parse(args[1:])
repo = GitRepository(os.path.curdir)
old_rev, new_rev, ref_name = sys.stdin.readline().split(' ')
if repo.is_revision_empty(old_rev):
old_rev = repo.get_default_branch_name()
if repo.is_revision_empty(new_rev):
# case of branch deletion
exit(0)
if config.php_cs_fixer_enabled:
fixer = PhpCsFixer(repo, config.php_cs_fixer_executable,
config.php_cs_fixer_config_path,
config.php_cs_fixer_dirs_to_create)
fixer.check_push(old_rev, new_rev)
each_commit_checker = EachCommitChecker(config, repo)
each_commit_checker.check(old_rev, new_rev)
def check_tap_output(self):
""" Returns is_tap, is_ok """
try:
with open(self.tmp_result, 'r') as f:
content = f.read()
tap = pytap13.TAP13()
tap.parse(content)
except (ValueError, UnicodeDecodeError) as e:
color_stdout('\nTAP13 parse failed (%s).\n' % str(e),
schema='error')
color_stdout('\nNo result file (%s) found.\n' % self.result,
schema='error')
if not Options().args.update_result:
msg = 'Run the test with --update-result option to write the new result file.\n'
color_stdout(msg, schema='error')
self.is_crash_reported = True
return False, False
is_ok = True
for test_case in tap.tests:
if test_case.result == 'ok':
continue
if is_ok:
color_stdout('\n')
color_stdout(
'%s %s %s # %s %s\n' %
(test_case.result, test_case.id or '', test_case.description
or '-', test_case.directive or '', test_case.comment or ''),
schema='error')
if test_case.yaml:
self.tap_parse_print_yaml(test_case.yaml)
is_ok = False
if not is_ok:
color_stdout('Rejected result file: %s\n' % self.reject,
schema='test_var')
self.is_crash_reported = True
return True, is_ok
import sys
from lib import Options
if __name__ == '__main__':
options = Options()
if len(sys.argv[1:]) == 0:
options.help()
sys.exit(0)
opts = options.parse(sys.argv[1:])
if opts.input_file == None:
print "Please supply an input file"
sys.exit(1)
else:
print "At this point I'll process", opts.input_file
# TODO Create an object for the processer and pass
# the input file to it
sys.exit(0)
def main_loop_parallel():
color_stdout("Started {0}\n".format(" ".join(sys.argv)), schema='tr_text')
args = Options().args
jobs = args.jobs
if jobs < 1:
# faster result I got was with 2 * cpu_count
jobs = 2 * multiprocessing.cpu_count()
if jobs > 0:
color_stdout("Running in parallel with %d workers\n\n" % jobs,
schema='tr_text')
randomize = True
color_stdout("Timeout options:\n", schema='tr_text')
color_stdout('-' * 19, "\n", schema='separator')
color_stdout("REPLICATION_SYNC_TIMEOUT:".ljust(26) +
"{}\n".format(args.replication_sync_timeout),
schema='tr_text')
color_stdout("TEST_TIMEOUT:".ljust(26) + "{}\n".format(args.test_timeout),
schema='tr_text')
color_stdout("NO_OUTPUT_TIMEOUT:".ljust(26) +
"{}\n".format(args.no_output_timeout),
schema='tr_text')
color_stdout("\n", schema='tr_text')
task_groups = get_task_groups()
if Options().args.reproduce:
task_groups = reproduce_task_groups(task_groups)
jobs = 1
randomize = False
dispatcher = Dispatcher(task_groups, jobs, randomize)
dispatcher.start()
print_greetings()
color_stdout("\n", '=' * 86, "\n", schema='separator')
color_stdout("WORKR".ljust(6), schema='t_name')
color_stdout("TEST".ljust(48), schema='t_name')
color_stdout("PARAMS".ljust(16), schema='test_var')
color_stdout("RESULT\n", schema='test_pass')
color_stdout('-' * 81, "\n", schema='separator')
try:
is_force = Options().args.is_force
dispatcher.wait()
dispatcher.wait_processes()
color_stdout('-' * 81, "\n", schema='separator')
has_failed = dispatcher.statistics.print_statistics()
has_undone = dispatcher.report_undone(
verbose=bool(is_force or not has_failed))
if has_failed:
dispatcher.artifacts.save_artifacts()
return EXIT_FAILED_TEST
if has_undone:
return EXIT_NOTDONE_TEST
except KeyboardInterrupt:
color_stdout('-' * 81, "\n", schema='separator')
dispatcher.statistics.print_statistics()
dispatcher.report_undone(verbose=False)
raise
except HangError:
color_stdout('-' * 81, "\n", schema='separator')
dispatcher.statistics.print_statistics()
dispatcher.report_undone(verbose=False)
return EXIT_HANG
return EXIT_SUCCESS
def run_loop(self, task_queue, result_queue):
""" called from 'run_all' """
while True:
task_id = self.task_get(task_queue)
# None is 'stop worker' marker
if task_id is None:
color_log('Worker "%s" exhausted task queue; '
'stopping the server...\n' % self.name,
schema='test_var')
self.stop_worker(task_queue, result_queue)
break
short_status = None
result_checksum = None
duration = 0.0
result_queue.put(self.current_task(task_id))
testname = os.path.basename(task_id[0])
fragile_checksums = self.suite.get_test_fragile_checksums(testname)
retries_left = self.suite.fragile_retries()
# let's run till short_status became 'pass'
while short_status != 'pass' and retries_left >= 0:
self.restart_server()
# print message only after some fails occurred
if short_status == 'fail':
color_stdout('Test "%s", conf: "%s"\n'
'\tfrom "fragile" list failed with results'
' file checksum: "%s", rerunning ...\n' %
(task_id[0], task_id[1], result_checksum),
schema='error')
# run task and save the result to short_status
short_status, result_checksum, duration = self.run_task(
task_id)
# check if the results file checksum set on fail and if
# the newly created results file is known by checksum
if not result_checksum or (result_checksum
not in fragile_checksums):
break
retries_left = retries_left - 1
result_queue.put(
self.wrap_result(task_id, short_status, result_checksum,
duration))
if short_status == 'fail':
if Options().args.is_force:
self.restart_server()
color_stdout(
'Worker "%s" got failed test; restarted the server\n' %
self.name,
schema='test_var')
else:
color_stdout(
'Worker "%s" got failed test; stopping the server...\n'
% self.name,
schema='test_var')
raise VoluntaryStopException()
if self.sigterm_received:
color_stdout('Worker "%s" got signal to terminate; '
'stopping the server...\n' % self.name,
schema='test_var')
raise VoluntaryStopException()
self.task_done(task_queue)
# So we hack the open() builtin.
#
# https://stackoverflow.com/a/53347548/1598057
if PY3 and sys.version_info[0:2] < (3, 7):
std_open = __builtins__.open
def open_as_utf8(*args, **kwargs):
if len(args) >= 2:
mode = args[1]
else:
mode = kwargs.get('mode', '')
if 'b' not in mode:
kwargs.setdefault('encoding', 'utf-8')
return std_open(*args, **kwargs)
__builtins__.open = open_as_utf8
# don't sure why, but it values 1 or 2 gives 1.5x speedup for parallel
# test-run (and almost doesn't affect consistent test-run)
os.environ['OMP_NUM_THREADS'] = '2'
status = 0
force_parallel = bool(Options().args.reproduce)
if not force_parallel and Options().args.jobs == -1:
status = main_consistent()
else:
status = main_parallel()
exit(status)
def get_reproduce_file(worker_name):
main_vardir = os.path.realpath(Options().args.vardir)
reproduce_dir = os.path.join(main_vardir, 'reproduce')
return os.path.join(reproduce_dir, '%s.list.yaml' % worker_name)
def run(args):
opts = Options()
opts.parse(args[1:])
w = Project(opts)
print w.welcome()
def run(self, server):
""" Execute the test assuming it's a python program. If the test
aborts, print its output to stdout, and raise an exception. Else,
comprare result and reject files. If there is a difference, print
it to stdout.
Returns short status of the test as a string: 'skip', 'pass',
'new', 'updated' or 'fail' and results file checksum on fail.
There is also one possible value for short_status, 'disabled',
but it returned in the caller, TestSuite.run_test().
"""
# Note: test was created before certain worker become known, so we need
# to update temporary result directory here as it depends on 'vardir'.
self.tmp_result = os.path.join(self.suite_ini['vardir'],
os.path.basename(self.result))
diagnostics = "unknown"
save_stdout = sys.stdout
try:
self.skip = False
if os.path.exists(self.skip_cond):
sys.stdout = FilteredStream(self.tmp_result)
stdout_fileno = sys.stdout.stream.fileno()
new_globals = dict(locals(), **server.__dict__)
with open(self.skip_cond, 'r') as f:
code = compile(f.read(), self.skip_cond, 'exec')
exec(code, new_globals)
sys.stdout.close()
sys.stdout = save_stdout
if not self.skip:
sys.stdout = FilteredStream(self.tmp_result)
stdout_fileno = sys.stdout.stream.fileno()
self.execute(server)
sys.stdout.flush()
self.is_executed_ok = True
except TestExecutionError:
self.is_executed_ok = False
except Exception as e:
if e.__class__.__name__ == 'TarantoolStartError':
# worker should stop
raise
color_stdout('\nTest.run() received the following error:\n'
'{0}\n'.format(traceback.format_exc()),
schema='error')
diagnostics = str(e)
finally:
if sys.stdout and sys.stdout != save_stdout:
sys.stdout.close()
sys.stdout = save_stdout
self.is_executed = True
sys.stdout.flush()
is_tap = False
if not self.skip:
if not os.path.exists(self.tmp_result):
self.is_executed_ok = False
self.is_equal_result = False
elif self.is_executed_ok and os.path.isfile(self.result):
self.is_equal_result = filecmp.cmp(self.result,
self.tmp_result)
elif self.is_executed_ok:
if Options().args.is_verbose:
color_stdout('\n')
with open(self.tmp_result, 'r') as f:
color_stdout(f.read(), schema='log')
is_tap, is_ok = self.check_tap_output()
self.is_equal_result = is_ok
else:
self.is_equal_result = 1
if self.args.valgrind:
non_empty_logs = non_empty_valgrind_logs(
server.current_valgrind_logs(for_test=True))
self.is_valgrind_clean = not bool(non_empty_logs)
short_status = None
result_checksum = None
if self.skip:
short_status = 'skip'
color_stdout("[ skip ]\n", schema='test_skip')
if os.path.exists(self.tmp_result):
os.remove(self.tmp_result)
elif (self.is_executed_ok and self.is_equal_result
and self.is_valgrind_clean):
short_status = 'pass'
color_stdout("[ pass ]\n", schema='test_pass')
if os.path.exists(self.tmp_result):
os.remove(self.tmp_result)
elif (self.is_executed_ok and not self.is_equal_result
and not os.path.isfile(self.result) and not is_tap
and Options().args.update_result):
shutil.copy(self.tmp_result, self.result)
short_status = 'new'
color_stdout("[ new ]\n", schema='test_new')
elif (self.is_executed_ok and not self.is_equal_result
and os.path.isfile(self.result) and not is_tap
and Options().args.update_result):
shutil.copy(self.tmp_result, self.result)
short_status = 'updated'
color_stdout("[ updated ]\n", schema='test_new')
else:
has_result = os.path.exists(self.tmp_result)
if has_result:
safe_makedirs(self.var_suite_path)
shutil.copy(self.tmp_result, self.reject)
with open(self.tmp_result, mode='rb') as result_file:
result_checksum = md5(result_file.read()).hexdigest()
short_status = 'fail'
color_stdout("[ fail ]\n", schema='test_fail')
where = ""
if not self.is_crash_reported and not has_result:
color_stdout('\nCannot open %s\n' % self.tmp_result,
schema='error')
elif not self.is_crash_reported and not self.is_executed_ok:
self.print_diagnostics(
self.reject, "Test failed! Output from reject file "
"{0}:\n".format(self.reject))
server.print_log(15)
where = ": test execution aborted, reason " \
"'{0}'".format(diagnostics)
elif not self.is_crash_reported and not self.is_equal_result:
self.print_unidiff()
server.print_log(15)
where = ": wrong test output"
elif not self.is_crash_reported and not self.is_valgrind_clean:
os.remove(self.reject)
for log_file in non_empty_logs:
self.print_diagnostics(
log_file, "Test failed! Output from log file "
"{0}:\n".format(log_file))
where = ": there were warnings in the valgrind log file(s)"
return short_status, result_checksum
def setUp(self): self.options = Options()
def _default_options(self):
options = Options()
opts, args = options.parse()
return opts
