def show_plans(self) -> None:
print()
print(
colorize(
"About to run {} test case{} for {} selected model{} "
"({} iteration{}):".format(self.num_cases(),
self.plural_s(self.num_cases()),
self.num_models(),
self.plural_s(self.num_models()),
self.iterations,
self.plural_s(self.iterations)),
color.BOLD))
for test in self.tests:
if not test['selected']:
print(
colorize(
colorize(" - {} (not selected)".format(test['model']),
color.BOLD), color.GRAY))
continue
print(colorize(" - {}:".format(test['model']), color.BOLD))
for key, case in test['cases'].items():
if case['selected']:
print(" - '{}'".format(key))
else:
print(
colorize(" - '{}' (not selected)".format(key),
color.GRAY))
print()
def run_iteration(self, iteration: int) -> bool:
iteration_success = True
for test in self.tests:
if not test['selected']:
continue
print(colorize(
"==> Running tests for {}".format(test['model']),
color.BOLD))
test_i = 0
for key, case_value in test['cases'].items():
if not case_value['selected']:
continue
print("--> Test case {} of {}: '{}' running ...".
format(test_i+1,
self.num_cases_for_test(test),
key))
log_dir = self.get_log_dir(iteration, test['model'], key)
if self.verbose:
print("creating log directory: {}"
.format(log_dir))
os.makedirs(log_dir, exist_ok=True)
was_success = self.run_test_case(test, key, log_dir)
print("--- Test case {} of {}: '{}' {}."
.format(test_i+1,
self.num_cases_for_test(test),
key,
colorize("succeeded", color.GREEN)
if was_success
else colorize("failed", color.RED)))
if not was_success:
iteration_success = False
if not was_success and self.abort_early:
print("Aborting early")
return False
test_i += 1
return iteration_success
def run_tests(self) -> None:
for iteration in range(self.iterations):
if self.iterations > 1:
print(colorize("%%% Test iteration: {} / {}".
format(iteration + 1, self.iterations), color.BOLD))
success = self.run_iteration(iteration)
if not success:
if self.abort_early:
break
def show_detailed_results(self) -> None:
print()
print(colorize("Results:", color.BOLD))
for test in self.tests:
if not test['selected']:
print(
colorize(
colorize(" - {} (not selected)".format(test['model']),
color.BOLD), color.GRAY))
continue
else:
print(colorize(" - {}:".format(test['model']), color.BOLD))
for name, case in test['cases'].items():
if not case['selected']:
continue
print(" - '{}': ".format(name), end="")
n_succeeded = [
result['success'] for result in case['results']
].count(True)
n_failed = [result['success']
for result in case['results']].count(False)
n_cancelled = self.iterations - len(case['results'])
notes = [
self.note_if_any(colorize("{}succeeded", color.GREEN),
n_succeeded, self.iterations),
self.note_if_any(colorize("{}failed", color.RED), n_failed,
self.iterations),
self.note_if_any(colorize("{}cancelled", color.GRAY),
n_cancelled, self.iterations)
]
notes_without_none = list(filter(None, notes))
print(", ".join(notes_without_none))
def show_overall_result(self) -> None:
print(
colorize(
"Overall result: {}".format(
colorize("PASS", color.GREEN) if self.was_overall_pass(
) else colorize("FAIL", color.RED)), color.BOLD))
def add_name_prefix(width: int, name: str, text: str) -> str:
return colorize("[" + name.ljust(width) + "] " + text, color.RESET)
self.start_combined_log(logfile_path)
test_timeout_s = test['timeout_min'] * 60
while self.active_runners[-1].time_elapsed_s() < test_timeout_s:
returncode = self.active_runners[-1].poll()
self.collect_runner_output()
if returncode is not None:
is_success = (returncode == 0)
break
else:
print(
colorize(
"Test timeout of {} mins triggered!".format(
test['timeout_min']), color.BOLD))
is_success = False
self.stop_runners()
# Collect what was left in output buffers.
self.collect_runner_output()
self.stop_combined_log()
result = {
'success':
is_success,
'logfiles':
[runner.get_log_filename() for runner in self.active_runners]
}
test['cases'][case]['results'].append(result)