def show_plans(self) -> None: print() print( colorize( "About to run {} test case{} for {} selected model{} " "({} iteration{}):".format(self.num_cases(), self.plural_s(self.num_cases()), self.num_models(), self.plural_s(self.num_models()), self.iterations, self.plural_s(self.iterations)), color.BOLD)) for test in self.tests: if not test['selected']: print( colorize( colorize(" - {} (not selected)".format(test['model']), color.BOLD), color.GRAY)) continue print(colorize(" - {}:".format(test['model']), color.BOLD)) for key, case in test['cases'].items(): if case['selected']: print(" - '{}'".format(key)) else: print( colorize(" - '{}' (not selected)".format(key), color.GRAY)) print()
def run_iteration(self, iteration: int) -> bool: iteration_success = True for test in self.tests: if not test['selected']: continue print(colorize( "==> Running tests for {}".format(test['model']), color.BOLD)) test_i = 0 for key, case_value in test['cases'].items(): if not case_value['selected']: continue print("--> Test case {} of {}: '{}' running ...". format(test_i+1, self.num_cases_for_test(test), key)) log_dir = self.get_log_dir(iteration, test['model'], key) if self.verbose: print("creating log directory: {}" .format(log_dir)) os.makedirs(log_dir, exist_ok=True) was_success = self.run_test_case(test, key, log_dir) print("--- Test case {} of {}: '{}' {}." .format(test_i+1, self.num_cases_for_test(test), key, colorize("succeeded", color.GREEN) if was_success else colorize("failed", color.RED))) if not was_success: iteration_success = False if not was_success and self.abort_early: print("Aborting early") return False test_i += 1 return iteration_success
def run_tests(self) -> None: for iteration in range(self.iterations): if self.iterations > 1: print(colorize("%%% Test iteration: {} / {}". format(iteration + 1, self.iterations), color.BOLD)) success = self.run_iteration(iteration) if not success: if self.abort_early: break
def show_detailed_results(self) -> None: print() print(colorize("Results:", color.BOLD)) for test in self.tests: if not test['selected']: print( colorize( colorize(" - {} (not selected)".format(test['model']), color.BOLD), color.GRAY)) continue else: print(colorize(" - {}:".format(test['model']), color.BOLD)) for name, case in test['cases'].items(): if not case['selected']: continue print(" - '{}': ".format(name), end="") n_succeeded = [ result['success'] for result in case['results'] ].count(True) n_failed = [result['success'] for result in case['results']].count(False) n_cancelled = self.iterations - len(case['results']) notes = [ self.note_if_any(colorize("{}succeeded", color.GREEN), n_succeeded, self.iterations), self.note_if_any(colorize("{}failed", color.RED), n_failed, self.iterations), self.note_if_any(colorize("{}cancelled", color.GRAY), n_cancelled, self.iterations) ] notes_without_none = list(filter(None, notes)) print(", ".join(notes_without_none))
def show_overall_result(self) -> None: print( colorize( "Overall result: {}".format( colorize("PASS", color.GREEN) if self.was_overall_pass( ) else colorize("FAIL", color.RED)), color.BOLD))
def add_name_prefix(width: int, name: str, text: str) -> str: return colorize("[" + name.ljust(width) + "] " + text, color.RESET)
self.start_combined_log(logfile_path) test_timeout_s = test['timeout_min'] * 60 while self.active_runners[-1].time_elapsed_s() < test_timeout_s: returncode = self.active_runners[-1].poll() self.collect_runner_output() if returncode is not None: is_success = (returncode == 0) break else: print( colorize( "Test timeout of {} mins triggered!".format( test['timeout_min']), color.BOLD)) is_success = False self.stop_runners() # Collect what was left in output buffers. self.collect_runner_output() self.stop_combined_log() result = { 'success': is_success, 'logfiles': [runner.get_log_filename() for runner in self.active_runners] } test['cases'][case]['results'].append(result)