def test_failures(self): """Tests that specified exception will cause FAIL not ERROR.""" @openhtf.PhaseOptions() def failure_phase(test): del test # Unused. raise self.TestDummyExceptionError # Configure test to throw exception midrun, and check that this causes # Outcome = ERROR. ev = threading.Event() group = phase_group.PhaseGroup( main=[failure_phase], teardown=[lambda: ev.set()], # pylint: disable=unnecessary-lambda ) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.ERROR) # Same as above, but now specify that the TestDummyExceptionError should # instead be a FAIL outcome. test.configure(failure_exceptions=[self.TestDummyExceptionError]) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.FAIL)
def test_cancel_phase(self): @openhtf.PhaseOptions() def cancel_phase(test): del test # Unused. # See above cancel_phase for explanations. inner_ev = threading.Event() def stop_executor(): executor.stop() inner_ev.set() threading.Thread(target=stop_executor).start() inner_ev.wait(1) ev = threading.Event() test = openhtf.Test(cancel_phase) # Cancel during test start phase. executor = test_executor.TestExecutor( test.descriptor, 'uid', start_phase, 'dut', teardown_function=lambda: ev.set()) # pylint: disable=unnecessary-lambda executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, start_phase.name) self.assertLessEqual(record.start_time_millis, util.time_millis()) self.assertLessEqual(record.start_time_millis, record.end_time_millis) self.assertLessEqual(record.end_time_millis, util.time_millis()) # Teardown function should be executed. self.assertTrue(ev.wait(1))
def setUp(self): super(TestExecutorExecuteBranchTest, self).setUp() self.diag_store = diagnoses_lib.DiagnosesStore() self.mock_test_record = mock.create_autospec(test_record.TestRecord) self.mock_logger = mock.create_autospec(logging.Logger) self.test_state = mock.MagicMock( spec=test_state.TestState, plug_manager=plugs.PlugManager(), diagnoses_manager=mock.MagicMock( spec=diagnoses_lib.DiagnosesManager, store=self.diag_store), execution_uid='01234567890', test_record=self.mock_test_record, state_logger=self.mock_logger) td = test_descriptor.TestDescriptor( phase_sequence=phase_collections.PhaseSequence( phase_group.PhaseGroup()), code_info=test_record.CodeInfo.uncaptured(), metadata={}) self.test_exec = test_executor.TestExecutor( td, td.uid, None, test_descriptor.TestOptions(), run_with_profiling=False) self.test_exec.test_state = self.test_state patcher = mock.patch.object(self.test_exec, '_execute_sequence') self.mock_execute_sequence = patcher.start()
def test_conf_stop_on_first_failure_phase(self): ev = threading.Event() group = phase_group.PhaseGroup( main=[phase_return_fail_and_continue, phase_one], teardown=[lambda: ev.set()]) # pylint: disable=unnecessary-lambda test = openhtf.Test(group) test.configure(default_dut_id='dut', ) conf.load(stop_on_first_failure=True) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, start_phase.name) self.assertTrue(record.outcome, Outcome.FAIL) # Verify phase_one was not run ran_phase = [phase.name for phase in record.phases] self.assertNotIn('phase_one', ran_phase) # Teardown function should be executed. self.assertTrue(ev.wait(1)) executor.close()
def test_log_during_teardown(self): message = 'hello' def teardown_log(test): test.logger.info(message) group = phase_group.PhaseGroup(main=[blank_phase], teardown=[teardown_log]) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.PASS) log_records = [ log_record for log_record in record.log_records if log_record.message == message ] self.assertTrue(log_records) executor.close()
def test_cancel_phase(self): @openhtf.PhaseOptions() def cancel_phase(): # See above cancel_phase for explanations. _abort_executor_in_thread(executor.abort) ev = threading.Event() group = phase_group.PhaseGroup(main=[cancel_phase], teardown=[lambda: ev.set()]) # pylint: disable=unnecessary-lambda test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, start_phase.name) self.assertLessEqual(record.start_time_millis, util.time_millis()) self.assertLessEqual(record.start_time_millis, record.end_time_millis) self.assertLessEqual(record.end_time_millis, util.time_millis()) # Teardown function should be executed. self.assertTrue(ev.wait(1)) executor.close()
def test_cancel_start(self): @openhtf.PhaseOptions() def cancel_phase(test): test.dut_id = 'DUT ID' # We have 'executor' because we're inside the test method's scope. # We have to run it in a thread to avoid getting a nasty series of # confusing errors: _abort_executor_in_thread(executor.abort) ev = threading.Event() group = phase_group.PhaseGroup( teardown=[lambda: ev.set()], # pylint: disable=unnecessary-lambda ) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) # Cancel during test start phase. executor = test_executor.TestExecutor(test.descriptor, 'uid', cancel_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, cancel_phase.name) # The test will end at the same time it starts because the test never # actually started, we canceled it inside of test_start, resulting in a # short vacuous start. Start and end times should be no more than a # few milliseconds apart in that case. self.assertLess(record.end_time_millis - record.start_time_millis, 4) self.assertLessEqual(record.end_time_millis, util.time_millis()) # Teardown function should not be executed. self.assertFalse(ev.wait(3))
def test_failure_during_plug_init(self): ev = threading.Event() def set_ev(): ev.set() group = phase_group.PhaseGroup(main=[fail_plug_phase], teardown=[set_ev]) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', None, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, test_record.Outcome.ERROR) self.assertEqual(record.outcome_details[0].code, FailedPlugError.__name__) self.assertEqual(record.outcome_details[0].description, FAIL_PLUG_MESSAGE) # Teardown function should *NOT* be executed. self.assertFalse(ev.is_set()) executor.close()
def test_failure_during_start_phase_plug_init(self): def never_gonna_run_phase(): ev2.set() ev = threading.Event() ev2 = threading.Event() group = phase_group.PhaseGroup( main=[never_gonna_run_phase], teardown=[lambda: ev.set()], # pylint: disable=unnecessary-lambda ) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', fail_plug_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.ERROR) self.assertEqual(record.outcome_details[0].code, FailedPlugError.__name__) self.assertEqual(record.outcome_details[0].description, FAIL_PLUG_MESSAGE) # Teardown function should *NOT* be executed. self.assertFalse(ev.is_set()) self.assertFalse(ev2.is_set())
def test_error_during_teardown(self): test = openhtf.Test(blank_phase) executor = test_executor.TestExecutor( test.descriptor, 'uid', start_phase, 'dut', teardown_function=teardown_fail) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.ERROR) self.assertEqual(record.outcome_details[0].code, TeardownError.__name__)
def setUp(self): super(TestExecutorExecutePhasesTest, self).setUp() self.test_state = mock.MagicMock(spec=test_state.TestState, plug_manager=plugs.PlugManager(), execution_uid='01234567890', state_logger=mock.MagicMock()) self.test_exec = test_executor.TestExecutor( None, 'uid', None, test_descriptor.TestOptions()) self.test_exec.test_state = self.test_state patcher = mock.patch.object(self.test_exec, '_handle_phase') self.mock_handle_phase = patcher.start()
def setUp(self): self.test_state = mock.MagicMock( spec=test_state.TestState, plug_manager=plugs.PlugManager( record_logger_name='mock.logger.for.openhtf'), execution_uid='01234567890', logger=mock.MagicMock()) self.test_exec = test_executor.TestExecutor( None, 'uid', None, test_descriptor.TestOptions()) self.test_exec.test_state = self.test_state patcher = mock.patch.object(self.test_exec, '_handle_phase') self.mock_handle_phase = patcher.start()
def test_failure_during_plug_init(self): ev = threading.Event() test = openhtf.Test(fail_plug_phase) executor = test_executor.TestExecutor(test.descriptor, 'uid', None, 'dut', teardown_function=lambda: ev.set()) # pylint: disable=unnecessary-lambda executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.ERROR) self.assertEqual(record.outcome_details[0].code, FailedPlugError.__name__) self.assertEqual(record.outcome_details[0].description, FAIL_PLUG_MESSAGE) # Teardown function should be executed. self.assertTrue(ev.wait(1))
def test_cancel_twice_phase(self): def abort_twice(): executor.abort() teardown_running.wait() executor.abort() @openhtf.PhaseOptions() def cancel_twice_phase(): # See above cancel_phase for explanations. _abort_executor_in_thread(abort_twice) @openhtf.PhaseOptions() def teardown_phase(): teardown_running.set() # Sleeping for the entire duration has a race condition with cancellation. timeout = timeouts.PolledTimeout(1) while not timeout.has_expired(): time.sleep(0.01) ev.set() @openhtf.PhaseOptions() def teardown2_phase(): ev2.set() teardown_running = threading.Event() ev = threading.Event() ev2 = threading.Event() group = phase_group.PhaseGroup( main=[cancel_twice_phase], teardown=[teardown_phase, teardown2_phase]) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, start_phase.name) self.assertLessEqual(record.start_time_millis, util.time_millis()) self.assertLessEqual(record.start_time_millis, record.end_time_millis) self.assertLessEqual(record.end_time_millis, util.time_millis()) # Teardown function should *NOT* be executed. self.assertFalse(ev.is_set()) self.assertFalse(ev2.is_set()) executor.close()
def test_error_during_teardown(self): group = phase_group.PhaseGroup(main=[blank_phase], teardown=[teardown_fail]) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.ERROR) self.assertEqual(record.outcome_details[0].code, TeardownError.__name__)
def test_log_during_teardown(self): message = 'hello' def teardown_log(test): test.logger.info(message) test = openhtf.Test(blank_phase) executor = test_executor.TestExecutor( test.descriptor, 'uid', start_phase, 'dut', teardown_function=teardown_log) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.PASS) log_records = [log_record for log_record in record.log_records if log_record.message == message] self.assertTrue(log_records)
def setUp(self): super(TestExecutorHandlePhaseTest, self).setUp() self.test_state = mock.MagicMock( spec=test_state.TestState, plug_manager=plugs.PlugManager(), execution_uid='01234567890', state_logger=mock.MagicMock(), test_options=test_descriptor.TestOptions(), test_record=mock.MagicMock()) self.phase_exec = mock.MagicMock(spec=phase_executor.PhaseExecutor) self.test_exec = test_executor.TestExecutor( None, 'uid', None, test_descriptor.TestOptions()) self.test_exec.test_state = self.test_state self.test_exec._phase_exec = self.phase_exec patcher = mock.patch.object(self.test_exec, '_execute_phase_group') self.mock_execute_phase_group = patcher.start()
def test_cancel_start(self): @openhtf.PhaseOptions() def cancel_phase(test): test.dut_id = 'DUT ID' # We have 'executor' because we're inside the test method's scope. # We have to run it in a thread to avoid getting a nasty series of # confusing errors: # If we were to stop it in this phase, it eventually causes the phase # to be killed using KillableThread, which raises ThreadTerminationError # inside here, which really raises it inside wherever executor.stop() is. # That leads to the stopping of the executor to get stopped itself at a # random point in time. To make this deterministic, we keep the phase # alive as long as the executor is running, which really just means that # the wait() call gets the error raised in it. inner_ev = threading.Event() def stop_executor(): executor.stop() inner_ev.set() threading.Thread(target=stop_executor).start() inner_ev.wait(1) ev = threading.Event() test = openhtf.Test() test.configure( teardown_function=lambda: ev.set(), # pylint: disable=unnecessary-lambda default_dut_id='dut', ) # Cancel during test start phase. executor = test_executor.TestExecutor(test.descriptor, 'uid', cancel_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, cancel_phase.name) # The test will end at the same time it starts because the test never # actually started, we canceled it inside of test_start, resulting in a # short vacuous start. Start and end times should be no more than a # millisecond or two apart in that case. self.assertLess(record.end_time_millis - record.start_time_millis, 2) self.assertLessEqual(record.end_time_millis, util.time_millis()) # Teardown function should not be executed. self.assertFalse(ev.wait(3))
def setUp(self): super(TestExecutorExecutePhaseGroupTest, self).setUp() self.test_state = mock.MagicMock(spec=test_state.TestState, plug_manager=plugs.PlugManager(), execution_uid='01234567890', state_logger=mock.MagicMock()) td = test_descriptor.TestDescriptor( phase_sequence=phase_collections.PhaseSequence( phase_group.PhaseGroup()), code_info=test_record.CodeInfo.uncaptured(), metadata={}) self.test_exec = test_executor.TestExecutor( td, td.uid, None, test_descriptor.TestOptions(), run_with_profiling=False) self.test_exec.test_state = self.test_state patcher = mock.patch.object(self.test_exec, '_execute_sequence') self.mock_execute_sequence = patcher.start() @phase_descriptor.PhaseOptions() def setup(): pass self._setup = phase_collections.PhaseSequence((setup, )) @phase_descriptor.PhaseOptions() def main(): pass self._main = phase_collections.PhaseSequence((main, )) @openhtf.PhaseOptions(timeout_s=30) def teardown(): pass self._teardown = phase_collections.PhaseSequence((teardown, )) self.group = phase_group.PhaseGroup(setup=self._setup, main=self._main, teardown=self._teardown, name='group')
def setUp(self): super(TestExecutorExecuteSequencesTest, self).setUp() self.test_state = mock.MagicMock(spec=test_state.TestState, plug_manager=plugs.PlugManager(), execution_uid='01234567890', state_logger=mock.MagicMock()) td = test_descriptor.TestDescriptor( phase_sequence=phase_collections.PhaseSequence( phase_group.PhaseGroup()), code_info=test_record.CodeInfo.uncaptured(), metadata={}) self.test_exec = test_executor.TestExecutor( td, td.uid, None, test_descriptor.TestOptions(), run_with_profiling=False) self.test_exec.test_state = self.test_state patcher = mock.patch.object(self.test_exec, '_execute_node') self.mock_execute_node = patcher.start()
def setUp(self): super(TestExecutorExecutePhaseGroupTest, self).setUp() self.test_state = mock.MagicMock(spec=test_state.TestState, plug_manager=plugs.PlugManager(), execution_uid='01234567890', state_logger=mock.MagicMock()) self.test_exec = test_executor.TestExecutor( None, 'uid', None, test_descriptor.TestOptions(), run_with_profiling=False) self.test_exec.test_state = self.test_state patcher = mock.patch.object(self.test_exec, '_execute_abortable_phases') self.mock_execute_abortable = patcher.start() patcher = mock.patch.object(self.test_exec, '_execute_teardown_phases') self.mock_execute_teardown = patcher.start() def setup(): pass self._setup = setup def main(): pass self._main = main @openhtf.PhaseOptions(timeout_s=30) def teardown(): pass self._teardown = teardown self.group = phase_group.PhaseGroup(setup=[setup], main=[main], teardown=[teardown], name='group')
def test_cancel_phase_with_diagnoser(self): class DiagResult(openhtf.DiagResultEnum): RESULT = 'result' @openhtf.PhaseDiagnoser(DiagResult) def diag(phase_record): del phase_record # Unused. return openhtf.Diagnosis(DiagResult.RESULT, 'result') @openhtf.diagnose(diag) @openhtf.PhaseOptions() def cancel_phase(): # See above cancel_phase for explanations. _abort_executor_in_thread(executor.abort) ev = threading.Event() def set_ev(): ev.set() group = phase_group.PhaseGroup(main=[cancel_phase], teardown=[set_ev]) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, start_phase.name) self.assertLessEqual(record.start_time_millis, util.time_millis()) self.assertLessEqual(record.start_time_millis, record.end_time_millis) self.assertLessEqual(record.end_time_millis, util.time_millis()) self.assertEqual([], record.diagnoses) # Teardown function should be executed. self.assertTrue(ev.wait(1)) executor.close()
def execute(self, test_start=None): """Starts the framework and executes the given test. Args: test_start: Either a trigger phase for starting the test, or a function that returns a DUT ID. If neither is provided, defaults to not setting the DUT ID. Returns: Boolean indicating whether the test failed (False) or passed (True). Raises: InvalidTestStateError: if this test is already being executed. """ # Lock this section so we don't .stop() the executor between instantiating # it and .Start()'ing it, doing so does weird things to the executor state. with self._lock: # Sanity check to make sure someone isn't doing something weird like # trying to Execute() the same test twice in two separate threads. We # hold the lock between here and Start()'ing the executor to guarantee # that only one thread is successfully executing the test. if self._executor: raise InvalidTestStateError('Test already running', self._executor) # Snapshot some things we care about and store them. self._test_desc.metadata['test_name'] = self._test_options.name self._test_desc.metadata['config'] = conf._asdict() self.last_run_time_millis = util.time_millis() if isinstance(test_start, LambdaType): @phase_descriptor.PhaseOptions() def trigger_phase(test): test.test_record.dut_id = test_start() trigger = trigger_phase else: trigger = test_start if conf.capture_source: trigger.code_info = test_record.CodeInfo.for_function( trigger.func) self._executor = test_executor.TestExecutor( self._test_desc, self.make_uid(), trigger, self._test_options.default_dut_id, self._test_options.teardown_function, self._test_options.failure_exceptions) _LOG.info('Executing test: %s', self.descriptor.code_info.name) self.TEST_INSTANCES[self.uid] = self self._executor.start() try: self._executor.wait() finally: try: final_state = self._executor.finalize() _LOG.debug('Test completed for %s, outputting now.', final_state.test_record.metadata['test_name']) for output_cb in self._test_options.output_callbacks: try: output_cb(final_state.test_record) except Exception: # pylint: disable=broad-except _LOG.error( 'Output callback %s raised; continuing anyway', output_cb) # Make sure the final outcome of the test is printed last and in a # noticeable color so it doesn't get scrolled off the screen or missed. if final_state.test_record.outcome == test_record.Outcome.ERROR: for detail in final_state.test_record.outcome_details: console_output.error_print(detail.description) else: colors = collections.defaultdict( lambda: 'colorama.Style.BRIGHT') colors[test_record.Outcome.PASS] = ''.join( (colorama.Style.BRIGHT, colorama.Fore.GREEN)) colors[test_record.Outcome.FAIL] = ''.join( (colorama.Style.BRIGHT, colorama.Fore.RED)) msg_template = "test: {name} outcome: {color}{outcome}{rst}" console_output.banner_print( msg_template.format( name=final_state.test_record.metadata['test_name'], color=colors[final_state.test_record.outcome], outcome=final_state.test_record.outcome.name, rst=colorama.Style.RESET_ALL)) finally: del self.TEST_INSTANCES[self.uid] self._executor = None return final_state.test_record.outcome == test_record.Outcome.PASS
def execute(self, test_start: Optional[phase_descriptor.PhaseT] = None, profile_filename: Optional[Text] = None) -> bool: """Starts the framework and executes the given test. Args: test_start: Either a trigger phase for starting the test, or a function that returns a DUT ID. If neither is provided, defaults to not setting the DUT ID. profile_filename: Name of file to put profiling stats into. This also enables profiling data collection. Returns: Boolean indicating whether the test failed (False) or passed (True). Raises: InvalidTestStateError: if this test is already being executed. """ phase_descriptor.check_for_duplicate_results( self._test_desc.phase_sequence.all_phases(), self._test_options.diagnosers) phase_collections.check_for_duplicate_subtest_names( self._test_desc.phase_sequence) # Lock this section so we don't .stop() the executor between instantiating # it and .Start()'ing it, doing so does weird things to the executor state. with self._lock: # Sanity check to make sure someone isn't doing something weird like # trying to Execute() the same test twice in two separate threads. We # hold the lock between here and Start()'ing the executor to guarantee # that only one thread is successfully executing the test. if self._executor: raise InvalidTestStateError('Test already running', self._executor) # Snapshot some things we care about and store them. self._test_desc.metadata['test_name'] = self._test_options.name self._test_desc.metadata['config'] = CONF._asdict() self.last_run_time_millis = util.time_millis() if isinstance(test_start, types.LambdaType): @phase_descriptor.PhaseOptions() def trigger_phase(test): test.test_record.dut_id = typing.cast( types.LambdaType, test_start)() trigger = trigger_phase else: trigger = test_start if CONF.capture_source: trigger.code_info = htf_test_record.CodeInfo.for_function( trigger.func) self._executor = test_executor.TestExecutor( self._test_desc, self.make_uid(), trigger, self._test_options, run_with_profiling=profile_filename is not None) _LOG.info('Executing test: %s', self.descriptor.code_info.name) self.TEST_INSTANCES[self.uid] = self self._executor.start() try: self._executor.wait() except KeyboardInterrupt: # The SIGINT handler only raises the KeyboardInterrupt once, so only retry # that once. self._executor.wait() raise finally: try: final_state = self._executor.finalize() _LOG.debug('Test completed for %s, outputting now.', final_state.test_record.metadata['test_name']) test_executor.combine_profile_stats( self._executor.phase_profile_stats, profile_filename) for output_cb in self._test_options.output_callbacks: try: output_cb(final_state.test_record) except Exception: # pylint: disable=broad-except stacktrace = traceback.format_exc() _LOG.error( 'Output callback %s raised:\n%s\nContinuing anyway...', output_cb, stacktrace) # Make sure the final outcome of the test is printed last and in a # noticeable color so it doesn't get scrolled off the screen or missed. if final_state.test_record.outcome == htf_test_record.Outcome.ERROR: for detail in final_state.test_record.outcome_details: console_output.error_print(detail.description) else: colors = collections.defaultdict( lambda: colorama.Style.BRIGHT) colors[htf_test_record.Outcome.PASS] = ''.join( (colorama.Style.BRIGHT, colorama.Fore.GREEN)) # pytype: disable=wrong-arg-types colors[htf_test_record.Outcome.FAIL] = ''.join( (colorama.Style.BRIGHT, colorama.Fore.RED)) # pytype: disable=wrong-arg-types msg_template = ( 'test: {name} outcome: {color}{outcome}{marginal}{rst}' ) console_output.banner_print( msg_template.format( name=final_state.test_record.metadata['test_name'], color=(colorama.Fore.YELLOW if final_state.test_record.marginal else colors[final_state.test_record.outcome]), outcome=final_state.test_record.outcome.name, marginal=(' (MARGINAL)' if final_state.test_record.marginal else ''), rst=colorama.Style.RESET_ALL)) finally: del self.TEST_INSTANCES[self.uid] self._executor.close() self._executor = None return final_state.test_record.outcome == htf_test_record.Outcome.PASS