def test_cancel_phase(self): @openhtf.PhaseOptions() def cancel_phase(): # See above cancel_phase for explanations. _abort_executor_in_thread(executor.abort) ev = threading.Event() group = phase_group.PhaseGroup(main=[cancel_phase], teardown=[lambda: ev.set()]) # pylint: disable=unnecessary-lambda test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, start_phase.name) self.assertLessEqual(record.start_time_millis, util.time_millis()) self.assertLessEqual(record.start_time_millis, record.end_time_millis) self.assertLessEqual(record.end_time_millis, util.time_millis()) # Teardown function should be executed. self.assertTrue(ev.wait(1)) executor.close()
def test_conf_stop_on_first_failure_phase(self): ev = threading.Event() group = phase_group.PhaseGroup( main=[phase_return_fail_and_continue, phase_one], teardown=[lambda: ev.set()]) # pylint: disable=unnecessary-lambda test = openhtf.Test(group) test.configure(default_dut_id='dut', ) conf.load(stop_on_first_failure=True) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, start_phase.name) self.assertTrue(record.outcome, Outcome.FAIL) # Verify phase_one was not run ran_phase = [phase.name for phase in record.phases] self.assertNotIn('phase_one', ran_phase) # Teardown function should be executed. self.assertTrue(ev.wait(1)) executor.close()
def test_cancel_start(self): @openhtf.PhaseOptions() def cancel_phase(test): test.dut_id = 'DUT ID' # We have 'executor' because we're inside the test method's scope. # We have to run it in a thread to avoid getting a nasty series of # confusing errors: _abort_executor_in_thread(executor.abort) ev = threading.Event() group = phase_group.PhaseGroup( teardown=[lambda: ev.set()], # pylint: disable=unnecessary-lambda ) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) # Cancel during test start phase. executor = test_executor.TestExecutor(test.descriptor, 'uid', cancel_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, cancel_phase.name) # The test will end at the same time it starts because the test never # actually started, we canceled it inside of test_start, resulting in a # short vacuous start. Start and end times should be no more than a # few milliseconds apart in that case. self.assertLess(record.end_time_millis - record.start_time_millis, 4) self.assertLessEqual(record.end_time_millis, util.time_millis()) # Teardown function should not be executed. self.assertFalse(ev.wait(3))
def test_failures(self): """Tests that specified exception will cause FAIL not ERROR.""" @openhtf.PhaseOptions() def failure_phase(test): del test # Unused. raise self.TestDummyExceptionError # Configure test to throw exception midrun, and check that this causes # Outcome = ERROR. ev = threading.Event() group = phase_group.PhaseGroup( main=[failure_phase], teardown=[lambda: ev.set()], # pylint: disable=unnecessary-lambda ) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.ERROR) # Same as above, but now specify that the TestDummyExceptionError should # instead be a FAIL outcome. test.configure(failure_exceptions=[self.TestDummyExceptionError]) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.FAIL)
def test_failure_during_start_phase_plug_init(self): def never_gonna_run_phase(): ev2.set() ev = threading.Event() ev2 = threading.Event() group = phase_group.PhaseGroup( main=[never_gonna_run_phase], teardown=[lambda: ev.set()], # pylint: disable=unnecessary-lambda ) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', fail_plug_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.ERROR) self.assertEqual(record.outcome_details[0].code, FailedPlugError.__name__) self.assertEqual(record.outcome_details[0].description, FAIL_PLUG_MESSAGE) # Teardown function should *NOT* be executed. self.assertFalse(ev.is_set()) self.assertFalse(ev2.is_set())
def test_failure_during_plug_init(self): ev = threading.Event() def set_ev(): ev.set() group = phase_group.PhaseGroup(main=[fail_plug_phase], teardown=[set_ev]) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', None, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, test_record.Outcome.ERROR) self.assertEqual(record.outcome_details[0].code, FailedPlugError.__name__) self.assertEqual(record.outcome_details[0].description, FAIL_PLUG_MESSAGE) # Teardown function should *NOT* be executed. self.assertFalse(ev.is_set()) executor.close()
def test_log_during_teardown(self): message = 'hello' def teardown_log(test): test.logger.info(message) group = phase_group.PhaseGroup(main=[blank_phase], teardown=[teardown_log]) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.PASS) log_records = [ log_record for log_record in record.log_records if log_record.message == message ] self.assertTrue(log_records) executor.close()
def setUp(self): super(TestExecutorExecuteBranchTest, self).setUp() self.diag_store = diagnoses_lib.DiagnosesStore() self.mock_test_record = mock.create_autospec(test_record.TestRecord) self.mock_logger = mock.create_autospec(logging.Logger) self.test_state = mock.MagicMock( spec=test_state.TestState, plug_manager=plugs.PlugManager(), diagnoses_manager=mock.MagicMock( spec=diagnoses_lib.DiagnosesManager, store=self.diag_store), execution_uid='01234567890', test_record=self.mock_test_record, state_logger=self.mock_logger) td = test_descriptor.TestDescriptor( phase_sequence=phase_collections.PhaseSequence( phase_group.PhaseGroup()), code_info=test_record.CodeInfo.uncaptured(), metadata={}) self.test_exec = test_executor.TestExecutor( td, td.uid, None, test_descriptor.TestOptions(), run_with_profiling=False) self.test_exec.test_state = self.test_state patcher = mock.patch.object(self.test_exec, '_execute_sequence') self.mock_execute_sequence = patcher.start()
def setUp(self): self.test_descriptor = test_descriptor.TestDescriptor( phase_group.PhaseGroup(main=[test_phase]), None, {'config': {}}) self.test_state = test_state.TestState(self.test_descriptor, 'testing-123', test_descriptor.TestOptions()) self.test_record = self.test_state.test_record self.running_phase_state = test_state.PhaseState.from_descriptor( test_phase, lambda *args: None) self.test_state.running_phase_state = self.running_phase_state self.test_api = self.test_state.test_api
def setUp(self): super(TestExecutorExecutePhaseGroupTest, self).setUp() self.test_state = mock.MagicMock(spec=test_state.TestState, plug_manager=plugs.PlugManager(), execution_uid='01234567890', state_logger=mock.MagicMock()) td = test_descriptor.TestDescriptor( phase_sequence=phase_collections.PhaseSequence( phase_group.PhaseGroup()), code_info=test_record.CodeInfo.uncaptured(), metadata={}) self.test_exec = test_executor.TestExecutor( td, td.uid, None, test_descriptor.TestOptions(), run_with_profiling=False) self.test_exec.test_state = self.test_state patcher = mock.patch.object(self.test_exec, '_execute_sequence') self.mock_execute_sequence = patcher.start() @phase_descriptor.PhaseOptions() def setup(): pass self._setup = phase_collections.PhaseSequence((setup, )) @phase_descriptor.PhaseOptions() def main(): pass self._main = phase_collections.PhaseSequence((main, )) @openhtf.PhaseOptions(timeout_s=30) def teardown(): pass self._teardown = phase_collections.PhaseSequence((teardown, )) self.group = phase_group.PhaseGroup(setup=self._setup, main=self._main, teardown=self._teardown, name='group')
def _get_running_test_descriptor(self): """If there is a teardown_function, wrap current descriptor with it.""" if not self._test_options.teardown_function: return self._test_desc teardown_phase = phase_descriptor.PhaseDescriptor.wrap_or_copy( self._test_options.teardown_function) if not teardown_phase.options.timeout_s: teardown_phase.options.timeout_s = conf.teardown_timeout_s return TestDescriptor( phase_group.PhaseGroup(main=[self._test_desc.phase_group], teardown=[teardown_phase]), self._test_desc.code_info, self._test_desc.metadata)
def test_cancel_twice_phase(self): def abort_twice(): executor.abort() teardown_running.wait() executor.abort() @openhtf.PhaseOptions() def cancel_twice_phase(): # See above cancel_phase for explanations. _abort_executor_in_thread(abort_twice) @openhtf.PhaseOptions() def teardown_phase(): teardown_running.set() # Sleeping for the entire duration has a race condition with cancellation. timeout = timeouts.PolledTimeout(1) while not timeout.has_expired(): time.sleep(0.01) ev.set() @openhtf.PhaseOptions() def teardown2_phase(): ev2.set() teardown_running = threading.Event() ev = threading.Event() ev2 = threading.Event() group = phase_group.PhaseGroup( main=[cancel_twice_phase], teardown=[teardown_phase, teardown2_phase]) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, start_phase.name) self.assertLessEqual(record.start_time_millis, util.time_millis()) self.assertLessEqual(record.start_time_millis, record.end_time_millis) self.assertLessEqual(record.end_time_millis, util.time_millis()) # Teardown function should *NOT* be executed. self.assertFalse(ev.is_set()) self.assertFalse(ev2.is_set()) executor.close()
def test_error_during_teardown(self): group = phase_group.PhaseGroup(main=[blank_phase], teardown=[teardown_fail]) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.outcome, Outcome.ERROR) self.assertEqual(record.outcome_details[0].code, TeardownError.__name__)
def setUp(self): super(TestTestApi, self).setUp() patcher = mock.patch.object(test_record.PhaseRecord, 'record_start_time', return_value=11235) self.mock_record_start_time = patcher.start() self.addCleanup(patcher.stop) self.test_descriptor = test_descriptor.TestDescriptor( phase_group.PhaseGroup(main=[test_phase]), None, {'config': {}}) self.test_state = test_state.TestState(self.test_descriptor, 'testing-123', test_descriptor.TestOptions()) self.test_record = self.test_state.test_record self.running_phase_state = test_state.PhaseState.from_descriptor( test_phase, self.test_state, logging.getLogger()) self.test_state.running_phase_state = self.running_phase_state self.test_api = self.test_state.test_api
def testNoTeardown(self): self.mock_execute_sequence.side_effect = [ test_executor._ExecutorReturn.CONTINUE, test_executor._ExecutorReturn.CONTINUE, ] group = phase_group.PhaseGroup(setup=self._setup, main=self._main, name='group') self.assertEqual( test_executor._ExecutorReturn.CONTINUE, self.test_exec._execute_phase_group(group, None, False)) self.assertEqual([ mock.call(self._setup, None, False, override_message='group:setup'), mock.call(self._main, None, False, override_message='group:main'), ], self.mock_execute_sequence.call_args_list)
def setUp(self): super(TestExecutorExecuteSequencesTest, self).setUp() self.test_state = mock.MagicMock(spec=test_state.TestState, plug_manager=plugs.PlugManager(), execution_uid='01234567890', state_logger=mock.MagicMock()) td = test_descriptor.TestDescriptor( phase_sequence=phase_collections.PhaseSequence( phase_group.PhaseGroup()), code_info=test_record.CodeInfo.uncaptured(), metadata={}) self.test_exec = test_executor.TestExecutor( td, td.uid, None, test_descriptor.TestOptions(), run_with_profiling=False) self.test_exec.test_state = self.test_state patcher = mock.patch.object(self.test_exec, '_execute_node') self.mock_execute_node = patcher.start()
def setUp(self): super(TestExecutorExecutePhaseGroupTest, self).setUp() self.test_state = mock.MagicMock(spec=test_state.TestState, plug_manager=plugs.PlugManager(), execution_uid='01234567890', state_logger=mock.MagicMock()) self.test_exec = test_executor.TestExecutor( None, 'uid', None, test_descriptor.TestOptions(), run_with_profiling=False) self.test_exec.test_state = self.test_state patcher = mock.patch.object(self.test_exec, '_execute_abortable_phases') self.mock_execute_abortable = patcher.start() patcher = mock.patch.object(self.test_exec, '_execute_teardown_phases') self.mock_execute_teardown = patcher.start() def setup(): pass self._setup = setup def main(): pass self._main = main @openhtf.PhaseOptions(timeout_s=30) def teardown(): pass self._teardown = teardown self.group = phase_group.PhaseGroup(setup=[setup], main=[main], teardown=[teardown], name='group')
def test_cancel_phase_with_diagnoser(self): class DiagResult(openhtf.DiagResultEnum): RESULT = 'result' @openhtf.PhaseDiagnoser(DiagResult) def diag(phase_record): del phase_record # Unused. return openhtf.Diagnosis(DiagResult.RESULT, 'result') @openhtf.diagnose(diag) @openhtf.PhaseOptions() def cancel_phase(): # See above cancel_phase for explanations. _abort_executor_in_thread(executor.abort) ev = threading.Event() def set_ev(): ev.set() group = phase_group.PhaseGroup(main=[cancel_phase], teardown=[set_ev]) test = openhtf.Test(group) test.configure(default_dut_id='dut', ) executor = test_executor.TestExecutor(test.descriptor, 'uid', start_phase, test._test_options, run_with_profiling=False) executor.start() executor.wait() record = executor.test_state.test_record self.assertEqual(record.phases[0].name, start_phase.name) self.assertLessEqual(record.start_time_millis, util.time_millis()) self.assertLessEqual(record.start_time_millis, record.end_time_millis) self.assertLessEqual(record.end_time_millis, util.time_millis()) self.assertEqual([], record.diagnoses) # Teardown function should be executed. self.assertTrue(ev.wait(1)) executor.close()
def testPhaseGroup_Terminal(self): self.mock_execute_phase_group.return_value = True group = phase_group.PhaseGroup(name='test') self.assertTrue(self.test_exec._handle_phase(group)) self.mock_execute_phase_group.assert_called_once_with(group)
def testEmptyGroup(self): group = phase_group.PhaseGroup() self.assertEqual( test_executor._ExecutorReturn.CONTINUE, self.test_exec._execute_phase_group(group, None, False)) self.mock_execute_sequence.assert_not_called()
return phase_descriptor.PhaseResult.FAIL_SUBTEST class BrokenError(Exception): pass def error_phase(): raise BrokenError('broken') def teardown_phase(): pass teardown_group = phase_group.PhaseGroup(teardown=teardown_phase) @phase_descriptor.PhaseOptions() def empty_phase(): pass @phase_descriptor.PhaseOptions() def skip_phase(): pass @phase_descriptor.PhaseOptions() def skip_phase0(): pass