def test_cancel_phase(self):
@openhtf.PhaseOptions()
def cancel_phase():
# See above cancel_phase for explanations.
_abort_executor_in_thread(executor.abort)
ev = threading.Event()
group = phase_group.PhaseGroup(main=[cancel_phase],
teardown=[lambda: ev.set()]) # pylint: disable=unnecessary-lambda
test = openhtf.Test(group)
test.configure(default_dut_id='dut', )
executor = test_executor.TestExecutor(test.descriptor,
'uid',
start_phase,
test._test_options,
run_with_profiling=False)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.phases[0].name, start_phase.name)
self.assertLessEqual(record.start_time_millis, util.time_millis())
self.assertLessEqual(record.start_time_millis, record.end_time_millis)
self.assertLessEqual(record.end_time_millis, util.time_millis())
# Teardown function should be executed.
self.assertTrue(ev.wait(1))
executor.close()
def test_conf_stop_on_first_failure_phase(self):
ev = threading.Event()
group = phase_group.PhaseGroup(
main=[phase_return_fail_and_continue, phase_one],
teardown=[lambda: ev.set()]) # pylint: disable=unnecessary-lambda
test = openhtf.Test(group)
test.configure(default_dut_id='dut', )
conf.load(stop_on_first_failure=True)
executor = test_executor.TestExecutor(test.descriptor,
'uid',
start_phase,
test._test_options,
run_with_profiling=False)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.phases[0].name, start_phase.name)
self.assertTrue(record.outcome, Outcome.FAIL)
# Verify phase_one was not run
ran_phase = [phase.name for phase in record.phases]
self.assertNotIn('phase_one', ran_phase)
# Teardown function should be executed.
self.assertTrue(ev.wait(1))
executor.close()
def test_cancel_start(self):
@openhtf.PhaseOptions()
def cancel_phase(test):
test.dut_id = 'DUT ID'
# We have 'executor' because we're inside the test method's scope.
# We have to run it in a thread to avoid getting a nasty series of
# confusing errors:
_abort_executor_in_thread(executor.abort)
ev = threading.Event()
group = phase_group.PhaseGroup(
teardown=[lambda: ev.set()], # pylint: disable=unnecessary-lambda
)
test = openhtf.Test(group)
test.configure(default_dut_id='dut', )
# Cancel during test start phase.
executor = test_executor.TestExecutor(test.descriptor, 'uid',
cancel_phase, test._test_options)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.phases[0].name, cancel_phase.name)
# The test will end at the same time it starts because the test never
# actually started, we canceled it inside of test_start, resulting in a
# short vacuous start. Start and end times should be no more than a
# few milliseconds apart in that case.
self.assertLess(record.end_time_millis - record.start_time_millis, 4)
self.assertLessEqual(record.end_time_millis, util.time_millis())
# Teardown function should not be executed.
self.assertFalse(ev.wait(3))
def test_failures(self):
"""Tests that specified exception will cause FAIL not ERROR."""
@openhtf.PhaseOptions()
def failure_phase(test):
del test # Unused.
raise self.TestDummyExceptionError
# Configure test to throw exception midrun, and check that this causes
# Outcome = ERROR.
ev = threading.Event()
group = phase_group.PhaseGroup(
main=[failure_phase],
teardown=[lambda: ev.set()], # pylint: disable=unnecessary-lambda
)
test = openhtf.Test(group)
test.configure(default_dut_id='dut', )
executor = test_executor.TestExecutor(test.descriptor, 'uid',
start_phase, test._test_options)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, Outcome.ERROR)
# Same as above, but now specify that the TestDummyExceptionError should
# instead be a FAIL outcome.
test.configure(failure_exceptions=[self.TestDummyExceptionError])
executor = test_executor.TestExecutor(test.descriptor, 'uid',
start_phase, test._test_options)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, Outcome.FAIL)
def test_failure_during_start_phase_plug_init(self):
def never_gonna_run_phase():
ev2.set()
ev = threading.Event()
ev2 = threading.Event()
group = phase_group.PhaseGroup(
main=[never_gonna_run_phase],
teardown=[lambda: ev.set()], # pylint: disable=unnecessary-lambda
)
test = openhtf.Test(group)
test.configure(default_dut_id='dut', )
executor = test_executor.TestExecutor(test.descriptor, 'uid',
fail_plug_phase,
test._test_options)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, Outcome.ERROR)
self.assertEqual(record.outcome_details[0].code,
FailedPlugError.__name__)
self.assertEqual(record.outcome_details[0].description,
FAIL_PLUG_MESSAGE)
# Teardown function should *NOT* be executed.
self.assertFalse(ev.is_set())
self.assertFalse(ev2.is_set())
def test_failure_during_plug_init(self):
ev = threading.Event()
def set_ev():
ev.set()
group = phase_group.PhaseGroup(main=[fail_plug_phase],
teardown=[set_ev])
test = openhtf.Test(group)
test.configure(default_dut_id='dut', )
executor = test_executor.TestExecutor(test.descriptor,
'uid',
None,
test._test_options,
run_with_profiling=False)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, test_record.Outcome.ERROR)
self.assertEqual(record.outcome_details[0].code,
FailedPlugError.__name__)
self.assertEqual(record.outcome_details[0].description,
FAIL_PLUG_MESSAGE)
# Teardown function should *NOT* be executed.
self.assertFalse(ev.is_set())
executor.close()
def test_log_during_teardown(self):
message = 'hello'
def teardown_log(test):
test.logger.info(message)
group = phase_group.PhaseGroup(main=[blank_phase],
teardown=[teardown_log])
test = openhtf.Test(group)
test.configure(default_dut_id='dut', )
executor = test_executor.TestExecutor(test.descriptor,
'uid',
start_phase,
test._test_options,
run_with_profiling=False)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, Outcome.PASS)
log_records = [
log_record for log_record in record.log_records
if log_record.message == message
]
self.assertTrue(log_records)
executor.close()
def setUp(self):
super(TestExecutorExecuteBranchTest, self).setUp()
self.diag_store = diagnoses_lib.DiagnosesStore()
self.mock_test_record = mock.create_autospec(test_record.TestRecord)
self.mock_logger = mock.create_autospec(logging.Logger)
self.test_state = mock.MagicMock(
spec=test_state.TestState,
plug_manager=plugs.PlugManager(),
diagnoses_manager=mock.MagicMock(
spec=diagnoses_lib.DiagnosesManager, store=self.diag_store),
execution_uid='01234567890',
test_record=self.mock_test_record,
state_logger=self.mock_logger)
td = test_descriptor.TestDescriptor(
phase_sequence=phase_collections.PhaseSequence(
phase_group.PhaseGroup()),
code_info=test_record.CodeInfo.uncaptured(),
metadata={})
self.test_exec = test_executor.TestExecutor(
td,
td.uid,
None,
test_descriptor.TestOptions(),
run_with_profiling=False)
self.test_exec.test_state = self.test_state
patcher = mock.patch.object(self.test_exec, '_execute_sequence')
self.mock_execute_sequence = patcher.start()
def setUp(self):
self.test_descriptor = test_descriptor.TestDescriptor(
phase_group.PhaseGroup(main=[test_phase]), None, {'config': {}})
self.test_state = test_state.TestState(self.test_descriptor, 'testing-123',
test_descriptor.TestOptions())
self.test_record = self.test_state.test_record
self.running_phase_state = test_state.PhaseState.from_descriptor(
test_phase, lambda *args: None)
self.test_state.running_phase_state = self.running_phase_state
self.test_api = self.test_state.test_api
def setUp(self):
super(TestExecutorExecutePhaseGroupTest, self).setUp()
self.test_state = mock.MagicMock(spec=test_state.TestState,
plug_manager=plugs.PlugManager(),
execution_uid='01234567890',
state_logger=mock.MagicMock())
td = test_descriptor.TestDescriptor(
phase_sequence=phase_collections.PhaseSequence(
phase_group.PhaseGroup()),
code_info=test_record.CodeInfo.uncaptured(),
metadata={})
self.test_exec = test_executor.TestExecutor(
td,
td.uid,
None,
test_descriptor.TestOptions(),
run_with_profiling=False)
self.test_exec.test_state = self.test_state
patcher = mock.patch.object(self.test_exec, '_execute_sequence')
self.mock_execute_sequence = patcher.start()
@phase_descriptor.PhaseOptions()
def setup():
pass
self._setup = phase_collections.PhaseSequence((setup, ))
@phase_descriptor.PhaseOptions()
def main():
pass
self._main = phase_collections.PhaseSequence((main, ))
@openhtf.PhaseOptions(timeout_s=30)
def teardown():
pass
self._teardown = phase_collections.PhaseSequence((teardown, ))
self.group = phase_group.PhaseGroup(setup=self._setup,
main=self._main,
teardown=self._teardown,
name='group')
def _get_running_test_descriptor(self):
"""If there is a teardown_function, wrap current descriptor with it."""
if not self._test_options.teardown_function:
return self._test_desc
teardown_phase = phase_descriptor.PhaseDescriptor.wrap_or_copy(
self._test_options.teardown_function)
if not teardown_phase.options.timeout_s:
teardown_phase.options.timeout_s = conf.teardown_timeout_s
return TestDescriptor(
phase_group.PhaseGroup(main=[self._test_desc.phase_group],
teardown=[teardown_phase]),
self._test_desc.code_info, self._test_desc.metadata)
def test_cancel_twice_phase(self):
def abort_twice():
executor.abort()
teardown_running.wait()
executor.abort()
@openhtf.PhaseOptions()
def cancel_twice_phase():
# See above cancel_phase for explanations.
_abort_executor_in_thread(abort_twice)
@openhtf.PhaseOptions()
def teardown_phase():
teardown_running.set()
# Sleeping for the entire duration has a race condition with cancellation.
timeout = timeouts.PolledTimeout(1)
while not timeout.has_expired():
time.sleep(0.01)
ev.set()
@openhtf.PhaseOptions()
def teardown2_phase():
ev2.set()
teardown_running = threading.Event()
ev = threading.Event()
ev2 = threading.Event()
group = phase_group.PhaseGroup(
main=[cancel_twice_phase],
teardown=[teardown_phase, teardown2_phase])
test = openhtf.Test(group)
test.configure(default_dut_id='dut', )
executor = test_executor.TestExecutor(test.descriptor,
'uid',
start_phase,
test._test_options,
run_with_profiling=False)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.phases[0].name, start_phase.name)
self.assertLessEqual(record.start_time_millis, util.time_millis())
self.assertLessEqual(record.start_time_millis, record.end_time_millis)
self.assertLessEqual(record.end_time_millis, util.time_millis())
# Teardown function should *NOT* be executed.
self.assertFalse(ev.is_set())
self.assertFalse(ev2.is_set())
executor.close()
def test_error_during_teardown(self):
group = phase_group.PhaseGroup(main=[blank_phase],
teardown=[teardown_fail])
test = openhtf.Test(group)
test.configure(default_dut_id='dut', )
executor = test_executor.TestExecutor(test.descriptor, 'uid',
start_phase, test._test_options)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, Outcome.ERROR)
self.assertEqual(record.outcome_details[0].code,
TeardownError.__name__)
def setUp(self):
super(TestTestApi, self).setUp()
patcher = mock.patch.object(test_record.PhaseRecord, 'record_start_time',
return_value=11235)
self.mock_record_start_time = patcher.start()
self.addCleanup(patcher.stop)
self.test_descriptor = test_descriptor.TestDescriptor(
phase_group.PhaseGroup(main=[test_phase]), None, {'config': {}})
self.test_state = test_state.TestState(self.test_descriptor, 'testing-123',
test_descriptor.TestOptions())
self.test_record = self.test_state.test_record
self.running_phase_state = test_state.PhaseState.from_descriptor(
test_phase, self.test_state, logging.getLogger())
self.test_state.running_phase_state = self.running_phase_state
self.test_api = self.test_state.test_api
def testNoTeardown(self):
self.mock_execute_sequence.side_effect = [
test_executor._ExecutorReturn.CONTINUE,
test_executor._ExecutorReturn.CONTINUE,
]
group = phase_group.PhaseGroup(setup=self._setup,
main=self._main,
name='group')
self.assertEqual(
test_executor._ExecutorReturn.CONTINUE,
self.test_exec._execute_phase_group(group, None, False))
self.assertEqual([
mock.call(self._setup, None, False,
override_message='group:setup'),
mock.call(self._main, None, False, override_message='group:main'),
], self.mock_execute_sequence.call_args_list)
def setUp(self):
super(TestExecutorExecuteSequencesTest, self).setUp()
self.test_state = mock.MagicMock(spec=test_state.TestState,
plug_manager=plugs.PlugManager(),
execution_uid='01234567890',
state_logger=mock.MagicMock())
td = test_descriptor.TestDescriptor(
phase_sequence=phase_collections.PhaseSequence(
phase_group.PhaseGroup()),
code_info=test_record.CodeInfo.uncaptured(),
metadata={})
self.test_exec = test_executor.TestExecutor(
td,
td.uid,
None,
test_descriptor.TestOptions(),
run_with_profiling=False)
self.test_exec.test_state = self.test_state
patcher = mock.patch.object(self.test_exec, '_execute_node')
self.mock_execute_node = patcher.start()
def setUp(self):
super(TestExecutorExecutePhaseGroupTest, self).setUp()
self.test_state = mock.MagicMock(spec=test_state.TestState,
plug_manager=plugs.PlugManager(),
execution_uid='01234567890',
state_logger=mock.MagicMock())
self.test_exec = test_executor.TestExecutor(
None,
'uid',
None,
test_descriptor.TestOptions(),
run_with_profiling=False)
self.test_exec.test_state = self.test_state
patcher = mock.patch.object(self.test_exec,
'_execute_abortable_phases')
self.mock_execute_abortable = patcher.start()
patcher = mock.patch.object(self.test_exec, '_execute_teardown_phases')
self.mock_execute_teardown = patcher.start()
def setup():
pass
self._setup = setup
def main():
pass
self._main = main
@openhtf.PhaseOptions(timeout_s=30)
def teardown():
pass
self._teardown = teardown
self.group = phase_group.PhaseGroup(setup=[setup],
main=[main],
teardown=[teardown],
name='group')
def test_cancel_phase_with_diagnoser(self):
class DiagResult(openhtf.DiagResultEnum):
RESULT = 'result'
@openhtf.PhaseDiagnoser(DiagResult)
def diag(phase_record):
del phase_record # Unused.
return openhtf.Diagnosis(DiagResult.RESULT, 'result')
@openhtf.diagnose(diag)
@openhtf.PhaseOptions()
def cancel_phase():
# See above cancel_phase for explanations.
_abort_executor_in_thread(executor.abort)
ev = threading.Event()
def set_ev():
ev.set()
group = phase_group.PhaseGroup(main=[cancel_phase], teardown=[set_ev])
test = openhtf.Test(group)
test.configure(default_dut_id='dut', )
executor = test_executor.TestExecutor(test.descriptor,
'uid',
start_phase,
test._test_options,
run_with_profiling=False)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.phases[0].name, start_phase.name)
self.assertLessEqual(record.start_time_millis, util.time_millis())
self.assertLessEqual(record.start_time_millis, record.end_time_millis)
self.assertLessEqual(record.end_time_millis, util.time_millis())
self.assertEqual([], record.diagnoses)
# Teardown function should be executed.
self.assertTrue(ev.wait(1))
executor.close()
def testPhaseGroup_Terminal(self):
self.mock_execute_phase_group.return_value = True
group = phase_group.PhaseGroup(name='test')
self.assertTrue(self.test_exec._handle_phase(group))
self.mock_execute_phase_group.assert_called_once_with(group)
def testEmptyGroup(self):
group = phase_group.PhaseGroup()
self.assertEqual(
test_executor._ExecutorReturn.CONTINUE,
self.test_exec._execute_phase_group(group, None, False))
self.mock_execute_sequence.assert_not_called()
return phase_descriptor.PhaseResult.FAIL_SUBTEST
class BrokenError(Exception):
pass
def error_phase():
raise BrokenError('broken')
def teardown_phase():
pass
teardown_group = phase_group.PhaseGroup(teardown=teardown_phase)
@phase_descriptor.PhaseOptions()
def empty_phase():
pass
@phase_descriptor.PhaseOptions()
def skip_phase():
pass
@phase_descriptor.PhaseOptions()
def skip_phase0():
pass
