def test_failures(self):
"""Tests that specified exception will cause FAIL not ERROR."""
@openhtf.PhaseOptions()
def failure_phase(test):
del test # Unused.
raise self.TestDummyExceptionError
# Configure test to throw exception midrun, and check that this causes
# Outcome = ERROR.
ev = threading.Event()
test = openhtf.Test(failure_phase)
executor = core.TestExecutor(test.descriptor,
'uid',
start_phase,
'dut',
teardown_function=lambda: ev.set()) # pylint: disable=unnecessary-lambda
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, Outcome.ERROR)
# Same as above, but now specify that the TestDummyExceptionError should
# instead be a FAIL outcome.
executor = core.TestExecutor(
test.descriptor,
'uid',
start_phase,
'dut',
teardown_function=lambda: ev.set(), # pylint: disable=unnecessary-lambda
failure_exceptions=[self.TestDummyExceptionError])
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, Outcome.FAIL)
def test_cancel_phase(self):
@openhtf.PhaseOptions()
def start_phase(test):
test.dut_id = 'DUT ID'
@openhtf.PhaseOptions()
def cancel_phase(test):
del test # Unused.
# See above cancel_phase for explanations.
inner_ev = threading.Event()
def stop_executor():
executor.stop()
inner_ev.set()
threading.Thread(target=stop_executor).start()
inner_ev.wait(1)
ev = threading.Event()
test = openhtf.Test(cancel_phase)
# Cancel during test start phase.
executor = core.TestExecutor(test.descriptor, 'uid', start_phase,
teardown_function=lambda: ev.set()) # pylint: disable=unnecessary-lambda
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.phases[0].name, start_phase.name)
self.assertLessEqual(record.start_time_millis, util.time_millis())
self.assertLessEqual(record.start_time_millis, record.end_time_millis)
self.assertLessEqual(record.end_time_millis, util.time_millis())
# Teardown function should be executed.
self.assertTrue(ev.wait(1))
def test_error_during_teardown(self):
test = openhtf.Test(blank_phase)
executor = core.TestExecutor(test.descriptor,
'uid',
start_phase,
'dut',
teardown_function=teardown_fail)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, Outcome.ERROR)
self.assertEqual(record.outcome_details[0].code,
TeardownError.__name__)
def test_failure_during_plug_init_with_dut_id(self):
ev = threading.Event()
test = openhtf.Test(fail_plug_phase)
executor = core.TestExecutor(test.descriptor,
'uid',
start_phase,
'dut',
teardown_function=lambda: ev.set()) # pylint: disable=unnecessary-lambda
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, Outcome.ERROR)
self.assertEqual(record.outcome_details[0].code,
FailedPlugError.__name__)
self.assertEqual(record.outcome_details[0].description,
FAIL_PLUG_MESSAGE)
# Teardown function should be executed.
self.assertTrue(ev.wait(1))
def test_cancel_start(self):
@openhtf.PhaseOptions()
def cancel_phase(test):
test.dut_id = 'DUT ID'
# We have 'executor' because we're inside the test method's scope.
# We have to run it in a thread to avoid getting a nasty series of
# confusing errors:
# If we were to stop it in this phase, it eventually causes the phase
# to be killed using KillableThread, which raises ThreadTerminationError
# inside here, which really raises it inside wherever executor.stop() is.
# That leads to the stopping of the executor to get stopped itself at a
# random point in time. To make this deterministic, we keep the phase
# alive as long as the executor is running, which really just means that
# the wait() call gets the error raised in it.
inner_ev = threading.Event()
def stop_executor():
executor.stop()
inner_ev.set()
threading.Thread(target=stop_executor).start()
inner_ev.wait(1)
ev = threading.Event()
test = openhtf.Test()
# Cancel during test start phase.
executor = core.TestExecutor(test.descriptor,
'uid',
cancel_phase,
'dut',
teardown_function=lambda: ev.set()) # pylint: disable=unnecessary-lambda
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.phases[0].name, cancel_phase.name)
# The test will end at the same time it starts because the test never
# actually started, we canceled it inside of test_start, resulting in a
# short vacuous start. Start and end times should be no more than a
# millisecond or two apart in that case.
self.assertLess(record.end_time_millis - record.start_time_millis, 2)
self.assertLessEqual(record.end_time_millis, util.time_millis())
# Teardown function should not be executed.
self.assertFalse(ev.wait(3))
def test_log_during_teardown(self):
message = 'hello'
def teardown_log(test):
test.logger.info(message)
test = openhtf.Test(blank_phase)
executor = core.TestExecutor(test.descriptor,
'uid',
start_phase,
'dut',
teardown_function=teardown_log)
executor.start()
executor.wait()
record = executor.test_state.test_record
self.assertEqual(record.outcome, Outcome.PASS)
log_records = [
log_record for log_record in record.log_records
if log_record.message == message
]
self.assertTrue(log_records)
def execute(self, test_start=None):
"""Starts the framework and executes the given test.
Args:
test_start: Either a trigger phase for starting the test, or a function
that returns a DUT ID. If neither is provided, defaults to not
setting the DUT ID.
"""
# Lock this section so we don't .stop() the executor between instantiating
# it and .Start()'ing it, doing so does weird things to the executor state.
with self._lock:
# Sanity check to make sure someone isn't doing something weird like
# trying to Execute() the same test twice in two separate threads. We
# hold the lock between here and Start()'ing the executor to guarantee
# that only one thread is successfully executing the test.
if self._executor:
raise InvalidTestStateError('Test already running',
self._executor)
# Snapshot some things we care about and store them.
self._test_desc.metadata['test_name'] = self._test_options.name
self._test_desc.metadata['config'] = conf._asdict()
self.last_run_time_millis = util.time_millis()
if isinstance(test_start, LambdaType):
@TestPhase()
def trigger_phase(test):
test.test_record.dut_id = test_start()
trigger = trigger_phase
else:
trigger = test_start
if conf.capture_source:
trigger.code_info = test_record.CodeInfo.for_function(
trigger.func)
self._executor = core.TestExecutor(
self._test_desc, self.make_uid(), trigger,
self._test_options.teardown_function)
_LOG.info('Executing test: %s', self.descriptor.code_info.name)
self.TEST_INSTANCES[self.uid] = self
self._executor.start()
try:
self._executor.wait()
finally:
try:
final_state = self._executor.finalize()
_LOG.debug(
'Test completed for %s, saving to history and outputting.',
final_state.test_record.metadata['test_name'])
for output_cb in (
self._test_options.output_callbacks +
[functools.partial(history.append_record, self.uid)]):
try:
output_cb(final_state.test_record)
except Exception: # pylint: disable=broad-except
_LOG.exception(
'Output callback %s raised; continuing anyway',
output_cb)
finally:
del self.TEST_INSTANCES[self.uid]
self._executor = None
return final_state.test_record.outcome == test_record.Outcome.PASS
def execute(self, test_start=None):
"""Starts the framework and executes the given test.
Args:
test_start: Either a trigger phase for starting the test, or a function
that returns a DUT ID. If neither is provided, defaults to not
setting the DUT ID.
"""
# Lock this section so we don't .stop() the executor between instantiating
# it and .Start()'ing it, doing so does weird things to the executor state.
with self._lock:
# Sanity check to make sure someone isn't doing something weird like
# trying to Execute() the same test twice in two separate threads. We
# hold the lock between here and Start()'ing the executor to guarantee
# that only one thread is successfully executing the test.
if self._executor:
raise InvalidTestStateError('Test already running',
self._executor)
# Snapshot some things we care about and store them.
self._test_desc.metadata['test_name'] = self._test_options.name
self._test_desc.metadata['config'] = conf._asdict()
self.last_run_time_millis = util.time_millis()
if isinstance(test_start, LambdaType):
@TestPhase()
def trigger_phase(test):
test.test_record.dut_id = test_start()
trigger = trigger_phase
else:
trigger = test_start
if conf.capture_source:
trigger.code_info = test_record.CodeInfo.for_function(
trigger.func)
self._executor = core.TestExecutor(
self._test_desc, self.make_uid(), trigger,
self._test_options.default_dut_id,
self._test_options.teardown_function,
self._test_options.failure_exceptions)
_LOG.info('Executing test: %s', self.descriptor.code_info.name)
self.TEST_INSTANCES[self.uid] = self
self._executor.start()
try:
self._executor.wait()
finally:
try:
final_state = self._executor.finalize()
_LOG.debug('Test completed for %s, outputting now.',
final_state.test_record.metadata['test_name'])
for output_cb in self._test_options.output_callbacks:
try:
output_cb(final_state.test_record)
except Exception: # pylint: disable=broad-except
_LOG.error(
'Output callback %s raised; continuing anyway',
output_cb)
# Make sure the final outcome of the test is printed last and in a
# noticeable color so it doesn't get scrolled off the screen or missed.
if final_state.test_record.outcome == test_record.Outcome.ERROR:
for detail in final_state.test_record.outcome_details:
console_output.error_print(detail.description)
else:
colors = collections.defaultdict(
lambda: 'colorama.Style.BRIGHT')
colors[test_record.Outcome.PASS] = ''.join(
(colorama.Style.BRIGHT, colorama.Fore.GREEN))
colors[test_record.Outcome.FAIL] = ''.join(
(colorama.Style.BRIGHT, colorama.Fore.RED))
msg_template = "test: {name} outcome: {color}{outcome}{rst}"
console_output.banner_print(
msg_template.format(
name=final_state.test_record.metadata['test_name'],
color=colors[final_state.test_record.outcome],
outcome=final_state.test_record.outcome.name,
rst=colorama.Style.RESET_ALL))
finally:
del self.TEST_INSTANCES[self.uid]
self._executor = None
return final_state.test_record.outcome == test_record.Outcome.PASS
