def _handle_phase(self, phase_desc):
"""Handle execution of a single test phase."""
self._initialize_plugs(phase_plug.cls
for phase_plug in phase_desc.plugs)
# Cobble together a fake TestState to pass to the test phase.
with mock.patch('openhtf.plugs.PlugManager',
new=lambda _, __: self.plug_manager):
test_state_ = test_state.TestState(
openhtf.TestDescriptor((phase_desc, ), phase_desc.code_info,
{}), 'Unittest:StubTest:UID')
test_state_.mark_test_started()
# Actually execute the phase, saving the result in our return value.
with test_state_.running_phase_context(phase_desc) as phase_state:
try:
phase_state.result = phase_executor.PhaseExecutionOutcome(
phase_desc(test_state_))
except Exception: # pylint:disable=broad-except
logging.exception('Exception executing phase %s',
phase_desc.name)
phase_state.result = phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(*sys.exc_info()))
return phase_state.phase_record
def _finalize_measurements(self):
"""Perform end-of-phase finalization steps for measurements.
Any UNSET measurements will cause the Phase to FAIL unless
conf.allow_unset_measurements is set True.
"""
for measurement in self.measurements.values():
# Clear notification callbacks for later serialization.
measurement.set_notification_callback(None)
# Validate multi-dimensional measurements now that we have all values.
if measurement.outcome is measurements.Outcome.PARTIALLY_SET:
try:
measurement.validate()
except Exception: # pylint: disable=broad-except
# Record the exception as the new result.
if self.phase_record.result.is_terminal:
_LOG.exception(
'Measurement validation raised an exception, but phase result '
'is already terminal; logging additional exception here.'
)
else:
self.phase_record.result = phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(*sys.exc_info()))
# Set final values on the PhaseRecord.
self.phase_record.measurements = self.measurements
def test_test_state_finalize_from_phase_outcome_exception_info(self):
try:
raise ValueError('Exception for unit testing.')
except ValueError:
phase_exe_outcome = phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(*sys.exc_info()))
self.test_state.finalize_from_phase_outcome(phase_exe_outcome)
self.assertEqual(self.test_state.test_record.outcome,
test_record.Outcome.ERROR)
def _execute_phase_diagnoser(self, diagnoser):
try:
self.test_state.diagnoses_manager.execute_phase_diagnoser(
diagnoser, self, self.test_state.test_record)
except Exception: # pylint: disable=broad-except
if self.phase_record.result.is_terminal:
self.logger.exception(
'Phase Diagnoser %s raised an exception, but phase result is '
'already terminal; logging additonal exception here.',
diagnoser.name)
else:
self.phase_record.result = phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(*sys.exc_info()))
def _execute_test_diagnoser(self, diagnoser):
try:
self.test_state.diagnoses_manager.execute_test_diagnoser(
diagnoser, self.test_state.test_record)
except Exception: # pylint: disable=broad-except
if self._last_outcome and self._last_outcome.is_terminal:
self.test_state.state_logger.exception(
'Test Diagnoser %s raised an exception, but the test outcome is '
'already terminal; logging additional exception here.',
diagnoser.name)
else:
# Record the equivalent failure outcome and exit early.
self._last_outcome = phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(*sys.exc_info()))
def _initialize_plugs(self, plug_types=None):
"""Initialize plugs.
Args:
plug_types: optional list of plug classes to initialize.
Returns:
True if there was an error initializing the plugs.
"""
try:
self.test_state.plug_manager.initialize_plugs(plug_types=plug_types)
return False
except Exception: # pylint: disable=broad-except
# Record the equivalent failure outcome and exit early.
self._latest_outcome = phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(*sys.exc_info()))
return True
def test_all__no_previous_phases(self):
self.test_start_function = None
test_rec = yield htf.Test(
phase_branches.PhaseFailureCheckpoint.all_previous('all_prev'))
self.assertTestError(test_rec)
self.assertTestOutcomeCode(test_rec, 'NoPhasesFoundError')
self.assertEqual([
test_record.CheckpointRecord(
name='all_prev',
action=htf.PhaseResult.STOP,
conditional=phase_branches.PreviousPhases.ALL,
subtest_name=None,
result=phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(
phase_branches.NoPhasesFoundError, mock.ANY,
mock.ANY)),
evaluated_millis=htf_test.VALID_TIMESTAMP),
], test_rec.checkpoints)
def test_all_fail_subtest__not_in_subtest(self):
test_rec = yield htf.Test(
fail_phase,
phase_branches.PhaseFailureCheckpoint.all_previous(
'all_subtest', action=htf.PhaseResult.FAIL_SUBTEST),
error_phase)
self.assertTestError(test_rec)
self.assertTestOutcomeCode(test_rec, 'InvalidPhaseResultError')
self.assertPhasesOutcomeByName(test_record.PhaseOutcome.FAIL, test_rec,
'fail_phase')
self.assertEqual([
test_record.CheckpointRecord(
name='all_subtest',
action=htf.PhaseResult.FAIL_SUBTEST,
conditional=phase_branches.PreviousPhases.ALL,
subtest_name=None,
result=phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(
phase_executor.InvalidPhaseResultError, mock.ANY,
mock.ANY)),
evaluated_millis=htf_test.VALID_TIMESTAMP),
], test_rec.checkpoints)
class TextTest(test.TestCase, parameterized.TestCase):
def testColorFromTestOutcome_HasCorrespondingTestOutcomeName(self):
"""Catches OpenHTF test outcome not added in _ColorFromTestOutcome."""
for member in test_record.Outcome.__members__:
self.assertIn(member, text._ColorFromTestOutcome.__members__)
def testHeadlineFromTestOutcome_HasCorrespondingTestOutcomeName(self):
"""Catches OpenHTF test outcome not added in _HeadlineFromTestOutcome."""
for member in test_record.Outcome.__members__:
self.assertIn(member, text._HeadlineFromTestOutcome.__members__)
def testColorText_GetsColorSuccessfully(self):
text_to_colorize = 'Foo Bar'
self.assertEqual(
text._ColorText(text_to_colorize, _GREEN),
f'{_GREEN}{text_to_colorize}{colorama.Style.RESET_ALL}')
# TODO(b/70517332): Pytype currently doesn't properly support the functional
# API of enums: https://github.com/google/pytype/issues/459. Remove
# disabling pytype once fixed.
@parameterized.named_parameters(
(headline_member.name, headline_member.name, headline_member.value)
for headline_member in text._HeadlineFromTestOutcome.__iter__()) # pytype: disable=attribute-error
def testGetTestOutcomeHeadline_TestNotColorized(self, outcome, headline):
record = test_record.TestRecord(dut_id='TestDutId',
station_id='test_station',
outcome=test_record.Outcome[outcome])
self.assertEqual(text._GetTestOutcomeHeadline(record), headline)
# TODO(b/70517332): Pytype currently doesn't properly support the functional
# API of enums: https://github.com/google/pytype/issues/459. Remove
# disabling pytype once fixed.
@parameterized.named_parameters(
(headline_member.name, headline_member.name, headline_member.value)
for headline_member in text._HeadlineFromTestOutcome.__iter__()) # pytype: disable=attribute-error
def testGetTestOutcomeHeadline_TestColorized(self, outcome, headline):
record = test_record.TestRecord(dut_id='TestDutId',
station_id='test_station',
outcome=test_record.Outcome[outcome])
# TODO(b/70517332): Pytype currently doesn't properly support the functional
# API of enums: https://github.com/google/pytype/issues/459. Remove
# disabling pytype once fixed.
self.assertEqual(
text._GetTestOutcomeHeadline(record, colorize_text=True),
f'{text._ColorFromTestOutcome[outcome].value}{headline}' # pytype: disable=unsupported-operands
f'{colorama.Style.RESET_ALL}')
def testStringFromMeasurement_SuccessfullyConvertsUnsetMeasurement(self):
self.assertEqual(
text.StringFromMeasurement(
openhtf.Measurement('text_measurement_a')),
'| text_measurement_a was not set')
def testStringFromMeasurement_SuccessfullyConvertsPassMeasurement(self):
measurement = openhtf.Measurement('text_measurement_a')
measurement._measured_value = measurements.MeasuredValue(
'text_measurement_a')
measurement._measured_value.set(10)
measurement.notify_value_set()
self.assertEqual(text.StringFromMeasurement(measurement),
'| text_measurement_a: 10')
def testStringFromMeasurement_SuccessfullyConvertsFailMeasurement(self):
measurement = openhtf.Measurement('text_measurement_a').in_range(
maximum=3)
measurement._measured_value = measurements.MeasuredValue(
'text_measurement_a')
measurement._measured_value.set(5)
measurement.notify_value_set()
output = text.StringFromMeasurement(measurement)
self.assertEqual(
output,
"| text_measurement_a failed because 5 failed these checks: ['x <= 3']"
)
self.assertNotIn(text._BRIGHT_RED_STYLE, output)
def testStringFromMeasurement_SuccessfullyConvertsFailMeasurementColorized(
self):
measurement = openhtf.Measurement('text_measurement_a').in_range(
maximum=3)
measurement._measured_value = measurements.MeasuredValue(
'text_measurement_a')
measurement._measured_value.set(5)
measurement.notify_value_set()
self.assertEqual(
text.StringFromMeasurement(measurement, colorize_text=True).count(
text._BRIGHT_RED_STYLE), 1)
def testStringFromAttachment_SuccessfullyConvertsPassMeasurement(self):
attachment = test_record.Attachment('content', 'text/plain')
self.assertEqual(
text.StringFromAttachment(attachment, 'attachment_a.txt'),
'| attachment: attachment_a.txt (mimetype=text/plain)')
@parameterized.named_parameters([
{
'testcase_name': 'None',
'phase_result': None,
'expected_str': ''
},
{
'testcase_name': 'PhaseResult',
'phase_result': phase_descriptor.PhaseResult.CONTINUE,
'expected_str': 'CONTINUE'
},
{
'testcase_name':
'ExceptionInfo',
'phase_result':
phase_executor.ExceptionInfo(
ValueError, ValueError('Invalid Value'),
mock.create_autospec(types.TracebackType, spec_set=True)),
'expected_str':
'ValueError'
},
{
'testcase_name': 'ThreadTerminationError',
'phase_result': threads.ThreadTerminationError(),
'expected_str': 'ThreadTerminationError'
},
])
def testStringFromPhaseExecutionOutcome_SuccessfullyConvertsOutcome(
self, phase_result, expected_str):
self.assertEqual(
text.StringFromPhaseExecutionOutcome(
phase_executor.PhaseExecutionOutcome(phase_result)),
expected_str)
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordPassPhase(
self):
record = self.execute_phase_or_test(PhaseThatSucceeds)
output = text.StringFromPhaseRecord(record)
self.assertEqual(
output, 'Phase PhaseThatSucceeds\n'
'+ Outcome: PASS Result: CONTINUE\n'
'| text_measurement_a: a\n'
'| text_measurement_b: b\n'
'| attachment: attachment_a.txt (mimetype=text/plain)\n'
'| attachment: attachment_b.json (mimetype=application/json)')
self.assertNotIn(text._BRIGHT_RED_STYLE, output)
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordFailPhase(
self):
record = self.execute_phase_or_test(PhaseWithFailure)
output = text.StringFromPhaseRecord(record)
self.assertEqual(
output, 'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]\n'
'| text_measurement_b was not set\n'
'| text_measurement_c: c')
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseFailLimitPhase(
self):
record = self.execute_phase_or_test(PhaseWithFailure)
output = text.StringFromPhaseRecord(record, maximum_num_measurements=2)
self.assertEqual(
output, 'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]\n'
'| text_measurement_b was not set\n'
'...')
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordOnlyFailPhase(
self):
record = self.execute_phase_or_test(PhaseWithFailure)
output = text.StringFromPhaseRecord(record, only_failures=True)
self.assertEqual(
output, 'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]')
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordFailPhaseColored(
self):
record = self.execute_phase_or_test(PhaseWithFailure)
self.assertEqual(
text.StringFromPhaseRecord(record, colorize_text=True).count(_RED),
3)
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordSkipPhaseColored(
self):
record = self.execute_phase_or_test(PhaseWithSkip)
self.assertNotIn(
text._BRIGHT_RED_STYLE,
text.StringFromPhaseRecord(record, colorize_text=True))
@parameterized.named_parameters([
{
'testcase_name':
'OneOutcome',
'outcome_details': [
test_record.OutcomeDetails(code=1,
description='Unknown exception.')
],
'expected_str': ('The test thinks this may be the reason:\n'
'1: Unknown exception.'),
},
{
'testcase_name':
'TwoOutcomes',
'outcome_details': [
test_record.OutcomeDetails(code=1,
description='Unknown exception.'),
test_record.OutcomeDetails(code='FooError',
description='Foo exception.')
],
'expected_str': ('The test thinks these may be the reason:\n'
'1: Unknown exception.\n'
'FooError: Foo exception.'),
},
])
def testStringFromOutcomeDetails_SuccessfullyConvertsOutcomeDetails(
self, outcome_details, expected_str):
self.assertEqual(text.StringFromOutcomeDetails(outcome_details),
expected_str)
def testStringFromTestRecord_SuccessfullyConvertsTestRecordSinglePassPhase(
self):
record = self.execute_phase_or_test(openhtf.Test(PhaseThatSucceeds))
self.assertEqual(
text.StringFromTestRecord(record), 'Test finished with a PASS!\n'
'Woohoo!\n'
'Phase trigger_phase\n'
'+ Outcome: PASS Result: CONTINUE\n'
'Phase PhaseThatSucceeds\n'
'+ Outcome: PASS Result: CONTINUE\n'
'| text_measurement_a: a\n'
'| text_measurement_b: b\n'
'| attachment: attachment_a.txt (mimetype=text/plain)\n'
'| attachment: attachment_b.json (mimetype=application/json)\n'
'Test finished with a PASS!')
def testStringFromTestRecord_SuccessfullyConvertsTestErrorPhase(self):
record = self.execute_phase_or_test(openhtf.Test(PhaseWithError))
self.assertEqual(
text.StringFromTestRecord(record), 'Test encountered an ERROR!!!\n'
'Phase trigger_phase\n'
'+ Outcome: PASS Result: CONTINUE\n'
'Phase PhaseWithError\n'
'+ Outcome: ERROR Result: Exception\n'
'| text_measurement_a was not set\n'
'| text_measurement_b was not set\n'
'The test thinks this may be the reason:\n'
'Exception: Intentional exception from test case.\n'
'Test encountered an ERROR!!!')
def testStringFromTestRecord_SuccessfullyConvertsTestFailurePhase(self):
record = self.execute_phase_or_test(openhtf.Test(PhaseWithFailure))
output = text.StringFromTestRecord(record)
self.assertEqual(
output, 'Test finished with a FAIL :(\n'
'Phase trigger_phase\n'
'+ Outcome: PASS Result: CONTINUE\n'
'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]\n'
'| text_measurement_b was not set\n'
'| text_measurement_c: c\n'
'Test finished with a FAIL :(')
self.assertNotIn(text._BRIGHT_RED_STYLE, output)
def testStringFromTestRecord_SuccessfullyConvertsTestOnlyFailurePhase(
self):
record = self.execute_phase_or_test(
openhtf.Test(PhaseThatSucceeds, PhaseWithFailure))
output = text.StringFromTestRecord(record, only_failures=True)
self.assertEqual(
output, 'Test finished with a FAIL :(\n'
'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]\n'
'Test finished with a FAIL :(')
self.assertNotIn(text._BRIGHT_RED_STYLE, output)
def testStringFromTestRecord_SuccessfullyConvertsTestFailurePhaseColored(
self):
record = self.execute_phase_or_test(openhtf.Test(PhaseWithFailure))
self.assertEqual(
text.StringFromTestRecord(record, colorize_text=True).count(_RED),
5)
def testStringFromTestRecord_SuccessfullyConvertsTestFailureMultiplePhases(
self):
record = self.execute_phase_or_test(
openhtf.Test(PhaseThatSucceeds, PhaseWithFailure))
self.assertEqual(
text.StringFromTestRecord(record), 'Test finished with a FAIL :(\n'
'Phase trigger_phase\n'
'+ Outcome: PASS Result: CONTINUE\n'
'Phase PhaseThatSucceeds\n'
'+ Outcome: PASS Result: CONTINUE\n'
'| text_measurement_a: a\n'
'| text_measurement_b: b\n'
'| attachment: attachment_a.txt (mimetype=text/plain)\n'
'| attachment: attachment_b.json (mimetype=application/json)\n'
'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]\n'
'| text_measurement_b was not set\n'
'| text_measurement_c: c\n'
'Test finished with a FAIL :(')
def testPrintTestRecord_SuccessfullyLogsNotColored(self):
record = self.execute_phase_or_test(openhtf.Test(PhaseThatSucceeds))
with mock.patch.object(sys, 'stdout', new_callable=io.StringIO) as cm:
with mock.patch.object(sys.stdout,
sys.stdout.isatty.__name__,
autospec=True,
spec_set=True,
return_value=False):
text.PrintTestRecord(record)
self.assertTrue(cm.getvalue())
self.assertNotIn(_GREEN, cm.getvalue())
def testPrintTestRecord_SuccessfullyLogsColored(self):
record = self.execute_phase_or_test(openhtf.Test(PhaseThatSucceeds))
with mock.patch.object(sys, 'stdout', new_callable=io.StringIO) as cm:
with mock.patch.object(sys.stdout,
sys.stdout.isatty.__name__,
autospec=True,
spec_set=True,
return_value=True):
text.PrintTestRecord(record)
self.assertIn(_GREEN, cm.getvalue())
def test_execute_phase_bad_phase_return(self):
result, _ = self.phase_executor.execute_phase(bad_return_phase)
self.assertEqual(
phase_executor.ExceptionInfo(
phase_executor.InvalidPhaseResultError, mock.ANY, mock.ANY),
result.phase_result)
