def _handle_phase(self, phase_desc): """Handle execution of a single test phase.""" self._initialize_plugs(phase_plug.cls for phase_plug in phase_desc.plugs) # Cobble together a fake TestState to pass to the test phase. with mock.patch('openhtf.plugs.PlugManager', new=lambda _, __: self.plug_manager): test_state_ = test_state.TestState( openhtf.TestDescriptor((phase_desc, ), phase_desc.code_info, {}), 'Unittest:StubTest:UID') test_state_.mark_test_started() # Actually execute the phase, saving the result in our return value. with test_state_.running_phase_context(phase_desc) as phase_state: try: phase_state.result = phase_executor.PhaseExecutionOutcome( phase_desc(test_state_)) except Exception: # pylint:disable=broad-except logging.exception('Exception executing phase %s', phase_desc.name) phase_state.result = phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo(*sys.exc_info())) return phase_state.phase_record
def _finalize_measurements(self): """Perform end-of-phase finalization steps for measurements. Any UNSET measurements will cause the Phase to FAIL unless conf.allow_unset_measurements is set True. """ for measurement in self.measurements.values(): # Clear notification callbacks for later serialization. measurement.set_notification_callback(None) # Validate multi-dimensional measurements now that we have all values. if measurement.outcome is measurements.Outcome.PARTIALLY_SET: try: measurement.validate() except Exception: # pylint: disable=broad-except # Record the exception as the new result. if self.phase_record.result.is_terminal: _LOG.exception( 'Measurement validation raised an exception, but phase result ' 'is already terminal; logging additional exception here.' ) else: self.phase_record.result = phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo(*sys.exc_info())) # Set final values on the PhaseRecord. self.phase_record.measurements = self.measurements
def test_test_state_finalize_from_phase_outcome_exception_info(self): try: raise ValueError('Exception for unit testing.') except ValueError: phase_exe_outcome = phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo(*sys.exc_info())) self.test_state.finalize_from_phase_outcome(phase_exe_outcome) self.assertEqual(self.test_state.test_record.outcome, test_record.Outcome.ERROR)
def _execute_phase_diagnoser(self, diagnoser): try: self.test_state.diagnoses_manager.execute_phase_diagnoser( diagnoser, self, self.test_state.test_record) except Exception: # pylint: disable=broad-except if self.phase_record.result.is_terminal: self.logger.exception( 'Phase Diagnoser %s raised an exception, but phase result is ' 'already terminal; logging additonal exception here.', diagnoser.name) else: self.phase_record.result = phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo(*sys.exc_info()))
def _execute_test_diagnoser(self, diagnoser): try: self.test_state.diagnoses_manager.execute_test_diagnoser( diagnoser, self.test_state.test_record) except Exception: # pylint: disable=broad-except if self._last_outcome and self._last_outcome.is_terminal: self.test_state.state_logger.exception( 'Test Diagnoser %s raised an exception, but the test outcome is ' 'already terminal; logging additional exception here.', diagnoser.name) else: # Record the equivalent failure outcome and exit early. self._last_outcome = phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo(*sys.exc_info()))
def _initialize_plugs(self, plug_types=None): """Initialize plugs. Args: plug_types: optional list of plug classes to initialize. Returns: True if there was an error initializing the plugs. """ try: self.test_state.plug_manager.initialize_plugs(plug_types=plug_types) return False except Exception: # pylint: disable=broad-except # Record the equivalent failure outcome and exit early. self._latest_outcome = phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo(*sys.exc_info())) return True
def test_all__no_previous_phases(self): self.test_start_function = None test_rec = yield htf.Test( phase_branches.PhaseFailureCheckpoint.all_previous('all_prev')) self.assertTestError(test_rec) self.assertTestOutcomeCode(test_rec, 'NoPhasesFoundError') self.assertEqual([ test_record.CheckpointRecord( name='all_prev', action=htf.PhaseResult.STOP, conditional=phase_branches.PreviousPhases.ALL, subtest_name=None, result=phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo( phase_branches.NoPhasesFoundError, mock.ANY, mock.ANY)), evaluated_millis=htf_test.VALID_TIMESTAMP), ], test_rec.checkpoints)
def test_all_fail_subtest__not_in_subtest(self): test_rec = yield htf.Test( fail_phase, phase_branches.PhaseFailureCheckpoint.all_previous( 'all_subtest', action=htf.PhaseResult.FAIL_SUBTEST), error_phase) self.assertTestError(test_rec) self.assertTestOutcomeCode(test_rec, 'InvalidPhaseResultError') self.assertPhasesOutcomeByName(test_record.PhaseOutcome.FAIL, test_rec, 'fail_phase') self.assertEqual([ test_record.CheckpointRecord( name='all_subtest', action=htf.PhaseResult.FAIL_SUBTEST, conditional=phase_branches.PreviousPhases.ALL, subtest_name=None, result=phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo( phase_executor.InvalidPhaseResultError, mock.ANY, mock.ANY)), evaluated_millis=htf_test.VALID_TIMESTAMP), ], test_rec.checkpoints)
class TextTest(test.TestCase, parameterized.TestCase): def testColorFromTestOutcome_HasCorrespondingTestOutcomeName(self): """Catches OpenHTF test outcome not added in _ColorFromTestOutcome.""" for member in test_record.Outcome.__members__: self.assertIn(member, text._ColorFromTestOutcome.__members__) def testHeadlineFromTestOutcome_HasCorrespondingTestOutcomeName(self): """Catches OpenHTF test outcome not added in _HeadlineFromTestOutcome.""" for member in test_record.Outcome.__members__: self.assertIn(member, text._HeadlineFromTestOutcome.__members__) def testColorText_GetsColorSuccessfully(self): text_to_colorize = 'Foo Bar' self.assertEqual( text._ColorText(text_to_colorize, _GREEN), f'{_GREEN}{text_to_colorize}{colorama.Style.RESET_ALL}') # TODO(b/70517332): Pytype currently doesn't properly support the functional # API of enums: https://github.com/google/pytype/issues/459. Remove # disabling pytype once fixed. @parameterized.named_parameters( (headline_member.name, headline_member.name, headline_member.value) for headline_member in text._HeadlineFromTestOutcome.__iter__()) # pytype: disable=attribute-error def testGetTestOutcomeHeadline_TestNotColorized(self, outcome, headline): record = test_record.TestRecord(dut_id='TestDutId', station_id='test_station', outcome=test_record.Outcome[outcome]) self.assertEqual(text._GetTestOutcomeHeadline(record), headline) # TODO(b/70517332): Pytype currently doesn't properly support the functional # API of enums: https://github.com/google/pytype/issues/459. Remove # disabling pytype once fixed. @parameterized.named_parameters( (headline_member.name, headline_member.name, headline_member.value) for headline_member in text._HeadlineFromTestOutcome.__iter__()) # pytype: disable=attribute-error def testGetTestOutcomeHeadline_TestColorized(self, outcome, headline): record = test_record.TestRecord(dut_id='TestDutId', station_id='test_station', outcome=test_record.Outcome[outcome]) # TODO(b/70517332): Pytype currently doesn't properly support the functional # API of enums: https://github.com/google/pytype/issues/459. Remove # disabling pytype once fixed. self.assertEqual( text._GetTestOutcomeHeadline(record, colorize_text=True), f'{text._ColorFromTestOutcome[outcome].value}{headline}' # pytype: disable=unsupported-operands f'{colorama.Style.RESET_ALL}') def testStringFromMeasurement_SuccessfullyConvertsUnsetMeasurement(self): self.assertEqual( text.StringFromMeasurement( openhtf.Measurement('text_measurement_a')), '| text_measurement_a was not set') def testStringFromMeasurement_SuccessfullyConvertsPassMeasurement(self): measurement = openhtf.Measurement('text_measurement_a') measurement._measured_value = measurements.MeasuredValue( 'text_measurement_a') measurement._measured_value.set(10) measurement.notify_value_set() self.assertEqual(text.StringFromMeasurement(measurement), '| text_measurement_a: 10') def testStringFromMeasurement_SuccessfullyConvertsFailMeasurement(self): measurement = openhtf.Measurement('text_measurement_a').in_range( maximum=3) measurement._measured_value = measurements.MeasuredValue( 'text_measurement_a') measurement._measured_value.set(5) measurement.notify_value_set() output = text.StringFromMeasurement(measurement) self.assertEqual( output, "| text_measurement_a failed because 5 failed these checks: ['x <= 3']" ) self.assertNotIn(text._BRIGHT_RED_STYLE, output) def testStringFromMeasurement_SuccessfullyConvertsFailMeasurementColorized( self): measurement = openhtf.Measurement('text_measurement_a').in_range( maximum=3) measurement._measured_value = measurements.MeasuredValue( 'text_measurement_a') measurement._measured_value.set(5) measurement.notify_value_set() self.assertEqual( text.StringFromMeasurement(measurement, colorize_text=True).count( text._BRIGHT_RED_STYLE), 1) def testStringFromAttachment_SuccessfullyConvertsPassMeasurement(self): attachment = test_record.Attachment('content', 'text/plain') self.assertEqual( text.StringFromAttachment(attachment, 'attachment_a.txt'), '| attachment: attachment_a.txt (mimetype=text/plain)') @parameterized.named_parameters([ { 'testcase_name': 'None', 'phase_result': None, 'expected_str': '' }, { 'testcase_name': 'PhaseResult', 'phase_result': phase_descriptor.PhaseResult.CONTINUE, 'expected_str': 'CONTINUE' }, { 'testcase_name': 'ExceptionInfo', 'phase_result': phase_executor.ExceptionInfo( ValueError, ValueError('Invalid Value'), mock.create_autospec(types.TracebackType, spec_set=True)), 'expected_str': 'ValueError' }, { 'testcase_name': 'ThreadTerminationError', 'phase_result': threads.ThreadTerminationError(), 'expected_str': 'ThreadTerminationError' }, ]) def testStringFromPhaseExecutionOutcome_SuccessfullyConvertsOutcome( self, phase_result, expected_str): self.assertEqual( text.StringFromPhaseExecutionOutcome( phase_executor.PhaseExecutionOutcome(phase_result)), expected_str) def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordPassPhase( self): record = self.execute_phase_or_test(PhaseThatSucceeds) output = text.StringFromPhaseRecord(record) self.assertEqual( output, 'Phase PhaseThatSucceeds\n' '+ Outcome: PASS Result: CONTINUE\n' '| text_measurement_a: a\n' '| text_measurement_b: b\n' '| attachment: attachment_a.txt (mimetype=text/plain)\n' '| attachment: attachment_b.json (mimetype=application/json)') self.assertNotIn(text._BRIGHT_RED_STYLE, output) def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordFailPhase( self): record = self.execute_phase_or_test(PhaseWithFailure) output = text.StringFromPhaseRecord(record) self.assertEqual( output, 'Phase PhaseWithFailure\n' '+ Outcome: FAIL Result: CONTINUE\n' '| text_measurement_a failed because intentionally wrong measurement ' 'failed these checks: ["\'x\' matches /^a$/"]\n' '| text_measurement_b was not set\n' '| text_measurement_c: c') def testStringFromPhaseRecord_SuccessfullyConvertsPhaseFailLimitPhase( self): record = self.execute_phase_or_test(PhaseWithFailure) output = text.StringFromPhaseRecord(record, maximum_num_measurements=2) self.assertEqual( output, 'Phase PhaseWithFailure\n' '+ Outcome: FAIL Result: CONTINUE\n' '| text_measurement_a failed because intentionally wrong measurement ' 'failed these checks: ["\'x\' matches /^a$/"]\n' '| text_measurement_b was not set\n' '...') def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordOnlyFailPhase( self): record = self.execute_phase_or_test(PhaseWithFailure) output = text.StringFromPhaseRecord(record, only_failures=True) self.assertEqual( output, 'Phase PhaseWithFailure\n' '+ Outcome: FAIL Result: CONTINUE\n' '| text_measurement_a failed because intentionally wrong measurement ' 'failed these checks: ["\'x\' matches /^a$/"]') def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordFailPhaseColored( self): record = self.execute_phase_or_test(PhaseWithFailure) self.assertEqual( text.StringFromPhaseRecord(record, colorize_text=True).count(_RED), 3) def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordSkipPhaseColored( self): record = self.execute_phase_or_test(PhaseWithSkip) self.assertNotIn( text._BRIGHT_RED_STYLE, text.StringFromPhaseRecord(record, colorize_text=True)) @parameterized.named_parameters([ { 'testcase_name': 'OneOutcome', 'outcome_details': [ test_record.OutcomeDetails(code=1, description='Unknown exception.') ], 'expected_str': ('The test thinks this may be the reason:\n' '1: Unknown exception.'), }, { 'testcase_name': 'TwoOutcomes', 'outcome_details': [ test_record.OutcomeDetails(code=1, description='Unknown exception.'), test_record.OutcomeDetails(code='FooError', description='Foo exception.') ], 'expected_str': ('The test thinks these may be the reason:\n' '1: Unknown exception.\n' 'FooError: Foo exception.'), }, ]) def testStringFromOutcomeDetails_SuccessfullyConvertsOutcomeDetails( self, outcome_details, expected_str): self.assertEqual(text.StringFromOutcomeDetails(outcome_details), expected_str) def testStringFromTestRecord_SuccessfullyConvertsTestRecordSinglePassPhase( self): record = self.execute_phase_or_test(openhtf.Test(PhaseThatSucceeds)) self.assertEqual( text.StringFromTestRecord(record), 'Test finished with a PASS!\n' 'Woohoo!\n' 'Phase trigger_phase\n' '+ Outcome: PASS Result: CONTINUE\n' 'Phase PhaseThatSucceeds\n' '+ Outcome: PASS Result: CONTINUE\n' '| text_measurement_a: a\n' '| text_measurement_b: b\n' '| attachment: attachment_a.txt (mimetype=text/plain)\n' '| attachment: attachment_b.json (mimetype=application/json)\n' 'Test finished with a PASS!') def testStringFromTestRecord_SuccessfullyConvertsTestErrorPhase(self): record = self.execute_phase_or_test(openhtf.Test(PhaseWithError)) self.assertEqual( text.StringFromTestRecord(record), 'Test encountered an ERROR!!!\n' 'Phase trigger_phase\n' '+ Outcome: PASS Result: CONTINUE\n' 'Phase PhaseWithError\n' '+ Outcome: ERROR Result: Exception\n' '| text_measurement_a was not set\n' '| text_measurement_b was not set\n' 'The test thinks this may be the reason:\n' 'Exception: Intentional exception from test case.\n' 'Test encountered an ERROR!!!') def testStringFromTestRecord_SuccessfullyConvertsTestFailurePhase(self): record = self.execute_phase_or_test(openhtf.Test(PhaseWithFailure)) output = text.StringFromTestRecord(record) self.assertEqual( output, 'Test finished with a FAIL :(\n' 'Phase trigger_phase\n' '+ Outcome: PASS Result: CONTINUE\n' 'Phase PhaseWithFailure\n' '+ Outcome: FAIL Result: CONTINUE\n' '| text_measurement_a failed because intentionally wrong measurement ' 'failed these checks: ["\'x\' matches /^a$/"]\n' '| text_measurement_b was not set\n' '| text_measurement_c: c\n' 'Test finished with a FAIL :(') self.assertNotIn(text._BRIGHT_RED_STYLE, output) def testStringFromTestRecord_SuccessfullyConvertsTestOnlyFailurePhase( self): record = self.execute_phase_or_test( openhtf.Test(PhaseThatSucceeds, PhaseWithFailure)) output = text.StringFromTestRecord(record, only_failures=True) self.assertEqual( output, 'Test finished with a FAIL :(\n' 'Phase PhaseWithFailure\n' '+ Outcome: FAIL Result: CONTINUE\n' '| text_measurement_a failed because intentionally wrong measurement ' 'failed these checks: ["\'x\' matches /^a$/"]\n' 'Test finished with a FAIL :(') self.assertNotIn(text._BRIGHT_RED_STYLE, output) def testStringFromTestRecord_SuccessfullyConvertsTestFailurePhaseColored( self): record = self.execute_phase_or_test(openhtf.Test(PhaseWithFailure)) self.assertEqual( text.StringFromTestRecord(record, colorize_text=True).count(_RED), 5) def testStringFromTestRecord_SuccessfullyConvertsTestFailureMultiplePhases( self): record = self.execute_phase_or_test( openhtf.Test(PhaseThatSucceeds, PhaseWithFailure)) self.assertEqual( text.StringFromTestRecord(record), 'Test finished with a FAIL :(\n' 'Phase trigger_phase\n' '+ Outcome: PASS Result: CONTINUE\n' 'Phase PhaseThatSucceeds\n' '+ Outcome: PASS Result: CONTINUE\n' '| text_measurement_a: a\n' '| text_measurement_b: b\n' '| attachment: attachment_a.txt (mimetype=text/plain)\n' '| attachment: attachment_b.json (mimetype=application/json)\n' 'Phase PhaseWithFailure\n' '+ Outcome: FAIL Result: CONTINUE\n' '| text_measurement_a failed because intentionally wrong measurement ' 'failed these checks: ["\'x\' matches /^a$/"]\n' '| text_measurement_b was not set\n' '| text_measurement_c: c\n' 'Test finished with a FAIL :(') def testPrintTestRecord_SuccessfullyLogsNotColored(self): record = self.execute_phase_or_test(openhtf.Test(PhaseThatSucceeds)) with mock.patch.object(sys, 'stdout', new_callable=io.StringIO) as cm: with mock.patch.object(sys.stdout, sys.stdout.isatty.__name__, autospec=True, spec_set=True, return_value=False): text.PrintTestRecord(record) self.assertTrue(cm.getvalue()) self.assertNotIn(_GREEN, cm.getvalue()) def testPrintTestRecord_SuccessfullyLogsColored(self): record = self.execute_phase_or_test(openhtf.Test(PhaseThatSucceeds)) with mock.patch.object(sys, 'stdout', new_callable=io.StringIO) as cm: with mock.patch.object(sys.stdout, sys.stdout.isatty.__name__, autospec=True, spec_set=True, return_value=True): text.PrintTestRecord(record) self.assertIn(_GREEN, cm.getvalue())
def test_execute_phase_bad_phase_return(self): result, _ = self.phase_executor.execute_phase(bad_return_phase) self.assertEqual( phase_executor.ExceptionInfo( phase_executor.InvalidPhaseResultError, mock.ANY, mock.ANY), result.phase_result)