def _initialize_plugs(self, plug_types=None): """Initialize plugs. Args: plug_types: optional list of plug classes to initialize. Returns: True if there was an error initializing the plugs. """ try: self.test_state.plug_manager.initialize_plugs( plug_types=plug_types) return False except Exception: # pylint: disable=broad-except # Record the equivalent failure outcome and exit early. self._last_outcome = phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo(*sys.exc_info())) return True
def test_last__no_failures(self): test_rec = yield htf.Test( phase0, phase_branches.PhaseFailureCheckpoint.last('last_pass'), phase1) self.assertTestPass(test_rec) self.assertPhasesOutcomeByName(test_record.PhaseOutcome.PASS, test_rec, 'phase0', 'phase1') self.assertEqual([ test_record.CheckpointRecord( name='last_pass', action=htf.PhaseResult.STOP, conditional=phase_branches.PreviousPhases.LAST, subtest_name=None, result=phase_executor.PhaseExecutionOutcome( htf.PhaseResult.CONTINUE), evaluated_millis=htf_test.VALID_TIMESTAMP), ], test_rec.checkpoints)
def _execute_phase(self, phase: phase_descriptor.PhaseDescriptor, subtest_rec: Optional[test_record.SubtestRecord], in_teardown: bool) -> _ExecutorReturn: if subtest_rec: self.logger.debug('Executing phase %s under subtest %s', phase.name, subtest_rec.name) else: self.logger.debug('Executing phase %s', phase.name) if not in_teardown and subtest_rec and subtest_rec.is_fail: self._phase_exec.skip_phase(phase, subtest_rec) return _ExecutorReturn.CONTINUE outcome, profile_stats = self._phase_exec.execute_phase( phase, run_with_profiling=self._run_with_profiling, subtest_rec=subtest_rec) if profile_stats is not None: self._phase_profile_stats.append(profile_stats) if (self.test_state.test_options.stop_on_first_failure or conf.stop_on_first_failure): # Stop Test on first measurement failure current_phase_result = self.test_state.test_record.phases[ len(self.test_state.test_record.phases) - 1] if current_phase_result.outcome == test_record.PhaseOutcome.FAIL: outcome = phase_executor.PhaseExecutionOutcome( phase_descriptor.PhaseResult.STOP) self.logger.error( 'Stopping test because stop_on_first_failure is True') if outcome.is_terminal: if not self._last_outcome: self._last_outcome = outcome return _ExecutorReturn.TERMINAL if outcome.is_fail_subtest: if not subtest_rec: raise TestExecutionError( 'INVALID STATE: Phase returned outcome FAIL_SUBTEST when not ' 'in subtest.') subtest_rec.outcome = test_record.SubtestOutcome.FAIL return _ExecutorReturn.CONTINUE
def test_all__fail(self): test_rec = yield htf.Test( fail_phase, phase_branches.PhaseFailureCheckpoint.all_previous('all_fail'), error_phase) self.assertTestFail(test_rec) self.assertPhasesOutcomeByName(test_record.PhaseOutcome.FAIL, test_rec, 'fail_phase') self.assertEqual([ test_record.CheckpointRecord( name='all_fail', action=htf.PhaseResult.STOP, conditional=phase_branches.PreviousPhases.ALL, subtest_name=None, result=phase_executor.PhaseExecutionOutcome( htf.PhaseResult.STOP), evaluated_millis=htf_test.VALID_TIMESTAMP), ], test_rec.checkpoints)
def test_all__no_previous_phases(self): self.test_start_function = None test_rec = yield htf.Test( phase_branches.PhaseFailureCheckpoint.all_previous('all_prev')) self.assertTestError(test_rec) self.assertTestOutcomeCode(test_rec, 'NoPhasesFoundError') self.assertEqual([ test_record.CheckpointRecord( name='all_prev', action=htf.PhaseResult.STOP, conditional=phase_branches.PreviousPhases.ALL, subtest_name=None, result=phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo( phase_branches.NoPhasesFoundError, mock.ANY, mock.ANY)), evaluated_millis=htf_test.VALID_TIMESTAMP), ], test_rec.checkpoints)
def _handle_phase(self, phase): if isinstance(phase, phase_group.PhaseGroup): return self._execute_phase_group(phase) self.test_state.state_logger.debug('Handling phase %s', phase.name) outcome = self._phase_exec.execute_phase(phase) if (self.test_state.test_options.stop_on_first_failure or conf.stop_on_first_failure): # Stop Test on first measurement failure current_phase_result = self.test_state.test_record.phases[ len(self.test_state.test_record.phases) - 1] if current_phase_result.outcome == test_record.PhaseOutcome.FAIL: outcome = phase_executor.PhaseExecutionOutcome( phase_descriptor.PhaseResult.STOP) self.test_state.state_logger.error( 'Stopping test because stop_on_first_failure is True') if outcome.is_terminal and not self._last_outcome: self._last_outcome = outcome return outcome.is_terminal
def test_all_fail_subtest__pass_in_subtest(self): test_rec = yield htf.Test( phase0, htf.Subtest( 'sub', phase1, phase_branches.PhaseFailureCheckpoint.all_previous( 'all_pass_subtest', action=htf.PhaseResult.FAIL_SUBTEST), phase2), phase3) self.assertTestPass(test_rec) self.assertPhasesOutcomeByName(test_record.PhaseOutcome.PASS, test_rec, 'phase0', 'phase1', 'phase2', 'phase3') self.assertEqual([ test_record.CheckpointRecord( name='all_pass_subtest', action=htf.PhaseResult.FAIL_SUBTEST, conditional=phase_branches.PreviousPhases.ALL, subtest_name='sub', result=phase_executor.PhaseExecutionOutcome( htf.PhaseResult.CONTINUE), evaluated_millis=htf_test.VALID_TIMESTAMP), ], test_rec.checkpoints)
def test_pass(self): test_rec = yield htf.Test( phase0, phase_branches.DiagnosisCheckpoint( 'diag_pass', phase_branches.DiagnosisCondition.on_all( BranchDiagResult.NOT_SET)), phase1) self.assertTestPass(test_rec) self.assertPhasesOutcomeByName(test_record.PhaseOutcome.PASS, test_rec, 'phase0', 'phase1') self.assertEqual([ test_record.CheckpointRecord( name='diag_pass', action=htf.PhaseResult.STOP, conditional=phase_branches.DiagnosisCondition( phase_branches.ConditionOn.ALL, (BranchDiagResult.NOT_SET, )), subtest_name=None, result=phase_executor.PhaseExecutionOutcome( htf.PhaseResult.CONTINUE), evaluated_millis=htf_test.VALID_TIMESTAMP), ], test_rec.checkpoints)
def test_all_fail_subtest__not_in_subtest(self): test_rec = yield htf.Test( fail_phase, phase_branches.PhaseFailureCheckpoint.all_previous( 'all_subtest', action=htf.PhaseResult.FAIL_SUBTEST), error_phase) self.assertTestError(test_rec) self.assertTestOutcomeCode(test_rec, 'InvalidPhaseResultError') self.assertPhasesOutcomeByName(test_record.PhaseOutcome.FAIL, test_rec, 'fail_phase') self.assertEqual([ test_record.CheckpointRecord( name='all_subtest', action=htf.PhaseResult.FAIL_SUBTEST, conditional=phase_branches.PreviousPhases.ALL, subtest_name=None, result=phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo( phase_executor.InvalidPhaseResultError, mock.ANY, mock.ANY)), evaluated_millis=htf_test.VALID_TIMESTAMP), ], test_rec.checkpoints)
def testStringFromPhaseExecutionOutcome_SuccessfullyConvertsOutcome( self, phase_result, expected_str): self.assertEqual( text.StringFromPhaseExecutionOutcome( phase_executor.PhaseExecutionOutcome(phase_result)), expected_str)
class TestTestApi(parameterized.TestCase): def setUp(self): super(TestTestApi, self).setUp() patcher = mock.patch.object(test_record.PhaseRecord, 'record_start_time', return_value=11235) self.mock_record_start_time = patcher.start() self.addCleanup(patcher.stop) self.test_descriptor = test_descriptor.TestDescriptor( phase_collections.PhaseSequence((test_phase, )), test_record.CodeInfo.uncaptured(), {'config': {}}) self.test_state = test_state.TestState(self.test_descriptor, 'testing-123', test_descriptor.TestOptions()) self.test_record = self.test_state.test_record self.running_phase_state = test_state.PhaseState.from_descriptor( test_phase, self.test_state, logging.getLogger()) self.test_state.running_phase_state = self.running_phase_state self.test_api = self.test_state.test_api def test_get_attachment(self): attachment_name = 'attachment.txt' input_contents = b'This is some attachment text!' mimetype = 'text/plain' self.test_api.attach(attachment_name, input_contents, mimetype) output_attachment = self.test_api.get_attachment(attachment_name) if not output_attachment: # Need branch to appease pytype. self.fail('output_attachment not found') self.assertEqual(input_contents, output_attachment.data) self.assertEqual(mimetype, output_attachment.mimetype) def test_get_attachment_strict(self): attachment_name = 'attachment.txt' with self.assertRaises(test_descriptor.AttachmentNotFoundError): self.test_api.get_attachment_strict(attachment_name) def test_get_measurement(self): measurement_val = [1, 2, 3] self.test_api.measurements['test_measurement'] = measurement_val measurement = self.test_api.get_measurement('test_measurement') if not measurement: # Need branch to appease pytype. self.fail('measurement not found.') self.assertEqual(measurement_val, measurement.value) self.assertEqual('test_measurement', measurement.name) def test_get_measurement_immutable(self): measurement_val = [1, 2, 3] self.test_api.measurements['test_measurement'] = measurement_val measurement = self.test_api.get_measurement('test_measurement') if not measurement: # Need branch to appease pytype. self.fail('measurement not found.') self.assertEqual(measurement_val, measurement.value) self.assertEqual('test_measurement', measurement.name) measurement.value.append(4) self.assertNotEqual(measurement_val, measurement.value) def test_infer_mime_type_from_file_name(self): with tempfile.NamedTemporaryFile(suffix='.txt') as f: f.write(b'Mock text contents.') f.flush() file_name = f.name self.test_api.attach_from_file(file_name, 'attachment') attachment = self.test_api.get_attachment('attachment') if not attachment: # Need branch to appease pytype. self.fail('attachment not found.') self.assertEqual(attachment.mimetype, 'text/plain') def test_infer_mime_type_from_attachment_name(self): with tempfile.NamedTemporaryFile() as f: f.write(b'Mock text contents.') f.flush() file_name = f.name self.test_api.attach_from_file(file_name, 'attachment.png') attachment = self.test_api.get_attachment('attachment.png') if not attachment: # Need branch to appease pytype. self.fail('attachment not found.') self.assertEqual(attachment.mimetype, 'image/png') def test_phase_state_cache(self): basetypes = self.running_phase_state.as_base_types() expected_initial_basetypes = copy.deepcopy( PHASE_STATE_BASE_TYPE_INITIAL) expected_initial_basetypes['descriptor_id'] = basetypes[ 'descriptor_id'] self.assertEqual(expected_initial_basetypes, basetypes) self.assertFalse(self.running_phase_state._update_measurements) self.test_api.measurements.test_measurement = 5 self.assertEqual({'test_measurement'}, self.running_phase_state._update_measurements) self.running_phase_state.as_base_types() expected_after_basetypes = copy.deepcopy(expected_initial_basetypes) expected_after_basetypes['measurements']['test_measurement'].update({ 'outcome': 'PASS', 'measured_value': 5, }) self.assertEqual(expected_after_basetypes, basetypes) self.assertFalse(self.running_phase_state._update_measurements) def test_test_state_cache(self): basetypes = self.test_state.as_base_types() # The descriptor id is not static, so grab it. expected_initial_basetypes = copy.deepcopy( TEST_STATE_BASE_TYPE_INITIAL) descriptor_id = basetypes['running_phase_state']['descriptor_id'] expected_initial_basetypes['running_phase_state']['descriptor_id'] = ( descriptor_id) self.assertEqual(expected_initial_basetypes, basetypes) self.running_phase_state._finalize_measurements() self.test_record.add_phase_record( self.running_phase_state.phase_record) self.test_state.running_phase_state = None basetypes2 = self.test_state.as_base_types() expected_after_phase_record_basetypes = copy.deepcopy( PHASE_RECORD_BASE_TYPE) expected_after_phase_record_basetypes['descriptor_id'] = descriptor_id self.assertEqual(expected_after_phase_record_basetypes, basetypes2['test_record']['phases'][0]) self.assertIsNone(basetypes2['running_phase_state']) @parameterized.parameters( (phase_executor.PhaseExecutionOutcome(None), test_record.Outcome.TIMEOUT), (phase_executor.PhaseExecutionOutcome( phase_descriptor.PhaseResult.STOP), test_record.Outcome.FAIL), (phase_executor.PhaseExecutionOutcome( threads.ThreadTerminationError()), test_record.Outcome.ERROR)) def test_test_state_finalize_from_phase_outcome( self, phase_exe_outcome: phase_executor.PhaseExecutionOutcome, test_record_outcome: test_record.Outcome): self.test_state.finalize_from_phase_outcome(phase_exe_outcome) self.assertEqual(self.test_state.test_record.outcome, test_record_outcome) def test_test_state_finalize_from_phase_outcome_exception_info(self): try: raise ValueError('Exception for unit testing.') except ValueError: phase_exe_outcome = phase_executor.PhaseExecutionOutcome( phase_executor.ExceptionInfo(*sys.exc_info())) self.test_state.finalize_from_phase_outcome(phase_exe_outcome) self.assertEqual(self.test_state.test_record.outcome, test_record.Outcome.ERROR)