def _initialize_plugs(self, plug_types=None):
"""Initialize plugs.
Args:
plug_types: optional list of plug classes to initialize.
Returns:
True if there was an error initializing the plugs.
"""
try:
self.test_state.plug_manager.initialize_plugs(
plug_types=plug_types)
return False
except Exception: # pylint: disable=broad-except
# Record the equivalent failure outcome and exit early.
self._last_outcome = phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(*sys.exc_info()))
return True
def test_last__no_failures(self):
test_rec = yield htf.Test(
phase0, phase_branches.PhaseFailureCheckpoint.last('last_pass'),
phase1)
self.assertTestPass(test_rec)
self.assertPhasesOutcomeByName(test_record.PhaseOutcome.PASS, test_rec,
'phase0', 'phase1')
self.assertEqual([
test_record.CheckpointRecord(
name='last_pass',
action=htf.PhaseResult.STOP,
conditional=phase_branches.PreviousPhases.LAST,
subtest_name=None,
result=phase_executor.PhaseExecutionOutcome(
htf.PhaseResult.CONTINUE),
evaluated_millis=htf_test.VALID_TIMESTAMP),
], test_rec.checkpoints)
def _execute_phase(self, phase: phase_descriptor.PhaseDescriptor,
subtest_rec: Optional[test_record.SubtestRecord],
in_teardown: bool) -> _ExecutorReturn:
if subtest_rec:
self.logger.debug('Executing phase %s under subtest %s',
phase.name, subtest_rec.name)
else:
self.logger.debug('Executing phase %s', phase.name)
if not in_teardown and subtest_rec and subtest_rec.is_fail:
self._phase_exec.skip_phase(phase, subtest_rec)
return _ExecutorReturn.CONTINUE
outcome, profile_stats = self._phase_exec.execute_phase(
phase,
run_with_profiling=self._run_with_profiling,
subtest_rec=subtest_rec)
if profile_stats is not None:
self._phase_profile_stats.append(profile_stats)
if (self.test_state.test_options.stop_on_first_failure
or conf.stop_on_first_failure):
# Stop Test on first measurement failure
current_phase_result = self.test_state.test_record.phases[
len(self.test_state.test_record.phases) - 1]
if current_phase_result.outcome == test_record.PhaseOutcome.FAIL:
outcome = phase_executor.PhaseExecutionOutcome(
phase_descriptor.PhaseResult.STOP)
self.logger.error(
'Stopping test because stop_on_first_failure is True')
if outcome.is_terminal:
if not self._last_outcome:
self._last_outcome = outcome
return _ExecutorReturn.TERMINAL
if outcome.is_fail_subtest:
if not subtest_rec:
raise TestExecutionError(
'INVALID STATE: Phase returned outcome FAIL_SUBTEST when not '
'in subtest.')
subtest_rec.outcome = test_record.SubtestOutcome.FAIL
return _ExecutorReturn.CONTINUE
def test_all__fail(self):
test_rec = yield htf.Test(
fail_phase,
phase_branches.PhaseFailureCheckpoint.all_previous('all_fail'),
error_phase)
self.assertTestFail(test_rec)
self.assertPhasesOutcomeByName(test_record.PhaseOutcome.FAIL, test_rec,
'fail_phase')
self.assertEqual([
test_record.CheckpointRecord(
name='all_fail',
action=htf.PhaseResult.STOP,
conditional=phase_branches.PreviousPhases.ALL,
subtest_name=None,
result=phase_executor.PhaseExecutionOutcome(
htf.PhaseResult.STOP),
evaluated_millis=htf_test.VALID_TIMESTAMP),
], test_rec.checkpoints)
def test_all__no_previous_phases(self):
self.test_start_function = None
test_rec = yield htf.Test(
phase_branches.PhaseFailureCheckpoint.all_previous('all_prev'))
self.assertTestError(test_rec)
self.assertTestOutcomeCode(test_rec, 'NoPhasesFoundError')
self.assertEqual([
test_record.CheckpointRecord(
name='all_prev',
action=htf.PhaseResult.STOP,
conditional=phase_branches.PreviousPhases.ALL,
subtest_name=None,
result=phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(
phase_branches.NoPhasesFoundError, mock.ANY,
mock.ANY)),
evaluated_millis=htf_test.VALID_TIMESTAMP),
], test_rec.checkpoints)
def _handle_phase(self, phase):
if isinstance(phase, phase_group.PhaseGroup):
return self._execute_phase_group(phase)
self.test_state.state_logger.debug('Handling phase %s', phase.name)
outcome = self._phase_exec.execute_phase(phase)
if (self.test_state.test_options.stop_on_first_failure
or conf.stop_on_first_failure):
# Stop Test on first measurement failure
current_phase_result = self.test_state.test_record.phases[
len(self.test_state.test_record.phases) - 1]
if current_phase_result.outcome == test_record.PhaseOutcome.FAIL:
outcome = phase_executor.PhaseExecutionOutcome(
phase_descriptor.PhaseResult.STOP)
self.test_state.state_logger.error(
'Stopping test because stop_on_first_failure is True')
if outcome.is_terminal and not self._last_outcome:
self._last_outcome = outcome
return outcome.is_terminal
def test_all_fail_subtest__pass_in_subtest(self):
test_rec = yield htf.Test(
phase0,
htf.Subtest(
'sub', phase1,
phase_branches.PhaseFailureCheckpoint.all_previous(
'all_pass_subtest', action=htf.PhaseResult.FAIL_SUBTEST),
phase2), phase3)
self.assertTestPass(test_rec)
self.assertPhasesOutcomeByName(test_record.PhaseOutcome.PASS, test_rec,
'phase0', 'phase1', 'phase2', 'phase3')
self.assertEqual([
test_record.CheckpointRecord(
name='all_pass_subtest',
action=htf.PhaseResult.FAIL_SUBTEST,
conditional=phase_branches.PreviousPhases.ALL,
subtest_name='sub',
result=phase_executor.PhaseExecutionOutcome(
htf.PhaseResult.CONTINUE),
evaluated_millis=htf_test.VALID_TIMESTAMP),
], test_rec.checkpoints)
def test_pass(self):
test_rec = yield htf.Test(
phase0,
phase_branches.DiagnosisCheckpoint(
'diag_pass',
phase_branches.DiagnosisCondition.on_all(
BranchDiagResult.NOT_SET)), phase1)
self.assertTestPass(test_rec)
self.assertPhasesOutcomeByName(test_record.PhaseOutcome.PASS, test_rec,
'phase0', 'phase1')
self.assertEqual([
test_record.CheckpointRecord(
name='diag_pass',
action=htf.PhaseResult.STOP,
conditional=phase_branches.DiagnosisCondition(
phase_branches.ConditionOn.ALL,
(BranchDiagResult.NOT_SET, )),
subtest_name=None,
result=phase_executor.PhaseExecutionOutcome(
htf.PhaseResult.CONTINUE),
evaluated_millis=htf_test.VALID_TIMESTAMP),
], test_rec.checkpoints)
def test_all_fail_subtest__not_in_subtest(self):
test_rec = yield htf.Test(
fail_phase,
phase_branches.PhaseFailureCheckpoint.all_previous(
'all_subtest', action=htf.PhaseResult.FAIL_SUBTEST),
error_phase)
self.assertTestError(test_rec)
self.assertTestOutcomeCode(test_rec, 'InvalidPhaseResultError')
self.assertPhasesOutcomeByName(test_record.PhaseOutcome.FAIL, test_rec,
'fail_phase')
self.assertEqual([
test_record.CheckpointRecord(
name='all_subtest',
action=htf.PhaseResult.FAIL_SUBTEST,
conditional=phase_branches.PreviousPhases.ALL,
subtest_name=None,
result=phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(
phase_executor.InvalidPhaseResultError, mock.ANY,
mock.ANY)),
evaluated_millis=htf_test.VALID_TIMESTAMP),
], test_rec.checkpoints)
def testStringFromPhaseExecutionOutcome_SuccessfullyConvertsOutcome(
self, phase_result, expected_str):
self.assertEqual(
text.StringFromPhaseExecutionOutcome(
phase_executor.PhaseExecutionOutcome(phase_result)),
expected_str)
class TestTestApi(parameterized.TestCase):
def setUp(self):
super(TestTestApi, self).setUp()
patcher = mock.patch.object(test_record.PhaseRecord,
'record_start_time',
return_value=11235)
self.mock_record_start_time = patcher.start()
self.addCleanup(patcher.stop)
self.test_descriptor = test_descriptor.TestDescriptor(
phase_collections.PhaseSequence((test_phase, )),
test_record.CodeInfo.uncaptured(), {'config': {}})
self.test_state = test_state.TestState(self.test_descriptor,
'testing-123',
test_descriptor.TestOptions())
self.test_record = self.test_state.test_record
self.running_phase_state = test_state.PhaseState.from_descriptor(
test_phase, self.test_state, logging.getLogger())
self.test_state.running_phase_state = self.running_phase_state
self.test_api = self.test_state.test_api
def test_get_attachment(self):
attachment_name = 'attachment.txt'
input_contents = b'This is some attachment text!'
mimetype = 'text/plain'
self.test_api.attach(attachment_name, input_contents, mimetype)
output_attachment = self.test_api.get_attachment(attachment_name)
if not output_attachment:
# Need branch to appease pytype.
self.fail('output_attachment not found')
self.assertEqual(input_contents, output_attachment.data)
self.assertEqual(mimetype, output_attachment.mimetype)
def test_get_attachment_strict(self):
attachment_name = 'attachment.txt'
with self.assertRaises(test_descriptor.AttachmentNotFoundError):
self.test_api.get_attachment_strict(attachment_name)
def test_get_measurement(self):
measurement_val = [1, 2, 3]
self.test_api.measurements['test_measurement'] = measurement_val
measurement = self.test_api.get_measurement('test_measurement')
if not measurement:
# Need branch to appease pytype.
self.fail('measurement not found.')
self.assertEqual(measurement_val, measurement.value)
self.assertEqual('test_measurement', measurement.name)
def test_get_measurement_immutable(self):
measurement_val = [1, 2, 3]
self.test_api.measurements['test_measurement'] = measurement_val
measurement = self.test_api.get_measurement('test_measurement')
if not measurement:
# Need branch to appease pytype.
self.fail('measurement not found.')
self.assertEqual(measurement_val, measurement.value)
self.assertEqual('test_measurement', measurement.name)
measurement.value.append(4)
self.assertNotEqual(measurement_val, measurement.value)
def test_infer_mime_type_from_file_name(self):
with tempfile.NamedTemporaryFile(suffix='.txt') as f:
f.write(b'Mock text contents.')
f.flush()
file_name = f.name
self.test_api.attach_from_file(file_name, 'attachment')
attachment = self.test_api.get_attachment('attachment')
if not attachment:
# Need branch to appease pytype.
self.fail('attachment not found.')
self.assertEqual(attachment.mimetype, 'text/plain')
def test_infer_mime_type_from_attachment_name(self):
with tempfile.NamedTemporaryFile() as f:
f.write(b'Mock text contents.')
f.flush()
file_name = f.name
self.test_api.attach_from_file(file_name, 'attachment.png')
attachment = self.test_api.get_attachment('attachment.png')
if not attachment:
# Need branch to appease pytype.
self.fail('attachment not found.')
self.assertEqual(attachment.mimetype, 'image/png')
def test_phase_state_cache(self):
basetypes = self.running_phase_state.as_base_types()
expected_initial_basetypes = copy.deepcopy(
PHASE_STATE_BASE_TYPE_INITIAL)
expected_initial_basetypes['descriptor_id'] = basetypes[
'descriptor_id']
self.assertEqual(expected_initial_basetypes, basetypes)
self.assertFalse(self.running_phase_state._update_measurements)
self.test_api.measurements.test_measurement = 5
self.assertEqual({'test_measurement'},
self.running_phase_state._update_measurements)
self.running_phase_state.as_base_types()
expected_after_basetypes = copy.deepcopy(expected_initial_basetypes)
expected_after_basetypes['measurements']['test_measurement'].update({
'outcome':
'PASS',
'measured_value':
5,
})
self.assertEqual(expected_after_basetypes, basetypes)
self.assertFalse(self.running_phase_state._update_measurements)
def test_test_state_cache(self):
basetypes = self.test_state.as_base_types()
# The descriptor id is not static, so grab it.
expected_initial_basetypes = copy.deepcopy(
TEST_STATE_BASE_TYPE_INITIAL)
descriptor_id = basetypes['running_phase_state']['descriptor_id']
expected_initial_basetypes['running_phase_state']['descriptor_id'] = (
descriptor_id)
self.assertEqual(expected_initial_basetypes, basetypes)
self.running_phase_state._finalize_measurements()
self.test_record.add_phase_record(
self.running_phase_state.phase_record)
self.test_state.running_phase_state = None
basetypes2 = self.test_state.as_base_types()
expected_after_phase_record_basetypes = copy.deepcopy(
PHASE_RECORD_BASE_TYPE)
expected_after_phase_record_basetypes['descriptor_id'] = descriptor_id
self.assertEqual(expected_after_phase_record_basetypes,
basetypes2['test_record']['phases'][0])
self.assertIsNone(basetypes2['running_phase_state'])
@parameterized.parameters(
(phase_executor.PhaseExecutionOutcome(None),
test_record.Outcome.TIMEOUT), (phase_executor.PhaseExecutionOutcome(
phase_descriptor.PhaseResult.STOP), test_record.Outcome.FAIL),
(phase_executor.PhaseExecutionOutcome(
threads.ThreadTerminationError()), test_record.Outcome.ERROR))
def test_test_state_finalize_from_phase_outcome(
self, phase_exe_outcome: phase_executor.PhaseExecutionOutcome,
test_record_outcome: test_record.Outcome):
self.test_state.finalize_from_phase_outcome(phase_exe_outcome)
self.assertEqual(self.test_state.test_record.outcome,
test_record_outcome)
def test_test_state_finalize_from_phase_outcome_exception_info(self):
try:
raise ValueError('Exception for unit testing.')
except ValueError:
phase_exe_outcome = phase_executor.PhaseExecutionOutcome(
phase_executor.ExceptionInfo(*sys.exc_info()))
self.test_state.finalize_from_phase_outcome(phase_exe_outcome)
self.assertEqual(self.test_state.test_record.outcome,
test_record.Outcome.ERROR)
