def test_test_diagnoser__exception_with_phase_excption(self):
test = htf.Test(exception_phase)
test.add_test_diagnosers(exception_test_diagnoser,
basic_wrapper_test_diagnoser)
test_rec = yield test
self.assertTestError(test_rec, exc_type=PhaseError)
self.assertEqual([test_record.OutcomeDetails('PhaseError', mock.ANY)],
test_rec.outcome_details)
self.assertEqual([htf.Diagnosis(OkayResult.TEST_OK, 'Okay')],
test_rec.diagnoses)
def test_test_diagnoser__single_not_diagnosis_list(self):
@htf.TestDiagnoser(BadResult)
def not_diag(test_record_, store):
del test_record_ # Unused.
del store # Unused.
return [BadResult.ONE]
test = htf.Test(basic_phase)
test.add_test_diagnosers(not_diag)
test_rec = yield test
self.assertTestError(test_rec)
self.assertEqual(
[test_record.OutcomeDetails('InvalidDiagnosisError', mock.ANY)],
test_rec.outcome_details)
def test_test_diagnoser__wrong_result_type_list(self):
@htf.TestDiagnoser(OkayResult)
def bad_result(test_record_, store):
del test_record_ # Unused.
del store # Unused.
return [htf.Diagnosis(BadResult.ONE, 'one')]
test = htf.Test(basic_phase)
test.add_test_diagnosers(bad_result)
test_rec = yield test
self.assertTestError(test_rec)
self.assertEqual(
[test_record.OutcomeDetails('InvalidDiagnosisError', mock.ANY)],
test_rec.outcome_details)
def test_test_diagnoser__internal_not_allowed(self):
@htf.TestDiagnoser(OkayResult)
def internal_diag(test_record_, store):
del test_record_ # Unused.
del store # Unused.
return htf.Diagnosis(OkayResult.OKAY, 'not really okay', is_internal=True)
test = htf.Test(basic_phase)
test.add_test_diagnosers(internal_diag)
test_rec = yield test
self.assertTestError(test_rec)
self.assertEqual(
[test_record.OutcomeDetails('InvalidDiagnosisError', mock.ANY)],
test_rec.outcome_details)
def test_populate_basic_data(self):
outcome_details = test_record.OutcomeDetails(
code='mock-code',
description='mock-description',
)
phase = test_record.PhaseRecord(
name='mock-phase-name',
descriptor_id=1,
codeinfo=self.create_codeinfo(),
start_time_millis=200,
end_time_millis=400,
)
log_record = test_logs.LogRecord(
level=logging.INFO,
logger_name='mock-logger-name',
source='mock-source',
lineno=123,
timestamp_millis=300,
message='mock-message',
)
record = test_record.TestRecord(
dut_id='mock-dut-id',
station_id='mock-station-id',
start_time_millis=100,
end_time_millis=500,
outcome=test_record.Outcome.PASS,
outcome_details=[outcome_details],
metadata={
'test_name': 'mock-test-name',
'operator_name': 'mock-operator-name',
'test_version': 1.0,
'test_description': 'mock-test-description',
},
phases=[phase],
log_records=[log_record],
)
mfg_event = mfg_event_pb2.MfgEvent()
mfg_event_converter._populate_basic_data(mfg_event, record)
self.assertEqual(mfg_event.dut_serial, 'mock-dut-id')
self.assertEqual(mfg_event.start_time_ms, 100)
self.assertEqual(mfg_event.end_time_ms, 500)
self.assertEqual(mfg_event.tester_name, 'mock-station-id')
self.assertEqual(mfg_event.test_name, 'mock-test-name')
self.assertEqual(mfg_event.test_version, '1.0')
self.assertEqual(mfg_event.test_description, 'mock-test-description')
self.assertEqual(mfg_event.test_status, test_runs_pb2.PASS)
# Phases.
self.assertEqual(mfg_event.phases[0].name, 'mock-phase-name')
self.assertEqual(mfg_event.phases[0].description, 'mock-sourcecode')
self.assertEqual(mfg_event.phases[0].timing.start_time_millis, 200)
self.assertEqual(mfg_event.phases[0].timing.end_time_millis, 400)
# Failure codes.
self.assertEqual(mfg_event.failure_codes[0].code, 'mock-code')
self.assertEqual(mfg_event.failure_codes[0].details, 'mock-description')
# Test logs.
self.assertEqual(mfg_event.test_logs[0].timestamp_millis, 300)
self.assertEqual(mfg_event.test_logs[0].log_message, 'mock-message')
self.assertEqual(mfg_event.test_logs[0].logger_name, 'mock-logger-name')
self.assertEqual(mfg_event.test_logs[0].levelno, logging.INFO)
self.assertEqual(mfg_event.test_logs[0].level,
test_runs_pb2.TestRunLogMessage.INFO)
self.assertEqual(mfg_event.test_logs[0].log_source, 'mock-source')
self.assertEqual(mfg_event.test_logs[0].lineno, 123)
class TextTest(test.TestCase, parameterized.TestCase):
def testColorFromTestOutcome_HasCorrespondingTestOutcomeName(self):
"""Catches OpenHTF test outcome not added in _ColorFromTestOutcome."""
for member in test_record.Outcome.__members__:
self.assertIn(member, text._ColorFromTestOutcome.__members__)
def testHeadlineFromTestOutcome_HasCorrespondingTestOutcomeName(self):
"""Catches OpenHTF test outcome not added in _HeadlineFromTestOutcome."""
for member in test_record.Outcome.__members__:
self.assertIn(member, text._HeadlineFromTestOutcome.__members__)
def testColorText_GetsColorSuccessfully(self):
text_to_colorize = 'Foo Bar'
self.assertEqual(
text._ColorText(text_to_colorize, _GREEN),
f'{_GREEN}{text_to_colorize}{colorama.Style.RESET_ALL}')
# TODO(b/70517332): Pytype currently doesn't properly support the functional
# API of enums: https://github.com/google/pytype/issues/459. Remove
# disabling pytype once fixed.
@parameterized.named_parameters(
(headline_member.name, headline_member.name, headline_member.value)
for headline_member in text._HeadlineFromTestOutcome.__iter__()) # pytype: disable=attribute-error
def testGetTestOutcomeHeadline_TestNotColorized(self, outcome, headline):
record = test_record.TestRecord(dut_id='TestDutId',
station_id='test_station',
outcome=test_record.Outcome[outcome])
self.assertEqual(text._GetTestOutcomeHeadline(record), headline)
# TODO(b/70517332): Pytype currently doesn't properly support the functional
# API of enums: https://github.com/google/pytype/issues/459. Remove
# disabling pytype once fixed.
@parameterized.named_parameters(
(headline_member.name, headline_member.name, headline_member.value)
for headline_member in text._HeadlineFromTestOutcome.__iter__()) # pytype: disable=attribute-error
def testGetTestOutcomeHeadline_TestColorized(self, outcome, headline):
record = test_record.TestRecord(dut_id='TestDutId',
station_id='test_station',
outcome=test_record.Outcome[outcome])
# TODO(b/70517332): Pytype currently doesn't properly support the functional
# API of enums: https://github.com/google/pytype/issues/459. Remove
# disabling pytype once fixed.
self.assertEqual(
text._GetTestOutcomeHeadline(record, colorize_text=True),
f'{text._ColorFromTestOutcome[outcome].value}{headline}' # pytype: disable=unsupported-operands
f'{colorama.Style.RESET_ALL}')
def testStringFromMeasurement_SuccessfullyConvertsUnsetMeasurement(self):
self.assertEqual(
text.StringFromMeasurement(
openhtf.Measurement('text_measurement_a')),
'| text_measurement_a was not set')
def testStringFromMeasurement_SuccessfullyConvertsPassMeasurement(self):
measurement = openhtf.Measurement('text_measurement_a')
measurement._measured_value = measurements.MeasuredValue(
'text_measurement_a')
measurement._measured_value.set(10)
measurement.notify_value_set()
self.assertEqual(text.StringFromMeasurement(measurement),
'| text_measurement_a: 10')
def testStringFromMeasurement_SuccessfullyConvertsFailMeasurement(self):
measurement = openhtf.Measurement('text_measurement_a').in_range(
maximum=3)
measurement._measured_value = measurements.MeasuredValue(
'text_measurement_a')
measurement._measured_value.set(5)
measurement.notify_value_set()
output = text.StringFromMeasurement(measurement)
self.assertEqual(
output,
"| text_measurement_a failed because 5 failed these checks: ['x <= 3']"
)
self.assertNotIn(text._BRIGHT_RED_STYLE, output)
def testStringFromMeasurement_SuccessfullyConvertsFailMeasurementColorized(
self):
measurement = openhtf.Measurement('text_measurement_a').in_range(
maximum=3)
measurement._measured_value = measurements.MeasuredValue(
'text_measurement_a')
measurement._measured_value.set(5)
measurement.notify_value_set()
self.assertEqual(
text.StringFromMeasurement(measurement, colorize_text=True).count(
text._BRIGHT_RED_STYLE), 1)
def testStringFromAttachment_SuccessfullyConvertsPassMeasurement(self):
attachment = test_record.Attachment('content', 'text/plain')
self.assertEqual(
text.StringFromAttachment(attachment, 'attachment_a.txt'),
'| attachment: attachment_a.txt (mimetype=text/plain)')
@parameterized.named_parameters([
{
'testcase_name': 'None',
'phase_result': None,
'expected_str': ''
},
{
'testcase_name': 'PhaseResult',
'phase_result': phase_descriptor.PhaseResult.CONTINUE,
'expected_str': 'CONTINUE'
},
{
'testcase_name':
'ExceptionInfo',
'phase_result':
phase_executor.ExceptionInfo(
ValueError, ValueError('Invalid Value'),
mock.create_autospec(types.TracebackType, spec_set=True)),
'expected_str':
'ValueError'
},
{
'testcase_name': 'ThreadTerminationError',
'phase_result': threads.ThreadTerminationError(),
'expected_str': 'ThreadTerminationError'
},
])
def testStringFromPhaseExecutionOutcome_SuccessfullyConvertsOutcome(
self, phase_result, expected_str):
self.assertEqual(
text.StringFromPhaseExecutionOutcome(
phase_executor.PhaseExecutionOutcome(phase_result)),
expected_str)
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordPassPhase(
self):
record = self.execute_phase_or_test(PhaseThatSucceeds)
output = text.StringFromPhaseRecord(record)
self.assertEqual(
output, 'Phase PhaseThatSucceeds\n'
'+ Outcome: PASS Result: CONTINUE\n'
'| text_measurement_a: a\n'
'| text_measurement_b: b\n'
'| attachment: attachment_a.txt (mimetype=text/plain)\n'
'| attachment: attachment_b.json (mimetype=application/json)')
self.assertNotIn(text._BRIGHT_RED_STYLE, output)
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordFailPhase(
self):
record = self.execute_phase_or_test(PhaseWithFailure)
output = text.StringFromPhaseRecord(record)
self.assertEqual(
output, 'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]\n'
'| text_measurement_b was not set\n'
'| text_measurement_c: c')
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseFailLimitPhase(
self):
record = self.execute_phase_or_test(PhaseWithFailure)
output = text.StringFromPhaseRecord(record, maximum_num_measurements=2)
self.assertEqual(
output, 'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]\n'
'| text_measurement_b was not set\n'
'...')
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordOnlyFailPhase(
self):
record = self.execute_phase_or_test(PhaseWithFailure)
output = text.StringFromPhaseRecord(record, only_failures=True)
self.assertEqual(
output, 'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]')
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordFailPhaseColored(
self):
record = self.execute_phase_or_test(PhaseWithFailure)
self.assertEqual(
text.StringFromPhaseRecord(record, colorize_text=True).count(_RED),
3)
def testStringFromPhaseRecord_SuccessfullyConvertsPhaseRecordSkipPhaseColored(
self):
record = self.execute_phase_or_test(PhaseWithSkip)
self.assertNotIn(
text._BRIGHT_RED_STYLE,
text.StringFromPhaseRecord(record, colorize_text=True))
@parameterized.named_parameters([
{
'testcase_name':
'OneOutcome',
'outcome_details': [
test_record.OutcomeDetails(code=1,
description='Unknown exception.')
],
'expected_str': ('The test thinks this may be the reason:\n'
'1: Unknown exception.'),
},
{
'testcase_name':
'TwoOutcomes',
'outcome_details': [
test_record.OutcomeDetails(code=1,
description='Unknown exception.'),
test_record.OutcomeDetails(code='FooError',
description='Foo exception.')
],
'expected_str': ('The test thinks these may be the reason:\n'
'1: Unknown exception.\n'
'FooError: Foo exception.'),
},
])
def testStringFromOutcomeDetails_SuccessfullyConvertsOutcomeDetails(
self, outcome_details, expected_str):
self.assertEqual(text.StringFromOutcomeDetails(outcome_details),
expected_str)
def testStringFromTestRecord_SuccessfullyConvertsTestRecordSinglePassPhase(
self):
record = self.execute_phase_or_test(openhtf.Test(PhaseThatSucceeds))
self.assertEqual(
text.StringFromTestRecord(record), 'Test finished with a PASS!\n'
'Woohoo!\n'
'Phase trigger_phase\n'
'+ Outcome: PASS Result: CONTINUE\n'
'Phase PhaseThatSucceeds\n'
'+ Outcome: PASS Result: CONTINUE\n'
'| text_measurement_a: a\n'
'| text_measurement_b: b\n'
'| attachment: attachment_a.txt (mimetype=text/plain)\n'
'| attachment: attachment_b.json (mimetype=application/json)\n'
'Test finished with a PASS!')
def testStringFromTestRecord_SuccessfullyConvertsTestErrorPhase(self):
record = self.execute_phase_or_test(openhtf.Test(PhaseWithError))
self.assertEqual(
text.StringFromTestRecord(record), 'Test encountered an ERROR!!!\n'
'Phase trigger_phase\n'
'+ Outcome: PASS Result: CONTINUE\n'
'Phase PhaseWithError\n'
'+ Outcome: ERROR Result: Exception\n'
'| text_measurement_a was not set\n'
'| text_measurement_b was not set\n'
'The test thinks this may be the reason:\n'
'Exception: Intentional exception from test case.\n'
'Test encountered an ERROR!!!')
def testStringFromTestRecord_SuccessfullyConvertsTestFailurePhase(self):
record = self.execute_phase_or_test(openhtf.Test(PhaseWithFailure))
output = text.StringFromTestRecord(record)
self.assertEqual(
output, 'Test finished with a FAIL :(\n'
'Phase trigger_phase\n'
'+ Outcome: PASS Result: CONTINUE\n'
'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]\n'
'| text_measurement_b was not set\n'
'| text_measurement_c: c\n'
'Test finished with a FAIL :(')
self.assertNotIn(text._BRIGHT_RED_STYLE, output)
def testStringFromTestRecord_SuccessfullyConvertsTestOnlyFailurePhase(
self):
record = self.execute_phase_or_test(
openhtf.Test(PhaseThatSucceeds, PhaseWithFailure))
output = text.StringFromTestRecord(record, only_failures=True)
self.assertEqual(
output, 'Test finished with a FAIL :(\n'
'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]\n'
'Test finished with a FAIL :(')
self.assertNotIn(text._BRIGHT_RED_STYLE, output)
def testStringFromTestRecord_SuccessfullyConvertsTestFailurePhaseColored(
self):
record = self.execute_phase_or_test(openhtf.Test(PhaseWithFailure))
self.assertEqual(
text.StringFromTestRecord(record, colorize_text=True).count(_RED),
5)
def testStringFromTestRecord_SuccessfullyConvertsTestFailureMultiplePhases(
self):
record = self.execute_phase_or_test(
openhtf.Test(PhaseThatSucceeds, PhaseWithFailure))
self.assertEqual(
text.StringFromTestRecord(record), 'Test finished with a FAIL :(\n'
'Phase trigger_phase\n'
'+ Outcome: PASS Result: CONTINUE\n'
'Phase PhaseThatSucceeds\n'
'+ Outcome: PASS Result: CONTINUE\n'
'| text_measurement_a: a\n'
'| text_measurement_b: b\n'
'| attachment: attachment_a.txt (mimetype=text/plain)\n'
'| attachment: attachment_b.json (mimetype=application/json)\n'
'Phase PhaseWithFailure\n'
'+ Outcome: FAIL Result: CONTINUE\n'
'| text_measurement_a failed because intentionally wrong measurement '
'failed these checks: ["\'x\' matches /^a$/"]\n'
'| text_measurement_b was not set\n'
'| text_measurement_c: c\n'
'Test finished with a FAIL :(')
def testPrintTestRecord_SuccessfullyLogsNotColored(self):
record = self.execute_phase_or_test(openhtf.Test(PhaseThatSucceeds))
with mock.patch.object(sys, 'stdout', new_callable=io.StringIO) as cm:
with mock.patch.object(sys.stdout,
sys.stdout.isatty.__name__,
autospec=True,
spec_set=True,
return_value=False):
text.PrintTestRecord(record)
self.assertTrue(cm.getvalue())
self.assertNotIn(_GREEN, cm.getvalue())
def testPrintTestRecord_SuccessfullyLogsColored(self):
record = self.execute_phase_or_test(openhtf.Test(PhaseThatSucceeds))
with mock.patch.object(sys, 'stdout', new_callable=io.StringIO) as cm:
with mock.patch.object(sys.stdout,
sys.stdout.isatty.__name__,
autospec=True,
spec_set=True,
return_value=True):
text.PrintTestRecord(record)
self.assertIn(_GREEN, cm.getvalue())
