def _teardown_class(self):
"""Proxy function to guarantee the base implementation of
teardown_class is called.
"""
stage_name = STAGE_NAME_TEARDOWN_CLASS
record = records.TestResultRecord(stage_name, self.TAG)
record.test_begin()
self.current_test_info = runtime_test_info.RuntimeTestInfo(
stage_name, self.log_path, record)
expects.recorder.reset_internal_states(record)
try:
with self._log_test_stage(stage_name):
self.teardown_class()
except signals.TestAbortAll as e:
setattr(e, 'results', self.results)
raise
except Exception as e:
logging.exception('Error encountered in %s.', stage_name)
record.test_error(e)
record.update_record()
self.results.add_class_error(record)
self.summary_writer.dump(record.to_dict(),
records.TestSummaryEntryType.RECORD)
else:
if expects.recorder.has_error:
record.update_record()
self.results.add_class_error(record)
self.summary_writer.dump(record.to_dict(),
records.TestSummaryEntryType.RECORD)
finally:
self._clean_up()
def _exec_one_test_with_retry(self, test_name, test_method, max_count):
"""Executes one test and retry the test if needed.
Repeatedly execute a test case until it passes or the maximum count of
iteration has been reached.
Args:
test_name: string, Name of the test.
test_method: function, The test method to execute.
max_count: int, the maximum number of iterations to execute the test for.
"""
def should_retry(record):
return record.result in [
records.TestResultEnums.TEST_RESULT_FAIL,
records.TestResultEnums.TEST_RESULT_ERROR,
]
previous_record = self.exec_one_test(test_name, test_method)
if not should_retry(previous_record):
return
for i in range(max_count - 1):
retry_name = f'{test_name}_retry_{i+1}'
new_record = records.TestResultRecord(retry_name, self.TAG)
new_record.retry_parent = previous_record
previous_record = self.exec_one_test(retry_name, test_method,
new_record)
if not should_retry(previous_record):
break
def test_result_record_fail_none(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
record.test_fail()
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_FAIL,
details=None,
extras=None)
def test_result_record_pass_none(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
record.test_pass()
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_PASS,
details=None,
extras=None)
def test_is_test_executed(self):
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(Exception("haha"))
tr = records.TestResult()
tr.add_record(record1)
self.assertTrue(tr.is_test_executed(record1.test_name))
self.assertFalse(tr.is_test_executed(self.tn + 'ha'))
def test_result_record_skip_none(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
record.test_skip()
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_SKIP,
details=None,
extras=None)
def test_result_add_operator_success(self):
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record1.test_pass(s)
tr1 = records.TestResult()
tr1.add_record(record1)
tr1.add_controller_info("MockDevice", ["magicA", "magicB"])
record2 = records.TestResultRecord(self.tn)
record2.test_begin()
s = signals.TestPass(self.details, self.json_extra)
record2.test_pass(s)
tr2 = records.TestResult()
tr2.add_record(record2)
tr2.add_controller_info("MockDevice", ["magicC"])
tr2 += tr1
self.assertTrue(tr2.passed, [tr1, tr2])
self.assertTrue(tr2.controller_info, {"MockDevice": ["magicC"]})
def test_result_record_fail_with_json_extra(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
s = signals.TestFailure(self.details, self.json_extra)
record.test_fail(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_FAIL,
details=self.details,
extras=self.json_extra)
def test_result_record_skip_with_json_extra(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
s = signals.TestSkip(self.details, self.json_extra)
record.test_skip(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_SKIP,
details=self.details,
extras=self.json_extra)
def test_result_record_pass_with_json_extra(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
s = signals.TestPass(self.details, self.json_extra)
record.test_pass(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_PASS,
details=self.details,
extras=self.json_extra)
def test_result_record_fail_none(self):
"""Verifies that `test_fail` can be called without an error object."""
record = records.TestResultRecord(self.tn)
record.test_begin()
record.test_fail()
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_FAIL,
details=None,
extras=None)
def test_result_record_explicit_pass_with_float_extra(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record.test_pass(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_PASS,
details=self.details,
termination_signal_type='TestPass',
extras=self.float_extra)
def test_is_all_pass_with_fail_class(self):
"""Verifies that is_all_pass yields correct value when fail_class is
used.
"""
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(Exception("haha"))
tr = records.TestResult()
tr.fail_class(record1)
self.assertFalse(tr.is_all_pass)
def test_result_record_fail_with_unicode_test_signal(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
details = u'\u2022'
s = signals.TestFailure(details, self.float_extra)
record.test_fail(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_FAIL,
details=details,
extras=self.float_extra)
def test_is_all_pass_with_add_class_error(self):
"""Verifies that is_all_pass yields correct value when add_class_error is
used.
"""
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(Exception('haha'))
tr = records.TestResult()
tr.add_class_error(record1)
self.assertFalse(tr.is_all_pass)
def test_result_record_skip_with_float_extra(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
s = signals.TestSkip(self.details, self.float_extra)
record.test_skip(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_SKIP,
details=self.details,
termination_signal_type='TestSkip',
extras=self.float_extra)
def test_result_record_fail_with_float_extra(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
s = signals.TestFailure(self.details, self.float_extra)
record.test_fail(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_FAIL,
details=self.details,
termination_signal_type='TestFailure',
extras=self.float_extra)
def test_result_add_operator_type_mismatch(self):
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record1.test_pass(s)
tr1 = records.TestResult()
tr1.add_record(record1)
expected_msg = "Operand .* of type .* is not a TestResult."
with self.assertRaisesRegexp(TypeError, expected_msg):
tr1 += "haha"
def test_result_record_fail_with_unicode_exception(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
details = u'\u2022'
s = Exception(details)
record.test_fail(s)
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_FAIL,
details=details,
extras=None)
def test_summary_write_dump(self):
s = signals.TestFailure(self.details, self.float_extra)
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(s)
dump_path = os.path.join(self.tmp_path, 'ha.yaml')
writer = records.TestSummaryWriter(dump_path)
writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD)
with open(dump_path, 'r') as f:
content = yaml.load(f)
self.assertEqual(content['Type'],
records.TestSummaryEntryType.RECORD.value)
def _teardown_class(self):
"""Proxy function to guarantee the base implementation of
teardown_class is called.
"""
record = records.TestResultRecord('teardown_class', self.TAG)
record.test_begin()
try:
self.teardown_class()
except Exception as e:
record.test_error(e)
record.update_record()
self.results.add_class_error(record)
def test_result_fail_class_with_special_error(self):
"""Call TestResult.fail_class with an error class that requires more
than one arg to instantiate.
"""
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record1.test_pass(s)
tr = records.TestResult()
tr.add_record(record1)
class SpecialError(Exception):
def __init__(self, arg1, arg2):
self.msg = "%s %s" % (arg1, arg2)
se = SpecialError("haha", 42)
record2 = records.TestResultRecord("SomeTest", se)
tr.fail_class(record2)
self.assertEqual(len(tr.passed), 1)
self.assertEqual(len(tr.failed), 1)
self.assertEqual(len(tr.executed), 2)
def test_result_add_operator_success(self):
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
s = signals.TestPass(self.details, self.float_extra)
record1.test_pass(s)
tr1 = records.TestResult()
tr1.add_record(record1)
controller_info = records.ControllerInfoRecord(
'SomeClass', 'MockDevice', ['magicA', 'magicB'])
tr1.add_controller_info_record(controller_info)
record2 = records.TestResultRecord(self.tn)
record2.test_begin()
s = signals.TestPass(self.details, self.json_extra)
record2.test_pass(s)
tr2 = records.TestResult()
tr2.add_record(record2)
controller_info = records.ControllerInfoRecord(
'SomeClass', 'MockDevice', ['magicC'])
tr2.add_controller_info_record(controller_info)
tr2 += tr1
self.assertTrue(tr2.passed, [tr1, tr2])
self.assertTrue(tr2.controller_info, {'MockDevice': ['magicC']})
def _teardown_class(self):
"""Proxy function to guarantee the base implementation of
teardown_class is called.
"""
record = records.TestResultRecord('teardown_class', self.TAG)
record.test_begin()
try:
self.teardown_class()
except Exception as e:
logging.exception('Error encountered in teardown_class.')
record.test_error(e)
record.update_record()
self.results.add_class_error(record)
self.summary_writer.dump(record.to_dict(),
records.TestSummaryEntryType.RECORD)
def test_summary_write_dump(self):
s = signals.TestFailure(self.details, self.float_extra)
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(s)
dump_path = os.path.join(self.tmp_path, 'ha.yaml')
writer = records.TestSummaryWriter(dump_path)
writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD)
with io.open(dump_path, 'r', encoding='utf-8') as f:
content = yaml.safe_load(f)
self.assertEqual(content['Type'],
records.TestSummaryEntryType.RECORD.value)
self.assertEqual(content[records.TestResultEnums.RECORD_DETAILS],
self.details)
self.assertEqual(content[records.TestResultEnums.RECORD_EXTRAS],
self.float_extra)
def test_result_record_fail_stacktrace(self):
record = records.TestResultRecord(self.tn)
record.test_begin()
try:
raise Exception('Something failed.')
except Exception as e:
record.test_fail(e)
# Verify stacktrace separately if we expect a non-None value.
# Because stacktrace includes file names and line numbers, we can't do
# a simple equality check.
self.verify_record(record=record,
result=records.TestResultEnums.TEST_RESULT_FAIL,
details='Something failed.',
extras=None,
stacktrace='in test_result_record_fail_stacktrace\n '
'raise Exception(\'Something failed.\')\nException: '
'Something failed.\n')
def _skip_remaining_tests(self, exception):
"""Marks any requested test that has not been executed in a class as
skipped.
This is useful for handling abort class signal.
Args:
exception: The exception object that was thrown to trigger the
skip.
"""
for test_name in self.results.requested:
if not self.results.is_test_executed(test_name):
test_record = records.TestResultRecord(test_name, self.TAG)
test_record.test_skip(exception)
self.results.add_record(test_record)
self.summary_writer.dump(test_record.to_dict(),
records.TestSummaryEntryType.RECORD)
def test_summary_write_dump_with_unicode(self):
unicode_details = u'\u901a' # utf-8 -> b'\xe9\x80\x9a'
unicode_extras = u'\u8fc7' # utf-8 -> b'\xe8\xbf\x87'
s = signals.TestFailure(unicode_details, unicode_extras)
record1 = records.TestResultRecord(self.tn)
record1.test_begin()
record1.test_fail(s)
dump_path = os.path.join(self.tmp_path, 'ha.yaml')
writer = records.TestSummaryWriter(dump_path)
writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD)
with io.open(dump_path, 'r', encoding='utf-8') as f:
content = yaml.load(f)
self.assertEqual(content['Type'],
records.TestSummaryEntryType.RECORD.value)
self.assertEqual(content[records.TestResultEnums.RECORD_DETAILS],
unicode_details)
self.assertEqual(content[records.TestResultEnums.RECORD_EXTRAS],
unicode_extras)
def _clean_up(self):
"""The final stage of a test class execution."""
stage_name = STAGE_NAME_CLEAN_UP
record = records.TestResultRecord(stage_name, self.TAG)
record.test_begin()
self.current_test_info = runtime_test_info.RuntimeTestInfo(
stage_name, self.log_path, record)
expects.recorder.reset_internal_states(record)
with self._log_test_stage(stage_name):
# Write controller info and summary to summary file.
self._record_controller_info()
self._controller_manager.unregister_controllers()
if expects.recorder.has_error:
record.test_error()
record.update_record()
self.results.add_class_error(record)
self.summary_writer.dump(record.to_dict(),
records.TestSummaryEntryType.RECORD)
def create_test_record(self, mobly_test_class):
"""Creates a TestResultRecord for the instrumentation block.
Args:
mobly_test_class: string, the name of the Mobly test case
executing the instrumentation run.
Returns:
A TestResultRecord with an appropriate signals exception
representing the instrumentation test method's result status.
"""
details = self._get_details()
extras = self._get_extras()
tr_record = records.TestResultRecord(
t_name=self._get_full_name(),
t_class=mobly_test_class,
)
if self._begin_time:
tr_record.begin_time = self._begin_time
if self._is_failed():
tr_record.test_fail(
e=signals.TestFailure(details=details, extras=extras))
elif self._status_code in _InstrumentationStatusCodeCategories.SKIPPED:
tr_record.test_skip(
e=signals.TestSkip(details=details, extras=extras))
elif self._status_code in _InstrumentationStatusCodeCategories.PASS:
tr_record.test_pass(
e=signals.TestPass(details=details, extras=extras))
elif self._status_code in _InstrumentationStatusCodeCategories.TIMING:
if self._error_message:
tr_record.test_error(
e=signals.TestError(details=details, extras=extras))
else:
tr_record = None
else:
tr_record.test_error(
e=signals.TestError(details=details, extras=extras))
if self._known_keys[_InstrumentationKnownStatusKeys.STACK]:
tr_record.termination_signal.stacktrace = self._known_keys[
_InstrumentationKnownStatusKeys.STACK]
return tr_record
