def _teardown_class(self): """Proxy function to guarantee the base implementation of teardown_class is called. """ stage_name = STAGE_NAME_TEARDOWN_CLASS record = records.TestResultRecord(stage_name, self.TAG) record.test_begin() self.current_test_info = runtime_test_info.RuntimeTestInfo( stage_name, self.log_path, record) expects.recorder.reset_internal_states(record) try: with self._log_test_stage(stage_name): self.teardown_class() except signals.TestAbortAll as e: setattr(e, 'results', self.results) raise except Exception as e: logging.exception('Error encountered in %s.', stage_name) record.test_error(e) record.update_record() self.results.add_class_error(record) self.summary_writer.dump(record.to_dict(), records.TestSummaryEntryType.RECORD) else: if expects.recorder.has_error: record.update_record() self.results.add_class_error(record) self.summary_writer.dump(record.to_dict(), records.TestSummaryEntryType.RECORD) finally: self._clean_up()
def _exec_one_test_with_retry(self, test_name, test_method, max_count): """Executes one test and retry the test if needed. Repeatedly execute a test case until it passes or the maximum count of iteration has been reached. Args: test_name: string, Name of the test. test_method: function, The test method to execute. max_count: int, the maximum number of iterations to execute the test for. """ def should_retry(record): return record.result in [ records.TestResultEnums.TEST_RESULT_FAIL, records.TestResultEnums.TEST_RESULT_ERROR, ] previous_record = self.exec_one_test(test_name, test_method) if not should_retry(previous_record): return for i in range(max_count - 1): retry_name = f'{test_name}_retry_{i+1}' new_record = records.TestResultRecord(retry_name, self.TAG) new_record.retry_parent = previous_record previous_record = self.exec_one_test(retry_name, test_method, new_record) if not should_retry(previous_record): break
def test_result_record_fail_none(self): record = records.TestResultRecord(self.tn) record.test_begin() record.test_fail() self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_FAIL, details=None, extras=None)
def test_result_record_pass_none(self): record = records.TestResultRecord(self.tn) record.test_begin() record.test_pass() self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_PASS, details=None, extras=None)
def test_is_test_executed(self): record1 = records.TestResultRecord(self.tn) record1.test_begin() record1.test_fail(Exception("haha")) tr = records.TestResult() tr.add_record(record1) self.assertTrue(tr.is_test_executed(record1.test_name)) self.assertFalse(tr.is_test_executed(self.tn + 'ha'))
def test_result_record_skip_none(self): record = records.TestResultRecord(self.tn) record.test_begin() record.test_skip() self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_SKIP, details=None, extras=None)
def test_result_add_operator_success(self): record1 = records.TestResultRecord(self.tn) record1.test_begin() s = signals.TestPass(self.details, self.float_extra) record1.test_pass(s) tr1 = records.TestResult() tr1.add_record(record1) tr1.add_controller_info("MockDevice", ["magicA", "magicB"]) record2 = records.TestResultRecord(self.tn) record2.test_begin() s = signals.TestPass(self.details, self.json_extra) record2.test_pass(s) tr2 = records.TestResult() tr2.add_record(record2) tr2.add_controller_info("MockDevice", ["magicC"]) tr2 += tr1 self.assertTrue(tr2.passed, [tr1, tr2]) self.assertTrue(tr2.controller_info, {"MockDevice": ["magicC"]})
def test_result_record_fail_with_json_extra(self): record = records.TestResultRecord(self.tn) record.test_begin() s = signals.TestFailure(self.details, self.json_extra) record.test_fail(s) self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_FAIL, details=self.details, extras=self.json_extra)
def test_result_record_skip_with_json_extra(self): record = records.TestResultRecord(self.tn) record.test_begin() s = signals.TestSkip(self.details, self.json_extra) record.test_skip(s) self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_SKIP, details=self.details, extras=self.json_extra)
def test_result_record_pass_with_json_extra(self): record = records.TestResultRecord(self.tn) record.test_begin() s = signals.TestPass(self.details, self.json_extra) record.test_pass(s) self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_PASS, details=self.details, extras=self.json_extra)
def test_result_record_fail_none(self): """Verifies that `test_fail` can be called without an error object.""" record = records.TestResultRecord(self.tn) record.test_begin() record.test_fail() self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_FAIL, details=None, extras=None)
def test_result_record_explicit_pass_with_float_extra(self): record = records.TestResultRecord(self.tn) record.test_begin() s = signals.TestPass(self.details, self.float_extra) record.test_pass(s) self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_PASS, details=self.details, termination_signal_type='TestPass', extras=self.float_extra)
def test_is_all_pass_with_fail_class(self): """Verifies that is_all_pass yields correct value when fail_class is used. """ record1 = records.TestResultRecord(self.tn) record1.test_begin() record1.test_fail(Exception("haha")) tr = records.TestResult() tr.fail_class(record1) self.assertFalse(tr.is_all_pass)
def test_result_record_fail_with_unicode_test_signal(self): record = records.TestResultRecord(self.tn) record.test_begin() details = u'\u2022' s = signals.TestFailure(details, self.float_extra) record.test_fail(s) self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_FAIL, details=details, extras=self.float_extra)
def test_is_all_pass_with_add_class_error(self): """Verifies that is_all_pass yields correct value when add_class_error is used. """ record1 = records.TestResultRecord(self.tn) record1.test_begin() record1.test_fail(Exception('haha')) tr = records.TestResult() tr.add_class_error(record1) self.assertFalse(tr.is_all_pass)
def test_result_record_skip_with_float_extra(self): record = records.TestResultRecord(self.tn) record.test_begin() s = signals.TestSkip(self.details, self.float_extra) record.test_skip(s) self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_SKIP, details=self.details, termination_signal_type='TestSkip', extras=self.float_extra)
def test_result_record_fail_with_float_extra(self): record = records.TestResultRecord(self.tn) record.test_begin() s = signals.TestFailure(self.details, self.float_extra) record.test_fail(s) self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_FAIL, details=self.details, termination_signal_type='TestFailure', extras=self.float_extra)
def test_result_add_operator_type_mismatch(self): record1 = records.TestResultRecord(self.tn) record1.test_begin() s = signals.TestPass(self.details, self.float_extra) record1.test_pass(s) tr1 = records.TestResult() tr1.add_record(record1) expected_msg = "Operand .* of type .* is not a TestResult." with self.assertRaisesRegexp(TypeError, expected_msg): tr1 += "haha"
def test_result_record_fail_with_unicode_exception(self): record = records.TestResultRecord(self.tn) record.test_begin() details = u'\u2022' s = Exception(details) record.test_fail(s) self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_FAIL, details=details, extras=None)
def test_summary_write_dump(self): s = signals.TestFailure(self.details, self.float_extra) record1 = records.TestResultRecord(self.tn) record1.test_begin() record1.test_fail(s) dump_path = os.path.join(self.tmp_path, 'ha.yaml') writer = records.TestSummaryWriter(dump_path) writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD) with open(dump_path, 'r') as f: content = yaml.load(f) self.assertEqual(content['Type'], records.TestSummaryEntryType.RECORD.value)
def _teardown_class(self): """Proxy function to guarantee the base implementation of teardown_class is called. """ record = records.TestResultRecord('teardown_class', self.TAG) record.test_begin() try: self.teardown_class() except Exception as e: record.test_error(e) record.update_record() self.results.add_class_error(record)
def test_result_fail_class_with_special_error(self): """Call TestResult.fail_class with an error class that requires more than one arg to instantiate. """ record1 = records.TestResultRecord(self.tn) record1.test_begin() s = signals.TestPass(self.details, self.float_extra) record1.test_pass(s) tr = records.TestResult() tr.add_record(record1) class SpecialError(Exception): def __init__(self, arg1, arg2): self.msg = "%s %s" % (arg1, arg2) se = SpecialError("haha", 42) record2 = records.TestResultRecord("SomeTest", se) tr.fail_class(record2) self.assertEqual(len(tr.passed), 1) self.assertEqual(len(tr.failed), 1) self.assertEqual(len(tr.executed), 2)
def test_result_add_operator_success(self): record1 = records.TestResultRecord(self.tn) record1.test_begin() s = signals.TestPass(self.details, self.float_extra) record1.test_pass(s) tr1 = records.TestResult() tr1.add_record(record1) controller_info = records.ControllerInfoRecord( 'SomeClass', 'MockDevice', ['magicA', 'magicB']) tr1.add_controller_info_record(controller_info) record2 = records.TestResultRecord(self.tn) record2.test_begin() s = signals.TestPass(self.details, self.json_extra) record2.test_pass(s) tr2 = records.TestResult() tr2.add_record(record2) controller_info = records.ControllerInfoRecord( 'SomeClass', 'MockDevice', ['magicC']) tr2.add_controller_info_record(controller_info) tr2 += tr1 self.assertTrue(tr2.passed, [tr1, tr2]) self.assertTrue(tr2.controller_info, {'MockDevice': ['magicC']})
def _teardown_class(self): """Proxy function to guarantee the base implementation of teardown_class is called. """ record = records.TestResultRecord('teardown_class', self.TAG) record.test_begin() try: self.teardown_class() except Exception as e: logging.exception('Error encountered in teardown_class.') record.test_error(e) record.update_record() self.results.add_class_error(record) self.summary_writer.dump(record.to_dict(), records.TestSummaryEntryType.RECORD)
def test_summary_write_dump(self): s = signals.TestFailure(self.details, self.float_extra) record1 = records.TestResultRecord(self.tn) record1.test_begin() record1.test_fail(s) dump_path = os.path.join(self.tmp_path, 'ha.yaml') writer = records.TestSummaryWriter(dump_path) writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD) with io.open(dump_path, 'r', encoding='utf-8') as f: content = yaml.safe_load(f) self.assertEqual(content['Type'], records.TestSummaryEntryType.RECORD.value) self.assertEqual(content[records.TestResultEnums.RECORD_DETAILS], self.details) self.assertEqual(content[records.TestResultEnums.RECORD_EXTRAS], self.float_extra)
def test_result_record_fail_stacktrace(self): record = records.TestResultRecord(self.tn) record.test_begin() try: raise Exception('Something failed.') except Exception as e: record.test_fail(e) # Verify stacktrace separately if we expect a non-None value. # Because stacktrace includes file names and line numbers, we can't do # a simple equality check. self.verify_record(record=record, result=records.TestResultEnums.TEST_RESULT_FAIL, details='Something failed.', extras=None, stacktrace='in test_result_record_fail_stacktrace\n ' 'raise Exception(\'Something failed.\')\nException: ' 'Something failed.\n')
def _skip_remaining_tests(self, exception): """Marks any requested test that has not been executed in a class as skipped. This is useful for handling abort class signal. Args: exception: The exception object that was thrown to trigger the skip. """ for test_name in self.results.requested: if not self.results.is_test_executed(test_name): test_record = records.TestResultRecord(test_name, self.TAG) test_record.test_skip(exception) self.results.add_record(test_record) self.summary_writer.dump(test_record.to_dict(), records.TestSummaryEntryType.RECORD)
def test_summary_write_dump_with_unicode(self): unicode_details = u'\u901a' # utf-8 -> b'\xe9\x80\x9a' unicode_extras = u'\u8fc7' # utf-8 -> b'\xe8\xbf\x87' s = signals.TestFailure(unicode_details, unicode_extras) record1 = records.TestResultRecord(self.tn) record1.test_begin() record1.test_fail(s) dump_path = os.path.join(self.tmp_path, 'ha.yaml') writer = records.TestSummaryWriter(dump_path) writer.dump(record1.to_dict(), records.TestSummaryEntryType.RECORD) with io.open(dump_path, 'r', encoding='utf-8') as f: content = yaml.load(f) self.assertEqual(content['Type'], records.TestSummaryEntryType.RECORD.value) self.assertEqual(content[records.TestResultEnums.RECORD_DETAILS], unicode_details) self.assertEqual(content[records.TestResultEnums.RECORD_EXTRAS], unicode_extras)
def _clean_up(self): """The final stage of a test class execution.""" stage_name = STAGE_NAME_CLEAN_UP record = records.TestResultRecord(stage_name, self.TAG) record.test_begin() self.current_test_info = runtime_test_info.RuntimeTestInfo( stage_name, self.log_path, record) expects.recorder.reset_internal_states(record) with self._log_test_stage(stage_name): # Write controller info and summary to summary file. self._record_controller_info() self._controller_manager.unregister_controllers() if expects.recorder.has_error: record.test_error() record.update_record() self.results.add_class_error(record) self.summary_writer.dump(record.to_dict(), records.TestSummaryEntryType.RECORD)
def create_test_record(self, mobly_test_class): """Creates a TestResultRecord for the instrumentation block. Args: mobly_test_class: string, the name of the Mobly test case executing the instrumentation run. Returns: A TestResultRecord with an appropriate signals exception representing the instrumentation test method's result status. """ details = self._get_details() extras = self._get_extras() tr_record = records.TestResultRecord( t_name=self._get_full_name(), t_class=mobly_test_class, ) if self._begin_time: tr_record.begin_time = self._begin_time if self._is_failed(): tr_record.test_fail( e=signals.TestFailure(details=details, extras=extras)) elif self._status_code in _InstrumentationStatusCodeCategories.SKIPPED: tr_record.test_skip( e=signals.TestSkip(details=details, extras=extras)) elif self._status_code in _InstrumentationStatusCodeCategories.PASS: tr_record.test_pass( e=signals.TestPass(details=details, extras=extras)) elif self._status_code in _InstrumentationStatusCodeCategories.TIMING: if self._error_message: tr_record.test_error( e=signals.TestError(details=details, extras=extras)) else: tr_record = None else: tr_record.test_error( e=signals.TestError(details=details, extras=extras)) if self._known_keys[_InstrumentationKnownStatusKeys.STACK]: tr_record.termination_signal.stacktrace = self._known_keys[ _InstrumentationKnownStatusKeys.STACK] return tr_record