def create_test_record(self, mobly_test_class): """Creates a TestResultRecord for the instrumentation block. Args: mobly_test_class: string, the name of the Mobly test case executing the instrumentation run. Returns: A TestResultRecord with an appropriate signals exception representing the instrumentation test method's result status. """ details = self._get_details() extras = self._get_extras() tr_record = records.TestResultRecord( t_name=self._get_full_name(), t_class=mobly_test_class, ) if self._begin_time: tr_record.begin_time = self._begin_time if self._is_failed(): tr_record.test_fail( e=signals.TestFailure(details=details, extras=extras)) elif self._status_code in _InstrumentationStatusCodeCategories.SKIPPED: tr_record.test_skip( e=signals.TestSkip(details=details, extras=extras)) elif self._status_code in _InstrumentationStatusCodeCategories.PASS: tr_record.test_pass( e=signals.TestPass(details=details, extras=extras)) elif self._status_code in _InstrumentationStatusCodeCategories.TIMING: if self._error_message: tr_record.test_error( e=signals.TestError(details=details, extras=extras)) else: tr_record = None else: tr_record.test_error( e=signals.TestError(details=details, extras=extras)) if self._known_keys[_InstrumentationKnownStatusKeys.STACK]: tr_record.termination_signal.stacktrace = self._known_keys[ _InstrumentationKnownStatusKeys.STACK] return tr_record
def has_completed_result_block_format(self, error_message): """Checks the instrumentation result block for a signal indicating normal completion. Args: error_message: string, the error message to give if the instrumentation run did not complete successfully.- Returns: A boolean indicating whether or not the instrumentation run passed or failed overall. Raises: signals.TestError: Error raised if the instrumentation run did not complete because of a crash or some other issue. """ extras = self._get_extras() if _InstrumentationResultSignals.PASS in extras: return True elif _InstrumentationResultSignals.FAIL in extras: return False else: raise signals.TestError(details=error_message, extras=extras)
def exec_one_test(self, test_name, test_method, args=(), **kwargs): """Executes one test and update test results. Executes setup_test, the test method, and teardown_test; then creates a records.TestResultRecord object with the execution information and adds the record to the test class's test results. Args: test_name: Name of the test. test_method: The test method. args: A tuple of params. kwargs: Extra kwargs. """ tr_record = records.TestResultRecord(test_name, self.TAG) tr_record.test_begin() logging.info('%s %s', TEST_CASE_TOKEN, test_name) teardown_test_failed = False try: try: try: self._setup_test(test_name) except signals.TestFailure as e: new_e = signals.TestError(e.details, e.extras) _, _, new_e.__traceback__ = sys.exc_info() raise new_e if args or kwargs: test_method(*args, **kwargs) else: test_method() except signals.TestPass: raise except Exception: logging.exception('Exception occurred in %s.', self.current_test_name) raise finally: try: self._teardown_test(test_name) except signals.TestAbortAll: raise except Exception as e: logging.exception(e) tr_record.add_error('teardown_test', e) teardown_test_failed = True except (signals.TestFailure, AssertionError) as e: tr_record.test_fail(e) except signals.TestSkip as e: # Test skipped. tr_record.test_skip(e) except (signals.TestAbortClass, signals.TestAbortAll) as e: # Abort signals, pass along. tr_record.test_fail(e) raise e except signals.TestPass as e: # Explicit test pass. tr_record.test_pass(e) except Exception as e: # Exception happened during test. tr_record.test_error(e) else: if not teardown_test_failed: tr_record.test_pass() finally: tr_record.update_record() if tr_record.result in (records.TestResultEnums.TEST_RESULT_ERROR, records.TestResultEnums.TEST_RESULT_FAIL): self._exec_procedure_func(self._on_fail, tr_record) elif tr_record.result == records.TestResultEnums.TEST_RESULT_PASS: self._exec_procedure_func(self._on_pass, tr_record) elif tr_record.result == records.TestResultEnums.TEST_RESULT_SKIP: self._exec_procedure_func(self._on_skip, tr_record) self.results.add_record(tr_record) self.summary_writer.dump(tr_record.to_dict(), records.TestSummaryEntryType.RECORD) self.current_test_name = None
def exec_one_test(self, test_name, test_method): """Executes one test and update test results. Executes setup_test, the test method, and teardown_test; then creates a records.TestResultRecord object with the execution information and adds the record to the test class's test results. Args: test_name: string, Name of the test. test_method: function, The test method to execute. """ tr_record = records.TestResultRecord(test_name, self.TAG) tr_record.uid = getattr(test_method, 'uid', None) tr_record.test_begin() self.current_test_info = runtime_test_info.RuntimeTestInfo( test_name, self.log_path, tr_record) expects.recorder.reset_internal_states(tr_record) logging.info('%s %s', TEST_CASE_TOKEN, test_name) # Did teardown_test throw an error. teardown_test_failed = False try: try: try: self._setup_test(test_name) except signals.TestFailure as e: raise_with_traceback(signals.TestError( e.details, e.extras)) test_method() except (signals.TestPass, signals.TestAbortSignal): raise except Exception: logging.exception('Exception occurred in %s.', self.current_test_name) raise finally: before_count = expects.recorder.error_count try: self._teardown_test(test_name) except signals.TestAbortSignal: raise except Exception as e: logging.exception(e) tr_record.test_error() tr_record.add_error(STAGE_NAME_TEARDOWN_TEST, e) teardown_test_failed = True else: # Check if anything failed by `expects`. if before_count < expects.recorder.error_count: teardown_test_failed = True except (signals.TestFailure, AssertionError) as e: tr_record.test_fail(e) except signals.TestSkip as e: # Test skipped. tr_record.test_skip(e) except signals.TestAbortSignal as e: # Abort signals, pass along. tr_record.test_fail(e) raise except signals.TestPass as e: # Explicit test pass. tr_record.test_pass(e) except Exception as e: # Exception happened during test. tr_record.test_error(e) else: # No exception is thrown from test and teardown, if `expects` has # error, the test should fail with the first error in `expects`. if expects.recorder.has_error and not teardown_test_failed: tr_record.test_fail() # Otherwise the test passed. elif not teardown_test_failed: tr_record.test_pass() finally: tr_record.update_record() try: if tr_record.result in ( records.TestResultEnums.TEST_RESULT_ERROR, records.TestResultEnums.TEST_RESULT_FAIL): self._exec_procedure_func(self._on_fail, tr_record) elif tr_record.result == records.TestResultEnums.TEST_RESULT_PASS: self._exec_procedure_func(self._on_pass, tr_record) elif tr_record.result == records.TestResultEnums.TEST_RESULT_SKIP: self._exec_procedure_func(self._on_skip, tr_record) finally: logging.info(RESULT_LINE_TEMPLATE, tr_record.test_name, tr_record.result) self.results.add_record(tr_record) self.summary_writer.dump(tr_record.to_dict(), records.TestSummaryEntryType.RECORD) self.current_test_info = None self.current_test_name = None
def exec_one_test(self, test_name, test_method, record=None): """Executes one test and update test results. Executes setup_test, the test method, and teardown_test; then creates a records.TestResultRecord object with the execution information and adds the record to the test class's test results. Args: test_name: string, Name of the test. test_method: function, The test method to execute. record: records.TestResultRecord, optional arg for injecting a record object to use for this test execution. If not set, a new one is created created. This is meant for passing information between consecutive test case execution for retry purposes. Do NOT abuse this for "magical" features. Returns: TestResultRecord, the test result record object of the test execution. This object is strictly for read-only purposes. Modifying this record will not change what is reported in the test run's summary yaml file. """ tr_record = record or records.TestResultRecord(test_name, self.TAG) tr_record.uid = getattr(test_method, 'uid', None) tr_record.test_begin() self.current_test_info = runtime_test_info.RuntimeTestInfo( test_name, self.log_path, tr_record) expects.recorder.reset_internal_states(tr_record) logging.info('%s %s', TEST_CASE_TOKEN, test_name) # Did teardown_test throw an error. teardown_test_failed = False try: try: try: self._setup_test(test_name) except signals.TestFailure as e: _, _, traceback = sys.exc_info() raise signals.TestError(e.details, e.extras).with_traceback(traceback) test_method() except (signals.TestPass, signals.TestAbortSignal): raise except Exception: logging.exception('Exception occurred in %s.', self.current_test_info.name) raise finally: before_count = expects.recorder.error_count try: self._teardown_test(test_name) except signals.TestAbortSignal: raise except Exception as e: logging.exception('Exception occurred in %s of %s.', STAGE_NAME_TEARDOWN_TEST, self.current_test_info.name) tr_record.test_error() tr_record.add_error(STAGE_NAME_TEARDOWN_TEST, e) teardown_test_failed = True else: # Check if anything failed by `expects`. if before_count < expects.recorder.error_count: teardown_test_failed = True except (signals.TestFailure, AssertionError) as e: tr_record.test_fail(e) except signals.TestSkip as e: # Test skipped. tr_record.test_skip(e) except signals.TestAbortSignal as e: # Abort signals, pass along. tr_record.test_fail(e) raise except signals.TestPass as e: # Explicit test pass. tr_record.test_pass(e) except Exception as e: # Exception happened during test. tr_record.test_error(e) else: # No exception is thrown from test and teardown, if `expects` has # error, the test should fail with the first error in `expects`. if expects.recorder.has_error and not teardown_test_failed: tr_record.test_fail() # Otherwise the test passed. elif not teardown_test_failed: tr_record.test_pass() finally: tr_record.update_record() try: if tr_record.result in ( records.TestResultEnums.TEST_RESULT_ERROR, records.TestResultEnums.TEST_RESULT_FAIL): self._exec_procedure_func(self._on_fail, tr_record) elif tr_record.result == records.TestResultEnums.TEST_RESULT_PASS: self._exec_procedure_func(self._on_pass, tr_record) elif tr_record.result == records.TestResultEnums.TEST_RESULT_SKIP: self._exec_procedure_func(self._on_skip, tr_record) finally: logging.info(RESULT_LINE_TEMPLATE, tr_record.test_name, tr_record.result) self.results.add_record(tr_record) self.summary_writer.dump(tr_record.to_dict(), records.TestSummaryEntryType.RECORD) self.current_test_info = None return tr_record