def RunTest(self, test): results = base_test_result.TestRunResults() timeout = (self._GetIndividualTestTimeoutSecs(test) * self._GetIndividualTestTimeoutScale(test) * self.tool.GetTimeoutScale()) cmdline_parameters = self._GetTestCmdlineParameters(test) for flag_modifiers in cmdline_parameters: start_ms = 0 duration_ms = 0 try: if self._IsFreTest(test): flag_modifiers.remove.append('--disable-fre') self.TestSetup(test, flag_modifiers) try: self.device.GoHome() except device_errors.CommandTimeoutError: logging.exception('Failed to focus the launcher.') time_ms = lambda: int(time.time() * 1000) start_ms = time_ms() raw_output = self._RunTest(test, timeout) duration_ms = time_ms() - start_ms # Parse the test output result_code, result_bundle, statuses = ( instrumentation_test_instance.ParseAmInstrumentRawOutput( raw_output)) result = self._GenerateTestResult( test, result_code, result_bundle, statuses, start_ms, duration_ms) if local_device_instrumentation_test_run.DidPackageCrashOnDevice( self.test_pkg.GetPackageName(), self.device): result.SetType(base_test_result.ResultType.CRASH) except device_errors.CommandTimeoutError as e: result = test_result.InstrumentationTestResult( test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms, log=str(e) or 'No information') if self.package_info: self.device.ForceStop(self.package_info.package) self.device.ForceStop(self.package_info.test_package) except device_errors.DeviceUnreachableError as e: result = test_result.InstrumentationTestResult( test, base_test_result.ResultType.CRASH, start_ms, duration_ms, log=str(e) or 'No information') if len(cmdline_parameters) > 1: # Specify commandline flag modifications used in the test run result_name = result.GetName() if flag_modifiers.add: result_name = '%s with {%s}' % ( result_name, ' '.join(flag_modifiers.add)) if flag_modifiers.remove: result_name = '%s without {%s}' % ( result_name, ' '.join(flag_modifiers.remove)) result.SetName(result_name) results.AddResult(result) self.TestTeardown(test, results) return (results, None if results.DidRunPass() else test)
def RunTest(self, test): results = base_test_result.TestRunResults() timeout = (self._GetIndividualTestTimeoutSecs(test) * self._GetIndividualTestTimeoutScale(test) * self.tool.GetTimeoutScale()) if (self.device.GetProp('ro.build.version.sdk') < constants.ANDROID_SDK_VERSION_CODES.JELLY_BEAN): timeout *= 10 start_ms = 0 duration_ms = 0 try: self.TestSetup(test) time_ms = lambda: int(time.time() * 1000) start_ms = time_ms() raw_output = self._RunTest(test, timeout) duration_ms = time_ms() - start_ms # Parse the test output _, _, statuses = self._ParseAmInstrumentRawOutput(raw_output) result = self._GenerateTestResult(test, statuses, start_ms, duration_ms) results.AddResult(result) except device_errors.CommandTimeoutError as e: results.AddResult(test_result.InstrumentationTestResult( test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms, log=str(e) or 'No information')) except device_errors.DeviceUnreachableError as e: results.AddResult(test_result.InstrumentationTestResult( test, base_test_result.ResultType.CRASH, start_ms, duration_ms, log=str(e) or 'No information')) self.TestTeardown(test, results) return (results, None if results.DidRunPass() else test)
def RunTest(self, test): results = base_test_result.TestRunResults() timeout = (self._GetIndividualTestTimeoutSecs(test) * self._GetIndividualTestTimeoutScale(test) * self.tool.GetTimeoutScale()) start_ms = 0 duration_ms = 0 try: self.TestSetup(test) time_ms = lambda: int(time.time() * 1000) start_ms = time_ms() raw_output = self._RunTest(test, timeout) duration_ms = time_ms() - start_ms # Parse the test output result_code, result_bundle, statuses = ( instrumentation_test_instance.ParseAmInstrumentRawOutput(raw_output)) result = self._GenerateTestResult( test, result_code, result_bundle, statuses, start_ms, duration_ms) if local_device_instrumentation_test_run.DidPackageCrashOnDevice( self.test_pkg.GetPackageName(), self.device): result.SetType(base_test_result.ResultType.CRASH) results.AddResult(result) except device_errors.CommandTimeoutError as e: results.AddResult(test_result.InstrumentationTestResult( test, base_test_result.ResultType.TIMEOUT, start_ms, duration_ms, log=str(e) or 'No information')) except device_errors.DeviceUnreachableError as e: results.AddResult(test_result.InstrumentationTestResult( test, base_test_result.ResultType.CRASH, start_ms, duration_ms, log=str(e) or 'No information')) self.TestTeardown(test, results) return (results, None if results.DidRunPass() else test)
def RunTest(self, test): raw_result = None start_date_ms = None results = base_test_result.TestRunResults() timeout = (self._GetIndividualTestTimeoutSecs(test) * self._GetIndividualTestTimeoutScale(test) * self.tool.GetTimeoutScale()) try: self.TestSetup(test) start_date_ms = int(time.time()) * 1000 raw_result = self._RunTest(test, timeout) duration_ms = int(time.time()) * 1000 - start_date_ms status_code = raw_result.GetStatusCode() if status_code: if self.options.screenshot_failures: self._TakeScreenshot(test) log = raw_result.GetFailureReason() if not log: log = 'No information.' result_type = base_test_result.ResultType.FAIL package = self.adb.DismissCrashDialogIfNeeded() # Assume test package convention of ".test" suffix if package and package in self.test_pkg.GetPackageName(): result_type = base_test_result.ResultType.CRASH result = test_result.InstrumentationTestResult(test, result_type, start_date_ms, duration_ms, log=log) else: result = test_result.InstrumentationTestResult( test, base_test_result.ResultType.PASS, start_date_ms, duration_ms) results.AddResult(result) # Catch exceptions thrown by StartInstrumentation(). # See ../../third_party/android/testrunner/adb_interface.py except (android_commands.errors.WaitForResponseTimedOutError, android_commands.errors.DeviceUnresponsiveError, android_commands.errors.InstrumentationError), e: if start_date_ms: duration_ms = int(time.time()) * 1000 - start_date_ms else: start_date_ms = int(time.time()) * 1000 duration_ms = 0 message = str(e) if not message: message = 'No information.' results.AddResult( test_result.InstrumentationTestResult( test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, log=message)) raw_result = None
def _RunJavaTestFilters(self, test_filters, additional_flags=None): """Calls a list of tests and stops at the first test failure. This method iterates until either it encounters a non-passing test or it exhausts the list of tests. Then it returns the appropriate overall result. Test cases may make use of this method internally to assist in running instrumentation tests. This function relies on instrumentation_options being defined. Args: test_filters: A list of Java test filters. additional_flags: A list of addition flags to add to the command line. Returns: A TestRunResults object containing an overall result for this set of Java tests. If any Java tests do not pass, this is a fail overall. """ test_type = base_test_result.ResultType.PASS log = '' test_pkg = test_package.TestPackage( self.instrumentation_options.test_apk_path, self.instrumentation_options.test_apk_jar_path, self.instrumentation_options.test_support_apk_path) start_ms = int(time.time()) * 1000 done = False for test_filter in test_filters: tests = test_pkg.GetAllMatchingTests(None, None, test_filter, [self.device]) # Filters should always result in >= 1 test. if len(tests) == 0: raise Exception('Java test filter "%s" returned no tests.' % test_filter) for test in tests: # We're only running one test at a time, so this TestRunResults object # will hold only one result. java_result = self.__RunJavaTest(test, test_pkg, additional_flags) assert len(java_result.GetAll()) == 1 if not java_result.DidRunPass(): result = java_result.GetNotPass().pop() log = result.GetLog() log += self.__GetHostForwarderLog() test_type = result.GetType() done = True break if done: break duration_ms = int(time.time()) * 1000 - start_ms overall_result = base_test_result.TestRunResults() overall_result.AddResult( test_result.InstrumentationTestResult(self.tagged_name, test_type, start_ms, duration_ms, log=log)) return overall_result
def _GenerateTestResult(self, test, instr_statuses, start_ms, duration_ms): """Generate the result of |test| from |instr_statuses|. Args: instr_statuses: A list of 2-tuples containing: - the status code as an integer - the bundle dump as a dict mapping string keys to string values Note that this is the same as the third item in the 3-tuple returned by |_ParseAmInstrumentRawOutput|. start_ms: The start time of the test in milliseconds. duration_ms: The duration of the test in milliseconds. Returns: An InstrumentationTestResult object. """ INSTR_STATUS_CODE_START = 1 INSTR_STATUS_CODE_OK = 0 INSTR_STATUS_CODE_ERROR = -1 INSTR_STATUS_CODE_FAIL = -2 log = '' result_type = base_test_result.ResultType.UNKNOWN for status_code, bundle in instr_statuses: if status_code == INSTR_STATUS_CODE_START: pass elif status_code == INSTR_STATUS_CODE_OK: bundle_test = '%s#%s' % ( ''.join(bundle.get('class', [''])), ''.join(bundle.get('test', ['']))) skipped = ''.join(bundle.get('test_skipped', [''])) if (test == bundle_test and result_type == base_test_result.ResultType.UNKNOWN): result_type = base_test_result.ResultType.PASS elif skipped.lower() in ('true', '1', 'yes'): result_type = base_test_result.ResultType.SKIP logging.info('Skipped ' + test) else: if status_code not in (INSTR_STATUS_CODE_ERROR, INSTR_STATUS_CODE_FAIL): logging.info('Unrecognized status code %d. Handling as an error.', status_code) result_type = base_test_result.ResultType.FAIL if 'stack' in bundle: log = '\n'.join(bundle['stack']) # Dismiss any error dialogs. Limit the number in case we have an error # loop or we are failing to dismiss. for _ in xrange(10): package = self.device.old_interface.DismissCrashDialogIfNeeded() if not package: break # Assume test package convention of ".test" suffix if package in self.test_pkg.GetPackageName(): result_type = base_test_result.ResultType.CRASH break return test_result.InstrumentationTestResult( test, result_type, start_ms, duration_ms, log=log)
def _GenerateTestResult(self, test, instr_result_code, instr_result_bundle, statuses, start_ms, duration_ms): results = instrumentation_test_instance.GenerateTestResults( instr_result_code, instr_result_bundle, statuses, start_ms, duration_ms) for r in results: if r.GetName() == test: return r logging.error('Could not find result for test: %s', test) return test_result.InstrumentationTestResult( test, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms)
def GenerateTestResult(test_name, instr_statuses, start_ms, duration_ms): """Generate the result of |test| from |instr_statuses|. Args: test_name: The name of the test as "class#method" instr_statuses: A list of 2-tuples containing: - the status code as an integer - the bundle dump as a dict mapping string keys to string values Note that this is the same as the third item in the 3-tuple returned by |_ParseAmInstrumentRawOutput|. start_ms: The start time of the test in milliseconds. duration_ms: The duration of the test in milliseconds. Returns: An InstrumentationTestResult object. """ INSTR_STATUS_CODE_START = 1 INSTR_STATUS_CODE_OK = 0 INSTR_STATUS_CODE_ERROR = -1 INSTR_STATUS_CODE_FAIL = -2 log = '' result_type = base_test_result.ResultType.UNKNOWN for status_code, bundle in instr_statuses: if status_code == INSTR_STATUS_CODE_START: pass elif status_code == INSTR_STATUS_CODE_OK: bundle_test = '%s#%s' % (''.join(bundle.get( 'class', [''])), ''.join(bundle.get('test', ['']))) skipped = ''.join(bundle.get('test_skipped', [''])) if (test_name == bundle_test and result_type == base_test_result.ResultType.UNKNOWN): result_type = base_test_result.ResultType.PASS elif skipped.lower() in ('true', '1', 'yes'): result_type = base_test_result.ResultType.SKIP logging.info('Skipped ' + test_name) else: if status_code not in (INSTR_STATUS_CODE_ERROR, INSTR_STATUS_CODE_FAIL): logging.error( 'Unrecognized status code %d. Handling as an error.', status_code) result_type = base_test_result.ResultType.FAIL if 'stack' in bundle: log = '\n'.join(bundle['stack']) return test_result.InstrumentationTestResult(test_name, result_type, start_ms, duration_ms, log=log)
def _RunJavaTests(self, package_name, tests): """Calls a list of tests and stops at the first test failure. This method iterates until either it encounters a non-passing test or it exhausts the list of tests. Then it returns the appropriate overall result. Test cases may make use of this method internally to assist in running instrumentation tests. This function relies on instrumentation_options being defined. Args: package_name: Package name in which the java tests live (e.g. foo.bar.baz.tests) tests: A list of Java test names which will be run Returns: A TestRunResults object containing an overall result for this set of Java tests. If any Java tests do not pass, this is a fail overall. """ test_type = base_test_result.ResultType.PASS log = '' start_ms = int(time.time()) * 1000 for test in tests: # We're only running one test at a time, so this TestRunResults object # will hold only one result. suite, test_name = test.split('.') java_result = self.__RunJavaTest(package_name, suite, test_name) assert len(java_result.GetAll()) == 1 if not java_result.DidRunPass(): result = java_result.GetNotPass().pop() log = result.GetLog() test_type = result.GetType() break duration_ms = int(time.time()) * 1000 - start_ms overall_result = base_test_result.TestRunResults() overall_result.AddResult( test_result.InstrumentationTestResult( self.tagged_name, test_type, start_ms, duration_ms, log=log)) return overall_result
def _RunJavaTests(self, fname, tests): """Calls a list of tests and stops at the first test failure. This method iterates until either it encounters a non-passing test or it exhausts the list of tests. Then it returns the appropriate Python result. Args: fname: filename for the Python test tests: a list of Java test names which will be run Returns: A TestRunResults object containing a result for this Python test. """ test_type = base_test_result.ResultType.PASS log = '' start_ms = int(time.time()) * 1000 for test in tests: # We're only running one test at a time, so this TestRunResults object # will hold only one result. suite, test_name = test.split('.') java_results = self._RunJavaTest(fname, suite, test_name) assert len(java_results.GetAll()) == 1 if not java_results.DidRunPass(): result = java_results.GetNotPass().pop() log = result.GetLog() test_type = result.GetType() break duration_ms = int(time.time()) * 1000 - start_ms python_results = base_test_result.TestRunResults() python_results.AddResult( test_result.InstrumentationTestResult(self.qualified_name, test_type, start_ms, duration_ms, log=log)) return python_results
def GenerateTestResults( result_code, result_bundle, statuses, start_ms, duration_ms, device_abi, symbolizer): """Generate test results from |statuses|. Args: result_code: The overall status code as an integer. result_bundle: The summary bundle dump as a dict. statuses: A list of 2-tuples containing: - the status code as an integer - the bundle dump as a dict mapping string keys to string values Note that this is the same as the third item in the 3-tuple returned by |_ParseAmInstrumentRawOutput|. start_ms: The start time of the test in milliseconds. duration_ms: The duration of the test in milliseconds. device_abi: The device_abi, which is needed for symbolization. symbolizer: The symbolizer used to symbolize stack. Returns: A list containing an instance of InstrumentationTestResult for each test parsed. """ results = [] current_result = None for status_code, bundle in statuses: test_class = bundle.get('class', '') test_method = bundle.get('test', '') if test_class and test_method: test_name = '%s#%s' % (test_class, test_method) else: continue if status_code == instrumentation_parser.STATUS_CODE_START: if current_result: results.append(current_result) current_result = test_result.InstrumentationTestResult( test_name, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms) else: if status_code == instrumentation_parser.STATUS_CODE_OK: if bundle.get('test_skipped', '').lower() in ('true', '1', 'yes'): current_result.SetType(base_test_result.ResultType.SKIP) elif current_result.GetType() == base_test_result.ResultType.UNKNOWN: current_result.SetType(base_test_result.ResultType.PASS) elif status_code == instrumentation_parser.STATUS_CODE_SKIP: current_result.SetType(base_test_result.ResultType.SKIP) elif status_code == instrumentation_parser.STATUS_CODE_ASSUMPTION_FAILURE: current_result.SetType(base_test_result.ResultType.SKIP) else: if status_code not in (instrumentation_parser.STATUS_CODE_ERROR, instrumentation_parser.STATUS_CODE_FAILURE): logging.error('Unrecognized status code %d. Handling as an error.', status_code) current_result.SetType(base_test_result.ResultType.FAIL) if 'stack' in bundle: if symbolizer and device_abi: current_result.SetLog( '%s\n%s' % ( bundle['stack'], '\n'.join(symbolizer.ExtractAndResolveNativeStackTraces( bundle['stack'], device_abi)))) else: current_result.SetLog(bundle['stack']) if current_result: if current_result.GetType() == base_test_result.ResultType.UNKNOWN: crashed = (result_code == _ACTIVITY_RESULT_CANCELED and any(_NATIVE_CRASH_RE.search(l) for l in result_bundle.itervalues())) if crashed: current_result.SetType(base_test_result.ResultType.CRASH) results.append(current_result) return results
def GenerateTestResults(result_code, result_bundle, statuses, duration_ms, device_abi, symbolizer): """Generate test results from |statuses|. Args: result_code: The overall status code as an integer. result_bundle: The summary bundle dump as a dict. statuses: A list of 2-tuples containing: - the status code as an integer - the bundle dump as a dict mapping string keys to string values Note that this is the same as the third item in the 3-tuple returned by |_ParseAmInstrumentRawOutput|. duration_ms: The duration of the test in milliseconds. device_abi: The device_abi, which is needed for symbolization. symbolizer: The symbolizer used to symbolize stack. Returns: A list containing an instance of InstrumentationTestResult for each test parsed. """ results = [] current_result = None cumulative_duration = 0 for status_code, bundle in statuses: # If the last test was a failure already, don't override that failure with # post-test failures that could be caused by the original failure. if (status_code == instrumentation_parser.STATUS_CODE_BATCH_FAILURE and current_result.GetType() != base_test_result.ResultType.FAIL): current_result.SetType(base_test_result.ResultType.FAIL) _MaybeSetLog(bundle, current_result, symbolizer, device_abi) continue if status_code == instrumentation_parser.STATUS_CODE_TEST_DURATION: # For the first result, duration will be set below to the difference # between the reported and actual durations to account for overhead like # starting instrumentation. if results: current_duration = int( bundle.get(_BUNDLE_DURATION_ID, duration_ms)) current_result.SetDuration(current_duration) cumulative_duration += current_duration continue test_class = bundle.get(_BUNDLE_CLASS_ID, '') test_method = bundle.get(_BUNDLE_TEST_ID, '') if test_class and test_method: test_name = '%s#%s' % (test_class, test_method) else: continue if status_code == instrumentation_parser.STATUS_CODE_START: if current_result: results.append(current_result) current_result = test_result.InstrumentationTestResult( test_name, base_test_result.ResultType.UNKNOWN, duration_ms) else: if status_code == instrumentation_parser.STATUS_CODE_OK: if bundle.get(_BUNDLE_SKIPPED_ID, '').lower() in ('true', '1', 'yes'): current_result.SetType(base_test_result.ResultType.SKIP) elif current_result.GetType( ) == base_test_result.ResultType.UNKNOWN: current_result.SetType(base_test_result.ResultType.PASS) elif status_code == instrumentation_parser.STATUS_CODE_SKIP: current_result.SetType(base_test_result.ResultType.SKIP) elif status_code == instrumentation_parser.STATUS_CODE_ASSUMPTION_FAILURE: current_result.SetType(base_test_result.ResultType.SKIP) else: if status_code not in ( instrumentation_parser.STATUS_CODE_ERROR, instrumentation_parser.STATUS_CODE_FAILURE): logging.error( 'Unrecognized status code %d. Handling as an error.', status_code) current_result.SetType(base_test_result.ResultType.FAIL) _MaybeSetLog(bundle, current_result, symbolizer, device_abi) if current_result: if current_result.GetType() == base_test_result.ResultType.UNKNOWN: crashed = (result_code == _ACTIVITY_RESULT_CANCELED and any( _NATIVE_CRASH_RE.search(l) for l in six.itervalues(result_bundle))) if crashed: current_result.SetType(base_test_result.ResultType.CRASH) results.append(current_result) if results: logging.info('Adding cumulative overhead to test %s: %dms', results[0].GetName(), duration_ms - cumulative_duration) results[0].SetDuration(duration_ms - cumulative_duration) return results
def RunTest(self, test): raw_result = None start_date_ms = None results = base_test_result.TestRunResults() timeout = (self._GetIndividualTestTimeoutSecs(test) * self._GetIndividualTestTimeoutScale(test) * self.tool.GetTimeoutScale()) try: self.TestSetup(test) start_date_ms = int(time.time()) * 1000 raw_result = self._RunTest(test, timeout) duration_ms = int(time.time()) * 1000 - start_date_ms status_code = raw_result.GetStatusCode() if status_code: if self.options.screenshot_failures: self._TakeScreenshot(test) log = raw_result.GetFailureReason() if not log: log = 'No information.' result_type = base_test_result.ResultType.FAIL # Dismiss any error dialogs. Limit the number in case we have an error # loop or we are failing to dismiss. for _ in xrange(10): package = self.device.old_interface.DismissCrashDialogIfNeeded( ) if not package: break # Assume test package convention of ".test" suffix if package in self.test_pkg.GetPackageName(): result_type = base_test_result.ResultType.CRASH break result = test_result.InstrumentationTestResult(test, result_type, start_date_ms, duration_ms, log=log) else: result = test_result.InstrumentationTestResult( test, base_test_result.ResultType.PASS, start_date_ms, duration_ms) results.AddResult(result) # Catch exceptions thrown by StartInstrumentation(). # See ../../third_party/android/testrunner/adb_interface.py except ( device_errors.CommandTimeoutError, device_errors.DeviceUnreachableError, # TODO(jbudorick) Remove these once the underlying implementations # for the above are switched or wrapped. android_commands.errors.WaitForResponseTimedOutError, android_commands.errors.DeviceUnresponsiveError, android_commands.errors.InstrumentationError), e: if start_date_ms: duration_ms = int(time.time()) * 1000 - start_date_ms else: start_date_ms = int(time.time()) * 1000 duration_ms = 0 message = str(e) if not message: message = 'No information.' results.AddResult( test_result.InstrumentationTestResult( test, base_test_result.ResultType.CRASH, start_date_ms, duration_ms, log=message)) raw_result = None