def RunTests(self): """Runs tests on a single device. Returns: A TestResults object. """ try: self.test_package.CreateTestRunnerScript(self._gtest_filter, self._test_arguments) self.test_results = self.test_package.RunTestsAndListResults() except errors.DeviceUnresponsiveError as e: # Make sure this device is not attached if android_commands.IsDeviceAttached(self.device): raise e # TODO(frankf): We should report these as "skipped" not "failures". # Wrap the results logging.warning(e) failed_tests = [] for t in self._gtest_filter.split(':'): failed_tests += [BaseTestResult(t, '')] self.test_results = TestResults.FromRun( failed=failed_tests, device_exception=self.device) return self.test_results
def RunTest(self, test): test_results = base_test_result.TestRunResults() if not test: return test_results, None try: self.test_package.ClearApplicationState() self.test_package.CreateTestRunnerScript(test, self._test_arguments) test_results = self.test_package.RunTestsAndListResults() except errors.DeviceUnresponsiveError as e: # Make sure this device is not attached logging.warning(e) if android_commands.IsDeviceAttached(self.device): raise finally: self.CleanupSpawningServerState() # Calculate unknown test results. all_tests = set(test.split(':')) all_tests_ran = set([t.GetName() for t in test_results.GetAll()]) unknown_tests = all_tests - all_tests_ran test_results.AddResults([ base_test_result.BaseTestResult( t, base_test_result.ResultType.UNKNOWN) for t in unknown_tests ]) retry = ':'.join([t.GetName() for t in test_results.GetNotPass()]) return test_results, retry
def RunTest(self, test): """Runs a test on a single device. Args: test: a gtest filter string to run. Returns: Tuple: (TestResults, test to retry or None) """ test_results = test_result.TestResults() if not test: return test_results, None try: self.test_package.ClearApplicationState() self.test_package.CreateTestRunnerScript(test, self._test_arguments) test_results = self.test_package.RunTestsAndListResults() except errors.DeviceUnresponsiveError as e: # Make sure this device is not attached logging.warning(e) if android_commands.IsDeviceAttached(self.device): raise # Calculate unknown test results. # TODO(frankf): Do not break TestResults encapsulation. all_tests = set(test.split(':')) all_tests_ran = set([t.name for t in test_results.GetAll()]) unknown_tests = all_tests - all_tests_ran test_results.unknown = [ test_result.BaseTestResult(t, '') for t in unknown_tests ] retry = ':'.join([t.name for t in test_results.GetAllBroken()]) return test_results, retry
def RunTests(self): """Runs tests on a single device. Returns: A TestResults object. """ if self._gtest_filter: try: self.test_package.CreateTestRunnerScript( self._gtest_filter, self._test_arguments) self.test_results = self.test_package.RunTestsAndListResults() except errors.DeviceUnresponsiveError as e: # Make sure this device is not attached logging.warning(e) if android_commands.IsDeviceAttached(self.device): raise e self.test_results.device_exception = device_exception # Calculate unknown test results. finally: # TODO(frankf): Do not break TestResults encapsulation. all_tests = set(self._gtest_filter.split(':')) all_tests_ran = set( [t.name for t in self.test_results.GetAll()]) unknown_tests = all_tests - all_tests_ran self.test_results.unknown = [ BaseTestResult(t, '') for t in unknown_tests ] return self.test_results
def _RunTestsFromQueue(runner, test_collection, out_results, watcher, num_retries): """Runs tests from the test_collection until empty using the given runner. Adds TestRunResults objects to the out_results list and may add tests to the out_retry list. Args: runner: A TestRunner object used to run the tests. test_collection: A _TestCollection from which to get _Test objects to run. out_results: A list to add TestRunResults to. watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout. num_retries: Number of retries for a test. """ for test in test_collection: watcher.Reset() try: if not android_commands.IsDeviceAttached(runner.device): # Device is unresponsive, stop handling tests on this device. msg = 'Device %s is unresponsive.' % runner.device logging.warning(msg) raise android_commands.errors.DeviceUnresponsiveError(msg) result, retry = runner.RunTest(test.test) test.tries += 1 if retry and test.tries <= num_retries: # Retry non-passing results, only record passing results. pass_results = base_test_result.TestRunResults() pass_results.AddResults(result.GetPass()) out_results.append(pass_results) logging.warning('Will retry test, try #%s.' % test.tries) test_collection.add(_Test(test=retry, tries=test.tries)) else: # All tests passed or retry limit reached. Either way, record results. out_results.append(result) except android_commands.errors.DeviceUnresponsiveError: # Device is unresponsive, stop handling tests on this device and ensure # current test gets runs by another device. Don't reraise this exception # on the main thread. test_collection.add(test) return except: # An unhandleable exception, ensure tests get run by another device and # reraise this exception on the main thread. test_collection.add(test) raise finally: # Retries count as separate tasks so always mark the popped test as done. test_collection.test_completed()
def _RunTestsFromQueue(runner, test_collection, out_results, watcher, num_retries, tag_results_with_device=False): """Runs tests from the test_collection until empty using the given runner. Adds TestRunResults objects to the out_results list and may add tests to the out_retry list. Args: runner: A TestRunner object used to run the tests. test_collection: A _TestCollection from which to get _Test objects to run. out_results: A list to add TestRunResults to. watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout. num_retries: Number of retries for a test. tag_results_with_device: If True, appends the name of the device on which the test was run to the test name. Used when replicating to identify which device ran each copy of the test, and to ensure each copy of the test is recorded separately. """ def TagTestRunResults(test_run_results): """Tags all results with the last 4 digits of the device id. Used when replicating tests to distinguish the same tests run on different devices. We use a set to store test results, so the hash (generated from name and tag) must be unique to be considered different results. """ new_test_run_results = base_test_result.TestRunResults() for test_result in test_run_results.GetAll(): test_result.SetName('%s_%s' % (runner.device[-4:], test_result.GetName())) new_test_run_results.AddResult(test_result) return new_test_run_results for test in test_collection: watcher.Reset() try: if not android_commands.IsDeviceAttached(runner.device): # Device is unresponsive, stop handling tests on this device. msg = 'Device %s is unresponsive.' % runner.device logging.warning(msg) raise android_commands.errors.DeviceUnresponsiveError(msg) result, retry = runner.RunTest(test.test) if tag_results_with_device: result = TagTestRunResults(result) test.tries += 1 if retry and test.tries <= num_retries: # Retry non-passing results, only record passing results. pass_results = base_test_result.TestRunResults() pass_results.AddResults(result.GetPass()) out_results.append(pass_results) logging.warning('Will retry test, try #%s.' % test.tries) test_collection.add(_Test(test=retry, tries=test.tries)) else: # All tests passed or retry limit reached. Either way, record results. out_results.append(result) except: # An unhandleable exception, ensure tests get run by another device and # reraise this exception on the main thread. test_collection.add(test) raise finally: # Retries count as separate tasks so always mark the popped test as done. test_collection.test_completed()