def RunTests(self, results): tests = self._GetTests() exit_now = threading.Event() @local_device_environment.handle_shard_failures def run_tests_on_device(dev, tests, results): consecutive_device_errors = 0 for test in tests: if exit_now.isSet(): thread.exit() result = None rerun = None try: result, rerun = crash_handler.RetryOnSystemCrash( lambda d, t=test: self._RunTest(d, t), device=dev) consecutive_device_errors = 0 if isinstance(result, base_test_result.BaseTestResult): results.AddResult(result) elif isinstance(result, list): results.AddResults(result) else: raise Exception('Unexpected result type: %s' % type(result).__name__) except device_errors.CommandTimeoutError: # Test timeouts don't count as device errors for the purpose # of bad device detection. consecutive_device_errors = 0 if isinstance(test, list): results.AddResults( base_test_result.BaseTestResult( self._GetUniqueTestName(t), base_test_result.ResultType.TIMEOUT) for t in test) else: results.AddResult( base_test_result.BaseTestResult( self._GetUniqueTestName(test), base_test_result.ResultType.TIMEOUT)) except Exception as e: # pylint: disable=broad-except if isinstance(tests, test_collection.TestCollection): rerun = test if (isinstance(e, device_errors.DeviceUnreachableError) or not isinstance(e, base_error.BaseError)): # If we get a device error but believe the device is still # reachable, attempt to continue using it. Otherwise, raise # the exception and terminate this run_tests_on_device call. raise consecutive_device_errors += 1 if consecutive_device_errors >= 3: # We believe the device is still reachable and may still be usable, # but if it fails repeatedly, we shouldn't attempt to keep using # it. logging.error( 'Repeated failures on device %s. Abandoning.', str(dev)) raise logging.exception( 'Attempting to continue using device %s despite failure (%d/3).', str(dev), consecutive_device_errors) finally: if isinstance(tests, test_collection.TestCollection): if rerun: tests.add(rerun) tests.test_completed() logging.info('Finished running tests on this device.') def stop_tests(_signum, _frame): logging.critical('Received SIGTERM. Stopping test execution.') exit_now.set() raise TestsTerminated() try: with signal_handler.AddSignalHandler(signal.SIGTERM, stop_tests): tries = 0 while tries < self._env.max_tries and tests: logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries) if tries > 0 and self._env.recover_devices: if any(d.build_version_sdk == version_codes.LOLLIPOP_MR1 for d in self._env.devices): logging.info( 'Attempting to recover devices due to known issue on L MR1. ' 'See crbug.com/787056 for details.') self._env.parallel_devices.pMap( device_recovery.RecoverDevice, None) elif tries + 1 == self._env.max_tries: logging.info( 'Attempting to recover devices prior to last test attempt.' ) self._env.parallel_devices.pMap( device_recovery.RecoverDevice, None) logging.info('Will run %d tests on %d devices: %s', len(tests), len(self._env.devices), ', '.join(str(d) for d in self._env.devices)) for t in tests: logging.debug(' %s', t) try_results = base_test_result.TestRunResults() test_names = (self._GetUniqueTestName(t) for t in tests) try_results.AddResults( base_test_result.BaseTestResult( t, base_test_result.ResultType.NOTRUN) for t in test_names if not t.endswith('*')) # As soon as we know the names of the tests, we populate |results|. # The tests in try_results will have their results updated by # try_results.AddResult() as they are run. results.append(try_results) try: if self._ShouldShard(): tc = test_collection.TestCollection( self._CreateShards(tests)) self._env.parallel_devices.pMap( run_tests_on_device, tc, try_results).pGet(None) else: self._env.parallel_devices.pMap( run_tests_on_device, tests, try_results).pGet(None) except TestsTerminated: for unknown_result in try_results.GetUnknown(): try_results.AddResult( base_test_result.BaseTestResult( unknown_result.GetName(), base_test_result.ResultType.TIMEOUT, log=_SIGTERM_TEST_LOG)) raise tries += 1 tests = self._GetTestsToRetry(tests, try_results) logging.info('FINISHED TRY #%d/%d', tries, self._env.max_tries) if tests: logging.info('%d failed tests remain.', len(tests)) else: logging.info('All tests completed.') except TestsTerminated: pass
def RunTests(self): tests = self._GetTests() exit_now = threading.Event() @handle_shard_failures def run_tests_on_device(dev, tests, results): for test in tests: if exit_now.isSet(): thread.exit() result = None try: result = self._RunTest(dev, test) if isinstance(result, base_test_result.BaseTestResult): results.AddResult(result) elif isinstance(result, list): results.AddResults(result) else: raise Exception('Unexpected result type: %s' % type(result).__name__) except: if isinstance(tests, test_collection.TestCollection): tests.add(test) raise finally: if isinstance(tests, test_collection.TestCollection): tests.test_completed() logging.info('Finished running tests on this device.') def stop_tests(_signum, _frame): exit_now.set() with signal_handler.AddSignalHandler(signal.SIGTERM, stop_tests): tries = 0 results = [] while tries < self._env.max_tries and tests: logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries) logging.info('Will run %d tests on %d devices: %s', len(tests), len(self._env.devices), ', '.join(str(d) for d in self._env.devices)) for t in tests: logging.debug(' %s', t) try_results = base_test_result.TestRunResults() if self._ShouldShard(): tc = test_collection.TestCollection( self._CreateShards(tests)) self._env.parallel_devices.pMap(run_tests_on_device, tc, try_results).pGet(None) else: self._env.parallel_devices.pMap(run_tests_on_device, tests, try_results).pGet(None) results.append(try_results) tries += 1 tests = self._GetTestsToRetry(tests, try_results) logging.info('FINISHED TRY #%d/%d', tries, self._env.max_tries) if tests: logging.info('%d failed tests remain.', len(tests)) else: logging.info('All tests completed.') return results
def RunTests(self): tests = self._GetTests() exit_now = threading.Event() @local_device_environment.handle_shard_failures def run_tests_on_device(dev, tests, results): for test in tests: if exit_now.isSet(): thread.exit() result = None try: result = self._RunTest(dev, test) if isinstance(result, base_test_result.BaseTestResult): results.AddResult(result) elif isinstance(result, list): results.AddResults(result) else: raise Exception('Unexpected result type: %s' % type(result).__name__) except: if isinstance(tests, test_collection.TestCollection): tests.add(test) raise finally: if isinstance(tests, test_collection.TestCollection): tests.test_completed() logging.info('Finished running tests on this device.') class TestsTerminated(Exception): pass def stop_tests(_signum, _frame): logging.critical('Received SIGTERM. Stopping test execution.') exit_now.set() raise TestsTerminated() try: with signal_handler.AddSignalHandler(signal.SIGTERM, stop_tests): tries = 0 results = [] while tries < self._env.max_tries and tests: logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries) logging.info('Will run %d tests on %d devices: %s', len(tests), len(self._env.devices), ', '.join(str(d) for d in self._env.devices)) for t in tests: logging.debug(' %s', t) try_results = base_test_result.TestRunResults() test_names = (self._GetUniqueTestName(t) for t in tests) try_results.AddResults( base_test_result.BaseTestResult( t, base_test_result.ResultType.UNKNOWN) for t in test_names if not t.endswith('*')) try: if self._ShouldShard(): tc = test_collection.TestCollection( self._CreateShards(tests)) self._env.parallel_devices.pMap( run_tests_on_device, tc, try_results).pGet(None) else: self._env.parallel_devices.pMap( run_tests_on_device, tests, try_results).pGet(None) except TestsTerminated: for unknown_result in try_results.GetUnknown(): try_results.AddResult( base_test_result.BaseTestResult( unknown_result.GetName(), base_test_result.ResultType.TIMEOUT, log=_SIGTERM_TEST_LOG)) raise finally: results.append(try_results) tries += 1 tests = self._GetTestsToRetry(tests, try_results) logging.info('FINISHED TRY #%d/%d', tries, self._env.max_tries) if tests: logging.info('%d failed tests remain.', len(tests)) else: logging.info('All tests completed.') except TestsTerminated: pass return results