def _CreateRunners(runner_factory, devices, timeout=None): """Creates a test runner for each device and calls SetUp() in parallel. Note: if a device is unresponsive the corresponding TestRunner will not be included in the returned list. Args: runner_factory: Callable that takes a device and index and returns a TestRunner object. devices: List of device serial numbers as strings. timeout: Watchdog timeout in seconds, defaults to the default timeout. Returns: A list of TestRunner objects. """ logging.warning('Creating %s test %s.', len(devices), 'runners' if len(devices) != 1 else 'runner') runners = [] counter = _ThreadSafeCounter() threads = reraiser_thread.ReraiserThreadGroup( [reraiser_thread.ReraiserThread(_SetUp, [runner_factory, d, runners, counter], name=str(d)[-4:]) for d in devices]) threads.StartAll() threads.JoinAll(watchdog_timer.WatchdogTimer(timeout)) return runners
def _JoinAll(self, watcher=None, timeout=None): """Join all threads without stack dumps. Reraises exceptions raised by the child threads and supports breaking immediately on exceptions raised on the main thread. Args: watcher: Watchdog object providing the thread timeout. If none is provided, the thread will never be timed out. timeout: An optional number of seconds to wait before timing out the join operation. This will not time out the threads. """ if watcher is None: watcher = watchdog_timer.WatchdogTimer(None) alive_threads = self._threads[:] end_time = (time.time() + timeout) if timeout else None try: while alive_threads and (end_time is None or end_time > time.time()): for thread in alive_threads[:]: if watcher.IsTimedOut(): raise TimeoutError( 'Timed out waiting for %d of %d threads.' % (len(alive_threads), len(self._threads))) # Allow the main thread to periodically check for interrupts. thread.join(0.1) if not thread.isAlive(): alive_threads.remove(thread) # All threads are allowed to complete before reraising exceptions. for thread in self._threads: thread.ReraiseIfException() finally: self.blocked_parent_thread_group = None
def _RunAllTests(runners, test_collection_factory, num_retries, timeout=None, tag_results_with_device=False): """Run all tests using the given TestRunners. Args: runners: A list of TestRunner objects. test_collection_factory: A callable to generate a TestCollection object for each test runner. num_retries: Number of retries for a test. timeout: Watchdog timeout in seconds. tag_results_with_device: If True, appends the name of the device on which the test was run to the test name. Used when replicating to identify which device ran each copy of the test, and to ensure each copy of the test is recorded separately. Returns: A tuple of (TestRunResults object, exit code) """ logging.warning('Running tests with %s test %s.', len(runners), 'runners' if len(runners) != 1 else 'runner') results = [] exit_code = 0 run_results = base_test_result.TestRunResults() watcher = watchdog_timer.WatchdogTimer(timeout) test_collections = [test_collection_factory() for _ in runners] threads = [ reraiser_thread.ReraiserThread( _RunTestsFromQueue, [r, tc, results, watcher, num_retries, tag_results_with_device], name=r.device_serial[-4:]) for r, tc in zip(runners, test_collections)] workers = reraiser_thread.ReraiserThreadGroup(threads) workers.StartAll() try: workers.JoinAll(watcher) except device_errors.CommandFailedError: logging.exception('Command failed on device.') except device_errors.CommandFailedError: logging.exception('Command timed out on device.') except device_errors.DeviceUnreachableError: logging.exception('Device became unreachable.') if not all((len(tc) == 0 for tc in test_collections)): logging.error('Only ran %d tests (all devices are likely offline).', len(results)) for tc in test_collections: run_results.AddResults(base_test_result.BaseTestResult( t, base_test_result.ResultType.UNKNOWN) for t in tc.test_names()) for r in results: run_results.AddTestRunResults(r) if not run_results.DidRunPass(): exit_code = constants.ERROR_EXIT_CODE return (run_results, exit_code)
def _RunTests(mock_runner, tests): results = [] tests = test_collection.TestCollection( [test_dispatcher._Test(t) for t in tests]) test_dispatcher._RunTestsFromQueue(mock_runner, tests, results, watchdog_timer.WatchdogTimer(None), 2) run_results = base_test_result.TestRunResults() for r in results: run_results.AddTestRunResults(r) return run_results
def _TearDownRunners(runners, timeout=None): """Calls TearDown() for each test runner in parallel. Args: runners: A list of TestRunner objects. timeout: Watchdog timeout in seconds, defaults to the default timeout. """ threads = reraiser_thread.ReraiserThreadGroup( [reraiser_thread.ReraiserThread(r.TearDown, name=r.device_serial[-4:]) for r in runners]) threads.StartAll() threads.JoinAll(watchdog_timer.WatchdogTimer(timeout))
def pFinish(self, timeout): """Finish any outstanding asynchronous operations. Args: timeout: The maximum number of seconds to wait for an individual result to return, or None to wait forever. Returns: self, now emulating the return values. """ self._assertNoShadow('pFinish') if isinstance(self._objs, reraiser_thread.ReraiserThreadGroup): self._objs.JoinAll() self._objs = self._objs.GetAllReturnValues( watchdog_timer.WatchdogTimer(timeout)) return self
def testJoinTimeout(self): def f(): pass event = threading.Event() def g(): event.wait() group = reraiser_thread.ReraiserThreadGroup( [reraiser_thread.ReraiserThread(g), reraiser_thread.ReraiserThread(f)]) group.StartAll() with self.assertRaises(reraiser_thread.TimeoutError): group.JoinAll(watchdog_timer.WatchdogTimer(0.01)) event.set()
def __init__(self, timeout, threads=None): super(TimeoutRetryThreadGroup, self).__init__(threads) self._watcher = watchdog_timer.WatchdogTimer(timeout)