def _CreateRunners(runner_factory, devices, timeout=None):
"""Creates a test runner for each device and calls SetUp() in parallel.
Note: if a device is unresponsive the corresponding TestRunner will not be
included in the returned list.
Args:
runner_factory: Callable that takes a device and index and returns a
TestRunner object.
devices: List of device serial numbers as strings.
timeout: Watchdog timeout in seconds, defaults to the default timeout.
Returns:
A list of TestRunner objects.
"""
logging.warning('Creating %s test %s.', len(devices),
'runners' if len(devices) != 1 else 'runner')
runners = []
counter = _ThreadSafeCounter()
threads = reraiser_thread.ReraiserThreadGroup(
[reraiser_thread.ReraiserThread(_SetUp,
[runner_factory, d, runners, counter],
name=str(d)[-4:])
for d in devices])
threads.StartAll()
threads.JoinAll(watchdog_timer.WatchdogTimer(timeout))
return runners
def _JoinAll(self, watcher=None, timeout=None):
"""Join all threads without stack dumps.
Reraises exceptions raised by the child threads and supports breaking
immediately on exceptions raised on the main thread.
Args:
watcher: Watchdog object providing the thread timeout. If none is
provided, the thread will never be timed out.
timeout: An optional number of seconds to wait before timing out the join
operation. This will not time out the threads.
"""
if watcher is None:
watcher = watchdog_timer.WatchdogTimer(None)
alive_threads = self._threads[:]
end_time = (time.time() + timeout) if timeout else None
try:
while alive_threads and (end_time is None
or end_time > time.time()):
for thread in alive_threads[:]:
if watcher.IsTimedOut():
raise TimeoutError(
'Timed out waiting for %d of %d threads.' %
(len(alive_threads), len(self._threads)))
# Allow the main thread to periodically check for interrupts.
thread.join(0.1)
if not thread.isAlive():
alive_threads.remove(thread)
# All threads are allowed to complete before reraising exceptions.
for thread in self._threads:
thread.ReraiseIfException()
finally:
self.blocked_parent_thread_group = None
def _RunAllTests(runners, test_collection_factory, num_retries, timeout=None,
tag_results_with_device=False):
"""Run all tests using the given TestRunners.
Args:
runners: A list of TestRunner objects.
test_collection_factory: A callable to generate a TestCollection object for
each test runner.
num_retries: Number of retries for a test.
timeout: Watchdog timeout in seconds.
tag_results_with_device: If True, appends the name of the device on which
the test was run to the test name. Used when replicating to identify
which device ran each copy of the test, and to ensure each copy of the
test is recorded separately.
Returns:
A tuple of (TestRunResults object, exit code)
"""
logging.warning('Running tests with %s test %s.',
len(runners), 'runners' if len(runners) != 1 else 'runner')
results = []
exit_code = 0
run_results = base_test_result.TestRunResults()
watcher = watchdog_timer.WatchdogTimer(timeout)
test_collections = [test_collection_factory() for _ in runners]
threads = [
reraiser_thread.ReraiserThread(
_RunTestsFromQueue,
[r, tc, results, watcher, num_retries, tag_results_with_device],
name=r.device_serial[-4:])
for r, tc in zip(runners, test_collections)]
workers = reraiser_thread.ReraiserThreadGroup(threads)
workers.StartAll()
try:
workers.JoinAll(watcher)
except device_errors.CommandFailedError:
logging.exception('Command failed on device.')
except device_errors.CommandFailedError:
logging.exception('Command timed out on device.')
except device_errors.DeviceUnreachableError:
logging.exception('Device became unreachable.')
if not all((len(tc) == 0 for tc in test_collections)):
logging.error('Only ran %d tests (all devices are likely offline).',
len(results))
for tc in test_collections:
run_results.AddResults(base_test_result.BaseTestResult(
t, base_test_result.ResultType.UNKNOWN) for t in tc.test_names())
for r in results:
run_results.AddTestRunResults(r)
if not run_results.DidRunPass():
exit_code = constants.ERROR_EXIT_CODE
return (run_results, exit_code)
def _RunTests(mock_runner, tests):
results = []
tests = test_collection.TestCollection(
[test_dispatcher._Test(t) for t in tests])
test_dispatcher._RunTestsFromQueue(mock_runner, tests, results,
watchdog_timer.WatchdogTimer(None), 2)
run_results = base_test_result.TestRunResults()
for r in results:
run_results.AddTestRunResults(r)
return run_results
def _TearDownRunners(runners, timeout=None):
"""Calls TearDown() for each test runner in parallel.
Args:
runners: A list of TestRunner objects.
timeout: Watchdog timeout in seconds, defaults to the default timeout.
"""
threads = reraiser_thread.ReraiserThreadGroup(
[reraiser_thread.ReraiserThread(r.TearDown, name=r.device_serial[-4:])
for r in runners])
threads.StartAll()
threads.JoinAll(watchdog_timer.WatchdogTimer(timeout))
def pFinish(self, timeout):
"""Finish any outstanding asynchronous operations.
Args:
timeout: The maximum number of seconds to wait for an individual
result to return, or None to wait forever.
Returns:
self, now emulating the return values.
"""
self._assertNoShadow('pFinish')
if isinstance(self._objs, reraiser_thread.ReraiserThreadGroup):
self._objs.JoinAll()
self._objs = self._objs.GetAllReturnValues(
watchdog_timer.WatchdogTimer(timeout))
return self
def testJoinTimeout(self):
def f():
pass
event = threading.Event()
def g():
event.wait()
group = reraiser_thread.ReraiserThreadGroup(
[reraiser_thread.ReraiserThread(g),
reraiser_thread.ReraiserThread(f)])
group.StartAll()
with self.assertRaises(reraiser_thread.TimeoutError):
group.JoinAll(watchdog_timer.WatchdogTimer(0.01))
event.set()
def __init__(self, timeout, threads=None):
super(TimeoutRetryThreadGroup, self).__init__(threads)
self._watcher = watchdog_timer.WatchdogTimer(timeout)
