def _CreateRunners(runner_factory, devices, timeout=None):
"""Creates a test runner for each device and calls SetUp() in parallel.
Note: if a device is unresponsive the corresponding TestRunner will not be
included in the returned list.
Args:
runner_factory: Callable that takes a device and index and returns a
TestRunner object.
devices: List of device serial numbers as strings.
timeout: Watchdog timeout in seconds, defaults to the default timeout.
Returns:
A list of TestRunner objects.
"""
logging.warning('Creating %s test runners.' % len(devices))
runners = []
counter = _ThreadSafeCounter()
threads = reraiser_thread.ReraiserThreadGroup([
reraiser_thread.ReraiserThread(_SetUp,
[runner_factory, d, runners, counter],
name=d[-4:]) for d in devices
])
threads.StartAll()
threads.JoinAll(watchdog_timer.WatchdogTimer(timeout))
return runners
def _JoinAll(self, watcher=None):
"""Join all threads without stack dumps.
Reraises exceptions raised by the child threads and supports breaking
immediately on exceptions raised on the main thread.
Args:
watcher: Watchdog object providing timeout, by default waits forever.
"""
if watcher is None:
watcher = watchdog_timer.WatchdogTimer(None)
alive_threads = self._threads[:]
while alive_threads:
for thread in alive_threads[:]:
if watcher.IsTimedOut():
raise TimeoutError(
'Timed out waiting for %d of %d threads.' %
(len(alive_threads), len(self._threads)))
# Allow the main thread to periodically check for interrupts.
thread.join(0.1)
if not thread.isAlive():
alive_threads.remove(thread)
# All threads are allowed to complete before reraising exceptions.
for thread in self._threads:
thread.ReraiseIfException()
def _RunAllTests(runners, tests, num_retries, timeout=None):
"""Run all tests using the given TestRunners.
Args:
runners: a list of TestRunner objects.
tests: a list of Tests to run using the given TestRunners.
num_retries: number of retries for a test.
timeout: watchdog timeout in seconds, defaults to the default timeout.
Returns:
A TestRunResults object.
"""
logging.warning('Running %s tests with %s test runners.' %
(len(tests), len(runners)))
tests_collection = _TestCollection([_Test(t) for t in tests])
results = []
watcher = watchdog_timer.WatchdogTimer(timeout)
workers = reraiser_thread.ReraiserThreadGroup([
reraiser_thread.ReraiserThread(
_RunTestsFromQueue,
[r, tests_collection, results, watcher, num_retries],
name=r.device[-4:]) for r in runners
])
workers.StartAll()
workers.JoinAll(watcher)
run_results = base_test_result.TestRunResults()
for r in results:
run_results.AddTestRunResults(r)
return run_results
def Run(func, timeout, retries, args=[], kwargs={}):
"""Runs the passed function in a separate thread with timeouts and retries.
Args:
func: the function to be wrapped.
timeout: the timeout in seconds for each try.
retries: the number of retries.
args: list of positional args to pass to |func|.
kwargs: dictionary of keyword args to pass to |func|.
Returns:
The return value of func(*args, **kwargs).
"""
# The return value uses a list because Python variables are references, not
# values. Closures make a copy of the reference, so updating the closure's
# reference wouldn't update where the original reference pointed.
ret = [None]
def RunOnTimeoutThread():
ret[0] = func(*args, **kwargs)
while True:
try:
name = 'TimeoutThread-for-%s' % threading.current_thread().name
thread_group = reraiser_thread.ReraiserThreadGroup([
reraiser_thread.ReraiserThread(RunOnTimeoutThread, name=name)
])
thread_group.StartAll()
thread_group.JoinAll(watchdog_timer.WatchdogTimer(timeout))
return ret[0]
except:
if retries <= 0:
raise
retries -= 1
def _RunAllTests(runners,
test_collection_factory,
num_retries,
timeout=None,
tag_results_with_device=False):
"""Run all tests using the given TestRunners.
Args:
runners: A list of TestRunner objects.
test_collection_factory: A callable to generate a _TestCollection object for
each test runner.
num_retries: Number of retries for a test.
timeout: Watchdog timeout in seconds.
tag_results_with_device: If True, appends the name of the device on which
the test was run to the test name. Used when replicating to identify
which device ran each copy of the test, and to ensure each copy of the
test is recorded separately.
Returns:
A tuple of (TestRunResults object, exit code)
"""
logging.warning('Running tests with %s test runners.' % (len(runners)))
results = []
exit_code = 0
run_results = base_test_result.TestRunResults()
watcher = watchdog_timer.WatchdogTimer(timeout)
test_collections = [test_collection_factory() for _ in runners]
threads = [
reraiser_thread.ReraiserThread(
_RunTestsFromQueue,
[r, tc, results, watcher, num_retries, tag_results_with_device],
name=r.device_serial[-4:])
for r, tc in zip(runners, test_collections)
]
workers = reraiser_thread.ReraiserThreadGroup(threads)
workers.StartAll()
# Catch DeviceUnreachableErrors and set a warning exit code
try:
workers.JoinAll(watcher)
except (
adb_wrapper.DeviceUnreachableError,
# TODO(jbudorick) Remove this once the underlying implementations
# for the above are switched or wrapped.
android_commands.errors.DeviceUnresponsiveError) as e:
logging.error(e)
exit_code = constants.WARNING_EXIT_CODE
assert all([len(tc) == 0 for tc in test_collections]), (
'Some tests were not run, all devices are likely offline (ran %d tests)'
% len(run_results.GetAll()))
for r in results:
run_results.AddTestRunResults(r)
if not run_results.DidRunPass():
exit_code = constants.ERROR_EXIT_CODE
return (run_results, exit_code)
def _RunTests(mock_runner, tests):
results = []
tests = shard._TestCollection([shard._Test(t) for t in tests])
shard._RunTestsFromQueue(mock_runner, tests, results,
watchdog_timer.WatchdogTimer(None), 2)
run_results = base_test_result.TestRunResults()
for r in results:
run_results.AddTestRunResults(r)
return run_results
def GetAllReturnValues(self, watcher=watchdog_timer.WatchdogTimer(None)):
"""Get all return values, joining all threads if necessary.
Args:
watcher: same as in |JoinAll|. Only used if threads are alive.
"""
if any([t.isAlive() for t in self._threads]):
self.JoinAll(watcher)
return [t.GetReturnValue() for t in self._threads]
def _RunAllTests(runners, test_collection_factory, num_retries, timeout=None,
tag_results_with_device=False):
"""Run all tests using the given TestRunners.
Args:
runners: A list of TestRunner objects.
test_collection_factory: A callable to generate a TestCollection object for
each test runner.
num_retries: Number of retries for a test.
timeout: Watchdog timeout in seconds.
tag_results_with_device: If True, appends the name of the device on which
the test was run to the test name. Used when replicating to identify
which device ran each copy of the test, and to ensure each copy of the
test is recorded separately.
Returns:
A tuple of (TestRunResults object, exit code)
"""
logging.warning('Running tests with %s test runners.' % (len(runners)))
results = []
exit_code = 0
run_results = base_test_result.TestRunResults()
watcher = watchdog_timer.WatchdogTimer(timeout)
test_collections = [test_collection_factory() for _ in runners]
threads = [
reraiser_thread.ReraiserThread(
_RunTestsFromQueue,
[r, tc, results, watcher, num_retries, tag_results_with_device],
name=r.device_serial[-4:])
for r, tc in zip(runners, test_collections)]
workers = reraiser_thread.ReraiserThreadGroup(threads)
workers.StartAll()
try:
workers.JoinAll(watcher)
except device_errors.CommandFailedError:
logging.exception('Command failed on device.')
except device_errors.CommandFailedError:
logging.exception('Command timed out on device.')
except device_errors.DeviceUnreachableError:
logging.exception('Device became unreachable.')
if not all((len(tc) == 0 for tc in test_collections)):
logging.error('Only ran %d tests (all devices are likely offline).' %
len(results))
for tc in test_collections:
run_results.AddResults(base_test_result.BaseTestResult(
t, base_test_result.ResultType.UNKNOWN) for t in tc.test_names())
for r in results:
run_results.AddTestRunResults(r)
if not run_results.DidRunPass():
exit_code = constants.ERROR_EXIT_CODE
return (run_results, exit_code)
def _TearDownRunners(runners, timeout=None):
"""Calls TearDown() for each test runner in parallel.
Args:
runners: A list of TestRunner objects.
timeout: Watchdog timeout in seconds, defaults to the default timeout.
"""
threads = reraiser_thread.ReraiserThreadGroup(
[reraiser_thread.ReraiserThread(r.TearDown, name=r.device[-4:])
for r in runners])
threads.StartAll()
threads.JoinAll(watchdog_timer.WatchdogTimer(timeout))
def testJoinTimeout(self):
def f():
pass
event = threading.Event()
def g():
event.wait()
group = reraiser_thread.ReraiserThreadGroup(
[reraiser_thread.ReraiserThread(g),
reraiser_thread.ReraiserThread(f)])
group.StartAll()
with self.assertRaises(reraiser_thread.TimeoutError):
group.JoinAll(watchdog_timer.WatchdogTimer(0.01))
event.set()
def _RunAllTests(runners,
test_collection_factory,
num_retries,
timeout=None,
tag_results_with_device=False):
"""Run all tests using the given TestRunners.
Args:
runners: A list of TestRunner objects.
test_collection_factory: A callable to generate a _TestCollection object for
each test runner.
num_retries: Number of retries for a test.
timeout: Watchdog timeout in seconds.
tag_results_with_device: If True, appends the name of the device on which
the test was run to the test name. Used when replicating to identify
which device ran each copy of the test, and to ensure each copy of the
test is recorded separately.
Returns:
A tuple of (TestRunResults object, exit code)
"""
logging.warning('Running tests with %s test runners.' % (len(runners)))
results = []
exit_code = 0
watcher = watchdog_timer.WatchdogTimer(timeout)
workers = reraiser_thread.ReraiserThreadGroup([
reraiser_thread.ReraiserThread(_RunTestsFromQueue, [
r,
test_collection_factory(), results, watcher, num_retries,
tag_results_with_device
],
name=r.device[-4:]) for r in runners
])
run_results = base_test_result.TestRunResults()
workers.StartAll()
# Catch DeviceUnresponsiveErrors and set a warning exit code
try:
workers.JoinAll(watcher)
except android_commands.errors.DeviceUnresponsiveError as e:
logging.error(e)
exit_code = constants.WARNING_EXIT_CODE
for r in results:
run_results.AddTestRunResults(r)
if not run_results.DidRunPass():
exit_code = constants.ERROR_EXIT_CODE
return (run_results, exit_code)
def pFinish(self, timeout):
"""Finish any outstanding asynchronous operations.
Args:
timeout: The maximum number of seconds to wait for an individual
result to return, or None to wait forever.
Returns:
self, now emulating the return values.
"""
self._assertNoShadow('pFinish')
if isinstance(self._objs, reraiser_thread.ReraiserThreadGroup):
self._objs.JoinAll()
self._objs = self._objs.GetAllReturnValues(
watchdog_timer.WatchdogTimer(timeout))
return self
def JoinAll(self, watcher=watchdog_timer.WatchdogTimer(None)):
"""Join all threads.
Reraises exceptions raised by the child threads and supports breaking
immediately on exceptions raised on the main thread. Unfinished threads'
stacks will be logged on watchdog timeout.
Args:
watcher: Watchdog object providing timeout, by default waits forever.
"""
try:
self._JoinAll(watcher)
except TimeoutError:
for thread in (t for t in self._threads if t.isAlive()):
LogThreadStack(thread)
raise
def __init__(self, func, timeout, name):
super(TimeoutRetryThread, self).__init__(func, name=name)
self._watcher = watchdog_timer.WatchdogTimer(timeout)
self._expired = False
