def _RunAllTests(runners, tests, num_retries, timeout=None):
"""Run all tests using the given TestRunners.
Args:
runners: a list of TestRunner objects.
tests: a list of Tests to run using the given TestRunners.
num_retries: number of retries for a test.
timeout: watchdog timeout in seconds, defaults to the default timeout.
Returns:
A TestRunResults object.
"""
logging.warning('Running %s tests with %s test runners.' %
(len(tests), len(runners)))
tests_collection = _TestCollection([_Test(t) for t in tests])
results = []
watcher = watchdog_timer.WatchdogTimer(timeout)
workers = reraiser_thread.ReraiserThreadGroup([
reraiser_thread.ReraiserThread(
_RunTestsFromQueue,
[r, tests_collection, results, watcher, num_retries],
name=r.device[-4:]) for r in runners
])
workers.StartAll()
workers.JoinAll(watcher)
run_results = base_test_result.TestRunResults()
for r in results:
run_results.AddTestRunResults(r)
return run_results
def RunTests(tests,
runner_factory,
wait_for_debugger,
test_device,
shard=True,
build_type='Debug',
test_timeout=DEFAULT_TIMEOUT,
setup_timeout=DEFAULT_TIMEOUT,
num_retries=2):
"""Run all tests on attached devices, retrying tests that don't pass.
Args:
tests: List of tests to run.
runner_factory: Callable that takes a device and index and returns a
TestRunner object.
wait_for_debugger: True if this test is using a debugger.
test_device: A specific device to run tests on, or None.
shard: True if we should shard, False if we should replicate tests.
- Sharding tests will distribute tests across all test runners through a
shared test collection.
- Replicating tests will copy all tests to each test runner through a
unique test collection for each test runner.
build_type: Either 'Debug' or 'Release'.
test_timeout: Watchdog timeout in seconds for running tests.
setup_timeout: Watchdog timeout in seconds for creating and cleaning up
test runners.
num_retries: Number of retries for a test.
Returns:
A tuple of (base_test_result.TestRunResults object, exit code).
"""
if not tests:
logging.error('No tests to run.')
return (base_test_result.TestRunResults(), constants.ERROR_EXIT_CODE)
if shard:
# Generate a shared _TestCollection object for all test runners, so they
# draw from a common pool of tests.
shared_test_collection = _TestCollection([_Test(t) for t in tests])
test_collection_factory = lambda: shared_test_collection
tag_results_with_device = False
else:
# Generate a unique _TestCollection object for each test runner, but use
# the same set of tests.
test_collection_factory = lambda: _TestCollection(
[_Test(t) for t in tests])
tag_results_with_device = True
devices = _GetAttachedDevices(wait_for_debugger, test_device)
logging.info('Will run %d tests: %s', len(tests), str(tests))
runners = _CreateRunners(runner_factory, devices, setup_timeout)
try:
return _RunAllTests(runners, test_collection_factory, num_retries,
test_timeout, tag_results_with_device)
finally:
try:
_TearDownRunners(runners, setup_timeout)
except android_commands.errors.DeviceUnresponsiveError as e:
logging.warning('Device unresponsive during TearDown: [%s]', e)
def _RunTests(mock_runner, tests):
results = []
tests = shard._TestCollection([shard._Test(t) for t in tests])
shard._RunTestsFromQueue(mock_runner, tests, results,
watchdog_timer.WatchdogTimer(None), 2)
run_results = base_test_result.TestRunResults()
for r in results:
run_results.AddTestRunResults(r)
return run_results
def RunTest(self, test):
self._fails += 1
results = base_test_result.TestRunResults()
if self._fails <= 2:
results.AddResult(base_test_result.BaseTestResult(
test, base_test_result.ResultType.FAIL))
return (results, test)
else:
results.AddResult(base_test_result.BaseTestResult(
test, base_test_result.ResultType.PASS))
return (results, None)
def TagTestRunResults(test_run_results):
"""Tags all results with the last 4 digits of the device id.
Used when replicating tests to distinguish the same tests run on different
devices. We use a set to store test results, so the hash (generated from
name and tag) must be unique to be considered different results.
"""
new_test_run_results = base_test_result.TestRunResults()
for test_result in test_run_results.GetAll():
test_result.SetName('%s_%s' % (runner.device[-4:], test_result.GetName()))
new_test_run_results.AddResult(test_result)
return new_test_run_results
def _RunAllTests(runners, test_collection_factory, num_retries, timeout=None,
tag_results_with_device=False):
"""Run all tests using the given TestRunners.
Args:
runners: A list of TestRunner objects.
test_collection_factory: A callable to generate a _TestCollection object for
each test runner.
num_retries: Number of retries for a test.
timeout: Watchdog timeout in seconds.
tag_results_with_device: If True, appends the name of the device on which
the test was run to the test name. Used when replicating to identify
which device ran each copy of the test, and to ensure each copy of the
test is recorded separately.
Returns:
A tuple of (TestRunResults object, exit code)
"""
logging.warning('Running tests with %s test runners.' % (len(runners)))
results = []
exit_code = 0
run_results = base_test_result.TestRunResults()
watcher = watchdog_timer.WatchdogTimer(timeout)
test_collections = [test_collection_factory() for _ in runners]
threads = [
reraiser_thread.ReraiserThread(
_RunTestsFromQueue,
[r, tc, results, watcher, num_retries, tag_results_with_device],
name=r.device[-4:])
for r, tc in zip(runners, test_collections)]
workers = reraiser_thread.ReraiserThreadGroup(threads)
workers.StartAll()
# Catch DeviceUnresponsiveErrors and set a warning exit code
try:
workers.JoinAll(watcher)
except android_commands.errors.DeviceUnresponsiveError as e:
logging.error(e)
exit_code = constants.WARNING_EXIT_CODE
assert all([len(tc) == 0 for tc in test_collections]), (
'Some tests were not run, all devices are likely offline (ran %d tests)' %
len(run_results.GetAll()))
for r in results:
run_results.AddTestRunResults(r)
if not run_results.DidRunPass():
exit_code = constants.ERROR_EXIT_CODE
return (run_results, exit_code)
def _RunTestsFromQueue(runner, test_collection, out_results, watcher,
num_retries):
"""Runs tests from the test_collection until empty using the given runner.
Adds TestRunResults objects to the out_results list and may add tests to the
out_retry list.
Args:
runner: A TestRunner object used to run the tests.
test_collection: A _TestCollection from which to get _Test objects to run.
out_results: A list to add TestRunResults to.
watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout.
num_retries: Number of retries for a test.
"""
for test in test_collection:
watcher.Reset()
try:
if not android_commands.IsDeviceAttached(runner.device):
# Device is unresponsive, stop handling tests on this device.
msg = 'Device %s is unresponsive.' % runner.device
logging.warning(msg)
raise android_commands.errors.DeviceUnresponsiveError(msg)
result, retry = runner.RunTest(test.test)
test.tries += 1
if retry and test.tries <= num_retries:
# Retry non-passing results, only record passing results.
pass_results = base_test_result.TestRunResults()
pass_results.AddResults(result.GetPass())
out_results.append(pass_results)
logging.warning('Will retry test, try #%s.' % test.tries)
test_collection.add(_Test(test=retry, tries=test.tries))
else:
# All tests passed or retry limit reached. Either way, record results.
out_results.append(result)
except android_commands.errors.DeviceUnresponsiveError:
# Device is unresponsive, stop handling tests on this device and ensure
# current test gets runs by another device. Don't reraise this exception
# on the main thread.
test_collection.add(test)
return
except:
# An unhandleable exception, ensure tests get run by another device and
# reraise this exception on the main thread.
test_collection.add(test)
raise
finally:
# Retries count as separate tasks so always mark the popped test as done.
test_collection.test_completed()
def ShardAndRunTests(runner_factory,
devices,
tests,
build_type='Debug',
test_timeout=DEFAULT_TIMEOUT,
setup_timeout=DEFAULT_TIMEOUT,
num_retries=2):
"""Run all tests on attached devices, retrying tests that don't pass.
Args:
runner_factory: callable that takes a device and index and returns a
TestRunner object.
devices: list of attached device serial numbers as strings.
tests: list of tests to run.
build_type: either 'Debug' or 'Release'.
test_timeout: watchdog timeout in seconds for running tests, defaults to the
default timeout.
setup_timeout: watchdog timeout in seconds for creating and cleaning up
test runners, defaults to the default timeout.
num_retries: number of retries for a test.
Returns:
A base_test_result.TestRunResults object.
"""
if not tests:
logging.warning('No tests to run.')
return base_test_result.TestRunResults()
logging.info('Will run %d tests: %s', len(tests), str(tests))
forwarder.Forwarder.KillHost(build_type)
runners = _CreateRunners(runner_factory, devices, setup_timeout)
try:
return _RunAllTests(runners, tests, num_retries, test_timeout)
finally:
try:
_TearDownRunners(runners, setup_timeout)
except android_commands.errors.DeviceUnresponsiveError as e:
logging.warning('Device unresponsive during TearDown: [%s]', e)
finally:
forwarder.Forwarder.KillHost(build_type)
def _RunTestsFromQueue(runner, test_collection, out_results, watcher,
num_retries, tag_results_with_device=False):
"""Runs tests from the test_collection until empty using the given runner.
Adds TestRunResults objects to the out_results list and may add tests to the
out_retry list.
Args:
runner: A TestRunner object used to run the tests.
test_collection: A _TestCollection from which to get _Test objects to run.
out_results: A list to add TestRunResults to.
watcher: A watchdog_timer.WatchdogTimer object, used as a shared timeout.
num_retries: Number of retries for a test.
tag_results_with_device: If True, appends the name of the device on which
the test was run to the test name. Used when replicating to identify
which device ran each copy of the test, and to ensure each copy of the
test is recorded separately.
"""
def TagTestRunResults(test_run_results):
"""Tags all results with the last 4 digits of the device id.
Used when replicating tests to distinguish the same tests run on different
devices. We use a set to store test results, so the hash (generated from
name and tag) must be unique to be considered different results.
"""
new_test_run_results = base_test_result.TestRunResults()
for test_result in test_run_results.GetAll():
test_result.SetName('%s_%s' % (runner.device[-4:], test_result.GetName()))
new_test_run_results.AddResult(test_result)
return new_test_run_results
for test in test_collection:
watcher.Reset()
try:
if not android_commands.IsDeviceAttached(runner.device):
# Device is unresponsive, stop handling tests on this device.
msg = 'Device %s is unresponsive.' % runner.device
logging.warning(msg)
raise android_commands.errors.DeviceUnresponsiveError(msg)
result, retry = runner.RunTest(test.test)
if tag_results_with_device:
result = TagTestRunResults(result)
test.tries += 1
if retry and test.tries <= num_retries:
# Retry non-passing results, only record passing results.
pass_results = base_test_result.TestRunResults()
pass_results.AddResults(result.GetPass())
out_results.append(pass_results)
logging.warning('Will retry test, try #%s.' % test.tries)
test_collection.add(_Test(test=retry, tries=test.tries))
else:
# All tests passed or retry limit reached. Either way, record results.
out_results.append(result)
except:
# An unhandleable exception, ensure tests get run by another device and
# reraise this exception on the main thread.
test_collection.add(test)
raise
finally:
# Retries count as separate tasks so always mark the popped test as done.
test_collection.test_completed()
def RunTest(self, test):
results = base_test_result.TestRunResults()
results.AddResult(
base_test_result.BaseTestResult(test,
base_test_result.ResultType.FAIL))
return (results, test)
