def RunShardedTests(self):
"""Runs tests in parallel using a pool of workers.
Returns:
A list of test results aggregated from all test runs.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
all_passed = []
test_results = TestResults()
tests_to_run = self.tests
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self._SetupSharding(self.tests)
test_runners = self._MakeTestRunners(self.attached_devices)
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[PythonTestSharder.tests_container])
# List of TestResults objects from each test execution.
try:
results_lists = pool.map(_DefaultRunnable, test_runners)
except Exception:
logging.exception('Unable to run tests. Something with the '
'PythonTestRunners has gone wrong.')
raise FatalTestException(
'PythonTestRunners were unable to run tests.')
test_results = TestResults.FromTestResults(results_lists)
# Accumulate passing results.
all_passed += test_results.ok
# If we have failed tests, map them to tests to retry.
failed_tests = test_results.GetAllBroken()
tests_to_run = self._GetTestsToRetry(self.tests, failed_tests)
# Bail out early if we have no more tests. This can happen if all tests
# pass before we're out of retries, for example.
if not tests_to_run:
break
final_results = TestResults()
# all_passed has accumulated all passing test results.
# test_results will have the results from the most recent run, which could
# include a variety of failure modes (unknown, crashed, failed, etc).
final_results = test_results
final_results.ok = all_passed
return final_results
def RunTests(self):
"""Runs tests from the shared pool of tests, aggregating results.
Returns:
A list of test results for all of the tests which this runner executed.
"""
tests = PythonTestSharder.tests_container
results = []
for t in tests:
res = CallPythonTest(t, self.options)
results.append(res)
return TestResults.FromTestResults(results)
def DispatchInstrumentationTests(options):
"""Dispatches the Java and Python instrumentation tests, sharding if possible.
Uses the logging module to print the combined final results and
summary of the Java and Python tests. If the java_only option is set, only
the Java tests run. If the python_only option is set, only the python tests
run. If neither are set, run both Java and Python tests.
Args:
options: command-line options for running the Java and Python tests.
Returns:
An integer representing the number of broken tests.
"""
if not options.keep_test_server_ports:
# Reset the test port allocation. It's important to do it before starting
# to dispatch any tests.
if not ports.ResetTestServerPortAllocation():
raise Exception('Failed to reset test server port.')
start_date = int(time.time() * 1000)
java_results = TestResults()
python_results = TestResults()
if options.run_java_tests:
java_results = run_java_tests.DispatchJavaTests(
options, [
apk_info.ApkInfo(options.test_apk_path,
options.test_apk_jar_path)
])
if options.run_python_tests:
python_results = run_python_tests.DispatchPythonTests(options)
all_results = TestResults.FromTestResults([java_results, python_results])
all_results.LogFull(test_type='Instrumentation',
test_package=options.test_apk,
annotation=options.annotation,
build_type=options.build_type,
flakiness_server=options.flakiness_dashboard_server)
return len(all_results.GetAllBroken())
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
self._KillHostForwarder()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
logging.warning('Attempting to run %d tests: %s'
% (len(self.tests), self.tests))
self.SetupSharding(self.tests)
test_runners = []
# Try to create N shards, and retrying on failure.
try:
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
except android_commands.errors.DeviceUnresponsiveError as e:
logging.critical('****Failed to create a shard: [%s]', e)
self.attached_devices.remove(device)
continue
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
# map can't handle KeyboardInterrupt exception. It's a python bug.
# So use map_async instead.
async_results = pool.map_async(_ShardedTestRunnable, test_runners)
try:
results_lists = async_results.get(999999)
except android_commands.errors.DeviceUnresponsiveError as e:
logging.critical('****Failed to run test: [%s]', e)
self.attached_devices = android_commands.GetAttachedDevices()
continue
test_results = TestResults.FromTestResults(results_lists)
# Re-check the attached devices for some devices may
# become offline
retry_devices = set(android_commands.GetAttachedDevices())
# Remove devices that had exceptions.
retry_devices -= TestResults.DeviceExceptions(results_lists)
# Retry on devices that didn't have any exception.
self.attached_devices = list(retry_devices)
# TODO(frankf): Do not break TestResults encapsulation.
if (retry == self.retries - 1 or
len(self.attached_devices) == 0):
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
else:
# We ran out retries, possibly out of healthy devices.
# There's no recovery at this point.
raise Exception('Unrecoverable error while retrying test runs.')
self._KillHostForwarder()
return final_results
