def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline,
timeout, performance_test, cleanup_test_files, tool,
log_dump_name):
"""Runs the tests.
Args:
device: Device to run the tests.
test_suite: A specific test suite to run, empty to run all.
gtest_filter: A gtest_filter flag.
test_arguments: Additional arguments to pass to the test binary.
rebaseline: Whether or not to run tests in isolation and update the filter.
timeout: Timeout for each test.
performance_test: Whether or not performance test(s).
cleanup_test_files: Whether or not to cleanup test files on device.
tool: Name of the Valgrind tool.
log_dump_name: Name of log dump file.
Returns:
A TestResults object.
"""
results = []
if test_suite:
global _TEST_SUITES
if not os.path.exists(test_suite):
logging.critical('Unrecognized test suite, supported: %s' %
_TEST_SUITES)
if test_suite in _TEST_SUITES:
logging.critical('(Remember to include the path: out/Release/%s)',
test_suite)
return TestResults.FromOkAndFailed([], [BaseTestResult(test_suite, '')])
_TEST_SUITES = [test_suite]
else:
# If not specified, assume the test suites are in out/Release
test_suite_dir = os.path.abspath(os.path.join(run_tests_helper.CHROME_DIR,
'out', 'Release'))
_TEST_SUITES = [os.path.join(test_suite_dir, t) for t in _TEST_SUITES]
debug_info_list = []
for t in _TEST_SUITES:
test = SingleTestRunner(device, t, gtest_filter, test_arguments,
timeout, rebaseline, performance_test,
cleanup_test_files, tool, not not log_dump_name)
test.RunTests()
results += [test.test_results]
# Collect debug info.
debug_info_list += [test.dump_debug_info]
if rebaseline:
test.UpdateFilter(test.test_results.failed)
elif test.test_results.failed:
# Stop running test if encountering failed test.
test.test_results.LogFull()
break
# Zip all debug info outputs into a file named by log_dump_name.
debug_info.GTestDebugInfo.ZipAndCleanResults(
os.path.join(run_tests_helper.CHROME_DIR, 'out', 'Release',
'debug_info_dumps'),
log_dump_name, [d for d in debug_info_list if d])
return TestResults.FromTestResults(results)
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
# map can't handle KeyboardInterrupt exception. It's a python bug.
# So use map_async instead.
async_results = pool.map_async(_ShardedTestRunnable, test_runners)
results_lists = async_results.get(999999)
test_results = TestResults.FromTestResults(results_lists)
# Re-check the attached devices for some devices may
# become offline
retry_devices = set(android_commands.GetAttachedDevices())
# Remove devices that had exceptions.
retry_devices -= TestResults.DeviceExceptions(results_lists)
# Retry on devices that didn't have any exception.
self.attached_devices = list(retry_devices)
if (retry == self.retries - 1 or len(self.attached_devices) == 0):
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
self.OnTestsCompleted(test_runners, final_results)
return final_results
def RunShardedTests(self):
"""Runs tests in parallel using a pool of workers.
Returns:
A list of test results aggregated from all test runs.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
all_passed = []
test_results = TestResults()
tests_to_run = self.tests
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self._SetupSharding(self.tests)
test_runners = self._MakeTestRunners(self.attached_devices)
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[PythonTestSharder.tests_container])
# List of TestResults objects from each test execution.
try:
results_lists = pool.map(_DefaultRunnable, test_runners)
except Exception:
logging.exception('Unable to run tests. Something with the '
'PythonTestRunners has gone wrong.')
raise FatalTestException(
'PythonTestRunners were unable to run tests.')
test_results = TestResults.FromTestResults(results_lists)
# Accumulate passing results.
all_passed += test_results.ok
# If we have failed tests, map them to tests to retry.
failed_tests = test_results.GetAllBroken()
tests_to_run = self._GetTestsToRetry(self.tests, failed_tests)
# Bail out early if we have no more tests. This can happen if all tests
# pass before we're out of retries, for example.
if not tests_to_run:
break
final_results = TestResults()
# all_passed has accumulated all passing test results.
# test_results will have the results from the most recent run, which could
# include a variety of failure modes (unknown, crashed, failed, etc).
final_results = test_results
final_results.ok = all_passed
return final_results
def RunTests(self):
"""Runs tests from the shared pool of tests, aggregating results.
Returns:
A list of test results for all of the tests which this runner executed.
"""
tests = PythonTestSharder.tests_container
results = []
for t in tests:
res = CallPythonTest(t, self.device_id, self.shard_index)
results.append(res)
return TestResults.FromTestResults(results)
def _RunPythonTests(tests_to_run, device_id):
"""Runs a list of Python tests serially on one device and returns results.
Args:
tests_to_run: a list of objects inheriting from PythonTestBase.
device_id: ID of the device to run tests on.
Returns:
A list of test results, aggregated across all the tests run.
"""
# This is a list of TestResults objects.
results = [CallPythonTest(t, device_id, 0) for t in tests_to_run]
# Merge the list of TestResults into one TestResults.
return TestResults.FromTestResults(results)
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
results_lists = pool.map(_ShardedTestRunnable, test_runners)
test_results = TestResults.FromTestResults(results_lists)
if retry == self.retries - 1:
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
self.OnTestsCompleted(test_runners, final_results)
return final_results
def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline,
timeout, performance_test, cleanup_test_files, tool,
log_dump_name, apk, annotate=False):
"""Runs the tests.
Args:
device: Device to run the tests.
test_suite: A specific test suite to run, empty to run all.
gtest_filter: A gtest_filter flag.
test_arguments: Additional arguments to pass to the test binary.
rebaseline: Whether or not to run tests in isolation and update the filter.
timeout: Timeout for each test.
performance_test: Whether or not performance test(s).
cleanup_test_files: Whether or not to cleanup test files on device.
tool: Name of the Valgrind tool.
log_dump_name: Name of log dump file.
apk: boolean to state if we are using the apk based test runner
annotate: should we print buildbot-style annotations?
Returns:
A TestResults object.
"""
results = []
if test_suite:
global _TEST_SUITES
if (not os.path.exists(test_suite) and
not os.path.splitext(test_suite)[1] == '.apk'):
logging.critical('Unrecognized test suite %s, supported: %s' %
(test_suite, _TEST_SUITES))
if test_suite in _TEST_SUITES:
logging.critical('(Remember to include the path: out/Release/%s)',
test_suite)
return TestResults.FromOkAndFailed([], [BaseTestResult(test_suite, '')],
False, False)
fully_qualified_test_suites = [test_suite]
else:
fully_qualified_test_suites = FullyQualifiedTestSuites(apk)
debug_info_list = []
print 'Known suites: ' + str(_TEST_SUITES)
print 'Running these: ' + str(fully_qualified_test_suites)
for t in fully_qualified_test_suites:
if annotate:
print '@@@BUILD_STEP Test suite %s@@@' % os.path.basename(t)
test = SingleTestRunner(device, t, gtest_filter, test_arguments,
timeout, rebaseline, performance_test,
cleanup_test_files, tool, 0, not not log_dump_name)
test.Run()
results += [test.test_results]
# Collect debug info.
debug_info_list += [test.dump_debug_info]
if rebaseline:
test.UpdateFilter(test.test_results.failed)
test.test_results.LogFull()
# Zip all debug info outputs into a file named by log_dump_name.
debug_info.GTestDebugInfo.ZipAndCleanResults(
os.path.join(run_tests_helper.CHROME_DIR, 'out', 'Release',
'debug_info_dumps'),
log_dump_name, [d for d in debug_info_list if d])
if annotate:
if test.test_results.timed_out:
print '@@@STEP_WARNINGS@@@'
elif test.test_results.failed:
print '@@@STEP_FAILURE@@@'
elif test.test_results.overall_fail:
print '@@@STEP_FAILURE@@@'
else:
print 'Step success!' # No annotation needed
return TestResults.FromTestResults(results)
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
self._KillHostForwarder()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
# Try to create N shards, and retrying on failure.
try:
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
except errors.DeviceUnresponsiveError as e:
logging.critical('****Failed to create a shard: [%s]', e)
self.attached_devices.remove(device)
continue
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
# map can't handle KeyboardInterrupt exception. It's a python bug.
# So use map_async instead.
async_results = pool.map_async(_ShardedTestRunnable, test_runners)
try:
results_lists = async_results.get(999999)
except errors.DeviceUnresponsiveError as e:
logging.critical('****Failed to run test: [%s]', e)
self.attached_devices = android_commands.GetAttachedDevices()
continue
test_results = TestResults.FromTestResults(results_lists)
# Re-check the attached devices for some devices may
# become offline
retry_devices = set(android_commands.GetAttachedDevices())
# Remove devices that had exceptions.
retry_devices -= TestResults.DeviceExceptions(results_lists)
# Retry on devices that didn't have any exception.
self.attached_devices = list(retry_devices)
if (retry == self.retries - 1 or len(self.attached_devices) == 0):
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
else:
# We ran out retries, possibly out of healthy devices.
# There's no recovery at this point.
raise Exception('Unrecoverable error while retrying test runs.')
self.OnTestsCompleted(test_runners, final_results)
self._KillHostForwarder()
return final_results
