def RunJavaTest(fname, suite, test, ports_to_forward):
device = android_commands.GetAttachedDevices()[0]
package_name = _GetPackageName(fname)
test = package_name + '.' + suite + '#' + test
java_test_runner = TestRunner(False, device, [test], False, False, False,
False, 0, ports_to_forward)
return java_test_runner.Run()
def _RunATestSuite(options):
"""Run a single test suite.
Helper for Dispatch() to allow stop/restart of the emulator across
test bundles. If using the emulator, we start it on entry and stop
it on exit.
Args:
options: options for running the tests.
Returns:
0 if successful, number of failing tests otherwise.
"""
attached_devices = []
buildbot_emulator = None
if options.use_emulator:
t = TimeProfile('Emulator launch')
buildbot_emulator = emulator.Emulator(options.fast_and_loose)
buildbot_emulator.Launch()
t.Stop()
attached_devices.append(buildbot_emulator.device)
else:
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
logging.critical('A device must be attached and online.')
return 1
test_results = RunTests(attached_devices[0],
options.test_suite,
options.gtest_filter,
options.test_arguments,
options.rebaseline,
options.timeout,
options.performance_test,
options.cleanup_test_files,
options.tool,
options.log_dump,
fast_and_loose=options.fast_and_loose,
annotate=options.annotate)
if buildbot_emulator:
buildbot_emulator.Shutdown()
# Another chance if we timed out? At this point It is safe(r) to
# run fast and loose since we just uploaded all the test data and
# binary.
if test_results.timed_out and options.repeat:
logging.critical('Timed out; repeating in fast_and_loose mode.')
options.fast_and_loose = True
options.repeat = options.repeat - 1
logging.critical('Repeats left: ' + str(options.repeat))
return _RunATestSuite(options)
return len(test_results.failed)
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
# map can't handle KeyboardInterrupt exception. It's a python bug.
# So use map_async instead.
async_results = pool.map_async(_ShardedTestRunnable, test_runners)
results_lists = async_results.get(999999)
test_results = TestResults.FromTestResults(results_lists)
# Re-check the attached devices for some devices may
# become offline
retry_devices = set(android_commands.GetAttachedDevices())
# Remove devices that had exceptions.
retry_devices -= TestResults.DeviceExceptions(results_lists)
# Retry on devices that didn't have any exception.
self.attached_devices = list(retry_devices)
if (retry == self.retries - 1 or len(self.attached_devices) == 0):
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
self.OnTestsCompleted(test_runners, final_results)
return final_results
def DispatchPythonTests(options):
"""Dispatches the Python tests. If there are multiple devices, use sharding.
Args:
options: command line options.
Returns:
A list of test results.
"""
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
raise FatalTestException('You have no devices attached or visible!')
if options.device:
attached_devices = [options.device]
test_collection = TestInfoCollection()
all_tests = _GetAllTests(options.python_test_root, options.official_build)
test_collection.AddTests(all_tests)
test_names = [t.qualified_name for t in all_tests]
logging.debug('All available tests: ' + str(test_names))
available_tests = test_collection.GetAvailableTests(
options.annotation, options.test_filter)
if not available_tests:
logging.warning('No Python tests to run with current args.')
return TestResults()
available_tests *= options.number_of_runs
test_names = [t.qualified_name for t in available_tests]
logging.debug('Final list of tests to run: ' + str(test_names))
# Copy files to each device before running any tests.
for device_id in attached_devices:
logging.debug('Pushing files to device %s', device_id)
apks = [
apk_info.ApkInfo(options.test_apk_path, options.test_apk_jar_path)
]
test_files_copier = run_java_tests.TestRunner(options, device_id, None,
False, 0, apks, [])
test_files_copier.CopyTestFilesOnce()
# Actually run the tests.
if (len(attached_devices) > 1 and not options.wait_for_debugger):
logging.debug('Sharding Python tests.')
sharder = PythonTestSharder(attached_devices, options.shard_retries,
available_tests)
test_results = sharder.RunShardedTests()
else:
logging.debug('Running Python tests serially.')
test_results = _RunPythonTests(available_tests, attached_devices[0])
return test_results
def Dispatch(options):
"""Dispatches the tests, sharding if possible.
If options.use_emulator is True, all tests will be run in a new emulator
instance.
Args:
options: options for running the tests.
Returns:
0 if successful, number of failing tests otherwise.
"""
if options.test_suite == 'help':
ListTestSuites()
return 0
buildbot_emulator = None
attached_devices = []
if options.use_xvfb:
xvfb = Xvfb()
xvfb.Start()
if options.use_emulator:
t = TimeProfile('Emulator launch')
buildbot_emulator = emulator.Emulator()
buildbot_emulator.Launch()
t.Stop()
attached_devices.append(buildbot_emulator.device)
else:
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
logging.critical('A device must be attached and online.')
return 1
test_results = RunTests(attached_devices[0], options.test_suite,
options.gtest_filter, options.test_arguments,
options.rebaseline, options.timeout,
options.performance_test,
options.cleanup_test_files, options.tool,
options.log_dump)
if buildbot_emulator:
buildbot_emulator.Shutdown()
if options.use_xvfb:
xvfb.Stop()
return len(test_results.failed)
def _RunATestSuite(options):
"""Run a single test suite.
Helper for Dispatch() to allow stop/restart of the emulator across
test bundles. If using the emulator, we start it on entry and stop
it on exit.
Args:
options: options for running the tests.
Returns:
0 if successful, number of failing tests otherwise.
"""
attached_devices = []
buildbot_emulators = []
if options.use_emulator:
for n in range(options.use_emulator):
t = TimeProfile('Emulator launch %d' % n)
buildbot_emulator = emulator.Emulator(options.fast_and_loose)
buildbot_emulator.Launch(kill_all_emulators=n == 0)
t.Stop()
buildbot_emulators.append(buildbot_emulator)
attached_devices.append(buildbot_emulator.device)
# Wait for all emulators to become available.
map(lambda buildbot_emulator:buildbot_emulator.ConfirmLaunch(),
buildbot_emulators)
else:
attached_devices = android_commands.GetAttachedDevices()
if not attached_devices:
logging.critical('A device must be attached and online.')
return 1
if (len(attached_devices) > 1 and options.test_suite and
not options.gtest_filter and not options.performance_test):
sharder = TestSharder(attached_devices, options.test_suite,
options.gtest_filter, options.test_arguments,
options.timeout, options.rebaseline,
options.performance_test,
options.cleanup_test_files, options.tool)
test_results = sharder.RunShardedTests()
else:
test_results = RunTests(attached_devices[0], options.test_suite,
options.gtest_filter, options.test_arguments,
options.rebaseline, options.timeout,
options.performance_test,
options.cleanup_test_files, options.tool,
options.log_dump,
options.apk,
annotate=options.annotate)
for buildbot_emulator in buildbot_emulators:
buildbot_emulator.Shutdown()
# Another chance if we timed out? At this point It is safe(r) to
# run fast and loose since we just uploaded all the test data and
# binary.
if test_results.timed_out and options.repeat:
logging.critical('Timed out; repeating in fast_and_loose mode.')
options.fast_and_loose = True
options.repeat = options.repeat - 1
logging.critical('Repeats left: ' + str(options.repeat))
return _RunATestSuite(options)
return len(test_results.failed)
def DispatchJavaTests(options, apks):
"""Dispatches Java tests onto connected device(s).
If possible, this method will attempt to shard the tests to
all connected devices. Otherwise, dispatch and run tests on one device.
Args:
options: Command line options.
apks: list of APKs to use.
Returns:
A TestResults object holding the results of the Java tests.
Raises:
FatalTestException: when there's no attached the devices.
"""
test_apk = apks[0]
if options.annotation:
available_tests = test_apk.GetAnnotatedTests(options.annotation)
if len(options.annotation
) == 1 and options.annotation[0] == 'SmallTest':
tests_without_annotation = [
m for m in test_apk.GetTestMethods()
if not test_apk.GetTestAnnotations(m)
and not apk_info.ApkInfo.IsPythonDrivenTest(m)
]
if tests_without_annotation:
tests_without_annotation.sort()
logging.warning(
'The following tests do not contain any annotation. '
'Assuming "SmallTest":\n%s',
'\n'.join(tests_without_annotation))
available_tests += tests_without_annotation
else:
available_tests = [
m for m in test_apk.GetTestMethods()
if not apk_info.ApkInfo.IsPythonDrivenTest(m)
]
coverage = os.environ.get('EMMA_INSTRUMENT') == 'true'
tests = []
if options.test_filter:
# |available_tests| are in adb instrument format: package.path.class#test.
filter_without_hash = options.test_filter.replace('#', '.')
tests = [
t for t in available_tests
if filter_without_hash in t.replace('#', '.')
]
else:
tests = available_tests
if not tests:
logging.warning('No Java tests to run with current args.')
return TestResults()
tests *= options.number_of_runs
attached_devices = android_commands.GetAttachedDevices()
test_results = TestResults()
if not attached_devices:
raise FatalTestException('You have no devices attached or visible!')
if options.device:
attached_devices = [options.device]
logging.info('Will run: %s', str(tests))
if len(attached_devices) > 1 and (coverage or options.wait_for_debugger):
logging.warning('Coverage / debugger can not be sharded, '
'using first available device')
attached_devices = attached_devices[:1]
sharder = TestSharder(attached_devices, options, tests, apks)
test_results = sharder.RunShardedTests()
return test_results
def RunShardedTests(self):
"""Runs the tests in all connected devices.
Returns:
A TestResults object.
"""
logging.warning('*' * 80)
logging.warning('Sharding in ' + str(len(self.attached_devices)) +
' devices.')
logging.warning('Note that the output is not synchronized.')
logging.warning('Look for the "Final result" banner in the end.')
logging.warning('*' * 80)
final_results = TestResults()
self._KillHostForwarder()
for retry in xrange(self.retries):
logging.warning('Try %d of %d', retry + 1, self.retries)
self.SetupSharding(self.tests)
test_runners = []
# Try to create N shards, and retrying on failure.
try:
for index, device in enumerate(self.attached_devices):
logging.warning('*' * 80)
logging.warning('Creating shard %d for %s', index, device)
logging.warning('*' * 80)
test_runner = self.CreateShardedTestRunner(device, index)
test_runners += [test_runner]
except errors.DeviceUnresponsiveError as e:
logging.critical('****Failed to create a shard: [%s]', e)
self.attached_devices.remove(device)
continue
logging.warning('Starting...')
pool = multiprocessing.Pool(len(self.attached_devices),
SetTestsContainer,
[BaseTestSharder.tests_container])
# map can't handle KeyboardInterrupt exception. It's a python bug.
# So use map_async instead.
async_results = pool.map_async(_ShardedTestRunnable, test_runners)
try:
results_lists = async_results.get(999999)
except errors.DeviceUnresponsiveError as e:
logging.critical('****Failed to run test: [%s]', e)
self.attached_devices = android_commands.GetAttachedDevices()
continue
test_results = TestResults.FromTestResults(results_lists)
# Re-check the attached devices for some devices may
# become offline
retry_devices = set(android_commands.GetAttachedDevices())
# Remove devices that had exceptions.
retry_devices -= TestResults.DeviceExceptions(results_lists)
# Retry on devices that didn't have any exception.
self.attached_devices = list(retry_devices)
if (retry == self.retries - 1 or len(self.attached_devices) == 0):
all_passed = final_results.ok + test_results.ok
final_results = test_results
final_results.ok = all_passed
break
else:
final_results.ok += test_results.ok
self.tests = []
for t in test_results.GetAllBroken():
self.tests += [t.name]
if not self.tests:
break
else:
# We ran out retries, possibly out of healthy devices.
# There's no recovery at this point.
raise Exception('Unrecoverable error while retrying test runs.')
self.OnTestsCompleted(test_runners, final_results)
self._KillHostForwarder()
return final_results
