def individual_device_set_up(dev):
def install_apk():
# Install test APK.
self._delegate.Install(dev)
def push_test_data():
# Push data dependencies.
external_storage = dev.GetExternalStoragePath()
data_deps = self._test_instance.GetDataDependencies()
host_device_tuples = [
(h, d if d is not None else external_storage)
for h, d in data_deps
]
dev.PushChangedFiles(host_device_tuples)
def init_tool_and_start_servers():
tool = self.GetTool(dev)
tool.CopyFiles(dev)
tool.SetupEnvironment()
self._servers[str(dev)] = []
if self.TestPackage() in _SUITE_REQUIRES_TEST_SERVER_SPAWNER:
self._servers[str(dev)].append(
local_test_server_spawner.LocalTestServerSpawner(
ports.AllocateTestServerPort(), dev, tool))
for s in self._servers[str(dev)]:
s.SetUp()
steps = (install_apk, push_test_data, init_tool_and_start_servers)
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
def RunTests(self):
# Affinitize the tests.
if self._test_instance.trace_output:
assert not trace_event.trace_is_enabled(
), 'Tracing already running.'
trace_event.trace_enable(self._test_instance.trace_output +
'.json')
self._SplitTestsByAffinity()
if not self._test_buckets and not self._no_device_tests:
raise local_device_test_run.NoTestsError()
def run_no_devices_tests():
if not self._no_device_tests:
return []
s = HostTestShard(self._env,
self._test_instance,
self._no_device_tests,
retries=3,
timeout=self._timeout)
return [s.RunTestsOnShard()]
def device_shard_helper(shard_id):
if device_status.IsBlacklisted(str(self._devices[shard_id]),
self._env.blacklist):
logging.warning(
'Device %s is not active. Will not create shard %s.',
str(self._devices[shard_id]), shard_id)
return None
s = DeviceTestShard(self._env,
self._test_instance,
self._devices[shard_id],
shard_id,
self._test_buckets[shard_id],
retries=self._env.max_tries,
timeout=self._timeout)
return s.RunTestsOnShard()
def run_devices_tests():
if not self._test_buckets:
return []
if self._devices is None:
self._devices = self._GetAllDevices(
self._env.devices, self._test_instance.known_devices_file)
device_indices = range(
min(len(self._devices), len(self._test_buckets)))
shards = parallelizer.Parallelizer(device_indices).pMap(
device_shard_helper)
return [x for x in shards.pGet(self._timeout) if x is not None]
host_test_results, device_test_results = reraiser_thread.RunAsync(
[run_no_devices_tests, run_devices_tests])
if self._test_instance.trace_output:
assert trace_event.trace_is_enabled(), 'Tracing not running.'
trace_event.trace_disable()
local_device_test_run.LocalDeviceTestRun._JsonToTrace(
self._test_instance.trace_output + '.json',
self._test_instance.trace_output)
return host_test_results + device_test_results
def individual_device_set_up(dev, host_device_tuples):
def install_apk():
if self._test_instance.apk_under_test_incremental_install_script:
local_device_test_run.IncrementalInstall(
dev, self._test_instance.apk_under_test,
self._test_instance.
apk_under_test_incremental_install_script)
else:
permissions = self._test_instance.apk_under_test.GetPermissions(
)
dev.Install(self._test_instance.apk_under_test,
permissions=permissions)
if self._test_instance.test_apk_incremental_install_script:
local_device_test_run.IncrementalInstall(
dev, self._test_instance.test_apk, self._test_instance.
test_apk_incremental_install_script)
else:
permissions = self._test_instance.test_apk.GetPermissions()
dev.Install(self._test_instance.test_apk,
permissions=permissions)
for apk in self._test_instance.additional_apks:
dev.Install(apk)
def push_test_data():
external_storage = dev.GetExternalStoragePath()
host_device_tuples_substituted = [
(h, substitute_external_storage(d, external_storage))
for h, d in host_device_tuples
]
logging.info('instrumentation data deps:')
for h, d in host_device_tuples_substituted:
logging.info('%r -> %r', h, d)
dev.PushChangedFiles(host_device_tuples_substituted)
def create_flag_changer():
if self._test_instance.flags:
if not self._test_instance.package_info:
logging.error("Couldn't set flags: no package info")
elif not self._test_instance.package_info.cmdline_file:
logging.error("Couldn't set flags: no cmdline_file")
else:
self._CreateFlagChangerIfNeeded(dev)
logging.debug('Attempting to set flags: %r',
self._test_instance.flags)
self._flag_changers[str(dev)].AddFlags(
self._test_instance.flags)
valgrind_tools.SetChromeTimeoutScale(
dev, self._test_instance.timeout_scale)
steps = (install_apk, push_test_data, create_flag_changer)
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
def individual_device_set_up(device, host_device_tuples):
def install_apk(dev):
# Install test APK.
self._delegate.Install(dev)
def push_test_data(dev):
# Push data dependencies.
device_root = self._delegate.GetTestDataRoot(dev)
host_device_tuples_substituted = [
(h,
local_device_test_run.SubstituteDeviceRoot(
d, device_root)) for h, d in host_device_tuples
]
local_device_environment.place_nomedia_on_device(
dev, device_root)
dev.PushChangedFiles(
host_device_tuples_substituted,
delete_device_stale=True,
# Some gtest suites, e.g. unit_tests, have data dependencies that
# can take longer than the default timeout to push. See
# crbug.com/791632 for context.
timeout=600)
if not host_device_tuples:
dev.RemovePath(device_root,
force=True,
recursive=True,
rename=True)
dev.RunShellCommand(['mkdir', '-p', device_root],
check_return=True)
def init_tool_and_start_servers(dev):
tool = self.GetTool(dev)
tool.CopyFiles(dev)
tool.SetupEnvironment()
self._servers[str(dev)] = []
if self.TestPackage() in _SUITE_REQUIRES_TEST_SERVER_SPAWNER:
self._servers[str(dev)].append(
local_test_server_spawner.LocalTestServerSpawner(
ports.AllocateTestServerPort(), dev, tool))
for s in self._servers[str(dev)]:
s.SetUp()
def bind_crash_handler(step, dev):
return lambda: crash_handler.RetryOnSystemCrash(step, dev)
steps = [
bind_crash_handler(s, device)
for s in (install_apk, push_test_data,
init_tool_and_start_servers)
]
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
def InnerFunc():
current_thread_group = timeout_retry.CurrentTimeoutThreadGroup()
self.assertIsNotNone(current_thread_group)
def InnerInnerFunc():
self.assertEqual(current_thread_group,
timeout_retry.CurrentTimeoutThreadGroup())
return True
return reraiser_thread.RunAsync((InnerInnerFunc,))[0]
def _Execute(concurrently, *funcs):
"""Calls all functions in |funcs| concurrently or in sequence."""
timer = time_profile.TimeProfile()
if concurrently:
reraiser_thread.RunAsync(funcs)
else:
for f in funcs:
f()
timer.Stop(log=False)
return timer
def RunTests(self):
def run_no_devices_tests():
if not self._no_device_tests:
return []
s = HostTestShard(self._env,
self._test_instance,
self._no_device_tests,
retries=3,
timeout=self._timeout)
return [s.RunTestsOnShard()]
def device_shard_helper(shard_id):
if device_status.IsBlacklisted(str(self._devices[shard_id]),
self._env.blacklist):
logging.warning(
'Device %s is not active. Will not create shard %s.',
str(self._devices[shard_id]), shard_id)
return None
s = DeviceTestShard(self._env,
self._test_instance,
self._devices[shard_id],
shard_id,
self._test_buckets[shard_id],
retries=self._env.max_tries,
timeout=self._timeout)
return s.RunTestsOnShard()
def run_devices_tests():
if not self._test_buckets:
return []
if self._devices is None:
self._devices = self._GetAllDevices(
self._env.devices, self._test_instance.known_devices_file)
device_indices = range(
min(len(self._devices), len(self._test_buckets)))
shards = parallelizer.Parallelizer(device_indices).pMap(
device_shard_helper)
return [x for x in shards.pGet(self._timeout) if x is not None]
# Run the tests.
with contextlib_ext.Optional(self._env.Tracing(),
self._env.trace_output):
# Affinitize the tests.
self._SplitTestsByAffinity()
if not self._test_buckets and not self._no_device_tests:
raise local_device_test_run.NoTestsError()
host_test_results, device_test_results = reraiser_thread.RunAsync(
[run_no_devices_tests, run_devices_tests])
return host_test_results + device_test_results
def individual_device_set_up(dev, host_device_tuples):
def install_apk(d):
# Install test APK.
self._delegate.Install(d)
def push_test_data():
# Push data dependencies.
device_root = self._delegate.GetTestDataRoot(dev)
host_device_tuples_substituted = [
(h,
local_device_test_run.SubstituteDeviceRoot(
d, device_root)) for h, d in host_device_tuples
]
dev.PushChangedFiles(host_device_tuples_substituted,
delete_device_stale=True)
if not host_device_tuples:
dev.RemovePath(device_root,
force=True,
recursive=True,
rename=True)
dev.RunShellCommand(['mkdir', '-p', device_root],
check_return=True)
def init_tool_and_start_servers():
tool = self.GetTool(dev)
tool.CopyFiles(dev)
tool.SetupEnvironment()
self._servers[str(dev)] = []
if self.TestPackage() in _SUITE_REQUIRES_TEST_SERVER_SPAWNER:
self._servers[str(dev)].append(
local_test_server_spawner.LocalTestServerSpawner(
ports.AllocateTestServerPort(), dev, tool))
for s in self._servers[str(dev)]:
s.SetUp()
steps = (
lambda: crash_handler.RetryOnSystemCrash(install_apk, dev),
push_test_data, init_tool_and_start_servers)
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
def RunTests(self, results):
def run_no_devices_tests():
if not self._no_device_tests:
return []
s = HostTestShard(self._env, self._test_instance, self._no_device_tests,
retries=3, timeout=self._timeout)
return [s.RunTestsOnShard()]
def device_shard_helper(shard_id):
if device_status.IsBlacklisted(
str(self._devices[shard_id]), self._env.blacklist):
logging.warning('Device %s is not active. Will not create shard %s.',
str(self._devices[shard_id]), shard_id)
return None
s = DeviceTestShard(self._env, self._test_instance,
self._devices[shard_id], shard_id,
self._test_buckets[shard_id],
retries=self._env.max_tries, timeout=self._timeout)
return s.RunTestsOnShard()
def run_devices_tests():
if not self._test_buckets:
return []
if self._devices is None:
self._devices = self._GetAllDevices(
self._env.devices, self._test_instance.known_devices_file)
device_indices = range(min(len(self._devices), len(self._test_buckets)))
shards = parallelizer.Parallelizer(device_indices).pMap(
device_shard_helper)
return [x for x in shards.pGet(self._timeout) if x is not None]
# Affinitize the tests.
self._SplitTestsByAffinity()
if not self._test_buckets and not self._no_device_tests:
raise local_device_test_run.NoTestsError()
host_test_results, device_test_results = reraiser_thread.RunAsync(
[run_no_devices_tests, run_devices_tests])
# Ideally, results would be populated as early as possible, so that in the
# event of an exception or timeout, the caller will still have partially
# populated results. This looks like it can be done prior to dispatching
# tests, but will hold off on making this change unless it looks like it
# might provide utility.
results.extend(host_test_results + device_test_results)
def testCrashDuringCommand(self):
self.device.EnableRoot()
with device_temp_file.DeviceTempFile(self.device.adb) as trigger_file:
trigger_text = 'hello world'
def victim():
trigger_cmd = 'echo -n %s > %s; sleep 20' % (
cmd_helper.SingleQuote(trigger_text),
cmd_helper.SingleQuote(trigger_file.name))
crash_handler.RetryOnSystemCrash(
lambda d: d.RunShellCommand(trigger_cmd,
shell=True,
check_return=True,
retries=1,
as_root=True,
timeout=180),
device=self.device)
self.assertEquals(
trigger_text,
self.device.ReadFile(trigger_file.name, retries=0).strip())
return True
def crasher():
def ready_to_crash():
try:
return trigger_text == self.device.ReadFile(
trigger_file.name, retries=0).strip()
except device_errors.CommandFailedError:
return False
timeout_retry.WaitFor(ready_to_crash,
wait_period=2,
max_tries=10)
if not ready_to_crash():
return False
self.device.adb.Shell('echo c > /proc/sysrq-trigger',
expect_status=None,
timeout=60,
retries=0)
return True
self.assertEquals([True, True],
reraiser_thread.RunAsync([crasher, victim]))
def individual_device_set_up(dev):
def install_apk():
# Install test APK.
self._delegate.Install(dev)
def push_test_data():
# Push data dependencies.
device_root = posixpath.join(dev.GetExternalStoragePath(),
'chromium_tests_root')
data_deps = self._test_instance.GetDataDependencies()
host_device_tuples = [(h, d if d is not None else device_root)
for h, d in data_deps]
dev.PushChangedFiles(host_device_tuples,
delete_device_stale=True)
if not host_device_tuples:
dev.RunShellCommand(['rm', '-rf', device_root],
check_return=True)
dev.RunShellCommand(['mkdir', '-p', device_root],
check_return=True)
def init_tool_and_start_servers():
tool = self.GetTool(dev)
tool.CopyFiles(dev)
tool.SetupEnvironment()
self._servers[str(dev)] = []
if self.TestPackage() in _SUITE_REQUIRES_TEST_SERVER_SPAWNER:
self._servers[str(dev)].append(
local_test_server_spawner.LocalTestServerSpawner(
ports.AllocateTestServerPort(), dev, tool))
for s in self._servers[str(dev)]:
s.SetUp()
steps = (install_apk, push_test_data, init_tool_and_start_servers)
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
def testTwoArgs(self):
a, b = reraiser_thread.RunAsync((lambda: 1, lambda: 2))
self.assertEqual(1, a)
self.assertEqual(2, b)
def testOneArg(self):
results = reraiser_thread.RunAsync([lambda: 1])
self.assertEqual([1], results)
def testNoArgs(self):
results = reraiser_thread.RunAsync([])
self.assertEqual([], results)
def individual_device_set_up(dev, host_device_tuples):
def install_apk():
if self._test_instance.apk_under_test:
if self._test_instance.apk_under_test_incremental_install_script:
local_device_test_run.IncrementalInstall(
dev, self._test_instance.apk_under_test,
self._test_instance.
apk_under_test_incremental_install_script)
else:
permissions = self._test_instance.apk_under_test.GetPermissions(
)
dev.Install(self._test_instance.apk_under_test,
permissions=permissions)
if self._test_instance.test_apk_incremental_install_script:
local_device_test_run.IncrementalInstall(
dev, self._test_instance.test_apk, self._test_instance.
test_apk_incremental_install_script)
else:
permissions = self._test_instance.test_apk.GetPermissions()
dev.Install(self._test_instance.test_apk,
permissions=permissions)
for apk in self._test_instance.additional_apks:
dev.Install(apk)
# Set debug app in order to enable reading command line flags on user
# builds
if self._test_instance.flags:
if not self._test_instance.package_info:
logging.error(
"Couldn't set debug app: no package info")
elif not self._test_instance.package_info.package:
logging.error(
"Couldn't set debug app: no package defined")
else:
dev.RunShellCommand([
'am', 'set-debug-app', '--persistent',
self._test_instance.package_info.package
],
check_return=True)
def push_test_data():
device_root = posixpath.join(dev.GetExternalStoragePath(),
'chromium_tests_root')
host_device_tuples_substituted = [
(h,
local_device_test_run.SubstituteDeviceRoot(
d, device_root)) for h, d in host_device_tuples
]
logging.info('instrumentation data deps:')
for h, d in host_device_tuples_substituted:
logging.info('%r -> %r', h, d)
dev.PushChangedFiles(host_device_tuples_substituted,
delete_device_stale=True)
if not host_device_tuples_substituted:
dev.RunShellCommand(['rm', '-rf', device_root],
check_return=True)
dev.RunShellCommand(['mkdir', '-p', device_root],
check_return=True)
def create_flag_changer():
if self._test_instance.flags:
if not self._test_instance.package_info:
logging.error("Couldn't set flags: no package info")
elif not self._test_instance.package_info.cmdline_file:
logging.error("Couldn't set flags: no cmdline_file")
else:
self._CreateFlagChangerIfNeeded(dev)
logging.debug('Attempting to set flags: %r',
self._test_instance.flags)
self._flag_changers[str(dev)].AddFlags(
self._test_instance.flags)
valgrind_tools.SetChromeTimeoutScale(
dev, self._test_instance.timeout_scale)
steps = (install_apk, push_test_data, create_flag_changer)
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(dev)
def individual_device_set_up(device, host_device_tuples):
def install_apk(dev):
# Install test APK.
self._delegate.Install(dev)
def push_test_data(dev):
# Push data dependencies.
device_root = self._delegate.GetTestDataRoot(dev)
host_device_tuples_substituted = [
(h,
local_device_test_run.SubstituteDeviceRoot(
d, device_root)) for h, d in host_device_tuples
]
local_device_environment.place_nomedia_on_device(
dev, device_root)
dev.PushChangedFiles(
host_device_tuples_substituted,
delete_device_stale=True,
# Some gtest suites, e.g. unit_tests, have data dependencies that
# can take longer than the default timeout to push. See
# crbug.com/791632 for context.
timeout=600)
if not host_device_tuples:
dev.RemovePath(device_root,
force=True,
recursive=True,
rename=True)
dev.RunShellCommand(['mkdir', '-p', device_root],
check_return=True)
def init_tool_and_start_servers(dev):
tool = self.GetTool(dev)
tool.CopyFiles(dev)
tool.SetupEnvironment()
try:
# See https://crbug.com/1030827.
# This is a hack that may break in the future. We're relying on the
# fact that adb doesn't use ipv6 for it's server, and so doesn't
# listen on ipv6, but ssh remote forwarding does. 5037 is the port
# number adb uses for its server.
if "[::1]:5037" in subprocess.check_output(
"ss -o state listening 'sport = 5037'",
shell=True):
logging.error(
'Test Server cannot be started with a remote-forwarded adb '
'server. Continuing anyways, but some tests may fail.'
)
return
except subprocess.CalledProcessError:
pass
self._servers[str(dev)] = []
if self.TestPackage() in _SUITE_REQUIRES_TEST_SERVER_SPAWNER:
self._servers[str(dev)].append(
local_test_server_spawner.LocalTestServerSpawner(
ports.AllocateTestServerPort(), dev, tool))
for s in self._servers[str(dev)]:
s.SetUp()
def bind_crash_handler(step, dev):
return lambda: crash_handler.RetryOnSystemCrash(step, dev)
steps = [
bind_crash_handler(s, device)
for s in (install_apk, push_test_data,
init_tool_and_start_servers)
]
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('apk_path',
help='The path to the APK to install.')
parser.add_argument('--split',
action='append',
dest='splits',
help='A glob matching the apk splits. '
'Can be specified multiple times.')
parser.add_argument('--lib-dir',
help='Path to native libraries directory.')
parser.add_argument('--dex-files',
help='List of dex files to push.',
action='append',
default=[])
parser.add_argument('-d', '--device', dest='device',
help='Target device for apk to install on.')
parser.add_argument('--uninstall',
action='store_true',
default=False,
help='Remove the app and all side-loaded files.')
parser.add_argument('--output-directory',
help='Path to the root build directory.')
parser.add_argument('--no-threading',
action='store_true',
default=False,
help='Do not install and push concurrently')
parser.add_argument('-v',
'--verbose',
dest='verbose_count',
default=0,
action='count',
help='Verbose level (multiple times for more)')
args = parser.parse_args()
run_tests_helper.SetLogLevel(args.verbose_count)
constants.SetBuildType('Debug')
if args.output_directory:
constants.SetOutputDirectory(args.output_directory)
main_timer = time_profile.TimeProfile()
install_timer = time_profile.TimeProfile()
push_native_timer = time_profile.TimeProfile()
push_dex_timer = time_profile.TimeProfile()
if args.device:
# Retries are annoying when commands fail for legitimate reasons. Might want
# to enable them if this is ever used on bots though.
device = device_utils.DeviceUtils(args.device, default_retries=0)
else:
devices = device_utils.DeviceUtils.HealthyDevices(default_retries=0)
if not devices:
raise device_errors.NoDevicesError()
elif len(devices) == 1:
device = devices[0]
else:
all_devices = device_utils.DeviceUtils.parallel(devices)
msg = ('More than one device available.\n'
'Use --device=SERIAL to select a device.\n'
'Available devices:\n')
descriptions = all_devices.pMap(lambda d: d.build_description).pGet(None)
for d, desc in zip(devices, descriptions):
msg += ' %s (%s)\n' % (d, desc)
raise Exception(msg)
apk_help = apk_helper.ApkHelper(args.apk_path)
apk_package = apk_help.GetPackageName()
device_incremental_dir = '/data/local/tmp/incremental-app-%s' % apk_package
if args.uninstall:
device.Uninstall(apk_package)
device.RunShellCommand(['rm', '-rf', device_incremental_dir],
check_return=True)
logging.info('Uninstall took %s seconds.', main_timer.GetDelta())
return
if device.build_version_sdk >= version_codes.MARSHMALLOW:
if apk_help.HasIsolatedProcesses():
raise Exception('Cannot use perform incremental installs on Android M+ '
'without first disabling isolated processes. Use GN arg: '
'disable_incremental_isolated_processes=true to do so.')
# Install .apk(s) if any of them have changed.
def do_install():
install_timer.Start()
if args.splits:
splits = []
for split_glob in args.splits:
splits.extend((f for f in glob.glob(split_glob)))
device.InstallSplitApk(args.apk_path, splits, reinstall=True,
allow_cached_props=True, permissions=())
else:
device.Install(args.apk_path, reinstall=True, permissions=())
install_timer.Stop(log=False)
# Push .so and .dex files to the device (if they have changed).
def do_push_files():
if args.lib_dir:
push_native_timer.Start()
device_lib_dir = posixpath.join(device_incremental_dir, 'lib')
device.PushChangedFiles([(args.lib_dir, device_lib_dir)],
delete_device_stale=True)
push_native_timer.Stop(log=False)
if args.dex_files:
push_dex_timer.Start()
# Put all .dex files to be pushed into a temporary directory so that we
# can use delete_device_stale=True.
with build_utils.TempDir() as temp_dir:
device_dex_dir = posixpath.join(device_incremental_dir, 'dex')
# Ensure no two files have the same name.
transformed_names = _TransformDexPaths(args.dex_files)
for src_path, dest_name in zip(args.dex_files, transformed_names):
shutil.copyfile(src_path, os.path.join(temp_dir, dest_name))
device.PushChangedFiles([(temp_dir, device_dex_dir)],
delete_device_stale=True)
push_dex_timer.Stop(log=False)
# Create 2 lock files:
# * install.lock tells the app to pause on start-up (until we release it).
# * firstrun.lock is used by the app to pause all secondary processes until
# the primary process finishes loading the .dex / .so files.
def create_lock_files():
# Creates or zeros out lock files.
cmd = ('D="%s";'
'mkdir -p $D &&'
'echo -n >$D/install.lock 2>$D/firstrun.lock')
device.RunShellCommand(cmd % device_incremental_dir, check_return=True)
# The firstrun.lock is released by the app itself.
def release_installer_lock():
device.RunShellCommand('echo > %s/install.lock' % device_incremental_dir,
check_return=True)
create_lock_files()
# Concurrency here speeds things up quite a bit, but DeviceUtils hasn't
# been designed for multi-threading. Enabling only because this is a
# developer-only tool.
if args.no_threading:
do_install()
do_push_files()
else:
reraiser_thread.RunAsync((do_install, do_push_files))
release_installer_lock()
logging.info('Took %s seconds (install=%s, libs=%s, dex=%s)',
main_timer.GetDelta(), install_timer.GetDelta(),
push_native_timer.GetDelta(), push_dex_timer.GetDelta())
def _RunTest(self, device, test):
extras = {}
# Provide package name under test for apk_under_test.
if self._test_instance.apk_under_test:
package_name = self._test_instance.apk_under_test.GetPackageName()
extras[_EXTRA_PACKAGE_UNDER_TEST] = package_name
flags_to_add = []
test_timeout_scale = None
if self._test_instance.coverage_directory:
coverage_basename = '%s.exec' % (
'%s_%s_group' %
(test[0]['class'], test[0]['method']) if isinstance(
test, list) else '%s_%s' % (test['class'], test['method']))
extras['coverage'] = 'true'
coverage_directory = os.path.join(device.GetExternalStoragePath(),
'chrome', 'test', 'coverage')
if not device.PathExists(coverage_directory):
device.RunShellCommand(['mkdir', '-p', coverage_directory],
check_return=True)
coverage_device_file = os.path.join(coverage_directory,
coverage_basename)
extras['coverageFile'] = coverage_device_file
# Save screenshot if screenshot dir is specified (save locally) or if
# a GS bucket is passed (save in cloud).
screenshot_device_file = device_temp_file.DeviceTempFile(
device.adb, suffix='.png', dir=device.GetExternalStoragePath())
extras[EXTRA_SCREENSHOT_FILE] = screenshot_device_file.name
# Set up the screenshot directory. This needs to be done for each test so
# that we only get screenshots created by that test. It has to be on
# external storage since the default location doesn't allow file creation
# from the instrumentation test app on Android L and M.
ui_capture_dir = device_temp_file.NamedDeviceTemporaryDirectory(
device.adb, dir=device.GetExternalStoragePath())
extras[EXTRA_UI_CAPTURE_DIR] = ui_capture_dir.name
if self._env.trace_output:
trace_device_file = device_temp_file.DeviceTempFile(
device.adb,
suffix='.json',
dir=device.GetExternalStoragePath())
extras[EXTRA_TRACE_FILE] = trace_device_file.name
if isinstance(test, list):
if not self._test_instance.driver_apk:
raise Exception('driver_apk does not exist. '
'Please build it and try again.')
if any(t.get('is_junit4') for t in test):
raise Exception('driver apk does not support JUnit4 tests')
def name_and_timeout(t):
n = instrumentation_test_instance.GetTestName(t)
i = self._GetTimeoutFromAnnotations(t['annotations'], n)
return (n, i)
test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
test_name = ','.join(test_names)
test_display_name = test_name
target = '%s/%s' % (self._test_instance.driver_package,
self._test_instance.driver_name)
extras.update(
self._test_instance.GetDriverEnvironmentVars(
test_list=test_names))
timeout = sum(timeouts)
else:
test_name = instrumentation_test_instance.GetTestName(test)
test_display_name = self._GetUniqueTestName(test)
if test['is_junit4']:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.junit4_runner_class)
else:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.junit3_runner_class)
extras['class'] = test_name
if 'flags' in test and test['flags']:
flags_to_add.extend(test['flags'])
timeout = self._GetTimeoutFromAnnotations(test['annotations'],
test_display_name)
test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
test['annotations'])
if test_timeout_scale and test_timeout_scale != 1:
valgrind_tools.SetChromeTimeoutScale(
device,
test_timeout_scale * self._test_instance.timeout_scale)
if self._test_instance.wait_for_java_debugger:
timeout = None
logging.info('preparing to run %s: %s', test_display_name, test)
render_tests_device_output_dir = None
if _IsRenderTest(test):
# TODO(mikecase): Add DeviceTempDirectory class and use that instead.
render_tests_device_output_dir = posixpath.join(
device.GetExternalStoragePath(), 'render_test_output_dir')
flags_to_add.append('--render-test-output-dir=%s' %
render_tests_device_output_dir)
if flags_to_add:
self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags(add=flags_to_add)
time_ms = lambda: int(time.time() * 1e3)
start_ms = time_ms()
stream_name = 'logcat_%s_%s_%s' % (test_name.replace(
'#', '.'), time.strftime('%Y%m%dT%H%M%S-UTC',
time.gmtime()), device.serial)
with ui_capture_dir:
with self._env.output_manager.ArchivedTempfile(
stream_name, 'logcat') as logcat_file:
logmon = None
try:
with logcat_monitor.LogcatMonitor(
device.adb,
filter_specs=local_device_environment.
LOGCAT_FILTERS,
output_file=logcat_file.name,
transform_func=self._test_instance.
MaybeDeobfuscateLines,
check_error=False) as logmon:
with _LogTestEndpoints(device, test_name):
with contextlib_ext.Optional(
trace_event.trace(test_name),
self._env.trace_output):
output = device.StartInstrumentation(
target,
raw=True,
extras=extras,
timeout=timeout,
retries=0)
finally:
if logmon:
logmon.Close()
if logcat_file.Link():
logging.info('Logcat saved to %s', logcat_file.Link())
duration_ms = time_ms() - start_ms
with contextlib_ext.Optional(trace_event.trace('ProcessResults'),
self._env.trace_output):
output = self._test_instance.MaybeDeobfuscateLines(output)
# TODO(jbudorick): Make instrumentation tests output a JSON so this
# doesn't have to parse the output.
result_code, result_bundle, statuses = (
self._test_instance.ParseAmInstrumentRawOutput(output))
results = self._test_instance.GenerateTestResults(
result_code, result_bundle, statuses, start_ms,
duration_ms, device.product_cpu_abi,
self._test_instance.symbolizer)
if self._env.trace_output:
self._SaveTraceData(trace_device_file, device, test['class'])
def restore_flags():
if flags_to_add:
self._flag_changers[str(device)].Restore()
def restore_timeout_scale():
if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale(
device, self._test_instance.timeout_scale)
def handle_coverage_data():
if self._test_instance.coverage_directory:
try:
if not os.path.exists(
self._test_instance.coverage_directory):
os.makedirs(self._test_instance.coverage_directory)
device.PullFile(coverage_device_file,
self._test_instance.coverage_directory)
device.RemovePath(coverage_device_file, True)
except (OSError, base_error.BaseError) as e:
logging.warning(
'Failed to handle coverage data after tests: %s',
e)
def handle_render_test_data():
if _IsRenderTest(test):
# Render tests do not cause test failure by default. So we have to
# check to see if any failure images were generated even if the test
# does not fail.
try:
self._ProcessRenderTestResults(
device, render_tests_device_output_dir, results)
finally:
device.RemovePath(render_tests_device_output_dir,
recursive=True,
force=True)
def pull_ui_screen_captures():
screenshots = []
for filename in device.ListDirectory(ui_capture_dir.name):
if filename.endswith('.json'):
screenshots.append(pull_ui_screenshot(filename))
if screenshots:
json_archive_name = 'ui_capture_%s_%s.json' % (
test_name.replace('#', '.'),
time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()))
with self._env.output_manager.ArchivedTempfile(
json_archive_name, 'ui_capture',
output_manager.Datatype.JSON) as json_archive:
json.dump(screenshots, json_archive)
for result in results:
result.SetLink('ui screenshot', json_archive.Link())
def pull_ui_screenshot(filename):
source_dir = ui_capture_dir.name
json_path = posixpath.join(source_dir, filename)
json_data = json.loads(device.ReadFile(json_path))
image_file_path = posixpath.join(source_dir,
json_data['location'])
with self._env.output_manager.ArchivedTempfile(
json_data['location'], 'ui_capture',
output_manager.Datatype.PNG) as image_archive:
device.PullFile(image_file_path, image_archive.name)
json_data['image_link'] = image_archive.Link()
return json_data
# While constructing the TestResult objects, we can parallelize several
# steps that involve ADB. These steps should NOT depend on any info in
# the results! Things such as whether the test CRASHED have not yet been
# determined.
post_test_steps = [
restore_flags, restore_timeout_scale, handle_coverage_data,
handle_render_test_data, pull_ui_screen_captures
]
if self._env.concurrent_adb:
reraiser_thread.RunAsync(post_test_steps)
else:
for step in post_test_steps:
step()
for result in results:
if logcat_file:
result.SetLink('logcat', logcat_file.Link())
# Update the result name if the test used flags.
if flags_to_add:
for r in results:
if r.GetName() == test_name:
r.SetName(test_display_name)
# Add UNKNOWN results for any missing tests.
iterable_test = test if isinstance(test, list) else [test]
test_names = set(self._GetUniqueTestName(t) for t in iterable_test)
results_names = set(r.GetName() for r in results)
results.extend(
base_test_result.BaseTestResult(
u, base_test_result.ResultType.UNKNOWN)
for u in test_names.difference(results_names))
# Update the result type if we detect a crash.
try:
if DidPackageCrashOnDevice(self._test_instance.test_package,
device):
for r in results:
if r.GetType() == base_test_result.ResultType.UNKNOWN:
r.SetType(base_test_result.ResultType.CRASH)
except device_errors.CommandTimeoutError:
logging.warning(
'timed out when detecting/dismissing error dialogs')
# Attach screenshot to the test to help with debugging the dialog boxes.
self._SaveScreenshot(device, screenshot_device_file,
test_display_name, results,
'dialog_box_screenshot')
# Handle failures by:
# - optionally taking a screenshot
# - logging the raw output at INFO level
# - clearing the application state while persisting permissions
if any(r.GetType() not in (base_test_result.ResultType.PASS,
base_test_result.ResultType.SKIP)
for r in results):
self._SaveScreenshot(device, screenshot_device_file,
test_display_name, results,
'post_test_screenshot')
logging.info('detected failure in %s. raw output:',
test_display_name)
for l in output:
logging.info(' %s', l)
if (not self._env.skip_clear_data
and self._test_instance.package_info):
permissions = (
self._test_instance.apk_under_test.GetPermissions()
if self._test_instance.apk_under_test else None)
device.ClearApplicationState(
self._test_instance.package_info.package,
permissions=permissions)
else:
logging.debug('raw output from %s:', test_display_name)
for l in output:
logging.debug(' %s', l)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True,
tombstone_symbolizer=self._test_instance.symbolizer
)
tombstone_filename = 'tombstones_%s_%s' % (
time.strftime('%Y%m%dT%H%M%S-UTC',
time.gmtime()), device.serial)
with self._env.output_manager.ArchivedTempfile(
tombstone_filename,
'tombstones') as tombstone_file:
tombstone_file.write(
'\n'.join(resolved_tombstones))
result.SetLink('tombstones', tombstone_file.Link())
return results, None
def individual_device_set_up(dev, host_device_tuples):
steps = []
if self._test_instance.replace_system_package:
# We need the context manager to be applied before modifying any shared
# preference files in case the replacement APK needs to be set up, and
# it needs to be applied while the test is running. Thus, it needs to
# be applied early during setup, but must still be applied during
# _RunTest, which isn't possible using 'with' without applying the
# context manager up in test_runner. Instead, we manually invoke
# its __enter__ and __exit__ methods in setup and teardown
self._replace_package_contextmanager = system_app.ReplaceSystemApp(
dev, self._test_instance.replace_system_package.package,
self._test_instance.replace_system_package.replacement_apk)
steps.append(self._replace_package_contextmanager.__enter__)
def install_helper(apk, permissions):
@trace_event.traced("apk_path")
def install_helper_internal(d, apk_path=apk.path):
# pylint: disable=unused-argument
d.Install(apk, permissions=permissions)
return lambda: crash_handler.RetryOnSystemCrash(
install_helper_internal, dev)
def incremental_install_helper(apk, script):
@trace_event.traced("apk_path")
def incremental_install_helper_internal(d, apk_path=apk.path):
# pylint: disable=unused-argument
local_device_test_run.IncrementalInstall(d, apk, script)
return lambda: crash_handler.RetryOnSystemCrash(
incremental_install_helper_internal, dev)
if self._test_instance.apk_under_test:
if self._test_instance.apk_under_test_incremental_install_script:
steps.append(
incremental_install_helper(
self._test_instance.apk_under_test,
self._test_instance.
apk_under_test_incremental_install_script))
else:
permissions = self._test_instance.apk_under_test.GetPermissions(
)
steps.append(
install_helper(self._test_instance.apk_under_test,
permissions))
if self._test_instance.test_apk_incremental_install_script:
steps.append(
incremental_install_helper(
self._test_instance.test_apk, self._test_instance.
test_apk_incremental_install_script))
else:
permissions = self._test_instance.test_apk.GetPermissions()
steps.append(
install_helper(self._test_instance.test_apk, permissions))
steps.extend(
install_helper(apk, None)
for apk in self._test_instance.additional_apks)
@trace_event.traced
def set_debug_app():
# Set debug app in order to enable reading command line flags on user
# builds
if self._test_instance.flags:
if not self._test_instance.package_info:
logging.error(
"Couldn't set debug app: no package info")
elif not self._test_instance.package_info.package:
logging.error(
"Couldn't set debug app: no package defined")
else:
dev.RunShellCommand([
'am', 'set-debug-app', '--persistent',
self._test_instance.package_info.package
],
check_return=True)
@trace_event.traced
def edit_shared_prefs():
shared_preference_utils.ApplySharedPreferenceSettings(
dev, self._test_instance.edit_shared_prefs)
@trace_event.traced
def push_test_data():
device_root = posixpath.join(dev.GetExternalStoragePath(),
'chromium_tests_root')
host_device_tuples_substituted = [
(h,
local_device_test_run.SubstituteDeviceRoot(
d, device_root)) for h, d in host_device_tuples
]
logging.info('instrumentation data deps:')
for h, d in host_device_tuples_substituted:
logging.info('%r -> %r', h, d)
dev.PushChangedFiles(host_device_tuples_substituted,
delete_device_stale=True)
if not host_device_tuples_substituted:
dev.RunShellCommand(['rm', '-rf', device_root],
check_return=True)
dev.RunShellCommand(['mkdir', '-p', device_root],
check_return=True)
@trace_event.traced
def create_flag_changer():
if self._test_instance.flags:
if not self._test_instance.package_info:
logging.error("Couldn't set flags: no package info")
elif not self._test_instance.package_info.cmdline_file:
logging.error("Couldn't set flags: no cmdline_file")
else:
self._CreateFlagChangerIfNeeded(dev)
logging.debug('Attempting to set flags: %r',
self._test_instance.flags)
self._flag_changers[str(dev)].AddFlags(
self._test_instance.flags)
valgrind_tools.SetChromeTimeoutScale(
dev, self._test_instance.timeout_scale)
@trace_event.traced
def setup_ui_capture_dir():
# Make sure the UI capture directory exists and is empty by deleting
# and recreating it.
# TODO (aberent) once DeviceTempDir exists use it here.
self._ui_capture_dir[dev] = posixpath.join(
dev.GetExternalStoragePath(), *UI_CAPTURE_DIRS)
if dev.PathExists(self._ui_capture_dir[dev]):
dev.RunShellCommand(
['rm', '-rf', self._ui_capture_dir[dev]])
dev.RunShellCommand(['mkdir', self._ui_capture_dir[dev]])
steps += [
set_debug_app, edit_shared_prefs, push_test_data,
create_flag_changer, setup_ui_capture_dir
]
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(dev)
def individual_device_set_up(dev, host_device_tuples):
steps = []
def install_helper(apk, permissions):
@trace_event.traced("apk_path")
def install_helper_internal(apk_path=apk.path):
# pylint: disable=unused-argument
dev.Install(apk, permissions=permissions)
return install_helper_internal
def incremental_install_helper(dev, apk, script):
@trace_event.traced("apk_path")
def incremental_install_helper_internal(apk_path=apk.path):
# pylint: disable=unused-argument
local_device_test_run.IncrementalInstall(dev, apk, script)
return incremental_install_helper_internal
if self._test_instance.apk_under_test:
if self._test_instance.apk_under_test_incremental_install_script:
steps.append(
incremental_install_helper(
dev, self._test_instance.apk_under_test,
self._test_instance.
apk_under_test_incremental_install_script))
else:
permissions = self._test_instance.apk_under_test.GetPermissions(
)
steps.append(
install_helper(self._test_instance.apk_under_test,
permissions))
if self._test_instance.test_apk_incremental_install_script:
steps.append(
incremental_install_helper(
dev, self._test_instance.test_apk, self._test_instance.
test_apk_incremental_install_script))
else:
permissions = self._test_instance.test_apk.GetPermissions()
steps.append(
install_helper(self._test_instance.test_apk, permissions))
steps.extend(
install_helper(apk, None)
for apk in self._test_instance.additional_apks)
@trace_event.traced
def set_debug_app():
# Set debug app in order to enable reading command line flags on user
# builds
if self._test_instance.flags:
if not self._test_instance.package_info:
logging.error(
"Couldn't set debug app: no package info")
elif not self._test_instance.package_info.package:
logging.error(
"Couldn't set debug app: no package defined")
else:
dev.RunShellCommand([
'am', 'set-debug-app', '--persistent',
self._test_instance.package_info.package
],
check_return=True)
@trace_event.traced
def edit_shared_prefs():
for pref in self._test_instance.edit_shared_prefs:
prefs = shared_prefs.SharedPrefs(dev, pref['package'],
pref['filename'])
prefs.Load()
for key in pref.get('remove', []):
try:
prefs.Remove(key)
except KeyError:
logging.warning(
"Attempted to remove non-existent key %s", key)
for key, value in pref.get('set', {}).iteritems():
if isinstance(value, bool):
prefs.SetBoolean(key, value)
elif isinstance(value, basestring):
prefs.SetString(key, value)
elif isinstance(value, long) or isinstance(value, int):
prefs.SetLong(key, value)
elif isinstance(value, list):
prefs.SetStringSet(key, value)
else:
raise ValueError(
"Given invalid value type %s for key %s" %
(str(type(value)), key))
prefs.Commit()
@trace_event.traced
def push_test_data():
device_root = posixpath.join(dev.GetExternalStoragePath(),
'chromium_tests_root')
host_device_tuples_substituted = [
(h,
local_device_test_run.SubstituteDeviceRoot(
d, device_root)) for h, d in host_device_tuples
]
logging.info('instrumentation data deps:')
for h, d in host_device_tuples_substituted:
logging.info('%r -> %r', h, d)
dev.PushChangedFiles(host_device_tuples_substituted,
delete_device_stale=True)
if not host_device_tuples_substituted:
dev.RunShellCommand(['rm', '-rf', device_root],
check_return=True)
dev.RunShellCommand(['mkdir', '-p', device_root],
check_return=True)
@trace_event.traced
def create_flag_changer():
if self._test_instance.flags:
if not self._test_instance.package_info:
logging.error("Couldn't set flags: no package info")
elif not self._test_instance.package_info.cmdline_file:
logging.error("Couldn't set flags: no cmdline_file")
else:
self._CreateFlagChangerIfNeeded(dev)
logging.debug('Attempting to set flags: %r',
self._test_instance.flags)
self._flag_changers[str(dev)].AddFlags(
self._test_instance.flags)
valgrind_tools.SetChromeTimeoutScale(
dev, self._test_instance.timeout_scale)
steps += [
set_debug_app, edit_shared_prefs, push_test_data,
create_flag_changer
]
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(dev)
def RunTests(self):
with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
json_file_path = os.path.join(temp_dir, 'results.json')
# Extract resources needed for test.
# TODO(mikecase): Investigate saving md5sums of zipfiles, and only
# extract zipfiles when they change.
def extract_resource_zip(resource_zip):
def helper():
extract_dest = os.path.join(
temp_dir,
os.path.splitext(os.path.basename(resource_zip))[0])
with zipfile.ZipFile(resource_zip, 'r') as zf:
zf.extractall(extract_dest)
return extract_dest
return helper
resource_dirs = reraiser_thread.RunAsync([
extract_resource_zip(resource_zip)
for resource_zip in self._test_instance.resource_zips
if os.path.exists(resource_zip)
])
java_script = os.path.join(constants.GetOutDirectory(), 'bin',
'helper', self._test_instance.suite)
command = [java_script]
# Add Jar arguments.
jar_args = [
'-test-jars', self._test_instance.suite + '.jar',
'-json-results-file', json_file_path
]
if self._test_instance.test_filter:
jar_args.extend(
['-gtest-filter', self._test_instance.test_filter])
if self._test_instance.package_filter:
jar_args.extend(
['-package-filter', self._test_instance.package_filter])
if self._test_instance.runner_filter:
jar_args.extend(
['-runner-filter', self._test_instance.runner_filter])
command.extend(['--jar-args', '"%s"' % ' '.join(jar_args)])
# Add JVM arguments.
jvm_args = [
'-Drobolectric.dependency.dir=%s' %
self._test_instance.robolectric_runtime_deps_dir,
'-Ddir.source.root=%s' % constants.DIR_SOURCE_ROOT,
]
if self._test_instance.android_manifest_path:
jvm_args += [
'-Dchromium.robolectric.manifest=%s' %
self._test_instance.android_manifest_path
]
if self._test_instance.package_name:
jvm_args += [
'-Dchromium.robolectric.package.name=%s' %
self._test_instance.package_name
]
if resource_dirs:
jvm_args += [
'-Dchromium.robolectric.resource.dirs=%s' %
':'.join(resource_dirs)
]
if self._test_instance.coverage_dir:
if not os.path.exists(self._test_instance.coverage_dir):
os.makedirs(self._test_instance.coverage_dir)
elif not os.path.isdir(self._test_instance.coverage_dir):
raise Exception(
'--coverage-dir takes a directory, not file path.')
jvm_args.append(
'-Demma.coverage.out.file=%s' %
os.path.join(self._test_instance.coverage_dir,
'%s.ec' % self._test_instance.suite))
if jvm_args:
command.extend(['--jvm-args', '"%s"' % ' '.join(jvm_args)])
cmd_helper.RunCmd(command)
with open(json_file_path, 'r') as f:
results_list = json_results.ParseResultsFromJson(
json.loads(f.read()))
test_run_results = base_test_result.TestRunResults()
test_run_results.AddResults(results_list)
return [test_run_results]
def individual_device_set_up(device, host_device_tuples):
steps = []
if self._test_instance.replace_system_package:
@trace_event.traced
def replace_package(dev):
# We need the context manager to be applied before modifying any
# shared preference files in case the replacement APK needs to be
# set up, and it needs to be applied while the test is running.
# Thus, it needs to be applied early during setup, but must still be
# applied during _RunTest, which isn't possible using 'with' without
# applying the context manager up in test_runner. Instead, we
# manually invoke its __enter__ and __exit__ methods in setup and
# teardown.
self._replace_package_contextmanager = system_app.ReplaceSystemApp(
dev,
self._test_instance.replace_system_package.package,
self._test_instance.replace_system_package.
replacement_apk)
# Pylint is not smart enough to realize that this field has
# an __enter__ method, and will complain loudly.
# pylint: disable=no-member
self._replace_package_contextmanager.__enter__()
# pylint: enable=no-member
steps.append(replace_package)
def install_helper(apk, permissions):
@instrumentation_tracing.no_tracing
@trace_event.traced("apk_path")
def install_helper_internal(d, apk_path=apk.path):
# pylint: disable=unused-argument
d.Install(apk, permissions=permissions)
return install_helper_internal
def incremental_install_helper(apk, json_path, permissions):
@trace_event.traced("apk_path")
def incremental_install_helper_internal(d, apk_path=apk.path):
# pylint: disable=unused-argument
installer.Install(d,
json_path,
apk=apk,
permissions=permissions)
return incremental_install_helper_internal
if self._test_instance.apk_under_test:
permissions = self._test_instance.apk_under_test.GetPermissions(
)
if self._test_instance.apk_under_test_incremental_install_json:
steps.append(
incremental_install_helper(
self._test_instance.apk_under_test,
self._test_instance.
apk_under_test_incremental_install_json,
permissions))
else:
steps.append(
install_helper(self._test_instance.apk_under_test,
permissions))
permissions = self._test_instance.test_apk.GetPermissions()
if self._test_instance.test_apk_incremental_install_json:
steps.append(
incremental_install_helper(
self._test_instance.test_apk,
self._test_instance.test_apk_incremental_install_json,
permissions))
else:
steps.append(
install_helper(self._test_instance.test_apk, permissions))
steps.extend(
install_helper(apk, None)
for apk in self._test_instance.additional_apks)
@trace_event.traced
def set_debug_app(dev):
# Set debug app in order to enable reading command line flags on user
# builds
package_name = None
if self._test_instance.apk_under_test:
package_name = self._test_instance.apk_under_test.GetPackageName(
)
elif self._test_instance.test_apk:
package_name = self._test_instance.test_apk.GetPackageName(
)
else:
logging.error(
"Couldn't set debug app: no package name found")
return
cmd = ['am', 'set-debug-app', '--persistent']
if self._test_instance.wait_for_java_debugger:
cmd.append('-w')
cmd.append(package_name)
dev.RunShellCommand(cmd, check_return=True)
@trace_event.traced
def edit_shared_prefs(dev):
for setting in self._test_instance.edit_shared_prefs:
shared_pref = shared_prefs.SharedPrefs(
dev,
setting['package'],
setting['filename'],
use_encrypted_path=setting.get(
'supports_encrypted_path', False))
pref_to_restore = copy.copy(shared_pref)
pref_to_restore.Load()
self._shared_prefs_to_restore.append(pref_to_restore)
shared_preference_utils.ApplySharedPreferenceSetting(
shared_pref, setting)
@instrumentation_tracing.no_tracing
def push_test_data(dev):
device_root = posixpath.join(dev.GetExternalStoragePath(),
'chromium_tests_root')
host_device_tuples_substituted = [
(h,
local_device_test_run.SubstituteDeviceRoot(
d, device_root)) for h, d in host_device_tuples
]
logging.info('instrumentation data deps:')
for h, d in host_device_tuples_substituted:
logging.info('%r -> %r', h, d)
dev.PushChangedFiles(host_device_tuples_substituted,
delete_device_stale=True)
if not host_device_tuples_substituted:
dev.RunShellCommand(['rm', '-rf', device_root],
check_return=True)
dev.RunShellCommand(['mkdir', '-p', device_root],
check_return=True)
@trace_event.traced
def create_flag_changer(dev):
if self._test_instance.flags:
self._CreateFlagChangerIfNeeded(dev)
logging.debug('Attempting to set flags: %r',
self._test_instance.flags)
self._flag_changers[str(dev)].AddFlags(
self._test_instance.flags)
valgrind_tools.SetChromeTimeoutScale(
dev, self._test_instance.timeout_scale)
steps += [
set_debug_app, edit_shared_prefs, push_test_data,
create_flag_changer
]
def bind_crash_handler(step, dev):
return lambda: crash_handler.RetryOnSystemCrash(step, dev)
steps = [bind_crash_handler(s, device) for s in steps]
try:
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
except device_errors.CommandFailedError:
# A bugreport can be large and take a while to generate, so only capture
# one if we're using a remote manager.
if isinstance(self._env.output_manager,
remote_output_manager.RemoteOutputManager):
logging.error(
'Error when setting up device for tests. Taking a bugreport for '
'investigation. This may take a while...')
report_name = '%s.bugreport' % device.serial
with self._env.output_manager.ArchivedTempfile(
report_name, 'bug_reports') as report_file:
device.TakeBugReport(report_file.name)
logging.error('Bug report saved to %s', report_file.Link())
raise
def RunTests(self, results):
with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
json_file_path = os.path.join(temp_dir, 'results.json')
# Extract resources needed for test.
# TODO(mikecase): Investigate saving md5sums of zipfiles, and only
# extract zipfiles when they change.
def extract_resource_zip(resource_zip):
def helper():
extract_dest = os.path.join(
temp_dir,
os.path.splitext(os.path.basename(resource_zip))[0])
with zipfile.ZipFile(resource_zip, 'r') as zf:
zf.extractall(extract_dest)
return extract_dest
return helper
resource_dirs = reraiser_thread.RunAsync([
extract_resource_zip(resource_zip)
for resource_zip in self._test_instance.resource_zips
if os.path.exists(resource_zip)
])
java_script = os.path.join(constants.GetOutDirectory(), 'bin',
'helper', self._test_instance.suite)
command = [java_script]
# Add Jar arguments.
jar_args = [
'-test-jars', self._test_instance.suite + '.jar',
'-json-results-file', json_file_path
]
if self._test_instance.test_filter:
jar_args.extend(
['-gtest-filter', self._test_instance.test_filter])
if self._test_instance.package_filter:
jar_args.extend(
['-package-filter', self._test_instance.package_filter])
if self._test_instance.runner_filter:
jar_args.extend(
['-runner-filter', self._test_instance.runner_filter])
command.extend(['--jar-args', '"%s"' % ' '.join(jar_args)])
# Add JVM arguments.
jvm_args = [
'-Drobolectric.dependency.dir=%s' %
self._test_instance.robolectric_runtime_deps_dir,
'-Ddir.source.root=%s' % constants.DIR_SOURCE_ROOT,
]
if self._test_instance.android_manifest_path:
jvm_args += [
'-Dchromium.robolectric.manifest=%s' %
self._test_instance.android_manifest_path
]
if self._test_instance.package_name:
jvm_args += [
'-Dchromium.robolectric.package.name=%s' %
self._test_instance.package_name
]
if resource_dirs:
jvm_args += [
'-Dchromium.robolectric.resource.dirs=%s' %
':'.join(resource_dirs)
]
if logging.getLogger().isEnabledFor(logging.INFO):
jvm_args += ['-Drobolectric.logging=stdout']
if self._test_instance.debug_socket:
jvm_args += [
'-agentlib:jdwp=transport=dt_socket'
',server=y,suspend=y,address=%s' %
self._test_instance.debug_socket
]
if self._test_instance.coverage_dir:
if not os.path.exists(self._test_instance.coverage_dir):
os.makedirs(self._test_instance.coverage_dir)
elif not os.path.isdir(self._test_instance.coverage_dir):
raise Exception(
'--coverage-dir takes a directory, not file path.')
jvm_args.append(
'-Demma.coverage.out.file=%s' %
os.path.join(self._test_instance.coverage_dir,
'%s.ec' % self._test_instance.suite))
if jvm_args:
command.extend(['--jvm-args', '"%s"' % ' '.join(jvm_args)])
cmd_helper.RunCmd(command)
try:
with open(json_file_path, 'r') as f:
results_list = json_results.ParseResultsFromJson(
json.loads(f.read()))
except IOError:
# In the case of a failure in the JUnit or Robolectric test runner
# the output json file may never be written.
results_list = [
base_test_result.BaseTestResult(
'Test Runner Failure',
base_test_result.ResultType.UNKNOWN)
]
test_run_results = base_test_result.TestRunResults()
test_run_results.AddResults(results_list)
results.append(test_run_results)
def individual_device_set_up(device, host_device_tuples):
steps = []
if self._test_instance.replace_system_package:
@trace_event.traced
def replace_package(dev):
# We need the context manager to be applied before modifying any
# shared preference files in case the replacement APK needs to be
# set up, and it needs to be applied while the test is running.
# Thus, it needs to be applied early during setup, but must still be
# applied during _RunTest, which isn't possible using 'with' without
# applying the context manager up in test_runner. Instead, we
# manually invoke its __enter__ and __exit__ methods in setup and
# teardown.
self._replace_package_contextmanager = system_app.ReplaceSystemApp(
dev,
self._test_instance.replace_system_package.package,
self._test_instance.replace_system_package.
replacement_apk)
# Pylint is not smart enough to realize that this field has
# an __enter__ method, and will complain loudly.
# pylint: disable=no-member
self._replace_package_contextmanager.__enter__()
# pylint: enable=no-member
steps.append(replace_package)
if self._test_instance.use_webview_provider:
@trace_event.traced
def use_webview_provider(dev):
# We need the context manager to be applied before modifying any
# shared preference files in case the replacement APK needs to be
# set up, and it needs to be applied while the test is running.
# Thus, it needs to be applied early during setup, but must still be
# applied during _RunTest, which isn't possible using 'with' without
# applying the context manager up in test_runner. Instead, we
# manually invoke its __enter__ and __exit__ methods in setup and
# teardown.
self._use_webview_contextmanager = webview_app.UseWebViewProvider(
dev, self._test_instance.use_webview_provider)
# Pylint is not smart enough to realize that this field has
# an __enter__ method, and will complain loudly.
# pylint: disable=no-member
self._use_webview_contextmanager.__enter__()
# pylint: enable=no-member
steps.append(use_webview_provider)
def install_helper(apk,
modules=None,
fake_modules=None,
permissions=None):
@instrumentation_tracing.no_tracing
@trace_event.traced
def install_helper_internal(d, apk_path=None):
# pylint: disable=unused-argument
logging.info('Start Installing %s', apk.path)
d.Install(apk,
modules=modules,
fake_modules=fake_modules,
permissions=permissions)
logging.info('Finished Installing %s', apk.path)
return install_helper_internal
def incremental_install_helper(apk, json_path, permissions):
@trace_event.traced
def incremental_install_helper_internal(d, apk_path=None):
# pylint: disable=unused-argument
logging.info('Start Incremental Installing %s', apk.path)
installer.Install(d,
json_path,
apk=apk,
permissions=permissions)
logging.info('Finished Incremental Installing %s',
apk.path)
return incremental_install_helper_internal
permissions = self._test_instance.test_apk.GetPermissions()
if self._test_instance.test_apk_incremental_install_json:
steps.append(
incremental_install_helper(
self._test_instance.test_apk,
self._test_instance.test_apk_incremental_install_json,
permissions))
else:
steps.append(
install_helper(self._test_instance.test_apk,
permissions=permissions))
steps.extend(
install_helper(apk)
for apk in self._test_instance.additional_apks)
# The apk under test needs to be installed last since installing other
# apks after will unintentionally clear the fake module directory.
# TODO(wnwen): Make this more robust, fix crbug.com/1010954.
if self._test_instance.apk_under_test:
permissions = self._test_instance.apk_under_test.GetPermissions(
)
if self._test_instance.apk_under_test_incremental_install_json:
steps.append(
incremental_install_helper(
self._test_instance.apk_under_test,
self._test_instance.
apk_under_test_incremental_install_json,
permissions))
else:
steps.append(
install_helper(self._test_instance.apk_under_test,
self._test_instance.modules,
self._test_instance.fake_modules,
permissions))
@trace_event.traced
def set_debug_app(dev):
# Set debug app in order to enable reading command line flags on user
# builds
package_name = None
if self._test_instance.apk_under_test:
package_name = self._test_instance.apk_under_test.GetPackageName(
)
elif self._test_instance.test_apk:
package_name = self._test_instance.test_apk.GetPackageName(
)
else:
logging.error(
"Couldn't set debug app: no package name found")
return
cmd = ['am', 'set-debug-app', '--persistent']
if self._test_instance.wait_for_java_debugger:
cmd.append('-w')
cmd.append(package_name)
dev.RunShellCommand(cmd, check_return=True)
@trace_event.traced
def edit_shared_prefs(dev):
for setting in self._test_instance.edit_shared_prefs:
shared_pref = shared_prefs.SharedPrefs(
dev,
setting['package'],
setting['filename'],
use_encrypted_path=setting.get(
'supports_encrypted_path', False))
pref_to_restore = copy.copy(shared_pref)
pref_to_restore.Load()
self._shared_prefs_to_restore.append(pref_to_restore)
shared_preference_utils.ApplySharedPreferenceSetting(
shared_pref, setting)
@trace_event.traced
def set_vega_permissions(dev):
# Normally, installation of VrCore automatically grants storage
# permissions. However, since VrCore is part of the system image on
# the Vega standalone headset, we don't install the APK as part of test
# setup. Instead, grant the permissions here so that it can take
# screenshots.
if dev.product_name == 'vega':
dev.GrantPermissions('com.google.vr.vrcore', [
'android.permission.WRITE_EXTERNAL_STORAGE',
'android.permission.READ_EXTERNAL_STORAGE'
])
@instrumentation_tracing.no_tracing
def push_test_data(dev):
device_root = posixpath.join(dev.GetExternalStoragePath(),
'chromium_tests_root')
host_device_tuples_substituted = [
(h,
local_device_test_run.SubstituteDeviceRoot(
d, device_root)) for h, d in host_device_tuples
]
logging.info('Pushing data dependencies.')
for h, d in host_device_tuples_substituted:
logging.debug(' %r -> %r', h, d)
dev.PushChangedFiles(host_device_tuples_substituted,
delete_device_stale=True)
if not host_device_tuples_substituted:
dev.RunShellCommand(['rm', '-rf', device_root],
check_return=True)
dev.RunShellCommand(['mkdir', '-p', device_root],
check_return=True)
@trace_event.traced
def create_flag_changer(dev):
if self._test_instance.flags:
self._CreateFlagChangerIfNeeded(dev)
logging.debug('Attempting to set flags: %r',
self._test_instance.flags)
self._flag_changers[str(dev)].AddFlags(
self._test_instance.flags)
valgrind_tools.SetChromeTimeoutScale(
dev, self._test_instance.timeout_scale)
steps += [
set_debug_app, edit_shared_prefs, push_test_data,
create_flag_changer, set_vega_permissions
]
def bind_crash_handler(step, dev):
return lambda: crash_handler.RetryOnSystemCrash(step, dev)
steps = [bind_crash_handler(s, device) for s in steps]
try:
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
except device_errors.CommandFailedError:
# A bugreport can be large and take a while to generate, so only capture
# one if we're using a remote manager.
if isinstance(self._env.output_manager,
remote_output_manager.RemoteOutputManager):
logging.error(
'Error when setting up device for tests. Taking a bugreport for '
'investigation. This may take a while...')
report_name = '%s.bugreport' % device.serial
with self._env.output_manager.ArchivedTempfile(
report_name, 'bug_reports') as report_file:
device.TakeBugReport(report_file.name)
logging.error('Bug report saved to %s', report_file.Link())
raise
def individual_device_set_up(device, host_device_tuples):
def install_apk(dev):
# Install test APK.
self._delegate.Install(dev)
def push_test_data(dev):
if self._test_instance.use_existing_test_data:
return
# Push data dependencies.
device_root = self._delegate.GetTestDataRoot(dev)
host_device_tuples_substituted = [
(h, local_device_test_run.SubstituteDeviceRoot(d, device_root))
for h, d in host_device_tuples]
local_device_environment.place_nomedia_on_device(dev, device_root)
dev.PushChangedFiles(
host_device_tuples_substituted,
delete_device_stale=True,
# Some gtest suites, e.g. unit_tests, have data dependencies that
# can take longer than the default timeout to push. See
# crbug.com/791632 for context.
timeout=600 * math.ceil(_GetDeviceTimeoutMultiplier() / 10))
if not host_device_tuples:
dev.RemovePath(device_root, force=True, recursive=True, rename=True)
dev.RunShellCommand(['mkdir', '-p', device_root], check_return=True)
def init_tool_and_start_servers(dev):
tool = self.GetTool(dev)
tool.CopyFiles(dev)
tool.SetupEnvironment()
try:
# See https://crbug.com/1030827.
# This is a hack that may break in the future. We're relying on the
# fact that adb doesn't use ipv6 for it's server, and so doesn't
# listen on ipv6, but ssh remote forwarding does. 5037 is the port
# number adb uses for its server.
if "[::1]:5037" in subprocess.check_output(
"ss -o state listening 'sport = 5037'", shell=True):
logging.error(
'Test Server cannot be started with a remote-forwarded adb '
'server. Continuing anyways, but some tests may fail.')
return
except subprocess.CalledProcessError:
pass
self._servers[str(dev)] = []
if self.TestPackage() in _SUITE_REQUIRES_TEST_SERVER_SPAWNER:
self._servers[str(dev)].append(
local_test_server_spawner.LocalTestServerSpawner(
ports.AllocateTestServerPort(), dev, tool))
for s in self._servers[str(dev)]:
s.SetUp()
def bind_crash_handler(step, dev):
return lambda: crash_handler.RetryOnSystemCrash(step, dev)
# Explicitly enable root to ensure that tests run under deterministic
# conditions. Without this explicit call, EnableRoot() is called from
# push_test_data() when PushChangedFiles() determines that it should use
# _PushChangedFilesZipped(), which is only most of the time.
# Root is required (amongst maybe other reasons) to pull the results file
# from the device, since it lives within the application's data directory
# (via GetApplicationDataDirectory()).
device.EnableRoot()
steps = [
bind_crash_handler(s, device)
for s in (install_apk, push_test_data, init_tool_and_start_servers)]
if self._env.concurrent_adb:
reraiser_thread.RunAsync(steps)
else:
for step in steps:
step()
