def RealMain(mapping, device_arg, package, status, hide_summary, workdir):
if mapping is None:
logging.warn(
'Skipping deobfuscation because no map file was provided.')
device = DetermineDeviceToUse(device_arg)
device.EnableRoot()
with device_temp_file.DeviceTempFile(device.adb) as file_on_device:
_AdbOatDumpForPackage(device, package, file_on_device.name)
file_on_host = os.path.join(workdir, 'out.dump')
device.PullFile(file_on_device.name, file_on_host)
proguard_mappings = (_ParseMappingFile(mapping) if mapping else None)
with open(file_on_host, 'r') as f:
java_classes = ListClassesAndVerificationStatus(f, proguard_mappings)
_PrintVerificationResults(status, java_classes, not hide_summary)
def Run(self, test, device, flags=None, **kwargs):
extras = dict(self._extras)
if ('timeout' in kwargs
and gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT not in extras):
# Make sure the instrumentation doesn't kill the test before the
# scripts do. The provided timeout value is in seconds, but the
# instrumentation deals with nanoseconds because that's how Android
# handles time.
extras[gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT] = int(
kwargs['timeout'] * _SECONDS_TO_NANOS)
with device_temp_file.DeviceTempFile(device.adb) as command_line_file:
device.WriteFile(command_line_file.name, '_ %s' % flags if flags else '_')
extras[_EXTRA_COMMAND_LINE_FILE] = command_line_file.name
with device_temp_file.DeviceTempFile(device.adb) as test_list_file:
if test:
device.WriteFile(test_list_file.name, '\n'.join(test))
extras[_EXTRA_TEST_LIST] = test_list_file.name
return device.StartInstrumentation(
self._component, extras=extras, raw=False, **kwargs)
def Run(self, test, device, flags=None, **kwargs):
extras = dict(self._extras)
if ('timeout' in kwargs
and gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT not in extras):
# Make sure the instrumentation doesn't kill the test before the
# scripts do. The provided timeout value is in seconds, but the
# instrumentation deals with nanoseconds because that's how Android
# handles time.
extras[gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT] = int(
kwargs['timeout'] * _SECONDS_TO_NANOS)
command_line_file = _NullContextManager()
if flags:
if len(flags) > _MAX_INLINE_FLAGS_LENGTH:
command_line_file = device_temp_file.DeviceTempFile(device.adb)
device.WriteFile(command_line_file.name, '_ %s' % flags)
extras[_EXTRA_COMMAND_LINE_FILE] = command_line_file.name
else:
extras[_EXTRA_COMMAND_LINE_FLAGS] = flags
test_list_file = _NullContextManager()
if test:
if len(test) > 1:
test_list_file = device_temp_file.DeviceTempFile(device.adb)
device.WriteFile(test_list_file.name, '\n'.join(test))
extras[_EXTRA_TEST_LIST] = test_list_file.name
else:
extras[_EXTRA_TEST] = test[0]
with command_line_file, test_list_file:
try:
return device.StartInstrumentation(
self._component, extras=extras, raw=False, **kwargs)
except Exception:
device.ForceStop(self._package)
raise
def Run(self, test, device, flags=None, **kwargs):
tool = self._test_run.GetTool(device).GetTestWrapper()
if tool:
cmd = [tool]
else:
cmd = []
cmd.append(posixpath.join(self._device_dist_dir, self._exe_file_name))
if test:
cmd.append('--gtest_filter=%s' % ':'.join(test))
if flags:
# TODO(agrieve): This won't work if multiple flags are passed.
cmd.append(flags)
cwd = constants.TEST_EXECUTABLE_DIR
env = {'LD_LIBRARY_PATH': self._device_dist_dir}
if self._coverage_dir:
coverage_device_file = device_temp_file.DeviceTempFile(
device.adb,
suffix='.profraw',
prefix=self._suite,
dir=device.GetExternalStoragePath())
env['LLVM_PROFILE_FILE'] = coverage_device_file.name
if self._tool != 'asan':
env['UBSAN_OPTIONS'] = constants.UBSAN_OPTIONS
try:
gcov_strip_depth = os.environ['NATIVE_COVERAGE_DEPTH_STRIP']
external = device.GetExternalStoragePath()
env['GCOV_PREFIX'] = '%s/gcov' % external
env['GCOV_PREFIX_STRIP'] = gcov_strip_depth
except (device_errors.CommandFailedError, KeyError):
pass
# Executable tests return a nonzero exit code on test failure, which is
# fine from the test runner's perspective; thus check_return=False.
output = device.RunShellCommand(cmd,
cwd=cwd,
env=env,
check_return=False,
large_output=True,
**kwargs)
if self._coverage_dir:
_PullCoverageFile(device, coverage_device_file, self._coverage_dir)
return output
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout *
self.GetTool(device).GetTimeoutScale())
with tempfile.NamedTemporaryFile(
suffix='.xml') as host_tmp_results_file:
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
flags = self._test_instance.test_arguments or ''
if self._test_instance.enable_xml_result_parsing:
flags += ' --gtest_output=xml:%s' % device_tmp_results_file.name
output = self._delegate.Run(test,
device,
flags=flags,
timeout=timeout,
retries=0)
if self._test_instance.enable_xml_result_parsing:
device.PullFile(device_tmp_results_file.name,
host_tmp_results_file.name)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device,
self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
with open(host_tmp_results_file.name) as xml_results_file:
results = gtest_test_instance.ParseGTestXML(
xml_results_file.read())
else:
results = gtest_test_instance.ParseGTestOutput(output)
# Check whether there are any crashed testcases.
self._crashes.update(
r.GetName() for r in results
if r.GetType() == base_test_result.ResultType.CRASH)
return results
def testCrashDuringCommand(self):
self.device.EnableRoot()
with device_temp_file.DeviceTempFile(self.device.adb) as trigger_file:
trigger_text = 'hello world'
def victim():
trigger_cmd = 'echo -n %s > %s; sleep 20' % (
cmd_helper.SingleQuote(trigger_text),
cmd_helper.SingleQuote(trigger_file.name))
crash_handler.RetryOnSystemCrash(
lambda d: d.RunShellCommand(trigger_cmd,
shell=True,
check_return=True,
retries=1,
as_root=True,
timeout=180),
device=self.device)
self.assertEquals(
trigger_text,
self.device.ReadFile(trigger_file.name, retries=0).strip())
return True
def crasher():
def ready_to_crash():
try:
return trigger_text == self.device.ReadFile(
trigger_file.name, retries=0).strip()
except device_errors.CommandFailedError:
return False
timeout_retry.WaitFor(ready_to_crash,
wait_period=2,
max_tries=10)
if not ready_to_crash():
return False
self.device.adb.Shell('echo c > /proc/sysrq-trigger',
expect_status=None,
timeout=60,
retries=0)
return True
self.assertEquals([True, True],
reraiser_thread.RunAsync([crasher, victim]))
def __init__(self, device, perf_binary, categories):
self._device = device
self._output_file = device_temp_file.DeviceTempFile(
self._device.adb, prefix='perf_output')
self._log_file = tempfile.TemporaryFile()
# TODO(jbudorick) Look at providing a way to unhandroll this once the
# adb rewrite has fully landed.
device_param = (['-s', str(self._device)] if str(self._device) else [])
cmd = ['adb'] + device_param + \
['shell', perf_binary, 'record',
'--output', self._output_file.name] + _PERF_OPTIONS
if categories:
cmd += ['--event', ','.join(categories)]
self._perf_control = perf_control.PerfControl(self._device)
self._perf_control.SetPerfProfilingMode()
self._perf_process = subprocess.Popen(cmd,
stdout=self._log_file,
stderr=subprocess.STDOUT)
def _GetRootUiNode(self, timeout=None, retries=None):
"""Get a node pointing to the root of the UI nodes on screen.
Note: This is currently implemented via adb calls to uiatomator and it
is *slow*, ~2 secs per call. Do not rely on low-level implementation
details that may change in the future.
TODO(crbug.com/567217): Swap to a more efficient implementation.
Args:
timeout: A number of seconds to wait for the uiautomator dump.
retries: Number of times to retry if the adb command fails.
Returns:
A UI node instance pointing to the root of the xml screenshot.
"""
with device_temp_file.DeviceTempFile(self._device.adb) as dtemp:
self._device.RunShellCommand(['uiautomator', 'dump', dtemp.name],
check_return=True)
xml_node = element_tree.fromstring(
self._device.ReadFile(dtemp.name, force_pull=True))
return _UiNode(self._device, xml_node, package=self._package)
def _run(dev):
with device_temp_file.DeviceTempFile(
dev.adb, suffix='.json', dir=dev.
GetExternalStoragePath()) as dev_test_list_json:
junit4_runner_class = self._test_instance.junit4_runner_class
test_package = self._test_instance.test_package
extras = {}
extras['log'] = 'true'
extras[_EXTRA_TEST_LIST] = dev_test_list_json.name
target = '%s/%s' % (test_package, junit4_runner_class)
test_list_run_output = dev.StartInstrumentation(
target, extras=extras)
if any(test_list_run_output):
logging.error('Unexpected output while listing tests:')
for line in test_list_run_output:
logging.error(' %s', line)
with tempfile_ext.NamedTemporaryDirectory() as host_dir:
host_file = os.path.join(host_dir, 'list_tests.json')
dev.PullFile(dev_test_list_json.name, host_file)
with open(host_file, 'r') as host_file:
return json.load(host_file)
def _RunTest(self, device, test):
extras = {}
# Provide package name under test for apk_under_test.
if self._test_instance.apk_under_test:
package_name = self._test_instance.apk_under_test.GetPackageName()
extras[_EXTRA_PACKAGE_UNDER_TEST] = package_name
flags_to_add = []
test_timeout_scale = None
if self._test_instance.coverage_directory:
coverage_basename = '%s.exec' % (
'%s_%s_group' %
(test[0]['class'], test[0]['method']) if isinstance(
test, list) else '%s_%s' % (test['class'], test['method']))
extras['coverage'] = 'true'
coverage_directory = os.path.join(device.GetExternalStoragePath(),
'chrome', 'test', 'coverage')
if not device.PathExists(coverage_directory):
device.RunShellCommand(['mkdir', '-p', coverage_directory],
check_return=True)
coverage_device_file = os.path.join(coverage_directory,
coverage_basename)
extras['coverageFile'] = coverage_device_file
# Save screenshot if screenshot dir is specified (save locally) or if
# a GS bucket is passed (save in cloud).
screenshot_device_file = device_temp_file.DeviceTempFile(
device.adb, suffix='.png', dir=device.GetExternalStoragePath())
extras[EXTRA_SCREENSHOT_FILE] = screenshot_device_file.name
# Set up the screenshot directory. This needs to be done for each test so
# that we only get screenshots created by that test. It has to be on
# external storage since the default location doesn't allow file creation
# from the instrumentation test app on Android L and M.
ui_capture_dir = device_temp_file.NamedDeviceTemporaryDirectory(
device.adb, dir=device.GetExternalStoragePath())
extras[EXTRA_UI_CAPTURE_DIR] = ui_capture_dir.name
if self._env.trace_output:
trace_device_file = device_temp_file.DeviceTempFile(
device.adb,
suffix='.json',
dir=device.GetExternalStoragePath())
extras[EXTRA_TRACE_FILE] = trace_device_file.name
if isinstance(test, list):
if not self._test_instance.driver_apk:
raise Exception('driver_apk does not exist. '
'Please build it and try again.')
if any(t.get('is_junit4') for t in test):
raise Exception('driver apk does not support JUnit4 tests')
def name_and_timeout(t):
n = instrumentation_test_instance.GetTestName(t)
i = self._GetTimeoutFromAnnotations(t['annotations'], n)
return (n, i)
test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
test_name = ','.join(test_names)
test_display_name = test_name
target = '%s/%s' % (self._test_instance.driver_package,
self._test_instance.driver_name)
extras.update(
self._test_instance.GetDriverEnvironmentVars(
test_list=test_names))
timeout = sum(timeouts)
else:
test_name = instrumentation_test_instance.GetTestName(test)
test_display_name = self._GetUniqueTestName(test)
if test['is_junit4']:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.junit4_runner_class)
else:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.junit3_runner_class)
extras['class'] = test_name
if 'flags' in test and test['flags']:
flags_to_add.extend(test['flags'])
timeout = self._GetTimeoutFromAnnotations(test['annotations'],
test_display_name)
test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
test['annotations'])
if test_timeout_scale and test_timeout_scale != 1:
valgrind_tools.SetChromeTimeoutScale(
device,
test_timeout_scale * self._test_instance.timeout_scale)
if self._test_instance.wait_for_java_debugger:
timeout = None
logging.info('preparing to run %s: %s', test_display_name, test)
render_tests_device_output_dir = None
if _IsRenderTest(test):
# TODO(mikecase): Add DeviceTempDirectory class and use that instead.
render_tests_device_output_dir = posixpath.join(
device.GetExternalStoragePath(), 'render_test_output_dir')
flags_to_add.append('--render-test-output-dir=%s' %
render_tests_device_output_dir)
if flags_to_add:
self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags(add=flags_to_add)
time_ms = lambda: int(time.time() * 1e3)
start_ms = time_ms()
stream_name = 'logcat_%s_%s_%s' % (test_name.replace(
'#', '.'), time.strftime('%Y%m%dT%H%M%S-UTC',
time.gmtime()), device.serial)
with ui_capture_dir:
with self._env.output_manager.ArchivedTempfile(
stream_name, 'logcat') as logcat_file:
try:
with logcat_monitor.LogcatMonitor(
device.adb,
filter_specs=local_device_environment.
LOGCAT_FILTERS,
output_file=logcat_file.name,
transform_func=self._test_instance.
MaybeDeobfuscateLines,
check_error=False) as logmon:
with _LogTestEndpoints(device, test_name):
with contextlib_ext.Optional(
trace_event.trace(test_name),
self._env.trace_output):
output = device.StartInstrumentation(
target,
raw=True,
extras=extras,
timeout=timeout,
retries=0)
finally:
logmon.Close()
if logcat_file.Link():
logging.info('Logcat saved to %s', logcat_file.Link())
duration_ms = time_ms() - start_ms
with contextlib_ext.Optional(trace_event.trace('ProcessResults'),
self._env.trace_output):
output = self._test_instance.MaybeDeobfuscateLines(output)
# TODO(jbudorick): Make instrumentation tests output a JSON so this
# doesn't have to parse the output.
result_code, result_bundle, statuses = (
self._test_instance.ParseAmInstrumentRawOutput(output))
results = self._test_instance.GenerateTestResults(
result_code, result_bundle, statuses, start_ms,
duration_ms, device.product_cpu_abi,
self._test_instance.symbolizer)
if self._env.trace_output:
self._SaveTraceData(trace_device_file, device, test['class'])
def restore_flags():
if flags_to_add:
self._flag_changers[str(device)].Restore()
def restore_timeout_scale():
if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale(
device, self._test_instance.timeout_scale)
def handle_coverage_data():
if self._test_instance.coverage_directory:
try:
if not os.path.exists(
self._test_instance.coverage_directory):
os.makedirs(self._test_instance.coverage_directory)
device.PullFile(coverage_device_file,
self._test_instance.coverage_directory)
device.RemovePath(coverage_device_file, True)
except (OSError, base_error.BaseError) as e:
logging.warning(
'Failed to handle coverage data after tests: %s',
e)
def handle_render_test_data():
if _IsRenderTest(test):
# Render tests do not cause test failure by default. So we have to
# check to see if any failure images were generated even if the test
# does not fail.
try:
self._ProcessRenderTestResults(
device, render_tests_device_output_dir, results)
finally:
device.RemovePath(render_tests_device_output_dir,
recursive=True,
force=True)
def pull_ui_screen_captures():
screenshots = []
for filename in device.ListDirectory(ui_capture_dir.name):
if filename.endswith('.json'):
screenshots.append(pull_ui_screenshot(filename))
if screenshots:
json_archive_name = 'ui_capture_%s_%s.json' % (
test_name.replace('#', '.'),
time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()))
with self._env.output_manager.ArchivedTempfile(
json_archive_name, 'ui_capture',
output_manager.Datatype.JSON) as json_archive:
json.dump(screenshots, json_archive)
for result in results:
result.SetLink('ui screenshot', json_archive.Link())
def pull_ui_screenshot(filename):
source_dir = ui_capture_dir.name
json_path = posixpath.join(source_dir, filename)
json_data = json.loads(device.ReadFile(json_path))
image_file_path = posixpath.join(source_dir,
json_data['location'])
with self._env.output_manager.ArchivedTempfile(
json_data['location'], 'ui_capture',
output_manager.Datatype.PNG) as image_archive:
device.PullFile(image_file_path, image_archive.name)
json_data['image_link'] = image_archive.Link()
return json_data
# While constructing the TestResult objects, we can parallelize several
# steps that involve ADB. These steps should NOT depend on any info in
# the results! Things such as whether the test CRASHED have not yet been
# determined.
post_test_steps = [
restore_flags, restore_timeout_scale, handle_coverage_data,
handle_render_test_data, pull_ui_screen_captures
]
if self._env.concurrent_adb:
post_test_step_thread_group = reraiser_thread.ReraiserThreadGroup(
reraiser_thread.ReraiserThread(f) for f in post_test_steps)
post_test_step_thread_group.StartAll(will_block=True)
else:
for step in post_test_steps:
step()
for result in results:
if logcat_file:
result.SetLink('logcat', logcat_file.Link())
# Update the result name if the test used flags.
if flags_to_add:
for r in results:
if r.GetName() == test_name:
r.SetName(test_display_name)
# Add UNKNOWN results for any missing tests.
iterable_test = test if isinstance(test, list) else [test]
test_names = set(self._GetUniqueTestName(t) for t in iterable_test)
results_names = set(r.GetName() for r in results)
results.extend(
base_test_result.BaseTestResult(
u, base_test_result.ResultType.UNKNOWN)
for u in test_names.difference(results_names))
# Update the result type if we detect a crash.
try:
if DidPackageCrashOnDevice(self._test_instance.test_package,
device):
for r in results:
if r.GetType() == base_test_result.ResultType.UNKNOWN:
r.SetType(base_test_result.ResultType.CRASH)
except device_errors.CommandTimeoutError:
logging.warning(
'timed out when detecting/dismissing error dialogs')
# Attach screenshot to the test to help with debugging the dialog boxes.
self._SaveScreenshot(device, screenshot_device_file,
test_display_name, results,
'dialog_box_screenshot')
# Handle failures by:
# - optionally taking a screenshot
# - logging the raw output at INFO level
# - clearing the application state while persisting permissions
if any(r.GetType() not in (base_test_result.ResultType.PASS,
base_test_result.ResultType.SKIP)
for r in results):
self._SaveScreenshot(device, screenshot_device_file,
test_display_name, results,
'post_test_screenshot')
logging.info('detected failure in %s. raw output:',
test_display_name)
for l in output:
logging.info(' %s', l)
if (not self._env.skip_clear_data
and self._test_instance.package_info):
permissions = (
self._test_instance.apk_under_test.GetPermissions()
if self._test_instance.apk_under_test else None)
device.ClearApplicationState(
self._test_instance.package_info.package,
permissions=permissions)
else:
logging.debug('raw output from %s:', test_display_name)
for l in output:
logging.debug(' %s', l)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True,
tombstone_symbolizer=self._test_instance.symbolizer
)
tombstone_filename = 'tombstones_%s_%s' % (
time.strftime('%Y%m%dT%H%M%S-UTC',
time.gmtime()), device.serial)
with self._env.output_manager.ArchivedTempfile(
tombstone_filename,
'tombstones') as tombstone_file:
tombstone_file.write(
'\n'.join(resolved_tombstones))
result.SetLink('tombstones', tombstone_file.Link())
if self._env.concurrent_adb:
post_test_step_thread_group.JoinAll()
return results, None
def _RunTest(self, device, test):
extras = {}
flags_to_add = []
test_timeout_scale = None
if self._test_instance.coverage_directory:
coverage_basename = '%s.ec' % ('%s_group' %
test[0]['method'] if isinstance(
test, list) else test['method'])
extras['coverage'] = 'true'
coverage_directory = os.path.join(device.GetExternalStoragePath(),
'chrome', 'test', 'coverage')
coverage_device_file = os.path.join(coverage_directory,
coverage_basename)
extras['coverageFile'] = coverage_device_file
# Save screenshot if screenshot dir is specified (save locally) or if
# a GS bucket is passed (save in cloud).
screenshot_device_file = None
if (self._test_instance.screenshot_dir
or self._test_instance.gs_results_bucket):
screenshot_device_file = device_temp_file.DeviceTempFile(
device.adb, suffix='.png', dir=device.GetExternalStoragePath())
extras[EXTRA_SCREENSHOT_FILE] = screenshot_device_file.name
extras[EXTRA_UI_CAPTURE_DIR] = self._ui_capture_dir[device]
if isinstance(test, list):
if not self._test_instance.driver_apk:
raise Exception('driver_apk does not exist. '
'Please build it and try again.')
if any(t.get('is_junit4') for t in test):
raise Exception('driver apk does not support JUnit4 tests')
def name_and_timeout(t):
n = instrumentation_test_instance.GetTestName(t)
i = self._GetTimeoutFromAnnotations(t['annotations'], n)
return (n, i)
test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
test_name = ','.join(test_names)
test_display_name = test_name
target = '%s/%s' % (self._test_instance.driver_package,
self._test_instance.driver_name)
extras.update(
self._test_instance.GetDriverEnvironmentVars(
test_list=test_names))
timeout = sum(timeouts)
else:
test_name = instrumentation_test_instance.GetTestName(test)
test_display_name = self._GetUniqueTestName(test)
if test['is_junit4']:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.test_runner_junit4)
else:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.test_runner)
extras['class'] = test_name
if 'flags' in test and test['flags']:
flags_to_add.extend(test['flags'])
timeout = self._GetTimeoutFromAnnotations(test['annotations'],
test_display_name)
test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
test['annotations'])
if test_timeout_scale and test_timeout_scale != 1:
valgrind_tools.SetChromeTimeoutScale(
device,
test_timeout_scale * self._test_instance.timeout_scale)
logging.info('preparing to run %s: %s', test_display_name, test)
render_tests_device_output_dir = None
if _IsRenderTest(test):
# TODO(mikecase): Add DeviceTempDirectory class and use that instead.
render_tests_device_output_dir = posixpath.join(
device.GetExternalStoragePath(), 'render_test_output_dir')
flags_to_add.append('--render-test-output-dir=%s' %
render_tests_device_output_dir)
if flags_to_add:
self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags(add=flags_to_add)
time_ms = lambda: int(time.time() * 1e3)
start_ms = time_ms()
stream_name = 'logcat_%s_%s_%s' % (test_name.replace(
'#', '.'), time.strftime('%Y%m%dT%H%M%S-UTC',
time.gmtime()), device.serial)
logmon = logdog_logcat_monitor.LogdogLogcatMonitor(
device.adb, stream_name, filter_specs=LOGCAT_FILTERS)
with contextlib_ext.Optional(logmon,
self._test_instance.should_save_logcat):
with _LogTestEndpoints(device, test_name):
with contextlib_ext.Optional(trace_event.trace(test_name),
self._env.trace_output):
output = device.StartInstrumentation(target,
raw=True,
extras=extras,
timeout=timeout,
retries=0)
logcat_url = logmon.GetLogcatURL()
duration_ms = time_ms() - start_ms
# TODO(jbudorick): Make instrumentation tests output a JSON so this
# doesn't have to parse the output.
result_code, result_bundle, statuses = (
self._test_instance.ParseAmInstrumentRawOutput(output))
results = self._test_instance.GenerateTestResults(
result_code, result_bundle, statuses, start_ms, duration_ms)
def restore_flags():
if flags_to_add:
self._flag_changers[str(device)].Restore()
def restore_timeout_scale():
if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale(
device, self._test_instance.timeout_scale)
def handle_coverage_data():
if self._test_instance.coverage_directory:
device.PullFile(coverage_directory,
self._test_instance.coverage_directory)
device.RunShellCommand('rm -f %s' %
posixpath.join(coverage_directory, '*'),
check_return=True,
shell=True)
def handle_render_test_data():
if _IsRenderTest(test):
# Render tests do not cause test failure by default. So we have to check
# to see if any failure images were generated even if the test does not
# fail.
try:
self._ProcessRenderTestResults(
device, render_tests_device_output_dir, results)
finally:
device.RemovePath(render_tests_device_output_dir,
recursive=True,
force=True)
# While constructing the TestResult objects, we can parallelize several
# steps that involve ADB. These steps should NOT depend on any info in
# the results! Things such as whether the test CRASHED have not yet been
# determined.
post_test_steps = [
restore_flags, restore_timeout_scale, handle_coverage_data,
handle_render_test_data
]
if self._env.concurrent_adb:
post_test_step_thread_group = reraiser_thread.ReraiserThreadGroup(
reraiser_thread.ReraiserThread(f) for f in post_test_steps)
post_test_step_thread_group.StartAll(will_block=True)
else:
for step in post_test_steps:
step()
for result in results:
if logcat_url:
result.SetLink('logcat', logcat_url)
# Update the result name if the test used flags.
if flags_to_add:
for r in results:
if r.GetName() == test_name:
r.SetName(test_display_name)
# Add UNKNOWN results for any missing tests.
iterable_test = test if isinstance(test, list) else [test]
test_names = set(self._GetUniqueTestName(t) for t in iterable_test)
results_names = set(r.GetName() for r in results)
results.extend(
base_test_result.BaseTestResult(
u, base_test_result.ResultType.UNKNOWN)
for u in test_names.difference(results_names))
# Update the result type if we detect a crash.
if DidPackageCrashOnDevice(self._test_instance.test_package, device):
for r in results:
if r.GetType() == base_test_result.ResultType.UNKNOWN:
r.SetType(base_test_result.ResultType.CRASH)
# Handle failures by:
# - optionally taking a screenshot
# - logging the raw output at INFO level
# - clearing the application state while persisting permissions
if any(r.GetType() not in (base_test_result.ResultType.PASS,
base_test_result.ResultType.SKIP)
for r in results):
with contextlib_ext.Optional(
tempfile_ext.NamedTemporaryDirectory(),
self._test_instance.screenshot_dir is None
and self._test_instance.gs_results_bucket
) as screenshot_host_dir:
screenshot_host_dir = (self._test_instance.screenshot_dir
or screenshot_host_dir)
self._SaveScreenshot(device, screenshot_host_dir,
screenshot_device_file, test_display_name,
results)
logging.info('detected failure in %s. raw output:',
test_display_name)
for l in output:
logging.info(' %s', l)
if (not self._env.skip_clear_data
and self._test_instance.package_info):
permissions = (
self._test_instance.apk_under_test.GetPermissions()
if self._test_instance.apk_under_test else None)
device.ClearApplicationState(
self._test_instance.package_info.package,
permissions=permissions)
else:
logging.debug('raw output from %s:', test_display_name)
for l in output:
logging.debug(' %s', l)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, '\n'.join(resolved_tombstones))
result.SetLink('tombstones', tombstones_url)
if self._env.concurrent_adb:
post_test_step_thread_group.JoinAll()
return results, None
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout *
self.GetTool(device).GetTimeoutScale())
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
flags = self._test_instance.test_arguments or ''
if self._test_instance.enable_xml_result_parsing:
flags += ' --gtest_output=xml:%s' % device_tmp_results_file.name
if self._test_instance.gtest_also_run_disabled_tests:
flags += ' --gtest_also_run_disabled_tests'
output = self._delegate.Run(test,
device,
flags=flags,
timeout=timeout,
retries=0)
if self._test_instance.enable_xml_result_parsing:
gtest_xml = device.ReadFile(device_tmp_results_file.name,
as_root=True)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
results = gtest_test_instance.ParseGTestXML(gtest_xml)
else:
results = gtest_test_instance.ParseGTestOutput(output)
# Check whether there are any crashed testcases.
self._crashes.update(
r.GetName() for r in results
if r.GetType() == base_test_result.ResultType.CRASH)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S', time.localtime()), device.serial)
tombstones_url = tombstones.LogdogTombstones(
resolved_tombstones, stream_name)
result.SetTombstonesUrl(tombstones_url)
not_run_tests = set(test).difference(set(r.GetName() for r in results))
return results, list(not_run_tests)
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout *
self.GetTool(device).GetTimeoutScale())
if self._test_instance.wait_for_java_debugger:
timeout = None
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
test_perf_output_filename = next(self._test_perf_output_filenames)
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
with contextlib_ext.Optional(
device_temp_file.NamedDeviceTemporaryDirectory(
adb=device.adb, dir='/sdcard/'), self._test_instance.
gs_test_artifacts_bucket) as test_artifacts_dir:
with (contextlib_ext.Optional(
device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device)),
test_perf_output_filename)
) as isolated_script_test_perf_output:
flags = list(self._test_instance.flags)
if self._test_instance.enable_xml_result_parsing:
flags.append('--gtest_output=xml:%s' %
device_tmp_results_file.name)
if self._test_instance.gs_test_artifacts_bucket:
flags.append('--test_artifacts_dir=%s' %
test_artifacts_dir.name)
if test_perf_output_filename:
flags.append('--isolated_script_test_perf_output=%s' %
isolated_script_test_perf_output.name)
logging.info('flags:')
for f in flags:
logging.info(' %s', f)
stream_name = 'logcat_%s_%s_%s' % (
hash(tuple(test)),
time.strftime('%Y%m%dT%H%M%S-UTC',
time.gmtime()), device.serial)
with self._env.output_manager.ArchivedTempfile(
stream_name, 'logcat') as logcat_file:
with logcat_monitor.LogcatMonitor(
device.adb,
filter_specs=local_device_environment.
LOGCAT_FILTERS,
output_file=logcat_file.name,
check_error=False) as logmon:
with contextlib_ext.Optional(
trace_event.trace(str(test)),
self._env.trace_output):
output = self._delegate.Run(
test,
device,
flags=' '.join(flags),
timeout=timeout,
retries=0)
logmon.Close()
if logcat_file.Link():
logging.info('Logcat saved to %s', logcat_file.Link())
if self._test_instance.enable_xml_result_parsing:
try:
gtest_xml = device.ReadFile(
device_tmp_results_file.name, as_root=True)
except device_errors.CommandFailedError as e:
logging.warning(
'Failed to pull gtest results XML file %s: %s',
device_tmp_results_file.name, str(e))
gtest_xml = None
if test_perf_output_filename:
try:
device.PullFile(
isolated_script_test_perf_output.name,
test_perf_output_filename)
except device_errors.CommandFailedError as e:
logging.warning(
'Failed to pull chartjson results %s: %s',
isolated_script_test_perf_output.name, str(e))
test_artifacts_url = self._UploadTestArtifacts(
device, test_artifacts_dir)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
for l in output:
logging.info(l)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
results = gtest_test_instance.ParseGTestXML(gtest_xml)
else:
results = gtest_test_instance.ParseGTestOutput(
output, self._test_instance.symbolizer, device.product_cpu_abi)
tombstones_url = None
for r in results:
if logcat_file:
r.SetLink('logcat', logcat_file.Link())
if self._test_instance.gs_test_artifacts_bucket:
r.SetLink('test_artifacts', test_artifacts_url)
if r.GetType() == base_test_result.ResultType.CRASH:
self._crashes.add(r.GetName())
if self._test_instance.store_tombstones:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S', time.localtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, '\n'.join(resolved_tombstones))
r.SetLink('tombstones', tombstones_url)
tests_stripped_disabled_prefix = set()
for t in test:
tests_stripped_disabled_prefix.add(
gtest_test_instance.TestNameWithoutDisabledPrefix(t))
not_run_tests = tests_stripped_disabled_prefix.difference(
set(r.GetName() for r in results))
return results, list(not_run_tests) if results else None
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout *
self.GetTool(device).GetTimeoutScale())
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
flags = list(self._test_instance.flags)
if self._test_instance.enable_xml_result_parsing:
flags.append('--gtest_output=xml:%s' %
device_tmp_results_file.name)
logging.info('flags:')
for f in flags:
logging.info(' %s', f)
with contextlib_ext.Optional(trace_event.trace(str(test)),
self._env.trace_output):
output = self._delegate.Run(test,
device,
flags=' '.join(flags),
timeout=timeout,
retries=0)
if self._test_instance.enable_xml_result_parsing:
gtest_xml = device.ReadFile(device_tmp_results_file.name,
as_root=True)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
results = gtest_test_instance.ParseGTestXML(gtest_xml)
else:
results = gtest_test_instance.ParseGTestOutput(output)
# Check whether there are any crashed testcases.
self._crashes.update(
r.GetName() for r in results
if r.GetType() == base_test_result.ResultType.CRASH)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S', time.localtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, resolved_tombstones)
result.SetLink('tombstones', tombstones_url)
tests_stripped_disabled_prefix = set()
for t in test:
tests_stripped_disabled_prefix.add(
gtest_test_instance.TestNameWithoutDisabledPrefix(t))
not_run_tests = tests_stripped_disabled_prefix.difference(
set(r.GetName() for r in results))
return results, list(not_run_tests) if results else None
def Run(self, test, device, flags=None, **kwargs):
extras = dict(self._extras)
device_api = device.build_version_sdk
if self._coverage_dir and device_api >= version_codes.LOLLIPOP:
coverage_device_file = device_temp_file.DeviceTempFile(
device.adb,
suffix='.profraw',
prefix=self._suite,
dir=device.GetExternalStoragePath())
extras[_EXTRA_COVERAGE_DEVICE_FILE] = coverage_device_file.name
if ('timeout' in kwargs and
gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT not in extras):
# Make sure the instrumentation doesn't kill the test before the
# scripts do. The provided timeout value is in seconds, but the
# instrumentation deals with nanoseconds because that's how Android
# handles time.
extras[gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT] = int(
kwargs['timeout'] * _SECONDS_TO_NANOS)
# pylint: disable=redefined-variable-type
command_line_file = _NullContextManager()
if flags:
if len(flags) > _MAX_INLINE_FLAGS_LENGTH:
command_line_file = device_temp_file.DeviceTempFile(device.adb)
device.WriteFile(command_line_file.name, '_ %s' % flags)
extras[_EXTRA_COMMAND_LINE_FILE] = command_line_file.name
else:
extras[_EXTRA_COMMAND_LINE_FLAGS] = flags
test_list_file = _NullContextManager()
if test:
if len(test) > 1:
test_list_file = device_temp_file.DeviceTempFile(device.adb)
device.WriteFile(test_list_file.name, '\n'.join(test))
extras[_EXTRA_TEST_LIST] = test_list_file.name
else:
extras[_EXTRA_TEST] = test[0]
# pylint: enable=redefined-variable-type
stdout_file = device_temp_file.DeviceTempFile(
device.adb,
dir=device.GetExternalStoragePath(),
suffix='.gtest_out')
extras[_EXTRA_STDOUT_FILE] = stdout_file.name
if self._wait_for_java_debugger:
cmd = ['am', 'set-debug-app', '-w', self._package]
device.RunShellCommand(cmd, check_return=True)
logging.warning('*' * 80)
logging.warning('Waiting for debugger to attach to process: %s',
self._package)
logging.warning('*' * 80)
with command_line_file, test_list_file, stdout_file:
try:
device.StartInstrumentation(self._component,
extras=extras,
raw=False,
**kwargs)
except device_errors.CommandFailedError:
logging.exception('gtest shard failed.')
except device_errors.CommandTimeoutError:
logging.exception('gtest shard timed out.')
except device_errors.DeviceUnreachableError:
logging.exception('gtest shard device unreachable.')
except Exception:
device.ForceStop(self._package)
raise
finally:
if self._coverage_dir and device_api >= version_codes.LOLLIPOP:
_PullCoverageFile(device, coverage_device_file,
self._coverage_dir)
# TODO(jbudorick): Remove this after resolving crbug.com/726880
if device.PathExists(stdout_file.name):
logging.info(
'%s size on device: %s', stdout_file.name,
device.StatPath(stdout_file.name).get('st_size', 0))
return device.ReadFile(stdout_file.name).splitlines()
else:
logging.info('%s does not exist?', stdout_file.name)
return []
def LogDeviceProperties(presentation.device):
props = presentation.device.RunShellCommand(['getprop'], check_return=True)
for prop in props:
logger.info(' %s', prop)
def CheckExternalStorage(presentation.device):
"""Checks that storage is writable and if not makes it writable.
Arguments:
presentation.device: The presentation.device to check.
"""
try:
with device_temp_file.DeviceTempFile(
presentation.device.adb, suffix='.sh', dir=presentation.device.GetExternalStoragePath()) as f:
presentation.device.WriteFile(f.name, 'test')
except device_errors.CommandFailedError:
logger.info('External storage not writable. Remounting / as RW')
presentation.device.RunShellCommand(['mount', '-o', 'remount,rw', '/'],
check_return=True, as_root=True)
presentation.device.EnableRoot()
with device_temp_file.DeviceTempFile(
presentation.device.adb, suffix='.sh', dir=presentation.device.GetExternalStoragePath()) as f:
presentation.device.WriteFile(f.name, 'test')
def presentation.main(raw_args):
# Recommended options on perf bots:
# --disable-network
# TODO(tonyg): We eventually want network on. However, currently radios
def _TempFile(self, **kwargs):
return device_temp_file.DeviceTempFile(self._device.adb, **kwargs)
def Run(self, test, device, flags=None, **kwargs):
extras = dict(self._extras)
device_api = device.build_version_sdk
if self._coverage_dir and device_api >= version_codes.LOLLIPOP:
device_coverage_dir = _GetDeviceCoverageDir(device)
extras[_EXTRA_COVERAGE_DEVICE_FILE] = _GetLLVMProfilePath(
device_coverage_dir, self._suite, self._coverage_index)
self._coverage_index += 1
if ('timeout' in kwargs and
gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT not in extras):
# Make sure the instrumentation doesn't kill the test before the
# scripts do. The provided timeout value is in seconds, but the
# instrumentation deals with nanoseconds because that's how Android
# handles time.
extras[gtest_test_instance.EXTRA_SHARD_NANO_TIMEOUT] = int(
kwargs['timeout'] * _SECONDS_TO_NANOS)
# pylint: disable=redefined-variable-type
command_line_file = _NullContextManager()
if flags:
if len(flags) > _MAX_INLINE_FLAGS_LENGTH:
command_line_file = device_temp_file.DeviceTempFile(device.adb)
device.WriteFile(command_line_file.name, '_ %s' % flags)
extras[_EXTRA_COMMAND_LINE_FILE] = command_line_file.name
else:
extras[_EXTRA_COMMAND_LINE_FLAGS] = flags
test_list_file = _NullContextManager()
if test:
if len(test) > 1:
test_list_file = device_temp_file.DeviceTempFile(device.adb)
device.WriteFile(test_list_file.name, '\n'.join(test))
extras[_EXTRA_TEST_LIST] = test_list_file.name
else:
extras[_EXTRA_TEST] = test[0]
# pylint: enable=redefined-variable-type
# We need to use GetAppWritablePath here instead of GetExternalStoragePath
# since we will not have yet applied legacy storage permission workarounds
# on R+.
stdout_file = device_temp_file.DeviceTempFile(
device.adb, dir=device.GetAppWritablePath(), suffix='.gtest_out')
extras[_EXTRA_STDOUT_FILE] = stdout_file.name
if self._wait_for_java_debugger:
cmd = ['am', 'set-debug-app', '-w', self._package]
device.RunShellCommand(cmd, check_return=True)
logging.warning('*' * 80)
logging.warning('Waiting for debugger to attach to process: %s',
self._package)
logging.warning('*' * 80)
with command_line_file, test_list_file, stdout_file:
try:
device.StartInstrumentation(self._component,
extras=extras,
raw=False,
**kwargs)
except device_errors.CommandFailedError:
logging.exception('gtest shard failed.')
except device_errors.CommandTimeoutError:
logging.exception('gtest shard timed out.')
except device_errors.DeviceUnreachableError:
logging.exception('gtest shard device unreachable.')
except Exception:
device.ForceStop(self._package)
raise
finally:
if self._coverage_dir and device_api >= version_codes.LOLLIPOP:
if not os.path.isdir(self._coverage_dir):
os.makedirs(self._coverage_dir)
# TODO(crbug.com/1179004) Use _MergeCoverageFiles when llvm-profdata
# not found is fixed.
_PullCoverageFiles(
device, device_coverage_dir,
os.path.join(self._coverage_dir,
str(self._coverage_index)))
return device.ReadFile(stdout_file.name).splitlines()
