def OptionalPerTestLogcat(
device, test_name, condition, additional_filter_specs=None,
deobfuscate_func=None):
"""Conditionally capture logcat and stream it to logdog.
Args:
device: (DeviceUtils) the device from which logcat should be captured.
test_name: (str) the test name to use in the stream name.
condition: (bool) whether or not to capture the logcat.
additional_filter_specs: (list) additional logcat filters.
deobfuscate_func: (callable) an optional unary function that
deobfuscates logcat lines. The callable should take an iterable
of logcat lines and return a list of deobfuscated logcat lines.
Yields:
A LogdogLogcatMonitor instance whether condition is true or not,
though it may not be active.
"""
stream_name = 'logcat_%s_%s_%s' % (
test_name,
time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()),
device.serial)
filter_specs = LOGCAT_FILTERS + (additional_filter_specs or [])
logmon = logdog_logcat_monitor.LogdogLogcatMonitor(
device.adb, stream_name, filter_specs=filter_specs,
deobfuscate_func=deobfuscate_func)
with contextlib_ext.Optional(logmon, condition):
yield logmon
def _RunTest(self, device, test):
extras = {}
flags_to_add = []
test_timeout_scale = None
if self._test_instance.coverage_directory:
coverage_basename = '%s.ec' % ('%s_group' %
test[0]['method'] if isinstance(
test, list) else test['method'])
extras['coverage'] = 'true'
coverage_directory = os.path.join(device.GetExternalStoragePath(),
'chrome', 'test', 'coverage')
coverage_device_file = os.path.join(coverage_directory,
coverage_basename)
extras['coverageFile'] = coverage_device_file
# Save screenshot if screenshot dir is specified (save locally) or if
# a GS bucket is passed (save in cloud).
screenshot_device_file = None
if (self._test_instance.screenshot_dir
or self._test_instance.gs_results_bucket):
screenshot_device_file = device_temp_file.DeviceTempFile(
device.adb, suffix='.png', dir=device.GetExternalStoragePath())
extras[EXTRA_SCREENSHOT_FILE] = screenshot_device_file.name
extras[EXTRA_UI_CAPTURE_DIR] = self._ui_capture_dir[device]
if isinstance(test, list):
if not self._test_instance.driver_apk:
raise Exception('driver_apk does not exist. '
'Please build it and try again.')
if any(t.get('is_junit4') for t in test):
raise Exception('driver apk does not support JUnit4 tests')
def name_and_timeout(t):
n = instrumentation_test_instance.GetTestName(t)
i = self._GetTimeoutFromAnnotations(t['annotations'], n)
return (n, i)
test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
test_name = ','.join(test_names)
test_display_name = test_name
target = '%s/%s' % (self._test_instance.driver_package,
self._test_instance.driver_name)
extras.update(
self._test_instance.GetDriverEnvironmentVars(
test_list=test_names))
timeout = sum(timeouts)
else:
test_name = instrumentation_test_instance.GetTestName(test)
test_display_name = self._GetUniqueTestName(test)
if test['is_junit4']:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.test_runner_junit4)
else:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.test_runner)
extras['class'] = test_name
if 'flags' in test and test['flags']:
flags_to_add.extend(test['flags'])
timeout = self._GetTimeoutFromAnnotations(test['annotations'],
test_display_name)
test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
test['annotations'])
if test_timeout_scale and test_timeout_scale != 1:
valgrind_tools.SetChromeTimeoutScale(
device,
test_timeout_scale * self._test_instance.timeout_scale)
logging.info('preparing to run %s: %s', test_display_name, test)
render_tests_device_output_dir = None
if _IsRenderTest(test):
# TODO(mikecase): Add DeviceTempDirectory class and use that instead.
render_tests_device_output_dir = posixpath.join(
device.GetExternalStoragePath(), 'render_test_output_dir')
flags_to_add.append('--render-test-output-dir=%s' %
render_tests_device_output_dir)
if flags_to_add:
self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags(add=flags_to_add)
time_ms = lambda: int(time.time() * 1e3)
start_ms = time_ms()
stream_name = 'logcat_%s_%s_%s' % (test_name.replace(
'#', '.'), time.strftime('%Y%m%dT%H%M%S-UTC',
time.gmtime()), device.serial)
logmon = logdog_logcat_monitor.LogdogLogcatMonitor(
device.adb, stream_name, filter_specs=LOGCAT_FILTERS)
with contextlib_ext.Optional(logmon,
self._test_instance.should_save_logcat):
with _LogTestEndpoints(device, test_name):
with contextlib_ext.Optional(trace_event.trace(test_name),
self._env.trace_output):
output = device.StartInstrumentation(target,
raw=True,
extras=extras,
timeout=timeout,
retries=0)
logcat_url = logmon.GetLogcatURL()
duration_ms = time_ms() - start_ms
# TODO(jbudorick): Make instrumentation tests output a JSON so this
# doesn't have to parse the output.
result_code, result_bundle, statuses = (
self._test_instance.ParseAmInstrumentRawOutput(output))
results = self._test_instance.GenerateTestResults(
result_code, result_bundle, statuses, start_ms, duration_ms)
def restore_flags():
if flags_to_add:
self._flag_changers[str(device)].Restore()
def restore_timeout_scale():
if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale(
device, self._test_instance.timeout_scale)
def handle_coverage_data():
if self._test_instance.coverage_directory:
device.PullFile(coverage_directory,
self._test_instance.coverage_directory)
device.RunShellCommand('rm -f %s' %
posixpath.join(coverage_directory, '*'),
check_return=True,
shell=True)
def handle_render_test_data():
if _IsRenderTest(test):
# Render tests do not cause test failure by default. So we have to check
# to see if any failure images were generated even if the test does not
# fail.
try:
self._ProcessRenderTestResults(
device, render_tests_device_output_dir, results)
finally:
device.RemovePath(render_tests_device_output_dir,
recursive=True,
force=True)
# While constructing the TestResult objects, we can parallelize several
# steps that involve ADB. These steps should NOT depend on any info in
# the results! Things such as whether the test CRASHED have not yet been
# determined.
post_test_steps = [
restore_flags, restore_timeout_scale, handle_coverage_data,
handle_render_test_data
]
if self._env.concurrent_adb:
post_test_step_thread_group = reraiser_thread.ReraiserThreadGroup(
reraiser_thread.ReraiserThread(f) for f in post_test_steps)
post_test_step_thread_group.StartAll(will_block=True)
else:
for step in post_test_steps:
step()
for result in results:
if logcat_url:
result.SetLink('logcat', logcat_url)
# Update the result name if the test used flags.
if flags_to_add:
for r in results:
if r.GetName() == test_name:
r.SetName(test_display_name)
# Add UNKNOWN results for any missing tests.
iterable_test = test if isinstance(test, list) else [test]
test_names = set(self._GetUniqueTestName(t) for t in iterable_test)
results_names = set(r.GetName() for r in results)
results.extend(
base_test_result.BaseTestResult(
u, base_test_result.ResultType.UNKNOWN)
for u in test_names.difference(results_names))
# Update the result type if we detect a crash.
if DidPackageCrashOnDevice(self._test_instance.test_package, device):
for r in results:
if r.GetType() == base_test_result.ResultType.UNKNOWN:
r.SetType(base_test_result.ResultType.CRASH)
# Handle failures by:
# - optionally taking a screenshot
# - logging the raw output at INFO level
# - clearing the application state while persisting permissions
if any(r.GetType() not in (base_test_result.ResultType.PASS,
base_test_result.ResultType.SKIP)
for r in results):
with contextlib_ext.Optional(
tempfile_ext.NamedTemporaryDirectory(),
self._test_instance.screenshot_dir is None
and self._test_instance.gs_results_bucket
) as screenshot_host_dir:
screenshot_host_dir = (self._test_instance.screenshot_dir
or screenshot_host_dir)
self._SaveScreenshot(device, screenshot_host_dir,
screenshot_device_file, test_display_name,
results)
logging.info('detected failure in %s. raw output:',
test_display_name)
for l in output:
logging.info(' %s', l)
if (not self._env.skip_clear_data
and self._test_instance.package_info):
permissions = (
self._test_instance.apk_under_test.GetPermissions()
if self._test_instance.apk_under_test else None)
device.ClearApplicationState(
self._test_instance.package_info.package,
permissions=permissions)
else:
logging.debug('raw output from %s:', test_display_name)
for l in output:
logging.debug(' %s', l)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, '\n'.join(resolved_tombstones))
result.SetLink('tombstones', tombstones_url)
if self._env.concurrent_adb:
post_test_step_thread_group.JoinAll()
return results, None
def _RunTest(self, device, test):
extras = {}
flags = None
test_timeout_scale = None
if self._test_instance.coverage_directory:
coverage_basename = '%s.ec' % ('%s_group' %
test[0]['method'] if isinstance(
test, list) else test['method'])
extras['coverage'] = 'true'
coverage_directory = os.path.join(device.GetExternalStoragePath(),
'chrome', 'test', 'coverage')
coverage_device_file = os.path.join(coverage_directory,
coverage_basename)
extras['coverageFile'] = coverage_device_file
if isinstance(test, list):
if not self._test_instance.driver_apk:
raise Exception('driver_apk does not exist. '
'Please build it and try again.')
if any(t.get('is_junit4') for t in test):
raise Exception('driver apk does not support JUnit4 tests')
def name_and_timeout(t):
n = instrumentation_test_instance.GetTestName(t)
i = self._GetTimeoutFromAnnotations(t['annotations'], n)
return (n, i)
test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
test_name = ','.join(test_names)
test_display_name = test_name
target = '%s/%s' % (self._test_instance.driver_package,
self._test_instance.driver_name)
extras.update(
self._test_instance.GetDriverEnvironmentVars(
test_list=test_names))
timeout = sum(timeouts)
else:
test_name = instrumentation_test_instance.GetTestName(test)
test_display_name = self._GetUniqueTestName(test)
if test['is_junit4']:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.test_runner_junit4)
else:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.test_runner)
extras['class'] = test_name
if 'flags' in test:
flags = test['flags']
timeout = self._GetTimeoutFromAnnotations(test['annotations'],
test_display_name)
test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
test['annotations'])
if test_timeout_scale and test_timeout_scale != 1:
valgrind_tools.SetChromeTimeoutScale(
device,
test_timeout_scale * self._test_instance.timeout_scale)
logging.info('preparing to run %s: %s', test_display_name, test)
if flags:
self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags(add=flags.add,
remove=flags.remove)
try:
device.RunShellCommand(
['log', '-p', 'i', '-t', _TAG,
'START %s' % test_name],
check_return=True)
logcat_url = None
time_ms = lambda: int(time.time() * 1e3)
start_ms = time_ms()
stream_name = 'logcat_%s_%s_%s' % (test_name.replace(
'#', '.'), time.strftime('%Y%m%dT%H%M%S',
time.localtime()), device.serial)
with contextlib_ext.Optional(
logdog_logcat_monitor.LogdogLogcatMonitor(
device.adb, stream_name),
self._test_instance.should_save_logcat) as logmon:
with contextlib_ext.Optional(trace_event.trace(test_name),
self._env.trace_output):
output = device.StartInstrumentation(target,
raw=True,
extras=extras,
timeout=timeout,
retries=0)
if logmon:
logcat_url = logmon.GetLogcatURL()
finally:
device.RunShellCommand(
['log', '-p', 'i', '-t', _TAG,
'END %s' % test_name],
check_return=True)
duration_ms = time_ms() - start_ms
if flags:
self._flag_changers[str(device)].Restore()
if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale(
device, self._test_instance.timeout_scale)
# TODO(jbudorick): Make instrumentation tests output a JSON so this
# doesn't have to parse the output.
result_code, result_bundle, statuses = (
self._test_instance.ParseAmInstrumentRawOutput(output))
results = self._test_instance.GenerateTestResults(
result_code, result_bundle, statuses, start_ms, duration_ms)
for result in results:
if logcat_url:
result.SetLink('logcat', logcat_url)
# Update the result name if the test used flags.
if flags:
for r in results:
if r.GetName() == test_name:
r.SetName(test_display_name)
# Add UNKNOWN results for any missing tests.
iterable_test = test if isinstance(test, list) else [test]
test_names = set(self._GetUniqueTestName(t) for t in iterable_test)
results_names = set(r.GetName() for r in results)
results.extend(
base_test_result.BaseTestResult(
u, base_test_result.ResultType.UNKNOWN)
for u in test_names.difference(results_names))
# Update the result type if we detect a crash.
if DidPackageCrashOnDevice(self._test_instance.test_package, device):
for r in results:
if r.GetType() == base_test_result.ResultType.UNKNOWN:
r.SetType(base_test_result.ResultType.CRASH)
# Handle failures by:
# - optionally taking a screenshot
# - logging the raw output at INFO level
# - clearing the application state while persisting permissions
if any(r.GetType() not in (base_test_result.ResultType.PASS,
base_test_result.ResultType.SKIP)
for r in results):
if self._test_instance.screenshot_dir:
file_name = '%s-%s.png' % (
test_display_name,
time.strftime('%Y%m%dT%H%M%S', time.localtime()))
saved_dir = device.TakeScreenshot(
os.path.join(self._test_instance.screenshot_dir,
file_name))
logging.info('Saved screenshot for %s to %s.',
test_display_name, saved_dir)
logging.info('detected failure in %s. raw output:',
test_display_name)
for l in output:
logging.info(' %s', l)
if (not self._env.skip_clear_data
and self._test_instance.package_info):
permissions = (
self._test_instance.apk_under_test.GetPermissions()
if self._test_instance.apk_under_test else None)
device.ClearApplicationState(
self._test_instance.package_info.package,
permissions=permissions)
else:
logging.debug('raw output from %s:', test_display_name)
for l in output:
logging.debug(' %s', l)
if self._test_instance.coverage_directory:
device.PullFile(coverage_directory,
self._test_instance.coverage_directory)
device.RunShellCommand('rm -f %s' %
os.path.join(coverage_directory, '*'))
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S', time.localtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, resolved_tombstones)
result.SetLink('tombstones', tombstones_url)
return results, None
