def OptionalPerTestLogcat( device, test_name, condition, additional_filter_specs=None, deobfuscate_func=None): """Conditionally capture logcat and stream it to logdog. Args: device: (DeviceUtils) the device from which logcat should be captured. test_name: (str) the test name to use in the stream name. condition: (bool) whether or not to capture the logcat. additional_filter_specs: (list) additional logcat filters. deobfuscate_func: (callable) an optional unary function that deobfuscates logcat lines. The callable should take an iterable of logcat lines and return a list of deobfuscated logcat lines. Yields: A LogdogLogcatMonitor instance whether condition is true or not, though it may not be active. """ stream_name = 'logcat_%s_%s_%s' % ( test_name, time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial) filter_specs = LOGCAT_FILTERS + (additional_filter_specs or []) logmon = logdog_logcat_monitor.LogdogLogcatMonitor( device.adb, stream_name, filter_specs=filter_specs, deobfuscate_func=deobfuscate_func) with contextlib_ext.Optional(logmon, condition): yield logmon
def _RunTest(self, device, test): extras = {} flags_to_add = [] test_timeout_scale = None if self._test_instance.coverage_directory: coverage_basename = '%s.ec' % ('%s_group' % test[0]['method'] if isinstance( test, list) else test['method']) extras['coverage'] = 'true' coverage_directory = os.path.join(device.GetExternalStoragePath(), 'chrome', 'test', 'coverage') coverage_device_file = os.path.join(coverage_directory, coverage_basename) extras['coverageFile'] = coverage_device_file # Save screenshot if screenshot dir is specified (save locally) or if # a GS bucket is passed (save in cloud). screenshot_device_file = None if (self._test_instance.screenshot_dir or self._test_instance.gs_results_bucket): screenshot_device_file = device_temp_file.DeviceTempFile( device.adb, suffix='.png', dir=device.GetExternalStoragePath()) extras[EXTRA_SCREENSHOT_FILE] = screenshot_device_file.name extras[EXTRA_UI_CAPTURE_DIR] = self._ui_capture_dir[device] if isinstance(test, list): if not self._test_instance.driver_apk: raise Exception('driver_apk does not exist. ' 'Please build it and try again.') if any(t.get('is_junit4') for t in test): raise Exception('driver apk does not support JUnit4 tests') def name_and_timeout(t): n = instrumentation_test_instance.GetTestName(t) i = self._GetTimeoutFromAnnotations(t['annotations'], n) return (n, i) test_names, timeouts = zip(*(name_and_timeout(t) for t in test)) test_name = ','.join(test_names) test_display_name = test_name target = '%s/%s' % (self._test_instance.driver_package, self._test_instance.driver_name) extras.update( self._test_instance.GetDriverEnvironmentVars( test_list=test_names)) timeout = sum(timeouts) else: test_name = instrumentation_test_instance.GetTestName(test) test_display_name = self._GetUniqueTestName(test) if test['is_junit4']: target = '%s/%s' % (self._test_instance.test_package, self._test_instance.test_runner_junit4) else: target = '%s/%s' % (self._test_instance.test_package, self._test_instance.test_runner) extras['class'] = test_name if 'flags' in test and test['flags']: flags_to_add.extend(test['flags']) timeout = self._GetTimeoutFromAnnotations(test['annotations'], test_display_name) test_timeout_scale = self._GetTimeoutScaleFromAnnotations( test['annotations']) if test_timeout_scale and test_timeout_scale != 1: valgrind_tools.SetChromeTimeoutScale( device, test_timeout_scale * self._test_instance.timeout_scale) logging.info('preparing to run %s: %s', test_display_name, test) render_tests_device_output_dir = None if _IsRenderTest(test): # TODO(mikecase): Add DeviceTempDirectory class and use that instead. render_tests_device_output_dir = posixpath.join( device.GetExternalStoragePath(), 'render_test_output_dir') flags_to_add.append('--render-test-output-dir=%s' % render_tests_device_output_dir) if flags_to_add: self._CreateFlagChangerIfNeeded(device) self._flag_changers[str(device)].PushFlags(add=flags_to_add) time_ms = lambda: int(time.time() * 1e3) start_ms = time_ms() stream_name = 'logcat_%s_%s_%s' % (test_name.replace( '#', '.'), time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial) logmon = logdog_logcat_monitor.LogdogLogcatMonitor( device.adb, stream_name, filter_specs=LOGCAT_FILTERS) with contextlib_ext.Optional(logmon, self._test_instance.should_save_logcat): with _LogTestEndpoints(device, test_name): with contextlib_ext.Optional(trace_event.trace(test_name), self._env.trace_output): output = device.StartInstrumentation(target, raw=True, extras=extras, timeout=timeout, retries=0) logcat_url = logmon.GetLogcatURL() duration_ms = time_ms() - start_ms # TODO(jbudorick): Make instrumentation tests output a JSON so this # doesn't have to parse the output. result_code, result_bundle, statuses = ( self._test_instance.ParseAmInstrumentRawOutput(output)) results = self._test_instance.GenerateTestResults( result_code, result_bundle, statuses, start_ms, duration_ms) def restore_flags(): if flags_to_add: self._flag_changers[str(device)].Restore() def restore_timeout_scale(): if test_timeout_scale: valgrind_tools.SetChromeTimeoutScale( device, self._test_instance.timeout_scale) def handle_coverage_data(): if self._test_instance.coverage_directory: device.PullFile(coverage_directory, self._test_instance.coverage_directory) device.RunShellCommand('rm -f %s' % posixpath.join(coverage_directory, '*'), check_return=True, shell=True) def handle_render_test_data(): if _IsRenderTest(test): # Render tests do not cause test failure by default. So we have to check # to see if any failure images were generated even if the test does not # fail. try: self._ProcessRenderTestResults( device, render_tests_device_output_dir, results) finally: device.RemovePath(render_tests_device_output_dir, recursive=True, force=True) # While constructing the TestResult objects, we can parallelize several # steps that involve ADB. These steps should NOT depend on any info in # the results! Things such as whether the test CRASHED have not yet been # determined. post_test_steps = [ restore_flags, restore_timeout_scale, handle_coverage_data, handle_render_test_data ] if self._env.concurrent_adb: post_test_step_thread_group = reraiser_thread.ReraiserThreadGroup( reraiser_thread.ReraiserThread(f) for f in post_test_steps) post_test_step_thread_group.StartAll(will_block=True) else: for step in post_test_steps: step() for result in results: if logcat_url: result.SetLink('logcat', logcat_url) # Update the result name if the test used flags. if flags_to_add: for r in results: if r.GetName() == test_name: r.SetName(test_display_name) # Add UNKNOWN results for any missing tests. iterable_test = test if isinstance(test, list) else [test] test_names = set(self._GetUniqueTestName(t) for t in iterable_test) results_names = set(r.GetName() for r in results) results.extend( base_test_result.BaseTestResult( u, base_test_result.ResultType.UNKNOWN) for u in test_names.difference(results_names)) # Update the result type if we detect a crash. if DidPackageCrashOnDevice(self._test_instance.test_package, device): for r in results: if r.GetType() == base_test_result.ResultType.UNKNOWN: r.SetType(base_test_result.ResultType.CRASH) # Handle failures by: # - optionally taking a screenshot # - logging the raw output at INFO level # - clearing the application state while persisting permissions if any(r.GetType() not in (base_test_result.ResultType.PASS, base_test_result.ResultType.SKIP) for r in results): with contextlib_ext.Optional( tempfile_ext.NamedTemporaryDirectory(), self._test_instance.screenshot_dir is None and self._test_instance.gs_results_bucket ) as screenshot_host_dir: screenshot_host_dir = (self._test_instance.screenshot_dir or screenshot_host_dir) self._SaveScreenshot(device, screenshot_host_dir, screenshot_device_file, test_display_name, results) logging.info('detected failure in %s. raw output:', test_display_name) for l in output: logging.info(' %s', l) if (not self._env.skip_clear_data and self._test_instance.package_info): permissions = ( self._test_instance.apk_under_test.GetPermissions() if self._test_instance.apk_under_test else None) device.ClearApplicationState( self._test_instance.package_info.package, permissions=permissions) else: logging.debug('raw output from %s:', test_display_name) for l in output: logging.debug(' %s', l) if self._test_instance.store_tombstones: tombstones_url = None for result in results: if result.GetType() == base_test_result.ResultType.CRASH: if not tombstones_url: resolved_tombstones = tombstones.ResolveTombstones( device, resolve_all_tombstones=True, include_stack_symbols=False, wipe_tombstones=True) stream_name = 'tombstones_%s_%s' % (time.strftime( '%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial) tombstones_url = logdog_helper.text( stream_name, '\n'.join(resolved_tombstones)) result.SetLink('tombstones', tombstones_url) if self._env.concurrent_adb: post_test_step_thread_group.JoinAll() return results, None
def _RunTest(self, device, test): extras = {} flags = None test_timeout_scale = None if self._test_instance.coverage_directory: coverage_basename = '%s.ec' % ('%s_group' % test[0]['method'] if isinstance( test, list) else test['method']) extras['coverage'] = 'true' coverage_directory = os.path.join(device.GetExternalStoragePath(), 'chrome', 'test', 'coverage') coverage_device_file = os.path.join(coverage_directory, coverage_basename) extras['coverageFile'] = coverage_device_file if isinstance(test, list): if not self._test_instance.driver_apk: raise Exception('driver_apk does not exist. ' 'Please build it and try again.') if any(t.get('is_junit4') for t in test): raise Exception('driver apk does not support JUnit4 tests') def name_and_timeout(t): n = instrumentation_test_instance.GetTestName(t) i = self._GetTimeoutFromAnnotations(t['annotations'], n) return (n, i) test_names, timeouts = zip(*(name_and_timeout(t) for t in test)) test_name = ','.join(test_names) test_display_name = test_name target = '%s/%s' % (self._test_instance.driver_package, self._test_instance.driver_name) extras.update( self._test_instance.GetDriverEnvironmentVars( test_list=test_names)) timeout = sum(timeouts) else: test_name = instrumentation_test_instance.GetTestName(test) test_display_name = self._GetUniqueTestName(test) if test['is_junit4']: target = '%s/%s' % (self._test_instance.test_package, self._test_instance.test_runner_junit4) else: target = '%s/%s' % (self._test_instance.test_package, self._test_instance.test_runner) extras['class'] = test_name if 'flags' in test: flags = test['flags'] timeout = self._GetTimeoutFromAnnotations(test['annotations'], test_display_name) test_timeout_scale = self._GetTimeoutScaleFromAnnotations( test['annotations']) if test_timeout_scale and test_timeout_scale != 1: valgrind_tools.SetChromeTimeoutScale( device, test_timeout_scale * self._test_instance.timeout_scale) logging.info('preparing to run %s: %s', test_display_name, test) if flags: self._CreateFlagChangerIfNeeded(device) self._flag_changers[str(device)].PushFlags(add=flags.add, remove=flags.remove) try: device.RunShellCommand( ['log', '-p', 'i', '-t', _TAG, 'START %s' % test_name], check_return=True) logcat_url = None time_ms = lambda: int(time.time() * 1e3) start_ms = time_ms() stream_name = 'logcat_%s_%s_%s' % (test_name.replace( '#', '.'), time.strftime('%Y%m%dT%H%M%S', time.localtime()), device.serial) with contextlib_ext.Optional( logdog_logcat_monitor.LogdogLogcatMonitor( device.adb, stream_name), self._test_instance.should_save_logcat) as logmon: with contextlib_ext.Optional(trace_event.trace(test_name), self._env.trace_output): output = device.StartInstrumentation(target, raw=True, extras=extras, timeout=timeout, retries=0) if logmon: logcat_url = logmon.GetLogcatURL() finally: device.RunShellCommand( ['log', '-p', 'i', '-t', _TAG, 'END %s' % test_name], check_return=True) duration_ms = time_ms() - start_ms if flags: self._flag_changers[str(device)].Restore() if test_timeout_scale: valgrind_tools.SetChromeTimeoutScale( device, self._test_instance.timeout_scale) # TODO(jbudorick): Make instrumentation tests output a JSON so this # doesn't have to parse the output. result_code, result_bundle, statuses = ( self._test_instance.ParseAmInstrumentRawOutput(output)) results = self._test_instance.GenerateTestResults( result_code, result_bundle, statuses, start_ms, duration_ms) for result in results: if logcat_url: result.SetLink('logcat', logcat_url) # Update the result name if the test used flags. if flags: for r in results: if r.GetName() == test_name: r.SetName(test_display_name) # Add UNKNOWN results for any missing tests. iterable_test = test if isinstance(test, list) else [test] test_names = set(self._GetUniqueTestName(t) for t in iterable_test) results_names = set(r.GetName() for r in results) results.extend( base_test_result.BaseTestResult( u, base_test_result.ResultType.UNKNOWN) for u in test_names.difference(results_names)) # Update the result type if we detect a crash. if DidPackageCrashOnDevice(self._test_instance.test_package, device): for r in results: if r.GetType() == base_test_result.ResultType.UNKNOWN: r.SetType(base_test_result.ResultType.CRASH) # Handle failures by: # - optionally taking a screenshot # - logging the raw output at INFO level # - clearing the application state while persisting permissions if any(r.GetType() not in (base_test_result.ResultType.PASS, base_test_result.ResultType.SKIP) for r in results): if self._test_instance.screenshot_dir: file_name = '%s-%s.png' % ( test_display_name, time.strftime('%Y%m%dT%H%M%S', time.localtime())) saved_dir = device.TakeScreenshot( os.path.join(self._test_instance.screenshot_dir, file_name)) logging.info('Saved screenshot for %s to %s.', test_display_name, saved_dir) logging.info('detected failure in %s. raw output:', test_display_name) for l in output: logging.info(' %s', l) if (not self._env.skip_clear_data and self._test_instance.package_info): permissions = ( self._test_instance.apk_under_test.GetPermissions() if self._test_instance.apk_under_test else None) device.ClearApplicationState( self._test_instance.package_info.package, permissions=permissions) else: logging.debug('raw output from %s:', test_display_name) for l in output: logging.debug(' %s', l) if self._test_instance.coverage_directory: device.PullFile(coverage_directory, self._test_instance.coverage_directory) device.RunShellCommand('rm -f %s' % os.path.join(coverage_directory, '*')) if self._test_instance.store_tombstones: tombstones_url = None for result in results: if result.GetType() == base_test_result.ResultType.CRASH: if not tombstones_url: resolved_tombstones = tombstones.ResolveTombstones( device, resolve_all_tombstones=True, include_stack_symbols=False, wipe_tombstones=True) stream_name = 'tombstones_%s_%s' % (time.strftime( '%Y%m%dT%H%M%S', time.localtime()), device.serial) tombstones_url = logdog_helper.text( stream_name, resolved_tombstones) result.SetLink('tombstones', tombstones_url) return results, None