def ProcessCommandLineArgs(cls, parser, args, environment): all_benchmarks = _Benchmarks(environment) if not args.positional_args: possible_browser = ( browser_finder.FindBrowser(args) if args.browser_type else None) PrintBenchmarkList(all_benchmarks, possible_browser) sys.exit(-1) input_benchmark_name = args.positional_args[0] matching_benchmarks = _MatchBenchmarkName(input_benchmark_name, environment) if not matching_benchmarks: print >> sys.stderr, 'No benchmark named "%s".' % input_benchmark_name print >> sys.stderr most_likely_matched_benchmarks = command_line.GetMostLikelyMatchedObject( all_benchmarks, input_benchmark_name, lambda x: x.Name()) if most_likely_matched_benchmarks: print >> sys.stderr, 'Do you mean any of those benchmarks below?' PrintBenchmarkList(most_likely_matched_benchmarks, None, sys.stderr) sys.exit(-1) if len(matching_benchmarks) > 1: print >> sys.stderr, ('Multiple benchmarks named "%s".' % input_benchmark_name) print >> sys.stderr, 'Did you mean one of these?' print >> sys.stderr PrintBenchmarkList(matching_benchmarks, None, sys.stderr) sys.exit(-1) benchmark_class = matching_benchmarks.pop() if len(args.positional_args) > 1: parser.error('Too many arguments.') assert issubclass(benchmark_class, benchmark.Benchmark), ( 'Trying to run a non-Benchmark?!') benchmark.ProcessCommandLineArgs(parser, args) benchmark_class.ProcessCommandLineArgs(parser, args) cls._benchmark = benchmark_class
def testWebPageReplay(self): story_set = example_domain.ExampleDomainPageSet() body = [] class TestWpr(legacy_page_test.LegacyPageTest): def ValidateAndMeasurePage(self, page, tab, results): del page, results # unused body.append(tab.EvaluateJavaScript('document.body.innerText')) def DidRunPage(self, platform): # Force the replay server to restart between pages; this verifies that # the restart mechanism works. platform.network_controller.StopReplay() test = TestWpr() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=test, story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, ) self.longMessage = True self.assertIn('Example Domain', body[0], msg='URL: %s' % story_set.stories[0].url) self.assertIn('Example Domain', body[1], msg='URL: %s' % story_set.stories[1].url) self.assertEquals(2, len(GetSuccessfulPageRuns(results))) self.assertFalse(results.had_failures)
def ProcessCommandLineArgs(cls, parser, options, environment): all_benchmarks = environment.GetBenchmarks() if environment.expectations_files: assert len(environment.expectations_files) == 1 expectations_file = environment.expectations_files[0] else: expectations_file = None if not options.positional_args: possible_browser = (browser_finder.FindBrowser(options) if options.browser_type else None) PrintBenchmarkList(all_benchmarks, possible_browser, expectations_file) sys.exit(-1) benchmark_name = options.positional_args[0] benchmark_class = environment.GetBenchmarkByName(benchmark_name) if benchmark_class is None: print >> sys.stderr, 'No benchmark named "%s".' % benchmark_name print >> sys.stderr most_likely_matched_benchmarks = matching.GetMostLikelyMatchedObject( all_benchmarks, benchmark_name, lambda x: x.Name()) if most_likely_matched_benchmarks: print >> sys.stderr, 'Do you mean any of those benchmarks below?' PrintBenchmarkList(most_likely_matched_benchmarks, None, expectations_file, sys.stderr) sys.exit(-1) if len(options.positional_args) > 1: parser.error('Too many arguments.') assert issubclass( benchmark_class, benchmark.Benchmark), ('Trying to run a non-Benchmark?!') benchmark.ProcessCommandLineArgs(parser, options) benchmark_class.ProcessCommandLineArgs(parser, options) cls._benchmark = benchmark_class cls._expectations_path = expectations_file
def testPushDefaultProfileDir(self): # Add a few files and directories to a temp directory, and ensure they are # copied to the device. with tempfile_ext.NamedTemporaryDirectory() as tempdir: foo_path = os.path.join(tempdir, 'foo') with open(foo_path, 'w') as f: f.write('foo_data') bar_path = os.path.join(tempdir, 'path', 'to', 'bar') os.makedirs(os.path.dirname(bar_path)) with open(bar_path, 'w') as f: f.write('bar_data') expected_profile_paths = [ 'foo', posixpath.join('path', 'to', 'bar') ] finder_options = options_for_unittests.GetCopy() finder_options.browser_options.profile_dir = tempdir browser_to_create = browser_finder.FindBrowser(finder_options) # SetUpEnvironment will end up calling PushProfile try: browser_to_create.SetUpEnvironment( finder_options.browser_options) profile_dir = browser_to_create.profile_directory device = browser_to_create._platform_backend.device absolute_expected_profile_paths = [ posixpath.join(profile_dir, path) for path in expected_profile_paths ] device = browser_to_create._platform_backend.device self.assertTrue( device.PathExists(absolute_expected_profile_paths), absolute_expected_profile_paths) finally: browser_to_create.CleanUpEnvironment()
def CustomizeBrowserOptions(self, options): # --test-type=gpu is used only to suppress the "Google API Keys are missing" # infobar, which causes flakiness in tests. options.AppendExtraBrowserArgs([ '--disable-gesture-requirement-for-media-playback', '--disable-domain-blocking-for-3d-apis', '--disable-gpu-process-crash-limit', '--js-flags=--expose-gc', '--test-type=gpu', '--enable-experimental-canvas-features' ]) browser = browser_finder.FindBrowser(options.finder_options) if (browser.target_os.startswith('android') and browser.browser_type == 'android-webview-shell'): # TODO(kbr): this is overly broad. We'd like to do this only on # Nexus 9. It'll go away shortly anyway. crbug.com/499928 # # The --ignore_egl_sync_failures is only there to work around # some strange failure on the Nexus 9 bot, not reproducible on # local hardware. options.AppendExtraBrowserArgs([ '--disable-gl-extensions=GL_EXT_disjoint_timer_query', '--ignore_egl_sync_failures', ])
def testRunPageWithStartupUrl(self): num_times_browser_closed = [0] class TestSharedState(shared_page_state.SharedPageState): def _StopBrowser(self): super(TestSharedState, self)._StopBrowser() num_times_browser_closed[0] += 1 story_set = story.StorySet() page = page_module.Page( 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), startup_url='about:blank', shared_page_state_class=TestSharedState) story_set.AddStory(page) class Measurement(legacy_page_test.LegacyPageTest): def __init__(self): super(Measurement, self).__init__() def ValidateAndMeasurePage(self, page, tab, results): del page, tab, results # not used options = options_for_unittests.GetCopy() options.pageset_repeat = 2 options.output_formats = ['none'] options.suppress_gtest_report = True if not browser_finder.FindBrowser(options): return test = Measurement() SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run(test, story_set, options, results, metadata=EmptyMetadataForTest()) self.assertEquals('about:blank', options.browser_options.startup_url) # _StopBrowser should be called 2 times: # 1. browser restarts after page 1 run # 2. in the TearDownState after all the pages have run. self.assertEquals(num_times_browser_closed[0], 2)
def setUp(self): """ Copy the manifest and background.js files of simple_extension to a number of temporary directories to load as extensions""" self._extension_dirs = [tempfile.mkdtemp() for _ in range(3)] src_extension_dir = os.path.join(util.GetUnittestDataDir(), 'simple_extension') manifest_path = os.path.join(src_extension_dir, 'manifest.json') script_path = os.path.join(src_extension_dir, 'background.js') for d in self._extension_dirs: shutil.copy(manifest_path, d) shutil.copy(script_path, d) options = options_for_unittests.GetCopy() self._extensions_to_load = [ extension_to_load.ExtensionToLoad(d, options.browser_type) for d in self._extension_dirs ] options.extensions_to_load = self._extensions_to_load browser_to_create = browser_finder.FindBrowser(options) self._browser = None # May not find a browser that supports extensions. if browser_to_create: self._browser = browser_to_create.Create(options)
def __init__(self, extra_browser_args=None, username=None, password=None): finder_options = browser_options.BrowserFinderOptions() finder_options.browser_type = ('system') if extra_browser_args: finder_options.browser_options.AppendExtraBrowserArgs( extra_browser_args) finder_options.verbosity = 0 finder_options.CreateParser().parse_args(args=[]) b_options = finder_options.browser_options b_options.disable_component_extensions_with_background_pages = False b_options.create_browser_with_oobe = True b_options.clear_enterprise_policy = True b_options.dont_override_profile = False b_options.disable_gaia_services = True b_options.disable_default_apps = True b_options.disable_component_extensions_with_background_pages = True b_options.auto_login = True b_options.gaia_login = False b_options.gaia_id = b_options.gaia_id open('/mnt/stateful_partition/etc/collect_chrome_crashes', 'w').close() browser_to_create = browser_finder.FindBrowser(finder_options) self._browser = browser_to_create.Create(finder_options)
def __init__(self, test, finder_options, story_set): """ Args: test: (unused) finder_options: A finder_options object story_set: (unused) """ super(SharedAndroidStoryState, self).__init__(test, finder_options, story_set) self._finder_options = finder_options self._possible_browser = browser_finder.FindBrowser( self._finder_options) self._current_story = None # This is an Android-only shared state. assert isinstance(self.platform, android_platform.AndroidPlatform) self._finder_options.browser_options.browser_user_agent_type = 'mobile' # TODO: This will always use live sites. Should use options to configure # network_controller properly. See e.g.: https://goo.gl/nAsyFr self.platform.network_controller.Open(wpr_modes.WPR_OFF) self.platform.Initialize()
def RunTest(self): class SinglePageBenchmark(benchmark_class): # pylint: disable=no-init def CreateStorySet(self, options): # pylint: disable=super-on-old-class story_set = super(SinglePageBenchmark, self).CreateStorySet(options) stories_to_remove = [s for s in story_set.stories if s != story_to_smoke_test] for s in stories_to_remove: story_set.RemoveStory(s) assert story_set.stories return story_set options = GenerateBenchmarkOptions(benchmark_class) # Prevent benchmarks from accidentally trying to upload too much data to the # chromeperf dashboard. The number of values uploaded is equal to (the # average number of values produced by a single story) * (1 + (the number of # stories)). The "1 + " accounts for values summarized across all stories. # We can approximate "the average number of values produced by a single # story" as the number of values produced by the given story. # pageset_repeat doesn't matter because values are summarized across # repetitions before uploading. story_set = benchmark_class().CreateStorySet(options) SinglePageBenchmark.MAX_NUM_VALUES = MAX_NUM_VALUES / len(story_set.stories) possible_browser = browser_finder.FindBrowser(options) if possible_browser is None: self.skipTest('Cannot find the browser to run the test.') if self.id() in _DISABLED_TESTS: self.skipTest('Test is explicitly disabled') single_page_benchmark = SinglePageBenchmark() with open(path_util.GetExpectationsPath()) as fp: single_page_benchmark.AugmentExpectationsWithParser(fp.read()) self.assertEqual(0, single_page_benchmark.Run(options), msg='Failed: %s' % benchmark_class)
def Run(self, options): """Kick off the process. Args: options: Instance of BrowserFinderOptions to search for proper browser. Returns: A 2-tuple (path, new_profile). path: The path of the generated profile or existing profile if --profile-dir is given. Could be None if it's generated on default location (e.g., cryptohome on CrOS). new_profile: Whether a new profile has been generated. If this is True, the caller is responsible for deleting the profile. """ possible_browser = browser_finder.FindBrowser(options) if possible_browser.browser_type.startswith('cros'): self.Create(options, None) return (None, False) # Use the given --profile-dir. if options.browser_options.profile_dir: return (options.browser_options.profile_dir, False) out_dir = os.path.abspath( os.path.join(tempfile.gettempdir(), self._profile_name, self._profile_name)) # Never reuse a generated profile, since it might be for a different version # of Chrome. if os.path.exists(out_dir): assert os.path.isdir(out_dir) shutil.rmtree(out_dir) self.Create(options, out_dir) return (out_dir, True)
def ProcessCommandLineArgs(cls, parser, options, environment): all_benchmarks = environment.GetBenchmarks() if environment.expectations_files: assert len(environment.expectations_files) == 1 expectations_file = environment.expectations_files[0] else: expectations_file = None if not options.positional_args: possible_browser = (browser_finder.FindBrowser(options) if options.browser_type else None) PrintBenchmarkList(all_benchmarks, possible_browser, expectations_file) parser.error('missing required argument: benchmark_name') benchmark_name = options.positional_args[0] benchmark_class = environment.GetBenchmarkByName(benchmark_name) if benchmark_class is None: most_likely_matched_benchmarks = matching.GetMostLikelyMatchedObject( all_benchmarks, benchmark_name, lambda x: x.Name()) if most_likely_matched_benchmarks: print >> sys.stderr, 'Do you mean any of those benchmarks below?' PrintBenchmarkList(most_likely_matched_benchmarks, None, expectations_file, sys.stderr) parser.error('no such benchmark: %s' % benchmark_name) if len(options.positional_args) > 1: parser.error('unrecognized arguments: %s' % ' '.join(options.positional_args[1:])) assert issubclass( benchmark_class, benchmark.Benchmark), ('Trying to run a non-Benchmark?!') story_runner.ProcessCommandLineArgs(parser, options) benchmark_class.ProcessCommandLineArgs(parser, options) cls._benchmark = benchmark_class cls._expectations_path = expectations_file
def BenchmarkSmokeTest(self): # Only measure a single page so that this test cycles reasonably quickly. benchmark.options['pageset_repeat'] = 1 benchmark.options['page_repeat'] = 1 class SinglePageBenchmark(benchmark): # pylint: disable=no-init def CreateStorySet(self, options): # pylint: disable=super-on-old-class story_set = super(SinglePageBenchmark, self).CreateStorySet(options) # Only smoke test the first story since smoke testing everything takes # too long. for s in story_set.stories[num_pages:]: story_set.RemoveStory(s) return story_set # Set the benchmark's default arguments. options = options_for_unittests.GetCopy() options.output_formats = ['none'] parser = options.CreateParser() benchmark.AddCommandLineArgs(parser) benchmark_module.AddCommandLineArgs(parser) benchmark.SetArgumentDefaults(parser) options.MergeDefaultValues(parser.get_default_values()) benchmark.ProcessCommandLineArgs(None, options) benchmark_module.ProcessCommandLineArgs(None, options) possible_browser = browser_finder.FindBrowser(options) if SinglePageBenchmark.ShouldDisable(possible_browser): self.skipTest('Benchmark %s has ShouldDisable return True' % SinglePageBenchmark.Name()) self.assertEqual(0, SinglePageBenchmark().Run(options), msg='Failed: %s' % benchmark)
def __init__(self, test, finder_options, story_set): """ Args: test: opaquely passed to parent class constructor. finder_options: A BrowserFinderOptions object. story_set: opaquely passed to parent class constructor. """ super(_MobileStartupSharedState, self).__init__( test, finder_options, story_set) self._finder_options = finder_options self._possible_browser = browser_finder.FindBrowser(self._finder_options) self._current_story = None # Allow using this shared state only on Android. assert isinstance(self.platform, android_platform.AndroidPlatform) self._finder_options.browser_options.browser_user_agent_type = 'mobile' self.platform.Initialize() assert finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD, ( 'Recording WPR archives is not supported for this benchmark.') wpr_mode = wpr_modes.WPR_REPLAY if finder_options.use_live_sites: wpr_mode = wpr_modes.WPR_OFF self.platform.network_controller.Open(wpr_mode) self._story_set = story_set
def RunTest(self): class SinglePageBenchmark(benchmark_class): # pylint: disable=no-init def CreateStorySet(self, options): # pylint: disable=super-on-old-class story_set = super(SinglePageBenchmark, self).CreateStorySet(options) assert story_to_smoke_test in story_set.stories story_set.stories = [story_to_smoke_test] return story_set options = GenerateBenchmarkOptions(benchmark_class) possible_browser = browser_finder.FindBrowser(options) if possible_browser is None: self.skipTest('Cannot find the browser to run the test.') if (SinglePageBenchmark.ShouldDisable(possible_browser) or not decorators.IsEnabled(benchmark_class, possible_browser)[0]): self.skipTest('Benchmark %s is disabled' % SinglePageBenchmark.Name()) if self.id() in _DISABLED_TESTS: self.skipTest('Test is explictly disabled') self.assertEqual(0, SinglePageBenchmark().Run(options), msg='Failed: %s' % benchmark_class)
def _PrepareBrowser(self, browser_type, options): """Add a browser to the dict of possible browsers. TODO(perezju): When available, use the GetBrowserForPlatform API instead. See: crbug.com/570348 Returns: The possible browser if found, or None otherwise. """ possible_browser = browser_finder.FindBrowser(options) if possible_browser is None: return None if self._platform is None: self._platform = possible_browser.platform # TODO(nedn): Remove the if condition once # https://codereview.chromium.org/2265593003/ is rolled to Chromium tree. if hasattr(self._platform.network_controller, 'InitializeIfNeeded'): self._platform.network_controller.InitializeIfNeeded() else: assert self._platform is possible_browser.platform self._possible_browsers[browser_type] = (possible_browser, options) return possible_browser
def ProcessCommandLineArgs(cls, parser, args, _): # We retry failures by default unless we're running a list of tests # explicitly. if not args.retry_limit and not args.positional_args: args.retry_limit = 3 if args.test_filter and args.positional_args: parser.error('Cannot specify test names in positional args and use' '--test-filter flag at the same time.') if args.no_browser: return if args.start_xvfb and xvfb.ShouldStartXvfb(): cls.xvfb_process = xvfb.StartXvfb() # Work around Mesa issues on Linux. See # https://github.com/catapult-project/catapult/issues/3074 args.browser_options.AppendExtraBrowserArgs('--disable-gpu') try: possible_browser = browser_finder.FindBrowser(args) except browser_finder_exceptions.BrowserFinderException, ex: parser.error(ex)
def _GetPossibleBrowser(self, test, finder_options): """Return a possible_browser with the given options. """ possible_browser = browser_finder.FindBrowser(finder_options) if not possible_browser: raise browser_finder_exceptions.BrowserFinderException( 'No browser found.\n\nAvailable browsers:\n%s\n' % '\n'.join( browser_finder.GetAllAvailableBrowserTypes(finder_options)) ) finder_options.browser_options.browser_type = ( possible_browser.browser_type) (enabled, msg) = decorators.IsEnabled(test, possible_browser) if (not enabled and not finder_options.run_disabled_tests): logging.warning(msg) logging.warning('You are trying to run a disabled test.') logging.warning( 'Pass --also-run-disabled-tests to squelch this message.') sys.exit(0) if possible_browser.IsRemote(): possible_browser.RunRemote() sys.exit(0) return possible_browser
def SetBrowserOptions(cls, browser_options): """Sets the browser option for the browser to create. Args: browser_options: Browser options object for the browser we want to test. """ cls._browser_options = browser_options cls._browser_to_create = browser_finder.FindBrowser(browser_options) if not cls._browser_to_create: raise browser_finder_exceptions.BrowserFinderException( 'Cannot find browser of type %s. \n\nAvailable browsers:\n%s\n' % ( browser_options.browser_options.browser_type, '\n'.join(browser_finder.GetAllAvailableBrowserTypes( browser_options)))) if not cls.platform: cls.platform = cls._browser_to_create.platform cls.platform.SetFullPerformanceModeEnabled( browser_options.full_performance_mode) cls.platform.network_controller.Open() else: assert cls.platform == cls._browser_to_create.platform, ( 'All browser launches within same test suite must use browsers on ' 'the same platform')
def _GetPossibleBrowser(self): """Return a possible_browser with the given options.""" possible_browser = browser_finder.FindBrowser(self._finder_options) if not possible_browser: raise browser_finder_exceptions.BrowserFinderException( 'Cannot find browser of type %s. \n\nAvailable browsers:\n%s\n' % (self._finder_options.browser_options.browser_type, '\n'.join( browser_finder.GetAllAvailableBrowserTypes( self._finder_options)))) self._finder_options.browser_options.browser_type = ( possible_browser.browser_type) if self._page_test: # Check for Enabled/Disabled decorators on page_test. skip, msg = decorators.ShouldSkip(self._page_test, possible_browser) if skip and not self._finder_options.run_disabled_tests: logging.warning(msg) logging.warning('You are trying to run a disabled test.') return possible_browser
def _CreateBrowser(self, autotest_ext=False, auto_login=True, gaia_login=False, username=None, password=None, gaia_id=None, dont_override_profile=False): """Finds and creates a browser for tests. if autotest_ext is True, also loads the autotest extension""" options = options_for_unittests.GetCopy() if autotest_ext: extension_path = os.path.join(util.GetUnittestDataDir(), 'autotest_ext') assert os.path.isdir(extension_path) self._load_extension = extension_to_load.ExtensionToLoad( path=extension_path, browser_type=options.browser_type, is_component=True) options.browser_options.extensions_to_load = [self._load_extension] browser_to_create = browser_finder.FindBrowser(options) self.assertTrue(browser_to_create) browser_options = options.browser_options browser_options.create_browser_with_oobe = True browser_options.auto_login = auto_login browser_options.gaia_login = gaia_login browser_options.dont_override_profile = dont_override_profile if username is not None: browser_options.username = username if password is not None: browser_options.password = password if gaia_id is not None: browser_options.gaia_id = gaia_id return browser_to_create.Create(options)
def __init__(self, test, finder_options, story_set, possible_browser=None): """ Args: test: opaquely passed to parent class constructor. finder_options: A BrowserFinderOptions object. story_set: opaquely passed to parent class constructor. """ super(_MobileStartupSharedState, self).__init__(test, finder_options, story_set, possible_browser) self._finder_options = finder_options if not self._possible_browser: self._possible_browser = browser_finder.FindBrowser( self._finder_options) self._current_story = None # Allow using this shared state only on Android. assert isinstance(self.platform, android_platform.AndroidPlatform) self._finder_options.browser_options.browser_user_agent_type = 'mobile' self._finder_options.browser_options.AppendExtraBrowserArgs( '--skip-webapk-verification') self.platform.Initialize() self.platform.SetFullPerformanceModeEnabled(True) maps_webapk = core_util.FindLatestApkOnHost(finder_options.chrome_root, 'MapsWebApk.apk') if not maps_webapk: raise Exception('MapsWebApk not found! Follow the Mini-HOWTO in ' 'startup_mobile.py') self.platform.InstallApplication(maps_webapk) wpr_mode = wpr_modes.WPR_REPLAY self._number_of_iterations = _NUMBER_OF_ITERATIONS if finder_options.use_live_sites: wpr_mode = wpr_modes.WPR_OFF elif finder_options.browser_options.wpr_mode == wpr_modes.WPR_RECORD: wpr_mode = wpr_modes.WPR_RECORD # When recording a WPR archive only load the story page once. self._number_of_iterations = 1 self.platform.network_controller.Open(wpr_mode) self._story_set = story_set
def ProcessCommandLineArgs(cls, parser, args, _): if args.verbosity >= 2: logging.getLogger().setLevel(logging.DEBUG) elif args.verbosity: logging.getLogger().setLevel(logging.INFO) else: logging.getLogger().setLevel(logging.WARNING) # We retry failures by default unless we're running a list of tests # explicitly. if not args.retry_limit and not args.positional_args: args.retry_limit = 3 if args.test_filter and args.positional_args: parser.error( 'Cannot specify test names in positional args and use' '--test-filter flag at the same time.') if args.no_browser: return if args.start_xvfb and xvfb.ShouldStartXvfb(): cls.xvfb_process = xvfb.StartXvfb() # Work around Mesa issues on Linux. See # https://github.com/catapult-project/catapult/issues/3074 args.browser_options.AppendExtraBrowserArgs('--disable-gpu') try: possible_browser = browser_finder.FindBrowser(args) except browser_finder_exceptions.BrowserFinderException as ex: parser.error(ex) if not possible_browser: parser.error('No browser found of type %s. Cannot run tests.\n' 'Re-run with --browser=list to see ' 'available browser types.' % args.browser_type)
def setUpClass(cls): global current_browser global current_browser_options options = options_for_unittests.GetCopy() cls.CustomizeBrowserOptions(options.browser_options) if not current_browser or (current_browser_options != options.browser_options): if current_browser: teardown_browser() browser_to_create = browser_finder.FindBrowser(options) if not browser_to_create: raise Exception('No browser found, cannot continue test.') try: current_browser = browser_to_create.Create(options) current_browser_options = options.browser_options except: cls.tearDownClass() raise cls._browser = current_browser cls._device = options.device
# Initialize the dependency manager from telemetry.internal.util import binary_manager from chrome_telemetry_build import chromium_config binary_manager.InitDependencyManager( chromium_config.ChromiumConfig().client_config) from telemetry.timeline import tracing_config from json import dumps options = browser_options.BrowserFinderOptions() parser = options.CreateParser() (_, args) = parser.parse_args() browserFactory = browser_finder.FindBrowser(options) with browserFactory.Create(options) as browser: tab = browser.tabs.New() tab.Activate() for i in browser.tabs: if i == tab: continue i.Close() config = tracing_config.TracingConfig() config.enable_chrome_trace = True browser.platform.tracing_controller.StartTracing(config) tab.Navigate(args[0]) tab.WaitForDocumentReadyStateToBeComplete() browser.platform.tracing_controller.StopTracing().Serialize(sys.stdout)
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: The number of failure values (up to 254) or 255 if there is an uncaught exception. """ start = time.time() benchmark.CustomizeBrowserOptions(finder_options.browser_options) benchmark_metadata = benchmark.GetMetadata() possible_browser = browser_finder.FindBrowser(finder_options) if not possible_browser: print( 'Cannot find browser of type %s. To list out all ' 'available browsers, rerun your command with ' '--browser=list' % finder_options.browser_options.browser_type) return 1 if (possible_browser and not decorators.IsBenchmarkEnabled(benchmark, possible_browser)): print '%s is disabled on the selected browser' % benchmark.Name() if finder_options.run_disabled_tests: print 'Running benchmark anyway due to: --also-run-disabled-tests' else: print 'Try --also-run-disabled-tests to force the benchmark to run.' # If chartjson is specified, this will print a dict indicating the # benchmark name and disabled state. with results_options.CreateResults( benchmark_metadata, finder_options, benchmark.ValueCanBeAddedPredicate, benchmark_enabled=False) as results: results.PrintSummary() # When a disabled benchmark is run we now want to return success since # we are no longer filtering these out in the buildbot recipes. return 0 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ disabled_attr_name = decorators.DisabledAttributeName(benchmark) # pylint: disable=protected-access pt._disabled_strings = getattr(benchmark, disabled_attr_name, set()) if hasattr(benchmark, '_enabled_strings'): # pylint: disable=protected-access pt._enabled_strings = benchmark._enabled_strings stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') should_tear_down_state_after_each_story_run = ( benchmark.ShouldTearDownStateAfterEachStoryRun()) # HACK: restarting domain.shared state has huge overhead on cros (crbug.com/645329), # hence we default this to False when test is run against CrOS. # TODO(cros-team): figure out ways to remove this hack. if (possible_browser.platform.GetOSName() == 'chromeos' and not benchmark.IsShouldTearDownStateAfterEachStoryRunOverriden()): should_tear_down_state_after_each_story_run = False with results_options.CreateResults(benchmark_metadata, finder_options, benchmark.ValueCanBeAddedPredicate, benchmark_enabled=True) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, should_tear_down_state_after_each_story_run, benchmark.ShouldTearDownStateAfterEachStorySetRun()) return_code = min(254, len(results.failures)) except Exception: exception_formatter.PrintFormattedException() return_code = 255 try: if finder_options.upload_results: bucket = finder_options.upload_bucket if bucket in cloud_storage.BUCKET_ALIASES: bucket = cloud_storage.BUCKET_ALIASES[bucket] results.UploadTraceFilesToCloud(bucket) results.UploadProfilingFilesToCloud(bucket) finally: duration = time.time() - start results.AddSummaryValue( scalar.ScalarValue(None, 'BenchmarkDuration', 'minutes', duration / 60.0)) results.PrintSummary() return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: -1 if the benchmark was skipped, 0 for success 1 if there was a failure 2 if there was an uncaught exception. """ benchmark.CustomizeOptions(finder_options) with results_options.CreateResults( finder_options, benchmark_name=benchmark.Name(), benchmark_description=benchmark.Description(), report_progress=not finder_options.suppress_gtest_report ) as results: possible_browser = browser_finder.FindBrowser(finder_options) if not possible_browser: print('No browser of type "%s" found for running benchmark "%s".' % (finder_options.browser_options.browser_type, benchmark.Name())) return -1 typ_expectation_tags = possible_browser.GetTypExpectationsTags() logging.info( 'The following expectations condition tags were generated %s', str(typ_expectation_tags)) try: benchmark.expectations.SetTags( typ_expectation_tags, not finder_options.skip_typ_expectations_tags_validation) except ValueError as e: # pylint: disable=broad-except traceback.print_exc(file=sys.stdout) logging.error( str(e) + '\nYou can use the --skip-typ-expectations-tags-validation ' 'argument to suppress this exception.') return -1 if not _ShouldRunBenchmark(benchmark, possible_browser, finder_options): return -1 test = benchmark.CreatePageTest(finder_options) test.__name__ = benchmark.__class__.__name__ story_set = benchmark.CreateStorySet(finder_options) if isinstance(test, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in story_set.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') try: RunStorySet(test, story_set, finder_options, results, benchmark.max_failures, expectations=benchmark.expectations, max_num_values=benchmark.MAX_NUM_VALUES) if results.benchmark_interrupted: return_code = 2 elif results.had_failures: return_code = 1 elif results.had_successes: return_code = 0 else: return_code = -1 # All stories were skipped. except Exception as exc: # pylint: disable=broad-except interruption = 'Benchmark execution interrupted: %r' % exc results.InterruptBenchmark(interruption) exception_formatter.PrintFormattedException() return_code = 2 # TODO(crbug.com/981349): merge two calls to AddSharedDiagnostics # (see RunStorySet() method for the second one). results.AddSharedDiagnostics( owners=benchmark.GetOwners(), bug_components=benchmark.GetBugComponents(), documentation_urls=benchmark.GetDocumentationLinks(), ) if finder_options.upload_results: results_processor.UploadArtifactsToCloud(results) return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: The number of failure values (up to 254) or 255 if there is an uncaught exception. """ benchmark.CustomizeBrowserOptions(finder_options.browser_options) possible_browser = browser_finder.FindBrowser(finder_options) if possible_browser and benchmark.ShouldDisable(possible_browser): logging.warning('%s is disabled on the selected browser', benchmark.Name()) if finder_options.run_disabled_tests: logging.warning( 'Running benchmark anyway due to: --also-run-disabled-tests') else: logging.warning( 'Try --also-run-disabled-tests to force the benchmark to run.') return 1 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ disabled_attr_name = decorators.DisabledAttributeName(benchmark) # pylint: disable=protected-access pt._disabled_strings = getattr(benchmark, disabled_attr_name, set()) if hasattr(benchmark, '_enabled_strings'): # pylint: disable=protected-access pt._enabled_strings = benchmark._enabled_strings stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') should_tear_down_state_after_each_story_run = ( benchmark.ShouldTearDownStateAfterEachStoryRun()) # HACK: restarting shared state has huge overhead on cros (crbug.com/645329), # hence we default this to False when test is run against CrOS. # TODO(cros-team): figure out ways to remove this hack. if (possible_browser.platform.GetOSName() == 'chromeos' and not benchmark.IsShouldTearDownStateAfterEachStoryRunOverriden()): should_tear_down_state_after_each_story_run = False benchmark_metadata = benchmark.GetMetadata() with results_options.CreateResults( benchmark_metadata, finder_options, benchmark.ValueCanBeAddedPredicate) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, should_tear_down_state_after_each_story_run, benchmark.ShouldTearDownStateAfterEachStorySetRun()) return_code = min(254, len(results.failures)) except Exception: exception_formatter.PrintFormattedException() return_code = 255 try: if finder_options.upload_results: bucket = finder_options.upload_bucket if bucket in cloud_storage.BUCKET_ALIASES: bucket = cloud_storage.BUCKET_ALIASES[bucket] results.UploadTraceFilesToCloud(bucket) results.UploadProfilingFilesToCloud(bucket) finally: results.PrintSummary() return return_code
def __init__(self, logged_in=True, extension_paths=None, autotest_ext=False, num_tries=3, extra_browser_args=None, clear_enterprise_policy=True, expect_policy_fetch=False, dont_override_profile=False, disable_gaia_services=True, disable_default_apps=True, auto_login=True, gaia_login=False, username=None, password=None, gaia_id=None, arc_mode=None, arc_timeout=None, disable_arc_opt_in=True, disable_arc_opt_in_verification=True, disable_arc_cpu_restriction=True, disable_app_sync=False, disable_play_auto_install=False, disable_locale_sync=True, disable_play_store_auto_update=True, enable_assistant=False, enterprise_arc_test=False, init_network_controller=False, mute_audio=False, proxy_server=None, login_delay=0): """ Constructor of telemetry wrapper. @param logged_in: Regular user (True) or guest user (False). @param extension_paths: path of unpacked extension to install. @param autotest_ext: Load a component extension with privileges to invoke chrome.autotestPrivate. @param num_tries: Number of attempts to log in. @param extra_browser_args: Additional argument(s) to pass to the browser. It can be a string or a list. @param clear_enterprise_policy: Clear enterprise policy before logging in. @param expect_policy_fetch: Expect that chrome can reach the device management server and download policy. @param dont_override_profile: Don't delete cryptohome before login. Telemetry will output a warning with this option. @param disable_gaia_services: For enterprise autotests, this option may be used to enable policy fetch. @param disable_default_apps: For tests that exercise default apps. @param auto_login: Does not login automatically if this is False. Useful if you need to examine oobe. @param gaia_login: Logs in to real gaia. @param username: Log in using this username instead of the default. @param password: Log in using this password instead of the default. @param gaia_id: Log in using this gaia_id instead of the default. @param arc_mode: How ARC instance should be started. Default is to not start. @param arc_timeout: Timeout to wait for ARC to boot. @param disable_arc_opt_in: For opt in flow autotest. This option is used to disable the arc opt in flow. @param disable_arc_opt_in_verification: Adds --disable-arc-opt-in-verification to browser args. This should generally be enabled when disable_arc_opt_in is enabled. However, for data migration tests where user's home data is already set up with opted-in state before login, this option needs to be set to False with disable_arc_opt_in=True to make ARC container work. @param disable_arc_cpu_restriction: Adds --disable-arc-cpu-restriction to browser args. This is enabled by default and will make tests run faster and is generally desirable unless a test is actually trying to test performance where ARC is running in the background for some porition of the test. @param disable_app_sync: Adds --arc-disable-app-sync to browser args and this disables ARC app sync flow. By default it is enabled. @param disable_play_auto_install: Adds --arc-disable-play-auto-install to browser args and this disables ARC Play Auto Install flow. By default it is enabled. @param enable_assistant: For tests that require to enable Google Assistant service. Default is False. @param enterprise_arc_test: Skips opt_in causing enterprise tests to fail @param disable_locale_sync: Adds --arc-disable-locale-sync to browser args and this disables locale sync between Chrome and Android container. In case of disabling sync, Android container is started with language and preference language list as it was set on the moment of starting full instance. Used to prevent random app restarts caused by racy locale change, coming from profile sync. By default locale sync is disabled. @param disable_play_store_auto_update: Adds --arc-play-store-auto-update=off to browser args and this disables Play Store, GMS Core and third-party apps auto-update. By default auto-update is off to have stable autotest environment. @param mute_audio: Mute audio. @param proxy_server: To launch the chrome with --proxy-server Adds '--proxy-server="http://$HTTP_PROXY:PORT"' to browser args. By default proxy-server is disabled @param login_delay: Time for idle in login screen to simulate the time required for password typing. """ self._autotest_ext_path = None # Force autotest extension if we need enable Play Store. if (utils.is_arc_available() and (arc_util.should_start_arc(arc_mode) or not disable_arc_opt_in)): autotest_ext = True if extension_paths is None: extension_paths = [] finder_options = browser_options.BrowserFinderOptions() if proxy_server: finder_options.browser_options.AppendExtraBrowserArgs( ['--proxy-server="%s"' % proxy_server]) if utils.is_arc_available() and arc_util.should_start_arc(arc_mode): if disable_arc_opt_in and disable_arc_opt_in_verification: finder_options.browser_options.AppendExtraBrowserArgs( ['--disable-arc-opt-in-verification']) if disable_arc_cpu_restriction: finder_options.browser_options.AppendExtraBrowserArgs( ['--disable-arc-cpu-restriction']) if disable_app_sync: finder_options.browser_options.AppendExtraBrowserArgs( ['--arc-disable-app-sync']) if disable_play_auto_install: finder_options.browser_options.AppendExtraBrowserArgs( ['--arc-disable-play-auto-install']) if disable_locale_sync: finder_options.browser_options.AppendExtraBrowserArgs( ['--arc-disable-locale-sync']) if disable_play_store_auto_update: finder_options.browser_options.AppendExtraBrowserArgs( ['--arc-play-store-auto-update=off']) logged_in = True if autotest_ext: self._autotest_ext_path = os.path.join(os.path.dirname(__file__), 'autotest_private_ext') extension_paths.append(self._autotest_ext_path) finder_options.browser_options.AppendExtraBrowserArgs( ['--whitelisted-extension-id=%s' % self.AUTOTEST_EXT_ID]) self._browser_type = (self.BROWSER_TYPE_LOGIN if logged_in else self.BROWSER_TYPE_GUEST) finder_options.browser_type = self.browser_type if extra_browser_args: finder_options.browser_options.AppendExtraBrowserArgs( extra_browser_args) # finder options must be set before parse_args(), browser options must # be set before Create(). # TODO(crbug.com/360890) Below MUST be '2' so that it doesn't inhibit # autotest debug logs finder_options.verbosity = 2 finder_options.CreateParser().parse_args(args=[]) b_options = finder_options.browser_options b_options.disable_component_extensions_with_background_pages = False b_options.create_browser_with_oobe = True b_options.clear_enterprise_policy = clear_enterprise_policy b_options.dont_override_profile = dont_override_profile b_options.disable_gaia_services = disable_gaia_services b_options.disable_default_apps = disable_default_apps b_options.disable_component_extensions_with_background_pages = disable_default_apps b_options.disable_background_networking = False b_options.expect_policy_fetch = expect_policy_fetch b_options.auto_login = auto_login b_options.gaia_login = gaia_login b_options.mute_audio = mute_audio b_options.login_delay = login_delay if utils.is_arc_available() and not disable_arc_opt_in: arc_util.set_browser_options_for_opt_in(b_options) self.username = b_options.username if username is None else username self.password = b_options.password if password is None else password self.username = NormalizeEmail(self.username) b_options.username = self.username b_options.password = self.password self.gaia_id = b_options.gaia_id if gaia_id is None else gaia_id b_options.gaia_id = self.gaia_id self.arc_mode = arc_mode if logged_in: extensions_to_load = b_options.extensions_to_load for path in extension_paths: extension = extension_to_load.ExtensionToLoad( path, self.browser_type) extensions_to_load.append(extension) self._extensions_to_load = extensions_to_load # Turn on collection of Chrome coredumps via creation of a magic file. # (Without this, Chrome coredumps are trashed.) open(constants.CHROME_CORE_MAGIC_FILE, 'w').close() self._browser_to_create = browser_finder.FindBrowser(finder_options) self._browser_to_create.SetUpEnvironment(b_options) for i in range(num_tries): try: self._browser = self._browser_to_create.Create() self._browser_pid = \ cros_interface.CrOSInterface().GetChromePid() if utils.is_arc_available(): if disable_arc_opt_in: if arc_util.should_start_arc(arc_mode): arc_util.enable_play_store(self.autotest_ext, True) else: if not enterprise_arc_test: wait_for_provisioning = \ arc_mode != arc_common.ARC_MODE_ENABLED_ASYNC arc_util.opt_in( browser=self.browser, autotest_ext=self.autotest_ext, wait_for_provisioning=wait_for_provisioning) arc_util.post_processing_after_browser(self, arc_timeout) if enable_assistant: assistant_util.enable_assistant(self.autotest_ext) break except exceptions.LoginException as e: logging.error('Timed out logging in, tries=%d, error=%s', i, repr(e)) if i == num_tries - 1: raise if init_network_controller: self._browser.platform.network_controller.Open()
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: 1 if there is failure or 2 if there is an uncaught exception. """ benchmark.CustomizeBrowserOptions(finder_options.browser_options) benchmark_metadata = benchmark.GetMetadata() possible_browser = browser_finder.FindBrowser(finder_options) expectations = benchmark.expectations target_platform = None if possible_browser: target_platform = possible_browser.platform else: target_platform = platform_module.GetHostPlatform() can_run_on_platform = benchmark._CanRunOnPlatform(target_platform, finder_options) expectations_disabled = False # For now, test expectations are only applicable in the cases where the # testing target involves a browser. if possible_browser: expectations_disabled = expectations.IsBenchmarkDisabled( possible_browser.platform, finder_options) if expectations_disabled or not can_run_on_platform: print '%s is disabled on the selected browser' % benchmark.Name() if finder_options.run_disabled_tests and can_run_on_platform: print 'Running benchmark anyway due to: --also-run-disabled-tests' else: if can_run_on_platform: print 'Try --also-run-disabled-tests to force the benchmark to run.' else: print( "This platform is not supported for this benchmark. If this is " "in error please add it to the benchmark's supported platforms." ) # If chartjson is specified, this will print a dict indicating the # benchmark name and disabled state. with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=False) as results: results.PrintSummary() # When a disabled benchmark is run we now want to return success since # we are no longer filtering these out in the buildbot recipes. return 0 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=True) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, expectations=expectations, max_num_values=benchmark.MAX_NUM_VALUES) return_code = 1 if results.had_failures else 0 # We want to make sure that all expectations are linked to real stories, # this will log error messages if names do not match what is in the set. benchmark.GetBrokenExpectations(stories) except Exception: # pylint: disable=broad-except logging.fatal( 'Benchmark execution interrupted by a fatal exception.') filtered_stories = story_module.StoryFilter.FilterStorySet(stories) results.InterruptBenchmark(filtered_stories, _GetPageSetRepeat(finder_options)) exception_formatter.PrintFormattedException() return_code = 2 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() benchmark_documentation_url = benchmark.GetDocumentationLink() if benchmark_owners: results.AddSharedDiagnostic(reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.AddSharedDiagnostic(reserved_infos.BUG_COMPONENTS.name, benchmark_component) if benchmark_documentation_url: results.AddSharedDiagnostic(reserved_infos.DOCUMENTATION_URLS.name, benchmark_documentation_url) try: if finder_options.upload_results: results.UploadTraceFilesToCloud() results.UploadArtifactsToCloud() finally: memory_debug.LogHostMemoryUsage() results.PrintSummary() return return_code