def test_find_all_browser_types_trybot(self): finder_options = browser_options.BrowserFinderOptions( browser_type='trybot-win') trybot_browser_finder.urllib2 = simple_mock.MockObject() trybot_browser_finder.urllib2.ExpectCall('urlopen').WithArgs( 'http://build.chromium.org/p/tryserver.chromium.perf/json').WillReturn( StringIO.StringIO(json.dumps({'builders': { 'android_nexus4_perf_bisect': 'stuff', 'mac_10_9_perf_bisect': 'otherstuff', 'win_perf_bisect_builder': 'not a trybot', }}))) self.assertEquals( ['trybot-android-nexus4', 'trybot-mac-10-9'], # pylint: disable=W0212 sorted(trybot_browser_finder.FindAllBrowserTypes(finder_options)))
def test_constructor_trybot_all_mac(self): finder_options = browser_options.BrowserFinderOptions() self._MockTryserverJson({ 'android_nexus4_perf_bisect': 'stuff', 'win_8_perf_bisect': 'otherstuff', 'mac_perf_bisect': 'otherstuff2', 'win_perf_bisect_builder': 'not a trybot', }) browser = trybot_browser_finder.PossibleTrybotBrowser( 'trybot-all-mac', finder_options) self.assertEquals('all', browser.target_os) # pylint: disable=W0212 self.assertEquals(['mac'], sorted(browser._builder_names)) self.assertEquals(['mac_perf_bisect'], sorted(browser._builder_names.get('mac')))
def Main(): options = browser_options.BrowserFinderOptions() parser = options.CreateParser( "%%prog <--profile-type-to-generate=...> <--browser=...> <--output-dir>" ) AddCommandLineArgs(parser) _, _ = parser.parse_args() ProcessCommandLineArgs(parser, options) # Generate profile. profile_extenders = _DiscoverProfileExtenderClasses() profile_extender_class = profile_extenders[ options.profile_type_to_generate] return GenerateProfiles(profile_extender_class, options.profile_type_to_generate, options)
def test_adb_permissions_error(self): finder_options = browser_options.BrowserFinderOptions() self._stubs.subprocess.Popen.communicate_result = ( """List of devices attached ????????????\tno permissions""", """* daemon not running. starting it now on port 5037 * * daemon started successfully * """) log_stub = LoggingStub() browsers = android_browser_finder.FindAllAvailableBrowsers( finder_options, log_stub) self.assertEquals(3, len(log_stub.warnings)) self.assertEquals(0, len(browsers))
def main(): usage = ('%prog [options]\n' 'Used to run the bisection script with a manual test.') options = browser_options.BrowserFinderOptions('release') parser = options.CreateParser(usage) parser.add_option('-b', '--bad_revision', type='str', help='A bad revision to start bisection. ' + 'Must be later than good revision. May be either a git' + ' or svn revision.') parser.add_option('-g', '--good_revision', type='str', help='A revision to start bisection where performance' + ' test is known to pass. Must be earlier than the ' + 'bad revision. May be either a git or svn revision.') parser.add_option('-w', '--working_directory', type='str', default='..', help='A working directory to supply to the bisection ' 'script, which will use it as the location to checkout ' 'a copy of the chromium depot.') options, args = parser.parse_args() error_msg = '' if not options.good_revision: error_msg += 'Error: missing required parameter: --good_revision\n' if not options.bad_revision: error_msg += 'Error: missing required parameter: --bad_revision\n' if error_msg: print error_msg parser.print_help() return 1 if 'android' not in options.browser_type and sys.platform.startswith( 'linux'): if not os.environ.get('CHROME_DEVEL_SANDBOX'): print 'SUID sandbox has not been setup.'\ ' See https://code.google.com/p/chromium/wiki/'\ 'LinuxSUIDSandboxDevelopment for more information.' return 1 return _RunBisectionScript(options)
def Main(): global scoreboard scores = dict() maxCell = dict() options = browser_options.BrowserFinderOptions() parser = options.CreateParser('telemetry_perf_test.py') options, _ = parser.parse_args(['--browser=system']) browser_to_create = browser_finder.FindBrowser(options) assert browser_to_create with browser_to_create.Create() as b: ai_list = enumerateAIs() for ai in ai_list: scores[ai[0]] = list() maxCell[ai[0]] = 2 if HAS_SCOREBOARD: scoreboard = ScoreBoard(b.tabs.New(), ITERATION) for ai in ai_list: scoreboard.addAI(ai[0]) waitStart(scoreboard.tab) b.tabs[0].Activate() for iteration in xrange(ITERATION): for ai in ai_list: try: s, mc = play(b.tabs[0], ai) except AIError, err: s = err.score mc = err.maxCell logMsg("%s(%d): %s" % (ai[0], iteration + 1, err.msg)) if mc > maxCell[ai[0]]: maxCell[ai[0]] = mc scoreboard.setScore(ai[0], ITERATION + 1, mc) scores[ai[0]].append(s) scoreboard.setScore(ai[0], iteration, s) if ITERATION > 2: for ai in ai_list: avg = ((sum(scores[ai[0]]) - max(scores[ai[0]]) - min(scores[ai[0]])) / (ITERATION - 2)) scoreboard.setScore(ai[0], ITERATION, avg) print "%s avg = %d\n" % (ai[0], avg) if HAS_SCOREBOARD: time.sleep(10000)
def test_adb_one_device(self): finder_options = browser_options.BrowserFinderOptions() self._stubs.adb_commands.attached_devices = ['015d14fec128220c'] def OnPM(args): assert args[0] == 'pm' assert args[1] == 'list' assert args[2] == 'packages' return ['package:org.chromium.content_shell_apk', 'package.com.google.android.setupwizard'] self._stubs.adb_commands.shell_command_handlers['pm'] = OnPM browsers = android_browser_finder.FindAllAvailableBrowsers(finder_options) self.assertEquals(1, len(browsers))
def test_constructor_trybot_all_android(self): finder_options = browser_options.BrowserFinderOptions() self._MockTryserverJson({ 'android_nexus4_perf_bisect': 'stuff', 'android_nexus5_perf_bisect': 'stuff2', 'win_8_perf_bisect': 'otherstuff', 'win_perf_bisect': 'otherstuff2', 'linux_perf_bisect': 'otherstuff3', 'win_x64_perf_bisect': 'otherstuff4', 'win_perf_bisect_builder': 'not a trybot', }) browser = trybot_browser_finder.PossibleTrybotBrowser( 'trybot-all-android', finder_options) self.assertEquals( ['android_nexus4_perf_bisect', 'android_nexus5_perf_bisect'], sorted(browser._builder_names.get('android')))
def test_find_all_browser_types_trybot(self): finder_options = browser_options.BrowserFinderOptions( browser_type='trybot-win') self._MockTryserverJson({ 'android_nexus4_perf_bisect': 'stuff', 'mac_10_9_perf_bisect': 'otherstuff', 'win_perf_bisect_builder': 'not a trybot', }) expected_trybots_list = [ 'trybot-all', 'trybot-all-android', 'trybot-all-linux', 'trybot-all-mac', 'trybot-all-win', 'trybot-android-nexus4', 'trybot-mac-10-9' ] self.assertEquals( expected_trybots_list, sorted(trybot_browser_finder.FindAllBrowserTypes(finder_options)))
def ParseCommandLine(self, args, env): self._options = browser_options.BrowserFinderOptions() self._parser = self._options.CreateParser( '%%prog [options] %s page_set' % self.test_class_name) test_constructors = self.FindTestConstructors(env.base_paths) test_name = self.FindTestName(test_constructors, args) test = None if test_name: test = test_constructors[test_name]() if isinstance(test, test_module.Test): page_test = test.test() test.AddCommandLineArgs(self._parser) else: page_test = test page_test.AddCommandLineArgs(self._parser) page_runner.AddCommandLineArgs(self._parser) _, self._args = self._parser.parse_args() if len(self._args) < 1: error_message = 'No %s specified.\nAvailable %ss:\n' % ( self.test_class_name, self.test_class_name) test_list_string = ',\n'.join(sorted(test_constructors.keys())) self.PrintParseError(error_message + test_list_string) if not test: error_message = 'No %s named %s.\nAvailable %ss:\n' % ( self.test_class_name, self._args[0], self.test_class_name) test_list_string = ',\n'.join(sorted(test_constructors.keys())) self.PrintParseError(error_message + test_list_string) if isinstance(test, test_module.Test): ps = test.CreatePageSet(self._options) expectations = test.CreateExpectations(ps) test.ProcessCommandLineArgs(self._parser, self._options) else: ps = self.GetPageSet(env) expectations = test.CreateExpectations(ps) page_test.ProcessCommandLineArgs(self._parser, self._options) page_runner.ProcessCommandLineArgs(self._parser, self._options) if len(self._args) > 2: self.PrintParseError('Too many arguments.') return page_test, ps, expectations
def _GetConfigForBrowser(self, name, branch, cfg_filename, is_blink=False): finder_options = browser_options.BrowserFinderOptions() browser = trybot_browser_finder.PossibleTrybotBrowser( name, finder_options) bot = '%s_perf_bisect' % name.replace('trybot-', '').replace('-', '_') first_processes = () if is_blink: first_processes = ((['git', 'rev-parse', '--abbrev-ref', 'HEAD'], (0, 'br', None)), (['git', 'update-index', '--refresh', '-q'], ( 0, None, None, )), (['git', 'diff-index', 'HEAD'], (0, '', None)), (['git', 'log', 'origin/master..HEAD'], (0, '', None))) self._ExpectProcesses(first_processes + ( (['git', 'rev-parse', '--abbrev-ref', 'HEAD'], (0, branch, None)), (['git', 'update-index', '--refresh', '-q'], ( 0, None, None, )), (['git', 'diff-index', 'HEAD'], (0, '', None)), (['git', 'log', 'origin/master..HEAD'], (0, 'logs here', None)), (['git', 'checkout', '-b', 'telemetry-tryjob'], (0, None, None)), (['git', 'branch', '--set-upstream-to', 'origin/master'], (0, None, None)), (['git', 'commit', '-a', '-m', 'bisect config'], (0, None, None)), ([ 'git', 'cl', 'upload', '-f', '--bypass-hooks', '-m', 'CL for perf tryjob' ], (0, 'stuff https://codereview.chromium.org/12345 stuff', None)), (['git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b', bot], (0, None, None)), (['git', 'checkout', branch], (0, None, None)), (['git', 'branch', '-D', 'telemetry-tryjob'], (0, None, None)))) self._stubs.sys.argv = [ 'tools/perf/run_benchmark', '--browser=%s' % browser, 'sunspider' ] cfg = StringIO.StringIO() self._stubs.open.files = {cfg_filename: cfg} browser.RunRemote() return cfg.getvalue()
def test_dirty_tree(self): finder_options = browser_options.BrowserFinderOptions() browser = trybot_browser_finder.PossibleTrybotBrowser( 'trybot-android-nexus4', finder_options) self._ExpectProcesses(( (['git', 'rev-parse', '--abbrev-ref', 'HEAD'], (0, 'br', None)), (['git', 'update-index', '--refresh', '-q'], ( 0, None, None, )), (['git', 'diff-index', 'HEAD'], (0, 'dirty tree', None)), )) browser.RunRemote() self.assertEquals( 'Cannot send a try job with a dirty tree. Commit locally first.\n', self.log_output.getvalue())
def Main(): profile_creators = _DiscoverProfileCreatorClasses() legal_profile_creators = '|'.join(profile_creators.keys()) options = browser_options.BrowserFinderOptions() parser = options.CreateParser( "%%prog <--profile-type-to-generate=...> <--browser=...>" " <--output-directory>") page_runner.AddCommandLineOptions(parser) group = optparse.OptionGroup(parser, 'Profile generation options') group.add_option('--profile-type-to-generate', dest='profile_type_to_generate', default=None, help='Type of profile to generate. ' 'Supported values: %s' % legal_profile_creators) group.add_option('--output-dir', dest='output_dir', help='Generated profile is placed in this directory.') parser.add_option_group(group) _, _ = parser.parse_args() # Sanity check arguments. if not options.profile_type_to_generate: raise Exception("Must specify --profile-type-to-generate option.") if options.profile_type_to_generate not in profile_creators.keys(): raise Exception("Invalid profile type, legal values are: %s." % legal_profile_creators) if not options.browser_type: raise Exception("Must specify --browser option.") if not options.output_dir: raise Exception("Must specify --output-dir option.") if options.browser_options.dont_override_profile: raise Exception("Can't use existing profile when generating profile.") # Generate profile. profile_creator_class = profile_creators[options.profile_type_to_generate] return GenerateProfiles(profile_creator_class, options.profile_type_to_generate, options)
def __init__(self, project): self._project = project self._browser = None self._tab = None self._server = None if telemetry == None: raise Exception( 'Not supported: trace-viewer must be inside a chrome checkout for this to work.' ) # If run in the context of the telemetry test runner, use # telemetry's browser options instead. if telemetry_options_for_unittests.AreSet(): finder_options = telemetry_options_for_unittests.GetCopy() else: finder_options = browser_options.BrowserFinderOptions() parser = finder_options.CreateParser('telemetry_perf_test.py') from tvcm import test_runner finder_options, _ = parser.parse_args( ['--browser', test_runner.BROWSER_TYPE_TO_USE]) finder_options.browser_options.warn_if_no_flash = False browser_to_create = browser_finder.FindBrowser(finder_options) if not browser_to_create: raise Exception('Failed to find the specified browser. ' + ' Its binary is probably broken.') self._browser = browser_to_create.Create() self._tab = None try: self._browser.Start() self._tab = self._browser.tabs[0] # Prevent the project from being mutated from here on out. project.Freeze() self._server = _LocalDevServer(self._project) self._browser.StartLocalServer(self._server) except: self._browser.Close() raise
def test_branch_checkout_fails(self): finder_options = browser_options.BrowserFinderOptions() browser = trybot_browser_finder.PossibleTrybotBrowser( 'trybot-android-nexus4', finder_options) self._ExpectProcesses(( (['git', 'rev-parse', '--abbrev-ref', 'HEAD'], (0, 'br', None)), (['git', 'update-index', '--refresh', '-q'], (0, None, None,)), (['git', 'diff-index', 'HEAD'], (0, '', None)), (['git', 'log', 'origin/master..HEAD'], (0, 'logs here', None)), (['git', 'checkout', '-b', 'telemetry-tryjob'], (1, None, 'fatal: A branch named \'telemetry-try\' already exists.')), )) browser.RunRemote() self.assertEquals( ('Error creating branch telemetry-tryjob. ' 'Please delete it if it exists.\n' 'fatal: A branch named \'telemetry-try\' already exists.\n'), self.log_output.getvalue())
def SetUpCycler(self, page_repeat=1, pageset_repeat=10, cold_load_percent=50, record_v8_object_stats=False, report_speed_index=False, setup_memory_module=False): cycler = page_cycler.PageCycler( page_repeat=page_repeat, pageset_repeat=pageset_repeat, cold_load_percent=cold_load_percent, record_v8_object_stats=record_v8_object_stats, report_speed_index=report_speed_index) options = browser_options.BrowserFinderOptions() options.browser_options.platform = FakePlatform() parser = options.CreateParser() user_story_runner.AddCommandLineArgs(parser) args = [ '--page-repeat=%i' % page_repeat, '--pageset-repeat=%i' % pageset_repeat ] parser.parse_args(args) user_story_runner.ProcessCommandLineArgs(parser, options) cycler.CustomizeBrowserOptions(options.browser_options) if setup_memory_module: # Mock out memory metrics; the real ones require a real browser. mock_memory_metric = MockMemoryMetric() mock_memory_module = simple_mock.MockObject() mock_memory_module.ExpectCall('MemoryMetric').WithArgs( simple_mock.DONT_CARE).WillReturn(mock_memory_metric) real_memory_module = page_cycler.memory try: page_cycler.memory = mock_memory_module browser = FakeBrowser() cycler.WillStartBrowser(options.browser_options.platform) cycler.DidStartBrowser(browser) finally: page_cycler.memory = real_memory_module return cycler
def testPageTestWithCompatibleUserStory(self): original_run_fn = story_runner.Run was_run = [False] def RunStub(*_arg, **_kwargs): was_run[0] = True story_runner.Run = RunStub try: options = browser_options.BrowserFinderOptions() options.output_formats = ['none'] options.suppress_gtest_report = True parser = optparse.OptionParser() benchmark.AddCommandLineArgs(parser) options.MergeDefaultValues(parser.get_default_values()) b = TestBenchmark(page.Page(url='about:blank')) b.Run(options) finally: story_runner.Run = original_run_fn self.assertTrue(was_run[0])
def test_no_local_commits(self): finder_options = browser_options.BrowserFinderOptions() browser = trybot_browser_finder.PossibleTrybotBrowser( 'trybot-android-nexus4', finder_options) self._ExpectProcesses(( (['git', 'rev-parse', '--abbrev-ref', 'HEAD'], (0, 'br', None)), (['git', 'update-index', '--refresh', '-q'], (0, None, None,)), (['git', 'diff-index', 'HEAD'], (0, '', None)), (['git', 'log', 'origin/master..HEAD'], (0, '', None)), (['git', 'rev-parse', '--abbrev-ref', 'HEAD'], (0, 'br', None)), (['git', 'update-index', '--refresh', '-q'], (0, None, None,)), (['git', 'diff-index', 'HEAD'], (0, '', None)), (['git', 'log', 'origin/master..HEAD'], (0, '', None)), )) browser.RunRemote() self.assertEquals( ('No local changes found in chromium or blink trees. ' 'browser=trybot-android-nexus4 argument sends local changes to the ' 'android_nexus4_perf_bisect perf trybot.\n'), self.log_output.getvalue())
def test_update_config_git_upload_tryboterror(self): finder_options = browser_options.BrowserFinderOptions() self._MockTryserverJson({'android_nexus4_perf_bisect': 'stuff'}) browser = trybot_browser_finder.PossibleTrybotBrowser( 'trybot-android-nexus4', finder_options) self._ExpectProcesses( ((['git', 'commit', '-a', '-m', 'bisect config: android'], (0, 'None', None)), ([ 'git', 'cl', 'upload', '-f', '--bypass-hooks', '-m', 'CL for perf tryjob on android' ], (128, None, 'error')), ([ 'git', 'cl', 'try', '-m', 'tryserver.chromium.perf', '-b', 'android_nexus4_perf_bisect' ], (0, None, None)))) self._stubs.sys.argv = [ 'tools/perf/run_benchmark', '--browser=%s' % browser, 'sunspider' ] cfg_filename = 'tools/run-perf-test.cfg' cfg = StringIO.StringIO() self._stubs.open.files = {cfg_filename: cfg} self.assertRaises(trybot_browser_finder.TrybotError, browser._UpdateConfigAndRunTryjob, 'android', cfg_filename)
def setupCycler(self, args, setup_memory_module=False): cycler = page_cycler.PageCycler() options = browser_options.BrowserFinderOptions() parser = options.CreateParser() cycler.AddCommandLineOptions(parser) parser.parse_args(args) cycler.CustomizeBrowserOptions(options) if setup_memory_module: # Mock out memory metrics; the real ones require a real browser. mock_memory_metric = MockMemoryMetric() mock_memory_module = simple_mock.MockObject() mock_memory_module.ExpectCall('MemoryMetric').WithArgs( simple_mock.DONT_CARE).WillReturn(mock_memory_metric) real_memory_module = page_cycler.memory try: page_cycler.memory = mock_memory_module cycler.DidStartBrowser(None) finally: page_cycler.memory = real_memory_module return cycler
def setUp(self): self.finder_options = browser_options.BrowserFinderOptions() # Mock out what's needed for testing with exact APKs self._android_browser_finder_stub = system_stub.Override( android_browser_finder, ['adb_commands', 'os'])
def Main(base_dir): measurements = { n: cls for n, cls in discover.DiscoverClasses( base_dir, base_dir, page_measurement.PageMeasurement).items() # Filter out unneeded ProfileCreators (crbug.com/319573). if not issubclass(cls, profile_creator.ProfileCreator) } tests = discover.DiscoverClasses(base_dir, base_dir, test.Test, index_by_class_name=True) options = browser_options.BrowserFinderOptions() parser = options.CreateParser('%prog <PageSet|Test|URL>') page_runner.AddCommandLineArgs(parser) recorder = RecordPage(measurements) recorder.AddCommandLineArgs(parser) quick_args = [a for a in sys.argv[1:] if not a.startswith('-')] if len(quick_args) != 1: parser.print_usage() sys.exit(1) target = quick_args[0] if target in tests: recorder.test = tests[target]().test() recorder.test.AddCommandLineArgs(parser) recorder.test.SetArgumentDefaults(parser) parser.parse_args() recorder.test.ProcessCommandLineArgs(parser, options) ps = tests[target]().CreatePageSet(options) elif discover.IsPageSetFile(target): parser.parse_args() ps = page_set.PageSet.FromFile(target) else: parser.print_usage() sys.exit(1) page_runner.ProcessCommandLineArgs(parser, options) recorder.ProcessCommandLineArgs(parser, options) expectations = test_expectations.TestExpectations() # Set the archive path to something temporary. temp_target_wpr_file_path = tempfile.mkstemp()[1] ps.wpr_archive_info.AddNewTemporaryRecording(temp_target_wpr_file_path) # Do the actual recording. options.browser_options.wpr_mode = wpr_modes.WPR_RECORD options.browser_options.no_proxy_server = True recorder.CustomizeBrowserOptions(options) results = page_runner.Run(recorder, ps, expectations, options) if results.errors or results.failures: logging.warning( 'Some pages failed. The recording has not been updated for ' 'these pages.') logging.warning('Failed pages:\n%s', '\n'.join(zip(*results.errors + results.failures)[0])) if results.skipped: logging.warning('Some pages were skipped. The recording has not been ' 'updated for these pages.') logging.warning('Skipped pages:\n%s', '\n'.join(zip(*results.skipped)[0])) if results.successes: # Update the metadata for the pages which were recorded. ps.wpr_archive_info.AddRecordedPages(results.successes) else: os.remove(temp_target_wpr_file_path) return min(255, len(results.failures))
import telemetry.core from telemetry.core import browser_options from telemetry.core import browser_finder from json import dumps options = browser_options.BrowserFinderOptions() parser = options.CreateParser() (_, args) = parser.parse_args() browserFactory = browser_finder.FindBrowser(options) with browserFactory.Create(options) as browser: tab = browser.tabs.New() tab.Activate() for i in browser.tabs: if i == tab.id: continue browser.tabs.GetTabById(i).Close() tab.Navigate(args[0]) tab.WaitForDocumentReadyStateToBeComplete() json = tab.EvaluateJavaScript("""(function() { function descend(node, out) { if (node.nodeType == Node.ELEMENT_NODE) { if (node.nodeName == 'STYLE' || node.nodeName == 'SCRIPT') return; out.push(node); for (var i = 0; i < node.attributes.length; ++i) { if (node.attributes[i].name == 'style')
def _CreateOptions(self): options = browser_options.BrowserFinderOptions() options.browser_options.wpr_mode = wpr_modes.WPR_RECORD options.browser_options.no_proxy_server = True return options
def CreateParser(cls): options = browser_options.BrowserFinderOptions() parser = options.CreateParser('%%prog %s %s' % (cls.Name(), cls.usage)) return parser
def test_adb_no_devices(self): finder_options = browser_options.BrowserFinderOptions() browsers = android_browser_finder.FindAllAvailableBrowsers( finder_options) self.assertEquals(0, len(browsers))
def CreateParser(cls): options = browser_options.BrowserFinderOptions() options.browser_type = 'any' parser = options.CreateParser('%%prog %s' % cls.usage) return parser
def testDefaultsPlusOverride(self): options = browser_options.BrowserFinderOptions() parser = options.CreateParser() parser.add_option('-x', action='store', default=3) parser.parse_args(['--browser', 'any', '-x', 10]) self.assertEquals(options.x, 10) # pylint: disable=E1101
def testCount2(self): options = browser_options.BrowserFinderOptions() parser = options.CreateParser() parser.add_option('-x', action='count', dest='v') parser.parse_args(['--browser', 'any', '-xx']) self.assertEquals(options.v, 2) # pylint: disable=E1101
def testProfileDirDefault(self): options = browser_options.BrowserFinderOptions() parser = options.CreateParser() parser.parse_args(['--browser', 'any']) self.assertEquals(options.browser_options.profile_dir, None)