def testNeedsBrowserRestartAfterEachPage(self): self.CaptureFormattedException() ps = page_set.PageSet() expectations = test_expectations.TestExpectations() ps.pages.append(page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) ps.pages.append(page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) class Test(page_test.PageTest): def __init__(self, *args, **kwargs): super(Test, self).__init__(*args, **kwargs) self.browser_starts = 0 def DidStartBrowser(self, *args): super(Test, self).DidStartBrowser(*args) self.browser_starts += 1 def ValidateAndMeasurePage(self, page, tab, results): pass options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True test = Test(needs_browser_restart_after_each_page=True) SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run(test, ps, expectations, options, results) self.assertEquals(2, len(GetSuccessfulPageRuns(results))) self.assertEquals(2, test.browser_starts) self.assertFormattedExceptionIsEmpty()
def testRunPageWithStartupUrl(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() expectations = test_expectations.TestExpectations() page = page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir(), startup_url='about:blank') ps.pages.append(page) class Measurement(page_test.PageTest): def __init__(self): super(Measurement, self).__init__() self.browser_restarted = False def CustomizeBrowserOptionsForSinglePage(self, ps, options): self.browser_restarted = True super(Measurement, self).CustomizeBrowserOptionsForSinglePage(ps, options) def ValidateAndMeasurePage(self, page, tab, results): pass options = options_for_unittests.GetCopy() options.page_repeat = 2 options.output_formats = ['none'] options.suppress_gtest_report = True if not browser_finder.FindBrowser(options): return test = Measurement() SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run(test, ps, expectations, options, results) self.assertEquals('about:blank', options.browser_options.startup_url) self.assertTrue(test.browser_restarted)
def testWebPageReplay(self): ps = example_domain.ExampleDomainPageSet() expectations = test_expectations.TestExpectations() body = [] class TestWpr(page_test.PageTest): def ValidateAndMeasurePage(self, _, tab, __): body.append(tab.EvaluateJavaScript('document.body.innerText')) test = TestWpr() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(test, ps, expectations, options, results) self.longMessage = True self.assertIn('Example Domain', body[0], msg='URL: %s' % ps.pages[0].url) self.assertIn('Example Domain', body[1], msg='URL: %s' % ps.pages[1].url) self.assertEquals(2, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures))
def testRunPageWithProfilingFlag(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() ps.pages.append( page_module.Page('file://blank.html', ps, base_dir=util.GetUnittestDataDir())) class Measurement(page_test.PageTest): pass options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True options.reset_results = None options.upload_results = None options.results_label = None options.output_dir = tempfile.mkdtemp() options.profiler = 'trace' try: SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(Measurement(), ps, expectations, options, results) self.assertEquals(1, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures)) self.assertEquals(0, len(results.all_page_specific_values)) self.assertTrue( os.path.isfile( os.path.join(options.output_dir, 'blank_html.json'))) finally: shutil.rmtree(options.output_dir)
def _CreateBrowser(self, autotest_ext=False, auto_login=True, gaia_login=False, username=None, password=None): """Finds and creates a browser for tests. if autotest_ext is True, also loads the autotest extension""" options = options_for_unittests.GetCopy() if autotest_ext: extension_path = os.path.join(util.GetUnittestDataDir(), 'autotest_ext') assert os.path.isdir(extension_path) self._load_extension = extension_to_load.ExtensionToLoad( path=extension_path, browser_type=options.browser_type, is_component=True) options.extensions_to_load = [self._load_extension] browser_to_create = browser_finder.FindBrowser(options) self.assertTrue(browser_to_create) browser_options = options.browser_options browser_options.create_browser_with_oobe = True browser_options.auto_login = auto_login browser_options.gaia_login = gaia_login if username is not None: browser_options.username = username if password is not None: browser_options.password = password return browser_to_create.Create(options)
def _RunPageTestThatRaisesAppCrashException(self, test, max_failures): class TestPage(page_module.Page): def RunNavigateSteps(self, _): raise exceptions.AppCrashException ps = page_set.PageSet() for _ in range(5): ps.AddUserStory( TestPage('file://blank.html', ps, base_dir=util.GetUnittestDataDir())) expectations = test_expectations.TestExpectations() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(test, ps, expectations, options, results, max_failures=max_failures) return results
def setUp(self): self._options = options_for_unittests.GetCopy() self._options.browser_options.wpr_mode = wpr_modes.WPR_OFF self._first_thread_name = ( task_execution_time.TaskExecutionTime._RENDERER_THREADS[0]) self._measurement = None self._page_set = None
def _GetAllPossiblePageTestInstances(): page_test_instances = [] measurements_dir = os.path.dirname(__file__) top_level_dir = os.path.dirname(measurements_dir) benchmarks_dir = os.path.join(top_level_dir, 'benchmarks') # Get all page test instances from measurement classes that are directly # constructable all_measurement_classes = discover.DiscoverClasses( measurements_dir, top_level_dir, page_test.PageTest).values() for measurement_class in all_measurement_classes: if classes.IsDirectlyConstructable(measurement_class): page_test_instances.append(measurement_class()) all_benchmarks_classes = discover.DiscoverClasses( benchmarks_dir, top_level_dir, benchmark_module.Benchmark).values() # Get all page test instances from defined benchmarks. # Note: since this depends on the command line options, there is no guaranteed # that this will generate all possible page test instances but it's worth # enough for smoke test purpose. for benchmark_class in all_benchmarks_classes: options = options_for_unittests.GetCopy() parser = options.CreateParser() benchmark_class.AddCommandLineArgs(parser) benchmark_module.AddCommandLineArgs(parser) benchmark_class.SetArgumentDefaults(parser) options.MergeDefaultValues(parser.get_default_values()) pt = benchmark_class().CreatePageTest(options) if not isinstance(pt, timeline_based_measurement.TimelineBasedMeasurement): page_test_instances.append(pt) return page_test_instances
def RunMeasurement(self, measurement, ps, expectations=test_expectations.TestExpectations(), options=None): """Runs a measurement against a pageset, returning the rows its outputs.""" if options is None: options = options_for_unittests.GetCopy() assert options temp_parser = options.CreateParser() story_runner.AddCommandLineArgs(temp_parser) defaults = temp_parser.get_default_values() for k, v in defaults.__dict__.items(): if hasattr(options, k): continue setattr(options, k, v) measurement.CustomizeBrowserOptions(options.browser_options) options.output_file = None options.output_formats = ['none'] options.suppress_gtest_report = True options.output_trace_tag = None story_runner.ProcessCommandLineArgs(temp_parser, options) results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run(measurement, ps, expectations, options, results) return results
def BenchmarkSmokeTest(self): # Only measure a single page so that this test cycles reasonably quickly. benchmark.options['pageset_repeat'] = 1 benchmark.options['page_repeat'] = 1 class SinglePageBenchmark(benchmark): # pylint: disable=W0232 def CreatePageSet(self, options): # pylint: disable=E1002 ps = super(SinglePageBenchmark, self).CreatePageSet(options) for p in ps.pages: p.skip_waits = True ps.user_stories = [p] break return ps # Set the benchmark's default arguments. options = options_for_unittests.GetCopy() options.output_format = 'none' options.suppress_gtest_report = True parser = options.CreateParser() benchmark.AddCommandLineArgs(parser) benchmark_module.AddCommandLineArgs(parser) benchmark.SetArgumentDefaults(parser) options.MergeDefaultValues(parser.get_default_values()) benchmark.ProcessCommandLineArgs(None, options) benchmark_module.ProcessCommandLineArgs(None, options) self.assertEqual(0, SinglePageBenchmark().Run(options), msg='Failed: %s' % benchmark)
def testBrowserBeforeLaunch(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() page = page_module.Page('file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page) class TestBeforeLaunch(page_test.PageTest): def __init__(self): super(TestBeforeLaunch, self).__init__() self._did_call_will_start = False self._did_call_did_start = False def WillStartBrowser(self, platform): self._did_call_will_start = True # TODO(simonjam): Test that the profile is available. def DidStartBrowser(self, browser): assert self._did_call_will_start self._did_call_did_start = True def ValidatePage(self, *_): assert self._did_call_did_start test = TestBeforeLaunch() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(test, ps, expectations, options, results)
def testCleanUpPage(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() page = page_module.Page('file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page) class Test(page_test.PageTest): def __init__(self): super(Test, self).__init__() self.did_call_clean_up = False def ValidatePage(self, *_): raise exceptions.IntentionalException def CleanUpAfterPage(self, page, tab): self.did_call_clean_up = True test = Test() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(test, ps, expectations, options, results) assert test.did_call_clean_up
def testOneTab(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() page = page_module.Page('file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page) class TestOneTab(page_test.PageTest): def __init__(self): super(TestOneTab, self).__init__() self._browser = None def DidStartBrowser(self, browser): self._browser = browser self._browser.tabs.New() def ValidatePage(self, *_): assert len(self._browser.tabs) == 1 test = TestOneTab() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(test, ps, expectations, options, results)
def testUserAgent(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() page = page_module.Page('file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page) ps.user_agent_type = 'tablet' class TestUserAgent(page_test.PageTest): def ValidatePage(self, _1, tab, _2): actual_user_agent = tab.EvaluateJavaScript( 'window.navigator.userAgent') expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet'] assert actual_user_agent.strip() == expected_user_agent # This is so we can check later that the test actually made it into this # function. Previously it was timing out before even getting here, which # should fail, but since it skipped all the asserts, it slipped by. self.hasRun = True # pylint: disable=W0201 test = TestUserAgent() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(test, ps, expectations, options, results) self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
def _RebuildCredentials(self): credentials = {} if self._credentials_path == None: pass elif os.path.exists(self._credentials_path): with open(self._credentials_path, 'r') as f: credentials = json.loads(f.read()) # TODO(nduca): use system keychain, if possible. homedir_credentials_path = os.path.expanduser('~/.telemetry-credentials') homedir_credentials = {} if (not options_for_unittests.GetCopy() and os.path.exists(homedir_credentials_path)): logging.info("Found ~/.telemetry-credentials. Its contents will be used " "when no other credentials can be found.") with open(homedir_credentials_path, 'r') as f: homedir_credentials = json.loads(f.read()) self._credentials = {} all_keys = set(credentials.keys()).union( homedir_credentials.keys()).union( self._extra_credentials.keys()) for k in all_keys: if k in credentials: self._credentials[k] = credentials[k] if k in homedir_credentials: logging.info("Will use ~/.telemetry-credentials for %s logins." % k) self._credentials[k] = homedir_credentials[k] if k in self._extra_credentials: self._credentials[k] = self._extra_credentials[k]
def setUpClass(cls): cls._number_of_tabs = 4 cls._profile_dir = _GenerateBrowserProfile(cls._number_of_tabs) cls._options = options_for_unittests.GetCopy() cls._options.browser_options.AppendExtraBrowserArgs( ['--restore-last-session']) cls._options.browser_options.profile_dir = cls._profile_dir cls._browser_to_create = browser_finder.FindBrowser(cls._options)
def setUp(self): options = options_for_unittests.GetCopy() self._cri = cros_interface.CrOSInterface(options.cros_remote, options.cros_ssh_identity) self._is_guest = options.browser_type == 'cros-chrome-guest' self._username = options.browser_options.username self._password = options.browser_options.password self._load_extension = None
def setUp(self): self._options = options_for_unittests.GetCopy() self._stubs = system_stub.Override( android_platform_backend, ['perf_control', 'adb_commands']) self.battery_patcher = mock.patch.object(battery_utils, 'BatteryUtils') self.battery_patcher.start() self._actual_ps_util = android_platform_backend.psutil
def testNonExistentExtensionPath(self): """Test that a non-existent extension path will raise an exception.""" extension_path = os.path.join(util.GetUnittestDataDir(), 'foo') options = options_for_unittests.GetCopy() self.assertRaises( extension_to_load.ExtensionPathNonExistentException, lambda: extension_to_load.ExtensionToLoad(extension_path, options. browser_type))
def _GetOptionForUnittest(): options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = False parser = options.CreateParser() story_runner.AddCommandLineArgs(parser) options.MergeDefaultValues(parser.get_default_values()) story_runner.ProcessCommandLineArgs(parser, options) return options
def setUp(self): self._options = options_for_unittests.GetCopy() self._stubs = system_stub.Override(android_platform_backend, [ 'perf_control', 'thermal_throttle', 'adb_commands', 'certutils', 'adb_install_cert' ]) # Skip _FixPossibleAdbInstability by setting psutil to None. self._actual_ps_util = android_platform_backend.psutil android_platform_backend.psutil = None
def testComponentExtensionNoPublicKey(self): # simple_extension does not have a public key. extension_path = os.path.join(util.GetUnittestDataDir(), 'simple_extension') options = options_for_unittests.GetCopy() self.assertRaises( extension_to_load.MissingPublicKeyException, lambda: extension_to_load.ExtensionToLoad(extension_path, browser_type=options. browser_type, is_component=True))
def setUp(self): self._options = options_for_unittests.GetCopy() self._stubs = system_stub.Override(android_platform_backend, ['perf_control', 'adb_commands']) self.battery_patcher = mock.patch.object(battery_utils, 'BatteryUtils') self.battery_patcher.start() self._actual_ps_util = android_platform_backend.psutil self.setup_prebuilt_tool_patcher = mock.patch( 'telemetry.core.platform.android_platform_backend._SetupPrebuiltTools' ) m = self.setup_prebuilt_tool_patcher.start() m.return_value = True
def testExtensionNotLoaded(self): """Querying an extension that was not loaded will return None""" extension_path = os.path.join(util.GetUnittestDataDir(), 'simple_extension') options = options_for_unittests.GetCopy() load_extension = extension_to_load.ExtensionToLoad( extension_path, options.browser_type) browser_to_create = browser_finder.FindBrowser(options) with browser_to_create.Create(options) as b: if b.supports_extensions: self.assertRaises(KeyError, lambda: b.extensions[load_extension])
def testPerformNavigations(self): maximum_batch_size = 15 options = options_for_unittests.GetCopy() extender = FastNavigationProfileExtender(options, maximum_batch_size) navigation_urls = [] for i in range(extender._NUM_TABS): navigation_urls.append('http://test%s.com' % i) batch_size = 5 navigation_urls_batch = navigation_urls[3:3 + batch_size] extender.GetUrlIterator = mock.MagicMock( return_value=iter(navigation_urls_batch)) extender.ShouldExitAfterBatchNavigation = mock.MagicMock(return_value=True) extender._WaitForQueuedTabsToLoad = mock.MagicMock() extender._browser = FakeBrowser() extender._BatchNavigateTabs = mock.MagicMock() # Set up a callback to record the tabs and urls in each navigation. callback_tabs_batch = [] callback_urls_batch = [] def SideEffect(*args, **_): batch = args[0] for tab, url in batch: callback_tabs_batch.append(tab) callback_urls_batch.append(url) extender._BatchNavigateTabs.side_effect = SideEffect # Perform the navigations. extender._PerformNavigations() # Each url in the batch should have been navigated to exactly once. self.assertEqual(set(callback_urls_batch), set(navigation_urls_batch)) # The other urls should not have been navigated to. navigation_urls_remaining = (set(navigation_urls) - set(navigation_urls_batch)) self.assertFalse(navigation_urls_remaining & set(callback_urls_batch)) # The first couple of tabs should have been navigated once. The remaining # tabs should not have been navigated. for i in range(len(extender._browser.tabs)): tab = extender._browser.tabs._tabs[i] if i < batch_size: expected_tab_navigation_count = 1 else: expected_tab_navigation_count = 0 count = callback_tabs_batch.count(tab) self.assertEqual(count, expected_tab_navigation_count)
def testRealLoginIfPossible(self): credentials_path = _GetCredentialsPath() if not credentials_path: logging.warning('Credentials file not found, skipping test.') return options = options_for_unittests.GetCopy() with browser_finder.FindBrowser(options).Create(options) as b: b.credentials.credentials_path = credentials_path if not b.credentials.CanLogin(self._credentials_type): return ret = b.credentials.LoginNeeded(b.tabs[0], self._credentials_type) self.assertTrue(ret)
def testEscapeCmdArguments(self): ''' Commands and their arguments that are executed through the cros interface should follow bash syntax. This test needs to run on remotely and locally on the device to check for consistency. ''' with cros_interface.CrOSInterface( options_for_unittests.GetCopy().cros_remote, options_for_unittests.GetCopy().cros_ssh_identity) as cri: # Check arguments with no special characters stdout, _ = cri.RunCmdOnDevice( ['echo', '--arg1=value1', '--arg2=value2', '--arg3="value3"']) assert stdout.strip( ) == '--arg1=value1 --arg2=value2 --arg3=value3' # Check argument with special characters escaped stdout, _ = cri.RunCmdOnDevice(['echo', '--arg=A\\; echo \\"B\\"']) assert stdout.strip() == '--arg=A; echo "B"' # Check argument with special characters in quotes stdout, _ = cri.RunCmdOnDevice(['echo', "--arg='$HOME;;$PATH'"]) assert stdout.strip() == "--arg=$HOME;;$PATH"
def CreateBrowserWithExtension(self, ext_path): extension_path = os.path.join(util.GetUnittestDataDir(), ext_path) options = options_for_unittests.GetCopy() load_extension = extension_to_load.ExtensionToLoad( extension_path, options.browser_type) options.extensions_to_load = [load_extension] browser_to_create = browser_finder.FindBrowser(options) if not browser_to_create: # May not find a browser that supports extensions. return False self._browser = browser_to_create.Create(options) self._extension = self._browser.extensions[load_extension] self._extension_id = load_extension.extension_id self.assertTrue(self._extension) return True
def _testMaxFailuresOptionIsRespectedAndOverridable( self, max_failures=None): self.SuppressExceptionFormatting() class TestPage(page_module.Page): def __init__(self, *args, **kwargs): super(TestPage, self).__init__(*args, **kwargs) self.was_run = False def RunNavigateSteps(self, action_runner): # pylint: disable=W0613 self.was_run = True raise Exception('Test exception') class Test(page_test.PageTest): def ValidatePage(self, *args): pass ps = page_set.PageSet() expectations = test_expectations.TestExpectations() for ii in range(5): ps.pages.append( TestPage('file://blank.html', ps, base_dir=util.GetUnittestDataDir())) options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True expected_max_failures = 2 if not max_failures is None: options.max_failures = max_failures expected_max_failures = max_failures SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(Test(max_failures=2), ps, expectations, options, results) self.assertEquals(0, len(GetSuccessfulPageRuns(results))) # Runs up to max_failures+1 failing tests before stopping, since # every tests after max_failures failures have been encountered # may all be passing. self.assertEquals(expected_max_failures + 1, len(results.failures)) for ii in range(len(ps.pages)): if ii <= expected_max_failures: self.assertTrue(ps.pages[ii].was_run) else: self.assertFalse(ps.pages[ii].was_run)
def setUp(self): self._options = options_for_unittests.GetCopy() self._stubs = system_stub.Override(android_platform_backend, [ 'perf_control', 'thermal_throttle', 'adb_commands', 'certutils', 'adb_install_cert', 'platformsettings' ]) # Skip _FixPossibleAdbInstability by setting psutil to None. self._actual_ps_util = android_platform_backend.psutil android_platform_backend.psutil = None self.battery_patcher = mock.patch.object(battery_utils, 'BatteryUtils') self.battery_patcher.start() self.setup_prebuilt_tool_patcher = mock.patch( 'telemetry.core.platform.android_platform_backend._SetupPrebuiltTools' ) m = self.setup_prebuilt_tool_patcher.start() m.return_value = True