def RunMeasurement(self, measurement, ps, options=None): """Runs a measurement against a pageset, returning the rows its outputs.""" if options is None: options = options_for_unittests.GetCopy() assert options temp_parser = options.CreateParser() story_runner.AddCommandLineArgs(temp_parser) defaults = temp_parser.get_default_values() for k, v in defaults.__dict__.items(): if hasattr(options, k): continue setattr(options, k, v) if isinstance(measurement, legacy_page_test.LegacyPageTest): measurement.CustomizeBrowserOptions(options.browser_options) options.output_file = None options.output_formats = ['none'] options.suppress_gtest_report = True options.output_trace_tag = None story_runner.ProcessCommandLineArgs(temp_parser, options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( measurement, ps, possible_browser, None, # expectations options.browser_options, options, results) return results
def RunPageTest(self, page_test, url, expect_status='PASS'): """Run a legacy page_test on a test url and return its measurements. Args: page_test: A legacy_page_test.LegacyPageTest instance. url: A URL for the test page to load, usually a local 'file://' URI to be served from telemetry/internal/testing. Clients can override the static method CreateStorySetForTestFile to change this behavior. expect_status: A string with the expected status of the test run. Returns: A dictionary with measurements recorded by the legacy_page_test. """ self.assertIsInstance(page_test, legacy_page_test.LegacyPageTest) page_test.CustomizeBrowserOptions(self.options.browser_options) story_set = self.CreateStorySetForTest(url) self.assertEqual(len(story_set), 1) with results_options.CreateResults(self.options) as results: story_runner.RunStorySet(page_test, story_set, self.options, results) test_results = results_options.ReadTestResults( self.options.intermediate_dir) self.assertEqual(len(test_results), 1) self.test_result = test_results[0] self.assertEqual(self.test_result['status'], expect_status) return results_options.ReadMeasurements(self.test_result)
def _RunPageTestThatRaisesAppCrashException(self, test, max_failures): class TestPage(page_module.Page): def RunNavigateSteps(self, _): raise exceptions.AppCrashException story_set = story.StorySet() for i in range(5): story_set.AddStory( TestPage('file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), name='foo%d' % i)) options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=test, story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, max_failures=max_failures, ) return results
def testTrafficSettings(self): story_set = story.StorySet() slow_page = page_module.Page( 'file://green_rect.html', story_set, base_dir=util.GetUnittestDataDir(), name='slow', traffic_setting=traffic_setting_module.GOOD_3G) fast_page = page_module.Page( 'file://green_rect.html', story_set, base_dir=util.GetUnittestDataDir(), name='fast', traffic_setting=traffic_setting_module.WIFI) story_set.AddStory(slow_page) story_set.AddStory(fast_page) latencies_by_page_in_ms = {} class MeasureLatency(legacy_page_test.LegacyPageTest): def __init__(self): super(MeasureLatency, self).__init__() self._will_navigate_time = None def WillNavigateToPage(self, page, tab): del page, tab # unused self._will_navigate_time = time.time() * 1000 def ValidateAndMeasurePage(self, page, tab, results): del results # unused latencies_by_page_in_ms[page.name] = ( time.time() * 1000 - self._will_navigate_time) test = MeasureLatency() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir: options.output_dir = tempdir SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=test, story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, ) failure_messages = [] for r in results.all_page_runs: if r.failure_str: failure_messages.append( 'Failure message of story %s:\n%s' % (r.story, r.failure_str)) self.assertFalse(results.had_failures, msg=''.join(failure_messages)) self.assertIn('slow', latencies_by_page_in_ms) self.assertIn('fast', latencies_by_page_in_ms) # Slow page should be slower than fast page by at least 40 ms (roundtrip # time of good 3G) - 2 ms (roundtrip time of Wifi) self.assertGreater(latencies_by_page_in_ms['slow'], latencies_by_page_in_ms['fast'] + 40 - 2)
def testOneTab(self): story_set = story.StorySet() page = page_module.Page( 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), name='blank.html') story_set.AddStory(page) class TestOneTab(legacy_page_test.LegacyPageTest): def DidStartBrowser(self, browser): browser.tabs.New() def ValidateAndMeasurePage(self, page, tab, results): del page, results # unused assert len(tab.browser.tabs) == 1 test = TestOneTab() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir: options.output_dir = tempdir SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=test, story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, )
def testNoScreenShotTakenForFailedPageDueToNoSupport(self): self.options.browser_options.take_screenshot_for_failed_page = True class FailingTestPage(page_module.Page): def RunNavigateSteps(self, action_runner): raise exceptions.AppCrashException story_set = story.StorySet() story_set.AddStory(page_module.Page('file://blank.html', story_set, name='blank.html')) failing_page = FailingTestPage('chrome://version', story_set, name='failing') story_set.AddStory(failing_page) results = results_options.CreateResults( EmptyMetadataForTest(), self.options) possible_browser = browser_finder.FindBrowser(self.options) story_runner.RunStorySet( test=DummyTest(), story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=self.options.browser_options, finder_options=self.options, results=results, max_failures=2, ) self.assertTrue(results.had_failures)
def RunStorySetAndGetResults(self, story_set): dummy_test = test_stories.DummyStoryTest() with results_options.CreateResults(self.options) as results: story_runner.RunStorySet(dummy_test, story_set, self.options, results) test_results = results_options.ReadIntermediateResults( self.options.intermediate_dir)['testResults'] self.assertEqual(len(test_results), 1) return test_results[0]
def Record(self, results): assert self._story_set.wpr_archive_info, ( 'Pageset archive_data_file path must be specified.') # Always record the benchmark one time only. self._options.pageset_repeat = 1 self._story_set.wpr_archive_info.AddNewTemporaryRecording() self._record_page_test.CustomizeBrowserOptions(self._options) story_runner.RunStorySet(self._record_page_test, self._story_set, self._options, results)
def testScreenShotTakenForFailedPageOnSupportedPlatform(self): fake_platform = self.options.fake_possible_browser.returned_browser.platform expected_png_base64 = """ iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91 JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A T8tgwbJAAAAABJRU5ErkJggg== """ fake_platform.screenshot_png_data = expected_png_base64 self.options.browser_options.take_screenshot_for_failed_page = True class FailingTestPage(page_module.Page): def RunNavigateSteps(self, action_runner): raise exceptions.AppCrashException story_set = story.StorySet() story_set.AddStory(page_module.Page('file://blank.html', story_set, name='blank.html')) failing_page = FailingTestPage('chrome://version', story_set, name='failing') story_set.AddStory(failing_page) self.options.output_formats = ['json-test-results'] with tempfile_ext.NamedTemporaryDirectory() as tempdir: self.options.output_dir = tempdir results = results_options.CreateResults( EmptyMetadataForTest(), self.options) # This ensures the output stream used by the json test results object is # closed. On windows, we can't delete the temp directory if a file in that # directory is still in use. with results: possible_browser = browser_finder.FindBrowser(self.options) story_runner.RunStorySet( test=DummyTest(), story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=self.options.browser_options, finder_options=self.options, results=results, max_failures=2, ) self.assertTrue(results.had_failures) artifacts = results._artifact_results.GetTestArtifacts( failing_page.name) self.assertIsNotNone(artifacts) self.assertIn('screenshot', artifacts) screenshot_file_path = os.path.join(tempdir, artifacts['screenshot'][0]) actual_screenshot_img = image_util.FromPngFile(screenshot_file_path) self.assertTrue( image_util.AreEqual( image_util.FromBase64Png(expected_png_base64), actual_screenshot_img))
def testScreenShotTakenForFailedPage(self): self.CaptureFormattedException() platform_screenshot_supported = [False] tab_screenshot_supported = [False] chrome_version_screen_shot = [None] class FailingTestPage(page_module.Page): def RunNavigateSteps(self, action_runner): action_runner.Navigate(self._url) platform_screenshot_supported[0] = ( action_runner.tab.browser.platform.CanTakeScreenshot) tab_screenshot_supported[0] = action_runner.tab.screenshot_supported if not platform_screenshot_supported[0] and tab_screenshot_supported[0]: chrome_version_screen_shot[0] = action_runner.tab.Screenshot() raise exceptions.AppCrashException story_set = story.StorySet() story_set.AddStory(page_module.Page('file://blank.html', story_set, name='blank.html')) failing_page = FailingTestPage('chrome://version', story_set, name='failing') story_set.AddStory(failing_page) with tempfile_ext.NamedTemporaryDirectory() as tempdir: options = options_for_unittests.GetCopy() options.output_dir = tempdir options.output_formats = ['none'] options.browser_options.take_screenshot_for_failed_page = True options.suppress_gtest_report = True SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=DummyTest(), story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, max_failures=2, ) self.assertTrue(results.had_failures) if not platform_screenshot_supported[0] and tab_screenshot_supported[0]: artifacts = results._artifact_results.GetTestArtifacts( failing_page.name) self.assertIsNotNone(artifacts) self.assertIn('screenshot', artifacts) screenshot_file_path = os.path.join(tempdir, artifacts['screenshot'][0]) actual_screenshot = image_util.FromPngFile(screenshot_file_path) self.assertEquals(image_util.Pixels(chrome_version_screen_shot[0]), image_util.Pixels(actual_screenshot))
def testSharedPageStateCannotRunOnBrowser(self): story_set = story.StorySet() class UnrunnableSharedState(shared_page_state.SharedPageState): def CanRunOnBrowser(self, browser_info, page): del browser_info, page # unused return False def ValidateAndMeasurePage(self, _): pass story_set.AddStory(page_module.Page( url='file://blank.html', page_set=story_set, base_dir=util.GetUnittestDataDir(), shared_page_state_class=UnrunnableSharedState, name='blank.html')) class Test(legacy_page_test.LegacyPageTest): def __init__(self, *args, **kwargs): super(Test, self).__init__(*args, **kwargs) self.will_navigate_to_page_called = False def ValidateAndMeasurePage(self, *args): del args # unused raise Exception('Exception should not be thrown') def WillNavigateToPage(self, page, tab): del page, tab # unused self.will_navigate_to_page_called = True test = Test() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=test, story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, ) self.assertFalse(test.will_navigate_to_page_called) self.assertEquals(1, len(GetSuccessfulPageRuns(results))) self.assertEquals(1, len(results.skipped_values)) self.assertFalse(results.had_failures)
def testBrowserRestartsAfterEachPage(self): self.CaptureFormattedException() story_set = story.StorySet() story_set.AddStory(page_module.Page( 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), name='foo')) story_set.AddStory(page_module.Page( 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), name='bar')) class Test(legacy_page_test.LegacyPageTest): def __init__(self, *args, **kwargs): super(Test, self).__init__(*args, **kwargs) self.browser_starts = 0 self.platform_name = None def DidStartBrowser(self, browser): super(Test, self).DidStartBrowser(browser) self.browser_starts += 1 self.platform_name = browser.platform.GetOSName() def ValidateAndMeasurePage(self, page, tab, results): pass options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir: options.output_dir = tempdir test = Test() SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test, story_set, possible_browser, None, # expectations options.browser_options, options, results) self.assertEquals(len(story_set), len(GetSuccessfulPageRuns(results))) # Browser is started once per story run, except in ChromeOS where a single # instance is reused for all stories. if test.platform_name == 'chromeos': self.assertEquals(1, test.browser_starts) else: self.assertEquals(len(story_set), test.browser_starts) self.assertFormattedExceptionIsEmpty()
def RunMeasurement(self, measurement, story_set, run_options): """Runs a measurement against a story set, returning a results object. Args: measurement: A test object: either a story_test.StoryTest or legacy_page_test.LegacyPageTest instance. story_set: A story set. run_options: An object with all options needed to run stories; can be created with the help of options_for_unittests.GetRunOptions(). """ if isinstance(measurement, legacy_page_test.LegacyPageTest): measurement.CustomizeBrowserOptions(run_options.browser_options) with results_options.CreateResults( run_options, benchmark_name=BENCHMARK_NAME) as results: story_runner.RunStorySet(measurement, story_set, run_options, results) return results
def Record(self, results): assert self._story_set.wpr_archive_info, ( 'Pageset archive_data_file path must be specified.') # Always record the benchmark one time only. self._options.pageset_repeat = 1 self._story_set.wpr_archive_info.AddNewTemporaryRecording() self._record_page_test.CustomizeBrowserOptions(self._options) possible_browser = browser_finder.FindBrowser(self._options) story_runner.RunStorySet( test=self._record_page_test, story_set=self._story_set, possible_browser=possible_browser, expectations=None, browser_options=self._options.browser_options, finder_options=self._options, results=results, )
def testBrowserBeforeLaunch(self): story_set = story.StorySet() page = page_module.Page( 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), name='blank.html') story_set.AddStory(page) class TestBeforeLaunch(legacy_page_test.LegacyPageTest): def __init__(self): super(TestBeforeLaunch, self).__init__() self._did_call_will_start = False self._did_call_did_start = False def WillStartBrowser(self, platform): self._did_call_will_start = True # TODO(simonjam): Test that the profile is available. def DidStartBrowser(self, browser): assert self._did_call_will_start self._did_call_did_start = True def ValidateAndMeasurePage(self, *_): assert self._did_call_did_start test = TestBeforeLaunch() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir: options.output_dir = tempdir SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=test, story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, )
def testUserAgent(self): story_set = story.StorySet() page = page_module.Page( 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), shared_page_state_class=shared_page_state.SharedTabletPageState, name='blank.html') story_set.AddStory(page) class TestUserAgent(legacy_page_test.LegacyPageTest): def ValidateAndMeasurePage(self, page, tab, results): del page, results # unused actual_user_agent = tab.EvaluateJavaScript( 'window.navigator.userAgent') expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet'] assert actual_user_agent.strip() == expected_user_agent # This is so we can check later that the test actually made it into this # function. Previously it was timing out before even getting here, which # should fail, but since it skipped all the asserts, it slipped by. self.hasRun = True # pylint: disable=attribute-defined-outside-init test = TestUserAgent() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir: options.output_dir = tempdir SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=test, story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, ) self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
def testCleanUpPage(self): story_set = story.StorySet() page = page_module.Page( 'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(), name='blank.html') story_set.AddStory(page) class Test(legacy_page_test.LegacyPageTest): def __init__(self): super(Test, self).__init__() self.did_call_clean_up = False def ValidateAndMeasurePage(self, *_): raise legacy_page_test.Failure def DidRunPage(self, platform): del platform # unused self.did_call_clean_up = True test = Test() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir: options.output_dir = tempdir SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=test, story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, ) assert test.did_call_clean_up
def testWebPageReplay(self): story_set = example_domain.ExampleDomainPageSet() body = [] class TestWpr(legacy_page_test.LegacyPageTest): def ValidateAndMeasurePage(self, page, tab, results): del page, results # unused body.append(tab.EvaluateJavaScript('document.body.innerText')) def DidRunPage(self, platform): # Force the replay server to restart between pages; this verifies that # the restart mechanism works. platform.network_controller.StopReplay() test = TestWpr() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=test, story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, ) self.longMessage = True self.assertIn('Example Domain', body[0], msg='URL: %s' % story_set.stories[0].url) self.assertIn('Example Domain', body[1], msg='URL: %s' % story_set.stories[1].url) self.assertEquals(2, len(GetSuccessfulPageRuns(results))) self.assertFalse(results.had_failures)
def RunStories(self, stories, **kwargs): story_set = test_stories.DummyStorySet(stories) with results_options.CreateResults( self.options, benchmark_name='benchmark') as results: story_runner.RunStorySet(self.mock_story_test, story_set, self.options, results, **kwargs)
def RunStorySet(test, story_set, options, **kwargs): with results_options.CreateResults(options) as results: story_runner.RunStorySet(test, story_set, options, results, **kwargs) return results
def testPageResetWhenBrowserReusedBetweenStories(self): class NoClosingBrowserSharedState(shared_page_state.SharedPageState): # Simulate what ChromeOS does. def ShouldStopBrowserAfterStoryRun(self, s): del s # unused return False # Loads a page and scrolls it to the end. class ScrollingPage(page_module.Page): def __init__(self, url, page_set, base_dir): super(ScrollingPage, self).__init__(page_set=page_set, base_dir=base_dir, shared_page_state_class= NoClosingBrowserSharedState, url=url, name='ScrollingPage') def RunPageInteractions(self, action_runner): action_runner.ScrollPage() # Loads same page as ScrollingPage() and records if the scroll position is # at the top of the page (in was_page_at_top_on_start). class CheckScrollPositionPage(page_module.Page): def __init__(self, url, page_set, base_dir): super(CheckScrollPositionPage, self).__init__( page_set=page_set, base_dir=base_dir, shared_page_state_class=NoClosingBrowserSharedState, url=url, name='CheckScroll') self.was_page_at_top_on_start = False def RunPageInteractions(self, action_runner): scroll_y = action_runner.tab.EvaluateJavaScript('window.scrollY') self.was_page_at_top_on_start = scroll_y == 0 class Test(legacy_page_test.LegacyPageTest): def ValidateAndMeasurePage(self, *_): pass story_set = story.StorySet() story_set.AddStory(ScrollingPage( url='file://page_with_swipeables.html', page_set=story_set, base_dir=util.GetUnittestDataDir())) test_page = CheckScrollPositionPage( url='file://page_with_swipeables.html', page_set=story_set, base_dir=util.GetUnittestDataDir()) story_set.AddStory(test_page) test = Test() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) possible_browser = browser_finder.FindBrowser(options) story_runner.RunStorySet( test=test, story_set=story_set, possible_browser=possible_browser, expectations=None, browser_options=options.browser_options, finder_options=options, results=results, ) self.assertTrue(test_page.was_page_at_top_on_start)