コード例 #1
0
  def testUnknownExceptionIsFatal(self):
    self.SuppressExceptionFormatting()
    story_set = story_module.StorySet()

    class UnknownException(Exception):
      pass

    # This erroneous test is set up to raise exception for the 2nd story
    # run.
    class Test(page_test.PageTest):
      def __init__(self, *args):
        super(Test, self).__init__(*args)
        self.run_count = 0

      def RunPage(self, *_):
        old_run_count = self.run_count
        self.run_count += 1
        if old_run_count == 1:
          raise UnknownException('FooBarzException')

      def ValidateAndMeasurePage(self, page, tab, results):
        pass

    s1 = DummyLocalStory(TestSharedPageState)
    s2 = DummyLocalStory(TestSharedPageState)
    story_set.AddStory(s1)
    story_set.AddStory(s2)
    test = Test()
    with self.assertRaises(UnknownException):
      story_runner.Run(
          test, story_set, self.options, self.results)
    self.assertEqual(set([s2]), self.results.pages_that_failed)
    self.assertEqual(set([s1]), self.results.pages_that_succeeded)
    self.assertIn('FooBarzException', self.fake_stdout.getvalue())
コード例 #2
0
    def testNoProfilingFilesCreatedForPageByDefault(self):
        self.CaptureFormattedException()

        class FailingTestPage(page_module.Page):
            def RunNavigateSteps(self, action_runner):
                action_runner.Navigate(self._url)
                raise exceptions.AppCrashException

        story_set = story.StorySet()
        story_set.AddStory(page_module.Page('file://blank.html', story_set))
        failing_page = FailingTestPage('chrome://version', story_set)
        story_set.AddStory(failing_page)
        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.suppress_gtest_report = True
        SetUpStoryRunnerArguments(options)
        results = results_options.CreateResults(EmptyMetadataForTest(),
                                                options)
        story_runner.Run(DummyTest(),
                         story_set,
                         options,
                         results,
                         max_failures=2)
        self.assertEquals(1, len(results.failures))
        self.assertEquals(0, len(results.pages_to_profiling_files))
コード例 #3
0
 def RunStoryTest(self, s, expected_successes):
   test = DummyTest()
   story_runner.Run(
       test, s, self.options, self.results)
   self.assertEquals(0, len(self.results.failures))
   self.assertEquals(expected_successes,
                     GetNumberOfSuccessfulPageRuns(self.results))
コード例 #4
0
    def testWebPageReplay(self):
        story_set = example_domain.ExampleDomainPageSet()
        body = []

        class TestWpr(legacy_page_test.LegacyPageTest):
            def ValidateAndMeasurePage(self, page, tab, results):
                del page, results  # unused
                body.append(tab.EvaluateJavaScript2('document.body.innerText'))

            def DidRunPage(self, platform):
                # Force the replay server to restart between pages; this verifies that
                # the restart mechanism works.
                platform.network_controller.StopReplay()

        test = TestWpr()
        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.suppress_gtest_report = True
        SetUpStoryRunnerArguments(options)
        results = results_options.CreateResults(EmptyMetadataForTest(),
                                                options)

        story_runner.Run(test, story_set, options, results)

        self.longMessage = True
        self.assertIn('Example Domain',
                      body[0],
                      msg='URL: %s' % story_set.stories[0].url)
        self.assertIn('Example Domain',
                      body[1],
                      msg='URL: %s' % story_set.stories[1].url)

        self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
        self.assertEquals(0, len(results.failures))
コード例 #5
0
    def testRunPageWithProfilingFlag(self):
        story_set = story.StorySet()
        story_set.AddStory(
            page_module.Page('file://blank.html',
                             story_set,
                             base_dir=util.GetUnittestDataDir()))

        class Measurement(legacy_page_test.LegacyPageTest):
            def ValidateAndMeasurePage(self, page, tab, results):
                pass

        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.suppress_gtest_report = True
        options.reset_results = None
        options.upload_results = None
        options.results_label = None
        options.output_dir = tempfile.mkdtemp()
        options.profiler = 'trace'
        try:
            SetUpStoryRunnerArguments(options)
            results = results_options.CreateResults(EmptyMetadataForTest(),
                                                    options)
            story_runner.Run(Measurement(), story_set, options, results)
            self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
            self.assertEquals(0, len(results.failures))
            self.assertEquals(0, len(results.all_page_specific_values))
            self.assertTrue(
                os.path.isfile(
                    os.path.join(options.output_dir, 'blank_html.html')))
        finally:
            shutil.rmtree(options.output_dir)
コード例 #6
0
    def testCleanUpPage(self):
        story_set = story.StorySet()
        page = page_module.Page('file://blank.html',
                                story_set,
                                base_dir=util.GetUnittestDataDir())
        story_set.AddStory(page)

        class Test(legacy_page_test.LegacyPageTest):
            def __init__(self):
                super(Test, self).__init__()
                self.did_call_clean_up = False

            def ValidateAndMeasurePage(self, *_):
                raise legacy_page_test.Failure

            def DidRunPage(self, platform):
                del platform  # unused
                self.did_call_clean_up = True

        test = Test()
        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.suppress_gtest_report = True
        SetUpStoryRunnerArguments(options)
        results = results_options.CreateResults(EmptyMetadataForTest(),
                                                options)
        story_runner.Run(test, story_set, options, results)
        assert test.did_call_clean_up
コード例 #7
0
    def testBrowserBeforeLaunch(self):
        story_set = story.StorySet()
        page = page_module.Page('file://blank.html',
                                story_set,
                                base_dir=util.GetUnittestDataDir())
        story_set.AddStory(page)

        class TestBeforeLaunch(legacy_page_test.LegacyPageTest):
            def __init__(self):
                super(TestBeforeLaunch, self).__init__()
                self._did_call_will_start = False
                self._did_call_did_start = False

            def WillStartBrowser(self, platform):
                self._did_call_will_start = True
                # TODO(simonjam): Test that the profile is available.

            def DidStartBrowser(self, browser):
                assert self._did_call_will_start
                self._did_call_did_start = True

            def ValidateAndMeasurePage(self, *_):
                assert self._did_call_did_start

        test = TestBeforeLaunch()
        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.suppress_gtest_report = True
        SetUpStoryRunnerArguments(options)
        results = results_options.CreateResults(EmptyMetadataForTest(),
                                                options)
        story_runner.Run(test, story_set, options, results)
コード例 #8
0
    def testUserAgent(self):
        story_set = story.StorySet()
        page = page_module.Page(
            'file://blank.html',
            story_set,
            base_dir=util.GetUnittestDataDir(),
            shared_page_state_class=shared_page_state.SharedTabletPageState)
        story_set.AddStory(page)

        class TestUserAgent(legacy_page_test.LegacyPageTest):
            def ValidateAndMeasurePage(self, page, tab, results):
                del page, results  # unused
                actual_user_agent = tab.EvaluateJavaScript2(
                    'window.navigator.userAgent')
                expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
                assert actual_user_agent.strip() == expected_user_agent

                # This is so we can check later that the test actually made it into this
                # function. Previously it was timing out before even getting here, which
                # should fail, but since it skipped all the asserts, it slipped by.
                self.hasRun = True  # pylint: disable=attribute-defined-outside-init

        test = TestUserAgent()
        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.suppress_gtest_report = True
        SetUpStoryRunnerArguments(options)
        results = results_options.CreateResults(EmptyMetadataForTest(),
                                                options)
        story_runner.Run(test, story_set, options, results)

        self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
コード例 #9
0
 def testRunStoryWithLongURLPage(self):
   story_set = story_module.StorySet()
   story_set.AddStory(page_module.Page('file://long' + 'g' * 180,
                                       story_set, name='test'))
   test = DummyTest()
   story_runner.Run(test, story_set,
                    self.options, self.results,
                    metadata=EmptyMetadataForTest())
コード例 #10
0
 def Record(self, results):
     assert self._page_set.wpr_archive_info, (
         'Pageset archive_data_file path must be specified.')
     self._page_set.wpr_archive_info.AddNewTemporaryRecording()
     self._record_page_test.CustomizeBrowserOptions(self._options)
     story_runner.Run(self._record_page_test, self._page_set,
                      test_expectations.TestExpectations(), self._options,
                      results)
コード例 #11
0
  def _testMaxFailuresOptionIsRespectedAndOverridable(
      self, num_failing_user_stories, runner_max_failures, options_max_failures,
      expected_num_failures):
    class SimpleSharedState(
        shared_state.SharedState):
      _fake_platform = FakePlatform()
      _current_user_story = None

      @property
      def platform(self):
        return self._fake_platform

      def WillRunUserStory(self, story):
        self._current_user_story = story

      def RunUserStory(self, results):
        self._current_user_story.Run()

      def DidRunUserStory(self, results):
        pass

      def GetTestExpectationAndSkipValue(self, expectations):
        return 'pass', None

      def TearDownState(self):
        pass

    class FailingUserStory(user_story.UserStory):
      def __init__(self):
        super(FailingUserStory, self).__init__(
            shared_state_class=SimpleSharedState,
            is_local=True)
        self.was_run = False

      def Run(self):
        self.was_run = True
        raise page_test.Failure

    self.SuppressExceptionFormatting()

    uss = user_story_set.UserStorySet()
    for _ in range(num_failing_user_stories):
      uss.AddUserStory(FailingUserStory())

    options = _GetOptionForUnittest()
    options.output_formats = ['none']
    options.suppress_gtest_report = True
    if options_max_failures:
      options.max_failures = options_max_failures

    results = results_options.CreateResults(EmptyMetadataForTest(), options)
    story_runner.Run(
        DummyTest(), uss, test_expectations.TestExpectations(), options,
        results, max_failures=runner_max_failures)
    self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results))
    self.assertEquals(expected_num_failures, len(results.failures))
    for ii, story in enumerate(uss.user_stories):
      self.assertEqual(story.was_run, ii < expected_num_failures)
コード例 #12
0
    def testTrafficSettings(self):
        story_set = story.StorySet()
        slow_page = page_module.Page(
            'file://green_rect.html',
            story_set,
            base_dir=util.GetUnittestDataDir(),
            name='slow',
            traffic_setting=traffic_setting_module.GOOD_3G)
        fast_page = page_module.Page(
            'file://green_rect.html',
            story_set,
            base_dir=util.GetUnittestDataDir(),
            name='fast',
            traffic_setting=traffic_setting_module.WIFI)
        story_set.AddStory(slow_page)
        story_set.AddStory(fast_page)

        latencies_by_page_in_ms = {}

        class MeasureLatency(legacy_page_test.LegacyPageTest):
            def __init__(self):
                super(MeasureLatency, self).__init__()
                self._will_navigate_time = None

            def WillNavigateToPage(self, page, tab):
                del page, tab  # unused
                self._will_navigate_time = time.time() * 1000

            def ValidateAndMeasurePage(self, page, tab, results):
                del results  # unused
                latencies_by_page_in_ms[page.name] = (time.time() * 1000 -
                                                      self._will_navigate_time)

        test = MeasureLatency()
        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.suppress_gtest_report = True

        with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
            options.output_dir = tempdir
            SetUpStoryRunnerArguments(options)
            results = results_options.CreateResults(EmptyMetadataForTest(),
                                                    options)
            story_runner.Run(test, story_set, options, results)
            failure_messages = []
            for r in results.all_page_runs:
                if r.failure_str:
                    failure_messages.append(
                        'Failure message of story %s:\n%s' %
                        (r.story, r.failure_str))
            self.assertFalse(results.had_failures,
                             msg=''.join(failure_messages))
            self.assertIn('slow', latencies_by_page_in_ms)
            self.assertIn('fast', latencies_by_page_in_ms)
            # Slow page should be slower than fast page by at least 40 ms (roundtrip
            # time of good 3G) - 2 ms (roundtrip time of Wifi)
            self.assertGreater(latencies_by_page_in_ms['slow'],
                               latencies_by_page_in_ms['fast'] + 40 - 2)
コード例 #13
0
    def testScreenShotTakenForFailedPage(self):
        self.CaptureFormattedException()
        platform_screenshot_supported = [False]
        tab_screenshot_supported = [False]
        chrome_version_screen_shot = [None]

        class FailingTestPage(page_module.Page):
            def RunNavigateSteps(self, action_runner):
                action_runner.Navigate(self._url)
                platform_screenshot_supported[0] = (
                    action_runner.tab.browser.platform.CanTakeScreenshot)
                tab_screenshot_supported[
                    0] = action_runner.tab.screenshot_supported
                if not platform_screenshot_supported[
                        0] and tab_screenshot_supported[0]:
                    chrome_version_screen_shot[
                        0] = action_runner.tab.Screenshot()
                raise exceptions.AppCrashException

        story_set = story.StorySet()
        story_set.AddStory(
            page_module.Page('file://blank.html', story_set,
                             name='blank.html'))
        failing_page = FailingTestPage('chrome://version',
                                       story_set,
                                       name='failing')
        story_set.AddStory(failing_page)
        with tempfile_ext.NamedTemporaryDirectory() as tempdir:
            options = options_for_unittests.GetCopy()
            options.output_dir = tempdir
            options.output_formats = ['none']
            options.browser_options.take_screenshot_for_failed_page = True
            options.suppress_gtest_report = True
            SetUpStoryRunnerArguments(options)
            results = results_options.CreateResults(EmptyMetadataForTest(),
                                                    options)
            story_runner.Run(DummyTest(),
                             story_set,
                             options,
                             results,
                             max_failures=2,
                             metadata=EmptyMetadataForTest())
            self.assertTrue(results.had_failures)
            if not platform_screenshot_supported[
                    0] and tab_screenshot_supported[0]:
                artifacts = results._artifact_results.GetTestArtifacts(
                    failing_page.name)
                self.assertIsNotNone(artifacts)
                self.assertIn('screenshot', artifacts)

                screenshot_file_path = os.path.join(tempdir,
                                                    artifacts['screenshot'][0])

                actual_screenshot = image_util.FromPngFile(
                    screenshot_file_path)
                self.assertEquals(
                    image_util.Pixels(chrome_version_screen_shot[0]),
                    image_util.Pixels(actual_screenshot))
コード例 #14
0
    def testPagesetRepeat(self):
        story_set = story_module.StorySet()

        # TODO(eakuefner): Factor this out after flattening page ref in Value
        blank_story = DummyLocalStory(TestSharedPageState, name='blank')
        green_story = DummyLocalStory(TestSharedPageState, name='green')
        story_set.AddStory(blank_story)
        story_set.AddStory(green_story)

        class Measurement(legacy_page_test.LegacyPageTest):
            i = 0

            def RunPage(self, page, _, results):
                self.i += 1
                results.AddValue(
                    scalar.ScalarValue(
                        page,
                        'metric',
                        'unit',
                        self.i,
                        improvement_direction=improvement_direction.UP))

            def ValidateAndMeasurePage(self, page, tab, results):
                pass

        self.options.page_repeat = 1
        self.options.pageset_repeat = 2
        self.options.output_formats = []
        results = results_options.CreateResults(EmptyMetadataForTest(),
                                                self.options)
        story_runner.Run(Measurement(), story_set, self.options, results)
        summary = summary_module.Summary(results.all_page_specific_values)
        values = summary.interleaved_computed_per_page_values_and_summaries

        blank_value = list_of_scalar_values.ListOfScalarValues(
            blank_story,
            'metric',
            'unit', [1, 3],
            improvement_direction=improvement_direction.UP)
        green_value = list_of_scalar_values.ListOfScalarValues(
            green_story,
            'metric',
            'unit', [2, 4],
            improvement_direction=improvement_direction.UP)
        merged_value = list_of_scalar_values.ListOfScalarValues(
            None,
            'metric',
            'unit',
            [1, 3, 2, 4],
            std=math.sqrt(2),  # Pooled standard deviation.
            improvement_direction=improvement_direction.UP)

        self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results))
        self.assertEquals(0, len(results.failures))
        self.assertEquals(3, len(values))
        self.assertIn(blank_value, values)
        self.assertIn(green_value, values)
        self.assertIn(merged_value, values)
コード例 #15
0
 def Record(self, results):
     assert self._story_set.wpr_archive_info, (
         'Pageset archive_data_file path must be specified.')
     self._story_set.wpr_archive_info.AddNewTemporaryRecording()
     self._record_page_test.CustomizeBrowserOptions(self._options)
     story_runner.Run(self._record_page_test,
                      self._story_set,
                      self._options,
                      results,
                      metadata=self._CreateBenchmarkMetadata())
コード例 #16
0
    def Record(self, results):
        assert self._story_set.wpr_archive_info, (
            'Pageset archive_data_file path must be specified.')

        # Always record the benchmark one time only.
        self._options.pageset_repeat = 1
        self._story_set.wpr_archive_info.AddNewTemporaryRecording()
        self._record_page_test.CustomizeBrowserOptions(self._options)
        story_runner.Run(self._record_page_test, self._story_set,
                         self._options, results)
コード例 #17
0
  def testPageResetWhenBrowserReusedBetweenStories(self):
    class NoClosingBrowserSharedState(shared_page_state.SharedPageState):
      # Simulate what ChromeOS does.
      def ShouldStopBrowserAfterStoryRun(self, s):
        del s  # unused
        return False

    # Loads a page and scrolls it to the end.
    class ScrollingPage(page_module.Page):
      def __init__(self, url, page_set, base_dir):
        super(ScrollingPage, self).__init__(page_set=page_set,
                                            base_dir=base_dir,
                                            shared_page_state_class=
                                            NoClosingBrowserSharedState,
                                            url=url, name='ScrollingPage')

      def RunPageInteractions(self, action_runner):
        action_runner.ScrollPage()

    # Loads same page as ScrollingPage() and records if the scroll position is
    # at the top of the page (in was_page_at_top_on_start).
    class CheckScrollPositionPage(page_module.Page):
      def __init__(self, url, page_set, base_dir):
        super(CheckScrollPositionPage, self).__init__(
            page_set=page_set, base_dir=base_dir,
            shared_page_state_class=NoClosingBrowserSharedState, url=url,
            name='CheckScroll')
        self.was_page_at_top_on_start = False

      def RunPageInteractions(self, action_runner):
        scroll_y = action_runner.tab.EvaluateJavaScript('window.scrollY')
        self.was_page_at_top_on_start = scroll_y == 0

    class Test(legacy_page_test.LegacyPageTest):
      def ValidateAndMeasurePage(self, *_):
        pass

    story_set = story.StorySet()
    story_set.AddStory(ScrollingPage(
        url='file://page_with_swipeables.html', page_set=story_set,
        base_dir=util.GetUnittestDataDir()))
    test_page = CheckScrollPositionPage(
        url='file://page_with_swipeables.html', page_set=story_set,
        base_dir=util.GetUnittestDataDir())
    story_set.AddStory(test_page)
    test = Test()
    options = options_for_unittests.GetCopy()
    SetUpStoryRunnerArguments(options)
    # Override defaults from parser creation and arg processing.
    options.output_formats = ['none']
    options.suppress_gtest_report = True
    options.output_dir = None
    results = results_options.CreateResults(options)
    story_runner.Run(test, story_set, options, results)
    self.assertTrue(test_page.was_page_at_top_on_start)
コード例 #18
0
    def testScreenShotTakenForFailedPage(self):
        self.CaptureFormattedException()
        platform_screenshot_supported = [False]
        tab_screenshot_supported = [False]
        chrome_version_screen_shot = [None]

        class FailingTestPage(page_module.Page):
            def RunNavigateSteps(self, action_runner):
                action_runner.Navigate(self._url)
                platform_screenshot_supported[0] = (
                    action_runner.tab.browser.platform.CanTakeScreenshot)
                tab_screenshot_supported[
                    0] = action_runner.tab.screenshot_supported
                if not platform_screenshot_supported[
                        0] and tab_screenshot_supported[0]:
                    chrome_version_screen_shot[
                        0] = action_runner.tab.Screenshot()
                raise exceptions.AppCrashException

        story_set = story.StorySet()
        story_set.AddStory(
            page_module.Page('file://blank.html', story_set,
                             name='blank.html'))
        failing_page = FailingTestPage('chrome://version',
                                       story_set,
                                       name='failing')
        story_set.AddStory(failing_page)
        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.browser_options.take_screenshot_for_failed_page = True
        options.suppress_gtest_report = True
        SetUpStoryRunnerArguments(options)
        results = results_options.CreateResults(EmptyMetadataForTest(),
                                                options)
        story_runner.Run(DummyTest(),
                         story_set,
                         options,
                         results,
                         max_failures=2,
                         metadata=EmptyMetadataForTest())
        self.assertEquals(1, len(results.failures))
        if not platform_screenshot_supported[0] and tab_screenshot_supported[0]:
            self.assertEquals(1, len(results.pages_to_profiling_files))
            self.assertIn(failing_page, results.pages_to_profiling_files)
            screenshot_file_path = (
                results.pages_to_profiling_files[failing_page][0].GetAbsPath())
            try:
                actual_screenshot = image_util.FromPngFile(
                    screenshot_file_path)
                self.assertEquals(
                    image_util.Pixels(chrome_version_screen_shot[0]),
                    image_util.Pixels(actual_screenshot))
            finally:  # Must clean up screenshot file if exists.
                os.remove(screenshot_file_path)
コード例 #19
0
    def testScreenShotTakenForFailedPageOnSupportedPlatform(self):
        fake_platform = self.options.fake_possible_browser.returned_browser.platform
        expected_png_base64 = """
 iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91
 JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A
 T8tgwbJAAAAABJRU5ErkJggg==
"""
        fake_platform.screenshot_png_data = expected_png_base64
        self.options.browser_options.take_screenshot_for_failed_page = True

        class FailingTestPage(page_module.Page):
            def RunNavigateSteps(self, action_runner):
                raise exceptions.AppCrashException

        story_set = story.StorySet()
        story_set.AddStory(
            page_module.Page('file://blank.html', story_set,
                             name='blank.html'))
        failing_page = FailingTestPage('chrome://version',
                                       story_set,
                                       name='failing')
        story_set.AddStory(failing_page)

        self.options.output_formats = ['json-test-results']
        with tempfile_ext.NamedTemporaryDirectory() as tempdir:
            self.options.output_dir = tempdir
            results = results_options.CreateResults(EmptyMetadataForTest(),
                                                    self.options)

            # This ensures the output stream used by the json test results object is
            # closed. On windows, we can't delete the temp directory if a file in that
            # directory is still in use.
            with results:
                story_runner.Run(DummyTest(),
                                 story_set,
                                 self.options,
                                 results,
                                 max_failures=2,
                                 metadata=EmptyMetadataForTest())
                self.assertTrue(results.had_failures)
                artifacts = results._artifact_results.GetTestArtifacts(
                    failing_page.name)
                self.assertIsNotNone(artifacts)
                self.assertIn('screenshot', artifacts)

                screenshot_file_path = os.path.join(tempdir,
                                                    artifacts['screenshot'][0])

                actual_screenshot_img = image_util.FromPngFile(
                    screenshot_file_path)
                self.assertTrue(
                    image_util.AreEqual(
                        image_util.FromBase64Png(expected_png_base64),
                        actual_screenshot_img))
コード例 #20
0
    def testExceptionRaisedInSharedStateTearDown(self):
        self.SuppressExceptionFormatting()
        story_set = story_module.StorySet()

        class SharedStoryThatCausesAppCrash(TestSharedPageState):
            def TearDownState(self):
                raise TestOnlyException()

        story_set.AddStory(DummyLocalStory(SharedStoryThatCausesAppCrash))
        with self.assertRaises(TestOnlyException):
            story_runner.Run(DummyTest(), story_set, self.options,
                             self.results)
コード例 #21
0
 def testRunStorySet(self):
   number_stories = 3
   story_set = story_module.StorySet()
   for i in xrange(number_stories):
     story_set.AddStory(DummyLocalStory(FooStoryState, name='story_%d' % i))
   test = DummyTest()
   story_runner.Run(
       test, story_set, self.options, self.results,
       metadata=EmptyMetadataForTest())
   self.assertEquals(0, len(self.results.failures))
   self.assertEquals(number_stories,
                     GetNumberOfSuccessfulPageRuns(self.results))
コード例 #22
0
    def testRaiseBrowserGoneExceptionFromRestartBrowserBeforeEachPage(self):
        self.CaptureFormattedException()
        story_set = story.StorySet()
        story_set.AddStory(
            page_module.Page('file://blank.html',
                             story_set,
                             base_dir=util.GetUnittestDataDir(),
                             name='foo'))
        story_set.AddStory(
            page_module.Page('file://blank.html',
                             story_set,
                             base_dir=util.GetUnittestDataDir(),
                             name='bar'))
        story_set.AddStory(
            page_module.Page('file://blank.html',
                             story_set,
                             base_dir=util.GetUnittestDataDir(),
                             name='baz'))

        class Test(legacy_page_test.LegacyPageTest):
            def __init__(self, *args):
                super(Test, self).__init__(
                    *args, needs_browser_restart_after_each_page=True)
                self.run_count = 0

            def RestartBrowserBeforeEachPage(self):
                # This will only be called twice with 3 pages.
                old_run_count = self.run_count
                self.run_count += 1
                if old_run_count == 1:
                    raise exceptions.BrowserGoneException(None)
                return self._needs_browser_restart_after_each_page

            def ValidateAndMeasurePage(self, page, tab, results):
                pass

        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.suppress_gtest_report = True
        test = Test()
        SetUpStoryRunnerArguments(options)
        results = results_options.CreateResults(EmptyMetadataForTest(),
                                                options)
        story_runner.Run(test,
                         story_set,
                         options,
                         results,
                         metadata=EmptyMetadataForTest())
        self.assertEquals(2, test.run_count)
        self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
        self.assertEquals(1, len(results.failures))
        self.assertFormattedExceptionIsEmpty()
コード例 #23
0
    def testBrowserRestartsAfterEachPage(self):
        self.CaptureFormattedException()
        story_set = story.StorySet()
        story_set.AddStory(
            page_module.Page('file://blank.html',
                             story_set,
                             base_dir=util.GetUnittestDataDir(),
                             name='foo'))
        story_set.AddStory(
            page_module.Page('file://blank.html',
                             story_set,
                             base_dir=util.GetUnittestDataDir(),
                             name='bar'))

        class Test(legacy_page_test.LegacyPageTest):
            def __init__(self, *args, **kwargs):
                super(Test, self).__init__(*args, **kwargs)
                self.browser_starts = 0
                self.platform_name = None

            def DidStartBrowser(self, browser):
                super(Test, self).DidStartBrowser(browser)
                self.browser_starts += 1
                self.platform_name = browser.platform.GetOSName()

            def ValidateAndMeasurePage(self, page, tab, results):
                pass

        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.suppress_gtest_report = True

        with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
            options.output_dir = tempdir
            test = Test()
            SetUpStoryRunnerArguments(options)
            results = results_options.CreateResults(EmptyMetadataForTest(),
                                                    options)
            story_runner.Run(test,
                             story_set,
                             options,
                             results,
                             metadata=EmptyMetadataForTest())
            self.assertEquals(len(story_set),
                              len(GetSuccessfulPageRuns(results)))
            # Browser is started once per story run, except in ChromeOS where a single
            # instance is reused for all stories.
            if test.platform_name == 'chromeos':
                self.assertEquals(1, test.browser_starts)
            else:
                self.assertEquals(len(story_set), test.browser_starts)
            self.assertFormattedExceptionIsEmpty()
コード例 #24
0
    def runCredentialsTest(self, credentials_backend):
        story_set = story.StorySet()
        did_run = [False]

        try:

            class TestSharedState(shared_page_state.SharedPageState):
                def ShouldStopBrowserAfterStoryRun(self, _):
                    # Do not close browser for LoginNoLongerNeeded to get called.
                    return False

            with tempfile.NamedTemporaryFile(delete=False) as f:
                page = page_module.Page(
                    'file://blank.html',
                    story_set,
                    base_dir=util.GetUnittestDataDir(),
                    shared_page_state_class=TestSharedState,
                    credentials_path=f.name,
                    name='blank.html')
                page.credentials = "test"
                story_set.AddStory(page)

                f.write(SIMPLE_CREDENTIALS_STRING)

            class TestThatInstallsCredentialsBackend(
                    legacy_page_test.LegacyPageTest):
                def __init__(self, credentials_backend):
                    super(TestThatInstallsCredentialsBackend, self).__init__()
                    self._credentials_backend = credentials_backend

                def DidStartBrowser(self, browser):
                    browser.credentials.AddBackend(self._credentials_backend)

                def ValidateAndMeasurePage(self, *_):
                    did_run[0] = True

            test = TestThatInstallsCredentialsBackend(credentials_backend)
            options = options_for_unittests.GetCopy()
            options.output_formats = ['none']
            options.suppress_gtest_report = True
            SetUpStoryRunnerArguments(options)
            results = results_options.CreateResults(EmptyMetadataForTest(),
                                                    options)
            story_runner.Run(test,
                             story_set,
                             options,
                             results,
                             metadata=EmptyMetadataForTest())
        finally:
            os.remove(f.name)

        return did_run[0]
コード例 #25
0
    def testTearDownStateAfterEachStoryOrStorySetRun(self):
        class TestSharedStateForTearDown(TestSharedState):
            num_of_tear_downs = 0

            def RunStory(self, results):
                pass

            def TearDownState(self):
                TestSharedStateForTearDown.num_of_tear_downs += 1

        story_set = story_module.StorySet()
        story_set.AddStory(
            DummyLocalStory(TestSharedStateForTearDown, name='foo'))
        story_set.AddStory(
            DummyLocalStory(TestSharedStateForTearDown, name='bar'))
        story_set.AddStory(
            DummyLocalStory(TestSharedStateForTearDown, name='baz'))

        TestSharedStateForTearDown.num_of_tear_downs = 0
        story_runner.Run(mock.MagicMock(), story_set, self.options,
                         self.results)
        self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 1)

        TestSharedStateForTearDown.num_of_tear_downs = 0
        story_runner.Run(mock.MagicMock(),
                         story_set,
                         self.options,
                         self.results,
                         tear_down_after_story=True)
        self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 3)

        self.options.pageset_repeat = 5
        TestSharedStateForTearDown.num_of_tear_downs = 0
        story_runner.Run(mock.MagicMock(),
                         story_set,
                         self.options,
                         self.results,
                         tear_down_after_story_set=True)
        self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 5)
コード例 #26
0
  def testAppCrashExceptionCausesFailureValue(self):
    self.SuppressExceptionFormatting()
    us = user_story_set.UserStorySet()
    class SharedUserStoryThatCausesAppCrash(TestSharedPageState):
      def WillRunUserStory(self, user_storyz):
        raise exceptions.AppCrashException('App Foo crashes')

    us.AddUserStory(DummyLocalUserStory(SharedUserStoryThatCausesAppCrash))
    story_runner.Run(
        DummyTest(), us, self.expectations, self.options, self.results)
    self.assertEquals(1, len(self.results.failures))
    self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results))
    self.assertIn('App Foo crashes', self.fake_stdout.getvalue())
コード例 #27
0
ファイル: runner.py プロジェクト: leer168/blink-misc
def run_benchmark(benchmark, finder_options):
    benchmark.CustomizeBrowserOptions(finder_options.browser_options)
    pt = benchmark.CreatePageTest(finder_options)
    pt.__name__ = benchmark.__class__.__name__
    stories = benchmark.CreateStorySet(finder_options)
    benchmark_metadata = benchmark.GetMetadata()
    with results_options.CreateResults(
            benchmark_metadata, finder_options,
            benchmark.ValueCanBeAddedPredicate) as results:
        failures = story_runner.Run(pt, stories, finder_options, results,
                                    benchmark.max_failures)
        results._SerializeTracesToDirPath(results._output_dir)
    return failures
コード例 #28
0
  def testTearDownStateAfterEachStoryRun(self):
    class TestSharedStateForTearDown(TestSharedState):
      num_of_tear_downs = 0

      def RunStory(self, results):
        pass

      def TearDownState(self):
        TestSharedStateForTearDown.num_of_tear_downs += 1

    story_set = story_module.StorySet()
    story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown))
    story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown))
    story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown))

    TestSharedStateForTearDown.num_of_tear_downs = 0
    story_runner.Run(mock.MagicMock(), story_set, self.options, self.results)
    self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 1)

    TestSharedStateForTearDown.num_of_tear_downs = 0
    story_runner.Run(mock.MagicMock(), story_set, self.options, self.results,
                     should_tear_down_state_after_each_story_run=True)
    self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 3)
コード例 #29
0
    def Run(self, finder_options):
        """Run this test with the given options.

    Returns:
      The number of failure values (up to 254) or 255 if there is an uncaught
      exception.
    """
        self.CustomizeBrowserOptions(finder_options.browser_options)

        pt = self.CreatePageTest(finder_options)
        pt.__name__ = self.__class__.__name__

        if hasattr(self, '_disabled_strings'):
            # pylint: disable=protected-access
            pt._disabled_strings = self._disabled_strings
        if hasattr(self, '_enabled_strings'):
            # pylint: disable=protected-access
            pt._enabled_strings = self._enabled_strings

        expectations = self.CreateExpectations()
        us = self.CreateStorySet(finder_options)
        if isinstance(pt, page_test.PageTest):
            if any(not isinstance(p, page.Page) for p in us.user_stories):
                raise Exception(
                    'PageTest must be used with StorySet containing only '
                    'telemetry.page.Page user stories.')

        benchmark_metadata = self.GetMetadata()
        with results_options.CreateResults(
                benchmark_metadata, finder_options,
                self.ValueCanBeAddedPredicate) as results:
            try:
                story_runner.Run(pt,
                                 us,
                                 expectations,
                                 finder_options,
                                 results,
                                 max_failures=self._max_failures)
                return_code = min(254, len(results.failures))
            except Exception:
                exception_formatter.PrintFormattedException()
                return_code = 255

            bucket = cloud_storage.BUCKET_ALIASES[finder_options.upload_bucket]
            if finder_options.upload_results:
                results.UploadTraceFilesToCloud(bucket)
                results.UploadProfilingFilesToCloud(bucket)

            results.PrintSummary()
        return return_code
コード例 #30
0
    def testTrafficSettings(self):
        story_set = story.StorySet()
        slow_page = page_module.Page(
            'file://green_rect.html',
            story_set,
            base_dir=util.GetUnittestDataDir(),
            name='slow',
            traffic_setting=traffic_setting_module.REGULAR_2G)
        fast_page = page_module.Page(
            'file://green_rect.html',
            story_set,
            base_dir=util.GetUnittestDataDir(),
            name='fast',
            traffic_setting=traffic_setting_module.WIFI)
        story_set.AddStory(slow_page)
        story_set.AddStory(fast_page)

        latencies_by_page_in_ms = {}

        class MeasureLatency(legacy_page_test.LegacyPageTest):
            def __init__(self):
                super(MeasureLatency, self).__init__()
                self._will_navigate_time = None

            def WillNavigateToPage(self, page, tab):
                del page, tab  # unused
                self._will_navigate_time = time.time() * 1000

            def ValidateAndMeasurePage(self, page, tab, results):
                del results  # unused
                latencies_by_page_in_ms[page.name] = (time.time() * 1000 -
                                                      self._will_navigate_time)

        test = MeasureLatency()
        options = options_for_unittests.GetCopy()
        options.output_formats = ['none']
        options.suppress_gtest_report = True
        SetUpStoryRunnerArguments(options)
        results = results_options.CreateResults(EmptyMetadataForTest(),
                                                options)
        story_runner.Run(test,
                         story_set,
                         options,
                         results,
                         metadata=EmptyMetadataForTest())
        # Slow page should be slower than fast page by at least 300 ms (roundtrip
        # time of 2G) - 2 ms (roundtrip time of Wifi)
        self.assertGreater(latencies_by_page_in_ms['slow'],
                           latencies_by_page_in_ms['fast'] + 300 - 2)