def testMultipleTabsOkayForMultiTabTest(self): story_set = story.StorySet() page = page_module.Page('file://blank.html', story_set, base_dir=util.GetUnittestDataDir()) story_set.AddStory(page) class TestMultiTabs(legacy_page_test.LegacyPageTest): def TabForPage(self, page, browser): del page # unused return browser.tabs.New() def ValidateAndMeasurePage(self, page, tab, results): del page, results # unused assert len(tab.browser.tabs) == 2 test = TestMultiTabs() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run(test, story_set, options, results)
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: 1 if there is failure or 2 if there is an uncaught exception. """ start = time.time() benchmark.CustomizeBrowserOptions(finder_options.browser_options) benchmark_metadata = benchmark.GetMetadata() possible_browser = browser_finder.FindBrowser(finder_options) expectations = benchmark.expectations if not possible_browser: print( 'Cannot find browser of type %s. To list out all ' 'available browsers, rerun your command with ' '--browser=list' % finder_options.browser_options.browser_type) return 1 can_run_on_platform = benchmark._CanRunOnPlatform( possible_browser.platform, finder_options) expectations_disabled = expectations.IsBenchmarkDisabled( possible_browser.platform, finder_options) if expectations_disabled or not can_run_on_platform: print '%s is disabled on the selected browser' % benchmark.Name() if finder_options.run_disabled_tests and can_run_on_platform: print 'Running benchmark anyway due to: --also-run-disabled-tests' else: if can_run_on_platform: print 'Try --also-run-disabled-tests to force the benchmark to run.' else: print( "This platform is not supported for this benchmark. If this is " "in error please add it to the benchmark's supported platforms." ) # If chartjson is specified, this will print a dict indicating the # benchmark name and disabled state. with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=False) as results: results.PrintSummary() # When a disabled benchmark is run we now want to return success since # we are no longer filtering these out in the buildbot recipes. return 0 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ disabled_attr_name = decorators.DisabledAttributeName(benchmark) # pylint: disable=protected-access pt._disabled_strings = getattr(benchmark, disabled_attr_name, set()) if hasattr(benchmark, '_enabled_strings'): # pylint: disable=protected-access pt._enabled_strings = benchmark._enabled_strings stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=True) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, expectations=expectations, metadata=benchmark.GetMetadata(), max_num_values=benchmark.MAX_NUM_VALUES) return_code = 1 if results.had_failures else 0 # We want to make sure that all expectations are linked to real stories, # this will log error messages if names do not match what is in the set. benchmark.GetBrokenExpectations(stories) except Exception: # pylint: disable=broad-except logging.fatal( 'Benchmark execution interrupted by a fatal exception.') results.telemetry_info.InterruptBenchmark() exception_formatter.PrintFormattedException() return_code = 2 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() if benchmark_owners: results.AddSharedDiagnostic(reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.AddSharedDiagnostic(reserved_infos.BUG_COMPONENTS.name, benchmark_component) try: if finder_options.upload_results: results.UploadTraceFilesToCloud() results.UploadProfilingFilesToCloud() results.UploadArtifactsToCloud() finally: duration = time.time() - start results.AddSummaryValue( scalar.ScalarValue(None, 'benchmark_duration', 'minutes', duration / 60.0)) results.AddDurationHistogram(duration * 1000.0) memory_debug.LogHostMemoryUsage() results.PrintSummary() return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: -1 if the benchmark was skipped, 0 for success 1 if there was a failure 2 if there was an uncaught exception. """ benchmark.CustomizeOptions(finder_options) with results_options.CreateResults( finder_options, benchmark_name=benchmark.Name(), benchmark_description=benchmark.Description(), report_progress=not finder_options.suppress_gtest_report, should_add_value=benchmark.ShouldAddValue) as results: possible_browser = browser_finder.FindBrowser(finder_options) if not possible_browser: print ('No browser of type "%s" found for running benchmark "%s".' % ( finder_options.browser_options.browser_type, benchmark.Name())) return -1 benchmark.expectations.SetTags( possible_browser.GetTypExpectationsTags()) if not _ShouldRunBenchmark(benchmark, possible_browser, finder_options): return -1 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ story_set = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in story_set.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') try: Run(pt, story_set, finder_options, results, benchmark.max_failures, expectations=benchmark.expectations, max_num_values=benchmark.MAX_NUM_VALUES) if results.benchmark_interrupted: return_code = 2 elif results.had_failures: return_code = 1 elif results.had_successes: return_code = 0 else: return_code = -1 # All stories were skipped. # We want to make sure that all expectations are linked to real stories, # this will log error messages if names do not match what is in the set. benchmark.GetBrokenExpectations(story_set) except Exception as exc: # pylint: disable=broad-except interruption = 'Benchmark execution interrupted: %r' % exc results.InterruptBenchmark(interruption) exception_formatter.PrintFormattedException() return_code = 2 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() benchmark_documentation_url = benchmark.GetDocumentationLink() if benchmark_owners: results.AddSharedDiagnosticToAllHistograms( reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.AddSharedDiagnosticToAllHistograms( reserved_infos.BUG_COMPONENTS.name, benchmark_component) if benchmark_documentation_url: results.AddSharedDiagnosticToAllHistograms( reserved_infos.DOCUMENTATION_URLS.name, benchmark_documentation_url) if finder_options.upload_results: results_processor.UploadArtifactsToCloud(results) return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: An exit code from exit_codes module describing what happened. """ benchmark_name = benchmark.Name() if not re.match(_RE_VALID_TEST_SUITE_NAME, benchmark_name): logging.fatal('Invalid benchmark name: %s', benchmark_name) return 2 # exit_codes.FATAL_ERROR possible_browser = browser_finder.FindBrowser(finder_options) if not possible_browser: print('No browser of type "%s" found for running benchmark "%s".' % (finder_options.browser_options.browser_type, benchmark.Name())) return exit_codes.ALL_TESTS_SKIPPED benchmark.CustomizeOptions(finder_options, possible_browser) with results_options.CreateResults( finder_options, benchmark_name=benchmark_name, benchmark_description=benchmark.Description(), report_progress=not finder_options.suppress_gtest_report ) as results: if not _ShouldRunBenchmark(benchmark, possible_browser, finder_options): return exit_codes.ALL_TESTS_SKIPPED test = benchmark.CreatePageTest(finder_options) test.__name__ = benchmark.__class__.__name__ story_set = benchmark.CreateStorySet(finder_options) if isinstance(test, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in story_set.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') results.AddSharedDiagnostics( architecture=possible_browser.platform.GetArchName(), device_id=possible_browser.platform.GetDeviceId(), os_name=possible_browser.platform.GetOSName(), os_version=possible_browser.platform.GetOSVersionName(), owners=benchmark.GetOwners(), bug_components=benchmark.GetBugComponents(), documentation_urls=benchmark.GetDocumentationLinks(), info_blurb=benchmark.GetInfoBlurb(), ) try: RunStorySet(test, story_set, finder_options, results, benchmark.max_failures, possible_browser) if results.benchmark_interrupted: return_code = exit_codes.FATAL_ERROR elif results.had_failures: return_code = exit_codes.TEST_FAILURE elif results.had_successes: return_code = exit_codes.SUCCESS else: return_code = exit_codes.ALL_TESTS_SKIPPED except Exception as exc: # pylint: disable=broad-except interruption = 'Benchmark execution interrupted: %r' % exc results.InterruptBenchmark(interruption) exception_formatter.PrintFormattedException() return_code = exit_codes.FATAL_ERROR return return_code
def RunStories(self, stories, **kwargs): story_set = test_stories.DummyStorySet(stories) with results_options.CreateResults( self.options, benchmark_name='benchmark') as results: story_runner.RunStorySet(self.mock_story_test, story_set, self.options, results, **kwargs)
def CreateResults(self): benchmark_metadata = self._CreateBenchmarkMetadata() return results_options.CreateResults(benchmark_metadata, self._options)
def RunBenchmark(benchmark, story_set, possible_browser, browser_options, finder_options): """Run the benchmark on the given browser with given options. Args: benchmark: an instance of Benchmark class, a benchmark to be run. story_set: and instance of StorySet, a collection of stories. possible_browser: an instance of PossibleBrowser. browser_options: options to be passed to browser. finder_options: options controlling the execution of benchmark. This can be an instance of BrowserFinderOptions class, but only options relevant to benchmark execution will be read. Returns: 1 if there is failure or 2 if there is an uncaught exception. """ benchmark_metadata = benchmark.GetMetadata() page_test = benchmark.CreatePageTest(finder_options) page_test.__name__ = benchmark.__class__.__name__ if isinstance(page_test, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in story_set): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=True) as results: try: RunStorySet( page_test, story_set, possible_browser, benchmark.expectations, browser_options, finder_options, results, benchmark.max_failures, benchmark.MAX_NUM_VALUES) return_code = 1 if results.had_failures else 0 except Exception as e: # pylint: disable=broad-except logging.fatal( 'Benchmark execution interrupted by a fatal exception: %s(%s)' % (type(e), e)) results.InterruptBenchmark(story_set, finder_options.pageset_repeat) exception_formatter.PrintFormattedException() return_code = 2 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() benchmark_documentation_url = benchmark.GetDocumentationLink() if benchmark_owners: results.AddSharedDiagnosticToAllHistograms( reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.AddSharedDiagnosticToAllHistograms( reserved_infos.BUG_COMPONENTS.name, benchmark_component) if benchmark_documentation_url: results.AddSharedDiagnosticToAllHistograms( reserved_infos.DOCUMENTATION_URLS.name, benchmark_documentation_url) try: if finder_options.upload_results: results.UploadTraceFilesToCloud() results.UploadArtifactsToCloud() finally: memory_debug.LogHostMemoryUsage() results.PrintSummary() return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: The number of failure values (up to 254) or 255 if there is an uncaught exception. """ benchmark.CustomizeBrowserOptions(finder_options.browser_options) possible_browser = browser_finder.FindBrowser(finder_options) if possible_browser and benchmark.ShouldDisable(possible_browser): logging.warning('%s is disabled on the selected browser', benchmark.Name()) if finder_options.run_disabled_tests: logging.warning( 'Running benchmark anyway due to: --also-run-disabled-tests') else: logging.warning( 'Try --also-run-disabled-tests to force the benchmark to run.') return 1 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ disabled_attr_name = decorators.DisabledAttributeName(benchmark) # pylint: disable=protected-access pt._disabled_strings = getattr(benchmark, disabled_attr_name, set()) if hasattr(benchmark, '_enabled_strings'): # pylint: disable=protected-access pt._enabled_strings = benchmark._enabled_strings stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') should_tear_down_state_after_each_story_run = ( benchmark.ShouldTearDownStateAfterEachStoryRun()) # HACK: restarting shared state has huge overhead on cros (crbug.com/645329), # hence we default this to False when test is run against CrOS. # TODO(cros-team): figure out ways to remove this hack. if (possible_browser.platform.GetOSName() == 'chromeos' and not benchmark.IsShouldTearDownStateAfterEachStoryRunOverriden()): should_tear_down_state_after_each_story_run = False benchmark_metadata = benchmark.GetMetadata() with results_options.CreateResults( benchmark_metadata, finder_options, benchmark.ValueCanBeAddedPredicate) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, should_tear_down_state_after_each_story_run, benchmark.ShouldTearDownStateAfterEachStorySetRun()) return_code = min(254, len(results.failures)) except Exception: exception_formatter.PrintFormattedException() return_code = 255 try: if finder_options.upload_results: bucket = finder_options.upload_bucket if bucket in cloud_storage.BUCKET_ALIASES: bucket = cloud_storage.BUCKET_ALIASES[bucket] results.UploadTraceFilesToCloud(bucket) results.UploadProfilingFilesToCloud(bucket) finally: results.PrintSummary() return return_code
def _testMaxFailuresOptionIsRespectedAndOverridable( self, num_failing_stories, runner_max_failures, options_max_failures, expected_num_failures): class SimpleSharedState(story_module.SharedState): _fake_platform = FakePlatform() _current_story = None @property def platform(self): return self._fake_platform def WillRunStory(self, story): self._current_story = story def RunStory(self, results): self._current_story.Run(self) def DidRunStory(self, results): pass def CanRunStory(self, story): return True def TearDownState(self): pass def DumpStateUponFailure(self, story, results): pass class FailingStory(story_module.Story): def __init__(self, name): super(FailingStory, self).__init__(shared_state_class=SimpleSharedState, is_local=True, name=name) self.was_run = False def Run(self, shared_state): self.was_run = True raise legacy_page_test.Failure @property def url(self): return 'data:,' self.SuppressExceptionFormatting() story_set = story_module.StorySet() for i in range(num_failing_stories): story_set.AddStory(FailingStory(name='failing%d' % i)) options = _GetOptionForUnittest() options.output_formats = ['none'] options.suppress_gtest_report = True if options_max_failures: options.max_failures = options_max_failures results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run(DummyTest(), story_set, options, results, max_failures=runner_max_failures) self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(expected_num_failures, len(results.failures)) for ii, story in enumerate(story_set.stories): self.assertEqual(story.was_run, ii < expected_num_failures)
def testPageResetWhenBrowserReusedBetweenStories(self): class NoClosingBrowserSharedState(shared_page_state.SharedPageState): # Simulate what ChromeOS does. def ShouldStopBrowserAfterStoryRun(self, s): del s # unused return False # Loads a page and scrolls it to the end. class ScrollingPage(page_module.Page): def __init__(self, url, page_set, base_dir): super(ScrollingPage, self).__init__( page_set=page_set, base_dir=base_dir, shared_page_state_class=NoClosingBrowserSharedState, url=url, name='ScrollingPage') def RunPageInteractions(self, action_runner): action_runner.ScrollPage() # Loads same page as ScrollingPage() and records if the scroll position is # at the top of the page (in was_page_at_top_on_start). class CheckScrollPositionPage(page_module.Page): def __init__(self, url, page_set, base_dir): super(CheckScrollPositionPage, self).__init__( page_set=page_set, base_dir=base_dir, shared_page_state_class=NoClosingBrowserSharedState, url=url, name='CheckScroll') self.was_page_at_top_on_start = False def RunPageInteractions(self, action_runner): scroll_y = action_runner.tab.EvaluateJavaScript( 'window.scrollY') self.was_page_at_top_on_start = scroll_y == 0 class Test(legacy_page_test.LegacyPageTest): def ValidateAndMeasurePage(self, *_): pass story_set = story.StorySet() story_set.AddStory( ScrollingPage(url='file://page_with_swipeables.html', page_set=story_set, base_dir=util.GetUnittestDataDir())) test_page = CheckScrollPositionPage( url='file://page_with_swipeables.html', page_set=story_set, base_dir=util.GetUnittestDataDir()) story_set.AddStory(test_page) test = Test() options = options_for_unittests.GetCopy() SetUpStoryRunnerArguments(options) # Override defaults from parser creation and arg processing. options.output_formats = ['none'] options.suppress_gtest_report = True options.output_dir = None results = results_options.CreateResults(options) story_runner.Run(test, story_set, options, results) self.assertTrue(test_page.was_page_at_top_on_start)
def RunStorySet(test, story_set, options, **kwargs): with results_options.CreateResults(options) as results: story_runner.RunStorySet(test, story_set, options, results, **kwargs) return results
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: -1 if the benchmark was skipped, 0 for success 1 if there was a failure 2 if there was an uncaught exception. """ benchmark.CustomizeOptions(finder_options) possible_browser = browser_finder.FindBrowser(finder_options) if not _ShouldRunBenchmark(benchmark, possible_browser, finder_options): return -1 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') with results_options.CreateResults( finder_options, benchmark_name=benchmark.Name(), benchmark_description=benchmark.Description(), benchmark_enabled=True, should_add_value=benchmark.ShouldAddValue) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, expectations=benchmark.expectations, max_num_values=benchmark.MAX_NUM_VALUES) if results.had_failures: return_code = 1 elif results.had_successes_not_skipped: return_code = 0 else: return_code = -1 # All stories were skipped. # We want to make sure that all expectations are linked to real stories, # this will log error messages if names do not match what is in the set. benchmark.GetBrokenExpectations(stories) except Exception as e: # pylint: disable=broad-except logging.fatal( 'Benchmark execution interrupted by a fatal exception: %s(%s)' % (type(e), e)) filtered_stories = story_module.StoryFilter.FilterStorySet(stories) results.InterruptBenchmark(filtered_stories, finder_options.pageset_repeat) exception_formatter.PrintFormattedException() return_code = 2 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() benchmark_documentation_url = benchmark.GetDocumentationLink() if benchmark_owners: results.AddSharedDiagnosticToAllHistograms( reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.AddSharedDiagnosticToAllHistograms( reserved_infos.BUG_COMPONENTS.name, benchmark_component) if benchmark_documentation_url: results.AddSharedDiagnosticToAllHistograms( reserved_infos.DOCUMENTATION_URLS.name, benchmark_documentation_url) try: if finder_options.upload_results: results.UploadTraceFilesToCloud() results.UploadArtifactsToCloud() finally: memory_debug.LogHostMemoryUsage() results.PrintSummary() return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: The number of failure values (up to 254) or 255 if there is an uncaught exception. """ start = time.time() benchmark.CustomizeBrowserOptions(finder_options.browser_options) benchmark_metadata = benchmark.GetMetadata() possible_browser = browser_finder.FindBrowser(finder_options) expectations = benchmark.InitializeExpectations() if not possible_browser: print ('Cannot find browser of type %s. To list out all ' 'available browsers, rerun your command with ' '--browser=list' % finder_options.browser_options.browser_type) return 1 can_run_on_platform = benchmark._CanRunOnPlatform(possible_browser.platform, finder_options) # TODO(rnephew): Remove decorators.IsBenchmarkEnabled and IsBenchmarkDisabled # when we have fully moved to _CanRunOnPlatform(). permanently_disabled = expectations.IsBenchmarkDisabled( possible_browser.platform, finder_options) temporarily_disabled = not decorators.IsBenchmarkEnabled( benchmark, possible_browser) if permanently_disabled or temporarily_disabled or not can_run_on_platform: print '%s is disabled on the selected browser' % benchmark.Name() if finder_options.run_disabled_tests and not permanently_disabled: print 'Running benchmark anyway due to: --also-run-disabled-tests' else: print 'Try --also-run-disabled-tests to force the benchmark to run.' # If chartjson is specified, this will print a dict indicating the # benchmark name and disabled state. with results_options.CreateResults( benchmark_metadata, finder_options, benchmark.ValueCanBeAddedPredicate, benchmark_enabled=False ) as results: results.PrintSummary() # When a disabled benchmark is run we now want to return success since # we are no longer filtering these out in the buildbot recipes. return 0 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ disabled_attr_name = decorators.DisabledAttributeName(benchmark) # pylint: disable=protected-access pt._disabled_strings = getattr(benchmark, disabled_attr_name, set()) if hasattr(benchmark, '_enabled_strings'): # pylint: disable=protected-access pt._enabled_strings = benchmark._enabled_strings stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') with results_options.CreateResults( benchmark_metadata, finder_options, benchmark.ValueCanBeAddedPredicate, benchmark_enabled=True) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, expectations=expectations, metadata=benchmark.GetMetadata()) return_code = min(254, len(results.failures)) # We want to make sure that all expectations are linked to real stories, # this will log error messages if names do not match what is in the set. benchmark.GetBrokenExpectations(stories) except Exception: # pylint: disable=broad-except results.telemetry_info.InterruptBenchmark() exception_formatter.PrintFormattedException() return_code = 255 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() if benchmark_owners: results.histograms.AddSharedDiagnostic( reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.histograms.AddSharedDiagnostic( reserved_infos.BUG_COMPONENTS.name, benchmark_component) try: if finder_options.upload_results: results.UploadTraceFilesToCloud() results.UploadProfilingFilesToCloud() finally: duration = time.time() - start results.AddSummaryValue(scalar.ScalarValue( None, 'benchmark_duration', 'minutes', duration / 60.0)) results.PrintSummary() return return_code
def _testMaxFailuresOptionIsRespectedAndOverridable( self, num_failing_user_stories, runner_max_failures, options_max_failures, expected_num_failures): class SimpleSharedState(shared_state.SharedState): _fake_platform = FakePlatform() _current_user_story = None @property def platform(self): return self._fake_platform def WillRunUserStory(self, user_story): self._current_user_story = user_story def RunUserStory(self, results): self._current_user_story.Run() def DidRunUserStory(self, results): pass def GetTestExpectationAndSkipValue(self, expectations): return 'pass', None def TearDownState(self): pass class FailingUserStory(user_story_module.UserStory): def __init__(self): super(FailingUserStory, self).__init__(shared_state_class=SimpleSharedState, is_local=True) self.was_run = False def Run(self): self.was_run = True raise page_test.Failure self.SuppressExceptionFormatting() story_set = story.StorySet() for _ in range(num_failing_user_stories): story_set.AddUserStory(FailingUserStory()) options = _GetOptionForUnittest() options.output_formats = ['none'] options.suppress_gtest_report = True if options_max_failures: options.max_failures = options_max_failures results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run(DummyTest(), story_set, test_expectations.TestExpectations(), options, results, max_failures=runner_max_failures) self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(expected_num_failures, len(results.failures)) for ii, user_story in enumerate(story_set.user_stories): self.assertEqual(user_story.was_run, ii < expected_num_failures)
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: The number of failure values (up to 254) or 255 if there is an uncaught exception. """ start = time.time() benchmark.CustomizeBrowserOptions(finder_options.browser_options) benchmark_metadata = benchmark.GetMetadata() possible_browser = browser_finder.FindBrowser(finder_options) if not possible_browser: print( 'Cannot find browser of type %s. To list out all ' 'available browsers, rerun your command with ' '--browser=list' % finder_options.browser_options.browser_type) return 1 if (possible_browser and not decorators.IsBenchmarkEnabled(benchmark, possible_browser)): print '%s is disabled on the selected browser' % benchmark.Name() if finder_options.run_disabled_tests: print 'Running benchmark anyway due to: --also-run-disabled-tests' else: print 'Try --also-run-disabled-tests to force the benchmark to run.' # If chartjson is specified, this will print a dict indicating the # benchmark name and disabled state. with results_options.CreateResults( benchmark_metadata, finder_options, benchmark.ValueCanBeAddedPredicate, benchmark_enabled=False) as results: results.PrintSummary() # When a disabled benchmark is run we now want to return success since # we are no longer filtering these out in the buildbot recipes. return 0 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ disabled_attr_name = decorators.DisabledAttributeName(benchmark) # pylint: disable=protected-access pt._disabled_strings = getattr(benchmark, disabled_attr_name, set()) if hasattr(benchmark, '_enabled_strings'): # pylint: disable=protected-access pt._enabled_strings = benchmark._enabled_strings stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') should_tear_down_state_after_each_story_run = ( benchmark.ShouldTearDownStateAfterEachStoryRun()) # HACK: restarting domain.shared state has huge overhead on cros (crbug.com/645329), # hence we default this to False when test is run against CrOS. # TODO(cros-team): figure out ways to remove this hack. if (possible_browser.platform.GetOSName() == 'chromeos' and not benchmark.IsShouldTearDownStateAfterEachStoryRunOverriden()): should_tear_down_state_after_each_story_run = False with results_options.CreateResults(benchmark_metadata, finder_options, benchmark.ValueCanBeAddedPredicate, benchmark_enabled=True) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, should_tear_down_state_after_each_story_run, benchmark.ShouldTearDownStateAfterEachStorySetRun()) return_code = min(254, len(results.failures)) except Exception: exception_formatter.PrintFormattedException() return_code = 255 try: if finder_options.upload_results: bucket = finder_options.upload_bucket if bucket in cloud_storage.BUCKET_ALIASES: bucket = cloud_storage.BUCKET_ALIASES[bucket] results.UploadTraceFilesToCloud(bucket) results.UploadProfilingFilesToCloud(bucket) finally: duration = time.time() - start results.AddSummaryValue( scalar.ScalarValue(None, 'BenchmarkDuration', 'minutes', duration / 60.0)) results.PrintSummary() return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: 1 if there is failure or 2 if there is an uncaught exception. """ benchmark.CustomizeBrowserOptions(finder_options.browser_options) benchmark_metadata = benchmark.GetMetadata() possible_browser = browser_finder.FindBrowser(finder_options) expectations = benchmark.expectations target_platform = None if possible_browser: target_platform = possible_browser.platform else: target_platform = platform_module.GetHostPlatform() can_run_on_platform = benchmark._CanRunOnPlatform(target_platform, finder_options) expectations_disabled = False # For now, test expectations are only applicable in the cases where the # testing target involves a browser. if possible_browser: expectations_disabled = expectations.IsBenchmarkDisabled( possible_browser.platform, finder_options) if expectations_disabled or not can_run_on_platform: print '%s is disabled on the selected browser' % benchmark.Name() if finder_options.run_disabled_tests and can_run_on_platform: print 'Running benchmark anyway due to: --also-run-disabled-tests' else: if can_run_on_platform: print 'Try --also-run-disabled-tests to force the benchmark to run.' else: print( "This platform is not supported for this benchmark. If this is " "in error please add it to the benchmark's supported platforms." ) # If chartjson is specified, this will print a dict indicating the # benchmark name and disabled state. with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=False) as results: results.PrintSummary() # When a disabled benchmark is run we now want to return success since # we are no longer filtering these out in the buildbot recipes. return 0 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=True) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, expectations=expectations, max_num_values=benchmark.MAX_NUM_VALUES) return_code = 1 if results.had_failures else 0 # We want to make sure that all expectations are linked to real stories, # this will log error messages if names do not match what is in the set. benchmark.GetBrokenExpectations(stories) except Exception: # pylint: disable=broad-except logging.fatal( 'Benchmark execution interrupted by a fatal exception.') filtered_stories = story_module.StoryFilter.FilterStorySet(stories) results.InterruptBenchmark(filtered_stories, _GetPageSetRepeat(finder_options)) exception_formatter.PrintFormattedException() return_code = 2 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() benchmark_documentation_url = benchmark.GetDocumentationLink() if benchmark_owners: results.AddSharedDiagnostic(reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.AddSharedDiagnostic(reserved_infos.BUG_COMPONENTS.name, benchmark_component) if benchmark_documentation_url: results.AddSharedDiagnostic(reserved_infos.DOCUMENTATION_URLS.name, benchmark_documentation_url) try: if finder_options.upload_results: results.UploadTraceFilesToCloud() results.UploadArtifactsToCloud() finally: memory_debug.LogHostMemoryUsage() results.PrintSummary() return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: -1 if the benchmark was skipped, 0 for success 1 if there was a failure 2 if there was an uncaught exception. """ benchmark.CustomizeOptions(finder_options) with results_options.CreateResults( finder_options, benchmark_name=benchmark.Name(), benchmark_description=benchmark.Description(), report_progress=not finder_options.suppress_gtest_report ) as results: possible_browser = browser_finder.FindBrowser(finder_options) if not possible_browser: print('No browser of type "%s" found for running benchmark "%s".' % (finder_options.browser_options.browser_type, benchmark.Name())) return -1 if not _ShouldRunBenchmark(benchmark, possible_browser, finder_options): return -1 test = benchmark.CreatePageTest(finder_options) test.__name__ = benchmark.__class__.__name__ story_set = benchmark.CreateStorySet(finder_options) if isinstance(test, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in story_set.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') results.AddSharedDiagnostics( architecture=possible_browser.platform.GetArchName(), device_id=possible_browser.platform.GetDeviceId(), os_name=possible_browser.platform.GetOSName(), os_version=possible_browser.platform.GetOSVersionName(), owners=benchmark.GetOwners(), bug_components=benchmark.GetBugComponents(), documentation_urls=benchmark.GetDocumentationLinks(), ) try: RunStorySet(test, story_set, finder_options, results, benchmark.max_failures) if results.benchmark_interrupted: return_code = 2 elif results.had_failures: return_code = 1 elif results.had_successes: return_code = 0 else: return_code = -1 # All stories were skipped. except Exception as exc: # pylint: disable=broad-except interruption = 'Benchmark execution interrupted: %r' % exc results.InterruptBenchmark(interruption) exception_formatter.PrintFormattedException() return_code = 2 return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: -1 if the benchmark was skipped, 0 for success 1 if there was a failure 2 if there was an uncaught exception. """ benchmark.CustomizeOptions(finder_options) with results_options.CreateResults( finder_options, benchmark_name=benchmark.Name(), benchmark_description=benchmark.Description(), report_progress=not finder_options.suppress_gtest_report ) as results: possible_browser = browser_finder.FindBrowser(finder_options) if not possible_browser: print('No browser of type "%s" found for running benchmark "%s".' % (finder_options.browser_options.browser_type, benchmark.Name())) return -1 typ_expectation_tags = possible_browser.GetTypExpectationsTags() logging.info( 'The following expectations condition tags were generated %s', str(typ_expectation_tags)) try: benchmark.expectations.SetTags( typ_expectation_tags, not finder_options.skip_typ_expectations_tags_validation) except ValueError as e: # pylint: disable=broad-except traceback.print_exc(file=sys.stdout) logging.error( str(e) + '\nYou can use the --skip-typ-expectations-tags-validation ' 'argument to suppress this exception.') return -1 if not _ShouldRunBenchmark(benchmark, possible_browser, finder_options): return -1 test = benchmark.CreatePageTest(finder_options) test.__name__ = benchmark.__class__.__name__ story_set = benchmark.CreateStorySet(finder_options) if isinstance(test, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in story_set.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') try: RunStorySet(test, story_set, finder_options, results, benchmark.max_failures, expectations=benchmark.expectations, max_num_values=benchmark.MAX_NUM_VALUES) if results.benchmark_interrupted: return_code = 2 elif results.had_failures: return_code = 1 elif results.had_successes: return_code = 0 else: return_code = -1 # All stories were skipped. except Exception as exc: # pylint: disable=broad-except interruption = 'Benchmark execution interrupted: %r' % exc results.InterruptBenchmark(interruption) exception_formatter.PrintFormattedException() return_code = 2 # TODO(crbug.com/981349): merge two calls to AddSharedDiagnostics # (see RunStorySet() method for the second one). results.AddSharedDiagnostics( owners=benchmark.GetOwners(), bug_components=benchmark.GetBugComponents(), documentation_urls=benchmark.GetDocumentationLinks(), ) if finder_options.upload_results: results_processor.UploadArtifactsToCloud(results) return return_code
def testTrafficSettings(self): story_set = story.StorySet() slow_page = page_module.Page( 'file://green_rect.html', story_set, base_dir=util.GetUnittestDataDir(), name='slow', traffic_setting=traffic_setting_module.GOOD_3G) fast_page = page_module.Page( 'file://green_rect.html', story_set, base_dir=util.GetUnittestDataDir(), name='fast', traffic_setting=traffic_setting_module.WIFI) story_set.AddStory(slow_page) story_set.AddStory(fast_page) latencies_by_page_in_ms = {} class MeasureLatency(legacy_page_test.LegacyPageTest): def __init__(self): super(MeasureLatency, self).__init__() self._will_navigate_time = None def WillNavigateToPage(self, page, tab): del page, tab # unused self._will_navigate_time = time.time() * 1000 def ValidateAndMeasurePage(self, page, tab, results): del results # unused latencies_by_page_in_ms[page.name] = (time.time() * 1000 - self._will_navigate_time) test = MeasureLatency() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir: options.output_dir = tempdir SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run(test, story_set, options, results, metadata=EmptyMetadataForTest()) failure_messages = [] for r in results.all_page_runs: if r.failure_str: failure_messages.append( 'Failure message of story %s:\n%s' % (r.story, r.failure_str)) self.assertFalse(results.had_failures, msg=''.join(failure_messages)) self.assertIn('slow', latencies_by_page_in_ms) self.assertIn('fast', latencies_by_page_in_ms) # Slow page should be slower than fast page by at least 40 ms (roundtrip # time of good 3G) - 2 ms (roundtrip time of Wifi) self.assertGreater(latencies_by_page_in_ms['slow'], latencies_by_page_in_ms['fast'] + 40 - 2)