def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: 1 if there is failure or 2 if there is an uncaught exception. """ benchmark.CustomizeBrowserOptions(finder_options.browser_options) benchmark_metadata = benchmark.GetMetadata() possible_browser = browser_finder.FindBrowser(finder_options) expectations = benchmark.expectations target_platform = None if possible_browser: target_platform = possible_browser.platform else: target_platform = platform_module.GetHostPlatform() can_run_on_platform = benchmark._CanRunOnPlatform(target_platform, finder_options) expectations_disabled = False # For now, test expectations are only applicable in the cases where the # testing target involves a browser. if possible_browser: expectations_disabled = expectations.IsBenchmarkDisabled( possible_browser.platform, finder_options) if expectations_disabled or not can_run_on_platform: print '%s is disabled on the selected browser' % benchmark.Name() if finder_options.run_disabled_tests and can_run_on_platform: print 'Running benchmark anyway due to: --also-run-disabled-tests' else: if can_run_on_platform: print 'Try --also-run-disabled-tests to force the benchmark to run.' else: print( "This platform is not supported for this benchmark. If this is " "in error please add it to the benchmark's supported platforms." ) # If chartjson is specified, this will print a dict indicating the # benchmark name and disabled state. with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=False) as results: results.PrintSummary() # When a disabled benchmark is run we now want to return success since # we are no longer filtering these out in the buildbot recipes. return 0 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=True) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, expectations=expectations, max_num_values=benchmark.MAX_NUM_VALUES) return_code = 1 if results.had_failures else 0 # We want to make sure that all expectations are linked to real stories, # this will log error messages if names do not match what is in the set. benchmark.GetBrokenExpectations(stories) except Exception: # pylint: disable=broad-except logging.fatal( 'Benchmark execution interrupted by a fatal exception.') filtered_stories = story_module.StoryFilter.FilterStorySet(stories) results.InterruptBenchmark(filtered_stories, _GetPageSetRepeat(finder_options)) exception_formatter.PrintFormattedException() return_code = 2 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() benchmark_documentation_url = benchmark.GetDocumentationLink() if benchmark_owners: results.AddSharedDiagnostic(reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.AddSharedDiagnostic(reserved_infos.BUG_COMPONENTS.name, benchmark_component) if benchmark_documentation_url: results.AddSharedDiagnostic(reserved_infos.DOCUMENTATION_URLS.name, benchmark_documentation_url) try: if finder_options.upload_results: results.UploadTraceFilesToCloud() results.UploadArtifactsToCloud() finally: memory_debug.LogHostMemoryUsage() results.PrintSummary() return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: -1 if the benchmark was skipped, 0 for success 1 if there was a failure 2 if there was an uncaught exception. """ benchmark.CustomizeOptions(finder_options) possible_browser = browser_finder.FindBrowser(finder_options) if not _ShouldRunBenchmark(benchmark, possible_browser, finder_options): return -1 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') with results_options.CreateResults( finder_options, benchmark_name=benchmark.Name(), benchmark_description=benchmark.Description(), benchmark_enabled=True, should_add_value=benchmark.ShouldAddValue) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, expectations=benchmark.expectations, max_num_values=benchmark.MAX_NUM_VALUES) if results.had_failures: return_code = 1 elif results.had_successes_not_skipped: return_code = 0 else: return_code = -1 # All stories were skipped. # We want to make sure that all expectations are linked to real stories, # this will log error messages if names do not match what is in the set. benchmark.GetBrokenExpectations(stories) except Exception as e: # pylint: disable=broad-except logging.fatal( 'Benchmark execution interrupted by a fatal exception: %s(%s)' % (type(e), e)) filtered_stories = story_module.StoryFilter.FilterStorySet(stories) results.InterruptBenchmark(filtered_stories, finder_options.pageset_repeat) exception_formatter.PrintFormattedException() return_code = 2 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() benchmark_documentation_url = benchmark.GetDocumentationLink() if benchmark_owners: results.AddSharedDiagnosticToAllHistograms( reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.AddSharedDiagnosticToAllHistograms( reserved_infos.BUG_COMPONENTS.name, benchmark_component) if benchmark_documentation_url: results.AddSharedDiagnosticToAllHistograms( reserved_infos.DOCUMENTATION_URLS.name, benchmark_documentation_url) try: if finder_options.upload_results: results.UploadTraceFilesToCloud() results.UploadArtifactsToCloud() finally: memory_debug.LogHostMemoryUsage() results.PrintSummary() return return_code
def RunBenchmark(benchmark, finder_options): """Run this test with the given options. Returns: 1 if there is failure or 2 if there is an uncaught exception. """ start = time.time() benchmark.CustomizeBrowserOptions(finder_options.browser_options) benchmark_metadata = benchmark.GetMetadata() possible_browser = browser_finder.FindBrowser(finder_options) expectations = benchmark.expectations if not possible_browser: print( 'Cannot find browser of type %s. To list out all ' 'available browsers, rerun your command with ' '--browser=list' % finder_options.browser_options.browser_type) return 1 can_run_on_platform = benchmark._CanRunOnPlatform( possible_browser.platform, finder_options) expectations_disabled = expectations.IsBenchmarkDisabled( possible_browser.platform, finder_options) if expectations_disabled or not can_run_on_platform: print '%s is disabled on the selected browser' % benchmark.Name() if finder_options.run_disabled_tests and can_run_on_platform: print 'Running benchmark anyway due to: --also-run-disabled-tests' else: if can_run_on_platform: print 'Try --also-run-disabled-tests to force the benchmark to run.' else: print( "This platform is not supported for this benchmark. If this is " "in error please add it to the benchmark's supported platforms." ) # If chartjson is specified, this will print a dict indicating the # benchmark name and disabled state. with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=False) as results: results.PrintSummary() # When a disabled benchmark is run we now want to return success since # we are no longer filtering these out in the buildbot recipes. return 0 pt = benchmark.CreatePageTest(finder_options) pt.__name__ = benchmark.__class__.__name__ disabled_attr_name = decorators.DisabledAttributeName(benchmark) # pylint: disable=protected-access pt._disabled_strings = getattr(benchmark, disabled_attr_name, set()) if hasattr(benchmark, '_enabled_strings'): # pylint: disable=protected-access pt._enabled_strings = benchmark._enabled_strings stories = benchmark.CreateStorySet(finder_options) if isinstance(pt, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in stories.stories): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=True) as results: try: Run(pt, stories, finder_options, results, benchmark.max_failures, expectations=expectations, metadata=benchmark.GetMetadata(), max_num_values=benchmark.MAX_NUM_VALUES) return_code = 1 if results.had_failures else 0 # We want to make sure that all expectations are linked to real stories, # this will log error messages if names do not match what is in the set. benchmark.GetBrokenExpectations(stories) except Exception: # pylint: disable=broad-except logging.fatal( 'Benchmark execution interrupted by a fatal exception.') results.telemetry_info.InterruptBenchmark() exception_formatter.PrintFormattedException() return_code = 2 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() if benchmark_owners: results.AddSharedDiagnostic(reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.AddSharedDiagnostic(reserved_infos.BUG_COMPONENTS.name, benchmark_component) try: if finder_options.upload_results: results.UploadTraceFilesToCloud() results.UploadProfilingFilesToCloud() results.UploadArtifactsToCloud() finally: duration = time.time() - start results.AddSummaryValue( scalar.ScalarValue(None, 'benchmark_duration', 'minutes', duration / 60.0)) results.AddDurationHistogram(duration * 1000.0) memory_debug.LogHostMemoryUsage() results.PrintSummary() return return_code
def RunBenchmark(benchmark, story_set, possible_browser, browser_options, finder_options): """Run the benchmark on the given browser with given options. Args: benchmark: an instance of Benchmark class, a benchmark to be run. story_set: and instance of StorySet, a collection of stories. possible_browser: an instance of PossibleBrowser. browser_options: options to be passed to browser. finder_options: options controlling the execution of benchmark. This can be an instance of BrowserFinderOptions class, but only options relevant to benchmark execution will be read. Returns: 1 if there is failure or 2 if there is an uncaught exception. """ benchmark_metadata = benchmark.GetMetadata() page_test = benchmark.CreatePageTest(finder_options) page_test.__name__ = benchmark.__class__.__name__ if isinstance(page_test, legacy_page_test.LegacyPageTest): if any(not isinstance(p, page.Page) for p in story_set): raise Exception( 'PageTest must be used with StorySet containing only ' 'telemetry.page.Page stories.') with results_options.CreateResults( benchmark_metadata, finder_options, should_add_value=benchmark.ShouldAddValue, benchmark_enabled=True) as results: try: RunStorySet( page_test, story_set, possible_browser, benchmark.expectations, browser_options, finder_options, results, benchmark.max_failures, benchmark.MAX_NUM_VALUES) return_code = 1 if results.had_failures else 0 except Exception as e: # pylint: disable=broad-except logging.fatal( 'Benchmark execution interrupted by a fatal exception: %s(%s)' % (type(e), e)) results.InterruptBenchmark(story_set, finder_options.pageset_repeat) exception_formatter.PrintFormattedException() return_code = 2 benchmark_owners = benchmark.GetOwners() benchmark_component = benchmark.GetBugComponents() benchmark_documentation_url = benchmark.GetDocumentationLink() if benchmark_owners: results.AddSharedDiagnosticToAllHistograms( reserved_infos.OWNERS.name, benchmark_owners) if benchmark_component: results.AddSharedDiagnosticToAllHistograms( reserved_infos.BUG_COMPONENTS.name, benchmark_component) if benchmark_documentation_url: results.AddSharedDiagnosticToAllHistograms( reserved_infos.DOCUMENTATION_URLS.name, benchmark_documentation_url) try: if finder_options.upload_results: results.UploadTraceFilesToCloud() results.UploadArtifactsToCloud() finally: memory_debug.LogHostMemoryUsage() results.PrintSummary() return return_code