def _IsBenchmarkEnabled(benchmark_class, possible_browser):
return (issubclass(benchmark_class, benchmark.Benchmark) and
decorators.IsBenchmarkEnabled(benchmark_class, possible_browser))
def RunBenchmark(benchmark, finder_options):
"""Run this test with the given options.
Returns:
The number of failure values (up to 254) or 255 if there is an uncaught
exception.
"""
start = time.time()
benchmark.CustomizeBrowserOptions(finder_options.browser_options)
benchmark_metadata = benchmark.GetMetadata()
possible_browser = browser_finder.FindBrowser(finder_options)
if not possible_browser:
print(
'Cannot find browser of type %s. To list out all '
'available browsers, rerun your command with '
'--browser=list' % finder_options.browser_options.browser_type)
return 1
if (possible_browser and
not decorators.IsBenchmarkEnabled(benchmark, possible_browser)):
print '%s is disabled on the selected browser' % benchmark.Name()
if finder_options.run_disabled_tests:
print 'Running benchmark anyway due to: --also-run-disabled-tests'
else:
print 'Try --also-run-disabled-tests to force the benchmark to run.'
# If chartjson is specified, this will print a dict indicating the
# benchmark name and disabled state.
with results_options.CreateResults(
benchmark_metadata,
finder_options,
benchmark.ValueCanBeAddedPredicate,
benchmark_enabled=False) as results:
results.PrintSummary()
# When a disabled benchmark is run we now want to return success since
# we are no longer filtering these out in the buildbot recipes.
return 0
pt = benchmark.CreatePageTest(finder_options)
pt.__name__ = benchmark.__class__.__name__
disabled_attr_name = decorators.DisabledAttributeName(benchmark)
# pylint: disable=protected-access
pt._disabled_strings = getattr(benchmark, disabled_attr_name, set())
if hasattr(benchmark, '_enabled_strings'):
# pylint: disable=protected-access
pt._enabled_strings = benchmark._enabled_strings
stories = benchmark.CreateStorySet(finder_options)
if isinstance(pt, legacy_page_test.LegacyPageTest):
if any(not isinstance(p, page.Page) for p in stories.stories):
raise Exception(
'PageTest must be used with StorySet containing only '
'telemetry.page.Page stories.')
should_tear_down_state_after_each_story_run = (
benchmark.ShouldTearDownStateAfterEachStoryRun())
# HACK: restarting domain.shared state has huge overhead on cros (crbug.com/645329),
# hence we default this to False when test is run against CrOS.
# TODO(cros-team): figure out ways to remove this hack.
if (possible_browser.platform.GetOSName() == 'chromeos' and
not benchmark.IsShouldTearDownStateAfterEachStoryRunOverriden()):
should_tear_down_state_after_each_story_run = False
with results_options.CreateResults(benchmark_metadata,
finder_options,
benchmark.ValueCanBeAddedPredicate,
benchmark_enabled=True) as results:
try:
Run(pt, stories, finder_options, results, benchmark.max_failures,
should_tear_down_state_after_each_story_run,
benchmark.ShouldTearDownStateAfterEachStorySetRun())
return_code = min(254, len(results.failures))
except Exception:
exception_formatter.PrintFormattedException()
return_code = 255
try:
if finder_options.upload_results:
bucket = finder_options.upload_bucket
if bucket in cloud_storage.BUCKET_ALIASES:
bucket = cloud_storage.BUCKET_ALIASES[bucket]
results.UploadTraceFilesToCloud(bucket)
results.UploadProfilingFilesToCloud(bucket)
finally:
duration = time.time() - start
results.AddSummaryValue(
scalar.ScalarValue(None, 'BenchmarkDuration', 'minutes',
duration / 60.0))
results.PrintSummary()
return return_code
def RunBenchmark(benchmark, finder_options):
"""Run this test with the given options.
Returns:
The number of failure values (up to 254) or 255 if there is an uncaught
exception.
"""
start = time.time()
benchmark.CustomizeBrowserOptions(finder_options.browser_options)
benchmark_metadata = benchmark.GetMetadata()
possible_browser = browser_finder.FindBrowser(finder_options)
expectations = benchmark.InitializeExpectations()
if not possible_browser:
print ('Cannot find browser of type %s. To list out all '
'available browsers, rerun your command with '
'--browser=list' % finder_options.browser_options.browser_type)
return 1
can_run_on_platform = benchmark._CanRunOnPlatform(possible_browser.platform,
finder_options)
# TODO(rnephew): Remove decorators.IsBenchmarkEnabled and IsBenchmarkDisabled
# when we have fully moved to _CanRunOnPlatform().
permanently_disabled = expectations.IsBenchmarkDisabled(
possible_browser.platform, finder_options)
temporarily_disabled = not decorators.IsBenchmarkEnabled(
benchmark, possible_browser)
if permanently_disabled or temporarily_disabled or not can_run_on_platform:
print '%s is disabled on the selected browser' % benchmark.Name()
if finder_options.run_disabled_tests and not permanently_disabled:
print 'Running benchmark anyway due to: --also-run-disabled-tests'
else:
print 'Try --also-run-disabled-tests to force the benchmark to run.'
# If chartjson is specified, this will print a dict indicating the
# benchmark name and disabled state.
with results_options.CreateResults(
benchmark_metadata, finder_options,
benchmark.ValueCanBeAddedPredicate, benchmark_enabled=False
) as results:
results.PrintSummary()
# When a disabled benchmark is run we now want to return success since
# we are no longer filtering these out in the buildbot recipes.
return 0
pt = benchmark.CreatePageTest(finder_options)
pt.__name__ = benchmark.__class__.__name__
disabled_attr_name = decorators.DisabledAttributeName(benchmark)
# pylint: disable=protected-access
pt._disabled_strings = getattr(benchmark, disabled_attr_name, set())
if hasattr(benchmark, '_enabled_strings'):
# pylint: disable=protected-access
pt._enabled_strings = benchmark._enabled_strings
stories = benchmark.CreateStorySet(finder_options)
if isinstance(pt, legacy_page_test.LegacyPageTest):
if any(not isinstance(p, page.Page) for p in stories.stories):
raise Exception(
'PageTest must be used with StorySet containing only '
'telemetry.page.Page stories.')
with results_options.CreateResults(
benchmark_metadata, finder_options,
benchmark.ValueCanBeAddedPredicate, benchmark_enabled=True) as results:
try:
Run(pt, stories, finder_options, results, benchmark.max_failures,
expectations=expectations, metadata=benchmark.GetMetadata())
return_code = min(254, len(results.failures))
# We want to make sure that all expectations are linked to real stories,
# this will log error messages if names do not match what is in the set.
benchmark.GetBrokenExpectations(stories)
except Exception: # pylint: disable=broad-except
results.telemetry_info.InterruptBenchmark()
exception_formatter.PrintFormattedException()
return_code = 255
benchmark_owners = benchmark.GetOwners()
benchmark_component = benchmark.GetBugComponents()
if benchmark_owners:
results.histograms.AddSharedDiagnostic(
reserved_infos.OWNERS.name, benchmark_owners)
if benchmark_component:
results.histograms.AddSharedDiagnostic(
reserved_infos.BUG_COMPONENTS.name, benchmark_component)
try:
if finder_options.upload_results:
results.UploadTraceFilesToCloud()
results.UploadProfilingFilesToCloud()
finally:
duration = time.time() - start
results.AddSummaryValue(scalar.ScalarValue(
None, 'benchmark_duration', 'minutes', duration / 60.0))
results.PrintSummary()
return return_code
