def _GetJsonTestList(possible_browser, possible_reference_browser,
test_classes, num_shards):
"""Returns a list of all enabled tests in a JSON format expected by buildbots.
JSON format (see build/android/pylib/perf/test_runner.py):
{ "version": <int>,
"steps": {
<string>: {
"device_affinity": <int>,
"cmd": <string>,
"perf_dashboard_id": <string>,
},
...
}
}
"""
output = {
'version': 1,
'steps': {
}
}
for test_class in test_classes:
if not issubclass(test_class, benchmark.Benchmark):
continue
if not decorators.IsEnabled(test_class, possible_browser):
continue
base_name = test_class.Name()
base_cmd = [sys.executable, os.path.realpath(sys.argv[0]),
'-v', '--output-format=buildbot', base_name]
perf_dashboard_id = base_name
# TODO(tonyg): Currently we set the device affinity to a stable hash of the
# test name. This somewhat evenly distributes benchmarks among the requested
# number of shards. However, it is far from optimal in terms of cycle time.
# We should add a test size decorator (e.g. small, medium, large) and let
# that inform sharding.
device_affinity = int(hashlib.sha1(base_name).hexdigest(), 16) % num_shards
output['steps'][base_name] = {
'cmd': ' '.join(base_cmd + [
'--browser=%s' % possible_browser.browser_type]),
'device_affinity': device_affinity,
'perf_dashboard_id': perf_dashboard_id,
}
if (possible_reference_browser and
decorators.IsEnabled(test_class, possible_reference_browser)):
output['steps'][base_name + '.reference'] = {
'cmd': ' '.join(base_cmd + [
'--browser=reference', '--output-trace-tag=_ref']),
'device_affinity': device_affinity,
'perf_dashboard_id': perf_dashboard_id,
}
return json.dumps(output, indent=2, sort_keys=True)
def RunTest(self):
class SinglePageBenchmark(benchmark_class): # pylint: disable=no-init
def CreateStorySet(self, options):
# pylint: disable=super-on-old-class
story_set = super(SinglePageBenchmark,
self).CreateStorySet(options)
stories_to_remove = [
s for s in story_set.stories if s != story_to_smoke_test
]
for s in stories_to_remove:
story_set.RemoveStory(s)
assert story_set.stories
return story_set
options = GenerateBenchmarkOptions(benchmark_class)
possible_browser = browser_finder.FindBrowser(options)
if possible_browser is None:
self.skipTest('Cannot find the browser to run the test.')
if (SinglePageBenchmark.ShouldDisable(possible_browser)
or not decorators.IsEnabled(benchmark_class,
possible_browser)[0]):
self.skipTest('Benchmark %s is disabled' %
SinglePageBenchmark.Name())
if self.id() in _DISABLED_TESTS:
self.skipTest('Test is explicitly disabled')
self.assertEqual(0,
SinglePageBenchmark().Run(options),
msg='Failed: %s' % benchmark_class)
def IsTestSelected(test):
if len(args) != 0:
found = False
for name in args:
if name in test.id():
found = True
if not found:
return False
if default_options.run_disabled_tests:
return True
# pylint: disable=W0212
if not hasattr(test, '_testMethodName'):
return True
method = getattr(test, test._testMethodName)
return decorators.IsEnabled(method, options.GetBrowserType(), platform)
def _GetPossibleBrowser(self, test, finder_options):
"""Return a possible_browser with the given options for |test|. """
possible_browser = self._FindBrowser(finder_options)
finder_options.browser_options.browser_type = (
possible_browser.browser_type)
enabled, msg = decorators.IsEnabled(test, possible_browser)
if not enabled and not finder_options.run_disabled_tests:
logging.warning(msg)
logging.warning('You are trying to run a disabled test.')
if possible_browser.IsRemote():
possible_browser.RunRemote()
sys.exit(0)
return possible_browser
def _GetPossibleBrowser(self, test, finder_options):
"""Return a possible_browser with the given options for |test|. """
possible_browser = self._FindBrowser(finder_options)
finder_options.browser_options.browser_type = (
possible_browser.browser_type)
(enabled, msg) = decorators.IsEnabled(test, possible_browser)
if (not enabled and not finder_options.run_disabled_tests):
logging.warning(msg)
logging.warning('You are trying to run a disabled test.')
logging.warning(
'Pass --also-run-disabled-tests to squelch this message.')
sys.exit(0)
if possible_browser.IsRemote():
possible_browser.RunRemote()
sys.exit(0)
return possible_browser
def RunTest(self):
class SinglePageBenchmark(benchmark_class): # pylint: disable=no-init
def CreateStorySet(self, options):
# pylint: disable=super-on-old-class
story_set = super(SinglePageBenchmark,
self).CreateStorySet(options)
stories_to_remove = [
s for s in story_set.stories if s != story_to_smoke_test
]
for s in stories_to_remove:
story_set.RemoveStory(s)
assert story_set.stories
return story_set
options = GenerateBenchmarkOptions(benchmark_class)
# Prevent benchmarks from accidentally trying to upload too much data to the
# chromeperf dashboard. The number of values uploaded is equal to (the
# average number of values produced by a single story) * (1 + (the number of
# stories)). The "1 + " accounts for values summarized across all stories.
# We can approximate "the average number of values produced by a single
# story" as the number of values produced by the given story.
# pageset_repeat doesn't matter because values are summarized across
# repetitions before uploading.
story_set = benchmark_class().CreateStorySet(options)
SinglePageBenchmark.MAX_NUM_VALUES = MAX_NUM_VALUES / len(
story_set.stories)
possible_browser = browser_finder.FindBrowser(options)
if possible_browser is None:
self.skipTest('Cannot find the browser to run the test.')
if (SinglePageBenchmark.ShouldDisable(possible_browser)
or not decorators.IsEnabled(benchmark_class,
possible_browser)[0]):
self.skipTest('Benchmark %s is disabled' %
SinglePageBenchmark.Name())
if self.id() in _DISABLED_TESTS:
self.skipTest('Test is explicitly disabled')
self.assertEqual(0,
SinglePageBenchmark().Run(options),
msg='Failed: %s' % benchmark_class)
def PrintBenchmarkList(benchmarks, possible_browser, output_pipe=sys.stdout):
""" Print benchmarks that are not filtered in the same order of benchmarks in
the |benchmarks| list.
Args:
benchmarks: the list of benchmarks to be printed (in the same order of the
list).
possible_browser: the possible_browser instance that's used for checking
which benchmarks are enabled.
output_pipe: the stream in which benchmarks are printed on.
"""
if not benchmarks:
print >> output_pipe, 'No benchmarks found!'
return
b = None # Need this to stop pylint from complaining undefined variable.
if any(not issubclass(b, benchmark.Benchmark) for b in benchmarks):
assert False, '|benchmarks| param contains non benchmark class: %s' % b
# Align the benchmark names to the longest one.
format_string = ' %%-%ds %%s' % max(len(b.Name()) for b in benchmarks)
disabled_benchmarks = []
print >> output_pipe, 'Available benchmarks %sare:' % (
'for %s ' % possible_browser.browser_type if possible_browser else '')
for benchmark_class in benchmarks:
if possible_browser and not decorators.IsEnabled(
benchmark_class, possible_browser)[0]:
disabled_benchmarks.append(benchmark_class)
continue
print >> output_pipe, format_string % (benchmark_class.Name(),
benchmark_class.Description())
if disabled_benchmarks:
print >> output_pipe
print >> output_pipe, (
'Disabled benchmarks for %s are (force run with -d):' %
possible_browser.browser_type)
for benchmark_class in disabled_benchmarks:
print >> output_pipe, format_string % (
benchmark_class.Name(), benchmark_class.Description())
print >> output_pipe, (
'Pass --browser to list benchmarks for another browser.')
print >> output_pipe
def IsTestSelected(test):
if selected_tests:
found = False
for name in selected_tests:
if selected_tests_are_exact:
if name == test.id():
found = True
else:
if name in test.id():
found = True
if not found:
return False
if run_disabled_tests:
return True
# pylint: disable=W0212
if not hasattr(test, '_testMethodName'):
return True
method = getattr(test, test._testMethodName)
return decorators.IsEnabled(method, possible_browser)
def _GetJsonTestList(possible_browser, test_classes, num_shards):
"""Returns a list of all enabled tests in a JSON format expected by buildbots.
JSON format (see build/android/pylib/perf/test_runner.py):
{ "version": int,
"steps": {
"foo": {
"device_affinity": int,
"cmd": "script_to_execute foo"
},
"bar": {
"device_affinity": int,
"cmd": "script_to_execute bar"
}
}
}
"""
output = {'version': 1, 'steps': {}}
for test_class in test_classes:
if not issubclass(test_class, benchmark.Benchmark):
continue
if not decorators.IsEnabled(test_class, possible_browser):
continue
name = test_class.Name()
output['steps'][name] = {
'cmd':
' '.join([
sys.executable,
os.path.realpath(sys.argv[0]),
'--browser=%s' % possible_browser.browser_type, '-v',
'--output-format=buildbot', name
]),
# TODO(tonyg): Currently we set the device affinity to a stable hash of
# the test name. This somewhat evenly distributes benchmarks among the
# requested number of shards. However, it is far from optimal in terms of
# cycle time. We should add a test size decorator (e.g. small, medium,
# large) and let that inform sharding.
'device_affinity':
int(hashlib.sha1(name).hexdigest(), 16) % num_shards
}
return json.dumps(output, indent=2, sort_keys=True)
def _GetPossibleBrowser(self, test, finder_options):
"""Return a possible_browser with the given options. """
possible_browser = browser_finder.FindBrowser(finder_options)
if not possible_browser:
raise browser_finder_exceptions.BrowserFinderException(
'No browser found.\n\nAvailable browsers:\n%s\n' %
'\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)))
finder_options.browser_options.browser_type = (
possible_browser.browser_type)
(enabled, msg) = decorators.IsEnabled(test, possible_browser)
if (not enabled and
not finder_options.run_disabled_tests):
logging.warning(msg)
logging.warning('You are trying to run a disabled test.')
logging.warning('Pass --also-run-disabled-tests to squelch this message.')
sys.exit(0)
if possible_browser.IsRemote():
possible_browser.RunRemote()
sys.exit(0)
return possible_browser
def _PrintBenchmarkList(benchmarks, possible_browser):
if not benchmarks:
print >> sys.stderr, 'No benchmarks found!'
return
# Align the benchmark names to the longest one.
format_string = ' %%-%ds %%s' % max(len(b.Name()) for b in benchmarks)
filtered_benchmarks = [
benchmark_class for benchmark_class in benchmarks
if issubclass(benchmark_class, benchmark.Benchmark)
]
disabled_benchmarks = []
if filtered_benchmarks:
print >> sys.stderr, 'Available benchmarks %sare:' % (
'for %s ' %
possible_browser.browser_type if possible_browser else '')
for benchmark_class in sorted(filtered_benchmarks,
key=lambda b: b.Name()):
if possible_browser and not decorators.IsEnabled(
benchmark_class, possible_browser):
disabled_benchmarks.append(benchmark_class)
continue
print >> sys.stderr, format_string % (
benchmark_class.Name(), benchmark_class.Description())
if disabled_benchmarks:
print >> sys.stderr, (
'Disabled benchmarks for %s are (force run with -d): ' %
possible_browser.browser_type)
for benchmark_class in disabled_benchmarks:
print >> sys.stderr, format_string % (
benchmark_class.Name(), benchmark_class.Description())
print >> sys.stderr, (
'Pass --browser to list benchmarks for another browser.')
print >> sys.stderr
def _IsBenchmarkEnabled(benchmark_class, possible_browser):
return (issubclass(benchmark_class, benchmark.Benchmark)
and not benchmark_class.ShouldDisable(possible_browser)
and decorators.IsEnabled(benchmark_class, possible_browser)[0])
try:
possible_browser = browser_finder.FindBrowser(finder_options)
except browser_finder.BrowserTypeRequiredException, e:
sys.stderr.write(str(e) + '\n')
sys.exit(-1)
if not possible_browser:
sys.stderr.write(
'No browser found. Available browsers:\n%s\n' %
'\n'.join(browser_finder.GetAllAvailableBrowserTypes(finder_options)))
sys.exit(-1)
browser_options = possible_browser.finder_options.browser_options
browser_options.browser_type = possible_browser.browser_type
test.CustomizeBrowserOptions(browser_options)
if (not decorators.IsEnabled(test, possible_browser) and
not finder_options.run_disabled_tests):
logging.warning('You are trying to run a disabled test.')
logging.warning('Pass --also-run-disabled-tests to squelch this message.')
return
if possible_browser.IsRemote():
possible_browser.RunRemote()
sys.exit(0)
# Reorder page set based on options.
pages = _ShuffleAndFilterPageSet(page_set, finder_options)
if not finder_options.use_live_sites:
_UpdateCredentials(page_set)
if browser_options.wpr_mode != wpr_modes.WPR_RECORD:
def MockPredicate(test):
method = getattr(test, test._testMethodName)
return decorators.IsEnabled(method, MockPossibleBrowser(
browser_type, os_name, os_version_name, supports_tab_control))
def _GetJsonBenchmarkList(possible_browser, possible_reference_browser,
benchmark_classes, num_shards):
"""Returns a list of all enabled benchmarks in a JSON format expected by
buildbots.
JSON format (see build/android/pylib/perf/benchmark_runner.py):
{ "version": <int>,
"steps": {
<string>: {
"device_affinity": <int>,
"cmd": <string>,
"perf_dashboard_id": <string>,
},
...
}
}
"""
output = {'version': 1, 'steps': {}}
for benchmark_class in benchmark_classes:
if not issubclass(benchmark_class, benchmark.Benchmark):
continue
enabled, _ = decorators.IsEnabled(benchmark_class, possible_browser)
if not enabled:
continue
base_name = benchmark_class.Name()
base_cmd = [
sys.executable,
os.path.realpath(sys.argv[0]), '-v', '--output-format=chartjson',
'--upload-results', base_name
]
perf_dashboard_id = base_name
# TODO(fmeawad): Currently we set the device affinity to a stable hash of
# the benchmark name. This somewhat evenly distributes benchmarks among the
# requested number of shards. However, it is far from optimal in terms of
# cycle time. We should add a benchmark size decorator (e.g. small, medium,
# large) and let that inform sharding.
# Based on the current timings, we shift the result of the hash function to
# achieve better load balancing. Those shift values are to be revised when
# necessary. (See tools/build/scripts/tools/perf/chrome-perf-step-timings.py
# for more details)
hash_shift = {2: 47, 5: 56, 8: 50}
shift = hash_shift.get(num_shards, 0)
base_name_hash = hashlib.sha1(base_name).hexdigest()
device_affinity = (int(base_name_hash, 16) >> shift) % num_shards
output['steps'][base_name] = {
'cmd':
' '.join(base_cmd +
['--browser=%s' % possible_browser.browser_type]),
'device_affinity':
device_affinity,
'perf_dashboard_id':
perf_dashboard_id,
}
if possible_reference_browser:
enabled, _ = decorators.IsEnabled(benchmark_class,
possible_reference_browser)
if enabled:
output['steps'][base_name + '.reference'] = {
'cmd':
' '.join(
base_cmd +
['--browser=reference', '--output-trace-tag=_ref']),
'device_affinity':
device_affinity,
'perf_dashboard_id':
perf_dashboard_id,
}
return json.dumps(output, indent=2, sort_keys=True)
def MockPredicate(test):
method = getattr(test, test._testMethodName)
return decorators.IsEnabled(method, browser_type, platform)
# Create a possible_browser with the given options.
try:
possible_browser = browser_finder.FindBrowser(finder_options)
except browser_finder.BrowserTypeRequiredException, e:
sys.stderr.write(str(e) + '\n')
sys.exit(-1)
if not possible_browser:
sys.stderr.write('No browser found. Available browsers:\n' + '\n'.join(
browser_finder.GetAllAvailableBrowserTypes(finder_options)) + '\n')
sys.exit(-1)
browser_options = possible_browser.finder_options.browser_options
browser_options.browser_type = possible_browser.browser_type
test.CustomizeBrowserOptions(browser_options)
should_run = decorators.IsEnabled(test, possible_browser)
should_run = should_run or finder_options.run_disabled_tests
if not should_run:
logging.warning('You are trying to run a disabled test.')
logging.warning(
'Pass --also-run-disabled-tests to squelch this message.')
return
# Reorder page set based on options.
pages = _ShuffleAndFilterPageSet(page_set, finder_options)
if (not finder_options.use_live_sites
and browser_options.wpr_mode != wpr_modes.WPR_RECORD):
_UpdatePageSetArchivesIfChanged(page_set)
# Create a possible_browser with the given options.
test.CustomizeBrowserOptions(finder_options)
try:
possible_browser = browser_finder.FindBrowser(finder_options)
except browser_finder.BrowserTypeRequiredException, e:
sys.stderr.write(str(e) + '\n')
sys.exit(1)
if not possible_browser:
sys.stderr.write('No browser found. Available browsers:\n' + '\n'.join(
browser_finder.GetAllAvailableBrowserTypes(finder_options)) + '\n')
sys.exit(1)
browser_options.browser_type = possible_browser.browser_type
if not decorators.IsEnabled(test, browser_options.browser_type,
possible_browser.platform):
return results
# Reorder page set based on options.
pages = _ShuffleAndFilterPageSet(page_set, finder_options)
if (not finder_options.allow_live_sites
and browser_options.wpr_mode != wpr_modes.WPR_RECORD):
pages = _CheckArchives(page_set, pages, results)
# Verify credentials path.
credentials_path = None
if page_set.credentials_path:
credentials_path = os.path.join(os.path.dirname(page_set.file_path),
page_set.credentials_path)
if not os.path.exists(credentials_path):
