def generate_telemetry_tests(tester_config, benchmarks, benchmark_sharding_map,
use_whitelist):
isolated_scripts = []
# First determine the browser that you need based on the tester
browser_name = ''
if tester_config['platform'] == 'android':
browser_name = 'android-chromium'
elif (tester_config['platform'] == 'win'
and tester_config['target_bits'] == 64):
browser_name = 'release_x64'
else:
browser_name = 'release'
num_shards = len(tester_config['swarming_dimensions'][0]['device_ids'])
current_shard = 0
for benchmark in benchmarks:
# First figure out swarming dimensions this test needs to be triggered on.
# For each set of dimensions it is only triggered on one of the devices
swarming_dimensions = []
for dimension in tester_config['swarming_dimensions']:
device_affinity = None
if benchmark_sharding_map:
sharding_map = benchmark_sharding_map.get(
str(num_shards), None)
if not sharding_map and not use_whitelist:
raise Exception(
'Invalid number of shards, generate new sharding map')
if use_whitelist:
device_affinity = current_shard
else:
device_affinity = sharding_map.get(benchmark.Name(), None)
else:
# No sharding map was provided, default to legacy device
# affinity algorithm
device_affinity = bot_utils.GetDeviceAffinity(
num_shards, benchmark.Name())
if device_affinity is None:
raise Exception('Device affinity for benchmark %s not found' %
benchmark.Name())
swarming_dimensions.append(
get_swarming_dimension(dimension, device_affinity))
test = generate_telemetry_test(swarming_dimensions, benchmark.Name(),
browser_name)
isolated_scripts.append(test)
# Now create another executable for this benchmark on the reference browser
reference_test = generate_telemetry_test(swarming_dimensions,
benchmark.Name(), 'reference')
isolated_scripts.append(reference_test)
if current_shard == (num_shards - 1):
current_shard = 0
else:
current_shard += 1
return isolated_scripts
def shard_benchmarks(num_shards, all_benchmarks):
benchmark_to_shard_dict = {}
shard_execution_times = [0] * num_shards
sorted_benchmark_list, new_benchmarks = get_sorted_benchmark_list_by_time(
all_benchmarks)
# Iterate over in reverse order and add them to the current smallest bucket.
for benchmark in sorted_benchmark_list:
# Find current smallest bucket
min_index = shard_execution_times.index(min(shard_execution_times))
benchmark_to_shard_dict[benchmark[0].Name()] = min_index
shard_execution_times[min_index] += benchmark[1]
# For all the benchmarks that didn't have avg run times, use the default
# device affinity algorithm
for benchmark in new_benchmarks:
device_affinity = bot_utils.GetDeviceAffinity(num_shards, benchmark.Name())
benchmark_to_shard_dict[benchmark.Name()] = device_affinity
return benchmark_to_shard_dict
def generate_telemetry_tests(tester_config, benchmarks):
isolated_scripts = []
# First determine the browser that you need based on the tester
browser_name = ''
if tester_config['platform'] == 'android':
browser_name = 'android-chromium'
elif (tester_config['platform'] == 'win'
and tester_config['target_bits'] == 64):
browser_name = 'release_x64'
else:
browser_name ='release'
for benchmark in benchmarks:
# First figure out swarming dimensions this test needs to be triggered on.
# For each set of dimensions it is only triggered on one of the devices
swarming_dimensions = []
for dimension in tester_config['swarming_dimensions']:
device_affinity = bot_utils.GetDeviceAffinity(
len(dimension['device_ids']), benchmark.Name())
device_id = dimension['device_ids'][device_affinity]
# Id is unique within the swarming pool so it is the only needed
# identifier for the bot to run the test on
swarming_dimensions.append({
'id': device_id
})
test = generate_telemetry_test(
swarming_dimensions, benchmark.Name(), browser_name)
isolated_scripts.append(test)
# Now create another executable for this benchmark on the reference browser
reference_test = generate_telemetry_test(
swarming_dimensions, benchmark.Name(),'reference')
isolated_scripts.append(reference_test)
return isolated_scripts
def _GetJsonBenchmarkList(possible_browser, possible_reference_browser,
benchmark_classes, num_shards):
"""Returns a list of all enabled benchmarks in a JSON format expected by
buildbots.
JSON format:
{ "version": <int>,
"steps": {
<string>: {
"device_affinity": <int>,
"cmd": <string>,
"perf_dashboard_id": <string>,
},
...
}
}
"""
# TODO(charliea): Remove this once we have more power perf bots.
only_run_battor_benchmarks = False
print 'Environment variables: ', os.environ
if os.environ.get('BUILDBOT_BUILDERNAME') in GOOD_POWER_PERF_BOT_WHITELIST:
only_run_battor_benchmarks = True
output = {'version': 1, 'steps': {}}
for benchmark_class in benchmark_classes:
# Filter out benchmarks in tools/perf/contrib/ directory
# This is a terrible hack but we should no longer need this
# _GetJsonBenchmarkList once all the perf bots are moved to swarming
# (crbug.com/715565)
if ('contrib' in os.path.abspath(
sys.modules[benchmark_class.__module__].__file__)):
continue
if not _IsBenchmarkEnabled(benchmark_class, possible_browser):
continue
base_name = benchmark_class.Name()
# TODO(charliea): Remove this once we have more power perf bots.
# Only run battor power benchmarks to reduce the cycle time of this bot.
# TODO(rnephew): Enable media.* and power.* tests when Mac BattOr issue
# is solved.
if only_run_battor_benchmarks and not base_name.startswith('battor'):
continue
base_cmd = [
sys.executable,
os.path.realpath(sys.argv[0]), '-v', '--output-format=chartjson',
'--upload-results', base_name
]
perf_dashboard_id = base_name
device_affinity = bot_utils.GetDeviceAffinity(num_shards, base_name)
output['steps'][base_name] = {
'cmd':
' '.join(base_cmd +
['--browser=%s' % possible_browser.browser_type]),
'device_affinity':
device_affinity,
'perf_dashboard_id':
perf_dashboard_id,
}
if (possible_reference_browser and _IsBenchmarkEnabled(
benchmark_class, possible_reference_browser)):
output['steps'][base_name + '.reference'] = {
'cmd':
' '.join(base_cmd +
['--browser=reference', '--output-trace-tag=_ref']),
'device_affinity':
device_affinity,
'perf_dashboard_id':
perf_dashboard_id,
}
return json.dumps(output, indent=2, sort_keys=True)
