def CreateStorySet(self, options):
spaceport_dir = os.path.join(path_util.GetChromiumSrcDir(), 'chrome',
'test', 'data', 'third_party',
'spaceport')
ps = story.StorySet(base_dir=spaceport_dir)
ps.AddStory(page_module.Page('file://index.html', ps, ps.base_dir))
return ps
def TearDownState(self):
super(AndroidSharedVrPageState, self).TearDownState()
# Re-apply Cardboard as the viewer to leave the device in a consistent
# state after a benchmark run
# TODO(bsheedy): Remove this after crbug.com/772969 is fixed
self._ConfigureVrCore(
os.path.join(path_util.GetChromiumSrcDir(), CARDBOARD_PATH))
def ZipDependencies(target_paths, dependencies, options):
base_dir = os.path.dirname(os.path.realpath(path_util.GetChromiumSrcDir()))
with zipfile.ZipFile(options.zip, 'w', zipfile.ZIP_DEFLATED) as zip_file:
# Add dependencies to archive.
for dependency_path in dependencies:
path_in_archive = os.path.join(
'telemetry', os.path.relpath(dependency_path, base_dir))
zip_file.write(dependency_path, path_in_archive)
# Add symlinks to executable paths, for ease of use.
for target_path in target_paths:
link_info = zipfile.ZipInfo(
os.path.join('telemetry', os.path.basename(target_path)))
link_info.create_system = 3 # Unix attributes.
# 010 is regular file, 0111 is the permission bits rwxrwxrwx.
link_info.external_attr = 0100777 << 16 # Octal.
relative_path = os.path.relpath(target_path, base_dir)
link_script = (
'#!/usr/bin/env vpython\n\n'
'import os\n'
'import sys\n\n\n'
'script = os.path.join(os.path.dirname(__file__), \'%s\')\n'
'os.execv(sys.executable, [sys.executable, script] + sys.argv[1:])'
% relative_path)
zip_file.writestr(link_info, link_script)
def _InstallVrCore(self):
# TODO(bsheedy): Add support for temporarily replacing it if it's still
# installed as a system app on the test device
self._platform.InstallApplication(
os.path.join(path_util.GetChromiumSrcDir(), 'third_party',
'gvr-android-sdk', 'test-apks', 'vr_services',
'vr_services_current.apk'))
def _RunFetchBenchmarkDepsTest(self,
benchmark_name,
expected_fetched_file_paths=None):
"""Simulates './fetch_benchmark_deps [benchmark_name]'
It checks if the paths returned are expected and have corresponding sha1
checksums. The expected result can be omitted if the dependencies of
specified benchmarks are subject to changes.
Args:
benchmark_name: benchmark name
expected_fetched_file_paths: the expected result.
"""
sys.argv[1] = benchmark_name
output = StringIO.StringIO()
with mock.patch('telemetry.wpr.archive_info.WprArchiveInfo'
'.DownloadArchivesIfNeeded') as mock_download:
with mock.patch('py_utils.cloud_storage'
'.GetFilesInDirectoryIfChanged') as mock_get:
mock_download.return_value = True
mock_get.GetFilesInDirectoryIfChanged.return_value = True
fetch_benchmark_deps.main(output)
for f in output.getvalue().splitlines():
fullpath = os.path.join(path_util.GetChromiumSrcDir(), f)
sha1path = fullpath + '.sha1'
self.assertTrue(os.path.isfile(sha1path))
if expected_fetched_file_paths:
self.assertEquals(expected_fetched_file_paths,
NormPaths(output.getvalue()))
def SetExtraBrowserOptions(self, options):
options.clear_sytem_cache_for_browser_and_profile_on_start = True
# This flag is required to enable the communication between the page and
# the test extension.
options.disable_background_networking = False
# TODO: find a better way to find extension location.
options.AppendExtraBrowserArgs([
'--load-extension=' + ','.join([
os.path.join(path_util.GetChromiumSrcDir(), 'out', 'Release',
'mr_extension', 'release'),
os.path.join(path_util.GetChromiumSrcDir(), 'out', 'Release',
'media_router', 'test_extension')
]), '--whitelisted-extension-id=enhhojjnijigcajfphajepfemndkmdlo',
'--media-router=1', '--enable-stats-collection-bindings'
])
def update_benchmark_csv():
"""Updates go/chrome-benchmarks.
Updates telemetry/perf/benchmark.csv containing the current benchmark names,
owners, and components. Requires that all benchmarks have owners.
"""
header_data = [['AUTOGENERATED FILE DO NOT EDIT'],
['See //tools/perf/generate_perf_data.py to make changes'],
['Benchmark name', 'Individual owners', 'Component']
]
csv_data = []
all_benchmarks = NON_TELEMETRY_BENCHMARKS
all_benchmarks.update(NON_WATERFALL_BENCHMARKS)
benchmark_metadata = get_all_benchmarks_metadata(all_benchmarks)
_verify_benchmark_owners(benchmark_metadata)
for benchmark_name in benchmark_metadata:
csv_data.append([
benchmark_name,
benchmark_metadata[benchmark_name].emails,
benchmark_metadata[benchmark_name].component
])
csv_data = sorted(csv_data, key=lambda b: b[0])
csv_data = header_data + csv_data
perf_dir = os.path.join(path_util.GetChromiumSrcDir(), 'tools', 'perf')
benchmark_file = os.path.join(perf_dir, 'benchmark.csv')
with open(benchmark_file, 'wb') as f:
writer = csv.writer(f, lineterminator="\n")
writer.writerows(csv_data)
def _EnumerateDependencies(story_set):
"""Enumerates paths of files needed by a user story set."""
deps = set()
# Enumerate WPRs
for story in story_set:
deps.add(story_set.WprFilePathForStory(story))
# Enumerate files in serving_dirs
for directory in story_set.serving_dirs:
if not os.path.isdir(directory):
raise ValueError('Must provide a valid directory.')
# Don't allow the root directory to be a serving_dir.
if directory == os.path.abspath(os.sep):
raise ValueError(
'Trying to serve root directory from HTTP server.')
for dirpath, _, filenames in os.walk(directory):
for filename in filenames:
path_name, extension = os.path.splitext(
os.path.join(dirpath, filename))
if extension == '.sha1':
deps.add(path_name)
# Return relative paths.
prefix_len = len(os.path.realpath(path_util.GetChromiumSrcDir())) + 1
return [dep[prefix_len:] for dep in deps if dep]
def MakeHistogramSetWithDiagnostics(histograms_file, test_name, bot,
buildername, buildnumber, project,
buildbucket, revisions_dict,
is_reference_build,
perf_dashboard_machine_group):
add_diagnostics_args = []
add_diagnostics_args.extend([
'--benchmarks',
test_name,
'--bots',
bot,
'--builds',
buildnumber,
'--masters',
perf_dashboard_machine_group,
'--is_reference_build',
'true' if is_reference_build else '',
])
stdio_url = _MakeStdioUrl(test_name, buildername, buildnumber)
if stdio_url:
add_diagnostics_args.extend(['--log_urls_k', 'Buildbot stdio'])
add_diagnostics_args.extend(['--log_urls_v', stdio_url])
build_status_url = _MakeBuildStatusUrl(project, buildbucket, buildername,
buildnumber)
if build_status_url:
add_diagnostics_args.extend(['--build_urls_k', 'Build Status'])
add_diagnostics_args.extend(['--build_urls_v', build_status_url])
for k, v in revisions_dict.iteritems():
add_diagnostics_args.extend((k, v))
add_diagnostics_args.append(histograms_file)
# Subprocess only accepts string args
add_diagnostics_args = [str(v) for v in add_diagnostics_args]
add_reserved_diagnostics_path = os.path.join(path_util.GetChromiumSrcDir(),
'third_party', 'catapult',
'tracing', 'bin',
'add_reserved_diagnostics')
tf = tempfile.NamedTemporaryFile(delete=False)
tf.close()
temp_histogram_output_file = tf.name
cmd = ([sys.executable, add_reserved_diagnostics_path] +
add_diagnostics_args +
['--output_path', temp_histogram_output_file])
try:
subprocess.check_call(cmd)
# TODO: Handle reference builds
with open(temp_histogram_output_file) as f:
hs = json.load(f)
return hs
finally:
os.remove(temp_histogram_output_file)
def FindBootstrapDependencies(base_dir):
deps_file = os.path.join(base_dir, DEPS_FILE)
if not os.path.exists(deps_file):
return []
deps_paths = bootstrap.ListAllDepsPaths(deps_file)
return set(os.path.realpath(os.path.join(
path_util.GetChromiumSrcDir(), '..', deps_path))
for deps_path in deps_paths)
def _ConfigureVrCore(self):
settings = shared_preference_utils.ExtractSettingsFromJson(
os.path.join(path_util.GetChromiumSrcDir(),
self._finder_options.shared_prefs_file))
for setting in settings:
shared_pref = self._platform.GetSharedPrefs(
setting['package'], setting['filename'])
shared_preference_utils.ApplySharedPreferenceSetting(
shared_pref, setting)
def main(args):
del args # unused
waterfall_file = os.path.join(path_util.GetChromiumSrcDir(), 'testing',
'buildbot', 'chromium.perf.json')
fyi_waterfall_file = os.path.join(path_util.GetChromiumSrcDir(), 'testing',
'buildbot', 'chromium.perf.fyi.json')
calibration_waterfall_file = os.path.join(
path_util.GetChromiumSrcDir(), 'testing', 'buildbot',
'chromium.perf.calibration.json')
with open(fyi_waterfall_file) as f:
ValidatePerfConfigFile(f, False)
with open(waterfall_file) as f:
ValidatePerfConfigFile(f, True)
with open(calibration_waterfall_file) as f:
ValidatePerfConfigFile(f, False)
def MakeHistogramSetWithDiagnostics(histograms_file,
test_name, bot, buildername, buildnumber,
project, buildbucket,
revisions_dict, is_reference_build,
perf_dashboard_machine_group, output_dir,
max_bytes=0):
"""Merges Histograms, adds Diagnostics, and batches the results.
Args:
histograms_file: input filename
output_dir: output directory
max_bytes: If non-zero, tries to produce files no larger than max_bytes.
(May generate a file that is larger than max_bytes if max_bytes is smaller
than a single Histogram.)
"""
add_diagnostics_args = []
add_diagnostics_args.extend([
'--benchmarks', test_name,
'--bots', bot,
'--builds', buildnumber,
'--masters', perf_dashboard_machine_group,
'--is_reference_build', 'true' if is_reference_build else '',
])
if max_bytes:
add_diagnostics_args.extend(['--max_bytes', max_bytes])
stdio_url = _MakeStdioUrl(test_name, buildername, buildnumber)
if stdio_url:
add_diagnostics_args.extend(['--log_urls_k', 'Buildbot stdio'])
add_diagnostics_args.extend(['--log_urls_v', stdio_url])
build_status_url = _MakeBuildStatusUrl(
project, buildbucket, buildername, buildnumber)
if build_status_url:
add_diagnostics_args.extend(['--build_urls_k', 'Build Status'])
add_diagnostics_args.extend(['--build_urls_v', build_status_url])
for k, v in revisions_dict.items():
add_diagnostics_args.extend((k, v))
add_diagnostics_args.append(histograms_file)
# Subprocess only accepts string args
add_diagnostics_args = [str(v) for v in add_diagnostics_args]
add_reserved_diagnostics_path = os.path.join(
path_util.GetChromiumSrcDir(), 'third_party', 'catapult', 'tracing',
'bin', 'add_reserved_diagnostics')
# This script may write multiple files to output_dir.
output_path = os.path.join(output_dir, test_name + '.json')
cmd = ([sys.executable, add_reserved_diagnostics_path] +
add_diagnostics_args + ['--output_path', output_path])
logging.info(cmd)
subprocess.check_call(cmd)
def _PerfettoRevision():
deps_line_re = re.compile(
r".*'/platform/external/perfetto.git' \+ '@' \+ '([a-f0-9]+)'")
deps_file = os.path.join(path_util.GetChromiumSrcDir(), 'DEPS')
with open(deps_file) as deps:
for line in deps:
match = deps_line_re.match(line)
if match:
return match.group(1)
raise RuntimeError("Couldn't parse perfetto revision from DEPS")
def current_benchmarks():
benchmarks_dir = os.path.join(
path_util.GetChromiumSrcDir(), 'tools', 'perf', 'benchmarks')
top_level_dir = os.path.dirname(benchmarks_dir)
all_benchmarks = discover.DiscoverClasses(
benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
index_by_class_name=True).values()
return sorted(all_benchmarks, key=lambda b: b.Name())
def SetExtraBrowserOptions(self, options):
options.clear_sytem_cache_for_browser_and_profile_on_start = True
# TODO: find a better way to find extension location.
options.AppendExtraBrowserArgs([
'--load-extension=' +
os.path.join(path_util.GetChromiumSrcDir(), 'out', 'Release',
'mr_extension'),
'--whitelisted-extension-id=enhhojjnijigcajfphajepfemndkmdlo',
'--media-router=1', '--enable-stats-collection-bindings'
])
def main(args):
parser = argparse.ArgumentParser(description=(
'Generate perf test\' json config and benchmark.csv. '
'This needs to be done anytime you add/remove any existing'
'benchmarks in tools/perf/benchmarks.'))
parser.add_argument(
'--validate-only',
action='store_true',
default=False,
help=(
'Validate whether the perf json generated will be the same as the '
'existing configs. This does not change the contain of existing '
'configs'))
options = parser.parse_args(args)
waterfall_file = os.path.join(path_util.GetChromiumSrcDir(), 'testing',
'buildbot', 'chromium.perf.json')
fyi_waterfall_file = os.path.join(path_util.GetChromiumSrcDir(), 'testing',
'buildbot', 'chromium.perf.fyi.json')
benchmark_file = os.path.join(path_util.GetChromiumSrcDir(), 'tools',
'perf', 'benchmark.csv')
labs_docs_file = os.path.join(path_util.GetChromiumSrcDir(), 'docs',
'speed', 'perf_lab_platforms.md')
if options.validate_only:
if validate_tests(get_waterfall_builder_config(), waterfall_file,
benchmark_file, labs_docs_file):
print 'All the perf config files are up-to-date. \\o/'
return 0
else:
print(
'Not all perf config files are up-to-date. Please run %s '
'to update them.') % sys.argv[0]
return 1
else:
load_and_update_fyi_json(fyi_waterfall_file)
update_all_tests(get_waterfall_builder_config(), waterfall_file)
update_benchmark_csv(benchmark_file)
update_labs_docs_md(labs_docs_file)
return 0
def __init__(self, test, finder_options, story_set, possible_browser=None):
super(AndroidSharedVrPageState,
self).__init__(test, finder_options, story_set, possible_browser)
if not self._finder_options.disable_vrcore_install:
self._InstallVrCore()
self._ConfigureVrCore(
os.path.join(path_util.GetChromiumSrcDir(),
self._finder_options.shared_prefs_file))
self._InstallNfcApk()
if not self._finder_options.disable_keyboard_install:
self._InstallKeyboardApk()
def SetExtraBrowserOptions(self, options):
options.clear_sytem_cache_for_browser_and_profile_on_start = True
# This flag is required to enable the communication between the page and
# the test extension.
options.disable_background_networking = False
options.AppendExtraBrowserArgs([
'--load-extension=' +
os.path.join(path_util.GetChromiumSrcDir(), 'out', 'Release',
'media_router', 'test_extension'), '--media-router=0',
'--enable-stats-collection-bindings'
])
def SetExtraBrowserOptions(self, options):
options.flush_os_page_caches_on_start = True
# This flag is required to enable the communication between the page and
# the test extension.
options.disable_background_networking = False
options.AppendExtraBrowserArgs([
'--load-extension=' +
os.path.join(path_util.GetChromiumSrcDir(), 'out', 'Release',
'media_router', 'telemetry_extension'),
'--disable-features=ViewsCastDialog', '--media-router=0',
'--enable-stats-collection-bindings'
])
def TearDownState(self):
super(SharedAndroidVrPageState, self).TearDownState()
# Reset the tracker type to use the actual sensor if it's been changed. When
# run on the bots, this shouldn't matter since the service will be killed
# during the automatic restart, but this could persist when run locally.
if self._did_set_tracker:
self.SetPoseTrackerType('sensor')
# Re-apply Cardboard as the viewer to leave the device in a consistent
# state after a benchmark run
# TODO(bsheedy): Remove this after crbug.com/772969 is fixed
self._ConfigureVrCore(
os.path.join(path_util.GetChromiumSrcDir(), CARDBOARD_PATH))
def FindPythonDependencies(module_path):
logging.info('Finding Python dependencies of %s', module_path)
if modulegraph is None:
raise import_error
prefixes = [sys.prefix]
if hasattr(sys, 'real_prefix'):
prefixes.append(sys.real_prefix)
logging.info('Excluding Prefixes: %r', prefixes)
sys_path = sys.path
sys.path = list(sys_path)
try:
# Load the module to inherit its sys.path modifications.
sys.path.insert(0, os.path.abspath(os.path.dirname(module_path)))
imp.load_source(
os.path.splitext(os.path.basename(module_path))[0], module_path)
# Analyze the module for its imports.
graph = modulegraph.ModuleGraph()
graph.run_script(module_path)
# Filter for only imports in Chromium.
for node in graph.nodes():
if not node.filename:
continue
module_path = os.path.realpath(node.filename)
_, incoming_edges = graph.get_edges(node)
message = 'Discovered %s (Imported by: %s)' % (
node.filename, ', '.join(
d.filename for d in incoming_edges
if d is not None and d.filename is not None))
logging.info(message)
# This check is done after the logging/printing above to make sure that
# we also print out the dependency edges that include python packages
# that are not in chromium.
if not path.IsSubpath(module_path, path_util.GetChromiumSrcDir()):
continue
# Exclude any dependencies which exist in the python installation.
if any(path.IsSubpath(module_path, pfx) for pfx in prefixes):
continue
yield module_path
if node.packagepath is not None:
for p in node.packagepath:
yield p
finally:
sys.path = sys_path
def current_benchmarks():
benchmarks_dir = os.path.join(
path_util.GetChromiumSrcDir(), 'tools', 'perf', 'benchmarks')
top_level_dir = os.path.dirname(benchmarks_dir)
all_benchmarks = []
for b in discover.DiscoverClasses(
benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
index_by_class_name=True).values():
if not b.Name() in _UNSCHEDULED_TELEMETRY_BENCHMARKS:
all_benchmarks.append(b)
return sorted(all_benchmarks, key=lambda b: b.Name())
def load_and_update_new_recipe_fyi_json():
tests = {}
filename = 'chromium.perf.fyi.json'
buildbot_dir = os.path.join(
path_util.GetChromiumSrcDir(), 'testing', 'buildbot')
fyi_filepath = os.path.join(buildbot_dir, filename)
with open(fyi_filepath) as fp_r:
tests = json.load(fp_r)
with open(fyi_filepath, 'w') as fp:
# We have loaded what is there, we want to update or add
# what we have listed here
get_new_recipe_testers(NEW_PERF_RECIPE_FYI_TESTERS, tests)
json.dump(tests, fp, indent=2, separators=(',', ': '), sort_keys=True)
fp.write('\n')
def __init__(self,
top_level_dir=None,
benchmark_dirs=None,
client_configs=None,
default_chrome_root=None):
if client_configs is None:
client_configs = [CLIENT_CONFIG_PATH]
if default_chrome_root is None:
default_chrome_root = path_util.GetChromiumSrcDir()
super(ChromiumConfig,
self).__init__(top_level_dir=top_level_dir,
benchmark_dirs=benchmark_dirs,
client_configs=client_configs,
default_chrome_root=default_chrome_root)
def _InstallNfcApk(self):
"""Installs the APK that allows VR tests to simulate a headset NFC scan."""
chromium_root = path_util.GetChromiumSrcDir()
# Find the most recently build APK
candidate_apks = []
for build_path in util.GetBuildDirectories(chromium_root):
apk_path = os.path.join(build_path, 'apks', 'VrNfcSimulator.apk')
if os.path.exists(apk_path):
last_changed = os.path.getmtime(apk_path)
candidate_apks.append((last_changed, apk_path))
if not candidate_apks:
raise RuntimeError(
'Could not find VrNfcSimulator.apk in a build output directory')
newest_apk_path = sorted(candidate_apks)[-1][1]
self._platform.InstallApplication(
os.path.join(chromium_root, newest_apk_path))
def MakeHistogramSetWithDiagnostics(histograms_file, test_name, bot,
buildername, buildnumber, revisions_dict,
is_reference_build,
perf_dashboard_machine_group):
add_diagnostics_args = []
add_diagnostics_args.extend([
'--benchmarks',
test_name,
'--bots',
bot,
'--builds',
buildnumber,
'--masters',
perf_dashboard_machine_group,
'--is_reference_build',
'true' if is_reference_build else '',
])
url = _MakeStdioUrl(test_name, buildername, buildnumber)
if url:
add_diagnostics_args.extend(['--log_urls', url])
for k, v in revisions_dict.iteritems():
add_diagnostics_args.extend((k, v))
add_diagnostics_args.append(histograms_file)
# Subprocess only accepts string args
add_diagnostics_args = [str(v) for v in add_diagnostics_args]
add_reserved_diagnostics_path = os.path.join(path_util.GetChromiumSrcDir(),
'third_party', 'catapult',
'tracing', 'bin',
'add_reserved_diagnostics')
cmd = [sys.executable, add_reserved_diagnostics_path
] + add_diagnostics_args
subprocess.call(cmd)
# TODO: Handle reference builds
with open(histograms_file) as f:
hs = json.load(f)
return hs
def load_and_update_new_recipe_json():
tests = {}
filename = 'chromium.perf.fyi.json'
buildbot_dir = os.path.join(
path_util.GetChromiumSrcDir(), 'testing', 'buildbot')
fyi_filepath = os.path.join(buildbot_dir, filename)
with open(fyi_filepath) as fp_r:
tests = json.load(fp_r)
with open(fyi_filepath, 'w') as fp:
# We have loaded what is there, we want to update or add
# what we have listed here
testers = NEW_PERF_RECIPE_FYI_TESTERS
for tester, tester_config in testers['testers'].iteritems():
isolated_scripts = [generate_performance_test_suite(tester_config)]
tests[tester] = {
'isolated_scripts': sorted(isolated_scripts, key=lambda x: x['name'])
}
json.dump(tests, fp, indent=2, separators=(',', ': '), sort_keys=True)
fp.write('\n')
def get_sorted_benchmark_list_by_time(all_benchmarks):
runtime_list = []
benchmark_avgs = {}
timing_file_path = os.path.join(
path_util.GetChromiumSrcDir(), 'tools', 'perf', 'core',
'desktop_benchmark_avg_times.json')
# Load in the avg times as calculated on Nov 1st, 2016
with open(timing_file_path) as f:
benchmark_avgs = json.load(f)
for benchmark in all_benchmarks:
benchmark_avg_time = benchmark_avgs.get(benchmark.Name(), None)
assert benchmark_avg_time
# Need to multiple the seconds by 2 since we will be generating two tests
# for each benchmark to be run on the same shard for the reference build
runtime_list.append((benchmark, benchmark_avg_time * 2.0))
# Return a reverse sorted list by runtime
runtime_list.sort(key=lambda tup: tup[1], reverse=True)
return runtime_list
def _GuessTraceProcessorPath():
"""Return path to trace processor binary.
When we run on bots, there's only one build directory, so we just return
the path to trace processor binary located in that directory. Otherwise
we don't guess, but leave it to the user to supply a path.
"""
build_dirs = ['build', 'out', 'xcodebuild']
build_types = ['Debug', 'Debug_x64', 'Release', 'Release_x64', 'Default']
candidate_paths = []
for build_dir in build_dirs:
for build_type in build_types:
candidate_path = os.path.join(path_util.GetChromiumSrcDir(),
build_dir, build_type,
trace_processor.TP_BINARY_NAME)
if os.path.isfile(candidate_path):
candidate_paths.append(candidate_path)
if len(candidate_paths) == 1:
return candidate_paths[0]
else:
return None
