def GetReplayBrowserStartupArgs(self):
replay_args = []
network_backend = self.platform_backend.network_controller_backend
if not network_backend.is_open:
return []
proxy_port = network_backend.forwarder.port_pair.remote_port
replay_args.append('--proxy-server=socks://localhost:%s' % proxy_port)
if not self.is_webview:
# Ignore certificate errors for certs that are signed with Wpr's root.
# For more details on this flag, see crbug.com/753948.
wpr_public_hash_file = os.path.join(util.GetCatapultDir(),
'web_page_replay_go',
'wpr_public_hash.txt')
if not os.path.exists(wpr_public_hash_file):
raise exceptions.PathMissingError('Unable to find %s' %
wpr_public_hash_file)
with open(wpr_public_hash_file) as f:
wpr_public_hash = f.readline().strip()
replay_args.append('--ignore-certificate-errors-spki-list=' +
wpr_public_hash)
elif self.is_webview:
# --ignore-certificate-errors-spki-list doesn't work with webview yet
# (crbug.com/753948)
replay_args.append('--ignore-certificate-errors')
return replay_args
def StartSymbolizerForProcessIfPossible(input_file, output_file,
build_id_file):
"""Starts an llvm symbolizer process if possible.
Args:
input_file: Input file to be symbolized.
output_file: Output file for symbolizer stdout and stderr.
build_id_file: Path to the ids.txt file which maps build IDs to
unstripped binaries on the filesystem.
Returns:
A subprocess.Popen object for the started process, None if symbolizer
fails to start."""
if (os.path.isfile(_LLVM_SYMBOLIZER_PATH)
and os.path.isfile(build_id_file)):
sdk_root = os.path.join(util.GetCatapultDir(), '..', 'fuchsia-sdk',
'sdk')
symbolizer = os.path.join(sdk_root, 'tools', 'x64', 'symbolize')
symbolizer_cmd = [
symbolizer, '-ids-rel', '-llvm-symbolizer', _LLVM_SYMBOLIZER_PATH,
'-build-id-dir',
os.path.join(sdk_root, '.build-id')
]
symbolizer_cmd.extend(['-ids', build_id_file])
logging.debug('Running "%s".' % ' '.join(symbolizer_cmd))
return subprocess.Popen(symbolizer_cmd,
stdin=input_file,
stdout=output_file,
stderr=subprocess.STDOUT,
close_fds=True)
else:
logging.info('Symbolizer cannot be started.')
return None
def GetReplayArgs(network_backend, supports_spki_list=True):
args = []
if not network_backend.is_open:
return args
proxy_port = network_backend.forwarder.remote_port
args.append('--proxy-server=socks://localhost:%s' % proxy_port)
if not network_backend.use_live_traffic:
if supports_spki_list:
# Ignore certificate errors for certs that are signed with Wpr's root.
# For more details on this flag, see crbug.com/753948.
wpr_public_hash_file = os.path.join(util.GetCatapultDir(),
'web_page_replay_go',
'wpr_public_hash.txt')
if not os.path.exists(wpr_public_hash_file):
raise exceptions.PathMissingError('Unable to find %s' %
wpr_public_hash_file)
with open(wpr_public_hash_file) as f:
wpr_public_hash = f.readline().strip()
args.append('--ignore-certificate-errors-spki-list=' +
wpr_public_hash)
else:
# If --ignore-certificate-errors-spki-list is not supported ignore all
# certificate errors.
args.append('--ignore-certificate-errors')
return args
def build_go_binary(binary_name, os_name, os_arch):
""" Build and return path to wpr go binary."""
# go build command recognizes 'amd64' but not 'x86_64', so we switch 'x86_64'
# to 'amd64' string here.
# The two names can be used interchangbly, see:
# https://wiki.debian.org/DebianAMD64Faq?action=recall&rev=65
if os_arch == 'x86_64' or os_arch == 'AMD64':
os_arch = 'amd64'
if os_arch == 'x86':
os_arch = '386'
if os_arch == 'armv7l':
os_arch = 'arm'
if os_arch == 'aarch64':
os_arch = 'arm64'
if os_arch == 'mips':
os_arch = 'mipsle'
# go build command recognizes 'darwin' but not 'mac'.
if os_name == 'mac':
os_name = 'darwin'
if os_name == 'win':
os_name = 'windows'
check_go_version()
try:
# We want to build wpr go binaries from the local source. We do this by
# making a temporary GOPATH that symlinks to our local directory.
go_path_dir = tempfile.mkdtemp()
repo_dir = os.path.join(go_path_dir, 'src/github.com/catapult-project')
os.makedirs(repo_dir)
os.symlink(util.GetCatapultDir(), os.path.join(repo_dir, 'catapult'))
env = os.environ.copy()
env['GOPATH'] = go_path_dir
env['GOOS'] = os_name
env['GOARCH'] = os_arch
env['CGO_ENABLED'] = '0'
print 'GOPATH=%s' % go_path_dir
print 'CWD=%s' % _WPR_GO_DIR
get_cmd = ['go', 'get', '-d', './...']
print 'Running get command: %s' % ' '.join(get_cmd)
subprocess.check_call(get_cmd, env=env, cwd=_WPR_GO_DIR)
build_cmd = ['go', 'build', '-v', '%s.go' % binary_name]
print 'Running build command: %s' % ' '.join(build_cmd)
subprocess.check_call(build_cmd, env=env, cwd=_WPR_GO_DIR)
cp_cmd = ['cp', 'wpr', '/output/']
subprocess.check_call(cp_cmd, env=env, cwd=_WPR_GO_DIR)
clean_cmd = ['go', 'clean', '-modcache']
print 'Running clean command: %s' % ' '.join(clean_cmd)
subprocess.check_call(clean_cmd, env=env, cwd=_WPR_GO_DIR)
finally:
if go_path_dir:
shutil.rmtree(go_path_dir)
if os_name == 'windows':
return os.path.join(_WPR_GO_DIR, '%s.exe' % binary_name)
return os.path.join(_WPR_GO_DIR, binary_name)
def _DownloadFuchsiaSDK(tar_file, dest=_SDK_ROOT_IN_CATAPULT):
if not os.path.isdir(dest):
os.makedirs(dest)
gsutil_path = os.path.join(util.GetCatapultDir(), 'third_party', 'gsutil',
'gsutil')
sdk_pkg = 'gs://fuchsia/sdk/core/linux-amd64/' + _SDK_SHA1
download_cmd = [gsutil_path, 'cp', sdk_pkg, tar_file]
subprocess.check_output(download_cmd, stderr=subprocess.STDOUT)
with tarfile.open(tar_file, 'r') as tar:
for f in _SDK_TOOLS:
# tarfile only accepts POSIX paths.
tar.extract(f.replace(os.path.sep, '/'), dest)
os.remove(tar_file)
def GetReplayArgs(network_backend, supports_spki_list=True):
args = []
if not network_backend.is_open:
return args
# Send all browser traffic (including requests to 127.0.0.1 and localhost) to
# ts_proxy_server.
# The proxy should NOT be set to "localhost", otherwise Chrome will first
# attempt to use the IPv6 version (::1) before falling back to IPv4. This
# causes issues if the IPv4 port we got randomly assigned on the device is
# also being used in IPv6 by some other process. See
# https://crbug.com/1005971 for more information.
proxy_port = network_backend.forwarder.remote_port
args.append('--proxy-server=socks://127.0.0.1:%s' % proxy_port)
args.append('--proxy-bypass-list=<-loopback>')
if not network_backend.use_live_traffic:
if supports_spki_list:
# Ignore certificate errors for certs that are signed with Wpr's root.
# For more details on this flag, see crbug.com/753948.
wpr_public_hash_file = os.path.join(util.GetCatapultDir(),
'web_page_replay_go',
'wpr_public_hash.txt')
if not os.path.exists(wpr_public_hash_file):
raise exceptions.PathMissingError('Unable to find %s' %
wpr_public_hash_file)
with open(wpr_public_hash_file) as f:
wpr_public_hash = f.readline().strip()
args.append('--ignore-certificate-errors-spki-list=' +
wpr_public_hash)
else:
# If --ignore-certificate-errors-spki-list is not supported ignore all
# certificate errors.
args.append('--ignore-certificate-errors')
return args
"""A Fuchsia device instance"""
import logging
import os
import platform
import subprocess
import tarfile
from telemetry.core import fuchsia_interface
from telemetry.core import util
from telemetry.internal.platform import device
from telemetry.util import cmd_util
_LIST_DEVICES_TIMEOUT_SECS = 5
_SDK_SHA1 = '8894838554076535504'
_SDK_ROOT_IN_CATAPULT = os.path.join(util.GetCatapultDir(), 'third_party',
'fuchsia-sdk', 'sdk')
_SDK_ROOT_IN_CHROMIUM = os.path.join(util.GetCatapultDir(), '..',
'fuchsia-sdk', 'sdk')
_SDK_TOOLS = [
os.path.join('tools', 'dev_finder'),
os.path.join('tools', 'symbolize')
]
class FuchsiaDevice(device.Device):
def __init__(self, target_name, host, output_dir, system_log_file, port):
super(FuchsiaDevice, self).__init__(name='Fuchsia with host localhost',
guid='fuchsia:%s' % target_name)
self._target_name = target_name
self._output_dir = output_dir
import re
import signal
import subprocess
import sys
import tempfile
import urllib
from telemetry.core import util
from telemetry.internal.util import binary_manager
import py_utils
from py_utils import atexit_with_log
_WPR_DIR = os.path.abspath(os.path.join(
util.GetCatapultDir(), 'web_page_replay_go'))
class ReplayError(Exception):
"""Catch-all exception for the module."""
pass
class ReplayNotFoundError(ReplayError):
def __init__(self, label, path):
super(ReplayNotFoundError, self).__init__()
self.args = (label, path)
def __str__(self):
label, path = self.args
return 'Path does not exist for %s: %s' % (label, path)
import os
from catapult_base import binary_manager
from dependency_manager import base_config
from dependency_manager import exceptions as dependency_manager_exceptions
from devil import devil_env
from telemetry.core import exceptions
from telemetry.core import util
TELEMETRY_PROJECT_CONFIG = os.path.join(
util.GetTelemetryDir(), 'telemetry', 'internal', 'binary_dependencies.json')
CHROME_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'catapult_base',
'catapult_base', 'chrome_binaries.json')
NoPathFoundError = dependency_manager_exceptions.NoPathFoundError
CloudStorageError = dependency_manager_exceptions.CloudStorageError
_binary_manager = None
def NeedsInit():
return not _binary_manager
def InitDependencyManager(environment_config):
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import copy
import json
import logging
import os
import shutil
import subprocess
import tempfile
from telemetry.core import util
_TRACE2HTML_PATH = os.path.join(util.GetCatapultDir(), 'tracing', 'bin',
'trace2html')
class NonSerializableTraceData(Exception):
"""Raised when raw trace data cannot be serialized to TraceData."""
pass
class TraceDataPart(object):
"""TraceData can have a variety of events.
These are called "parts" and are accessed by the following fixed field names.
"""
def __init__(self, raw_field_name):
self._raw_field_name = raw_field_name
import logging
import os
from py_utils import binary_manager
from py_utils import dependency_util
import dependency_manager
from devil import devil_env
from telemetry.core import exceptions
from telemetry.core import util
from telemetry.core import platform as platform_module
TELEMETRY_PROJECT_CONFIG = os.path.join(util.GetTelemetryDir(), 'telemetry',
'internal', 'binary_dependencies.json')
CHROME_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'common',
'py_utils', 'py_utils',
'chrome_binaries.json')
BATTOR_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'common', 'battor',
'battor',
'battor_binary_dependencies.json')
NoPathFoundError = dependency_manager.NoPathFoundError
CloudStorageError = dependency_manager.CloudStorageError
_binary_manager = None
def NeedsInit():
return not _binary_manager
from py_utils import dependency_util
import dependency_manager
from dependency_manager import base_config
from devil import devil_env
from telemetry.core import exceptions
from telemetry.core import util
TELEMETRY_PROJECT_CONFIG = os.path.join(
util.GetTelemetryDir(), 'telemetry', 'binary_dependencies.json')
CHROME_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'common', 'py_utils',
'py_utils', 'chrome_binaries.json')
SUPPORTED_DEP_PLATFORMS = (
'linux_aarch64', 'linux_x86_64', 'linux_armv7l', 'linux_mips',
'mac_x86_64',
'win_x86', 'win_AMD64',
'android_arm64-v8a', 'android_armeabi-v7a', 'android_arm', 'android_x64',
'android_x86'
)
PLATFORMS_TO_DOWNLOAD_FOLDER_MAP = {
'linux_aarch64': 'bin/linux/aarch64',
'linux_x86_64': 'bin/linux/x86_64',
'linux_armv7l': 'bin/linux/armv7l',
def __init__(self, browser_backend, platform_backend, output_path, state,
presentation.device=None):
super(AndroidSystraceProfiler, self).__init__(
browser_backend, platform_backend, output_path, state)
assert self._browser_backend.supports_tracing
self._output_path = output_path + '-trace.zip'
self._systrace_output_path = output_path + '.systrace'
# Use telemetry's own tracing backend instead the combined mode in
# adb_profile_chrome because some benchmarks also do tracing of their own
# and the two methods conflict.
config = tracing_config.TracingConfig()
config.enable_chrome_trace = True
self._browser_backend.StartTracing(config, timeout=10)
command = ['python', os.path.join(util.GetCatapultDir(), 'systrace', 'bin',
'adb_profile_chrome'),
'--categories', '', '--continuous', '--output',
self._systrace_output_path, '--json', '--systrace',
','.join(_SYSTRACE_CATEGORIES)]
if presentation.device:
command.extend(['--presentation.device', presentation.device])
self._profiler = subprocess.Popen(command, stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
@classmethod
def name(cls):
return 'android-systrace'
@classmethod
def is_supported(cls, browser_type):
import shutil
import sys
import subprocess
import tempfile
import platform
TELEMETRY_DIR = os.path.join(
'/root/go/src/github.com/catapult-project/catapult/telemetry')
sys.path.insert(1, TELEMETRY_DIR)
from telemetry.core import util
from telemetry.core import platform as platform_module
from telemetry.internal.util import binary_manager
import py_utils
_WPR_GO_DIR = os.path.join(util.GetCatapultDir(), 'web_page_replay_go', 'src')
_SUPPORTED_GO_VERSIONS = ('go1.8', 'go1.9')
def check_go_version():
try:
out = subprocess.check_output(['go', 'version'])
except subprocess.CalledProcessError:
out = 'no go binary found'
assert any(v in out for v in _SUPPORTED_GO_VERSIONS), (
'Require go version 1.8 or higher. Found: %s' % out)
def build_wpr_go(os_name, os_arch):
""" Build and return path to wpr binary."""
# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A wrapper for common operations on a Fuchsia device"""
import logging
import os
import subprocess
from telemetry.core import util
FUCHSIA_BROWSERS = ['web-engine-shell']
_LLVM_SYMBOLIZER_PATH = os.path.join(
util.GetCatapultDir(), '..', 'llvm-build', 'Release+Asserts', 'bin',
'llvm-symbolizer')
class CommandRunner(object):
"""Helper class used to execute commands on Fuchsia devices on a remote host
over SSH."""
def __init__(self, config_path, host, port):
"""Creates a CommandRunner that connects to the specified |host| and |port|
using the ssh config at the specified |config_path|.
Args:
config_path: Full path to SSH configuration.
host: The hostname or IP address of the remote host.
port: The port to connect to."""
self._config_path = config_path
self._host = host
import logging
import os
import re
import signal
import subprocess
import sys
import tempfile
import urllib
from telemetry.core import util
from telemetry.internal.util import binary_manager
import py_utils
_WPR_DIR = os.path.abspath(
os.path.join(util.GetCatapultDir(), 'web_page_replay_go'))
class ReplayError(Exception):
"""Catch-all exception for the module."""
pass
class ReplayNotFoundError(ReplayError):
def __init__(self, label, path):
super(ReplayNotFoundError, self).__init__()
self.args = (label, path)
def __str__(self):
label, path = self.args
return 'Path does not exist for %s: %s' % (label, path)
# Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A wrapper for common operations on a Fuchsia device"""
import logging
import os
import subprocess
from telemetry.core import util
FUCHSIA_BROWSERS = ['web-engine-shell']
_LLVM_SYMBOLIZER_PATH = os.path.join(util.GetCatapultDir(), '..', 'llvm-build',
'Release+Asserts', 'bin',
'llvm-symbolizer')
class CommandRunner(object):
"""Helper class used to execute commands on Fuchsia devices on a remote host
over SSH."""
def __init__(self, config_path, host, port):
"""Creates a CommandRunner that connects to the specified |host| and |port|
using the ssh config at the specified |config_path|.
Args:
config_path: Full path to SSH configuration.
host: The hostname or IP address of the remote host.
port: The port to connect to."""
self._config_path = config_path
self._host = host
self._port = port
