def GetReplayBrowserStartupArgs(self): replay_args = [] network_backend = self.platform_backend.network_controller_backend if not network_backend.is_open: return [] proxy_port = network_backend.forwarder.port_pair.remote_port replay_args.append('--proxy-server=socks://localhost:%s' % proxy_port) if not self.is_webview: # Ignore certificate errors for certs that are signed with Wpr's root. # For more details on this flag, see crbug.com/753948. wpr_public_hash_file = os.path.join(util.GetCatapultDir(), 'web_page_replay_go', 'wpr_public_hash.txt') if not os.path.exists(wpr_public_hash_file): raise exceptions.PathMissingError('Unable to find %s' % wpr_public_hash_file) with open(wpr_public_hash_file) as f: wpr_public_hash = f.readline().strip() replay_args.append('--ignore-certificate-errors-spki-list=' + wpr_public_hash) elif self.is_webview: # --ignore-certificate-errors-spki-list doesn't work with webview yet # (crbug.com/753948) replay_args.append('--ignore-certificate-errors') return replay_args
def StartSymbolizerForProcessIfPossible(input_file, output_file, build_id_file): """Starts an llvm symbolizer process if possible. Args: input_file: Input file to be symbolized. output_file: Output file for symbolizer stdout and stderr. build_id_file: Path to the ids.txt file which maps build IDs to unstripped binaries on the filesystem. Returns: A subprocess.Popen object for the started process, None if symbolizer fails to start.""" if (os.path.isfile(_LLVM_SYMBOLIZER_PATH) and os.path.isfile(build_id_file)): sdk_root = os.path.join(util.GetCatapultDir(), '..', 'fuchsia-sdk', 'sdk') symbolizer = os.path.join(sdk_root, 'tools', 'x64', 'symbolize') symbolizer_cmd = [ symbolizer, '-ids-rel', '-llvm-symbolizer', _LLVM_SYMBOLIZER_PATH, '-build-id-dir', os.path.join(sdk_root, '.build-id') ] symbolizer_cmd.extend(['-ids', build_id_file]) logging.debug('Running "%s".' % ' '.join(symbolizer_cmd)) return subprocess.Popen(symbolizer_cmd, stdin=input_file, stdout=output_file, stderr=subprocess.STDOUT, close_fds=True) else: logging.info('Symbolizer cannot be started.') return None
def GetReplayArgs(network_backend, supports_spki_list=True): args = [] if not network_backend.is_open: return args proxy_port = network_backend.forwarder.remote_port args.append('--proxy-server=socks://localhost:%s' % proxy_port) if not network_backend.use_live_traffic: if supports_spki_list: # Ignore certificate errors for certs that are signed with Wpr's root. # For more details on this flag, see crbug.com/753948. wpr_public_hash_file = os.path.join(util.GetCatapultDir(), 'web_page_replay_go', 'wpr_public_hash.txt') if not os.path.exists(wpr_public_hash_file): raise exceptions.PathMissingError('Unable to find %s' % wpr_public_hash_file) with open(wpr_public_hash_file) as f: wpr_public_hash = f.readline().strip() args.append('--ignore-certificate-errors-spki-list=' + wpr_public_hash) else: # If --ignore-certificate-errors-spki-list is not supported ignore all # certificate errors. args.append('--ignore-certificate-errors') return args
def build_go_binary(binary_name, os_name, os_arch): """ Build and return path to wpr go binary.""" # go build command recognizes 'amd64' but not 'x86_64', so we switch 'x86_64' # to 'amd64' string here. # The two names can be used interchangbly, see: # https://wiki.debian.org/DebianAMD64Faq?action=recall&rev=65 if os_arch == 'x86_64' or os_arch == 'AMD64': os_arch = 'amd64' if os_arch == 'x86': os_arch = '386' if os_arch == 'armv7l': os_arch = 'arm' if os_arch == 'aarch64': os_arch = 'arm64' if os_arch == 'mips': os_arch = 'mipsle' # go build command recognizes 'darwin' but not 'mac'. if os_name == 'mac': os_name = 'darwin' if os_name == 'win': os_name = 'windows' check_go_version() try: # We want to build wpr go binaries from the local source. We do this by # making a temporary GOPATH that symlinks to our local directory. go_path_dir = tempfile.mkdtemp() repo_dir = os.path.join(go_path_dir, 'src/github.com/catapult-project') os.makedirs(repo_dir) os.symlink(util.GetCatapultDir(), os.path.join(repo_dir, 'catapult')) env = os.environ.copy() env['GOPATH'] = go_path_dir env['GOOS'] = os_name env['GOARCH'] = os_arch env['CGO_ENABLED'] = '0' print 'GOPATH=%s' % go_path_dir print 'CWD=%s' % _WPR_GO_DIR get_cmd = ['go', 'get', '-d', './...'] print 'Running get command: %s' % ' '.join(get_cmd) subprocess.check_call(get_cmd, env=env, cwd=_WPR_GO_DIR) build_cmd = ['go', 'build', '-v', '%s.go' % binary_name] print 'Running build command: %s' % ' '.join(build_cmd) subprocess.check_call(build_cmd, env=env, cwd=_WPR_GO_DIR) cp_cmd = ['cp', 'wpr', '/output/'] subprocess.check_call(cp_cmd, env=env, cwd=_WPR_GO_DIR) clean_cmd = ['go', 'clean', '-modcache'] print 'Running clean command: %s' % ' '.join(clean_cmd) subprocess.check_call(clean_cmd, env=env, cwd=_WPR_GO_DIR) finally: if go_path_dir: shutil.rmtree(go_path_dir) if os_name == 'windows': return os.path.join(_WPR_GO_DIR, '%s.exe' % binary_name) return os.path.join(_WPR_GO_DIR, binary_name)
def _DownloadFuchsiaSDK(tar_file, dest=_SDK_ROOT_IN_CATAPULT): if not os.path.isdir(dest): os.makedirs(dest) gsutil_path = os.path.join(util.GetCatapultDir(), 'third_party', 'gsutil', 'gsutil') sdk_pkg = 'gs://fuchsia/sdk/core/linux-amd64/' + _SDK_SHA1 download_cmd = [gsutil_path, 'cp', sdk_pkg, tar_file] subprocess.check_output(download_cmd, stderr=subprocess.STDOUT) with tarfile.open(tar_file, 'r') as tar: for f in _SDK_TOOLS: # tarfile only accepts POSIX paths. tar.extract(f.replace(os.path.sep, '/'), dest) os.remove(tar_file)
def GetReplayArgs(network_backend, supports_spki_list=True): args = [] if not network_backend.is_open: return args # Send all browser traffic (including requests to 127.0.0.1 and localhost) to # ts_proxy_server. # The proxy should NOT be set to "localhost", otherwise Chrome will first # attempt to use the IPv6 version (::1) before falling back to IPv4. This # causes issues if the IPv4 port we got randomly assigned on the device is # also being used in IPv6 by some other process. See # https://crbug.com/1005971 for more information. proxy_port = network_backend.forwarder.remote_port args.append('--proxy-server=socks://127.0.0.1:%s' % proxy_port) args.append('--proxy-bypass-list=<-loopback>') if not network_backend.use_live_traffic: if supports_spki_list: # Ignore certificate errors for certs that are signed with Wpr's root. # For more details on this flag, see crbug.com/753948. wpr_public_hash_file = os.path.join(util.GetCatapultDir(), 'web_page_replay_go', 'wpr_public_hash.txt') if not os.path.exists(wpr_public_hash_file): raise exceptions.PathMissingError('Unable to find %s' % wpr_public_hash_file) with open(wpr_public_hash_file) as f: wpr_public_hash = f.readline().strip() args.append('--ignore-certificate-errors-spki-list=' + wpr_public_hash) else: # If --ignore-certificate-errors-spki-list is not supported ignore all # certificate errors. args.append('--ignore-certificate-errors') return args
"""A Fuchsia device instance""" import logging import os import platform import subprocess import tarfile from telemetry.core import fuchsia_interface from telemetry.core import util from telemetry.internal.platform import device from telemetry.util import cmd_util _LIST_DEVICES_TIMEOUT_SECS = 5 _SDK_SHA1 = '8894838554076535504' _SDK_ROOT_IN_CATAPULT = os.path.join(util.GetCatapultDir(), 'third_party', 'fuchsia-sdk', 'sdk') _SDK_ROOT_IN_CHROMIUM = os.path.join(util.GetCatapultDir(), '..', 'fuchsia-sdk', 'sdk') _SDK_TOOLS = [ os.path.join('tools', 'dev_finder'), os.path.join('tools', 'symbolize') ] class FuchsiaDevice(device.Device): def __init__(self, target_name, host, output_dir, system_log_file, port): super(FuchsiaDevice, self).__init__(name='Fuchsia with host localhost', guid='fuchsia:%s' % target_name) self._target_name = target_name self._output_dir = output_dir
import re import signal import subprocess import sys import tempfile import urllib from telemetry.core import util from telemetry.internal.util import binary_manager import py_utils from py_utils import atexit_with_log _WPR_DIR = os.path.abspath(os.path.join( util.GetCatapultDir(), 'web_page_replay_go')) class ReplayError(Exception): """Catch-all exception for the module.""" pass class ReplayNotFoundError(ReplayError): def __init__(self, label, path): super(ReplayNotFoundError, self).__init__() self.args = (label, path) def __str__(self): label, path = self.args return 'Path does not exist for %s: %s' % (label, path)
import os from catapult_base import binary_manager from dependency_manager import base_config from dependency_manager import exceptions as dependency_manager_exceptions from devil import devil_env from telemetry.core import exceptions from telemetry.core import util TELEMETRY_PROJECT_CONFIG = os.path.join( util.GetTelemetryDir(), 'telemetry', 'internal', 'binary_dependencies.json') CHROME_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'catapult_base', 'catapult_base', 'chrome_binaries.json') NoPathFoundError = dependency_manager_exceptions.NoPathFoundError CloudStorageError = dependency_manager_exceptions.CloudStorageError _binary_manager = None def NeedsInit(): return not _binary_manager def InitDependencyManager(environment_config):
# Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. import copy import json import logging import os import shutil import subprocess import tempfile from telemetry.core import util _TRACE2HTML_PATH = os.path.join(util.GetCatapultDir(), 'tracing', 'bin', 'trace2html') class NonSerializableTraceData(Exception): """Raised when raw trace data cannot be serialized to TraceData.""" pass class TraceDataPart(object): """TraceData can have a variety of events. These are called "parts" and are accessed by the following fixed field names. """ def __init__(self, raw_field_name): self._raw_field_name = raw_field_name
import logging import os from py_utils import binary_manager from py_utils import dependency_util import dependency_manager from devil import devil_env from telemetry.core import exceptions from telemetry.core import util from telemetry.core import platform as platform_module TELEMETRY_PROJECT_CONFIG = os.path.join(util.GetTelemetryDir(), 'telemetry', 'internal', 'binary_dependencies.json') CHROME_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'common', 'py_utils', 'py_utils', 'chrome_binaries.json') BATTOR_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'common', 'battor', 'battor', 'battor_binary_dependencies.json') NoPathFoundError = dependency_manager.NoPathFoundError CloudStorageError = dependency_manager.CloudStorageError _binary_manager = None def NeedsInit(): return not _binary_manager
from py_utils import dependency_util import dependency_manager from dependency_manager import base_config from devil import devil_env from telemetry.core import exceptions from telemetry.core import util TELEMETRY_PROJECT_CONFIG = os.path.join( util.GetTelemetryDir(), 'telemetry', 'binary_dependencies.json') CHROME_BINARY_CONFIG = os.path.join(util.GetCatapultDir(), 'common', 'py_utils', 'py_utils', 'chrome_binaries.json') SUPPORTED_DEP_PLATFORMS = ( 'linux_aarch64', 'linux_x86_64', 'linux_armv7l', 'linux_mips', 'mac_x86_64', 'win_x86', 'win_AMD64', 'android_arm64-v8a', 'android_armeabi-v7a', 'android_arm', 'android_x64', 'android_x86' ) PLATFORMS_TO_DOWNLOAD_FOLDER_MAP = { 'linux_aarch64': 'bin/linux/aarch64', 'linux_x86_64': 'bin/linux/x86_64', 'linux_armv7l': 'bin/linux/armv7l',
def __init__(self, browser_backend, platform_backend, output_path, state, presentation.device=None): super(AndroidSystraceProfiler, self).__init__( browser_backend, platform_backend, output_path, state) assert self._browser_backend.supports_tracing self._output_path = output_path + '-trace.zip' self._systrace_output_path = output_path + '.systrace' # Use telemetry's own tracing backend instead the combined mode in # adb_profile_chrome because some benchmarks also do tracing of their own # and the two methods conflict. config = tracing_config.TracingConfig() config.enable_chrome_trace = True self._browser_backend.StartTracing(config, timeout=10) command = ['python', os.path.join(util.GetCatapultDir(), 'systrace', 'bin', 'adb_profile_chrome'), '--categories', '', '--continuous', '--output', self._systrace_output_path, '--json', '--systrace', ','.join(_SYSTRACE_CATEGORIES)] if presentation.device: command.extend(['--presentation.device', presentation.device]) self._profiler = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE) @classmethod def name(cls): return 'android-systrace' @classmethod def is_supported(cls, browser_type):
import shutil import sys import subprocess import tempfile import platform TELEMETRY_DIR = os.path.join( '/root/go/src/github.com/catapult-project/catapult/telemetry') sys.path.insert(1, TELEMETRY_DIR) from telemetry.core import util from telemetry.core import platform as platform_module from telemetry.internal.util import binary_manager import py_utils _WPR_GO_DIR = os.path.join(util.GetCatapultDir(), 'web_page_replay_go', 'src') _SUPPORTED_GO_VERSIONS = ('go1.8', 'go1.9') def check_go_version(): try: out = subprocess.check_output(['go', 'version']) except subprocess.CalledProcessError: out = 'no go binary found' assert any(v in out for v in _SUPPORTED_GO_VERSIONS), ( 'Require go version 1.8 or higher. Found: %s' % out) def build_wpr_go(os_name, os_arch): """ Build and return path to wpr binary."""
# Copyright 2019 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """A wrapper for common operations on a Fuchsia device""" import logging import os import subprocess from telemetry.core import util FUCHSIA_BROWSERS = ['web-engine-shell'] _LLVM_SYMBOLIZER_PATH = os.path.join( util.GetCatapultDir(), '..', 'llvm-build', 'Release+Asserts', 'bin', 'llvm-symbolizer') class CommandRunner(object): """Helper class used to execute commands on Fuchsia devices on a remote host over SSH.""" def __init__(self, config_path, host, port): """Creates a CommandRunner that connects to the specified |host| and |port| using the ssh config at the specified |config_path|. Args: config_path: Full path to SSH configuration. host: The hostname or IP address of the remote host. port: The port to connect to.""" self._config_path = config_path self._host = host
import logging import os import re import signal import subprocess import sys import tempfile import urllib from telemetry.core import util from telemetry.internal.util import binary_manager import py_utils _WPR_DIR = os.path.abspath( os.path.join(util.GetCatapultDir(), 'web_page_replay_go')) class ReplayError(Exception): """Catch-all exception for the module.""" pass class ReplayNotFoundError(ReplayError): def __init__(self, label, path): super(ReplayNotFoundError, self).__init__() self.args = (label, path) def __str__(self): label, path = self.args return 'Path does not exist for %s: %s' % (label, path)
# Copyright 2019 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """A wrapper for common operations on a Fuchsia device""" import logging import os import subprocess from telemetry.core import util FUCHSIA_BROWSERS = ['web-engine-shell'] _LLVM_SYMBOLIZER_PATH = os.path.join(util.GetCatapultDir(), '..', 'llvm-build', 'Release+Asserts', 'bin', 'llvm-symbolizer') class CommandRunner(object): """Helper class used to execute commands on Fuchsia devices on a remote host over SSH.""" def __init__(self, config_path, host, port): """Creates a CommandRunner that connects to the specified |host| and |port| using the ssh config at the specified |config_path|. Args: config_path: Full path to SSH configuration. host: The hostname or IP address of the remote host. port: The port to connect to.""" self._config_path = config_path self._host = host self._port = port