def ParseArgs(args=None):
defaults = parser.get_default_values()
for k, v in defaults.__dict__.items():
if k in self.__dict__ and self.__dict__[k] != None:
continue
self.__dict__[k] = v
ret = real_parse(args, self) # pylint: disable=E1121
if self.verbosity >= 2:
logging.getLogger().setLevel(logging.DEBUG)
elif self.verbosity:
logging.getLogger().setLevel(logging.INFO)
else:
logging.getLogger().setLevel(logging.WARNING)
if self.chromium_output_dir:
os.environ['CHROMIUM_OUTPUT_DIR'] = self.chromium_output_dir
if self.device == 'list':
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager(None)
devices = device_finder.GetDevicesMatchingOptions(self)
print 'Available devices:'
for device in devices:
print ' ', device.name
sys.exit(0)
if self.browser_executable and not self.browser_type:
self.browser_type = 'exact'
if self.browser_type == 'list':
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager(None)
devices = device_finder.GetDevicesMatchingOptions(self)
if not devices:
sys.exit(0)
browser_types = {}
for device in devices:
try:
possible_browsers = browser_finder.GetAllAvailableBrowsers(
self, device)
browser_types[device.name] = sorted([
browser.browser_type
for browser in possible_browsers
])
except browser_finder_exceptions.BrowserFinderException as ex:
print >> sys.stderr, 'ERROR: ', ex
sys.exit(1)
print 'Available browsers:'
if len(browser_types) == 0:
print ' No devices were found.'
for device_name in sorted(browser_types.keys()):
print ' ', device_name
for browser_type in browser_types[device_name]:
print ' ', browser_type
sys.exit(0)
# Parse browser options.
self.browser_options.UpdateFromParseResults(self)
return ret
def _SetUpProcess(child, context): # pylint: disable=unused-argument
ps_util.EnableListingStrayProcessesUponExitHook()
# Make sure that we don't invokes cloud storage I/Os when we run the tests in
# parallel.
# TODO(nednguyen): always do this once telemetry tests in Chromium is updated
# to prefetch files.
# (https://github.com/catapult-project/catapult/issues/2192)
args = context
if args.disable_cloud_storage_io:
os.environ[cloud_storage.DISABLE_CLOUD_STORAGE_IO] = '1'
if binary_manager.NeedsInit():
# Typ doesn't keep the DependencyManager initialization in the child
# processes.
binary_manager.InitDependencyManager(context.client_configs)
# We need to reset the handlers in case some other parts of telemetry already
# set it to make this work.
if not args.disable_logging_config:
logging.getLogger().handlers = []
logging.basicConfig(
level=logging.INFO,
format='(%(levelname)s) %(asctime)s pid=%(process)d'
' %(module)s.%(funcName)s:%(lineno)d'
' %(message)s')
if args.remote_platform_options.device == 'android':
android_devices = android_device.FindAllAvailableDevices(args)
if not android_devices:
raise RuntimeError("No Android device found")
android_devices.sort(key=lambda device: device.name)
args.remote_platform_options.device = (
android_devices[child.worker_num - 1].guid)
options_for_unittests.Push(args)
def _GetGoBinaryPath(cls):
if not cls._go_binary_path:
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager(None)
cls._go_binary_path = binary_manager.FetchPath(
'wpr_go', py_utils.GetHostArchName(), py_utils.GetHostOsName())
return cls._go_binary_path
def _SetUpProcess(child, context): # pylint: disable=unused-argument
ps_util.EnableListingStrayProcessesUponExitHook()
if binary_manager.NeedsInit():
# Typ doesn't keep the DependencyManager initialization in the child
# processes.
binary_manager.InitDependencyManager(context.client_config)
# We need to reset the handlers in case some other parts of telemetry already
# set it to make this work.
logging.getLogger().handlers = []
logging.basicConfig(
level=logging.INFO,
format='(%(levelname)s) %(asctime)s %(module)s.%(funcName)s:%(lineno)d '
'%(message)s')
args = context
if not args.disable_logging_config:
logging.getLogger().handlers = []
logging.basicConfig(
level=logging.INFO,
format=
'(%(levelname)s) %(asctime)s %(module)s.%(funcName)s:%(lineno)d'
' %(message)s')
if args.device and args.device == 'android':
android_devices = android_device.FindAllAvailableDevices(args)
if not android_devices:
raise RuntimeError("No Android device found")
android_devices.sort(key=lambda device: device.name)
args.device = android_devices[child.worker_num - 1].guid
options_for_unittests.Push(args)
def main(environment, argv=None):
# The log level is set in browser_options.
# Clear the log handlers to ensure we can set up logging properly here.
logging.getLogger().handlers = []
logging.basicConfig(format=DEFAULT_LOG_FORMAT)
ps_util.EnableListingStrayProcessesUponExitHook()
if argv is None:
argv = sys.argv
# Get the command name from the command line.
if len(argv) > 1 and argv[1] == '--help':
argv[1] = 'help'
command_name = 'run'
for arg in argv[1:]:
if not arg.startswith('-'):
command_name = arg
break
# TODO(eakuefner): Remove this hack after we port to argparse.
if command_name == 'help' and len(argv) > 2 and argv[2] == 'run':
command_name = 'run'
argv[2] = '--help'
# Validate and interpret the command name.
commands = _MatchingCommands(command_name)
if len(commands) > 1:
print >> sys.stderr, (
'"%s" is not a %s command. Did you mean one of these?' %
(command_name, _ScriptName()))
for command in commands:
print >> sys.stderr, ' %-10s %s' % (command.Name(),
command.Description())
return 1
if commands:
command = commands[0]
else:
command = Run
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager(environment.client_configs)
# Parse and run the command.
parser = command.CreateParser()
command.AddCommandLineArgs(parser, environment)
# Set the default chrome root variable.
parser.set_defaults(chrome_root=environment.default_chrome_root)
options, args = parser.parse_args(argv[1:])
if commands:
args = args[1:]
options.positional_args = args
command.ProcessCommandLineArgs(parser, options, environment)
return command().Run(options)
def FetchTelemetryDependencies(
platform=None, client_configs=None, chrome_reference_browser=False):
if not platform:
platform = platform_module.GetHostPlatform()
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager(client_configs)
else:
raise Exception('Binary manager already initialized with other configs.')
binary_manager.FetchBinaryDependencies(
platform, client_configs, chrome_reference_browser)
def _BringUpWpr(self):
"""Start the WPR server on the host and the forwarder on the device."""
print 'Starting WPR on host...'
_DownloadFromCloudStorage(self._WPR_BUCKET, self._wpr_archive_hash)
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager([])
self._wpr_server = webpagereplay_go_server.ReplayServer(
self._wpr_archive, '127.0.0.1', 0, 0, replay_options=[])
ports = self._wpr_server.StartServer()[:-1]
self._host_http_port = ports[0]
self._host_https_port = ports[1]
def _SetUpProcess(child, context):
args = context.finder_options
if binary_manager.NeedsInit():
# On windows, typ doesn't keep the DependencyManager initialization in the
# child processes.
binary_manager.InitDependencyManager(context.client_configs)
if args.remote_platform_options.device == 'android':
android_devices = android_device.FindAllAvailableDevices(args)
if not android_devices:
raise RuntimeError("No Android device found")
android_devices.sort(key=lambda device: device.name)
args.remote_platform_options.device = (
android_devices[child.worker_num - 1].guid)
browser_test_context._global_test_context = context
context.test_class.SetUpProcess()
def __init__(self, archive_path, replay_host, http_port, https_port,
replay_options):
"""Initialize ReplayServer.
Args:
archive_path: a path to a specific WPR archive (required).
replay_host: the hostname to serve traffic.
http_port: an integer port on which to serve HTTP traffic. May be zero
to let the OS choose an available port.
https_port: an integer port on which to serve HTTPS traffic. May be zero
to let the OS choose an available port.
replay_options: an iterable of option strings to forward to replay.py
"""
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager(None)
self._wpr_server = webpagereplay_go_server.ReplayServer(
archive_path, replay_host, http_port, https_port, replay_options,
binary_manager.FetchPath)
def _FetchBinaryManagerPath(self, dependency_name):
"""Fetches the path for the dependency via binary_manager.
Initializes binary_manager with defaults if it is not already initialized.
Args:
dependency_name: A string containing the name of the dependency.
Returns:
A string containing the path to the dependency, or None if it could not be
found.
"""
if binary_manager.NeedsInit():
logging.info(
'binary_manager was not initialized. Initializing with default '
'values.')
binary_manager.InitDependencyManager(None)
return binary_manager.FetchPath(dependency_name, self._os, self._arch,
self._os_version)
def ParseArgs(args=None):
defaults = parser.get_default_values()
for k, v in defaults.__dict__.items():
if k in self.__dict__ and self.__dict__[k] != None:
continue
self.__dict__[k] = v
ret = real_parse(args, self) # pylint: disable=E1121
if self.chromium_output_dir:
os.environ['CHROMIUM_OUTPUT_DIR'] = self.chromium_output_dir
# Parse remote platform options.
self.BuildRemotePlatformOptions()
if self.remote_platform_options.device == 'list':
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager([])
devices = device_finder.GetDevicesMatchingOptions(self)
print 'Available devices:'
for device in devices:
print ' ', device.name
sys.exit(0)
if self.browser_executable and not self.browser_type:
self.browser_type = 'exact'
if self.browser_type == 'list':
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager([])
devices = device_finder.GetDevicesMatchingOptions(self)
if not devices:
sys.exit(0)
browser_types = {}
for device in devices:
try:
possible_browsers = browser_finder.GetAllAvailableBrowsers(
self, device)
browser_types[device.name] = sorted([
browser.browser_type
for browser in possible_browsers
])
except browser_finder_exceptions.BrowserFinderException as ex:
print >> sys.stderr, 'ERROR: ', ex
sys.exit(1)
print 'Available browsers:'
if len(browser_types) == 0:
print ' No devices were found.'
for device_name in sorted(browser_types.keys()):
print ' ', device_name
for browser_type in browser_types[device_name]:
print ' ', browser_type
if len(browser_types[device_name]) == 0:
print ' No browsers found for this device'
sys.exit(0)
# Profiling other periods along with the story_run period leads to running
# multiple profiling processes at the same time. The effects of performing
# muliple CPU profiling at the same time is unclear and may generate
# incorrect profiles so this will not be supported.
if (len(self.interval_profiling_periods) > 1
and 'story_run' in self.interval_profiling_periods):
print 'Cannot specify other periods along with the story_run period.'
sys.exit(1)
self.interval_profiler_options = shlex.split(
self.interval_profiler_options)
# Parse browser options.
self.browser_options.UpdateFromParseResults(self)
return ret
def _PrepareDevice(device):
if not 'BUILDTYPE' in os.environ:
os.environ['BUILDTYPE'] = 'Release'
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager(None)
return android_profiling_helper.PrepareDeviceForPerf(device)
self._perf_binary = self._PrepareDevice(presentation.device)
self._perf_instance = None
self._categories = None
def __repr__(self):
return 'perf profile'
@staticmethod
def IsSupported():
return bool(android_profiling_helper)
@staticmethod
def _PrepareDevice(presentation.device):
if not 'BUILDTYPE' in os.environ:
os.environ['BUILDTYPE'] = 'Release'
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager(None)
return android_profiling_helper.PrepareDeviceForPerf(presentation.device)
@classmethod
def GetCategories(cls, presentation.device):
perf_binary = cls._PrepareDevice(presentation.device)
# Perf binary returns non-zero exit status on "list" command.
return presentation.device.RunShellCommand([perf_binary, 'list'], check_return=False)
@py_utils.Timeout(tracing_agents.START_STOP_TIMEOUT)
def StartAgentTracing(self, config, timeout=None):
self._categories = _ComputePerfCategories(config)
self._perf_instance = _PerfProfiler(self._device,
self._perf_binary,
self._categories)
def ParseArgs(args=None):
defaults = parser.get_default_values()
for k, v in defaults.__dict__.items():
if k in self.__dict__ and self.__dict__[k] != None:
continue
self.__dict__[k] = v
ret = real_parse(args, self) # pylint: disable=E1121
if self.chromium_output_dir:
os.environ['CHROMIUM_OUTPUT_DIR'] = self.chromium_output_dir
# Set up Android emulator if necessary.
self.ParseAndroidEmulatorOptions()
# Parse remote platform options.
self.BuildRemotePlatformOptions()
if self.remote_platform_options.device == 'list':
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager([])
devices = device_finder.GetDevicesMatchingOptions(self)
print 'Available devices:'
for device in devices:
print ' ', device.name
sys.exit(0)
if self.browser_executable and not self.browser_type:
self.browser_type = 'exact'
if self.browser_type == 'list':
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager([])
devices = device_finder.GetDevicesMatchingOptions(self)
if not devices:
sys.exit(0)
browser_types = {}
for device in devices:
try:
possible_browsers = browser_finder.GetAllAvailableBrowsers(
self, device)
browser_types[device.name] = sorted([
browser.browser_type
for browser in possible_browsers
])
except browser_finder_exceptions.BrowserFinderException as ex:
print >> sys.stderr, 'ERROR: ', ex
sys.exit(1)
print 'Available browsers:'
if len(browser_types) == 0:
print ' No devices were found.'
for device_name in sorted(browser_types.keys()):
print ' ', device_name
for browser_type in browser_types[device_name]:
print ' ', browser_type
if len(browser_types[device_name]) == 0:
print ' No browsers found for this device'
sys.exit(0)
if ((self.browser_type == 'cros-chrome'
or self.browser_type == 'lacros-chrome') and self.cros_remote
and (self.cros_remote_ssh_port < 0)):
try:
self.cros_remote_ssh_port = socket.getservbyname('ssh')
except OSError as e:
raise RuntimeError(
'Running a CrOS test in remote mode, but failed to retrieve port '
'used by SSH service. This likely means SSH is not installed on '
'the system. Original error: %s' % e)
# Profiling other periods along with the story_run period leads to running
# multiple profiling processes at the same time. The effects of performing
# muliple CPU profiling at the same time is unclear and may generate
# incorrect profiles so this will not be supported.
if (len(self.interval_profiling_periods) > 1
and 'story_run' in self.interval_profiling_periods):
print 'Cannot specify other periods along with the story_run period.'
sys.exit(1)
self.interval_profiler_options = shlex.split(
self.interval_profiler_options, posix=(not _IsWin()))
# Parse browser options.
self.browser_options.UpdateFromParseResults(self)
return ret
def _GetAllCrashpadMinidumps(self, minidump_dir):
if not minidump_dir:
self._explanation.append(
'No minidump directory provided. Likely '
'attempted to retrieve the Crashpad minidumps '
'after the browser was already closed.')
return None
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager(None)
try:
crashpad_database_util = binary_manager.FetchPath(
'crashpad_database_util', self._os, self._arch)
if not crashpad_database_util:
self._explanation.append(
'Unable to find crashpad_database_util. This '
'is likely due to running on a platform that '
'does not support Crashpad.')
return None
except dependency_manager.NoPathFoundError:
self._explanation.append(
'Could not find a path to look for crashpad_database_util.')
return None
self._explanation.append('Found crashpad_database_util at %s' %
crashpad_database_util)
report_output = subprocess.check_output([
crashpad_database_util, '--database=' + minidump_dir,
'--show-pending-reports', '--show-completed-reports',
'--show-all-report-info'
])
last_indentation = -1
reports_list = []
report_dict = {}
for report_line in report_output.splitlines():
# Report values are grouped together by the same indentation level.
current_indentation = 0
for report_char in report_line:
if not report_char.isspace():
break
current_indentation += 1
# Decrease in indentation level indicates a new report is being printed.
if current_indentation >= last_indentation:
report_key, report_value = report_line.split(':', 1)
if report_value:
report_dict[report_key.strip()] = report_value.strip()
elif report_dict:
try:
report_time = _ParseCrashpadDateTime(
report_dict['Creation time'])
report_path = report_dict['Path'].strip()
reports_list.append((report_time, report_path))
except (ValueError, KeyError) as e:
self._explanation.append(
'Expected to find keys "Path" and "Creation '
'time" in Crashpad report, but one or both '
'don\'t exist: %s' % e)
finally:
report_dict = {}
last_indentation = current_indentation
# Include the last report.
if report_dict:
try:
report_time = _ParseCrashpadDateTime(
report_dict['Creation time'])
report_path = report_dict['Path'].strip()
reports_list.append((report_time, report_path))
except (ValueError, KeyError) as e:
self._explanation.append(
'Expected to find keys "Path" and "Creation '
'time" in Crashpad report, but one or both '
'don\'t exist: %s' % e)
return reports_list
def main(environment, extra_commands=None, **log_config_kwargs):
# The log level is set in browser_options.
log_config_kwargs.pop('level', None)
log_config_kwargs.setdefault('format', DEFAULT_LOG_FORMAT)
logging.basicConfig(**log_config_kwargs)
ps_util.EnableListingStrayProcessesUponExitHook()
# Get the command name from the command line.
if len(sys.argv) > 1 and sys.argv[1] == '--help':
sys.argv[1] = 'help'
command_name = 'run'
for arg in sys.argv[1:]:
if not arg.startswith('-'):
command_name = arg
break
# TODO(eakuefner): Remove this hack after we port to argparse.
if command_name == 'help' and len(sys.argv) > 2 and sys.argv[2] == 'run':
command_name = 'run'
sys.argv[2] = '--help'
if extra_commands is None:
extra_commands = []
all_commands = [Help, CheckIndependent, List, Run] + extra_commands
# Validate and interpret the command name.
commands = _MatchingCommands(command_name, all_commands)
if len(commands) > 1:
print >> sys.stderr, (
'"%s" is not a %s command. Did you mean one of these?' %
(command_name, _ScriptName()))
for command in commands:
print >> sys.stderr, ' %-10s %s' % (command.Name(),
command.Description())
return 1
if commands:
command = commands[0]
else:
command = Run
if binary_manager.NeedsInit():
binary_manager.InitDependencyManager(environment.client_configs)
# Parse and run the command.
parser = command.CreateParser()
command.AddCommandLineArgs(parser, environment)
# Set the default chrome root variable.
parser.set_defaults(chrome_root=environment.default_chrome_root)
if isinstance(parser, argparse.ArgumentParser):
commandline_args = sys.argv[1:]
options, args = parser.parse_known_args(commandline_args[1:])
command.ProcessCommandLineArgs(parser, options, args, environment)
else:
options, args = parser.parse_args()
if commands:
args = args[1:]
options.positional_args = args
command.ProcessCommandLineArgs(parser, options, environment)
if command == Help:
command_instance = command(all_commands)
else:
command_instance = command()
if isinstance(command_instance, command_line.OptparseCommand):
return command_instance.Run(options)
else:
return command_instance.Run(options, args)
