def ParseArgs(args=None): defaults = parser.get_default_values() for k, v in defaults.__dict__.items(): if k in self.__dict__ and self.__dict__[k] != None: continue self.__dict__[k] = v ret = real_parse(args, self) # pylint: disable=E1121 if self.verbosity >= 2: logging.getLogger().setLevel(logging.DEBUG) elif self.verbosity: logging.getLogger().setLevel(logging.INFO) else: logging.getLogger().setLevel(logging.WARNING) if self.chromium_output_dir: os.environ['CHROMIUM_OUTPUT_DIR'] = self.chromium_output_dir if self.device == 'list': if binary_manager.NeedsInit(): binary_manager.InitDependencyManager(None) devices = device_finder.GetDevicesMatchingOptions(self) print 'Available devices:' for device in devices: print ' ', device.name sys.exit(0) if self.browser_executable and not self.browser_type: self.browser_type = 'exact' if self.browser_type == 'list': if binary_manager.NeedsInit(): binary_manager.InitDependencyManager(None) devices = device_finder.GetDevicesMatchingOptions(self) if not devices: sys.exit(0) browser_types = {} for device in devices: try: possible_browsers = browser_finder.GetAllAvailableBrowsers( self, device) browser_types[device.name] = sorted([ browser.browser_type for browser in possible_browsers ]) except browser_finder_exceptions.BrowserFinderException as ex: print >> sys.stderr, 'ERROR: ', ex sys.exit(1) print 'Available browsers:' if len(browser_types) == 0: print ' No devices were found.' for device_name in sorted(browser_types.keys()): print ' ', device_name for browser_type in browser_types[device_name]: print ' ', browser_type sys.exit(0) # Parse browser options. self.browser_options.UpdateFromParseResults(self) return ret
def _SetUpProcess(child, context): # pylint: disable=unused-argument ps_util.EnableListingStrayProcessesUponExitHook() # Make sure that we don't invokes cloud storage I/Os when we run the tests in # parallel. # TODO(nednguyen): always do this once telemetry tests in Chromium is updated # to prefetch files. # (https://github.com/catapult-project/catapult/issues/2192) args = context if args.disable_cloud_storage_io: os.environ[cloud_storage.DISABLE_CLOUD_STORAGE_IO] = '1' if binary_manager.NeedsInit(): # Typ doesn't keep the DependencyManager initialization in the child # processes. binary_manager.InitDependencyManager(context.client_configs) # We need to reset the handlers in case some other parts of telemetry already # set it to make this work. if not args.disable_logging_config: logging.getLogger().handlers = [] logging.basicConfig( level=logging.INFO, format='(%(levelname)s) %(asctime)s pid=%(process)d' ' %(module)s.%(funcName)s:%(lineno)d' ' %(message)s') if args.remote_platform_options.device == 'android': android_devices = android_device.FindAllAvailableDevices(args) if not android_devices: raise RuntimeError("No Android device found") android_devices.sort(key=lambda device: device.name) args.remote_platform_options.device = ( android_devices[child.worker_num - 1].guid) options_for_unittests.Push(args)
def _GetGoBinaryPath(cls): if not cls._go_binary_path: if binary_manager.NeedsInit(): binary_manager.InitDependencyManager(None) cls._go_binary_path = binary_manager.FetchPath( 'wpr_go', py_utils.GetHostArchName(), py_utils.GetHostOsName()) return cls._go_binary_path
def _SetUpProcess(child, context): # pylint: disable=unused-argument ps_util.EnableListingStrayProcessesUponExitHook() if binary_manager.NeedsInit(): # Typ doesn't keep the DependencyManager initialization in the child # processes. binary_manager.InitDependencyManager(context.client_config) # We need to reset the handlers in case some other parts of telemetry already # set it to make this work. logging.getLogger().handlers = [] logging.basicConfig( level=logging.INFO, format='(%(levelname)s) %(asctime)s %(module)s.%(funcName)s:%(lineno)d ' '%(message)s') args = context if not args.disable_logging_config: logging.getLogger().handlers = [] logging.basicConfig( level=logging.INFO, format= '(%(levelname)s) %(asctime)s %(module)s.%(funcName)s:%(lineno)d' ' %(message)s') if args.device and args.device == 'android': android_devices = android_device.FindAllAvailableDevices(args) if not android_devices: raise RuntimeError("No Android device found") android_devices.sort(key=lambda device: device.name) args.device = android_devices[child.worker_num - 1].guid options_for_unittests.Push(args)
def main(environment, argv=None): # The log level is set in browser_options. # Clear the log handlers to ensure we can set up logging properly here. logging.getLogger().handlers = [] logging.basicConfig(format=DEFAULT_LOG_FORMAT) ps_util.EnableListingStrayProcessesUponExitHook() if argv is None: argv = sys.argv # Get the command name from the command line. if len(argv) > 1 and argv[1] == '--help': argv[1] = 'help' command_name = 'run' for arg in argv[1:]: if not arg.startswith('-'): command_name = arg break # TODO(eakuefner): Remove this hack after we port to argparse. if command_name == 'help' and len(argv) > 2 and argv[2] == 'run': command_name = 'run' argv[2] = '--help' # Validate and interpret the command name. commands = _MatchingCommands(command_name) if len(commands) > 1: print >> sys.stderr, ( '"%s" is not a %s command. Did you mean one of these?' % (command_name, _ScriptName())) for command in commands: print >> sys.stderr, ' %-10s %s' % (command.Name(), command.Description()) return 1 if commands: command = commands[0] else: command = Run if binary_manager.NeedsInit(): binary_manager.InitDependencyManager(environment.client_configs) # Parse and run the command. parser = command.CreateParser() command.AddCommandLineArgs(parser, environment) # Set the default chrome root variable. parser.set_defaults(chrome_root=environment.default_chrome_root) options, args = parser.parse_args(argv[1:]) if commands: args = args[1:] options.positional_args = args command.ProcessCommandLineArgs(parser, options, environment) return command().Run(options)
def FetchTelemetryDependencies( platform=None, client_configs=None, chrome_reference_browser=False): if not platform: platform = platform_module.GetHostPlatform() if binary_manager.NeedsInit(): binary_manager.InitDependencyManager(client_configs) else: raise Exception('Binary manager already initialized with other configs.') binary_manager.FetchBinaryDependencies( platform, client_configs, chrome_reference_browser)
def _BringUpWpr(self): """Start the WPR server on the host and the forwarder on the device.""" print 'Starting WPR on host...' _DownloadFromCloudStorage(self._WPR_BUCKET, self._wpr_archive_hash) if binary_manager.NeedsInit(): binary_manager.InitDependencyManager([]) self._wpr_server = webpagereplay_go_server.ReplayServer( self._wpr_archive, '127.0.0.1', 0, 0, replay_options=[]) ports = self._wpr_server.StartServer()[:-1] self._host_http_port = ports[0] self._host_https_port = ports[1]
def _SetUpProcess(child, context): args = context.finder_options if binary_manager.NeedsInit(): # On windows, typ doesn't keep the DependencyManager initialization in the # child processes. binary_manager.InitDependencyManager(context.client_configs) if args.remote_platform_options.device == 'android': android_devices = android_device.FindAllAvailableDevices(args) if not android_devices: raise RuntimeError("No Android device found") android_devices.sort(key=lambda device: device.name) args.remote_platform_options.device = ( android_devices[child.worker_num - 1].guid) browser_test_context._global_test_context = context context.test_class.SetUpProcess()
def __init__(self, archive_path, replay_host, http_port, https_port, replay_options): """Initialize ReplayServer. Args: archive_path: a path to a specific WPR archive (required). replay_host: the hostname to serve traffic. http_port: an integer port on which to serve HTTP traffic. May be zero to let the OS choose an available port. https_port: an integer port on which to serve HTTPS traffic. May be zero to let the OS choose an available port. replay_options: an iterable of option strings to forward to replay.py """ if binary_manager.NeedsInit(): binary_manager.InitDependencyManager(None) self._wpr_server = webpagereplay_go_server.ReplayServer( archive_path, replay_host, http_port, https_port, replay_options, binary_manager.FetchPath)
def _FetchBinaryManagerPath(self, dependency_name): """Fetches the path for the dependency via binary_manager. Initializes binary_manager with defaults if it is not already initialized. Args: dependency_name: A string containing the name of the dependency. Returns: A string containing the path to the dependency, or None if it could not be found. """ if binary_manager.NeedsInit(): logging.info( 'binary_manager was not initialized. Initializing with default ' 'values.') binary_manager.InitDependencyManager(None) return binary_manager.FetchPath(dependency_name, self._os, self._arch, self._os_version)
def ParseArgs(args=None): defaults = parser.get_default_values() for k, v in defaults.__dict__.items(): if k in self.__dict__ and self.__dict__[k] != None: continue self.__dict__[k] = v ret = real_parse(args, self) # pylint: disable=E1121 if self.chromium_output_dir: os.environ['CHROMIUM_OUTPUT_DIR'] = self.chromium_output_dir # Parse remote platform options. self.BuildRemotePlatformOptions() if self.remote_platform_options.device == 'list': if binary_manager.NeedsInit(): binary_manager.InitDependencyManager([]) devices = device_finder.GetDevicesMatchingOptions(self) print 'Available devices:' for device in devices: print ' ', device.name sys.exit(0) if self.browser_executable and not self.browser_type: self.browser_type = 'exact' if self.browser_type == 'list': if binary_manager.NeedsInit(): binary_manager.InitDependencyManager([]) devices = device_finder.GetDevicesMatchingOptions(self) if not devices: sys.exit(0) browser_types = {} for device in devices: try: possible_browsers = browser_finder.GetAllAvailableBrowsers( self, device) browser_types[device.name] = sorted([ browser.browser_type for browser in possible_browsers ]) except browser_finder_exceptions.BrowserFinderException as ex: print >> sys.stderr, 'ERROR: ', ex sys.exit(1) print 'Available browsers:' if len(browser_types) == 0: print ' No devices were found.' for device_name in sorted(browser_types.keys()): print ' ', device_name for browser_type in browser_types[device_name]: print ' ', browser_type if len(browser_types[device_name]) == 0: print ' No browsers found for this device' sys.exit(0) # Profiling other periods along with the story_run period leads to running # multiple profiling processes at the same time. The effects of performing # muliple CPU profiling at the same time is unclear and may generate # incorrect profiles so this will not be supported. if (len(self.interval_profiling_periods) > 1 and 'story_run' in self.interval_profiling_periods): print 'Cannot specify other periods along with the story_run period.' sys.exit(1) self.interval_profiler_options = shlex.split( self.interval_profiler_options) # Parse browser options. self.browser_options.UpdateFromParseResults(self) return ret
def _PrepareDevice(device): if not 'BUILDTYPE' in os.environ: os.environ['BUILDTYPE'] = 'Release' if binary_manager.NeedsInit(): binary_manager.InitDependencyManager(None) return android_profiling_helper.PrepareDeviceForPerf(device)
self._perf_binary = self._PrepareDevice(presentation.device) self._perf_instance = None self._categories = None def __repr__(self): return 'perf profile' @staticmethod def IsSupported(): return bool(android_profiling_helper) @staticmethod def _PrepareDevice(presentation.device): if not 'BUILDTYPE' in os.environ: os.environ['BUILDTYPE'] = 'Release' if binary_manager.NeedsInit(): binary_manager.InitDependencyManager(None) return android_profiling_helper.PrepareDeviceForPerf(presentation.device) @classmethod def GetCategories(cls, presentation.device): perf_binary = cls._PrepareDevice(presentation.device) # Perf binary returns non-zero exit status on "list" command. return presentation.device.RunShellCommand([perf_binary, 'list'], check_return=False) @py_utils.Timeout(tracing_agents.START_STOP_TIMEOUT) def StartAgentTracing(self, config, timeout=None): self._categories = _ComputePerfCategories(config) self._perf_instance = _PerfProfiler(self._device, self._perf_binary, self._categories)
def ParseArgs(args=None): defaults = parser.get_default_values() for k, v in defaults.__dict__.items(): if k in self.__dict__ and self.__dict__[k] != None: continue self.__dict__[k] = v ret = real_parse(args, self) # pylint: disable=E1121 if self.chromium_output_dir: os.environ['CHROMIUM_OUTPUT_DIR'] = self.chromium_output_dir # Set up Android emulator if necessary. self.ParseAndroidEmulatorOptions() # Parse remote platform options. self.BuildRemotePlatformOptions() if self.remote_platform_options.device == 'list': if binary_manager.NeedsInit(): binary_manager.InitDependencyManager([]) devices = device_finder.GetDevicesMatchingOptions(self) print 'Available devices:' for device in devices: print ' ', device.name sys.exit(0) if self.browser_executable and not self.browser_type: self.browser_type = 'exact' if self.browser_type == 'list': if binary_manager.NeedsInit(): binary_manager.InitDependencyManager([]) devices = device_finder.GetDevicesMatchingOptions(self) if not devices: sys.exit(0) browser_types = {} for device in devices: try: possible_browsers = browser_finder.GetAllAvailableBrowsers( self, device) browser_types[device.name] = sorted([ browser.browser_type for browser in possible_browsers ]) except browser_finder_exceptions.BrowserFinderException as ex: print >> sys.stderr, 'ERROR: ', ex sys.exit(1) print 'Available browsers:' if len(browser_types) == 0: print ' No devices were found.' for device_name in sorted(browser_types.keys()): print ' ', device_name for browser_type in browser_types[device_name]: print ' ', browser_type if len(browser_types[device_name]) == 0: print ' No browsers found for this device' sys.exit(0) if ((self.browser_type == 'cros-chrome' or self.browser_type == 'lacros-chrome') and self.cros_remote and (self.cros_remote_ssh_port < 0)): try: self.cros_remote_ssh_port = socket.getservbyname('ssh') except OSError as e: raise RuntimeError( 'Running a CrOS test in remote mode, but failed to retrieve port ' 'used by SSH service. This likely means SSH is not installed on ' 'the system. Original error: %s' % e) # Profiling other periods along with the story_run period leads to running # multiple profiling processes at the same time. The effects of performing # muliple CPU profiling at the same time is unclear and may generate # incorrect profiles so this will not be supported. if (len(self.interval_profiling_periods) > 1 and 'story_run' in self.interval_profiling_periods): print 'Cannot specify other periods along with the story_run period.' sys.exit(1) self.interval_profiler_options = shlex.split( self.interval_profiler_options, posix=(not _IsWin())) # Parse browser options. self.browser_options.UpdateFromParseResults(self) return ret
def _GetAllCrashpadMinidumps(self, minidump_dir): if not minidump_dir: self._explanation.append( 'No minidump directory provided. Likely ' 'attempted to retrieve the Crashpad minidumps ' 'after the browser was already closed.') return None if binary_manager.NeedsInit(): binary_manager.InitDependencyManager(None) try: crashpad_database_util = binary_manager.FetchPath( 'crashpad_database_util', self._os, self._arch) if not crashpad_database_util: self._explanation.append( 'Unable to find crashpad_database_util. This ' 'is likely due to running on a platform that ' 'does not support Crashpad.') return None except dependency_manager.NoPathFoundError: self._explanation.append( 'Could not find a path to look for crashpad_database_util.') return None self._explanation.append('Found crashpad_database_util at %s' % crashpad_database_util) report_output = subprocess.check_output([ crashpad_database_util, '--database=' + minidump_dir, '--show-pending-reports', '--show-completed-reports', '--show-all-report-info' ]) last_indentation = -1 reports_list = [] report_dict = {} for report_line in report_output.splitlines(): # Report values are grouped together by the same indentation level. current_indentation = 0 for report_char in report_line: if not report_char.isspace(): break current_indentation += 1 # Decrease in indentation level indicates a new report is being printed. if current_indentation >= last_indentation: report_key, report_value = report_line.split(':', 1) if report_value: report_dict[report_key.strip()] = report_value.strip() elif report_dict: try: report_time = _ParseCrashpadDateTime( report_dict['Creation time']) report_path = report_dict['Path'].strip() reports_list.append((report_time, report_path)) except (ValueError, KeyError) as e: self._explanation.append( 'Expected to find keys "Path" and "Creation ' 'time" in Crashpad report, but one or both ' 'don\'t exist: %s' % e) finally: report_dict = {} last_indentation = current_indentation # Include the last report. if report_dict: try: report_time = _ParseCrashpadDateTime( report_dict['Creation time']) report_path = report_dict['Path'].strip() reports_list.append((report_time, report_path)) except (ValueError, KeyError) as e: self._explanation.append( 'Expected to find keys "Path" and "Creation ' 'time" in Crashpad report, but one or both ' 'don\'t exist: %s' % e) return reports_list
def main(environment, extra_commands=None, **log_config_kwargs): # The log level is set in browser_options. log_config_kwargs.pop('level', None) log_config_kwargs.setdefault('format', DEFAULT_LOG_FORMAT) logging.basicConfig(**log_config_kwargs) ps_util.EnableListingStrayProcessesUponExitHook() # Get the command name from the command line. if len(sys.argv) > 1 and sys.argv[1] == '--help': sys.argv[1] = 'help' command_name = 'run' for arg in sys.argv[1:]: if not arg.startswith('-'): command_name = arg break # TODO(eakuefner): Remove this hack after we port to argparse. if command_name == 'help' and len(sys.argv) > 2 and sys.argv[2] == 'run': command_name = 'run' sys.argv[2] = '--help' if extra_commands is None: extra_commands = [] all_commands = [Help, CheckIndependent, List, Run] + extra_commands # Validate and interpret the command name. commands = _MatchingCommands(command_name, all_commands) if len(commands) > 1: print >> sys.stderr, ( '"%s" is not a %s command. Did you mean one of these?' % (command_name, _ScriptName())) for command in commands: print >> sys.stderr, ' %-10s %s' % (command.Name(), command.Description()) return 1 if commands: command = commands[0] else: command = Run if binary_manager.NeedsInit(): binary_manager.InitDependencyManager(environment.client_configs) # Parse and run the command. parser = command.CreateParser() command.AddCommandLineArgs(parser, environment) # Set the default chrome root variable. parser.set_defaults(chrome_root=environment.default_chrome_root) if isinstance(parser, argparse.ArgumentParser): commandline_args = sys.argv[1:] options, args = parser.parse_known_args(commandline_args[1:]) command.ProcessCommandLineArgs(parser, options, args, environment) else: options, args = parser.parse_args() if commands: args = args[1:] options.positional_args = args command.ProcessCommandLineArgs(parser, options, environment) if command == Help: command_instance = command(all_commands) else: command_instance = command() if isinstance(command_instance, command_line.OptparseCommand): return command_instance.Run(options) else: return command_instance.Run(options, args)