def parse_args(argv): p = argparse.ArgumentParser() p.add_argument( '--interval', default=10, type=int, help='time (in seconds) between sampling system metrics') p.add_argument( '--root-setup', action='store_true', help='if this is set sysmon will run once to initialise configs in /etc ' 'and then exit immediately. Used on GCE bots to bootstrap sysmon') logs.add_argparse_options(p) ts_mon.add_argparse_options(p) outer_loop.add_argparse_options(p) p.set_defaults(ts_mon_flush='manual') opts = p.parse_args(argv) logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) loop_opts = outer_loop.process_argparse_options(opts) return opts, loop_opts
def parse_args(args): # pragma: no cover parser = argparse.ArgumentParser('./run.py %s' % __package__) parser.add_argument( '-c', '--configfile', help='Local JSON poller configuration file to override ' 'confilg file from luci-config.') parser.add_argument( '-d', '--credentials_db', help='File to use for Codesite OAuth2 credentials storage.') parser.add_argument( '--datadir', default=DATADIR, help='Directory where persistent app data should be stored.') logs.add_argparse_options(parser) ts_mon.add_argparse_options(parser) outer_loop.add_argparse_options(parser) parser.set_defaults(ts_mon_target_type='task', ts_mon_task_service_name='bugdroid', ts_mon_task_job_name='bugdroid_job') opts = parser.parse_args(args) logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) loop_opts = outer_loop.process_argparse_options(opts) return opts, loop_opts
def parse_args(argv): p = argparse.ArgumentParser() p.add_argument('--interval', default=10, type=int, help='time (in seconds) between sampling system metrics') p.add_argument( '--root-setup', action='store_true', help='if this is set sysmon will run once to initialise configs in /etc ' 'and then exit immediately. Used on GCE bots to bootstrap sysmon') logs.add_argparse_options(p) ts_mon.add_argparse_options(p) outer_loop.add_argparse_options(p) p.set_defaults(ts_mon_flush='manual') opts = p.parse_args(argv) logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) loop_opts = outer_loop.process_argparse_options(opts) return opts, loop_opts
def main(argv): args = get_arguments(argv) infra_libs.logs.process_argparse_options(args) ts_mon.process_argparse_options(args) arg_to_metric_type = ( (args.gauge, ts_mon.GaugeMetric), (args.float, ts_mon.FloatMetric), (args.string, ts_mon.StringMetric), (args.bool, ts_mon.BooleanMetric), (args.counter, ts_mon.CounterMetric), (args.cumulative, ts_mon.CumulativeMetric), ) for arg, metric in arg_to_metric_type: set_metrics(arg, metric) argfile_to_metric_type = ( (args.gauge_file, ts_mon.GaugeMetric), (args.float_file, ts_mon.FloatMetric), (args.string_file, ts_mon.StringMetric), (args.bool_file, ts_mon.BooleanMetric), (args.counter_file, ts_mon.CounterMetric), (args.cumulative_file, ts_mon.CumulativeMetric), ) for arg, metric in argfile_to_metric_type: set_metrics_file(arg, metric) ts_mon.flush()
def parse_args(argv): if sys.platform == 'win32': default_state_directory = 'C:\\chrome-infra\\service-state' default_config_directory = 'C:\\chrome-infra\\service-config' default_root_directory = 'C:\\infra-python' else: default_state_directory = '/var/run/infra-services' default_config_directory = '/etc/infra-services' default_root_directory = '/opt/infra-python' p = argparse.ArgumentParser( description='Starts and stops machine-wide infra services with arguments ' 'from config files') p.add_argument( '--state-directory', default=default_state_directory, help='directory to store PID files (default %(default)s)') p.add_argument( '--config-directory', default=default_config_directory, help='directory to read JSON config files (default %(default)s)') p.add_argument( '--root-directory', default=default_root_directory, help='directory where the service_manager package is deployed. If this ' 'package is updated the process will exit') p.add_argument( '--config-poll-interval', default=10, help='how frequently (in seconds) to poll the config directory') p.add_argument( '--service-poll-interval', default=10, help='how frequently (in seconds) to restart failed services') p.add_argument( '--root-setup', action='store_true', help='if this is set service_manager will run once to initialise configs ' 'in /etc and then exit immediately. Used on GCE bots to bootstrap ' 'service_manager') logs.add_argparse_options(p) ts_mon.add_argparse_options(p) p.set_defaults( ts_mon_target_type='task', ts_mon_task_service_name='service_manager', ts_mon_task_job_name=socket.getfqdn(), ) opts = p.parse_args(argv) logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) return opts
def process_argparse_options(self, options): """Process any commandline arguments. Args: options: An argparse.Namespace object. """ logs.process_argparse_options(options) ts_mon.process_argparse_options(options)
def parse_args(argv): p = argparse.ArgumentParser() group = p.add_mutually_exclusive_group(required=True) group.add_argument( '--url', help='URL of one buildbot master to monitor') group.add_argument('--build-dir', help='location of the tools/build directory. Used with --hostname to get ' 'the list of all buildbot masters on this host to monitor. Cannot be ' 'used with --url') p.add_argument('--hostname', default=socket.getfqdn(), help='override local hostname (currently %(default)s). Used with ' '--build-dir to get the list of all buildbot masters on this host to ' 'monitor') p.add_argument( '--interval', default=300, type=int, help='time (in seconds) between sampling the buildbot master') logs.add_argparse_options(p) ts_mon.add_argparse_options(p) outer_loop.add_argparse_options(p) DEFAULT_ARG_VALUE = '(default)' p.set_defaults( ts_mon_flush='manual', ts_mon_target_type='task', ts_mon_task_service_name='mastermon', ts_mon_task_job_name=DEFAULT_ARG_VALUE, ) opts = p.parse_args(argv) if opts.ts_mon_task_job_name == DEFAULT_ARG_VALUE: # The ts_mon job name defaults to either the hostname when monitoring all # masters on a host, or the name of the master extracted from the URL. if opts.build_dir: opts.ts_mon_task_job_name = opts.hostname else: parsed_url = urlparse.urlsplit(opts.url) path_components = [x for x in parsed_url.path.split('/') if x] if path_components: opts.ts_mon_task_job_name = path_components[-1] else: opts.ts_mon_task_job_name = parsed_url.netloc logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) loop_opts = outer_loop.process_argparse_options(opts) return opts, loop_opts
def process_argparse_options(self, options): """Process any commandline arguments. Args: options: An argparse.Namespace object. """ if self.USES_STANDARD_LOGGING: logs.process_argparse_options(options) if self.USES_TS_MON: ts_mon.process_argparse_options(options)
def parse_args(argv): p = argparse.ArgumentParser() group = p.add_mutually_exclusive_group(required=True) group.add_argument( '--url', help='URL of one buildbot master to monitor') group.add_argument('--build-dir', help='location of the tools/build directory. Used with --hostname to get ' 'the list of all buildbot masters on this host to monitor. Cannot be ' 'used with --url') p.add_argument('--hostname', default=socket.getfqdn(), help='override local hostname (currently %(default)s). Used with ' '--build-dir to get the list of all buildbot masters on this host to ' 'monitor') p.add_argument( '--interval', default=60, type=int, help='time (in seconds) between sampling the buildbot master') logs.add_argparse_options(p) ts_mon.add_argparse_options(p) outer_loop.add_argparse_options(p) DEFAULT_ARG_VALUE = '(default)' p.set_defaults( ts_mon_flush='manual', ts_mon_target_type='task', ts_mon_task_service_name='mastermon', ts_mon_task_job_name=DEFAULT_ARG_VALUE, ) opts = p.parse_args(argv) if opts.ts_mon_task_job_name == DEFAULT_ARG_VALUE: # The ts_mon job name defaults to either the hostname when monitoring all # masters on a host, or the name of the master extracted from the URL. if opts.build_dir: opts.ts_mon_task_job_name = opts.hostname else: parsed_url = urlparse.urlsplit(opts.url) path_components = [x for x in parsed_url.path.split('/') if x] if path_components: opts.ts_mon_task_job_name = path_components[-1] else: opts.ts_mon_task_job_name = parsed_url.netloc logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) loop_opts = outer_loop.process_argparse_options(opts) return opts, loop_opts
def parse_args(): parser = argparse.ArgumentParser( description='Launches master_manager for every master on a host. NOTE: ' 'does not perform any action unless --prod is set.') parser.add_argument('build_dir', nargs='?', help='location of the tools/build directory') parser.add_argument('--hostname', default=socket.getfqdn(), help='override local hostname (currently %(default)s)') parser.add_argument('--json-file', help='load desired master state from a file on disk') parser.add_argument('--json-gitiles', help='load desired master state from a gitiles location') parser.add_argument('--netrc', help='location of the netrc file when connecting to gitiles') parser.add_argument('--command-timeout', help='apply a timeout in seconds to each master_manager process') parser.add_argument('--verify', action='store_true', help='verify the desired master state JSON is valid, then exit') parser.add_argument('--prod', action='store_true', help='actually perform actions instead of doing a dry run') parser.add_argument('--processes', default=16, type=int, help='maximum number of master_manager processes to run simultaneously ' '(default %(default)d)') ts_mon.add_argparse_options(parser) logs.add_argparse_options(parser) parser.set_defaults( ts_mon_target_type='task', ts_mon_task_job_name=socket.getfqdn().split(".")[0], ts_mon_task_service_name='master_manager_launcher', ts_mon_flush_mode='manual', ) args = parser.parse_args() ts_mon.process_argparse_options(args) logs.process_argparse_options(args) if args.json_file and args.json_gitiles: parser.error("Can't specify --json-file and --json-gitiles simultaneously") if not args.json_gitiles and not args.json_file: parser.error('Must specify either --json-gitiles or --json-file.') if not args.verify: if not args.build_dir: parser.error('A build/ directory must be specified.') return args
def main(argv): # pragma: no cover args = get_arguments(argv) infra_libs.logs.process_argparse_options(args) ts_mon.process_argparse_options(args) set_metrics(args.gauge, ts_mon.GaugeMetric) set_metrics(args.float, ts_mon.FloatMetric) set_metrics(args.string, ts_mon.StringMetric) set_metrics(args.bool, ts_mon.BooleanMetric) set_metrics(args.counter, ts_mon.CounterMetric) set_metrics(args.cumulative, ts_mon.CumulativeMetric) ts_mon.flush()
def parse_args(argv): p = argparse.ArgumentParser( description='Starts and stops machine-wide infra services with arguments ' 'from config files') p.add_argument( '--state-directory', default='/var/run/infra-services', help='directory to store PID files (default %(default)s)') p.add_argument( '--config-directory', default='/etc/infra-services', help='directory to read JSON config files (default %(default)s)') p.add_argument( '--root-directory', default='/opt/infra-python', help='directory where the service_manager package is deployed. If this ' 'package is updated the process will exit') p.add_argument( '--config-poll-interval', default=10, help='how frequently (in seconds) to poll the config directory') p.add_argument( '--service-poll-interval', default=10, help='how frequently (in seconds) to restart failed services') p.add_argument( '--root-setup', action='store_true', help='if this is set service_manager will run once to initialise configs ' 'in /etc and then exit immediately. Used on GCE bots to bootstrap ' 'service_manager') logs.add_argparse_options(p) ts_mon.add_argparse_options(p) p.set_defaults( ts_mon_target_type='task', ts_mon_task_service_name='service_manager', ts_mon_task_job_name=socket.getfqdn(), ) opts = p.parse_args(argv) logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) return opts
def parse_args(args): # pragma: no cover def check_url(s): parsed = urlparse.urlparse(s) if parsed.scheme not in ('https', 'git', 'file'): raise argparse.ArgumentTypeError( 'Repo URL must use https, git or file protocol.') if not parsed.path.strip('/'): raise argparse.ArgumentTypeError('URL is missing a path?') return git2.Repo(s) parser = argparse.ArgumentParser('./run.py %s' % __package__) parser.add_argument('--dry_run', action='store_true', help='Do not actually push anything.') parser.add_argument('--repo_dir', metavar='DIR', default='gsubtreed_repos', help=('The directory to use for git clones ' '(default: %(default)s)')) parser.add_argument('--json_output', metavar='PATH', help='Path to write JSON with results of the run to') parser.add_argument('repo', nargs=1, help='The url of the repo to act on.', type=check_url) logs.add_argparse_options(parser) ts_mon.add_argparse_options(parser) outer_loop.add_argparse_options(parser) parser.set_defaults( ts_mon_target_type='task', ts_mon_task_service_name='gsubtreed', ) opts = parser.parse_args(args) repo = opts.repo[0] repo.repos_dir = os.path.abspath(opts.repo_dir) if not opts.ts_mon_task_job_name: parsed_repo_url = urlparse.urlparse(repo.url) opts.ts_mon_task_job_name = '%s%s' % (parsed_repo_url.netloc, parsed_repo_url.path) logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) loop_opts = outer_loop.process_argparse_options(opts) return Options(repo, loop_opts, opts.json_output, opts.dry_run)
def main(args): parser = argparse.ArgumentParser() parser.add_argument( '-n', '--nice', type=int, metavar='VALUE', help='Set the nice level of the process to VALUE prior to execution.') parser.add_argument( 'master_paths', nargs='+', help='The paths to the master base directories to monitor. Consider ' 'the /path/to/build/masters/master.* wildcard to specify all of ' 'them.') logs.add_argparse_options(parser) ts_mon.add_argparse_options(parser) # Parse arguments. args = parser.parse_args(args) logs.process_argparse_options(args) ts_mon.process_argparse_options(args) # Try setting the nice value; if it fails, eat the error and continue. if args.nice: logging.debug('Setting process "nice" to: %d', args.nice) try: os.nice(args.nice) except OSError as e: logging.error('Failed to update "nice" to %d: %s', args.nice, e) # Update global state calculations. logging.info('Pulling master state from: %s', args.master_paths) for master_path in args.master_paths: master_name = master_path_to_name(master_path) # Log to the target: buildbot/master/<master_name> target = ts_mon.TaskTarget('buildbot/master', master_name, args.ts_mon_task_region, args.ts_mon_task_hostname, args.ts_mon_task_number) logging.info('Collecting log state for master "%s" at: %s', master_name, master_path) get_master_state(master_path, target) logging.info('Flushing collected information.') ts_mon.flush() return 0
def parse_args(args): # pragma: no cover parser = argparse.ArgumentParser('./run.py %s' % __package__) parser.add_argument( '-c', '--configfile', help='Local JSON poller configuration file to override ' 'config file from luci-config.') parser.add_argument( '-d', '--credentials_db', help= 'File to use for OAuth2 credentials storage if not running on LUCI.') parser.add_argument( '--datadir', default=DATADIR, help='Directory where persistent app data should be stored.') parser.add_argument( '--dryrun', action='store_true', help='Don\'t update monorail issues or update issues to the bugdroid ' 'appengine app.') logs.add_argparse_options(parser) ts_mon.add_argparse_options(parser) outer_loop.add_argparse_options(parser) parser.set_defaults(log_level=logging.DEBUG, ts_mon_target_type='task', ts_mon_task_service_name='bugdroid', ts_mon_task_job_name='bugdroid_job') opts = parser.parse_args(args) logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) loop_opts = outer_loop.process_argparse_options(opts) # We need to include the logger ID (i.e. "%(name)s") in the formatter string. # Override the root logging handler set by infra_libs.logs. logging.root.handlers[0].setFormatter( logging.Formatter( '[%(severity)s%(iso8601)s %(process)d %(thread)d ' '%(fullModuleName)s:%(lineno)s] (%(name)s) %(message)s')) return opts, loop_opts
def main(args): parser = argparse.ArgumentParser() parser.add_argument('-n', '--nice', type=int, metavar='VALUE', help='Set the nice level of the process to VALUE prior to execution.') parser.add_argument('master_paths', nargs='+', help='The paths to the master base directories to monitor. Consider ' 'the /path/to/build/masters/master.* wildcard to specify all of ' 'them.') logs.add_argparse_options(parser) ts_mon.add_argparse_options(parser) # Parse arguments. args = parser.parse_args(args) logs.process_argparse_options(args) ts_mon.process_argparse_options(args) # Try setting the nice value; if it fails, eat the error and continue. if args.nice: logging.debug('Setting process "nice" to: %d', args.nice) try: os.nice(args.nice) except OSError as e: logging.error('Failed to update "nice" to %d: %s', args.nice, e) # Update global state calculations. logging.info('Pulling master state from: %s', args.master_paths) for master_path in args.master_paths: master_name = master_path_to_name(master_path) # Log to the target: buildbot/master/<master_name> target = ts_mon.TaskTarget( 'buildbot/master', master_name, args.ts_mon_task_region, args.ts_mon_task_hostname, args.ts_mon_task_number) logging.info('Collecting log state for master "%s" at: %s', master_name, master_path) get_master_state(master_path, target) logging.info('Flushing collected information.') ts_mon.flush() return 0
def parse_args(args): # pragma: no cover def check_url(s): parsed = urlparse.urlparse(s) if parsed.scheme not in ('https', 'git', 'file'): raise argparse.ArgumentTypeError( 'Repo URL must use https, git or file protocol.') if not parsed.path.strip('/'): raise argparse.ArgumentTypeError('URL is missing a path?') return git2.Repo(s) parser = argparse.ArgumentParser('./run.py %s' % __package__) parser.add_argument('--dry_run', action='store_true', help='Do not actually push anything.') parser.add_argument('--repo_dir', metavar='DIR', default='gsubtreed_repos', help=('The directory to use for git clones ' '(default: %(default)s)')) parser.add_argument('--json_output', metavar='PATH', help='Path to write JSON with results of the run to') parser.add_argument('repo', nargs=1, help='The url of the repo to act on.', type=check_url) logs.add_argparse_options(parser) ts_mon.add_argparse_options(parser) outer_loop.add_argparse_options(parser) parser.set_defaults( ts_mon_target_type='task', ts_mon_task_service_name='gsubtreed', ) opts = parser.parse_args(args) repo = opts.repo[0] repo.dry_run = opts.dry_run repo.repos_dir = os.path.abspath(opts.repo_dir) if not opts.ts_mon_task_job_name: opts.ts_mon_task_job_name = urlparse.urlparse(repo.url).path logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) loop_opts = outer_loop.process_argparse_options(opts) return Options(repo, loop_opts, opts.json_output)
def parse_args(args): parser = argparse.ArgumentParser('python -m %s' % __package__) parser.add_argument('--project', required=True) parser.add_argument('--range', required=True) logs.add_argparse_options(parser) ts_mon.add_argparse_options(parser) parser.set_defaults( logs_directory='', ts_mon_target_type='task', ts_mon_task_service_name='cq_stats_uploader', ) opts = parser.parse_args(args) if not opts.ts_mon_task_job_name: opts.ts_mon_task_job_name = '%s-%s' % (opts.project, opts.range) logs.process_argparse_options(opts) ts_mon.process_argparse_options(opts) return opts
def parse_args(): parser = argparse.ArgumentParser( description='Launches master_manager for every master on a host. NOTE: ' 'does not perform any action unless --prod is set.') parser.add_argument('build_dir', nargs='?', help='location of the tools/build directory') parser.add_argument('--hostname', default=socket.getfqdn(), help='override local hostname (currently %(default)s)') parser.add_argument('--json-file', help='load desired master state from a file on disk') parser.add_argument( '--json-gitiles', help='load desired master state from a gitiles location') parser.add_argument( '--netrc', help='location of the netrc file when connecting to gitiles') parser.add_argument( '--command-timeout', help='apply a timeout in seconds to each master_manager process') parser.add_argument( '--verify', action='store_true', help='verify the desired master state JSON is valid, then exit') parser.add_argument( '--prod', action='store_true', help='actually perform actions instead of doing a dry run') parser.add_argument( '--processes', default=16, type=int, help='maximum number of master_manager processes to run simultaneously ' '(default %(default)d)') ts_mon.add_argparse_options(parser) logs.add_argparse_options(parser) parser.set_defaults( ts_mon_target_type='task', ts_mon_task_job_name=socket.getfqdn().split(".")[0], ts_mon_task_service_name='master_manager_launcher', ts_mon_flush_mode='manual', ) args = parser.parse_args() ts_mon.process_argparse_options(args) logs.process_argparse_options(args) if args.json_file and args.json_gitiles: parser.error( "Can't specify --json-file and --json-gitiles simultaneously") if not args.json_gitiles and not args.json_file: parser.error('Must specify either --json-gitiles or --json-file.') if not args.verify: if not args.build_dir: parser.error('A build/ directory must be specified.') return args
def process_argparse_options(args): # pragma: no cover infra_libs.logs.process_argparse_options(args) event_mon.process_argparse_options(args) ts_mon.process_argparse_options(args)
def process_argparse_options(args): # pragma: no cover infra_libs.logs.process_argparse_options(args) event_mon.process_argparse_options(args) # Put here because we want to send the presence metric as late as possible. ts_mon.process_argparse_options(args) _process_logrequest_path(args)