def parse_args(argv):
p = argparse.ArgumentParser()
p.add_argument(
'--interval',
default=10, type=int,
help='time (in seconds) between sampling system metrics')
p.add_argument(
'--root-setup',
action='store_true',
help='if this is set sysmon will run once to initialise configs in /etc '
'and then exit immediately. Used on GCE bots to bootstrap sysmon')
logs.add_argparse_options(p)
ts_mon.add_argparse_options(p)
outer_loop.add_argparse_options(p)
p.set_defaults(ts_mon_flush='manual')
opts = p.parse_args(argv)
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
loop_opts = outer_loop.process_argparse_options(opts)
return opts, loop_opts
def parse_args(args): # pragma: no cover
parser = argparse.ArgumentParser('./run.py %s' % __package__)
parser.add_argument(
'-c',
'--configfile',
help='Local JSON poller configuration file to override '
'confilg file from luci-config.')
parser.add_argument(
'-d',
'--credentials_db',
help='File to use for Codesite OAuth2 credentials storage.')
parser.add_argument(
'--datadir',
default=DATADIR,
help='Directory where persistent app data should be stored.')
logs.add_argparse_options(parser)
ts_mon.add_argparse_options(parser)
outer_loop.add_argparse_options(parser)
parser.set_defaults(ts_mon_target_type='task',
ts_mon_task_service_name='bugdroid',
ts_mon_task_job_name='bugdroid_job')
opts = parser.parse_args(args)
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
loop_opts = outer_loop.process_argparse_options(opts)
return opts, loop_opts
def parse_args(argv):
p = argparse.ArgumentParser()
p.add_argument('--interval',
default=10,
type=int,
help='time (in seconds) between sampling system metrics')
p.add_argument(
'--root-setup',
action='store_true',
help='if this is set sysmon will run once to initialise configs in /etc '
'and then exit immediately. Used on GCE bots to bootstrap sysmon')
logs.add_argparse_options(p)
ts_mon.add_argparse_options(p)
outer_loop.add_argparse_options(p)
p.set_defaults(ts_mon_flush='manual')
opts = p.parse_args(argv)
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
loop_opts = outer_loop.process_argparse_options(opts)
return opts, loop_opts
def main(argv):
args = get_arguments(argv)
infra_libs.logs.process_argparse_options(args)
ts_mon.process_argparse_options(args)
arg_to_metric_type = (
(args.gauge, ts_mon.GaugeMetric),
(args.float, ts_mon.FloatMetric),
(args.string, ts_mon.StringMetric),
(args.bool, ts_mon.BooleanMetric),
(args.counter, ts_mon.CounterMetric),
(args.cumulative, ts_mon.CumulativeMetric),
)
for arg, metric in arg_to_metric_type:
set_metrics(arg, metric)
argfile_to_metric_type = (
(args.gauge_file, ts_mon.GaugeMetric),
(args.float_file, ts_mon.FloatMetric),
(args.string_file, ts_mon.StringMetric),
(args.bool_file, ts_mon.BooleanMetric),
(args.counter_file, ts_mon.CounterMetric),
(args.cumulative_file, ts_mon.CumulativeMetric),
)
for arg, metric in argfile_to_metric_type:
set_metrics_file(arg, metric)
ts_mon.flush()
def parse_args(argv):
if sys.platform == 'win32':
default_state_directory = 'C:\\chrome-infra\\service-state'
default_config_directory = 'C:\\chrome-infra\\service-config'
default_root_directory = 'C:\\infra-python'
else:
default_state_directory = '/var/run/infra-services'
default_config_directory = '/etc/infra-services'
default_root_directory = '/opt/infra-python'
p = argparse.ArgumentParser(
description='Starts and stops machine-wide infra services with arguments '
'from config files')
p.add_argument(
'--state-directory',
default=default_state_directory,
help='directory to store PID files (default %(default)s)')
p.add_argument(
'--config-directory',
default=default_config_directory,
help='directory to read JSON config files (default %(default)s)')
p.add_argument(
'--root-directory',
default=default_root_directory,
help='directory where the service_manager package is deployed. If this '
'package is updated the process will exit')
p.add_argument(
'--config-poll-interval',
default=10,
help='how frequently (in seconds) to poll the config directory')
p.add_argument(
'--service-poll-interval',
default=10,
help='how frequently (in seconds) to restart failed services')
p.add_argument(
'--root-setup',
action='store_true',
help='if this is set service_manager will run once to initialise configs '
'in /etc and then exit immediately. Used on GCE bots to bootstrap '
'service_manager')
logs.add_argparse_options(p)
ts_mon.add_argparse_options(p)
p.set_defaults(
ts_mon_target_type='task',
ts_mon_task_service_name='service_manager',
ts_mon_task_job_name=socket.getfqdn(),
)
opts = p.parse_args(argv)
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
return opts
def process_argparse_options(self, options):
"""Process any commandline arguments.
Args:
options: An argparse.Namespace object.
"""
logs.process_argparse_options(options)
ts_mon.process_argparse_options(options)
def parse_args(argv):
p = argparse.ArgumentParser()
group = p.add_mutually_exclusive_group(required=True)
group.add_argument(
'--url',
help='URL of one buildbot master to monitor')
group.add_argument('--build-dir',
help='location of the tools/build directory. Used with --hostname to get '
'the list of all buildbot masters on this host to monitor. Cannot be '
'used with --url')
p.add_argument('--hostname',
default=socket.getfqdn(),
help='override local hostname (currently %(default)s). Used with '
'--build-dir to get the list of all buildbot masters on this host to '
'monitor')
p.add_argument(
'--interval',
default=300, type=int,
help='time (in seconds) between sampling the buildbot master')
logs.add_argparse_options(p)
ts_mon.add_argparse_options(p)
outer_loop.add_argparse_options(p)
DEFAULT_ARG_VALUE = '(default)'
p.set_defaults(
ts_mon_flush='manual',
ts_mon_target_type='task',
ts_mon_task_service_name='mastermon',
ts_mon_task_job_name=DEFAULT_ARG_VALUE,
)
opts = p.parse_args(argv)
if opts.ts_mon_task_job_name == DEFAULT_ARG_VALUE:
# The ts_mon job name defaults to either the hostname when monitoring all
# masters on a host, or the name of the master extracted from the URL.
if opts.build_dir:
opts.ts_mon_task_job_name = opts.hostname
else:
parsed_url = urlparse.urlsplit(opts.url)
path_components = [x for x in parsed_url.path.split('/') if x]
if path_components:
opts.ts_mon_task_job_name = path_components[-1]
else:
opts.ts_mon_task_job_name = parsed_url.netloc
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
loop_opts = outer_loop.process_argparse_options(opts)
return opts, loop_opts
def process_argparse_options(self, options):
"""Process any commandline arguments.
Args:
options: An argparse.Namespace object.
"""
if self.USES_STANDARD_LOGGING:
logs.process_argparse_options(options)
if self.USES_TS_MON:
ts_mon.process_argparse_options(options)
def parse_args(argv):
p = argparse.ArgumentParser()
group = p.add_mutually_exclusive_group(required=True)
group.add_argument(
'--url',
help='URL of one buildbot master to monitor')
group.add_argument('--build-dir',
help='location of the tools/build directory. Used with --hostname to get '
'the list of all buildbot masters on this host to monitor. Cannot be '
'used with --url')
p.add_argument('--hostname',
default=socket.getfqdn(),
help='override local hostname (currently %(default)s). Used with '
'--build-dir to get the list of all buildbot masters on this host to '
'monitor')
p.add_argument(
'--interval',
default=60, type=int,
help='time (in seconds) between sampling the buildbot master')
logs.add_argparse_options(p)
ts_mon.add_argparse_options(p)
outer_loop.add_argparse_options(p)
DEFAULT_ARG_VALUE = '(default)'
p.set_defaults(
ts_mon_flush='manual',
ts_mon_target_type='task',
ts_mon_task_service_name='mastermon',
ts_mon_task_job_name=DEFAULT_ARG_VALUE,
)
opts = p.parse_args(argv)
if opts.ts_mon_task_job_name == DEFAULT_ARG_VALUE:
# The ts_mon job name defaults to either the hostname when monitoring all
# masters on a host, or the name of the master extracted from the URL.
if opts.build_dir:
opts.ts_mon_task_job_name = opts.hostname
else:
parsed_url = urlparse.urlsplit(opts.url)
path_components = [x for x in parsed_url.path.split('/') if x]
if path_components:
opts.ts_mon_task_job_name = path_components[-1]
else:
opts.ts_mon_task_job_name = parsed_url.netloc
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
loop_opts = outer_loop.process_argparse_options(opts)
return opts, loop_opts
def parse_args():
parser = argparse.ArgumentParser(
description='Launches master_manager for every master on a host. NOTE: '
'does not perform any action unless --prod is set.')
parser.add_argument('build_dir', nargs='?',
help='location of the tools/build directory')
parser.add_argument('--hostname',
default=socket.getfqdn(),
help='override local hostname (currently %(default)s)')
parser.add_argument('--json-file',
help='load desired master state from a file on disk')
parser.add_argument('--json-gitiles',
help='load desired master state from a gitiles location')
parser.add_argument('--netrc',
help='location of the netrc file when connecting to gitiles')
parser.add_argument('--command-timeout',
help='apply a timeout in seconds to each master_manager process')
parser.add_argument('--verify', action='store_true',
help='verify the desired master state JSON is valid, then exit')
parser.add_argument('--prod', action='store_true',
help='actually perform actions instead of doing a dry run')
parser.add_argument('--processes',
default=16, type=int,
help='maximum number of master_manager processes to run simultaneously '
'(default %(default)d)')
ts_mon.add_argparse_options(parser)
logs.add_argparse_options(parser)
parser.set_defaults(
ts_mon_target_type='task',
ts_mon_task_job_name=socket.getfqdn().split(".")[0],
ts_mon_task_service_name='master_manager_launcher',
ts_mon_flush_mode='manual',
)
args = parser.parse_args()
ts_mon.process_argparse_options(args)
logs.process_argparse_options(args)
if args.json_file and args.json_gitiles:
parser.error("Can't specify --json-file and --json-gitiles simultaneously")
if not args.json_gitiles and not args.json_file:
parser.error('Must specify either --json-gitiles or --json-file.')
if not args.verify:
if not args.build_dir:
parser.error('A build/ directory must be specified.')
return args
def main(argv): # pragma: no cover args = get_arguments(argv) infra_libs.logs.process_argparse_options(args) ts_mon.process_argparse_options(args) set_metrics(args.gauge, ts_mon.GaugeMetric) set_metrics(args.float, ts_mon.FloatMetric) set_metrics(args.string, ts_mon.StringMetric) set_metrics(args.bool, ts_mon.BooleanMetric) set_metrics(args.counter, ts_mon.CounterMetric) set_metrics(args.cumulative, ts_mon.CumulativeMetric) ts_mon.flush()
def parse_args(argv):
p = argparse.ArgumentParser(
description='Starts and stops machine-wide infra services with arguments '
'from config files')
p.add_argument(
'--state-directory',
default='/var/run/infra-services',
help='directory to store PID files (default %(default)s)')
p.add_argument(
'--config-directory',
default='/etc/infra-services',
help='directory to read JSON config files (default %(default)s)')
p.add_argument(
'--root-directory',
default='/opt/infra-python',
help='directory where the service_manager package is deployed. If this '
'package is updated the process will exit')
p.add_argument(
'--config-poll-interval',
default=10,
help='how frequently (in seconds) to poll the config directory')
p.add_argument(
'--service-poll-interval',
default=10,
help='how frequently (in seconds) to restart failed services')
p.add_argument(
'--root-setup',
action='store_true',
help='if this is set service_manager will run once to initialise configs '
'in /etc and then exit immediately. Used on GCE bots to bootstrap '
'service_manager')
logs.add_argparse_options(p)
ts_mon.add_argparse_options(p)
p.set_defaults(
ts_mon_target_type='task',
ts_mon_task_service_name='service_manager',
ts_mon_task_job_name=socket.getfqdn(),
)
opts = p.parse_args(argv)
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
return opts
def parse_args(args): # pragma: no cover
def check_url(s):
parsed = urlparse.urlparse(s)
if parsed.scheme not in ('https', 'git', 'file'):
raise argparse.ArgumentTypeError(
'Repo URL must use https, git or file protocol.')
if not parsed.path.strip('/'):
raise argparse.ArgumentTypeError('URL is missing a path?')
return git2.Repo(s)
parser = argparse.ArgumentParser('./run.py %s' % __package__)
parser.add_argument('--dry_run',
action='store_true',
help='Do not actually push anything.')
parser.add_argument('--repo_dir',
metavar='DIR',
default='gsubtreed_repos',
help=('The directory to use for git clones '
'(default: %(default)s)'))
parser.add_argument('--json_output',
metavar='PATH',
help='Path to write JSON with results of the run to')
parser.add_argument('repo',
nargs=1,
help='The url of the repo to act on.',
type=check_url)
logs.add_argparse_options(parser)
ts_mon.add_argparse_options(parser)
outer_loop.add_argparse_options(parser)
parser.set_defaults(
ts_mon_target_type='task',
ts_mon_task_service_name='gsubtreed',
)
opts = parser.parse_args(args)
repo = opts.repo[0]
repo.repos_dir = os.path.abspath(opts.repo_dir)
if not opts.ts_mon_task_job_name:
parsed_repo_url = urlparse.urlparse(repo.url)
opts.ts_mon_task_job_name = '%s%s' % (parsed_repo_url.netloc,
parsed_repo_url.path)
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
loop_opts = outer_loop.process_argparse_options(opts)
return Options(repo, loop_opts, opts.json_output, opts.dry_run)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument(
'-n',
'--nice',
type=int,
metavar='VALUE',
help='Set the nice level of the process to VALUE prior to execution.')
parser.add_argument(
'master_paths',
nargs='+',
help='The paths to the master base directories to monitor. Consider '
'the /path/to/build/masters/master.* wildcard to specify all of '
'them.')
logs.add_argparse_options(parser)
ts_mon.add_argparse_options(parser)
# Parse arguments.
args = parser.parse_args(args)
logs.process_argparse_options(args)
ts_mon.process_argparse_options(args)
# Try setting the nice value; if it fails, eat the error and continue.
if args.nice:
logging.debug('Setting process "nice" to: %d', args.nice)
try:
os.nice(args.nice)
except OSError as e:
logging.error('Failed to update "nice" to %d: %s', args.nice, e)
# Update global state calculations.
logging.info('Pulling master state from: %s', args.master_paths)
for master_path in args.master_paths:
master_name = master_path_to_name(master_path)
# Log to the target: buildbot/master/<master_name>
target = ts_mon.TaskTarget('buildbot/master', master_name,
args.ts_mon_task_region,
args.ts_mon_task_hostname,
args.ts_mon_task_number)
logging.info('Collecting log state for master "%s" at: %s',
master_name, master_path)
get_master_state(master_path, target)
logging.info('Flushing collected information.')
ts_mon.flush()
return 0
def parse_args(args): # pragma: no cover
parser = argparse.ArgumentParser('./run.py %s' % __package__)
parser.add_argument(
'-c',
'--configfile',
help='Local JSON poller configuration file to override '
'config file from luci-config.')
parser.add_argument(
'-d',
'--credentials_db',
help=
'File to use for OAuth2 credentials storage if not running on LUCI.')
parser.add_argument(
'--datadir',
default=DATADIR,
help='Directory where persistent app data should be stored.')
parser.add_argument(
'--dryrun',
action='store_true',
help='Don\'t update monorail issues or update issues to the bugdroid '
'appengine app.')
logs.add_argparse_options(parser)
ts_mon.add_argparse_options(parser)
outer_loop.add_argparse_options(parser)
parser.set_defaults(log_level=logging.DEBUG,
ts_mon_target_type='task',
ts_mon_task_service_name='bugdroid',
ts_mon_task_job_name='bugdroid_job')
opts = parser.parse_args(args)
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
loop_opts = outer_loop.process_argparse_options(opts)
# We need to include the logger ID (i.e. "%(name)s") in the formatter string.
# Override the root logging handler set by infra_libs.logs.
logging.root.handlers[0].setFormatter(
logging.Formatter(
'[%(severity)s%(iso8601)s %(process)d %(thread)d '
'%(fullModuleName)s:%(lineno)s] (%(name)s) %(message)s'))
return opts, loop_opts
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('-n', '--nice', type=int, metavar='VALUE',
help='Set the nice level of the process to VALUE prior to execution.')
parser.add_argument('master_paths', nargs='+',
help='The paths to the master base directories to monitor. Consider '
'the /path/to/build/masters/master.* wildcard to specify all of '
'them.')
logs.add_argparse_options(parser)
ts_mon.add_argparse_options(parser)
# Parse arguments.
args = parser.parse_args(args)
logs.process_argparse_options(args)
ts_mon.process_argparse_options(args)
# Try setting the nice value; if it fails, eat the error and continue.
if args.nice:
logging.debug('Setting process "nice" to: %d', args.nice)
try:
os.nice(args.nice)
except OSError as e:
logging.error('Failed to update "nice" to %d: %s', args.nice, e)
# Update global state calculations.
logging.info('Pulling master state from: %s', args.master_paths)
for master_path in args.master_paths:
master_name = master_path_to_name(master_path)
# Log to the target: buildbot/master/<master_name>
target = ts_mon.TaskTarget(
'buildbot/master',
master_name,
args.ts_mon_task_region,
args.ts_mon_task_hostname,
args.ts_mon_task_number)
logging.info('Collecting log state for master "%s" at: %s',
master_name, master_path)
get_master_state(master_path, target)
logging.info('Flushing collected information.')
ts_mon.flush()
return 0
def parse_args(args): # pragma: no cover
def check_url(s):
parsed = urlparse.urlparse(s)
if parsed.scheme not in ('https', 'git', 'file'):
raise argparse.ArgumentTypeError(
'Repo URL must use https, git or file protocol.')
if not parsed.path.strip('/'):
raise argparse.ArgumentTypeError('URL is missing a path?')
return git2.Repo(s)
parser = argparse.ArgumentParser('./run.py %s' % __package__)
parser.add_argument('--dry_run', action='store_true',
help='Do not actually push anything.')
parser.add_argument('--repo_dir', metavar='DIR', default='gsubtreed_repos',
help=('The directory to use for git clones '
'(default: %(default)s)'))
parser.add_argument('--json_output', metavar='PATH',
help='Path to write JSON with results of the run to')
parser.add_argument('repo', nargs=1, help='The url of the repo to act on.',
type=check_url)
logs.add_argparse_options(parser)
ts_mon.add_argparse_options(parser)
outer_loop.add_argparse_options(parser)
parser.set_defaults(
ts_mon_target_type='task',
ts_mon_task_service_name='gsubtreed',
)
opts = parser.parse_args(args)
repo = opts.repo[0]
repo.dry_run = opts.dry_run
repo.repos_dir = os.path.abspath(opts.repo_dir)
if not opts.ts_mon_task_job_name:
opts.ts_mon_task_job_name = urlparse.urlparse(repo.url).path
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
loop_opts = outer_loop.process_argparse_options(opts)
return Options(repo, loop_opts, opts.json_output)
def parse_args(args):
parser = argparse.ArgumentParser('python -m %s' % __package__)
parser.add_argument('--project', required=True)
parser.add_argument('--range', required=True)
logs.add_argparse_options(parser)
ts_mon.add_argparse_options(parser)
parser.set_defaults(
logs_directory='',
ts_mon_target_type='task',
ts_mon_task_service_name='cq_stats_uploader',
)
opts = parser.parse_args(args)
if not opts.ts_mon_task_job_name:
opts.ts_mon_task_job_name = '%s-%s' % (opts.project, opts.range)
logs.process_argparse_options(opts)
ts_mon.process_argparse_options(opts)
return opts
def parse_args():
parser = argparse.ArgumentParser(
description='Launches master_manager for every master on a host. NOTE: '
'does not perform any action unless --prod is set.')
parser.add_argument('build_dir',
nargs='?',
help='location of the tools/build directory')
parser.add_argument('--hostname',
default=socket.getfqdn(),
help='override local hostname (currently %(default)s)')
parser.add_argument('--json-file',
help='load desired master state from a file on disk')
parser.add_argument(
'--json-gitiles',
help='load desired master state from a gitiles location')
parser.add_argument(
'--netrc',
help='location of the netrc file when connecting to gitiles')
parser.add_argument(
'--command-timeout',
help='apply a timeout in seconds to each master_manager process')
parser.add_argument(
'--verify',
action='store_true',
help='verify the desired master state JSON is valid, then exit')
parser.add_argument(
'--prod',
action='store_true',
help='actually perform actions instead of doing a dry run')
parser.add_argument(
'--processes',
default=16,
type=int,
help='maximum number of master_manager processes to run simultaneously '
'(default %(default)d)')
ts_mon.add_argparse_options(parser)
logs.add_argparse_options(parser)
parser.set_defaults(
ts_mon_target_type='task',
ts_mon_task_job_name=socket.getfqdn().split(".")[0],
ts_mon_task_service_name='master_manager_launcher',
ts_mon_flush_mode='manual',
)
args = parser.parse_args()
ts_mon.process_argparse_options(args)
logs.process_argparse_options(args)
if args.json_file and args.json_gitiles:
parser.error(
"Can't specify --json-file and --json-gitiles simultaneously")
if not args.json_gitiles and not args.json_file:
parser.error('Must specify either --json-gitiles or --json-file.')
if not args.verify:
if not args.build_dir:
parser.error('A build/ directory must be specified.')
return args
def process_argparse_options(args): # pragma: no cover
infra_libs.logs.process_argparse_options(args)
event_mon.process_argparse_options(args)
ts_mon.process_argparse_options(args)
def process_argparse_options(args): # pragma: no cover infra_libs.logs.process_argparse_options(args) event_mon.process_argparse_options(args) # Put here because we want to send the presence metric as late as possible. ts_mon.process_argparse_options(args) _process_logrequest_path(args)
def process_argparse_options(args): # pragma: no cover
infra_libs.logs.process_argparse_options(args)
event_mon.process_argparse_options(args)
ts_mon.process_argparse_options(args)