def get_slaves(master_paths, slavelist):
"""Return slaves split up by OS.
Takes a list of master paths and an optional slave whitelist."""
slavedict = {}
for path in master_paths:
for slave in chromium_utils.GetSlavesFromMasterPath(path):
if 'hostname' in slave:
slavedict[slave['hostname']] = slave
slaves = slaves_list.BaseSlavesList(slavedict.values())
def F(os_type):
out = slaves.GetSlaves(os=os_type)
named_slaves = [s.get('hostname') for s in out]
if slavelist:
return [s for s in named_slaves if s in slavelist]
else:
return named_slaves
slave_dict = {}
slave_dict['win'] = list(set(F('win')))
slave_dict['linux'] = list(set(F('linux')))
slave_dict['mac'] = list(set(F('mac')))
return slave_dict
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument(
'-i',
'--input',
metavar='PATH',
type=argparse.FileType('r'),
default=None,
help='Read slave names from this file (use - for STDIN).')
parser.add_argument(
'slave_names',
nargs='*',
help='Individual names of slaves. Leave blank to read from STDIN.')
opts = parser.parse_args(args)
slave_names = set(opts.slave_names)
if not slave_names and not opts.input:
opts.input = sys.stdin
if opts.input:
slave_names.update(s.strip() for s in opts.input.read().split())
opts.input.close()
slaves = {}
for path in chromium_utils.ListMastersWithSlaves():
for slave in chromium_utils.GetSlavesFromMasterPath(path):
hostname = slave.get('hostname')
if hostname in slave_names:
slaves[hostname] = slave
json.dump(slaves, sys.stdout, indent=1, sort_keys=True)
def LoadMaster(slaves, path):
"""Add list of slaves for a given master to 'slaves'.
Args:
slaves(list): this list is extended with dicts describing each slave from
the specified master. *This is the output of this function*.
path(str): path to a master directory
(e.g. '<leading dirs>/build/masters/master.chromium.infra/')
"""
cur_slaves = chromium_utils.GetSlavesFromMasterPath(path)
cur_master = os.path.basename(path)
for cur_slave in cur_slaves:
cur_slave['mastername'] = cur_master
slaves.extend(cur_slaves)
def LoadMaster(slaves, path):
cur_slaves = chromium_utils.GetSlavesFromMasterPath(path)
cur_master = os.path.basename(path)
for cur_slave in cur_slaves:
cur_slave['mastername'] = cur_master
slaves.extend(cur_slaves)
def Main(argv):
usage = """%prog [options]
Sample usage:
%prog -x t.c --index 5 -i -W "cmd rd /q /s c:\\b\\build\\slave\\win\\build"
%prog -x chromium -l -c
Note: t is replaced with 'tryserver', 'c' with chromium' and
co with 'chromiumos'."""
# Generate the list of available masters.
masters_path = chromium_utils.ListMasters()
masters = [os.path.basename(f) for f in masters_path]
# Strip off 'master.'
masters = [re.match(r'(master\.|)(.*)', m).group(2) for m in masters]
parser = optparse.OptionParser(usage=usage)
group = optparse.OptionGroup(parser, 'Slaves to process')
group.add_option(
'-x',
'--master',
help=('Master to use to load the slaves list. If omitted, all masters '
'that were started at least once are included. If \'all\', all '
'masters are selected. Choices are: %s.') % ', '.join(masters))
group.add_option('-w', '--win', action='store_true')
group.add_option('-l', '--linux', action='store_true')
group.add_option('-m', '--mac', action='store_true')
group.add_option('--bits', help='Slave os bitness', type='int')
group.add_option('--version', help='Slave os version')
group.add_option('-b',
'--builder',
help='Only slaves attached to a specific builder')
group.add_option('--min', type='int')
group.add_option('--max', type='int', help='Inclusive')
group.add_option('--index', type='int', help='execute on only one slave')
group.add_option('-s', '--slave', action='append')
group.add_option('--raw',
help='Line separated list of slaves to use. Must '
'still use -l, -m or -w to let the script '
'know what command to run')
parser.add_option_group(group)
parser.add_option('-i',
'--ignore_failure',
action='store_true',
help='Continue even if ssh returned an error')
group = optparse.OptionGroup(parser, 'Premade commands')
group.add_option('-c', '--clobber', action='store_true')
group.add_option('-r', '--restart', action='store_true')
group.add_option('--revert',
action='store_true',
help='Execute gclient revert')
group.add_option('--sync_scripts', action='store_true')
group.add_option('--taskkill', action='store_true')
group.add_option('--scp',
action='store_true',
help='with the source and dest files')
group.add_option('-q',
'--quiet',
action='store_true',
help='Quiet mode - do not print the commands')
group.add_option(
'-p',
'--print_only',
action='store_true',
help='Print which slaves would have been processed but do '
'nothing. With no command, just print the list of '
'slaves for the given platform(s).')
group.add_option('-N',
'--no_cygwin',
action='store_true',
help='By default cygwin\'s bash is called to execute the '
'command')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Custom commands')
group.add_option('-W', '--win_cmd', help='Run a custom command instead')
group.add_option('-L', '--linux_cmd')
group.add_option('-M', '--mac_cmd')
parser.add_option_group(group)
options, args = parser.parse_args(argv)
# If a command is specified, the corresponding platform is automatically
# enabled.
if options.linux_cmd:
options.linux = True
if options.mac_cmd:
options.mac = True
if options.win_cmd:
options.win = True
if options.raw:
# Remove extra spaces and empty lines.
options.slave = filter(None,
(s.strip() for s in open(options.raw, 'r')))
if not options.slave:
if not options.master:
# Populates by defaults with every masters with a twistd.pid, thus has
# been started.
slaves = []
for m_p in masters_path:
if os.path.isfile(os.path.join(m_p, 'twistd.pid')):
slaves.extend(chromium_utils.GetSlavesFromMasterPath(m_p))
slaves = slaves_list.BaseSlavesList(slaves)
elif options.master == 'all':
slaves = []
for m_p in masters_path:
slaves.extend(chromium_utils.GetSlavesFromMasterPath(m_p))
slaves = slaves_list.BaseSlavesList(slaves)
else:
if not options.master in masters:
options.master = ProcessShortName(options.master)
if not options.master in masters:
parser.error('Unknown master \'%s\'.\nChoices are: %s' %
(options.master, ', '.join(masters)))
master_path = masters_path[masters.index(options.master)]
slaves = chromium_utils.GetSlavesFromMasterPath(master_path)
slaves = slaves_list.BaseSlavesList(slaves)
def F(os_type):
out = slaves.GetSlaves(os=os_type,
bits=options.bits,
version=options.version,
builder=options.builder)
# Skips slave without a hostname.
return [s.get('hostname') for s in out if s.get('hostname')]
options.win_names = F('win')
options.linux_names = F('linux')
options.mac_names = F('mac')
else:
slaves = options.slave
options.win_names = slaves
options.linux_names = slaves
options.mac_names = slaves
if not options.linux and not options.mac and not options.win:
parser.print_help()
return 0
if options.index:
options.min = options.index
options.max = options.index
if options.scp:
if len(args) != 2:
parser.error('Need 2 args')
return RunSCP(options, args[0], args[1])
if args:
parser.error('Only --scp expects arguments')
if options.restart:
return Restart(options)
elif options.clobber:
return Clobber(options)
elif options.sync_scripts:
return SyncScripts(options)
elif options.taskkill:
return TaskKill(options)
elif options.revert:
return Revert(options)
elif options.print_only and not (options.win_cmd or options.linux_cmd
or options.mac_cmd):
names_list = []
if not options.min:
options.min = 1
if options.win:
max_i = len(options.win_names)
if options.max:
max_i = options.max
names_list += options.win_names[options.min - 1:max_i]
if options.linux:
max_i = len(options.linux_names)
if options.max:
max_i = options.max
names_list += options.linux_names[options.min - 1:max_i]
if options.mac:
max_i = len(options.mac_names)
if options.max:
max_i = options.max
names_list += options.mac_names[options.min - 1:max_i]
print '\n'.join(names_list)
else:
if ((options.win and not options.win_cmd)
or (options.linux and not options.linux_cmd)
or (options.mac and not options.mac_cmd)):
parser.error('Need to specify a command')
return RunSSH(options)