def get_slaves(master_paths, slavelist):
"""Return slaves split up by OS.
Takes a list of master paths and an optional slave whitelist."""
slavedict = {}
for path in master_paths:
for slave in chromium_utils.GetSlavesFromMasterPath(path):
if 'hostname' in slave:
slavedict[slave['hostname']] = slave
slaves = slaves_list.BaseSlavesList(slavedict.values())
def F(os_type):
out = slaves.GetSlaves(os=os_type)
named_slaves = [s.get('hostname') for s in out]
if slavelist:
return [s for s in named_slaves if s in slavelist]
else:
return named_slaves
slave_dict = {}
slave_dict['win'] = list(set(F('win')))
slave_dict['linux'] = list(set(F('linux')))
slave_dict['mac'] = list(set(F('mac')))
return slave_dict
def _Populate(BuildmasterConfig, builders, active_master_cls):
m_annotator = annotator_factory.AnnotatorFactory(active_master_cls)
c = BuildmasterConfig
c['logCompressionLimit'] = False
c['projectName'] = active_master_cls.project_name
c['projectURL'] = Master.project_url
c['buildbotURL'] = active_master_cls.buildbot_url
# This sets c['db_url'] to the database connect string in found in
# the .dbconfig in the master directory, if it exists. If this is
# a production host, it must exist.
chromium_utils.DatabaseSetup(
c, require_dbconfig=active_master_cls.is_production_host)
c['builders'] = _ComputeBuilders(builders, m_annotator)
c['schedulers'] = _ComputeSchedulers(builders)
c['change_source'], tag_comparator = _ComputeChangeSourceAndTagComparator(
builders)
# The 'slaves' list defines the set of allowable buildslaves. List all the
# slaves registered to a builder. Remove dupes.
c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
Master.GetBotPassword())
# This does some sanity checks on the configuration.
slaves = slaves_list.BaseSlavesList(
chromium_utils.GetSlavesFromBuilders(builders),
builders['master_classname'])
master_utils.VerifySetup(c, slaves)
default_public_html = os.path.join(chromium_utils.BUILD_DIR, 'masters',
'master.chromium', 'public_html')
public_html = builders.get('public_html', default_public_html)
# Adds common status and tools to this master.
# TODO: Look at the logic in this routine to see if any of the logic
# in this routine can be moved there to simplify things.
master_utils.AutoSetupMaster(
c,
active_master_cls,
public_html=public_html,
templates=builders['templates'],
tagComparator=tag_comparator,
enable_http_status_push=active_master_cls.is_production_host)
# TODO: AutoSetupMaster's settings for the following are too low to be
# useful for most projects. We should fix that.
c['buildHorizon'] = 3000
c['logHorizon'] = 3000
# Must be at least 2x the number of slaves.
c['eventHorizon'] = 200
def Main(argv):
usage = """%prog [options]
Sample usage:
%prog -x t.c --index 5 -i -W "cmd rd /q /s c:\\b\\build\\slave\\win\\build"
%prog -x chromium -l -c
Note: t is replaced with 'tryserver', 'c' with chromium' and
co with 'chromiumos'."""
# Generate the list of available masters.
masters_path = chromium_utils.ListMasters()
masters = [os.path.basename(f) for f in masters_path]
# Strip off 'master.'
masters = [re.match(r'(master\.|)(.*)', m).group(2) for m in masters]
parser = optparse.OptionParser(usage=usage)
group = optparse.OptionGroup(parser, 'Slaves to process')
group.add_option(
'-x',
'--master',
help=('Master to use to load the slaves list. If omitted, all masters '
'that were started at least once are included. If \'all\', all '
'masters are selected. Choices are: %s.') % ', '.join(masters))
group.add_option('-w', '--win', action='store_true')
group.add_option('-l', '--linux', action='store_true')
group.add_option('-m', '--mac', action='store_true')
group.add_option('--bits', help='Slave os bitness', type='int')
group.add_option('--version', help='Slave os version')
group.add_option('-b',
'--builder',
help='Only slaves attached to a specific builder')
group.add_option('--min', type='int')
group.add_option('--max', type='int', help='Inclusive')
group.add_option('--index', type='int', help='execute on only one slave')
group.add_option('-s', '--slave', action='append')
group.add_option('--raw',
help='Line separated list of slaves to use. Must '
'still use -l, -m or -w to let the script '
'know what command to run')
parser.add_option_group(group)
parser.add_option('-i',
'--ignore_failure',
action='store_true',
help='Continue even if ssh returned an error')
group = optparse.OptionGroup(parser, 'Premade commands')
group.add_option('-c', '--clobber', action='store_true')
group.add_option('-r', '--restart', action='store_true')
group.add_option('--revert',
action='store_true',
help='Execute gclient revert')
group.add_option('--sync_scripts', action='store_true')
group.add_option('--taskkill', action='store_true')
group.add_option('--scp',
action='store_true',
help='with the source and dest files')
group.add_option('-q',
'--quiet',
action='store_true',
help='Quiet mode - do not print the commands')
group.add_option(
'-p',
'--print_only',
action='store_true',
help='Print which slaves would have been processed but do '
'nothing. With no command, just print the list of '
'slaves for the given platform(s).')
group.add_option('-N',
'--no_cygwin',
action='store_true',
help='By default cygwin\'s bash is called to execute the '
'command')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Custom commands')
group.add_option('-W', '--win_cmd', help='Run a custom command instead')
group.add_option('-L', '--linux_cmd')
group.add_option('-M', '--mac_cmd')
parser.add_option_group(group)
options, args = parser.parse_args(argv)
# If a command is specified, the corresponding platform is automatically
# enabled.
if options.linux_cmd:
options.linux = True
if options.mac_cmd:
options.mac = True
if options.win_cmd:
options.win = True
if options.raw:
# Remove extra spaces and empty lines.
options.slave = filter(None,
(s.strip() for s in open(options.raw, 'r')))
if not options.slave:
if not options.master:
# Populates by defaults with every masters with a twistd.pid, thus has
# been started.
slaves = []
for m_p in masters_path:
if os.path.isfile(os.path.join(m_p, 'twistd.pid')):
slaves.extend(
chromium_utils.RunSlavesCfg(
os.path.join(m_p, 'slaves.cfg')))
slaves = slaves_list.BaseSlavesList(slaves)
elif options.master == 'all':
slaves = []
for m_p in masters_path:
slaves.extend(
chromium_utils.RunSlavesCfg(os.path.join(
m_p, 'slaves.cfg')))
slaves = slaves_list.BaseSlavesList(slaves)
else:
if not options.master in masters:
options.master = ProcessShortName(options.master)
if not options.master in masters:
parser.error('Unknown master \'%s\'.\nChoices are: %s' %
(options.master, ', '.join(masters)))
master_path = masters_path[masters.index(options.master)]
slaves = slaves_list.SlavesList(
os.path.join(master_path, 'slaves.cfg'))
def F(os_type):
out = slaves.GetSlaves(os=os_type,
bits=options.bits,
version=options.version,
builder=options.builder)
# Skips slave without a hostname.
return [s.get('hostname') for s in out if s.get('hostname')]
options.win_names = F('win')
options.linux_names = F('linux')
options.mac_names = F('mac')
else:
slaves = options.slave
options.win_names = slaves
options.linux_names = slaves
options.mac_names = slaves
if not options.linux and not options.mac and not options.win:
parser.print_help()
return 0
if options.index:
options.min = options.index
options.max = options.index
if options.scp:
if len(args) != 2:
parser.error('Need 2 args')
return RunSCP(options, args[0], args[1])
if args:
parser.error('Only --scp expects arguments')
if options.restart:
return Restart(options)
elif options.clobber:
return Clobber(options)
elif options.sync_scripts:
return SyncScripts(options)
elif options.taskkill:
return TaskKill(options)
elif options.revert:
return Revert(options)
elif options.print_only and not (options.win_cmd or options.linux_cmd
or options.mac_cmd):
names_list = []
if not options.min:
options.min = 1
if options.win:
max_i = len(options.win_names)
if options.max:
max_i = options.max
names_list += options.win_names[options.min - 1:max_i]
if options.linux:
max_i = len(options.linux_names)
if options.max:
max_i = options.max
names_list += options.linux_names[options.min - 1:max_i]
if options.mac:
max_i = len(options.mac_names)
if options.max:
max_i = options.max
names_list += options.mac_names[options.min - 1:max_i]
print '\n'.join(names_list)
else:
if ((options.win and not options.win_cmd)
or (options.linux and not options.linux_cmd)
or (options.mac and not options.mac_cmd)):
parser.error('Need to specify a command')
return RunSSH(options)