def ImportMasterConfigs(master_name=None):
"""Import master configs.
Normally a slave can use GetActiveMaster() to find itself and
determine which ActiveMaster to use. In that case, the active
master name is passed in as an arg, and we only load the
site_config.py that defines it. When testing, the current "slave"
won't be found. In that case, we don't know which config to use, so
load them all. In either case, masters are assigned as attributes
to the config.Master object."""
for master in chromium_utils.ListMasters():
path = os.path.join(master, 'master_site_config.py')
if os.path.exists(path):
local_vars = {}
try:
execfile(path, local_vars)
# pylint: disable=W0703
except Exception, e:
# Naked exceptions are banned by the style guide but we are
# trying to be resilient here.
print >> sys.stderr, 'WARNING: cannot exec ' + path
print >> sys.stderr, e
for (symbol_name, symbol) in local_vars.items():
if inspect.isclass(local_vars[symbol_name]):
setattr(config.Master, symbol_name, symbol)
# If we have a master_name and it matches, set
# config.Master.active_master.
if master_name and master_name == symbol_name:
setattr(config.Master, 'active_master', symbol)
def main():
slaves = []
for master in chromium_utils.ListMasters():
masterbase = os.path.basename(master)
master_slaves = {}
execfile(os.path.join(master, 'slaves.cfg'), master_slaves)
for slave in master_slaves.get('slaves', []):
slave['master'] = masterbase
slaves.extend(master_slaves.get('slaves', []))
for slave in sorted(slaves, cmp=None, key=lambda x: x.get('hostname', '')):
slavename = slave.get('hostname')
if not slavename:
continue
master = slave.get('master', '?')
builder = slave.get('builder', '?')
if builder == []:
builder = '?'
osname = slave.get('os', '?')
if type(builder) is list:
for b in sorted(builder):
print '%-30s %-35s %-35s %-10s' % (slavename, master, b,
osname)
else:
print '%-30s %-35s %-35s %-10s' % (slavename, master, builder,
osname)
def main(argv):
hack_subprocess()
parser = optparse.OptionParser()
parser.add_option(
'--maxlines',
type=int,
default=40,
help='Max number of lines to send')
parser.add_option(
'--testing',
action='store_true',
help='Override to run on non production hardware')
parser.add_option('-v', '--verbose', action='store_true')
options, args = parser.parse_args(argv)
if args:
parser.error('Following arguments unsupported:' % args)
if not socket.getfqdn().endswith('.golo.chromium.org'):
if not options.testing:
parser.error('Not running on production hardware')
else:
print('WARNING: By running this program you agree to send all the data\n'
'it generates to Google owned servers and grant rights to use it\n'
'at will.\n'
'This program is meant to be used by Google employees only.\n'
'Use at your own risk and peril.\n')
if options.verbose:
level = logging.DEBUG
else:
level = logging.INFO
logging.basicConfig(
level=level,
format='%(asctime)s %(levelname)-7s %(threadName)-11s %(message)s',
datefmt='%m/%d %H:%M:%S')
masters_path = chromium_utils.ListMasters()
# Starts tail for each master.
threads = []
for master_path in masters_path:
name = os.path.basename(master_path)
name = (name
.replace('master.', '')
.replace('client.', '')
.replace('experimental', 'experi')
.replace('chromiumos', 'cros')
.replace('tryserver', 't')
.replace('toolchain', 'tlch')
.replace('chromium', 'c'))
thread = threading.Thread(
target=process,
args=(master_path, options.maxlines),
name=name)
thread.daemon = True
threads.append(thread)
thread.start()
for p in threads:
p.join()
return 0
def GetActiveMaster():
"""Parses all the slaves.cfg and returns the name of the active master
determined by the host name. Returns None otherwise."""
hostname = socket.getfqdn().split('.', 1)[0].lower()
for master in chromium_utils.ListMasters():
path = os.path.join(master, 'slaves.cfg')
if os.path.exists(path):
for slave in chromium_utils.RunSlavesCfg(path):
if slave.get('hostname', None) == hostname:
return slave['master']
def get_masters():
"""Return a pair of (mastername, path) for all masters found."""
# note: ListMasters uses master.cfg hardcoded as part of its search path
def parse_master_name(masterpath):
"""Returns a mastername from a pathname to a master."""
_, tail = os.path.split(masterpath)
sep = '.'
hdr = 'master'
chunks = tail.split(sep)
if not chunks or chunks[0] != hdr or len(chunks) < 2:
raise ValueError('unable to parse mastername from path! (%s)' %
tail)
return sep.join(chunks[1:])
return [(parse_master_name(m), m) for m in chromium_utils.ListMasters()]
def TemporaryMasterPasswords():
all_paths = [os.path.join(BUILD_DIR, 'site_config', '.bot_password')]
all_paths.extend(os.path.join(path, '.apply_issue_password')
for path in chromium_utils.ListMasters())
created_paths = []
for path in all_paths:
if not os.path.exists(path):
try:
os.symlink(os.devnull, path)
created_paths.append(path)
except OSError:
pass
yield
for path in created_paths:
try:
os.remove(path)
except OSError:
print 'WARNING: Could not remove %s!' % path
def get_masters(parser, options):
"""Given parser options, find suitable master directories."""
paths = []
masters_path = chromium_utils.ListMasters()
# Populates by defaults with every masters with a twistd.pid, thus has
# been started.
if not options.master:
for m_p in masters_path:
if os.path.isfile(os.path.join(m_p, 'twistd.pid')):
paths.append(m_p)
elif options.master == 'all':
paths.extend(masters_path)
elif options.master in (os.path.basename(p) for p in masters_path):
full_master = next(p for p in masters_path
if os.path.basename(p) == options.master)
paths.append(full_master)
else:
parser.error('Unknown master \'%s\'.\nChoices are:\n %s' %
(options.master, '\n '.join(
(os.path.basename(p) for p in masters_path))))
return paths
def TemporaryMasterPasswords():
all_paths = [os.path.join(BASE_DIR, 'site_config', '.bot_password')]
all_paths.extend(
os.path.join(path, '.apply_issue_password')
for path in chromium_utils.ListMasters())
created_paths = []
for path in all_paths:
if not os.path.exists(path):
try:
with open(path, 'w') as f:
f.write('reindeer flotilla\n')
created_paths.append(path)
except OSError:
pass
try:
yield
finally:
for path in created_paths:
try:
os.remove(path)
except OSError:
print 'WARNING: Could not remove %s!' % path
def Main(argv):
usage = """%prog [options]
Note: t is replaced with 'tryserver', 'c' with chromium' and
co with 'chromiumos', 'cr' with chrome, 'cros' with 'chromeos'."""
# Generate the list of available masters. We want any with slaves.
masters_path = chromium_utils.ListMasters(cue='slaves.cfg')
masters = [os.path.basename(f) for f in masters_path]
# Strip off 'master.'
masters = [re.match(r'(master\.|)(.*)', m).group(2) for m in masters]
parser = optparse.OptionParser(usage=usage)
group = optparse.OptionGroup(parser, 'Slaves to process')
group.add_option('-x',
'--master',
default=[],
action='append',
help='Master to use to load the slaves list.')
group.add_option('-k',
'--kind',
action='append',
default=[],
help='Only slaves with a substring present in a builder')
group.add_option('-b',
'--builder',
action='append',
default=[],
help='Only slaves attached to a specific builder')
group.add_option('--os',
action='append',
default=[],
help='Only slaves using a specific OS')
group.add_option('--os-version',
action='append',
default=[],
help='Only slaves using a specific OS version')
group.add_option('-s', '--slave', action='append', default=[])
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Output format')
group.add_option('-n',
'--name',
default='host',
dest='fmt',
action='store_const',
const='host',
help='Output slave hostname')
group.add_option('-r',
'--raw',
dest='fmt',
action='store_const',
const='raw',
help='Output all slave info')
group.add_option('-t',
'--assignment',
dest='fmt',
action='store_const',
const='assignment',
help='Output slave tasks too')
group.add_option('',
'--botmap',
dest='fmt',
action='store_const',
const='botmap',
help='Output botmap style')
group.add_option('-w',
'--waterfall',
dest='fmt',
action='store_const',
const='waterfall',
help='Output slave master and tasks')
parser.add_option_group(group)
options, args = parser.parse_args(argv)
if args:
parser.error('Unknown argument(s): %s\n' % args)
slaves = []
if not options.master:
# Populates by default with every master with a twistd.pid, thus has
# been started.
for m_p in masters_path:
if os.path.isfile(os.path.join(m_p, 'twistd.pid')):
LoadMaster(slaves, m_p)
else:
for master in options.master:
if master == 'allcros':
for m in (m for m in masters
if (m.startswith('chromeos')
or m.startswith('chromiumos'))):
LoadMaster(slaves, masters_path[masters.index(m)])
elif master == 'all':
for m_p in masters_path:
LoadMaster(slaves, m_p)
slaves.sort(key=lambda x: (x['mastername'], x.get('hostname')))
else:
if not master in masters:
master = ProcessShortName(master)
if not master in masters:
parser.error('Unknown master \'%s\'.\nChoices are: %s' %
(master, ', '.join(masters)))
LoadMaster(slaves, masters_path[masters.index(master)])
if options.kind:
def kind_interested_in_any(builders):
if isinstance(builders, basestring):
return any(builders.find(k) >= 0 for k in options.kind)
return any(
any(x.find(k) >= 0 for k in options.kind) for x in builders)
slaves = [
s for s in slaves if kind_interested_in_any(s.get('builder'))
]
if options.builder:
builders = set(options.builder)
def builder_interested_in_any(x):
return builders.intersection(set(x))
slaves = [
s for s in slaves if builder_interested_in_any(s.get('builder'))
]
if options.os:
selected = set(options.os)
slaves = [s for s in slaves if s.get('os', 'unknown') in selected]
if options.os_version:
selected = set(options.os_version)
slaves = [s for s in slaves if s.get('version', 'unknown') in selected]
if options.slave:
selected = set(options.slave)
slaves = [s for s in slaves if s.get('hostname') in selected]
for s in slaves:
if options.fmt == 'raw':
print s
elif options.fmt == 'assignment':
print s.get('hostname',
'unknown'), ':', s.get('builder', 'unknown')
elif options.fmt == 'waterfall':
print s.get('hostname', 'unknown'), ':', s.get('master', 'unknown'), \
':', s.get('builder', 'unknown')
elif options.fmt == 'master':
print s.get('hostname', 'unknown'), ':', s.get('mastername', 'unknown'), \
':', s.get('builder', 'unknown')
elif options.fmt == 'botmap':
host = s.get('hostname')
if host:
master = s.get('mastername') or '?'
slaveos = s.get('os') or '?'
pathsep = '\\' if s.get('os') == 'win' else '/'
if 'subdir' in s:
d = pathsep + 'c' + pathsep + s['subdir']
else:
d = pathsep + 'b'
builders = s.get('builder') or '?'
if type(builders) is not list:
builders = [builders]
for b in sorted(builders):
print '%-30s %-20s %-35s %-35s %-10s' % (host, d, master,
b, slaveos)
else:
print s.get('hostname', 'unknown')
def Main(argv):
usage = """%prog [options]
Sample usage:
%prog -x t.c --index 5 -i -W "cmd rd /q /s c:\\b\\build\\slave\\win\\build"
%prog -x chromium -l -c
Note: t is replaced with 'tryserver', 'c' with chromium' and
co with 'chromiumos'."""
# Generate the list of available masters.
masters_path = chromium_utils.ListMasters()
masters = [os.path.basename(f) for f in masters_path]
# Strip off 'master.'
masters = [re.match(r'(master\.|)(.*)', m).group(2) for m in masters]
parser = optparse.OptionParser(usage=usage)
group = optparse.OptionGroup(parser, 'Slaves to process')
group.add_option(
'-x',
'--master',
help=('Master to use to load the slaves list. If omitted, all masters '
'that were started at least once are included. If \'all\', all '
'masters are selected. Choices are: %s.') % ', '.join(masters))
group.add_option('-w', '--win', action='store_true')
group.add_option('-l', '--linux', action='store_true')
group.add_option('-m', '--mac', action='store_true')
group.add_option('--bits', help='Slave os bitness', type='int')
group.add_option('--version', help='Slave os version')
group.add_option('-b',
'--builder',
help='Only slaves attached to a specific builder')
group.add_option('--min', type='int')
group.add_option('--max', type='int', help='Inclusive')
group.add_option('--index', type='int', help='execute on only one slave')
group.add_option('-s', '--slave', action='append')
group.add_option('--raw',
help='Line separated list of slaves to use. Must '
'still use -l, -m or -w to let the script '
'know what command to run')
parser.add_option_group(group)
parser.add_option('-i',
'--ignore_failure',
action='store_true',
help='Continue even if ssh returned an error')
group = optparse.OptionGroup(parser, 'Premade commands')
group.add_option('-c', '--clobber', action='store_true')
group.add_option('-r', '--restart', action='store_true')
group.add_option('--revert',
action='store_true',
help='Execute gclient revert')
group.add_option('--sync_scripts', action='store_true')
group.add_option('--taskkill', action='store_true')
group.add_option('--scp',
action='store_true',
help='with the source and dest files')
group.add_option('-q',
'--quiet',
action='store_true',
help='Quiet mode - do not print the commands')
group.add_option(
'-p',
'--print_only',
action='store_true',
help='Print which slaves would have been processed but do '
'nothing. With no command, just print the list of '
'slaves for the given platform(s).')
group.add_option('-N',
'--no_cygwin',
action='store_true',
help='By default cygwin\'s bash is called to execute the '
'command')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Custom commands')
group.add_option('-W', '--win_cmd', help='Run a custom command instead')
group.add_option('-L', '--linux_cmd')
group.add_option('-M', '--mac_cmd')
parser.add_option_group(group)
options, args = parser.parse_args(argv)
# If a command is specified, the corresponding platform is automatically
# enabled.
if options.linux_cmd:
options.linux = True
if options.mac_cmd:
options.mac = True
if options.win_cmd:
options.win = True
if options.raw:
# Remove extra spaces and empty lines.
options.slave = filter(None,
(s.strip() for s in open(options.raw, 'r')))
if not options.slave:
if not options.master:
# Populates by defaults with every masters with a twistd.pid, thus has
# been started.
slaves = []
for m_p in masters_path:
if os.path.isfile(os.path.join(m_p, 'twistd.pid')):
slaves.extend(
chromium_utils.RunSlavesCfg(
os.path.join(m_p, 'slaves.cfg')))
slaves = slaves_list.BaseSlavesList(slaves)
elif options.master == 'all':
slaves = []
for m_p in masters_path:
slaves.extend(
chromium_utils.RunSlavesCfg(os.path.join(
m_p, 'slaves.cfg')))
slaves = slaves_list.BaseSlavesList(slaves)
else:
if not options.master in masters:
options.master = ProcessShortName(options.master)
if not options.master in masters:
parser.error('Unknown master \'%s\'.\nChoices are: %s' %
(options.master, ', '.join(masters)))
master_path = masters_path[masters.index(options.master)]
slaves = slaves_list.SlavesList(
os.path.join(master_path, 'slaves.cfg'))
def F(os_type):
out = slaves.GetSlaves(os=os_type,
bits=options.bits,
version=options.version,
builder=options.builder)
# Skips slave without a hostname.
return [s.get('hostname') for s in out if s.get('hostname')]
options.win_names = F('win')
options.linux_names = F('linux')
options.mac_names = F('mac')
else:
slaves = options.slave
options.win_names = slaves
options.linux_names = slaves
options.mac_names = slaves
if not options.linux and not options.mac and not options.win:
parser.print_help()
return 0
if options.index:
options.min = options.index
options.max = options.index
if options.scp:
if len(args) != 2:
parser.error('Need 2 args')
return RunSCP(options, args[0], args[1])
if args:
parser.error('Only --scp expects arguments')
if options.restart:
return Restart(options)
elif options.clobber:
return Clobber(options)
elif options.sync_scripts:
return SyncScripts(options)
elif options.taskkill:
return TaskKill(options)
elif options.revert:
return Revert(options)
elif options.print_only and not (options.win_cmd or options.linux_cmd
or options.mac_cmd):
names_list = []
if not options.min:
options.min = 1
if options.win:
max_i = len(options.win_names)
if options.max:
max_i = options.max
names_list += options.win_names[options.min - 1:max_i]
if options.linux:
max_i = len(options.linux_names)
if options.max:
max_i = options.max
names_list += options.linux_names[options.min - 1:max_i]
if options.mac:
max_i = len(options.mac_names)
if options.max:
max_i = options.max
names_list += options.mac_names[options.min - 1:max_i]
print '\n'.join(names_list)
else:
if ((options.win and not options.win_cmd)
or (options.linux and not options.linux_cmd)
or (options.mac and not options.mac_cmd)):
parser.error('Need to specify a command')
return RunSSH(options)