def ImportMasterConfigs(master_name=None): """Import master configs. Normally a slave can use GetActiveMaster() to find itself and determine which ActiveMaster to use. In that case, the active master name is passed in as an arg, and we only load the site_config.py that defines it. When testing, the current "slave" won't be found. In that case, we don't know which config to use, so load them all. In either case, masters are assigned as attributes to the config.Master object.""" for master in chromium_utils.ListMasters(): path = os.path.join(master, 'master_site_config.py') if os.path.exists(path): local_vars = {} try: execfile(path, local_vars) # pylint: disable=W0703 except Exception, e: # Naked exceptions are banned by the style guide but we are # trying to be resilient here. print >> sys.stderr, 'WARNING: cannot exec ' + path print >> sys.stderr, e for (symbol_name, symbol) in local_vars.items(): if inspect.isclass(local_vars[symbol_name]): setattr(config.Master, symbol_name, symbol) # If we have a master_name and it matches, set # config.Master.active_master. if master_name and master_name == symbol_name: setattr(config.Master, 'active_master', symbol)
def main(): slaves = [] for master in chromium_utils.ListMasters(): masterbase = os.path.basename(master) master_slaves = {} execfile(os.path.join(master, 'slaves.cfg'), master_slaves) for slave in master_slaves.get('slaves', []): slave['master'] = masterbase slaves.extend(master_slaves.get('slaves', [])) for slave in sorted(slaves, cmp=None, key=lambda x: x.get('hostname', '')): slavename = slave.get('hostname') if not slavename: continue master = slave.get('master', '?') builder = slave.get('builder', '?') if builder == []: builder = '?' osname = slave.get('os', '?') if type(builder) is list: for b in sorted(builder): print '%-30s %-35s %-35s %-10s' % (slavename, master, b, osname) else: print '%-30s %-35s %-35s %-10s' % (slavename, master, builder, osname)
def main(argv): hack_subprocess() parser = optparse.OptionParser() parser.add_option( '--maxlines', type=int, default=40, help='Max number of lines to send') parser.add_option( '--testing', action='store_true', help='Override to run on non production hardware') parser.add_option('-v', '--verbose', action='store_true') options, args = parser.parse_args(argv) if args: parser.error('Following arguments unsupported:' % args) if not socket.getfqdn().endswith('.golo.chromium.org'): if not options.testing: parser.error('Not running on production hardware') else: print('WARNING: By running this program you agree to send all the data\n' 'it generates to Google owned servers and grant rights to use it\n' 'at will.\n' 'This program is meant to be used by Google employees only.\n' 'Use at your own risk and peril.\n') if options.verbose: level = logging.DEBUG else: level = logging.INFO logging.basicConfig( level=level, format='%(asctime)s %(levelname)-7s %(threadName)-11s %(message)s', datefmt='%m/%d %H:%M:%S') masters_path = chromium_utils.ListMasters() # Starts tail for each master. threads = [] for master_path in masters_path: name = os.path.basename(master_path) name = (name .replace('master.', '') .replace('client.', '') .replace('experimental', 'experi') .replace('chromiumos', 'cros') .replace('tryserver', 't') .replace('toolchain', 'tlch') .replace('chromium', 'c')) thread = threading.Thread( target=process, args=(master_path, options.maxlines), name=name) thread.daemon = True threads.append(thread) thread.start() for p in threads: p.join() return 0
def GetActiveMaster(): """Parses all the slaves.cfg and returns the name of the active master determined by the host name. Returns None otherwise.""" hostname = socket.getfqdn().split('.', 1)[0].lower() for master in chromium_utils.ListMasters(): path = os.path.join(master, 'slaves.cfg') if os.path.exists(path): for slave in chromium_utils.RunSlavesCfg(path): if slave.get('hostname', None) == hostname: return slave['master']
def get_masters(): """Return a pair of (mastername, path) for all masters found.""" # note: ListMasters uses master.cfg hardcoded as part of its search path def parse_master_name(masterpath): """Returns a mastername from a pathname to a master.""" _, tail = os.path.split(masterpath) sep = '.' hdr = 'master' chunks = tail.split(sep) if not chunks or chunks[0] != hdr or len(chunks) < 2: raise ValueError('unable to parse mastername from path! (%s)' % tail) return sep.join(chunks[1:]) return [(parse_master_name(m), m) for m in chromium_utils.ListMasters()]
def TemporaryMasterPasswords(): all_paths = [os.path.join(BUILD_DIR, 'site_config', '.bot_password')] all_paths.extend(os.path.join(path, '.apply_issue_password') for path in chromium_utils.ListMasters()) created_paths = [] for path in all_paths: if not os.path.exists(path): try: os.symlink(os.devnull, path) created_paths.append(path) except OSError: pass yield for path in created_paths: try: os.remove(path) except OSError: print 'WARNING: Could not remove %s!' % path
def get_masters(parser, options): """Given parser options, find suitable master directories.""" paths = [] masters_path = chromium_utils.ListMasters() # Populates by defaults with every masters with a twistd.pid, thus has # been started. if not options.master: for m_p in masters_path: if os.path.isfile(os.path.join(m_p, 'twistd.pid')): paths.append(m_p) elif options.master == 'all': paths.extend(masters_path) elif options.master in (os.path.basename(p) for p in masters_path): full_master = next(p for p in masters_path if os.path.basename(p) == options.master) paths.append(full_master) else: parser.error('Unknown master \'%s\'.\nChoices are:\n %s' % (options.master, '\n '.join( (os.path.basename(p) for p in masters_path)))) return paths
def TemporaryMasterPasswords(): all_paths = [os.path.join(BASE_DIR, 'site_config', '.bot_password')] all_paths.extend( os.path.join(path, '.apply_issue_password') for path in chromium_utils.ListMasters()) created_paths = [] for path in all_paths: if not os.path.exists(path): try: with open(path, 'w') as f: f.write('reindeer flotilla\n') created_paths.append(path) except OSError: pass try: yield finally: for path in created_paths: try: os.remove(path) except OSError: print 'WARNING: Could not remove %s!' % path
def Main(argv): usage = """%prog [options] Note: t is replaced with 'tryserver', 'c' with chromium' and co with 'chromiumos', 'cr' with chrome, 'cros' with 'chromeos'.""" # Generate the list of available masters. We want any with slaves. masters_path = chromium_utils.ListMasters(cue='slaves.cfg') masters = [os.path.basename(f) for f in masters_path] # Strip off 'master.' masters = [re.match(r'(master\.|)(.*)', m).group(2) for m in masters] parser = optparse.OptionParser(usage=usage) group = optparse.OptionGroup(parser, 'Slaves to process') group.add_option('-x', '--master', default=[], action='append', help='Master to use to load the slaves list.') group.add_option('-k', '--kind', action='append', default=[], help='Only slaves with a substring present in a builder') group.add_option('-b', '--builder', action='append', default=[], help='Only slaves attached to a specific builder') group.add_option('--os', action='append', default=[], help='Only slaves using a specific OS') group.add_option('--os-version', action='append', default=[], help='Only slaves using a specific OS version') group.add_option('-s', '--slave', action='append', default=[]) parser.add_option_group(group) group = optparse.OptionGroup(parser, 'Output format') group.add_option('-n', '--name', default='host', dest='fmt', action='store_const', const='host', help='Output slave hostname') group.add_option('-r', '--raw', dest='fmt', action='store_const', const='raw', help='Output all slave info') group.add_option('-t', '--assignment', dest='fmt', action='store_const', const='assignment', help='Output slave tasks too') group.add_option('', '--botmap', dest='fmt', action='store_const', const='botmap', help='Output botmap style') group.add_option('-w', '--waterfall', dest='fmt', action='store_const', const='waterfall', help='Output slave master and tasks') parser.add_option_group(group) options, args = parser.parse_args(argv) if args: parser.error('Unknown argument(s): %s\n' % args) slaves = [] if not options.master: # Populates by default with every master with a twistd.pid, thus has # been started. for m_p in masters_path: if os.path.isfile(os.path.join(m_p, 'twistd.pid')): LoadMaster(slaves, m_p) else: for master in options.master: if master == 'allcros': for m in (m for m in masters if (m.startswith('chromeos') or m.startswith('chromiumos'))): LoadMaster(slaves, masters_path[masters.index(m)]) elif master == 'all': for m_p in masters_path: LoadMaster(slaves, m_p) slaves.sort(key=lambda x: (x['mastername'], x.get('hostname'))) else: if not master in masters: master = ProcessShortName(master) if not master in masters: parser.error('Unknown master \'%s\'.\nChoices are: %s' % (master, ', '.join(masters))) LoadMaster(slaves, masters_path[masters.index(master)]) if options.kind: def kind_interested_in_any(builders): if isinstance(builders, basestring): return any(builders.find(k) >= 0 for k in options.kind) return any( any(x.find(k) >= 0 for k in options.kind) for x in builders) slaves = [ s for s in slaves if kind_interested_in_any(s.get('builder')) ] if options.builder: builders = set(options.builder) def builder_interested_in_any(x): return builders.intersection(set(x)) slaves = [ s for s in slaves if builder_interested_in_any(s.get('builder')) ] if options.os: selected = set(options.os) slaves = [s for s in slaves if s.get('os', 'unknown') in selected] if options.os_version: selected = set(options.os_version) slaves = [s for s in slaves if s.get('version', 'unknown') in selected] if options.slave: selected = set(options.slave) slaves = [s for s in slaves if s.get('hostname') in selected] for s in slaves: if options.fmt == 'raw': print s elif options.fmt == 'assignment': print s.get('hostname', 'unknown'), ':', s.get('builder', 'unknown') elif options.fmt == 'waterfall': print s.get('hostname', 'unknown'), ':', s.get('master', 'unknown'), \ ':', s.get('builder', 'unknown') elif options.fmt == 'master': print s.get('hostname', 'unknown'), ':', s.get('mastername', 'unknown'), \ ':', s.get('builder', 'unknown') elif options.fmt == 'botmap': host = s.get('hostname') if host: master = s.get('mastername') or '?' slaveos = s.get('os') or '?' pathsep = '\\' if s.get('os') == 'win' else '/' if 'subdir' in s: d = pathsep + 'c' + pathsep + s['subdir'] else: d = pathsep + 'b' builders = s.get('builder') or '?' if type(builders) is not list: builders = [builders] for b in sorted(builders): print '%-30s %-20s %-35s %-35s %-10s' % (host, d, master, b, slaveos) else: print s.get('hostname', 'unknown')
def Main(argv): usage = """%prog [options] Sample usage: %prog -x t.c --index 5 -i -W "cmd rd /q /s c:\\b\\build\\slave\\win\\build" %prog -x chromium -l -c Note: t is replaced with 'tryserver', 'c' with chromium' and co with 'chromiumos'.""" # Generate the list of available masters. masters_path = chromium_utils.ListMasters() masters = [os.path.basename(f) for f in masters_path] # Strip off 'master.' masters = [re.match(r'(master\.|)(.*)', m).group(2) for m in masters] parser = optparse.OptionParser(usage=usage) group = optparse.OptionGroup(parser, 'Slaves to process') group.add_option( '-x', '--master', help=('Master to use to load the slaves list. If omitted, all masters ' 'that were started at least once are included. If \'all\', all ' 'masters are selected. Choices are: %s.') % ', '.join(masters)) group.add_option('-w', '--win', action='store_true') group.add_option('-l', '--linux', action='store_true') group.add_option('-m', '--mac', action='store_true') group.add_option('--bits', help='Slave os bitness', type='int') group.add_option('--version', help='Slave os version') group.add_option('-b', '--builder', help='Only slaves attached to a specific builder') group.add_option('--min', type='int') group.add_option('--max', type='int', help='Inclusive') group.add_option('--index', type='int', help='execute on only one slave') group.add_option('-s', '--slave', action='append') group.add_option('--raw', help='Line separated list of slaves to use. Must ' 'still use -l, -m or -w to let the script ' 'know what command to run') parser.add_option_group(group) parser.add_option('-i', '--ignore_failure', action='store_true', help='Continue even if ssh returned an error') group = optparse.OptionGroup(parser, 'Premade commands') group.add_option('-c', '--clobber', action='store_true') group.add_option('-r', '--restart', action='store_true') group.add_option('--revert', action='store_true', help='Execute gclient revert') group.add_option('--sync_scripts', action='store_true') group.add_option('--taskkill', action='store_true') group.add_option('--scp', action='store_true', help='with the source and dest files') group.add_option('-q', '--quiet', action='store_true', help='Quiet mode - do not print the commands') group.add_option( '-p', '--print_only', action='store_true', help='Print which slaves would have been processed but do ' 'nothing. With no command, just print the list of ' 'slaves for the given platform(s).') group.add_option('-N', '--no_cygwin', action='store_true', help='By default cygwin\'s bash is called to execute the ' 'command') parser.add_option_group(group) group = optparse.OptionGroup(parser, 'Custom commands') group.add_option('-W', '--win_cmd', help='Run a custom command instead') group.add_option('-L', '--linux_cmd') group.add_option('-M', '--mac_cmd') parser.add_option_group(group) options, args = parser.parse_args(argv) # If a command is specified, the corresponding platform is automatically # enabled. if options.linux_cmd: options.linux = True if options.mac_cmd: options.mac = True if options.win_cmd: options.win = True if options.raw: # Remove extra spaces and empty lines. options.slave = filter(None, (s.strip() for s in open(options.raw, 'r'))) if not options.slave: if not options.master: # Populates by defaults with every masters with a twistd.pid, thus has # been started. slaves = [] for m_p in masters_path: if os.path.isfile(os.path.join(m_p, 'twistd.pid')): slaves.extend( chromium_utils.RunSlavesCfg( os.path.join(m_p, 'slaves.cfg'))) slaves = slaves_list.BaseSlavesList(slaves) elif options.master == 'all': slaves = [] for m_p in masters_path: slaves.extend( chromium_utils.RunSlavesCfg(os.path.join( m_p, 'slaves.cfg'))) slaves = slaves_list.BaseSlavesList(slaves) else: if not options.master in masters: options.master = ProcessShortName(options.master) if not options.master in masters: parser.error('Unknown master \'%s\'.\nChoices are: %s' % (options.master, ', '.join(masters))) master_path = masters_path[masters.index(options.master)] slaves = slaves_list.SlavesList( os.path.join(master_path, 'slaves.cfg')) def F(os_type): out = slaves.GetSlaves(os=os_type, bits=options.bits, version=options.version, builder=options.builder) # Skips slave without a hostname. return [s.get('hostname') for s in out if s.get('hostname')] options.win_names = F('win') options.linux_names = F('linux') options.mac_names = F('mac') else: slaves = options.slave options.win_names = slaves options.linux_names = slaves options.mac_names = slaves if not options.linux and not options.mac and not options.win: parser.print_help() return 0 if options.index: options.min = options.index options.max = options.index if options.scp: if len(args) != 2: parser.error('Need 2 args') return RunSCP(options, args[0], args[1]) if args: parser.error('Only --scp expects arguments') if options.restart: return Restart(options) elif options.clobber: return Clobber(options) elif options.sync_scripts: return SyncScripts(options) elif options.taskkill: return TaskKill(options) elif options.revert: return Revert(options) elif options.print_only and not (options.win_cmd or options.linux_cmd or options.mac_cmd): names_list = [] if not options.min: options.min = 1 if options.win: max_i = len(options.win_names) if options.max: max_i = options.max names_list += options.win_names[options.min - 1:max_i] if options.linux: max_i = len(options.linux_names) if options.max: max_i = options.max names_list += options.linux_names[options.min - 1:max_i] if options.mac: max_i = len(options.mac_names) if options.max: max_i = options.max names_list += options.mac_names[options.min - 1:max_i] print '\n'.join(names_list) else: if ((options.win and not options.win_cmd) or (options.linux and not options.linux_cmd) or (options.mac and not options.mac_cmd)): parser.error('Need to specify a command') return RunSSH(options)