def GetArchiveUrl(project, machine_name, builder_name, zip_os_name):
# This is slightly tricky since it depends on os.getcwd() being right.
slaves = slaves_list.SlavesList('slaves.cfg', project)
host = slaves.GetSlaveName(project, machine_name)
if not host:
raise ValueError("%s isn't reachable" % machine_name)
return ('http://%s/b/build/slave/%s/chrome_staging/full-build-%s.zip' %
(host, builder_name, zip_os_name))
def GetArchiveUrl(project, machine_name, builder_name, zip_os_name,
static_host=None):
# static host can be used for connections made through an intermediary
if static_host:
host = static_host
else:
# This is slightly tricky since it depends on os.getcwd() being right.
slaves = slaves_list.SlavesList('slaves.cfg', project)
host = slaves.GetSlaveName(project, machine_name)
if not host:
raise ValueError("%s isn't reachable" % machine_name)
return ('http://%s/b/build/slave/%s/chrome_staging/full-build-%s.zip' % (
host, builder_name, zip_os_name))
def SetupMaster(ActiveMaster):
# Buildmaster config dict.
c = {}
config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host)
####### CHANGESOURCES
# Polls config.Master.trunk_url for changes
poller = GitilesPoller(
repo_url=ActiveMaster.repo_url,
branches=['master'],
pollInterval=10,
revlinktmpl='https://skia.googlesource.com/skia/+/%s')
c['change_source'] = [poller]
####### SLAVES
# Load the slave list. We need some information from it in order to
# produce the builders.
slaves = slaves_list.SlavesList('slaves.cfg', ActiveMaster.project_name)
####### BUILDERS
# Load the builders list.
builders = chromium_utils.ParsePythonCfg('builders.cfg')['builders']
# Configure the Builders and Schedulers.
SetupBuildersAndSchedulers(c=c,
builders=builders,
slaves=slaves,
ActiveMaster=ActiveMaster)
####### BUILDSLAVES
# The 'slaves' list defines the set of allowable buildslaves. List all the
# slaves registered to a builder. Remove dupes.
c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
config.Master.GetBotPassword())
master_utils.VerifySetup(c, slaves)
####### STATUS TARGETS
c['buildbotURL'] = ActiveMaster.buildbot_url
# Adds common status and tools to this master.
master_utils.AutoSetupMaster(
c,
ActiveMaster,
public_html='../../../build/masters/master.client.skia/public_html',
templates=[
'../../../build/masters/master.client.skia/templates',
'../../../build/masters/master.chromium/templates'
],
tagComparator=poller.comparator,
enable_http_status_push=ActiveMaster.is_production_host,
order_console_by_time=True,
console_repo_filter=ActiveMaster.repo_url,
console_builder_filter=lambda b: not builder_name_schema.IsTrybot(b))
with status_json.JsonStatusHelper() as json_helper:
json_helper.putChild('trybots', status_json.TryBuildersJsonResource)
if ActiveMaster.is_production_host:
# Build result emails.
c['status'].append(
skia_notifier.SkiaMailNotifier(
fromaddr=ActiveMaster.from_address,
mode='change',
relayhost=config.Master.smtp,
lookup=master_utils.UsersAreEmails()))
# Try job result emails.
c['status'].append(
skia_notifier.SkiaTryMailNotifier(
fromaddr=ActiveMaster.from_address,
subject="try %(result)s for %(reason)s @ r%(revision)s",
mode='all',
relayhost=config.Master.smtp,
lookup=master_utils.UsersAreEmails()))
# Rietveld status push.
c['status'].append(
TryServerHttpStatusPush(serverUrl=ActiveMaster.code_review_site))
return c
def Main(argv):
usage = """%prog [options]
Sample usage:
%prog -x t.c --index 5 -i -W "cmd rd /q /s c:\\b\\build\\slave\\win\\build"
%prog -x chromium -l -c
Note: t is replaced with 'tryserver', 'c' with chromium' and
co with 'chromiumos'."""
# Generate the list of available masters.
masters_path = chromium_utils.ListMasters()
masters = [os.path.basename(f) for f in masters_path]
# Strip off 'master.'
masters = [re.match(r'(master\.|)(.*)', m).group(2) for m in masters]
parser = optparse.OptionParser(usage=usage)
group = optparse.OptionGroup(parser, 'Slaves to process')
group.add_option(
'-x',
'--master',
help=('Master to use to load the slaves list. If omitted, all masters '
'that were started at least once are included. If \'all\', all '
'masters are selected. Choices are: %s.') % ', '.join(masters))
group.add_option('-w', '--win', action='store_true')
group.add_option('-l', '--linux', action='store_true')
group.add_option('-m', '--mac', action='store_true')
group.add_option('--bits', help='Slave os bitness', type='int')
group.add_option('--version', help='Slave os version')
group.add_option('-b',
'--builder',
help='Only slaves attached to a specific builder')
group.add_option('--min', type='int')
group.add_option('--max', type='int', help='Inclusive')
group.add_option('--index', type='int', help='execute on only one slave')
group.add_option('-s', '--slave', action='append')
group.add_option('--raw',
help='Line separated list of slaves to use. Must '
'still use -l, -m or -w to let the script '
'know what command to run')
parser.add_option_group(group)
parser.add_option('-i',
'--ignore_failure',
action='store_true',
help='Continue even if ssh returned an error')
group = optparse.OptionGroup(parser, 'Premade commands')
group.add_option('-c', '--clobber', action='store_true')
group.add_option('-r', '--restart', action='store_true')
group.add_option('--revert',
action='store_true',
help='Execute gclient revert')
group.add_option('--sync_scripts', action='store_true')
group.add_option('--taskkill', action='store_true')
group.add_option('--scp',
action='store_true',
help='with the source and dest files')
group.add_option('-q',
'--quiet',
action='store_true',
help='Quiet mode - do not print the commands')
group.add_option(
'-p',
'--print_only',
action='store_true',
help='Print which slaves would have been processed but do '
'nothing. With no command, just print the list of '
'slaves for the given platform(s).')
group.add_option('-N',
'--no_cygwin',
action='store_true',
help='By default cygwin\'s bash is called to execute the '
'command')
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Custom commands')
group.add_option('-W', '--win_cmd', help='Run a custom command instead')
group.add_option('-L', '--linux_cmd')
group.add_option('-M', '--mac_cmd')
parser.add_option_group(group)
options, args = parser.parse_args(argv)
# If a command is specified, the corresponding platform is automatically
# enabled.
if options.linux_cmd:
options.linux = True
if options.mac_cmd:
options.mac = True
if options.win_cmd:
options.win = True
if options.raw:
# Remove extra spaces and empty lines.
options.slave = filter(None,
(s.strip() for s in open(options.raw, 'r')))
if not options.slave:
if not options.master:
# Populates by defaults with every masters with a twistd.pid, thus has
# been started.
slaves = []
for m_p in masters_path:
if os.path.isfile(os.path.join(m_p, 'twistd.pid')):
slaves.extend(
chromium_utils.RunSlavesCfg(
os.path.join(m_p, 'slaves.cfg')))
slaves = slaves_list.BaseSlavesList(slaves)
elif options.master == 'all':
slaves = []
for m_p in masters_path:
slaves.extend(
chromium_utils.RunSlavesCfg(os.path.join(
m_p, 'slaves.cfg')))
slaves = slaves_list.BaseSlavesList(slaves)
else:
if not options.master in masters:
options.master = ProcessShortName(options.master)
if not options.master in masters:
parser.error('Unknown master \'%s\'.\nChoices are: %s' %
(options.master, ', '.join(masters)))
master_path = masters_path[masters.index(options.master)]
slaves = slaves_list.SlavesList(
os.path.join(master_path, 'slaves.cfg'))
def F(os_type):
out = slaves.GetSlaves(os=os_type,
bits=options.bits,
version=options.version,
builder=options.builder)
# Skips slave without a hostname.
return [s.get('hostname') for s in out if s.get('hostname')]
options.win_names = F('win')
options.linux_names = F('linux')
options.mac_names = F('mac')
else:
slaves = options.slave
options.win_names = slaves
options.linux_names = slaves
options.mac_names = slaves
if not options.linux and not options.mac and not options.win:
parser.print_help()
return 0
if options.index:
options.min = options.index
options.max = options.index
if options.scp:
if len(args) != 2:
parser.error('Need 2 args')
return RunSCP(options, args[0], args[1])
if args:
parser.error('Only --scp expects arguments')
if options.restart:
return Restart(options)
elif options.clobber:
return Clobber(options)
elif options.sync_scripts:
return SyncScripts(options)
elif options.taskkill:
return TaskKill(options)
elif options.revert:
return Revert(options)
elif options.print_only and not (options.win_cmd or options.linux_cmd
or options.mac_cmd):
names_list = []
if not options.min:
options.min = 1
if options.win:
max_i = len(options.win_names)
if options.max:
max_i = options.max
names_list += options.win_names[options.min - 1:max_i]
if options.linux:
max_i = len(options.linux_names)
if options.max:
max_i = options.max
names_list += options.linux_names[options.min - 1:max_i]
if options.mac:
max_i = len(options.mac_names)
if options.max:
max_i = options.max
names_list += options.mac_names[options.min - 1:max_i]
print '\n'.join(names_list)
else:
if ((options.win and not options.win_cmd)
or (options.linux and not options.linux_cmd)
or (options.mac and not options.mac_cmd)):
parser.error('Need to specify a command')
return RunSSH(options)
def SetupMaster(ActiveMaster):
# Buildmaster config dict.
c = {}
config.DatabaseSetup(c)
####### CHANGESOURCES
# Polls config.Master.trunk_url for changes
poller = GitilesPoller(
repo_url=ActiveMaster.repo_url,
branches=[POLLING_BRANCH],
pollInterval=10,
revlinktmpl='https://skia.googlesource.com/skia/+/%s')
c['change_source'] = [poller]
####### SLAVES
# Load the slave list. We need some information from it in order to
# produce the builders.
slaves = slaves_list.SlavesList('slaves.cfg', ActiveMaster.project_name)
####### BUILDERS
# Load the builders list.
builders = chromium_utils.ParsePythonCfg('builders.cfg')['builders']
# Configure the Builders and Schedulers.
SetupBuildersAndSchedulers(c=c,
builders=builders,
slaves=slaves,
ActiveMaster=ActiveMaster)
####### BUILDSLAVES
# The 'slaves' list defines the set of allowable buildslaves. List all the
# slaves registered to a builder. Remove dupes.
c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
config.Master.GetBotPassword())
master_utils.VerifySetup(c, slaves)
####### STATUS TARGETS
c['buildbotURL'] = ActiveMaster.buildbot_url
# Adds common status and tools to this master.
master_utils.AutoSetupMaster(
c,
ActiveMaster,
public_html='../../../build/masters/master.client.skia/public_html',
templates=[
'../../../build/masters/master.client.skia/templates',
'../../../build/masters/master.chromium/templates'
],
tagComparator=poller.comparator,
enable_http_status_push=ActiveMaster.is_production_host,
order_console_by_time=True,
console_repo_filter=ActiveMaster.repo_url,
console_builder_filter=lambda b: not builder_name_schema.IsTrybot(b))
with status_json.JsonStatusHelper() as json_helper:
json_helper.putChild('trybots', status_json.TryBuildersJsonResource)
if (ActiveMaster.is_production_host
and ActiveMaster.project_name != 'SkiaInternal'):
# Build result emails.
c['status'].append(status_logger.StatusEventLogger())
c['status'].append(
skia_notifier.SkiaMailNotifier(
fromaddr=ActiveMaster.from_address,
mode='change',
relayhost=config.Master.smtp,
lookup=master_utils.UsersAreEmails()))
# Try job result emails.
c['status'].append(
skia_notifier.SkiaTryMailNotifier(
fromaddr=ActiveMaster.from_address,
subject="try %(result)s for %(reason)s @ r%(revision)s",
mode='all',
relayhost=config.Master.smtp,
lookup=master_utils.UsersAreEmails()))
# Push status updates to GrandCentral.
c['status'].append(
HttpStatusPush(serverUrl='https://grandcentral.skia.org/buildbot'))
c['mergeRequests'] = CanMergeBuildRequests
###### LOGS
# Skia bots have been known to have run away builds continously dumping to
# stdout and creating ~100GB logs. See crbug.com/589654 for context.
c['logMaxSize'] = 1024 * 1024 * 100 # 100MB
c['logMaxTailSize'] = 1024 * 32 # 32KB
return c