def _Populate(BuildmasterConfig, builders, active_master_cls):
m_annotator = annotator_factory.AnnotatorFactory(active_master_cls)
c = BuildmasterConfig
c['logCompressionLimit'] = False
c['projectName'] = active_master_cls.project_name
c['projectURL'] = Master.project_url
c['buildbotURL'] = active_master_cls.buildbot_url
# This sets c['db_url'] to the database connect string in found in
# the .dbconfig in the master directory, if it exists. If this is
# a production host, it must exist.
chromium_utils.DatabaseSetup(
c, require_dbconfig=active_master_cls.is_production_host)
c['builders'] = _ComputeBuilders(builders, m_annotator)
c['schedulers'] = _ComputeSchedulers(builders)
c['change_source'], tag_comparator = _ComputeChangeSourceAndTagComparator(
builders)
# The 'slaves' list defines the set of allowable buildslaves. List all the
# slaves registered to a builder. Remove dupes.
c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
Master.GetBotPassword())
# This does some sanity checks on the configuration.
slaves = slaves_list.BaseSlavesList(
chromium_utils.GetSlavesFromBuilders(builders),
builders['master_classname'])
master_utils.VerifySetup(c, slaves)
default_public_html = os.path.join(chromium_utils.BUILD_DIR, 'masters',
'master.chromium', 'public_html')
public_html = builders.get('public_html', default_public_html)
# Adds common status and tools to this master.
# TODO: Look at the logic in this routine to see if any of the logic
# in this routine can be moved there to simplify things.
master_utils.AutoSetupMaster(
c,
active_master_cls,
public_html=public_html,
templates=builders['templates'],
tagComparator=tag_comparator,
enable_http_status_push=active_master_cls.is_production_host)
# TODO: AutoSetupMaster's settings for the following are too low to be
# useful for most projects. We should fix that.
c['buildHorizon'] = 3000
c['logHorizon'] = 3000
# Must be at least 2x the number of slaves.
c['eventHorizon'] = 200
def Update(_config, active_master, c):
factory_obj = annotator_factory.AnnotatorFactory(
active_master=active_master)
builders = [
Builder(factory_obj, 'x86-generic'),
Builder(factory_obj, 'amd64-generic'),
]
c['schedulers'] += [
Scheduler(name='chromium_src_asan',
branch='master',
treeStableTimer=60,
builderNames=[b['name'] for b in builders],
),
]
c['builders'] += builders
'build_config': 'Release',
'perf_id': 'gpu-webkit-linux-nvidia',
'triggered_by': 'GPU Linux Builder',
'auto_reboot': False,
},
{
'builder': 'GPU Linux (dbg) (NVIDIA)',
'factory_id': 'f_gpu_linux_dbg',
'recipe': 'gpu/download_and_test',
'build_config': 'Debug',
'triggered_by': 'GPU Linux Builder (dbg)',
'auto_reboot': False,
},
]
m_annotator = annotator_factory.AnnotatorFactory()
defaults['category'] = 'gpu'
# Maps the parent builder to a set of the names of the builders it triggers.
trigger_map = collections.defaultdict(list)
# Maps the name of the parent builder to the (synthesized) name of its
# trigger, wrapped in a list.
trigger_name_map = {}
next_group_id = 0
# Note this code is very similar to that in recipe_master_helper.py.
# Unfortunately due to the different structure of this waterfall it's
# impossible to share the code.
def BuilderExists(builder_name):
def SetupBuildersAndSchedulers(c, builders, slaves, ActiveMaster):
"""Set up builders and schedulers for the build master."""
# List of dicts for every builder.
builder_dicts = []
# Builder names by scheduler.
builders_by_scheduler = {s: [] for s in SCHEDULERS}
# Maps a triggering builder to its triggered builders.
triggered_builders = collections.defaultdict(list)
def process_builder(builder, is_trybot=False):
"""Create a dict for the given builder and place its name in the
appropriate scheduler list.
"""
builder_name = builder['name']
if is_trybot:
builder_name = builder_name_schema.TrybotName(builder_name)
# Categorize the builder based on its role.
try:
category = builder_name_schema.DictForBuilderName(
builder_name)['role']
subcategory = builder_name.split(
builder_name_schema.BUILDER_NAME_SEP)[1]
category = '|'.join((category, subcategory))
except ValueError:
# Assume that all builders whose names don't play by our rules are named
# upstream and are therefore canaries.
category = builder_name_schema.BUILDER_ROLE_CANARY
builder_dict = {
'name': builder_name,
'gatekeeper': builder.get('gatekeeper_categories', ''),
'auto_reboot': builder.get('auto_reboot', DEFAULT_AUTO_REBOOT),
'slavenames': slaves.GetSlavesName(builder=builder['name']),
'category': category,
'recipe': builder.get('recipe', DEFAULT_RECIPE),
}
builder_dicts.append(builder_dict)
parent_builder = builder.get('triggered_by')
if parent_builder is not None:
assert builder.get('scheduler') is None
if is_trybot:
parent_builder = builder_name_schema.TrybotName(parent_builder)
triggered_builders[parent_builder].append(builder_name)
elif is_trybot:
builders_by_scheduler[TRY_SCHEDULER_NAME].append(builder_name)
else:
scheduler = builder.get('scheduler', PERCOMMIT_SCHEDULER_NAME)
builders_by_scheduler[scheduler].append(builder_name)
# Create builders and trybots.
for builder in builders:
process_builder(builder)
if builder.get('do_trybot', DEFAULT_DO_TRYBOT):
process_builder(builder, is_trybot=True)
# Verify that all parent builders exist.
all_nontriggered_builders = set(
builders_by_scheduler[PERCOMMIT_SCHEDULER_NAME]).union(
set(builders_by_scheduler[TRY_SCHEDULER_NAME]))
trigger_parents = set(triggered_builders.keys())
nonexistent_parents = trigger_parents - all_nontriggered_builders
if nonexistent_parents:
raise Exception('Could not find parent builders: %s' %
', '.join(nonexistent_parents))
# Create the schedulers.
def trigger_name(parent_builder):
"""Given a parent builder name, return a triggerable scheduler name."""
return 'triggers_%s' % parent_builder
c['schedulers'] = []
s = Scheduler(name=PERCOMMIT_SCHEDULER_NAME,
branch=POLLING_BRANCH,
treeStableTimer=60,
builderNames=builders_by_scheduler[PERCOMMIT_SCHEDULER_NAME])
c['schedulers'].append(s)
s = timed.Nightly(
name=PERIODIC_15MINS_SCHEDULER_NAME,
branch=POLLING_BRANCH,
builderNames=builders_by_scheduler[PERIODIC_15MINS_SCHEDULER_NAME],
minute=[i * 15 for i in xrange(60 / 15)],
hour='*',
dayOfMonth='*',
month='*',
dayOfWeek='*')
c['schedulers'].append(s)
for parent, builders_to_trigger in triggered_builders.iteritems():
c['schedulers'].append(
Triggerable(name=trigger_name(parent),
builderNames=builders_to_trigger))
pools = BuildersPools(TRY_SCHEDULER_NAME)
pools[TRY_SCHEDULER_NAME].extend(builders_by_scheduler[TRY_SCHEDULER_NAME])
c['schedulers'].append(
TryJobRietveld(name=TRY_SCHEDULER_NAME,
code_review_sites={
TRY_SCHEDULER_PROJECT: ActiveMaster.code_review_site
},
pools=pools,
project=TRY_SCHEDULER_PROJECT,
filter_master=True))
# Create the BuildFactorys.
annotator = annotator_factory.AnnotatorFactory()
for builder_dict in builder_dicts:
triggers = ([trigger_name(builder_dict['name'])]
if builder_dict['name'] in triggered_builders else None)
builder_dict['factory'] = annotator.BaseFactory(builder_dict['recipe'],
triggers=triggers)
# Finished!
c['builders'] = builder_dicts
factory_properties={
'android_bot_id': 'main-tests-rel',
'build_url': android_rel_archive,
}))
B('Android Clang Builder (dbg)',
'f_android_clang_dbg',
'android',
'android',
notify_on_missing=True)
F(
'f_android_clang_dbg',
linux_android().ChromiumAnnotationFactory(
target='Debug',
annotation_script='src/build/android/buildbot/bb_run_bot.py',
factory_properties={
'android_bot_id': 'main-clang-builder-dbg',
}))
B('Android Webview AOSP Builder',
'f_android_webview_aosp_rel',
'android',
'android',
notify_on_missing=True)
F('f_android_webview_aosp_rel',
annotator_factory.AnnotatorFactory().BaseFactory('android_webview_aosp'))
def Update(_config_arg, _active_master, c):
return helper.Update(c)
def SetupBuildersAndSchedulers(c, builders, slaves, ActiveMaster):
"""Set up builders and schedulers for the build master."""
# List of dicts for every builder.
builder_dicts = []
# Builder names by scheduler.
builders_by_scheduler = {s: [] for s in SCHEDULERS}
# Maps a triggering builder to its triggered builders.
triggered_builders = collections.defaultdict(list)
def process_builder(builder, is_trybot=False):
"""Create a dict for the given builder and place its name in the
appropriate scheduler list.
"""
builder_name = builder['name']
if is_trybot:
builder_name = builder_name_schema.TrybotName(builder_name)
# Categorize the builder based on its role.
try:
category = builder_name_schema.DictForBuilderName(
builder_name)['role']
subcategory = builder_name.split(
builder_name_schema.BUILDER_NAME_SEP)[1]
category = '|'.join((category, subcategory))
except ValueError:
# Assume that all builders whose names don't play by our rules are named
# upstream and are therefore canaries.
category = builder_name_schema.BUILDER_ROLE_CANARY
properties = builder.get('properties', {})
cc = builder.get('cc')
if cc:
if isinstance(cc, basestring):
cc = [cc]
properties['owners'] = cc
builder_dict = {
'name':
builder_name,
'auto_reboot':
builder.get('auto_reboot', DEFAULT_AUTO_REBOOT),
'slavenames':
slaves.GetSlavesName(builder=builder['name']),
'category':
category,
'recipe':
builder.get('recipe', DEFAULT_RECIPE),
'properties':
properties,
'mergeRequests':
builder.get('can_merge_requests', CanMergeBuildRequests),
'slavebuilddir':
SLAVE_WORKDIR,
}
builder_dicts.append(builder_dict)
parent_builder = builder.get('triggered_by')
if parent_builder is not None:
assert builder.get('scheduler') is None
if is_trybot:
parent_builder = builder_name_schema.TrybotName(parent_builder)
triggered_builders[parent_builder].append(builder_name)
else:
scheduler = builder.get('scheduler', BUILDBUCKET_SCHEDULER_NAME)
# Setting the scheduler to BUILDBUCKET_SCHEDULER_NAME indicates that
# BuildBucket is the only way to schedule builds for this bot; just
# pretend to add a scheduler in those cases.
builders_by_scheduler[scheduler].append(builder_name)
# Create builders and trybots.
for builder in builders:
if builder.get('trybot_only', DEFAULT_TRYBOT_ONLY):
# trybot_only=True should only be used in combination with do_trybot=True
# Also, the buildername then needs to already have the '-Trybot' suffix.
assert builder.get('do_trybot', DEFAULT_DO_TRYBOT)
assert builder['name'] == builder_name_schema.TrybotName(
builder['name'])
else:
process_builder(builder)
if builder.get('do_trybot', DEFAULT_DO_TRYBOT):
process_builder(builder, is_trybot=True)
# Verify that all parent builders exist.
all_nontriggered_builders = set(
builders_by_scheduler[BUILDBUCKET_SCHEDULER_NAME])
trigger_parents = set(triggered_builders.keys())
nonexistent_parents = trigger_parents - all_nontriggered_builders
if nonexistent_parents:
raise Exception('Could not find parent builders: %s' %
', '.join(nonexistent_parents))
# Create the schedulers.
infra_change_filter = change_filter.ChangeFilter(
project='buildbot', repository=global_constants.INFRA_REPO)
skia_master_only_change_filter = change_filter.ChangeFilter(
project='skia', repository=ActiveMaster.repo_url, branch=MASTER_BRANCH)
c['schedulers'] = []
s = Scheduler(
name=MASTER_ONLY_SCHEDULER_NAME,
treeStableTimer=60,
change_filter=skia_master_only_change_filter,
builderNames=builders_by_scheduler[MASTER_ONLY_SCHEDULER_NAME])
c['schedulers'].append(s)
s = timed.Nightly(
name=PERIODIC_15MINS_SCHEDULER_NAME,
branch=MASTER_BRANCH,
builderNames=builders_by_scheduler[PERIODIC_15MINS_SCHEDULER_NAME],
minute=[i * 15 for i in xrange(60 / 15)],
hour='*',
dayOfMonth='*',
month='*',
dayOfWeek='*')
c['schedulers'].append(s)
s = timed.Nightly(
name=NIGHTLY_SCHEDULER_NAME,
branch=MASTER_BRANCH,
builderNames=builders_by_scheduler[NIGHTLY_SCHEDULER_NAME],
minute=0,
hour=22,
dayOfMonth='*',
month='*',
dayOfWeek='*')
c['schedulers'].append(s)
s = timed.Nightly(
name=WEEKLY_SCHEDULER_NAME,
branch=MASTER_BRANCH,
builderNames=builders_by_scheduler[WEEKLY_SCHEDULER_NAME],
minute=0,
hour=0,
dayOfMonth='*',
month='*',
dayOfWeek=6) # Sunday (Monday = 0).
c['schedulers'].append(s)
s = AnyBranchScheduler(
name=INFRA_PERCOMMIT_SCHEDULER_NAME,
treeStableTimer=0,
change_filter=infra_change_filter,
builderNames=builders_by_scheduler[INFRA_PERCOMMIT_SCHEDULER_NAME])
c['schedulers'].append(s)
# Don't add triggerable schedulers for triggered_builders; triggers are now
# handled on the slave-side through buildbucket.
# Create the BuildFactorys.
annotator = annotator_factory.AnnotatorFactory(ActiveMaster)
for builder_dict in builder_dicts:
factory = annotator.BaseFactory(builder_dict['recipe'], timeout=2400)
factory.properties.update(builder_dict['properties'], 'BuildFactory')
builder_dict['factory'] = factory
# Finished!
c['builders'] = builder_dicts
