def process_builder(builder, is_trybot=False):
"""Create a dict for the given builder and place its name in the
appropriate scheduler list.
"""
builder_name = builder['name']
if is_trybot:
builder_name = builder_name_schema.TrybotName(builder_name)
# Categorize the builder based on its role.
try:
category = builder_name_schema.DictForBuilderName(
builder_name)['role']
subcategory = builder_name.split(
builder_name_schema.BUILDER_NAME_SEP)[1]
category = '|'.join((category, subcategory))
except ValueError:
# Assume that all builders whose names don't play by our rules are named
# upstream and are therefore canaries.
category = builder_name_schema.BUILDER_ROLE_CANARY
properties = builder.get('properties', {})
cc = builder.get('cc')
if cc:
if isinstance(cc, basestring):
cc = [cc]
properties['owners'] = cc
builder_dict = {
'name':
builder_name,
'auto_reboot':
builder.get('auto_reboot', DEFAULT_AUTO_REBOOT),
'slavenames':
slaves.GetSlavesName(builder=builder['name']),
'category':
category,
'recipe':
builder.get('recipe', DEFAULT_RECIPE),
'properties':
properties,
'mergeRequests':
builder.get('can_merge_requests', CanMergeBuildRequests),
'slavebuilddir':
SLAVE_WORKDIR,
}
builder_dicts.append(builder_dict)
parent_builder = builder.get('triggered_by')
if parent_builder is not None:
assert builder.get('scheduler') is None
if is_trybot:
parent_builder = builder_name_schema.TrybotName(parent_builder)
triggered_builders[parent_builder].append(builder_name)
elif is_trybot:
builders_by_scheduler[FAKE_TRY_SCHEDULER_NAME].append(builder_name)
else:
scheduler = builder.get('scheduler', BUILDBUCKET_SCHEDULER_NAME)
# Setting the scheduler to BUILDBUCKET_SCHEDULER_NAME indicates that
# BuildBucket is the only way to schedule builds for this bot; just
# pretend to add a scheduler in those cases.
builders_by_scheduler[scheduler].append(builder_name)
def process_builder(builder, is_trybot=False):
"""Create a dict for the given builder and place its name in the
appropriate scheduler list.
"""
builder_name = builder['name']
if is_trybot:
builder_name = builder_name_schema.TrybotName(builder_name)
# Categorize the builder based on its role.
try:
category = builder_name_schema.DictForBuilderName(
builder_name)['role']
subcategory = builder_name.split(
builder_name_schema.BUILDER_NAME_SEP)[1]
category = '|'.join((category, subcategory))
except ValueError:
# Assume that all builders whose names don't play by our rules are named
# upstream and are therefore canaries.
category = builder_name_schema.BUILDER_ROLE_CANARY
builder_dict = {
'name': builder_name,
'gatekeeper': builder.get('gatekeeper_categories', ''),
'auto_reboot': builder.get('auto_reboot', DEFAULT_AUTO_REBOOT),
'slavenames': slaves.GetSlavesName(builder=builder['name']),
'category': category,
'recipe': builder.get('recipe', DEFAULT_RECIPE),
}
builder_dicts.append(builder_dict)
parent_builder = builder.get('triggered_by')
if parent_builder is not None:
assert builder.get('scheduler') is None
if is_trybot:
parent_builder = builder_name_schema.TrybotName(parent_builder)
triggered_builders[parent_builder].append(builder_name)
elif is_trybot:
builders_by_scheduler[TRY_SCHEDULER_NAME].append(builder_name)
else:
scheduler = builder.get('scheduler', PERCOMMIT_SCHEDULER_NAME)
builders_by_scheduler[scheduler].append(builder_name)
def SetupBuildersAndSchedulers(c, builders, slaves, ActiveMaster):
"""Set up builders and schedulers for the build master."""
# List of dicts for every builder.
builder_dicts = []
# Builder names by scheduler.
builders_by_scheduler = {s: [] for s in SCHEDULERS}
# Maps a triggering builder to its triggered builders.
triggered_builders = collections.defaultdict(list)
def process_builder(builder, is_trybot=False):
"""Create a dict for the given builder and place its name in the
appropriate scheduler list.
"""
builder_name = builder['name']
if is_trybot:
builder_name = builder_name_schema.TrybotName(builder_name)
# Categorize the builder based on its role.
try:
category = builder_name_schema.DictForBuilderName(
builder_name)['role']
subcategory = builder_name.split(
builder_name_schema.BUILDER_NAME_SEP)[1]
category = '|'.join((category, subcategory))
except ValueError:
# Assume that all builders whose names don't play by our rules are named
# upstream and are therefore canaries.
category = builder_name_schema.BUILDER_ROLE_CANARY
properties = builder.get('properties', {})
cc = builder.get('cc')
if cc:
if isinstance(cc, basestring):
cc = [cc]
properties['owners'] = cc
builder_dict = {
'name':
builder_name,
'auto_reboot':
builder.get('auto_reboot', DEFAULT_AUTO_REBOOT),
'slavenames':
slaves.GetSlavesName(builder=builder['name']),
'category':
category,
'recipe':
builder.get('recipe', DEFAULT_RECIPE),
'properties':
properties,
'mergeRequests':
builder.get('can_merge_requests', CanMergeBuildRequests),
'slavebuilddir':
SLAVE_WORKDIR,
}
builder_dicts.append(builder_dict)
parent_builder = builder.get('triggered_by')
if parent_builder is not None:
assert builder.get('scheduler') is None
if is_trybot:
parent_builder = builder_name_schema.TrybotName(parent_builder)
triggered_builders[parent_builder].append(builder_name)
else:
scheduler = builder.get('scheduler', BUILDBUCKET_SCHEDULER_NAME)
# Setting the scheduler to BUILDBUCKET_SCHEDULER_NAME indicates that
# BuildBucket is the only way to schedule builds for this bot; just
# pretend to add a scheduler in those cases.
builders_by_scheduler[scheduler].append(builder_name)
# Create builders and trybots.
for builder in builders:
if builder.get('trybot_only', DEFAULT_TRYBOT_ONLY):
# trybot_only=True should only be used in combination with do_trybot=True
# Also, the buildername then needs to already have the '-Trybot' suffix.
assert builder.get('do_trybot', DEFAULT_DO_TRYBOT)
assert builder['name'] == builder_name_schema.TrybotName(
builder['name'])
else:
process_builder(builder)
if builder.get('do_trybot', DEFAULT_DO_TRYBOT):
process_builder(builder, is_trybot=True)
# Verify that all parent builders exist.
all_nontriggered_builders = set(
builders_by_scheduler[BUILDBUCKET_SCHEDULER_NAME])
trigger_parents = set(triggered_builders.keys())
nonexistent_parents = trigger_parents - all_nontriggered_builders
if nonexistent_parents:
raise Exception('Could not find parent builders: %s' %
', '.join(nonexistent_parents))
# Create the schedulers.
infra_change_filter = change_filter.ChangeFilter(
project='buildbot', repository=global_constants.INFRA_REPO)
skia_master_only_change_filter = change_filter.ChangeFilter(
project='skia', repository=ActiveMaster.repo_url, branch=MASTER_BRANCH)
c['schedulers'] = []
s = Scheduler(
name=MASTER_ONLY_SCHEDULER_NAME,
treeStableTimer=60,
change_filter=skia_master_only_change_filter,
builderNames=builders_by_scheduler[MASTER_ONLY_SCHEDULER_NAME])
c['schedulers'].append(s)
s = timed.Nightly(
name=PERIODIC_15MINS_SCHEDULER_NAME,
branch=MASTER_BRANCH,
builderNames=builders_by_scheduler[PERIODIC_15MINS_SCHEDULER_NAME],
minute=[i * 15 for i in xrange(60 / 15)],
hour='*',
dayOfMonth='*',
month='*',
dayOfWeek='*')
c['schedulers'].append(s)
s = timed.Nightly(
name=NIGHTLY_SCHEDULER_NAME,
branch=MASTER_BRANCH,
builderNames=builders_by_scheduler[NIGHTLY_SCHEDULER_NAME],
minute=0,
hour=22,
dayOfMonth='*',
month='*',
dayOfWeek='*')
c['schedulers'].append(s)
s = timed.Nightly(
name=WEEKLY_SCHEDULER_NAME,
branch=MASTER_BRANCH,
builderNames=builders_by_scheduler[WEEKLY_SCHEDULER_NAME],
minute=0,
hour=0,
dayOfMonth='*',
month='*',
dayOfWeek=6) # Sunday (Monday = 0).
c['schedulers'].append(s)
s = AnyBranchScheduler(
name=INFRA_PERCOMMIT_SCHEDULER_NAME,
treeStableTimer=0,
change_filter=infra_change_filter,
builderNames=builders_by_scheduler[INFRA_PERCOMMIT_SCHEDULER_NAME])
c['schedulers'].append(s)
# Don't add triggerable schedulers for triggered_builders; triggers are now
# handled on the slave-side through buildbucket.
# Create the BuildFactorys.
annotator = annotator_factory.AnnotatorFactory(ActiveMaster)
for builder_dict in builder_dicts:
factory = annotator.BaseFactory(builder_dict['recipe'], timeout=2400)
factory.properties.update(builder_dict['properties'], 'BuildFactory')
builder_dict['factory'] = factory
# Finished!
c['builders'] = builder_dicts
# use an up-to-date Skia.
# This list specifies which Chromium builders to "copy".
_builders = [
# SPEC Module Test Spec File Builder Names
(chromium_linux, 'chromium.linux.json', ['Linux Builder', 'Linux Tests']),
(chromium_win, 'chromium.win.json', ['Win Builder', 'Win7 Tests (1)']),
(chromium_mac, 'chromium.mac.json', ['Mac Builder', 'Mac10.9 Tests']),
]
SPEC = {
'settings': {
'build_gs_bucket': 'chromium-skia-gm',
},
'builders': {},
}
for spec_module, test_spec_file, builders_list in _builders:
for builder in builders_list:
for builder_name in (builder, builder_name_schema.TrybotName(builder)):
builder_cfg = copy.deepcopy(spec_module.SPEC['builders'][builder])
builder_cfg['gclient_config'] = 'chromium_skia'
parent = builder_cfg.get('parent_buildername')
if parent:
if builder_name_schema.IsTrybot(builder_name):
parent = builder_name_schema.TrybotName(parent)
builder_cfg['parent_buildername'] = parent
builder_cfg['patch_root'] = 'src/third_party/skia'
builder_cfg['testing']['test_spec_file'] = test_spec_file
SPEC['builders'][builder_name] = builder_cfg