def m_remote_run(recipe, **kwargs): return remote_run_factory.RemoteRunFactory( active_master=ActiveMaster, repository='https://chromium.googlesource.com/chromium/tools/build.git', recipe=recipe, factory_properties={'path_config': 'kitchen'}, **kwargs)
def _ComputeBuilders(builders, m_annotator, active_master_cls): actual_builders = [] def cmp_fn(a, b): a_cat = builders['builders'][a].get('category') b_cat = builders['builders'][b].get('category') if a_cat != b_cat: return 1 if a_cat > b_cat else -1 if a != b: return 1 if a > b else -1 return 0 for builder_name in sorted(builders['builders'], cmp=cmp_fn): builder_data = builders['builders'][builder_name] has_schedulers = bool( builder_data.get('scheduler', builder_data.get('schedulers'))) # We will automatically merge all build requests for any # builder that can be scheduled; this is normally the behavior # we want for repo-triggered builders and cron-triggered builders. # You can override this behavior by setting the mergeRequests field though. merge_requests = builder_data.get('mergeRequests', has_schedulers) slavebuilddir = builder_data.get('slavebuilddir', util.safeTranslate(builder_name)) props = {} props.update(builders.get('default_properties', {}).copy()) if builder_data.get('use_remote_run'): props.update( builders.get('default_remote_run_properties', {}).copy()) props.update(builder_data.get('properties', {})) if builder_data.get('use_remote_run'): factory = remote_run_factory.RemoteRunFactory( active_master=active_master_cls, repository=builder_data.get( 'repository', builders.get('default_remote_run_repository')), recipe=builder_data['recipe'], max_time=builder_data.get('builder_timeout_s'), factory_properties=props, ) else: factory = m_annotator.BaseFactory( recipe=builder_data['recipe'], max_time=builder_data.get('builder_timeout_s'), factory_properties=props, ) actual_builders.append({ 'auto_reboot': builder_data.get('auto_reboot', True), 'mergeRequests': merge_requests, 'name': builder_name, 'factory': factory, 'slavebuilddir': slavebuilddir, 'slavenames': chromium_utils.GetSlaveNamesForBuilder(builders, builder_name), 'category': builder_data.get('category'), 'trybot': builder_data.get('trybot'), }) return actual_builders
def SetupBuildersAndSchedulers(c, builders, slaves, ActiveMaster): """Set up builders and schedulers for the build master.""" # List of dicts for every builder. builder_dicts = [] # Builder names by scheduler. builders_by_scheduler = {s: [] for s in SCHEDULERS} # Maps a triggering builder to its triggered builders. triggered_builders = collections.defaultdict(list) def process_builder(builder, is_trybot=False): """Create a dict for the given builder and place its name in the appropriate scheduler list. """ builder_name = builder['name'] if is_trybot: builder_name = builder_name_schema.TrybotName(builder_name) # Categorize the builder based on its role. try: category = builder_name_schema.DictForBuilderName( builder_name)['role'] subcategory = builder_name.split( builder_name_schema.BUILDER_NAME_SEP)[1] category = '|'.join((category, subcategory)) except ValueError: # Assume that all builders whose names don't play by our rules are named # upstream and are therefore canaries. category = builder_name_schema.BUILDER_ROLE_CANARY properties = builder.get('properties', {}) cc = builder.get('cc') if cc: if isinstance(cc, basestring): cc = [cc] properties['owners'] = cc builder_dict = { 'name': builder_name, 'auto_reboot': builder.get('auto_reboot', DEFAULT_AUTO_REBOOT), 'slavenames': slaves.GetSlavesName(builder=builder['name']), 'category': category, 'recipe': builder.get('recipe', DEFAULT_RECIPE), 'remote_run': builder.get('remote_run', DEFAULT_REMOTE_RUN), 'properties': properties, 'mergeRequests': builder.get('can_merge_requests', CanMergeBuildRequests), 'slavebuilddir': SLAVE_WORKDIR, } builder_dicts.append(builder_dict) parent_builder = builder.get('triggered_by') if parent_builder is not None: assert builder.get('scheduler') is None if is_trybot: parent_builder = builder_name_schema.TrybotName(parent_builder) triggered_builders[parent_builder].append(builder_name) elif is_trybot: builders_by_scheduler[FAKE_TRY_SCHEDULER_NAME].append(builder_name) else: scheduler = builder.get('scheduler', BUILDBUCKET_SCHEDULER_NAME) # Setting the scheduler to BUILDBUCKET_SCHEDULER_NAME indicates that # BuildBucket is the only way to schedule builds for this bot; just # pretend to add a scheduler in those cases. builders_by_scheduler[scheduler].append(builder_name) # Create builders and trybots. for builder in builders: if builder.get('trybot_only', DEFAULT_TRYBOT_ONLY): # trybot_only=True should only be used in combination with do_trybot=True # Also, the buildername then needs to already have the '-Trybot' suffix. assert builder.get('do_trybot', DEFAULT_DO_TRYBOT) assert builder['name'] == builder_name_schema.TrybotName( builder['name']) else: process_builder(builder) if builder.get('do_trybot', DEFAULT_DO_TRYBOT): process_builder(builder, is_trybot=True) # Verify that all parent builders exist. buildbucket_builders = set( builders_by_scheduler[BUILDBUCKET_SCHEDULER_NAME]) trybots = set(builders_by_scheduler[FAKE_TRY_SCHEDULER_NAME]) trigger_parents = set(triggered_builders.keys()) nonexistent_parents = trigger_parents - buildbucket_builders - trybots if nonexistent_parents: raise Exception('Could not find parent builders: %s' % ', '.join(nonexistent_parents)) # Create the schedulers. skia_master_only_change_filter = change_filter.ChangeFilter( project='skia', repository=ActiveMaster.repo_url, branch=MASTER_BRANCH) c['schedulers'] = [] s = Scheduler( name=MASTER_ONLY_SCHEDULER_NAME, treeStableTimer=60, change_filter=skia_master_only_change_filter, builderNames=builders_by_scheduler[MASTER_ONLY_SCHEDULER_NAME]) c['schedulers'].append(s) s = timed.Nightly( name=PERIODIC_15MINS_SCHEDULER_NAME, branch=MASTER_BRANCH, builderNames=builders_by_scheduler[PERIODIC_15MINS_SCHEDULER_NAME], minute=[i * 15 for i in xrange(60 / 15)], hour='*', dayOfMonth='*', month='*', dayOfWeek='*') c['schedulers'].append(s) s = timed.Nightly( name=NIGHTLY_SCHEDULER_NAME, branch=MASTER_BRANCH, builderNames=builders_by_scheduler[NIGHTLY_SCHEDULER_NAME], minute=0, hour=22, dayOfMonth='*', month='*', dayOfWeek='*') c['schedulers'].append(s) s = timed.Nightly( name=WEEKLY_SCHEDULER_NAME, branch=MASTER_BRANCH, builderNames=builders_by_scheduler[WEEKLY_SCHEDULER_NAME], minute=0, hour=0, dayOfMonth='*', month='*', dayOfWeek=6) # Sunday (Monday = 0). c['schedulers'].append(s) # Don't add triggerable schedulers for triggered_builders; triggers are now # handled on the slave-side through buildbucket. # Create the BuildFactorys. annotator = annotator_factory.AnnotatorFactory(ActiveMaster) for builder_dict in builder_dicts: if builder_dict['remote_run']: factory = remote_run_factory.RemoteRunFactory( active_master=ActiveMaster, repository= 'https://chromium.googlesource.com/chromium/tools/build.git', recipe=builder_dict['recipe'], factory_properties={'path_config': 'kitchen'}) else: factory = annotator.BaseFactory(builder_dict['recipe'], timeout=2400) factory.properties.update(builder_dict['properties'], 'BuildFactory') builder_dict['factory'] = factory # Finished! c['builders'] = builder_dicts