def _Populate(BuildmasterConfig, builders, active_master_cls): m_annotator = annotator_factory.AnnotatorFactory(active_master_cls) c = BuildmasterConfig c['logCompressionLimit'] = False c['projectName'] = active_master_cls.project_name c['projectURL'] = Master.project_url c['buildbotURL'] = active_master_cls.buildbot_url # This sets c['db_url'] to the database connect string in found in # the .dbconfig in the master directory, if it exists. If this is # a production host, it must exist. chromium_utils.DatabaseSetup( c, require_dbconfig=active_master_cls.is_production_host) c['builders'] = _ComputeBuilders(builders, m_annotator) c['schedulers'] = _ComputeSchedulers(builders) c['change_source'], tag_comparator = _ComputeChangeSourceAndTagComparator( builders) # The 'slaves' list defines the set of allowable buildslaves. List all the # slaves registered to a builder. Remove dupes. c['slaves'] = master_utils.AutoSetupSlaves(c['builders'], Master.GetBotPassword()) # This does some sanity checks on the configuration. slaves = slaves_list.BaseSlavesList( chromium_utils.GetSlavesFromBuilders(builders), builders['master_classname']) master_utils.VerifySetup(c, slaves) default_public_html = os.path.join(chromium_utils.BUILD_DIR, 'masters', 'master.chromium', 'public_html') public_html = builders.get('public_html', default_public_html) # Adds common status and tools to this master. # TODO: Look at the logic in this routine to see if any of the logic # in this routine can be moved there to simplify things. master_utils.AutoSetupMaster( c, active_master_cls, public_html=public_html, templates=builders['templates'], tagComparator=tag_comparator, enable_http_status_push=active_master_cls.is_production_host) # TODO: AutoSetupMaster's settings for the following are too low to be # useful for most projects. We should fix that. c['buildHorizon'] = 3000 c['logHorizon'] = 3000 # Must be at least 2x the number of slaves. c['eventHorizon'] = 200
def Update(_config, active_master, c): factory_obj = annotator_factory.AnnotatorFactory( active_master=active_master) builders = [ Builder(factory_obj, 'x86-generic'), Builder(factory_obj, 'amd64-generic'), ] c['schedulers'] += [ Scheduler(name='chromium_src_asan', branch='master', treeStableTimer=60, builderNames=[b['name'] for b in builders], ), ] c['builders'] += builders
'build_config': 'Release', 'perf_id': 'gpu-webkit-linux-nvidia', 'triggered_by': 'GPU Linux Builder', 'auto_reboot': False, }, { 'builder': 'GPU Linux (dbg) (NVIDIA)', 'factory_id': 'f_gpu_linux_dbg', 'recipe': 'gpu/download_and_test', 'build_config': 'Debug', 'triggered_by': 'GPU Linux Builder (dbg)', 'auto_reboot': False, }, ] m_annotator = annotator_factory.AnnotatorFactory() defaults['category'] = 'gpu' # Maps the parent builder to a set of the names of the builders it triggers. trigger_map = collections.defaultdict(list) # Maps the name of the parent builder to the (synthesized) name of its # trigger, wrapped in a list. trigger_name_map = {} next_group_id = 0 # Note this code is very similar to that in recipe_master_helper.py. # Unfortunately due to the different structure of this waterfall it's # impossible to share the code. def BuilderExists(builder_name):
def SetupBuildersAndSchedulers(c, builders, slaves, ActiveMaster): """Set up builders and schedulers for the build master.""" # List of dicts for every builder. builder_dicts = [] # Builder names by scheduler. builders_by_scheduler = {s: [] for s in SCHEDULERS} # Maps a triggering builder to its triggered builders. triggered_builders = collections.defaultdict(list) def process_builder(builder, is_trybot=False): """Create a dict for the given builder and place its name in the appropriate scheduler list. """ builder_name = builder['name'] if is_trybot: builder_name = builder_name_schema.TrybotName(builder_name) # Categorize the builder based on its role. try: category = builder_name_schema.DictForBuilderName( builder_name)['role'] subcategory = builder_name.split( builder_name_schema.BUILDER_NAME_SEP)[1] category = '|'.join((category, subcategory)) except ValueError: # Assume that all builders whose names don't play by our rules are named # upstream and are therefore canaries. category = builder_name_schema.BUILDER_ROLE_CANARY builder_dict = { 'name': builder_name, 'gatekeeper': builder.get('gatekeeper_categories', ''), 'auto_reboot': builder.get('auto_reboot', DEFAULT_AUTO_REBOOT), 'slavenames': slaves.GetSlavesName(builder=builder['name']), 'category': category, 'recipe': builder.get('recipe', DEFAULT_RECIPE), } builder_dicts.append(builder_dict) parent_builder = builder.get('triggered_by') if parent_builder is not None: assert builder.get('scheduler') is None if is_trybot: parent_builder = builder_name_schema.TrybotName(parent_builder) triggered_builders[parent_builder].append(builder_name) elif is_trybot: builders_by_scheduler[TRY_SCHEDULER_NAME].append(builder_name) else: scheduler = builder.get('scheduler', PERCOMMIT_SCHEDULER_NAME) builders_by_scheduler[scheduler].append(builder_name) # Create builders and trybots. for builder in builders: process_builder(builder) if builder.get('do_trybot', DEFAULT_DO_TRYBOT): process_builder(builder, is_trybot=True) # Verify that all parent builders exist. all_nontriggered_builders = set( builders_by_scheduler[PERCOMMIT_SCHEDULER_NAME]).union( set(builders_by_scheduler[TRY_SCHEDULER_NAME])) trigger_parents = set(triggered_builders.keys()) nonexistent_parents = trigger_parents - all_nontriggered_builders if nonexistent_parents: raise Exception('Could not find parent builders: %s' % ', '.join(nonexistent_parents)) # Create the schedulers. def trigger_name(parent_builder): """Given a parent builder name, return a triggerable scheduler name.""" return 'triggers_%s' % parent_builder c['schedulers'] = [] s = Scheduler(name=PERCOMMIT_SCHEDULER_NAME, branch=POLLING_BRANCH, treeStableTimer=60, builderNames=builders_by_scheduler[PERCOMMIT_SCHEDULER_NAME]) c['schedulers'].append(s) s = timed.Nightly( name=PERIODIC_15MINS_SCHEDULER_NAME, branch=POLLING_BRANCH, builderNames=builders_by_scheduler[PERIODIC_15MINS_SCHEDULER_NAME], minute=[i * 15 for i in xrange(60 / 15)], hour='*', dayOfMonth='*', month='*', dayOfWeek='*') c['schedulers'].append(s) for parent, builders_to_trigger in triggered_builders.iteritems(): c['schedulers'].append( Triggerable(name=trigger_name(parent), builderNames=builders_to_trigger)) pools = BuildersPools(TRY_SCHEDULER_NAME) pools[TRY_SCHEDULER_NAME].extend(builders_by_scheduler[TRY_SCHEDULER_NAME]) c['schedulers'].append( TryJobRietveld(name=TRY_SCHEDULER_NAME, code_review_sites={ TRY_SCHEDULER_PROJECT: ActiveMaster.code_review_site }, pools=pools, project=TRY_SCHEDULER_PROJECT, filter_master=True)) # Create the BuildFactorys. annotator = annotator_factory.AnnotatorFactory() for builder_dict in builder_dicts: triggers = ([trigger_name(builder_dict['name'])] if builder_dict['name'] in triggered_builders else None) builder_dict['factory'] = annotator.BaseFactory(builder_dict['recipe'], triggers=triggers) # Finished! c['builders'] = builder_dicts
factory_properties={ 'android_bot_id': 'main-tests-rel', 'build_url': android_rel_archive, })) B('Android Clang Builder (dbg)', 'f_android_clang_dbg', 'android', 'android', notify_on_missing=True) F( 'f_android_clang_dbg', linux_android().ChromiumAnnotationFactory( target='Debug', annotation_script='src/build/android/buildbot/bb_run_bot.py', factory_properties={ 'android_bot_id': 'main-clang-builder-dbg', })) B('Android Webview AOSP Builder', 'f_android_webview_aosp_rel', 'android', 'android', notify_on_missing=True) F('f_android_webview_aosp_rel', annotator_factory.AnnotatorFactory().BaseFactory('android_webview_aosp')) def Update(_config_arg, _active_master, c): return helper.Update(c)
def SetupBuildersAndSchedulers(c, builders, slaves, ActiveMaster): """Set up builders and schedulers for the build master.""" # List of dicts for every builder. builder_dicts = [] # Builder names by scheduler. builders_by_scheduler = {s: [] for s in SCHEDULERS} # Maps a triggering builder to its triggered builders. triggered_builders = collections.defaultdict(list) def process_builder(builder, is_trybot=False): """Create a dict for the given builder and place its name in the appropriate scheduler list. """ builder_name = builder['name'] if is_trybot: builder_name = builder_name_schema.TrybotName(builder_name) # Categorize the builder based on its role. try: category = builder_name_schema.DictForBuilderName( builder_name)['role'] subcategory = builder_name.split( builder_name_schema.BUILDER_NAME_SEP)[1] category = '|'.join((category, subcategory)) except ValueError: # Assume that all builders whose names don't play by our rules are named # upstream and are therefore canaries. category = builder_name_schema.BUILDER_ROLE_CANARY properties = builder.get('properties', {}) cc = builder.get('cc') if cc: if isinstance(cc, basestring): cc = [cc] properties['owners'] = cc builder_dict = { 'name': builder_name, 'auto_reboot': builder.get('auto_reboot', DEFAULT_AUTO_REBOOT), 'slavenames': slaves.GetSlavesName(builder=builder['name']), 'category': category, 'recipe': builder.get('recipe', DEFAULT_RECIPE), 'properties': properties, 'mergeRequests': builder.get('can_merge_requests', CanMergeBuildRequests), 'slavebuilddir': SLAVE_WORKDIR, } builder_dicts.append(builder_dict) parent_builder = builder.get('triggered_by') if parent_builder is not None: assert builder.get('scheduler') is None if is_trybot: parent_builder = builder_name_schema.TrybotName(parent_builder) triggered_builders[parent_builder].append(builder_name) else: scheduler = builder.get('scheduler', BUILDBUCKET_SCHEDULER_NAME) # Setting the scheduler to BUILDBUCKET_SCHEDULER_NAME indicates that # BuildBucket is the only way to schedule builds for this bot; just # pretend to add a scheduler in those cases. builders_by_scheduler[scheduler].append(builder_name) # Create builders and trybots. for builder in builders: if builder.get('trybot_only', DEFAULT_TRYBOT_ONLY): # trybot_only=True should only be used in combination with do_trybot=True # Also, the buildername then needs to already have the '-Trybot' suffix. assert builder.get('do_trybot', DEFAULT_DO_TRYBOT) assert builder['name'] == builder_name_schema.TrybotName( builder['name']) else: process_builder(builder) if builder.get('do_trybot', DEFAULT_DO_TRYBOT): process_builder(builder, is_trybot=True) # Verify that all parent builders exist. all_nontriggered_builders = set( builders_by_scheduler[BUILDBUCKET_SCHEDULER_NAME]) trigger_parents = set(triggered_builders.keys()) nonexistent_parents = trigger_parents - all_nontriggered_builders if nonexistent_parents: raise Exception('Could not find parent builders: %s' % ', '.join(nonexistent_parents)) # Create the schedulers. infra_change_filter = change_filter.ChangeFilter( project='buildbot', repository=global_constants.INFRA_REPO) skia_master_only_change_filter = change_filter.ChangeFilter( project='skia', repository=ActiveMaster.repo_url, branch=MASTER_BRANCH) c['schedulers'] = [] s = Scheduler( name=MASTER_ONLY_SCHEDULER_NAME, treeStableTimer=60, change_filter=skia_master_only_change_filter, builderNames=builders_by_scheduler[MASTER_ONLY_SCHEDULER_NAME]) c['schedulers'].append(s) s = timed.Nightly( name=PERIODIC_15MINS_SCHEDULER_NAME, branch=MASTER_BRANCH, builderNames=builders_by_scheduler[PERIODIC_15MINS_SCHEDULER_NAME], minute=[i * 15 for i in xrange(60 / 15)], hour='*', dayOfMonth='*', month='*', dayOfWeek='*') c['schedulers'].append(s) s = timed.Nightly( name=NIGHTLY_SCHEDULER_NAME, branch=MASTER_BRANCH, builderNames=builders_by_scheduler[NIGHTLY_SCHEDULER_NAME], minute=0, hour=22, dayOfMonth='*', month='*', dayOfWeek='*') c['schedulers'].append(s) s = timed.Nightly( name=WEEKLY_SCHEDULER_NAME, branch=MASTER_BRANCH, builderNames=builders_by_scheduler[WEEKLY_SCHEDULER_NAME], minute=0, hour=0, dayOfMonth='*', month='*', dayOfWeek=6) # Sunday (Monday = 0). c['schedulers'].append(s) s = AnyBranchScheduler( name=INFRA_PERCOMMIT_SCHEDULER_NAME, treeStableTimer=0, change_filter=infra_change_filter, builderNames=builders_by_scheduler[INFRA_PERCOMMIT_SCHEDULER_NAME]) c['schedulers'].append(s) # Don't add triggerable schedulers for triggered_builders; triggers are now # handled on the slave-side through buildbucket. # Create the BuildFactorys. annotator = annotator_factory.AnnotatorFactory(ActiveMaster) for builder_dict in builder_dicts: factory = annotator.BaseFactory(builder_dict['recipe'], timeout=2400) factory.properties.update(builder_dict['properties'], 'BuildFactory') builder_dict['factory'] = factory # Finished! c['builders'] = builder_dicts