def testAutoSetupSlaves(self): def B(name, slavenames, auto_reboot): return { 'name': name, 'slavenames': slavenames, 'auto_reboot': auto_reboot, } builders = [ # Bot sharing two slaves. B('B1', ['S1', 'S2'], True), B('B2', ['S3', 'S4'], False), # Slave sharing two bots. B('B3', ['S5'], True), B('B4', ['S5'], False), # Slave sharing two bots (inverse auto-reboot). B('B5', ['S6'], False), B('B6', ['S6'], True), # Two builders heterogeneously sharing one slave. B('B7', ['S7'], True), B('B8', ['S7', 'S8'], False), ] slaves = dict( (slave.slavename, slave) for slave in master_utils.AutoSetupSlaves(builders, 'pwd')) self.assertTrue(isinstance(slaves['S1'], AutoRebootBuildSlave)) self.assertTrue(isinstance(slaves['S2'], AutoRebootBuildSlave)) self.assertFalse(isinstance(slaves['S3'], AutoRebootBuildSlave)) self.assertFalse(isinstance(slaves['S4'], AutoRebootBuildSlave)) self.assertTrue(isinstance(slaves['S5'], AutoRebootBuildSlave)) self.assertTrue(isinstance(slaves['S6'], AutoRebootBuildSlave)) self.assertTrue(isinstance(slaves['S7'], AutoRebootBuildSlave)) self.assertFalse(isinstance(slaves['S8'], AutoRebootBuildSlave))
def _Populate(BuildmasterConfig, builders, active_master_cls): m_annotator = annotator_factory.AnnotatorFactory(active_master_cls) c = BuildmasterConfig c['logCompressionLimit'] = False c['projectName'] = active_master_cls.project_name c['projectURL'] = Master.project_url c['buildbotURL'] = active_master_cls.buildbot_url # This sets c['db_url'] to the database connect string in found in # the .dbconfig in the master directory, if it exists. If this is # a production host, it must exist. chromium_utils.DatabaseSetup( c, require_dbconfig=active_master_cls.is_production_host) c['builders'] = _ComputeBuilders(builders, m_annotator) c['schedulers'] = _ComputeSchedulers(builders) c['change_source'], tag_comparator = _ComputeChangeSourceAndTagComparator( builders) # The 'slaves' list defines the set of allowable buildslaves. List all the # slaves registered to a builder. Remove dupes. c['slaves'] = master_utils.AutoSetupSlaves(c['builders'], Master.GetBotPassword()) # This does some sanity checks on the configuration. slaves = slaves_list.BaseSlavesList( chromium_utils.GetSlavesFromBuilders(builders), builders['master_classname']) master_utils.VerifySetup(c, slaves) default_public_html = os.path.join(chromium_utils.BUILD_DIR, 'masters', 'master.chromium', 'public_html') public_html = builders.get('public_html', default_public_html) # Adds common status and tools to this master. # TODO: Look at the logic in this routine to see if any of the logic # in this routine can be moved there to simplify things. master_utils.AutoSetupMaster( c, active_master_cls, public_html=public_html, templates=builders['templates'], tagComparator=tag_comparator, enable_http_status_push=active_master_cls.is_production_host) # TODO: AutoSetupMaster's settings for the following are too low to be # useful for most projects. We should fix that. c['buildHorizon'] = 3000 c['logHorizon'] = 3000 # Must be at least 2x the number of slaves. c['eventHorizon'] = 200
def SetupMaster(ActiveMaster): # Buildmaster config dict. c = {} config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host) ####### CHANGESOURCES # Polls config.Master.trunk_url for changes poller = GitilesPoller( repo_url=ActiveMaster.repo_url, branches=['master'], pollInterval=10, revlinktmpl='https://skia.googlesource.com/skia/+/%s') c['change_source'] = [poller] ####### SLAVES # Load the slave list. We need some information from it in order to # produce the builders. slaves = slaves_list.SlavesList('slaves.cfg', ActiveMaster.project_name) ####### BUILDERS # Load the builders list. builders = chromium_utils.ParsePythonCfg('builders.cfg')['builders'] # Configure the Builders and Schedulers. SetupBuildersAndSchedulers(c=c, builders=builders, slaves=slaves, ActiveMaster=ActiveMaster) ####### BUILDSLAVES # The 'slaves' list defines the set of allowable buildslaves. List all the # slaves registered to a builder. Remove dupes. c['slaves'] = master_utils.AutoSetupSlaves(c['builders'], config.Master.GetBotPassword()) master_utils.VerifySetup(c, slaves) ####### STATUS TARGETS c['buildbotURL'] = ActiveMaster.buildbot_url # Adds common status and tools to this master. master_utils.AutoSetupMaster( c, ActiveMaster, public_html='../../../build/masters/master.client.skia/public_html', templates=[ '../../../build/masters/master.client.skia/templates', '../../../build/masters/master.chromium/templates' ], tagComparator=poller.comparator, enable_http_status_push=ActiveMaster.is_production_host, order_console_by_time=True, console_repo_filter=ActiveMaster.repo_url, console_builder_filter=lambda b: not builder_name_schema.IsTrybot(b)) with status_json.JsonStatusHelper() as json_helper: json_helper.putChild('trybots', status_json.TryBuildersJsonResource) if ActiveMaster.is_production_host: # Build result emails. c['status'].append( skia_notifier.SkiaMailNotifier( fromaddr=ActiveMaster.from_address, mode='change', relayhost=config.Master.smtp, lookup=master_utils.UsersAreEmails())) # Try job result emails. c['status'].append( skia_notifier.SkiaTryMailNotifier( fromaddr=ActiveMaster.from_address, subject="try %(result)s for %(reason)s @ r%(revision)s", mode='all', relayhost=config.Master.smtp, lookup=master_utils.UsersAreEmails())) # Rietveld status push. c['status'].append( TryServerHttpStatusPush(serverUrl=ActiveMaster.code_review_site)) return c
def get_slaves(builders): # The 'slaves' list defines the set of allowable buildslaves. List all the # slaves registered to a builder. Remove dupes. return master_utils.AutoSetupSlaves(builders, config.Master.GetBotPassword())
def SetupMaster(ActiveMaster): # Buildmaster config dict. c = {} config.DatabaseSetup(c) ####### CHANGESOURCES # Polls config.Master.trunk_url for changes poller = GitilesPoller( repo_url=ActiveMaster.repo_url, branches=[POLLING_BRANCH], pollInterval=10, revlinktmpl='https://skia.googlesource.com/skia/+/%s') c['change_source'] = [poller] ####### SLAVES # Load the slave list. We need some information from it in order to # produce the builders. slaves = slaves_list.SlavesList('slaves.cfg', ActiveMaster.project_name) ####### BUILDERS # Load the builders list. builders = chromium_utils.ParsePythonCfg('builders.cfg')['builders'] # Configure the Builders and Schedulers. SetupBuildersAndSchedulers(c=c, builders=builders, slaves=slaves, ActiveMaster=ActiveMaster) ####### BUILDSLAVES # The 'slaves' list defines the set of allowable buildslaves. List all the # slaves registered to a builder. Remove dupes. c['slaves'] = master_utils.AutoSetupSlaves(c['builders'], config.Master.GetBotPassword()) master_utils.VerifySetup(c, slaves) ####### STATUS TARGETS c['buildbotURL'] = ActiveMaster.buildbot_url # Adds common status and tools to this master. master_utils.AutoSetupMaster( c, ActiveMaster, public_html='../../../build/masters/master.client.skia/public_html', templates=[ '../../../build/masters/master.client.skia/templates', '../../../build/masters/master.chromium/templates' ], tagComparator=poller.comparator, enable_http_status_push=ActiveMaster.is_production_host, order_console_by_time=True, console_repo_filter=ActiveMaster.repo_url, console_builder_filter=lambda b: not builder_name_schema.IsTrybot(b)) with status_json.JsonStatusHelper() as json_helper: json_helper.putChild('trybots', status_json.TryBuildersJsonResource) if (ActiveMaster.is_production_host and ActiveMaster.project_name != 'SkiaInternal'): # Build result emails. c['status'].append(status_logger.StatusEventLogger()) c['status'].append( skia_notifier.SkiaMailNotifier( fromaddr=ActiveMaster.from_address, mode='change', relayhost=config.Master.smtp, lookup=master_utils.UsersAreEmails())) # Try job result emails. c['status'].append( skia_notifier.SkiaTryMailNotifier( fromaddr=ActiveMaster.from_address, subject="try %(result)s for %(reason)s @ r%(revision)s", mode='all', relayhost=config.Master.smtp, lookup=master_utils.UsersAreEmails())) # Push status updates to GrandCentral. c['status'].append( HttpStatusPush(serverUrl='https://grandcentral.skia.org/buildbot')) c['mergeRequests'] = CanMergeBuildRequests ###### LOGS # Skia bots have been known to have run away builds continously dumping to # stdout and creating ~100GB logs. See crbug.com/589654 for context. c['logMaxSize'] = 1024 * 1024 * 100 # 100MB c['logMaxTailSize'] = 1024 * 32 # 32KB return c