class JhBuildMaster(BuildMaster): jhbuild_config = config def loadConfig(self, f): # modified from parent method to get slaves, projects, change # sources, schedulers, builders and web status ouf of # master.cfg [it would have been cleaner if jhbuild didn't # have to copy all that code.] localDict = {'basedir': os.path.expanduser(self.basedir)} try: exec f in localDict except: log.msg("error while parsing config file") raise jhbuild_config.load() try: config = localDict['BuildmasterConfig'] except KeyError: log.err("missing config dictionary") log.err("config file must define BuildmasterConfig") raise known_keys = ("bots", "slaves", "sources", "change_source", "schedulers", "builders", "mergeRequests", "slavePortnum", "debugPassword", "logCompressionLimit", "manhole", "status", "projectName", "projectURL", "buildbotURL", "properties", "prioritizeBuilders", "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon", "changeHorizon", "logMaxSize", "logMaxTailSize", "logCompressionMethod", ) for k in config.keys(): if k not in known_keys: log.msg("unknown key '%s' defined in config dictionary" % k) # the 'slaves' list is read from the 'slaves.csv' file in the # current directory (unless instructed different from command line) # it is a CSV file structured like this: # slavename,password config['slaves'] = [] slaves_csv_file = os.path.join(slaves_dir, 'slaves.csv') if os.path.exists(slaves_csv_file): for x in csv.reader(file(slaves_csv_file)): if not x or x[0].startswith('#'): continue kw = {} build_slave = JhBuildSlave(x[0], x[1]) build_slave.load_extra_configuration(slaves_dir) config['slaves'].append(build_slave) if len(config['slaves']) == 0: log.msg('you must fill slaves.csv with slaves') module_set = jhbuild.moduleset.load(self.jhbuild_config) module_list = module_set.get_module_list( self.jhbuild_config.modules, self.jhbuild_config.skip, include_optional_modules=True) config['projects'] = [x.name for x in module_list \ if not x.name.startswith('meta-')] if self.jhbuild_config.jhbuildbot_svn_commits_box: # trigger builds from mails to svn-commit-list # (note Maildir must be correct, or everything will fail) from jhbuild.buildbot.changes import GnomeMaildirSource config['change_source'] = GnomeMaildirSource( self.jhbuild_config.jhbuildbot_svn_commits_box, modules=module_list, prefix=None) else: # support injection (use 'buildbot sendchange') from buildbot.changes.pb import PBChangeSource config['change_source'] = PBChangeSource() # Schedulers from jhbuild.buildbot.scheduler import SerialScheduler, NightlySerialScheduler, OnCommitScheduler config['schedulers'] = [] for slave in config['slaves']: s = None for project in config['projects']: buildername = str('%s-%s' % (project, slave.slavename)) scheduler_kwargs = {} if slave.scheduler == 'nightly': scheduler_class = NightlySerialScheduler scheduler_kwargs = slave.nightly_kwargs else: scheduler_class = SerialScheduler s = scheduler_class(buildername, project, upstream=s, builderNames=[buildername], **scheduler_kwargs) config['schedulers'].append(s) if self.jhbuild_config.jhbuildbot_svn_commits_box: # schedulers that will launch job when receiving # change notifications s2 = OnCommitScheduler('oc-' + buildername, project, builderNames=[buildername]) config['schedulers'].append(s2) # Builders from jhbuild.buildbot.factory import JHBuildFactory config['builders'] = [] for project in config['projects']: for slave in config['slaves']: f = JHBuildFactory(project, slave) config['builders'].append({ 'name' : "%s-%s" % (project, slave.slavename), 'slavename' : slave.slavename, 'builddir' : 'builddir/%s.%s' % (project, slave.slavename), 'factory' : f, 'category' : project }) # Status targets if not config.has_key('status'): # let it be possible to define additional status in # master.cfg config['status'] = [] from jhbuild.buildbot.status.web import JHBuildWebStatus config['status'].append( JHBuildWebStatus( self.jhbuild_config.moduleset, config['projects'], [x.slavename for x in config['slaves']], http_port=8080, allowForce=True) ) # remaining of the method is a straight copy from buildbot # ... try: # required schedulers = config['schedulers'] builders = config['builders'] slavePortnum = config['slavePortnum'] #slaves = config['slaves'] #change_source = config['change_source'] # optional debugPassword = config.get('debugPassword') manhole = config.get('manhole') status = config.get('status', []) projectName = config.get('projectName') projectURL = config.get('projectURL') buildbotURL = config.get('buildbotURL') properties = config.get('properties', {}) buildCacheSize = config.get('buildCacheSize', None) eventHorizon = config.get('eventHorizon', None) logHorizon = config.get('logHorizon', None) buildHorizon = config.get('buildHorizon', None) logCompressionLimit = config.get('logCompressionLimit', 4*1024) if logCompressionLimit is not None and not \ isinstance(logCompressionLimit, int): raise ValueError("logCompressionLimit needs to be bool or int") logCompressionMethod = config.get('logCompressionMethod', "bz2") if logCompressionMethod not in ('bz2', 'gz'): raise ValueError("logCompressionMethod needs to be 'bz2', or 'gz'") logMaxSize = config.get('logMaxSize') if logMaxSize is not None and not \ isinstance(logMaxSize, int): raise ValueError("logMaxSize needs to be None or int") logMaxTailSize = config.get('logMaxTailSize') if logMaxTailSize is not None and not \ isinstance(logMaxTailSize, int): raise ValueError("logMaxTailSize needs to be None or int") mergeRequests = config.get('mergeRequests') if mergeRequests is not None and not callable(mergeRequests): raise ValueError("mergeRequests must be a callable") prioritizeBuilders = config.get('prioritizeBuilders') if prioritizeBuilders is not None and not callable(prioritizeBuilders): raise ValueError("prioritizeBuilders must be callable") changeHorizon = config.get("changeHorizon") if changeHorizon is not None and not isinstance(changeHorizon, int): raise ValueError("changeHorizon needs to be an int") except KeyError, e: log.msg("config dictionary is missing a required parameter") log.msg("leaving old configuration in place") raise #if "bots" in config: # raise KeyError("c['bots'] is no longer accepted") slaves = config.get('slaves', []) if "bots" in config: m = ("c['bots'] is deprecated as of 0.7.6 and will be " "removed by 0.8.0 . Please use c['slaves'] instead.") log.msg(m) warnings.warn(m, DeprecationWarning) for name, passwd in config['bots']: slaves.append(JhBuildSlave(name, passwd)) if "bots" not in config and "slaves" not in config: log.msg("config dictionary must have either 'bots' or 'slaves'") log.msg("leaving old configuration in place") raise KeyError("must have either 'bots' or 'slaves'") #if "sources" in config: # raise KeyError("c['sources'] is no longer accepted") if changeHorizon is not None: self.change_svc.changeHorizon = changeHorizon change_source = config.get('change_source', []) if isinstance(change_source, (list, tuple)): change_sources = change_source else: change_sources = [change_source] if "sources" in config: m = ("c['sources'] is deprecated as of 0.7.6 and will be " "removed by 0.8.0 . Please use c['change_source'] instead.") log.msg(m) warnings.warn(m, DeprecationWarning) for s in config['sources']: change_sources.append(s) # do some validation first for s in slaves: assert interfaces.IBuildSlave.providedBy(s) if s.slavename in ("debug", "change", "status"): raise KeyError( "reserved name '%s' used for a bot" % s.slavename) if config.has_key('interlocks'): raise KeyError("c['interlocks'] is no longer accepted") assert isinstance(change_sources, (list, tuple)) for s in change_sources: assert interfaces.IChangeSource(s, None) # this assertion catches c['schedulers'] = Scheduler(), since # Schedulers are service.MultiServices and thus iterable. errmsg = "c['schedulers'] must be a list of Scheduler instances" assert isinstance(schedulers, (list, tuple)), errmsg for s in schedulers: assert interfaces.IScheduler(s, None), errmsg assert isinstance(status, (list, tuple)) for s in status: assert interfaces.IStatusReceiver(s, None) slavenames = [s.slavename for s in slaves] buildernames = [] dirnames = [] # convert builders from objects to config dictionaries builders_dicts = [] for b in builders: if isinstance(b, buildbot.config.BuilderConfig): builders_dicts.append(b.getConfigDict()) elif type(b) is dict: builders_dicts.append(b) else: raise ValueError("builder %s is not a BuilderConfig object (or a dict)" % b) builders = builders_dicts for b in builders: if b.has_key('slavename') and b['slavename'] not in slavenames: raise ValueError("builder %s uses undefined slave %s" \ % (b['name'], b['slavename'])) for n in b.get('slavenames', []): if n not in slavenames: raise ValueError("builder %s uses undefined slave %s" \ % (b['name'], n)) if b['name'] in buildernames: raise ValueError("duplicate builder name %s" % b['name']) buildernames.append(b['name']) # sanity check name (BuilderConfig does this too) if b['name'].startswith("_"): errmsg = ("builder names must not start with an " "underscore: " + b['name']) log.err(errmsg) raise ValueError(errmsg) # Fix the dictionnary with default values, in case this wasn't # specified with a BuilderConfig object (which sets the same defaults) b.setdefault('builddir', buildbot.util.safeTranslate(b['name'])) b.setdefault('slavebuilddir', b['builddir']) if b['builddir'] in dirnames: raise ValueError("builder %s reuses builddir %s" % (b['name'], b['builddir'])) dirnames.append(b['builddir']) unscheduled_buildernames = buildernames[:] schedulernames = [] for s in schedulers: for b in s.listBuilderNames(): assert b in buildernames, \ "%s uses unknown builder %s" % (s, b) if b in unscheduled_buildernames: unscheduled_buildernames.remove(b) if s.name in schedulernames: # TODO: schedulers share a namespace with other Service # children of the BuildMaster node, like status plugins, the # Manhole, the ChangeMaster, and the BotMaster (although most # of these don't have names) msg = ("Schedulers must have unique names, but " "'%s' was a duplicate" % (s.name,)) raise ValueError(msg) schedulernames.append(s.name) if unscheduled_buildernames: log.msg("Warning: some Builders have no Schedulers to drive them:" " %s" % (unscheduled_buildernames,)) # assert that all locks used by the Builds and their Steps are # uniquely named. lock_dict = {} for b in builders: for l in b.get('locks', []): if isinstance(l, locks.LockAccess): # User specified access to the lock l = l.lockid if lock_dict.has_key(l.name): if lock_dict[l.name] is not l: raise ValueError("Two different locks (%s and %s) " "share the name %s" % (l, lock_dict[l.name], l.name)) else: lock_dict[l.name] = l # TODO: this will break with any BuildFactory that doesn't use a # .steps list, but I think the verification step is more # important. for s in b['factory'].steps: for l in s[1].get('locks', []): if isinstance(l, locks.LockAccess): # User specified access to the lock l = l.lockid if lock_dict.has_key(l.name): if lock_dict[l.name] is not l: raise ValueError("Two different locks (%s and %s)" " share the name %s" % (l, lock_dict[l.name], l.name)) else: lock_dict[l.name] = l if not isinstance(properties, dict): raise ValueError("c['properties'] must be a dictionary") # slavePortnum supposed to be a strports specification if type(slavePortnum) is int: slavePortnum = "tcp:%d" % slavePortnum # now we're committed to implementing the new configuration, so do # it atomically # TODO: actually, this is spread across a couple of Deferreds, so it # really isn't atomic. d = defer.succeed(None) self.projectName = projectName self.projectURL = projectURL self.buildbotURL = buildbotURL self.properties = Properties() self.properties.update(properties, self.configFileName) self.status.logCompressionLimit = logCompressionLimit self.status.logCompressionMethod = logCompressionMethod self.status.logMaxSize = logMaxSize self.status.logMaxTailSize = logMaxTailSize # Update any of our existing builders with the current log parameters. # This is required so that the new value is picked up after a # reconfig. for builder in self.botmaster.builders.values(): builder.builder_status.setLogCompressionLimit(logCompressionLimit) builder.builder_status.setLogCompressionMethod(logCompressionMethod) builder.builder_status.setLogMaxSize(logMaxSize) builder.builder_status.setLogMaxTailSize(logMaxTailSize) if mergeRequests is not None: self.botmaster.mergeRequests = mergeRequests if prioritizeBuilders is not None: self.botmaster.prioritizeBuilders = prioritizeBuilders self.buildCacheSize = buildCacheSize self.eventHorizon = eventHorizon self.logHorizon = logHorizon self.buildHorizon = buildHorizon # self.slaves: Disconnect any that were attached and removed from the # list. Update self.checker with the new list of passwords, including # debug/change/status. d.addCallback(lambda res: self.loadConfig_Slaves(slaves)) # self.debugPassword if debugPassword: self.checker.addUser("debug", debugPassword) self.debugPassword = debugPassword # self.manhole if manhole != self.manhole: # changing if self.manhole: # disownServiceParent may return a Deferred d.addCallback(lambda res: self.manhole.disownServiceParent()) def _remove(res): self.manhole = None return res d.addCallback(_remove) if manhole: def _add(res): self.manhole = manhole manhole.setServiceParent(self) d.addCallback(_add) # add/remove self.botmaster.builders to match builders. The # botmaster will handle startup/shutdown issues. d.addCallback(lambda res: self.loadConfig_Builders(builders)) d.addCallback(lambda res: self.loadConfig_status(status)) # Schedulers are added after Builders in case they start right away d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers)) # and Sources go after Schedulers for the same reason d.addCallback(lambda res: self.loadConfig_Sources(change_sources)) # self.slavePort if self.slavePortnum != slavePortnum: if self.slavePort: def closeSlavePort(res): d1 = self.slavePort.disownServiceParent() self.slavePort = None return d1 d.addCallback(closeSlavePort) if slavePortnum is not None: def openSlavePort(res): self.slavePort = strports.service(slavePortnum, self.slaveFactory) self.slavePort.setServiceParent(self) d.addCallback(openSlavePort) log.msg("BuildMaster listening on port %s" % slavePortnum) self.slavePortnum = slavePortnum log.msg("configuration update started") def _done(res): self.readConfig = True log.msg("configuration update complete") d.addCallback(_done) d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds()) return d
raise KeyError, "reserved name '%s' used for a bot" % s.slavename if config.has_key('interlocks'): raise KeyError("c['interlocks'] is no longer accepted") assert isinstance(change_sources, (list, tuple)) for s in change_sources: assert interfaces.IChangeSource(s, None) # this assertion catches c['schedulers'] = Scheduler(), since # Schedulers are service.MultiServices and thus iterable. errmsg = "c['schedulers'] must be a list of Scheduler instances" assert isinstance(schedulers, (list, tuple)), errmsg for s in schedulers: assert interfaces.IScheduler(s, None), errmsg assert isinstance(status, (list, tuple)) for s in status: assert interfaces.IStatusReceiver(s, None) slavenames = [s.slavename for s in slaves] buildernames = [] dirnames = [] for b in builders: if type(b) is tuple: raise ValueError("builder %s must be defined with a dict, " "not a tuple" % b[0]) if b.has_key('slavename') and b['slavename'] not in slavenames: raise ValueError("builder %s uses undefined slave %s" \ % (b['name'], b['slavename'])) for n in b.get('slavenames', []): if n not in slavenames: raise ValueError("builder %s uses undefined slave %s" \ % (b['name'], n))