def reconfigServiceWithSibling(self, sibling): # only reconfigure if sibling is configured differently. # sibling == self is using ComparableMixin's implementation # only compare compare_attrs if self.configured and sibling == self: defer.returnValue(None) self.configured = True # render renderables in parallel # Properties import to resolve cyclic import issue from buildbot.process.properties import Properties p = Properties() p.master = self.master # render renderables in parallel secrets = [] kwargs = {} accumulateClassList(self.__class__, 'secrets', secrets) for k, v in sibling._config_kwargs.items(): if k in secrets: value = yield p.render(v) setattr(self, k, value) kwargs.update({k: value}) else: kwargs.update({k: v}) d = yield self.reconfigService(*sibling._config_args, **sibling._config_kwargs) defer.returnValue(d)
def getAndCheckProperties(req): """ Fetch custom build properties from the HTTP request of a "Force build" or "Resubmit build" HTML form. Check the names for valid strings, and return None if a problem is found. Return a new Properties object containing each property found in req. """ master = req.site.buildbot_service.master pname_validate = master.config.validation['property_name'] pval_validate = master.config.validation['property_value'] properties = Properties() i = 1 while True: pname = req.args.get("property%dname" % i, [""])[0] pvalue = req.args.get("property%dvalue" % i, [""])[0] if not pname: break if not pname_validate.match(pname) \ or not pval_validate.match(pvalue): log.msg("bad property name='%s', value='%s'" % (pname, pvalue)) return None properties.setProperty(pname, pvalue, "Force Build Form") i = i + 1 return properties
def _downloadSshPrivateKeyIfNeeded(self): if self.sshPrivateKey is None: defer.returnValue(RC_SUCCESS) p = Properties() p.master = self.master private_key = yield p.render(self.sshPrivateKey) # not using self.workdir because it may be changed depending on step # options workdir = self._getSshDataWorkDir() rel_key_path = self.build.path_module.relpath( self._getSshPrivateKeyPath(), workdir) rel_wrapper_script_path = self.build.path_module.relpath( self._getSshWrapperScriptPath(), workdir) yield self.runMkdir(self._getSshDataPath()) if not self.supportsSshPrivateKeyAsEnvOption: yield self.downloadFileContentToWorker(rel_wrapper_script_path, self._getSshWrapperScript(), workdir=workdir, mode=0o700) yield self.downloadFileContentToWorker(rel_key_path, private_key, workdir=workdir, mode=0o400) self.didDownloadSshPrivateKey = True defer.returnValue(RC_SUCCESS)
class BaseScheduler(service.MultiService, util.ComparableMixin): """ A Schduler creates BuildSets and submits them to the BuildMaster. @ivar name: name of the scheduler @ivar properties: additional properties specified in this scheduler's configuration @type properties: Properties object """ implements(interfaces.IScheduler) def __init__(self, name, properties={}): """ @param name: name for this scheduler @param properties: properties to be propagated from this scheduler @type properties: dict """ service.MultiService.__init__(self) self.name = name self.properties = Properties() self.properties.update(properties, "Scheduler") self.properties.setProperty("scheduler", name, "Scheduler") def __repr__(self): # TODO: why can't id() return a positive number? %d is ugly. return "<Scheduler '%s' at %d>" % (self.name, id(self)) def submitBuildSet(self, bs): self.parent.submitBuildSet(bs) def addChange(self, change): pass
def _downloadSshPrivateKeyIfNeeded(self): if self.sshPrivateKey is None: defer.returnValue(RC_SUCCESS) p = Properties() p.master = self.master private_key = yield p.render(self.sshPrivateKey) host_key = yield p.render(self.sshHostKey) # not using self.workdir because it may be changed depending on step # options workdir = self._getSshDataWorkDir() yield self.runMkdir(self._getSshDataPath()) if not self.supportsSshPrivateKeyAsEnvOption: yield self.downloadFileContentToWorker(self._getSshWrapperScriptPath(), self._getSshWrapperScript(), workdir=workdir, mode=0o700) yield self.downloadFileContentToWorker(self._getSshPrivateKeyPath(), private_key, workdir=workdir, mode=0o400) if self.sshHostKey is not None: known_hosts_contents = getSshKnownHostsContents(host_key) yield self.downloadFileContentToWorker(self._getSshHostKeyPath(), known_hosts_contents, workdir=workdir, mode=0o400) self.didDownloadSshPrivateKey = True defer.returnValue(RC_SUCCESS)
def trigger(self, ssid, set_props=None): """Trigger this scheduler with the given sourcestamp ID. Returns a deferred that will fire when the buildset is finished.""" # properties for this buildset are composed of our own properties, # potentially overridden by anything from the triggering build props = Properties() props.updateFromProperties(self.properties) if set_props: props.updateFromProperties(set_props) # note that this does not use the buildset subscriptions mechanism, as # the duration of interest to the caller is bounded by the lifetime of # this process. if ssid: d = self.addBuildsetForSourceStamp(reason=self.reason, ssid=ssid, properties=props) else: d = self.addBuildsetForLatest(reason=self.reason, properties=props) def setup_waiter((bsid,brids)): d = defer.Deferred() self._waiters[bsid] = (d, brids) self._updateWaiters() return d d.addCallback(setup_waiter) return d
def perspective_try(self, branch, revision, patch, builderNames, properties={}): log.msg("user %s requesting build on builders %s" % (self.username, builderNames)) for b in builderNames: if not b in self.parent.builderNames: log.msg("%s got job with builder %s" % (self, b)) log.msg(" but that wasn't in our list: %s" % (self.parent.builderNames,)) return ss = SourceStamp(branch, revision, patch) reason = "'try' job from user %s" % self.username # roll the specified props in with our inherited props combined_props = Properties() combined_props.updateFromProperties(self.parent.properties) combined_props.update(properties, "try build") bs = buildset.BuildSet(builderNames, ss, reason=reason, properties=combined_props) self.parent.submitBuildSet(bs) # return a remotely-usable BuildSetStatus object from buildbot.status.client import makeRemote return makeRemote(bs.status)
class Try_Userpass(TryBase): compare_attrs = ( 'name', 'builderNames', 'port', 'userpass', 'properties' ) def __init__(self, name, builderNames, port, userpass, properties={}): base.BaseScheduler.__init__(self, name, builderNames, properties) self.port = port self.userpass = userpass self.properties = Properties() self.properties.update(properties, 'Scheduler') def startService(self): TryBase.startService(self) master = self.parent.parent # register each user/passwd with the pbmanager def factory(mind, username): return Try_Userpass_Perspective(self, username) self.registrations = [] for user, passwd in self.userpass: self.registrations.append( master.pbmanager.register(self.port, user, passwd, factory)) def stopService(self): d = defer.maybeDeferred(TryBase.stopService, self) def unreg(_): return defer.gatherResults( [ reg.unregister() for reg in self.registrations ]) d.addCallback(unreg)
def testBasic(self): s = transfer.JSONPropertiesDownload("props.json") s.build = Mock() props = Properties() props.setProperty('key1', 'value1', 'test') s.build.getProperties.return_value = props s.build.getSlaveCommandVersion.return_value = 1 ss = Mock() ss.asDict.return_value = dict(revision="12345") s.build.getSourceStamp.return_value = ss s.step_status = Mock() s.buildslave = Mock() s.remote = Mock() s.start() for c in s.remote.method_calls: name, command, args = c commandName = command[3] kwargs = command[-1] if commandName == 'downloadFile': self.assertEquals(kwargs['slavedest'], 'props.json') reader = kwargs['reader'] data = reader.remote_read(100) self.assertEquals(data, json.dumps(dict(sourcestamp=ss.asDict(), properties={'key1': 'value1'}))) break else: self.assert_(False, "No downloadFile command found")
class TestInterpolatePositional(unittest.TestCase): def setUp(self): self.props = Properties() self.build = FakeBuild(self.props) def test_string(self): command = Interpolate("test %s", "one fish") d = self.build.render(command) d.addCallback(self.failUnlessEqual, "test one fish") def test_twoString(self): command = Interpolate("test %s, %s", "one fish", "two fish") d = self.build.render(command) d.addCallback(self.failUnlessEqual, "test one fish, two fish") def test_deferred(self): renderable = DeferredRenderable() command = Interpolate("echo '%s'", renderable) d = self.build.render(command) d.addCallback(self.failUnlessEqual, "echo 'red fish'") renderable.callback("red fish") return d def test_renderable(self): self.props.setProperty("buildername", "blue fish", "test") command = Interpolate("echo '%s'", Property("buildername")) d = self.build.render(command) d.addCallback(self.failUnlessEqual, "echo 'blue fish'") return d
def testBasic(self): s = transfer.JSONPropertiesDownload("props.json") s.build = Mock() props = Properties() props.setProperty('key1', 'value1', 'test') s.build.getProperties.return_value = props s.build.getWorkerCommandVersion.return_value = '3.0' ss = Mock() ss.asDict.return_value = dict(revision="12345") s.build.getAllSourceStamps.return_value = [ss] s.worker = Mock() s.remote = Mock() s.start() for c in s.remote.method_calls: name, command, args = c commandName = command[3] kwargs = command[-1] if commandName == 'downloadFile': self.assertEquals(kwargs['workerdest'], 'props.json') reader = kwargs['reader'] data = reader.remote_read(100) self.assertEquals( data, json.dumps(dict(sourcestamps=[ss.asDict()], properties={'key1': 'value1'}))) break else: raise ValueError("No downloadFile command found")
def perspective_try(self, branch, revision, patch, repository, project, builderNames, who="", comment="", properties=None): log.msg("user %s requesting build on builders %s" % (self.username, builderNames)) if properties is None: properties = {} # build the intersection of the request and our configured list builderNames = self.scheduler.filterBuilderList(builderNames) if not builderNames: return reason = u"'try' job" if who: reason += u" by user %s" % ascii2unicode(who) if comment: reason += u" (%s)" % ascii2unicode(comment) sourcestamp = dict( branch=branch, revision=revision, repository=repository, project=project, patch_level=patch[0], patch_body=patch[1], patch_subdir='', patch_author=who or '', patch_comment=comment or '', codebase='', ) # note: no way to specify patch subdir - #1769 requested_props = Properties() requested_props.update(properties, "try build") (bsid, brids) = yield self.scheduler.addBuildsetForSourceStamps( sourcestamps=[sourcestamp], reason=reason, properties=requested_props, builderNames=builderNames) # return a remotely-usable BuildSetStatus object bss = RemoteBuildSetStatus(self.scheduler.master, bsid, brids) defer.returnValue(bss)
def reconfigServiceWithSibling(self, sibling): # only reconfigure if sibling is configured differently. # sibling == self is using ComparableMixin's implementation # only compare compare_attrs if self.configured and sibling == self: return None self.configured = True # render renderables in parallel # Properties import to resolve cyclic import issue from buildbot.process.properties import Properties p = Properties() p.master = self.master # render renderables in parallel secrets = [] kwargs = {} accumulateClassList(self.__class__, 'secrets', secrets) for k, v in sibling._config_kwargs.items(): if k in secrets: # for non reconfigurable services, we force the attribute v = yield p.render(v) setattr(sibling, k, v) setattr(self, k, v) kwargs[k] = v d = yield self.reconfigService(*sibling._config_args, **kwargs) return d
def canStartWithWorkerForBuilder(self, workerforbuilder, buildrequests=None): locks = self.config.locks if IRenderable.providedBy(locks): if buildrequests is None: raise RuntimeError("buildrequests parameter must be specified " " when using renderable builder locks. Not " "specifying buildrequests is deprecated") # collect properties that would be set for a build if we # started it now and render locks using it props = Properties() Build.setupPropertiesKnownBeforeBuildStarts(props, buildrequests, self, workerforbuilder) locks = yield props.render(locks) # Make sure we don't warn and throw an exception at the same time if buildrequests is None: warnings.warn( "Not passing corresponding buildrequests to " "Builder.canStartWithWorkerForBuilder is deprecated") locks = [(self.botmaster.getLockFromLockAccess(access), access) for access in locks] can_start = Build.canStartWithWorkerForBuilder(locks, workerforbuilder) defer.returnValue(can_start)
def testBasic(self): s = transfer.JSONPropertiesDownload("props.json") s.build = Mock() props = Properties() props.setProperty("key1", "value1", "test") s.build.getProperties.return_value = props s.build.getWorkerCommandVersion.return_value = 1 ss = Mock() ss.asDict.return_value = dict(revision="12345") s.build.getAllSourceStamps.return_value = [ss] s.worker = Mock() s.remote = Mock() s.start() for c in s.remote.method_calls: name, command, args = c commandName = command[3] kwargs = command[-1] if commandName == "downloadFile": self.assertEquals(kwargs["slavedest"], "props.json") reader = kwargs["reader"] data = reader.remote_read(100) self.assertEquals(data, json.dumps(dict(sourcestamps=[ss.asDict()], properties={"key1": "value1"}))) break else: raise ValueError("No downloadFile command found")
def buildIDSchedFunc(sched, t, ssid): """Generates a unique buildid for this change. Returns a Properties instance with 'buildid' set to the buildid to use. scheduler `sched`'s state is modified as a result.""" state = sched.get_state(t) # Get the last buildid we scheduled from the database lastid = state.get('last_buildid', '19700101000000') incrementedid = incrementBuildID(lastid) nowid = genBuildID() # Our new buildid will be the highest of the last buildid incremented or # the buildid based on the current date newid = str(max(int(nowid), int(incrementedid))) # Save it in the scheduler's state so we don't generate the same one again. state['last_buildid'] = newid sched.set_state(t, state) props = Properties() props.setProperty('buildid', newid, 'buildIDSchedFunc') return props
def testBasicWorker2_16(self): s = transfer.JSONPropertiesDownload("props.json") s.build = Mock() props = Properties() props.setProperty('key1', 'value1', 'test') s.build.getProperties.return_value = props s.build.getWorkerCommandVersion.return_value = '2.16' ss = Mock() ss.asDict.return_value = dict(revision="12345") s.build.getAllSourceStamps.return_value = [ss] s.worker = Mock() s.remote = Mock() s.start() for c in s.remote.method_calls: name, command, args = c commandName = command[3] kwargs = command[-1] if commandName == 'downloadFile': self.assertEqual(kwargs['slavedest'], 'props.json') reader = kwargs['reader'] data = reader.remote_read(100) data = bytes2unicode(data) actualJson = json.loads(data) expectedJson = dict(sourcestamps=[ss.asDict()], properties={'key1': 'value1'}) self.assertEqual(actualJson, expectedJson) break else: raise ValueError("No downloadFile command found")
def perspective_requestBuild(self, buildername, reason, branch, revision, properties={}): c = interfaces.IControl(self.master) bc = c.getBuilder(buildername) ss = SourceStamp(branch, revision) bpr = Properties() bpr.update(properties, "remote requestBuild") return bc.submitBuildRequest(ss, reason, bpr)
def canStartBuild(self, workerforbuilder, buildrequest): can_start = True # check whether the locks that the build will acquire can actually be # acquired locks = self.config.locks if IRenderable.providedBy(locks): # collect properties that would be set for a build if we # started it now and render locks using it props = Properties() Build.setupPropertiesKnownBeforeBuildStarts(props, [buildrequest], self, workerforbuilder) locks = yield props.render(locks) locks = [(self.botmaster.getLockFromLockAccess(access), access) for access in locks] if locks: can_start = Build._canAcquireLocks(locks, workerforbuilder) if can_start is False: defer.returnValue(can_start) if callable(self.config.canStartBuild): can_start = yield self.config.canStartBuild(self, workerforbuilder, buildrequest) defer.returnValue(can_start)
def trigger(self, waited_for, sourcestamps=None, set_props=None, parent_buildid=None, parent_relationship=None): """Trigger this scheduler with the optional given list of sourcestamps Returns two deferreds: idsDeferred -- yields the ids of the buildset and buildrequest, as soon as they are available. resultsDeferred -- yields the build result(s), when they finish.""" # properties for this buildset are composed of our own properties, # potentially overridden by anything from the triggering build props = Properties() props.updateFromProperties(self.properties) if set_props: props.updateFromProperties(set_props) # note that this does not use the buildset subscriptions mechanism, as # the duration of interest to the caller is bounded by the lifetime of # this process. idsDeferred = self.addBuildsetForSourceStampsWithDefaults( self.reason, sourcestamps, waited_for, properties=props, parent_buildid=parent_buildid, parent_relationship=parent_relationship) resultsDeferred = defer.Deferred() @idsDeferred.addCallback def setup_waiter(ids): bsid, brids = ids self._waiters[bsid] = (resultsDeferred, brids) self._updateWaiters() return ids return idsDeferred, resultsDeferred
def verifyCode(self, code): # everything in deferToThread is not counted with trial --coverage :-( def thd(client_id, client_secret): url = self.tokenUri data = {'redirect_uri': self.loginUri, 'code': code, 'grant_type': self.grantType} auth = None if self.getTokenUseAuthHeaders: auth = (client_id, client_secret) else: data.update( {'client_id': client_id, 'client_secret': client_secret}) data.update(self.tokenUriAdditionalParams) response = requests.post( url, data=data, auth=auth, verify=self.sslVerify) response.raise_for_status() responseContent = bytes2unicode(response.content) try: content = json.loads(responseContent) except ValueError: content = parse_qs(responseContent) for k, v in content.items(): content[k] = v[0] except TypeError: content = responseContent session = self.createSessionFromToken(content) return self.getUserInfoFromOAuthClient(session) p = Properties() p.master = self.master client_id = yield p.render(self.clientId) client_secret = yield p.render(self.clientSecret) result = yield threads.deferToThread(thd, client_id, client_secret) return result
def TryJobBaseGetProps(self, builder, options): """ Override of try_job_base.TryJobBase.get_props: http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_base.py?view=markup We modify it to add "baseurl". """ keys = ( ############################### Added by borenet ############################### 'baseurl', ################################################################################ 'clobber', 'issue', 'patchset', 'requester', 'rietveld', 'root', 'try_job_key', ) # All these settings have no meaning when False or not set, so don't set # them in that case. properties = dict((i, options[i]) for i in keys if options.get(i)) properties['testfilter'] = options['bot'].get(builder, None) # pylint: disable=W0212 props = Properties() props.updateFromProperties(self.properties) props.update(properties, self._PROPERTY_SOURCE) return props
def perspective_try(self, branch, revision, patch, repository, project, builderNames, properties={}, ): db = self.scheduler.master.db log.msg("user %s requesting build on builders %s" % (self.username, builderNames)) # build the intersection of the request and our configured list builderNames = self.scheduler.filterBuilderList(builderNames) if not builderNames: return wfd = defer.waitForDeferred( db.sourcestamps.createSourceStamp(branch=branch, revision=revision, repository=repository, project=project, patch_level=patch[0], patch_body=patch[1], patch_subdir='')) # note: no way to specify patch subdir - #1769 yield wfd ssid = wfd.getResult() reason = "'try' job from user %s" % self.username requested_props = Properties() requested_props.update(properties, "try build") wfd = defer.waitForDeferred( self.scheduler.addBuildsetForSourceStamp(ssid=ssid, reason=reason, properties=requested_props, builderNames=builderNames)) yield wfd bsid = wfd.getResult() # return a remotely-usable BuildSetStatus object bss = BuildSetStatus(bsid, self.scheduler.master.status, db) from buildbot.status.client import makeRemote r = makeRemote(bss) yield r # return value
def perspective_try(self, branch, revision, patch, repository, project, builderNames, properties={}, ): log.msg("user %s requesting build on builders %s" % (self.username, builderNames)) # build the intersection of the request and our configured list builderNames = self.parent.filterBuilderList(builderNames) if not builderNames: return ss = SourceStamp(branch, revision, patch, repository=repository, project=project) reason = "'try' job from user %s" % self.username # roll the specified props in with our inherited props combined_props = Properties() combined_props.updateFromProperties(self.parent.properties) combined_props.update(properties, "try build") status = self.parent.parent.parent.status db = self.parent.parent.db d = db.runInteraction(self._try, ss, builderNames, reason, combined_props, db) def _done(bsid): # return a remotely-usable BuildSetStatus object bss = BuildSetStatus(bsid, status, db) from buildbot.status.client import makeRemote r = makeRemote(bss) #self.parent.parent.loop_done() # so it will notify builder loop return r d.addCallback(_done) return d
def MockBuild(my_builder, buildsetup, mastername, slavename, basepath=None, build_properties=None, slavedir=None): """Given a builder object and configuration, mock a Buildbot setup around it. This sets up a mock BuildMaster, BuildSlave, Build, BuildStatus, and all other superstructure required for BuildSteps inside the provided builder to render properly. These BuildSteps are returned to the user in an array. It additionally returns the build object (in order to get its properties if desired). buildsetup is passed straight into the FakeSource's init method and contains sourcestamp information (revision, branch, etc). basepath is the directory of the build (what goes under build/slave/, for example 'Chromium_Linux_Builder'. It is nominally inferred from the builder name, but it can be overridden. This is useful when pointing the buildrunner at a different builder than what it's running under. build_properties will update and override build_properties after all builder-derived defaults have been set. """ my_factory = my_builder['factory'] steplist = ListSteps(my_factory) build = base.Build([FakeRequest(buildsetup)]) safename = buildbot.util.safeTranslate(my_builder['name']) if not basepath: basepath = safename if not slavedir: slavedir = os.path.join(SCRIPT_DIR, '..', '..', 'slave') basedir = os.path.join(slavedir, basepath) build.basedir = basedir builderstatus = builder.BuilderStatus('test') builderstatus.nextBuildNumber = 2 builderstatus.basedir = basedir my_builder['builddir'] = safename my_builder['slavebuilddir'] = safename mybuilder = real_builder.Builder(my_builder, builderstatus) build.setBuilder(mybuilder) build_status = build_module.BuildStatus(builderstatus, 1) build_status.setProperty('blamelist', [], 'Build') build_status.setProperty('mastername', mastername, 'Build') build_status.setProperty('slavename', slavename, 'Build') build_status.setProperty('gtest_filter', [], 'Build') # if build_properties are passed in, overwrite the defaults above: buildprops = Properties() if build_properties: buildprops.update(build_properties, 'Botmaster') mybuilder.setBotmaster(FakeBotmaster(mastername, buildprops)) buildslave = FakeSlave(safename, slavename) build.build_status = build_status build.setupSlaveBuilder(buildslave) build.setupProperties() process_steps(steplist, build, buildslave, build_status, basedir) return steplist, build
def addBuildsetForSourceStamps(self, waited_for=False, sourcestamps=None, reason='', external_idstring=None, properties=None, builderNames=None, **kw): if sourcestamps is None: sourcestamps = [] # combine properties if properties: properties.updateFromProperties(self.properties) else: properties = self.properties # make a fresh copy that we actually can modify safely properties = Properties.fromDict(properties.asDict()) # make extra info available from properties.render() properties.master = self.master properties.sourcestamps = [] properties.changes = [] for ss in sourcestamps: if isinstance(ss, int): # fetch actual sourcestamp and changes from data API properties.sourcestamps.append( (yield self.master.data.get(('sourcestamps', ss)))) properties.changes.extend( (yield self.master.data.get(('sourcestamps', ss, 'changes')))) else: # sourcestamp with no change, see addBuildsetForChanges properties.sourcestamps.append(ss) for c in properties.changes: properties.updateFromProperties(Properties.fromDict(c['properties'])) # apply the default builderNames if not builderNames: builderNames = self.builderNames # dynamically get the builder list to schedule builderNames = yield properties.render(builderNames) # Get the builder ids # Note that there is a data.updates.findBuilderId(name) # but that would merely only optimize the single builder case, while # probably the multiple builder case will be severely impacted by the # several db requests needed. builderids = list() for bldr in (yield self.master.data.get(('builders', ))): if bldr['name'] in builderNames: builderids.append(bldr['builderid']) # translate properties object into a dict as required by the # addBuildset method properties_dict = properties.asDict() bsid, brids = yield self.master.data.updates.addBuildset( scheduler=self.name, sourcestamps=sourcestamps, reason=reason, waited_for=waited_for, properties=properties_dict, builderids=builderids, external_idstring=external_idstring, **kw) return (bsid, brids)
def createTriggerProperties(self): # make a new properties object from a dict rendered by the old # properties object trigger_properties = Properties() trigger_properties.update(self.set_properties, "Trigger") for prop in trigger_properties.asDict(): if prop not in self.set_properties: self.set_properties[prop] = Property(prop) return trigger_properties
def getProperties(self, req): properties = Properties() for i in itertools.count(0): pname = req.args.get("property%dname" % i, [""])[0] pvalue = req.args.get("property%dvalue" % i, [""])[0] if not pname or not pvalue: break properties.setProperty(pname, pvalue, "Custom Force Build Form") return [properties, None]
def renderSecrets(self, *args): # Properties import to resolve cyclic import issue from buildbot.process.properties import Properties p = Properties() p.master = self.master if len(args) == 1: return p.render(args[0]) return defer.gatherResults([p.render(s) for s in args], consumeErrors=True)
def requestBuild(self, builder, locale): # returns a Deferred that fires with an IBuildStatus object when the # build is finished props = Properties() props.setProperty('locale', locale, 'scheduler') props.setProperty('tree', 'app', 'scheduler') req = BuildRequest("forced build", SourceStamp(), builder, properties=props) self.control.getBuilder(builder).requestBuild(req) return req.waitUntilFinished()
def MockBuild(my_builder, buildsetup, mastername, slavename, basepath=None, build_properties=None, slavedir=None): """Given a builder object and configuration, mock a Buildbot setup around it. This sets up a mock BuildMaster, BuildSlave, Build, BuildStatus, and all other superstructure required for BuildSteps inside the provided builder to render properly. These BuildSteps are returned to the user in an array. It additionally returns the build object (in order to get its properties if desired). buildsetup is passed straight into the FakeSource's init method and contains sourcestamp information (revision, branch, etc). basepath is the directory of the build (what goes under build/slave/, for example 'Chromium_Linux_Builder'. It is nominally inferred from the builder name, but it can be overridden. This is useful when pointing the buildrunner at a different builder than what it's running under. build_properties will update and override build_properties after all builder-derived defaults have been set. """ my_factory = my_builder['factory'] steplist = ListSteps(my_factory) build = base.Build([FakeRequest(buildsetup)]) safename = buildbot.util.safeTranslate(my_builder['name']) my_builder.setdefault('builddir', safename) my_builder.setdefault('slavebuilddir', my_builder['builddir']) workdir_root = None if not slavedir: workdir_root = os.path.join(SCRIPT_DIR, '..', '..', 'slave', my_builder['slavebuilddir']) if not basepath: basepath = safename if not slavedir: slavedir = os.path.join(SCRIPT_DIR, '..', '..', 'slave') basedir = os.path.join(slavedir, basepath) build.basedir = basedir if not workdir_root: workdir_root = basedir builderstatus = builder.BuilderStatus('test') builderstatus.basedir = basedir buildnumber = build_properties.get('buildnumber', 1) builderstatus.nextBuildNumber = buildnumber + 1 mybuilder = real_builder.Builder(my_builder, builderstatus) build.setBuilder(mybuilder) build_status = build_module.BuildStatus(builderstatus, buildnumber) build_status.setProperty('blamelist', [], 'Build') build_status.setProperty('mastername', mastername, 'Build') build_status.setProperty('slavename', slavename, 'Build') build_status.setProperty('gtest_filter', [], 'Build') build_status.setProperty('extra_args', [], 'Build') build_status.setProperty('build_id', buildnumber, 'Build') # if build_properties are passed in, overwrite the defaults above: buildprops = Properties() if build_properties: buildprops.update(build_properties, 'Botmaster') mybuilder.setBotmaster(FakeBotmaster(mastername, buildprops)) buildslave = FakeSlave(safename, my_builder.get('slavebuilddir'), slavename) build.build_status = build_status build.setupSlaveBuilder(buildslave) build.setupProperties() process_steps(steplist, build, buildslave, build_status, workdir_root) return steplist, build
class TestWithProperties(unittest.TestCase): def setUp(self): self.props = Properties() def testBasic(self): # test basic substitution with WithProperties self.props.setProperty("revision", "47", "test") command = WithProperties("build-%s.tar.gz", "revision") self.failUnlessEqual(self.props.render(command), "build-47.tar.gz") def testDict(self): # test dict-style substitution with WithProperties self.props.setProperty("other", "foo", "test") command = WithProperties("build-%(other)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-foo.tar.gz") def testDictColonMinus(self): # test dict-style substitution with WithProperties self.props.setProperty("prop1", "foo", "test") command = WithProperties( "build-%(prop1:-empty)s-%(prop2:-empty)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-foo-empty.tar.gz") def testDictColonPlus(self): # test dict-style substitution with WithProperties self.props.setProperty("prop1", "foo", "test") command = WithProperties( "build-%(prop1:+exists)s-%(prop2:+exists)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-exists-.tar.gz") def testEmpty(self): # None should render as '' self.props.setProperty("empty", None, "test") command = WithProperties("build-%(empty)s.tar.gz") self.failUnlessEqual(self.props.render(command), "build-.tar.gz") def testRecursiveList(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = [ WithProperties("%(x)s %(y)s"), "and", WithProperties("%(y)s %(x)s") ] self.failUnlessEqual(self.props.render(command), ["10 20", "and", "20 10"]) def testRecursiveTuple(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = (WithProperties("%(x)s %(y)s"), "and", WithProperties("%(y)s %(x)s")) self.failUnlessEqual(self.props.render(command), ("10 20", "and", "20 10")) def testRecursiveDict(self): self.props.setProperty("x", 10, "test") self.props.setProperty("y", 20, "test") command = { WithProperties("%(x)s %(y)s"): WithProperties("%(y)s %(x)s") } self.failUnlessEqual(self.props.render(command), {"10 20": "20 10"}) def testLambdaSubst(self): command = WithProperties('%(foo)s', foo=lambda _: 'bar') self.failUnlessEqual(self.props.render(command), 'bar') def testLambdaOverride(self): self.props.setProperty('x', 10, 'test') command = WithProperties('%(x)s', x=lambda _: 20) self.failUnlessEqual(self.props.render(command), '20') def testLambdaCallable(self): self.assertRaises(ValueError, lambda: WithProperties('%(foo)s', foo='bar')) def testLambdaUseExisting(self): self.props.setProperty('x', 10, 'test') self.props.setProperty('y', 20, 'test') command = WithProperties('%(z)s', z=lambda pmap: pmap['x'] + pmap['y']) self.failUnlessEqual(self.props.render(command), '30')
def setUp(self): self.props = Properties()
def createEmail(self, msgdict, builderName, title, results, builds=None, patches=None, logs=None): text = msgdict['body'].encode(ENCODING) type = msgdict['type'] if 'subject' in msgdict: subject = msgdict['subject'].encode(ENCODING) else: subject = self.subject % { 'result': Results[results], 'projectName': title, 'title': title, 'builder': builderName } assert '\n' not in subject, \ "Subject cannot contain newlines" assert type in ('plain', 'html'), \ "'%s' message type must be 'plain' or 'html'." % type if patches or logs: m = MIMEMultipart() txt = MIMEText(text, type, ENCODING) m.attach(txt) else: m = Message() m.set_payload(text, ENCODING) m.set_type("text/%s" % type) m['Date'] = formatdate(localtime=True) m['Subject'] = subject m['From'] = self.fromaddr # m['To'] is added later if patches: for (i, patch) in enumerate(patches): a = self.patch_to_attachment(patch, i) m.attach(a) if logs: for log in logs: name = "%s.%s" % (log['stepname'], log['name']) if (self._shouldAttachLog(log['name']) or self._shouldAttachLog(name)): # Use distinct filenames for the e-mail summary if self.buildSetSummary: filename = "%s.%s" % (log['buildername'], name) else: filename = name text = log['content']['content'] a = MIMEText(text.encode(ENCODING), _charset=ENCODING) a.add_header('Content-Disposition', "attachment", filename=filename) m.attach(a) # @todo: is there a better way to do this? # Add any extra headers that were requested, doing WithProperties # interpolation if only one build was given if self.extraHeaders: extraHeaders = self.extraHeaders if len(builds) == 1: props = Properties.fromDict(builds[0]['properties']) extraHeaders = yield props.render(extraHeaders) for k, v in iteritems(extraHeaders): if k in m: twlog.msg("Warning: Got header " + k + " in self.extraHeaders " "but it already exists in the Message - " "not adding it.") m[k] = v defer.returnValue(m)
def __init__(self, name, password, max_builds=None, notify_on_missing=[], missing_timeout=3600, properties={}, locks=None, keepalive_interval=3600): """ @param name: botname this machine will supply when it connects @param password: password this machine will supply when it connects @param max_builds: maximum number of simultaneous builds that will be run concurrently on this buildslave (the default is None for no limit) @param properties: properties that will be applied to builds run on this slave @type properties: dictionary @param locks: A list of locks that must be acquired before this slave can be used @type locks: dictionary """ service.MultiService.__init__(self) self.slavename = name self.password = password # PB registration self.registration = None self.registered_port = None # these are set when the service is started, and unset when it is # stopped self.botmaster = None self.master = None self.slave_status = SlaveStatus(name) self.slave = None # a RemoteReference to the Bot, when connected self.slave_commands = None self.slavebuilders = {} self.max_builds = max_builds self.access = [] if locks: self.access = locks self.lock_subscriptions = [] self.properties = Properties() self.properties.update(properties, "BuildSlave") self.properties.setProperty("slavename", name, "BuildSlave") self.lastMessageReceived = 0 if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing for i in notify_on_missing: if not isinstance(i, str): config.error('notify_on_missing arg %r is not a string' % (i, )) self.missing_timeout = missing_timeout self.missing_timer = None self.keepalive_interval = keepalive_interval self.detached_subs = None self._old_builder_list = None
class BuildRequest: """I represent a request to a specific Builder to run a single build. I have a SourceStamp which specifies what sources I will build. This may specify a specific revision of the source tree (so source.branch, source.revision, and source.patch are used). The .patch attribute is either None or a tuple of (patchlevel, diff), consisting of a number to use in 'patch -pN', and a unified-format context diff. Alternatively, the SourceStamp may specify a set of Changes to be built, contained in source.changes. In this case, I may be mergeable with other BuildRequests on the same branch. I may be part of a BuildSet, in which case I will report status results to it. I am paired with a BuildRequestStatus object, to which I feed status information. @type source: a L{buildbot.sourcestamp.SourceStamp} instance. @ivar source: the source code that this BuildRequest use @type reason: string @ivar reason: the reason this Build is being requested. Schedulers provide this, but for forced builds the user requesting the build will provide a string. @type properties: Properties object @ivar properties: properties that should be applied to this build 'owner' property is used by Build objects to collect the list returned by getInterestedUsers @ivar status: the IBuildStatus object which tracks our status @ivar submittedAt: a timestamp (seconds since epoch) when this request was submitted to the Builder. This is used by the CVS step to compute a checkout timestamp, as well as the master to prioritize build requests from oldest to newest. """ source = None builder = None startCount = 0 # how many times we have tried to start this build submittedAt = None implements(interfaces.IBuildRequestControl) def __init__(self, reason, source, builderName, properties=None): assert interfaces.ISourceStamp(source, None) self.reason = reason self.source = source self.properties = Properties() if properties: self.properties.updateFromProperties(properties) self.start_watchers = [] self.finish_watchers = [] self.status = BuildRequestStatus(source, builderName) def canBeMergedWith(self, other): return self.source.canBeMergedWith(other.source) def mergeWith(self, others): return self.source.mergeWith([o.source for o in others]) def mergeReasons(self, others): """Return a reason for the merged build request.""" reasons = [] for req in [self] + others: if req.reason and req.reason not in reasons: reasons.append(req.reason) return ", ".join(reasons) def waitUntilFinished(self): """Get a Deferred that will fire (with a L{buildbot.interfaces.IBuildStatus} instance when the build finishes.""" d = defer.Deferred() self.finish_watchers.append(d) return d # these are called by the Builder def requestSubmitted(self, builder): # the request has been placed on the queue self.builder = builder def buildStarted(self, build, buildstatus): """This is called by the Builder when a Build has been started in the hopes of satifying this BuildRequest. It may be called multiple times, since interrupted builds and lost buildslaves may force multiple Builds to be run until the fate of the BuildRequest is known for certain.""" for o in self.start_watchers[:]: # these observers get the IBuildControl o(build) # while these get the IBuildStatus self.status.buildStarted(buildstatus) def finished(self, buildstatus): """This is called by the Builder when the BuildRequest has been retired. This happens when its Build has either succeeded (yay!) or failed (boo!). TODO: If it is halted due to an exception (oops!), or some other retryable error, C{finished} will not be called yet.""" for w in self.finish_watchers: w.callback(buildstatus) self.finish_watchers = [] # IBuildRequestControl def subscribe(self, observer): self.start_watchers.append(observer) def unsubscribe(self, observer): self.start_watchers.remove(observer) def cancel(self): """Cancel this request. This can only be successful if the Build has not yet been started. @return: a boolean indicating if the cancel was successful.""" if self.builder: return self.builder.cancelBuildRequest(self) return False def setSubmitTime(self, t): self.submittedAt = t self.status.setSubmitTime(t) def getSubmitTime(self): return self.submittedAt
def command_FORCE(self, args): # FIXME: NEED TO THINK ABOUT! errReply = "try 'force build [--branch=BRANCH] [--revision=REVISION] [--props=PROP1=VAL1,PROP2=VAL2...] <WHICH> <REASON>'" args = self.splitArgs(args) if not args: raise UsageError(errReply) what = args.pop(0) if what != "build": raise UsageError(errReply) opts = ForceOptions() opts.parseOptions(args) builderName = opts['builder'] branch = opts['branch'] revision = opts['revision'] reason = opts['reason'] props = opts['props'] if builderName is None: raise UsageError("you must provide a Builder, " + errReply) # keep weird stuff out of the branch, revision, and properties args. branch_validate = self.master.config.validation['branch'] revision_validate = self.master.config.validation['revision'] pname_validate = self.master.config.validation['property_name'] pval_validate = self.master.config.validation['property_value'] if branch and not branch_validate.match(branch): log.msg("bad branch '%s'" % branch) self.send("sorry, bad branch '%s'" % branch) return if revision and not revision_validate.match(revision): log.msg("bad revision '%s'" % revision) self.send("sorry, bad revision '%s'" % revision) return properties = Properties() if props: # split props into name:value dict pdict = {} propertylist = props.split(",") for i in range(0, len(propertylist)): splitproperty = propertylist[i].split("=", 1) pdict[splitproperty[0]] = splitproperty[1] # set properties for prop in pdict: pname = prop pvalue = pdict[prop] if not pname_validate.match(pname) \ or not pval_validate.match(pvalue): log.msg("bad property name='%s', value='%s'" % (pname, pvalue)) self.send("sorry, bad property name='%s', value='%s'" % (pname, pvalue)) return properties.setProperty(pname, pvalue, "Force Build chat") reason = u"forced: by %s: %s" % (self.describeUser(), reason) d = self.master.data.updates.addBuildset( builderNames=[builderName], scheduler=u"status.words", # For now, we just use this as the id. sourcestamps=[{ 'branch': branch, 'revision': revision }], reason=reason, properties=properties.asDict(), waited_for=False) @d.addCallback def subscribe(xxx_todo_changeme): (bsid, brids) = xxx_todo_changeme assert 0, "rewrite to not use the status hierarchy" # TODO # ireq = BuildRequest(self, self.useRevisions) # buildreq.subscribe(ireq.started) d.addErrback(log.err, "while forcing a build")
class BaseScheduler(ClusteredBuildbotService, StateMixin): DEFAULT_CODEBASES = {'': {}} compare_attrs = ClusteredBuildbotService.compare_attrs + \ ('builderNames', 'properties', 'codebases') def __init__(self, name, builderNames, properties=None, codebases=DEFAULT_CODEBASES): super(BaseScheduler, self).__init__(name=name) ok = True if interfaces.IRenderable.providedBy(builderNames): pass elif isinstance(builderNames, (list, tuple)): for b in builderNames: if not isinstance(b, string_types) and \ not interfaces.IRenderable.providedBy(b): ok = False else: ok = False if not ok: config.error( "The builderNames argument to a scheduler must be a list " "of Builder names or an IRenderable object that will render" "to a list of builder names.") self.builderNames = builderNames if properties is None: properties = {} self.properties = Properties() self.properties.update(properties, "Scheduler") self.properties.setProperty("scheduler", name, "Scheduler") self.objectid = None # Set the codebases that are necessary to process the changes # These codebases will always result in a sourcestamp with or without # changes known_keys = set(['branch', 'repository', 'revision']) if codebases is None: config.error("Codebases cannot be None") elif isinstance(codebases, list): codebases = dict((codebase, {}) for codebase in codebases) elif not isinstance(codebases, dict): config.error( "Codebases must be a dict of dicts, or list of strings") else: for codebase, attrs in codebases.items(): if not isinstance(attrs, dict): config.error("Codebases must be a dict of dicts") else: unk = set(attrs) - known_keys if unk: config.error( "Unknown codebase keys %s for codebase %s" % (', '.join(unk), codebase)) self.codebases = codebases # internal variables self._change_consumer = None self._enable_consumer = None self._change_consumption_lock = defer.DeferredLock() self.enabled = True def reconfigService(self, *args, **kwargs): raise NotImplementedError() # activity handling @defer.inlineCallbacks def activate(self): if not self.enabled: return None # even if we aren't called via _activityPoll(), at this point we # need to ensure the service id is set correctly if self.serviceid is None: self.serviceid = yield self._getServiceId() assert self.serviceid is not None schedulerData = yield self._getScheduler(self.serviceid) if schedulerData: self.enabled = schedulerData['enabled'] if not self._enable_consumer: yield self.startConsumingEnableEvents() def _enabledCallback(self, key, msg): if msg['enabled']: self.enabled = True d = self.activate() else: d = self.deactivate() def fn(x): self.enabled = False d.addCallback(fn) return d @defer.inlineCallbacks def deactivate(self): if not self.enabled: return yield defer.maybeDeferred(self._stopConsumingChanges) # service handling def _getServiceId(self): return self.master.data.updates.findSchedulerId(self.name) def _getScheduler(self, sid): return self.master.db.schedulers.getScheduler(sid) def _claimService(self): return self.master.data.updates.trySetSchedulerMaster( self.serviceid, self.master.masterid) def _unclaimService(self): return self.master.data.updates.trySetSchedulerMaster( self.serviceid, None) # status queries # deprecated: these aren't compatible with distributed schedulers def listBuilderNames(self): return self.builderNames # change handling @defer.inlineCallbacks def startConsumingChanges(self, fileIsImportant=None, change_filter=None, onlyImportant=False): assert fileIsImportant is None or callable(fileIsImportant) # register for changes with the data API assert not self._change_consumer self._change_consumer = yield self.master.mq.startConsuming( lambda k, m: self._changeCallback(k, m, fileIsImportant, change_filter, onlyImportant), ('changes', None, 'new')) @defer.inlineCallbacks def startConsumingEnableEvents(self): assert not self._enable_consumer self._enable_consumer = yield self.master.mq.startConsuming( self._enabledCallback, ('schedulers', str(self.serviceid), 'updated')) @defer.inlineCallbacks def _changeCallback(self, key, msg, fileIsImportant, change_filter, onlyImportant): # ignore changes delivered while we're not running if not self._change_consumer: return # get a change object, since the API requires it chdict = yield self.master.db.changes.getChange(msg['changeid']) change = yield changes.Change.fromChdict(self.master, chdict) # filter it if change_filter and not change_filter.filter_change(change): return if change.codebase not in self.codebases: log.msg(format='change contains codebase %(codebase)s that is ' 'not processed by scheduler %(name)s', codebase=change.codebase, name=self.name) return if fileIsImportant: try: important = fileIsImportant(change) if not important and onlyImportant: return except Exception: log.err(failure.Failure(), 'in fileIsImportant check for %s' % change) return else: important = True # use change_consumption_lock to ensure the service does not stop # while this change is being processed d = self._change_consumption_lock.run(self.gotChange, change, important) d.addErrback(log.err, 'while processing change') def _stopConsumingChanges(self): # (note: called automatically in deactivate) # acquire the lock change consumption lock to ensure that any change # consumption is complete before we are done stopping consumption def stop(): if self._change_consumer: self._change_consumer.stopConsuming() self._change_consumer = None return self._change_consumption_lock.run(stop) def gotChange(self, change, important): raise NotImplementedError # starting builds @defer.inlineCallbacks def addBuildsetForSourceStampsWithDefaults(self, reason, sourcestamps=None, waited_for=False, properties=None, builderNames=None, **kw): if sourcestamps is None: sourcestamps = [] # convert sourcestamps to a dictionary keyed by codebase stampsByCodebase = {} for ss in sourcestamps: cb = ss['codebase'] if cb in stampsByCodebase: raise RuntimeError("multiple sourcestamps with same codebase") stampsByCodebase[cb] = ss # Merge codebases with the passed list of sourcestamps # This results in a new sourcestamp for each codebase stampsWithDefaults = [] for codebase in self.codebases: cb = yield self.getCodebaseDict(codebase) ss = { 'codebase': codebase, 'repository': cb.get('repository', ''), 'branch': cb.get('branch', None), 'revision': cb.get('revision', None), 'project': '', } # apply info from passed sourcestamps onto the configured default # sourcestamp attributes for this codebase. ss.update(stampsByCodebase.get(codebase, {})) stampsWithDefaults.append(ss) # fill in any supplied sourcestamps that aren't for a codebase in the # scheduler's codebase dictionary for codebase in set(stampsByCodebase) - set(self.codebases): cb = stampsByCodebase[codebase] ss = { 'codebase': codebase, 'repository': cb.get('repository', ''), 'branch': cb.get('branch', None), 'revision': cb.get('revision', None), 'project': '', } stampsWithDefaults.append(ss) rv = yield self.addBuildsetForSourceStamps( sourcestamps=stampsWithDefaults, reason=reason, waited_for=waited_for, properties=properties, builderNames=builderNames, **kw) return rv def getCodebaseDict(self, codebase): # Hook for subclasses to change codebase parameters when a codebase does # not have a change associated with it. try: return defer.succeed(self.codebases[codebase]) except KeyError: return defer.fail() @defer.inlineCallbacks def addBuildsetForChanges(self, waited_for=False, reason='', external_idstring=None, changeids=None, builderNames=None, properties=None, **kw): if changeids is None: changeids = [] changesByCodebase = {} def get_last_change_for_codebase(codebase): return max(changesByCodebase[codebase], key=lambda change: change["changeid"]) # Changes are retrieved from database and grouped by their codebase for changeid in changeids: chdict = yield self.master.db.changes.getChange(changeid) changesByCodebase.setdefault(chdict["codebase"], []).append(chdict) sourcestamps = [] for codebase in sorted(self.codebases): if codebase not in changesByCodebase: # codebase has no changes # create a sourcestamp that has no changes cb = yield self.getCodebaseDict(codebase) ss = { 'codebase': codebase, 'repository': cb.get('repository', ''), 'branch': cb.get('branch', None), 'revision': cb.get('revision', None), 'project': '', } else: lastChange = get_last_change_for_codebase(codebase) ss = lastChange['sourcestampid'] sourcestamps.append(ss) # add one buildset, using the calculated sourcestamps bsid, brids = yield self.addBuildsetForSourceStamps( waited_for, sourcestamps=sourcestamps, reason=reason, external_idstring=external_idstring, builderNames=builderNames, properties=properties, **kw) return (bsid, brids) @defer.inlineCallbacks def addBuildsetForSourceStamps(self, waited_for=False, sourcestamps=None, reason='', external_idstring=None, properties=None, builderNames=None, **kw): if sourcestamps is None: sourcestamps = [] # combine properties if properties: properties.updateFromProperties(self.properties) else: properties = self.properties # make a fresh copy that we actually can modify safely properties = Properties.fromDict(properties.asDict()) # make extra info available from properties.render() properties.master = self.master properties.sourcestamps = [] properties.changes = [] for ss in sourcestamps: if isinstance(ss, integer_types): # fetch actual sourcestamp and changes from data API properties.sourcestamps.append((yield self.master.data.get( ('sourcestamps', ss)))) properties.changes.extend((yield self.master.data.get( ('sourcestamps', ss, 'changes')))) else: # sourcestamp with no change, see addBuildsetForChanges properties.sourcestamps.append(ss) for c in properties.changes: properties.updateFromProperties( Properties.fromDict(c['properties'])) # apply the default builderNames if not builderNames: builderNames = self.builderNames # dynamically get the builder list to schedule builderNames = yield properties.render(builderNames) # Get the builder ids # Note that there is a data.updates.findBuilderId(name) # but that would merely only optimize the single builder case, while # probably the multiple builder case will be severely impacted by the # several db requests needed. builderids = list() for bldr in (yield self.master.data.get(('builders', ))): if bldr['name'] in builderNames: builderids.append(bldr['builderid']) # translate properties object into a dict as required by the # addBuildset method properties_dict = properties.asDict() bsid, brids = yield self.master.data.updates.addBuildset( scheduler=self.name, sourcestamps=sourcestamps, reason=reason, waited_for=waited_for, properties=properties_dict, builderids=builderids, external_idstring=external_idstring, **kw) return (bsid, brids)
class Change: """I represent a single change to the source tree. This may involve several files, but they are all changed by the same person, and there is a change comment for the group as a whole.""" implements(interfaces.IStatusEvent) number = None branch = None category = None revision = None # used to create a source-stamp links = [] # links are gone, but upgrade code expects this attribute @classmethod def fromChdict(cls, master, chdict): """ Class method to create a L{Change} from a dictionary as returned by L{ChangesConnectorComponent.getChange}. @param master: build master instance @param ssdict: change dictionary @returns: L{Change} via Deferred """ cache = master.caches.get_cache("Changes", cls._make_ch) return cache.get(chdict['changeid'], chdict=chdict, master=master) @classmethod def _make_ch(cls, changeid, master, chdict): change = cls(None, None, None, _fromChdict=True) change.who = chdict['author'] change.comments = chdict['comments'] change.isdir = chdict['is_dir'] change.revision = chdict['revision'] change.branch = chdict['branch'] change.category = chdict['category'] change.revlink = chdict['revlink'] change.repository = chdict['repository'] change.codebase = chdict['codebase'] change.project = chdict['project'] change.number = chdict['changeid'] when = chdict['when_timestamp'] if when: when = datetime2epoch(when) change.when = when change.files = chdict['files'][:] change.files.sort() change.properties = Properties() for n, (v, s) in chdict['properties'].iteritems(): change.properties.setProperty(n, v, s) return defer.succeed(change) def __init__(self, who, files, comments, isdir=0, revision=None, when=None, branch=None, category=None, revlink='', properties={}, repository='', codebase='', project='', _fromChdict=False): # skip all this madness if we're being built from the database if _fromChdict: return self.who = who self.comments = comments self.isdir = isdir def none_or_unicode(x): if x is None: return x return unicode(x) self.revision = none_or_unicode(revision) now = util.now() if when is None: self.when = now elif when > now: # this happens when the committing system has an incorrect clock, for example. # handle it gracefully log.msg( "received a Change with when > now; assuming the change happened now" ) self.when = now else: self.when = when self.branch = none_or_unicode(branch) self.category = none_or_unicode(category) self.revlink = revlink self.properties = Properties() self.properties.update(properties, "Change") self.repository = repository self.codebase = codebase self.project = project # keep a sorted list of the files, for easier display self.files = (files or [])[:] self.files.sort() def __setstate__(self, dict): self.__dict__ = dict # Older Changes won't have a 'properties' attribute in them if not hasattr(self, 'properties'): self.properties = Properties() if not hasattr(self, 'revlink'): self.revlink = "" def __str__(self): return (u"Change(revision=%r, who=%r, branch=%r, comments=%r, " + u"when=%r, category=%r, project=%r, repository=%r, " + u"codebase=%r)") % (self.revision, self.who, self.branch, self.comments, self.when, self.category, self.project, self.repository, self.codebase) def __cmp__(self, other): return self.number - other.number def asText(self): data = "" data += self.getFileContents() if self.repository: data += "On: %s\n" % self.repository if self.project: data += "For: %s\n" % self.project data += "At: %s\n" % self.getTime() data += "Changed By: %s\n" % self.who data += "Comments: %s" % self.comments data += "Properties: \n%s\n\n" % self.getProperties() return data def asDict(self): '''returns a dictonary with suitable info for html/mail rendering''' result = {} files = [dict(name=f) for f in self.files] files.sort(cmp=lambda a, b: a['name'] < b['name']) # Constant result['number'] = self.number result['branch'] = self.branch result['category'] = self.category result['who'] = self.getShortAuthor() result['comments'] = self.comments result['revision'] = self.revision result['rev'] = self.revision result['when'] = self.when result['at'] = self.getTime() result['files'] = files result['revlink'] = getattr(self, 'revlink', None) result['properties'] = self.properties.asList() result['repository'] = getattr(self, 'repository', None) result['codebase'] = getattr(self, 'codebase', '') result['project'] = getattr(self, 'project', None) return result def getShortAuthor(self): return self.who def getTime(self): if not self.when: return "?" return time.strftime("%a %d %b %Y %H:%M:%S", time.localtime(self.when)) def getTimes(self): return (self.when, None) def getText(self): return [html.escape(self.who)] def getLogs(self): return {} def getFileContents(self): data = "" if len(self.files) == 1: if self.isdir: data += "Directory: %s\n" % self.files[0] else: data += "File: %s\n" % self.files[0] else: data += "Files:\n" for f in self.files: data += " %s\n" % f return data def getProperties(self): data = "" for prop in self.properties.asList(): data += " %s: %s" % (prop[0], prop[1]) return data
def start(self): properties = self.build.getProperties() # make a new properties object from a dict rendered by the old # properties object props_to_set = Properties() props_to_set.update(properties.render(self.set_properties), "Trigger") for p in self.copy_properties: if p not in properties: raise RuntimeError( "copy_property '%s' is not set in the triggering build" % p) props_to_set.setProperty( p, properties[p], "%s (in triggering build)" % properties.getPropertySource(p)) self.running = True ss = self.build.getSourceStamp() if self.updateSourceStamp: got = properties.getProperty('got_revision') if got: ss = ss.getAbsoluteSourceStamp(got) # (is there an easier way to find the BuildMaster?) all_schedulers = self.build.builder.botmaster.parent.allSchedulers() all_schedulers = dict([(sch.name, sch) for sch in all_schedulers]) unknown_schedulers = [] triggered_schedulers = [] # TODO: don't fire any schedulers if we discover an unknown one for scheduler in self.schedulerNames: scheduler = properties.render(scheduler) if all_schedulers.has_key(scheduler): sch = all_schedulers[scheduler] if isinstance(sch, Triggerable): triggered_schedulers.append(scheduler) else: unknown_schedulers.append(scheduler) else: unknown_schedulers.append(scheduler) if unknown_schedulers: self.step_status.setText(['no scheduler:'] + unknown_schedulers) return self.end(FAILURE) master = self.build.builder.botmaster.parent # seriously?! if self.alwaysUseLatest: d = defer.succeed(None) else: d = ss.getSourceStampId(master) def start_builds(ssid): dl = [] for scheduler in triggered_schedulers: sch = all_schedulers[scheduler] dl.append(sch.trigger(ssid, set_props=props_to_set)) self.step_status.setText(['triggered'] + triggered_schedulers) d = defer.DeferredList(dl, consumeErrors=1) if self.waitForFinish: return d else: # do something to handle errors d.addErrback( log.err, '(ignored) while invoking Triggerable schedulers:') return None d.addCallback(start_builds) def cb(rclist): rc = SUCCESS # (this rc is not the same variable as that above) for was_cb, results in rclist: # TODO: make this algo more configurable if not was_cb: rc = EXCEPTION log.err(results) break if results == FAILURE: rc = FAILURE return self.end(rc) def eb(why): return self.end(FAILURE) if self.waitForFinish: d.addCallbacks(cb, eb) d.addErrback(log.err, '(ignored) while triggering builds:')
def __init__(self): self.sources = [] self.reason = "Because" self.properties = Properties() self.id = 9385
class BuildStatus(styles.Versioned): implements(interfaces.IBuildStatus, interfaces.IStatusEvent) persistenceVersion = 3 persistenceForgets = ( 'wasUpgraded', ) source = None reason = None changes = [] blamelist = [] progress = None started = None finished = None currentStep = None text = [] results = None slavename = "???" # these lists/dicts are defined here so that unserialized instances have # (empty) values. They are set in __init__ to new objects to make sure # each instance gets its own copy. watchers = [] updates = {} finishedWatchers = [] testResults = {} def __init__(self, parent, number): """ @type parent: L{BuilderStatus} @type number: int """ assert interfaces.IBuilderStatus(parent) self.builder = parent self.number = number self.watchers = [] self.updates = {} self.finishedWatchers = [] self.steps = [] self.testResults = {} self.properties = Properties() def __repr__(self): return "<%s #%s>" % (self.__class__.__name__, self.number) # IBuildStatus def getBuilder(self): """ @rtype: L{BuilderStatus} """ return self.builder def getProperty(self, propname): return self.properties[propname] def getProperties(self): return self.properties def getNumber(self): return self.number def getPreviousBuild(self): if self.number == 0: return None return self.builder.getBuild(self.number-1) def getSourceStamp(self, absolute=False): if not absolute or not self.properties.has_key('got_revision'): return self.source return self.source.getAbsoluteSourceStamp(self.properties['got_revision']) def getReason(self): return self.reason def getChanges(self): return self.changes def getResponsibleUsers(self): return self.blamelist def getInterestedUsers(self): # TODO: the Builder should add others: sheriffs, domain-owners return self.blamelist + self.properties.getProperty('owners', []) def getSteps(self): """Return a list of IBuildStepStatus objects. For invariant builds (those which always use the same set of Steps), this should be the complete list, however some of the steps may not have started yet (step.getTimes()[0] will be None). For variant builds, this may not be complete (asking again later may give you more of them).""" return self.steps def getTimes(self): return (self.started, self.finished) _sentinel = [] # used as a sentinel to indicate unspecified initial_value def getSummaryStatistic(self, name, summary_fn, initial_value=_sentinel): """Summarize the named statistic over all steps in which it exists, using combination_fn and initial_value to combine multiple results into a single result. This translates to a call to Python's X{reduce}:: return reduce(summary_fn, step_stats_list, initial_value) """ step_stats_list = [ st.getStatistic(name) for st in self.steps if st.hasStatistic(name) ] if initial_value is self._sentinel: return reduce(summary_fn, step_stats_list) else: return reduce(summary_fn, step_stats_list, initial_value) def isFinished(self): return (self.finished is not None) def waitUntilFinished(self): if self.finished: d = defer.succeed(self) else: d = defer.Deferred() self.finishedWatchers.append(d) return d # while the build is running, the following methods make sense. # Afterwards they return None def getETA(self): if self.finished is not None: return None if not self.progress: return None eta = self.progress.eta() if eta is None: return None return eta - util.now() def getCurrentStep(self): return self.currentStep # Once you know the build has finished, the following methods are legal. # Before ths build has finished, they all return None. def getText(self): text = [] text.extend(self.text) for s in self.steps: text.extend(s.text2) return text def getResults(self): return self.results def getSlavename(self): return self.slavename def getTestResults(self): return self.testResults def getTestResultsOrd(self): trs = self.testResults.keys() trs.sort() ret = [ self.testResults[t] for t in trs] return ret def getLogs(self): # TODO: steps should contribute significant logs instead of this # hack, which returns every log from every step. The logs should get # names like "compile" and "test" instead of "compile.output" logs = [] for s in self.steps: for loog in s.getLogs(): logs.append(loog) return logs # subscription interface def subscribe(self, receiver, updateInterval=None): # will receive stepStarted and stepFinished messages # and maybe buildETAUpdate self.watchers.append(receiver) if updateInterval is not None: self.sendETAUpdate(receiver, updateInterval) def sendETAUpdate(self, receiver, updateInterval): self.updates[receiver] = None ETA = self.getETA() if ETA is not None: receiver.buildETAUpdate(self, self.getETA()) # they might have unsubscribed during buildETAUpdate if receiver in self.watchers: self.updates[receiver] = reactor.callLater(updateInterval, self.sendETAUpdate, receiver, updateInterval) def unsubscribe(self, receiver): if receiver in self.watchers: self.watchers.remove(receiver) if receiver in self.updates: if self.updates[receiver] is not None: self.updates[receiver].cancel() del self.updates[receiver] # methods for the base.Build to invoke def addStepWithName(self, name): """The Build is setting up, and has added a new BuildStep to its list. Create a BuildStepStatus object to which it can send status updates.""" s = BuildStepStatus(self, len(self.steps)) s.setName(name) self.steps.append(s) return s def setProperty(self, propname, value, source, runtime=True): self.properties.setProperty(propname, value, source, runtime) def addTestResult(self, result): self.testResults[result.getName()] = result def setSourceStamp(self, sourceStamp): self.source = sourceStamp self.changes = self.source.changes def setReason(self, reason): self.reason = reason def setBlamelist(self, blamelist): self.blamelist = blamelist def setProgress(self, progress): self.progress = progress def buildStarted(self, build): """The Build has been set up and is about to be started. It can now be safely queried, so it is time to announce the new build.""" self.started = util.now() # now that we're ready to report status, let the BuilderStatus tell # the world about us self.builder.buildStarted(self) def setSlavename(self, slavename): self.slavename = slavename def setText(self, text): assert isinstance(text, (list, tuple)) self.text = text def setResults(self, results): self.results = results def buildFinished(self): self.currentStep = None self.finished = util.now() for r in self.updates.keys(): if self.updates[r] is not None: self.updates[r].cancel() del self.updates[r] watchers = self.finishedWatchers self.finishedWatchers = [] for w in watchers: w.callback(self) # methods called by our BuildStepStatus children def stepStarted(self, step): self.currentStep = step for w in self.watchers: receiver = w.stepStarted(self, step) if receiver: if type(receiver) == type(()): step.subscribe(receiver[0], receiver[1]) else: step.subscribe(receiver) d = step.waitUntilFinished() d.addCallback(lambda step: step.unsubscribe(receiver)) step.waitUntilFinished().addCallback(self._stepFinished) def _stepFinished(self, step): results = step.getResults() for w in self.watchers: w.stepFinished(self, step, results) # methods called by our BuilderStatus parent def pruneSteps(self): # this build is very old: remove the build steps too self.steps = [] # persistence stuff def generateLogfileName(self, stepname, logname): """Return a filename (relative to the Builder's base directory) where the logfile's contents can be stored uniquely. The base filename is made by combining our build number, the Step's name, and the log's name, then removing unsuitable characters. The filename is then made unique by appending _0, _1, etc, until it does not collide with any other logfile. These files are kept in the Builder's basedir (rather than a per-Build subdirectory) because that makes cleanup easier: cron and find will help get rid of the old logs, but the empty directories are more of a hassle to remove.""" starting_filename = "%d-log-%s-%s" % (self.number, stepname, logname) starting_filename = re.sub(r'[^\w\.\-]', '_', starting_filename) # now make it unique unique_counter = 0 filename = starting_filename while filename in [l.filename for step in self.steps for l in step.getLogs() if l.filename]: filename = "%s_%d" % (starting_filename, unique_counter) unique_counter += 1 return filename def __getstate__(self): d = styles.Versioned.__getstate__(self) # for now, a serialized Build is always "finished". We will never # save unfinished builds. if not self.finished: d['finished'] = util.now() # TODO: push an "interrupted" step so it is clear that the build # was interrupted. The builder will have a 'shutdown' event, but # someone looking at just this build will be confused as to why # the last log is truncated. for k in 'builder', 'watchers', 'updates', 'finishedWatchers': if k in d: del d[k] return d def __setstate__(self, d): styles.Versioned.__setstate__(self, d) # self.builder must be filled in by our parent when loading for step in self.steps: step.build = self self.watchers = [] self.updates = {} self.finishedWatchers = [] def upgradeToVersion1(self): if hasattr(self, "sourceStamp"): # the old .sourceStamp attribute wasn't actually very useful maxChangeNumber, patch = self.sourceStamp changes = getattr(self, 'changes', []) source = sourcestamp.SourceStamp(branch=None, revision=None, patch=patch, changes=changes) self.source = source self.changes = source.changes del self.sourceStamp self.wasUpgraded = True def upgradeToVersion2(self): self.properties = {} self.wasUpgraded = True def upgradeToVersion3(self): # in version 3, self.properties became a Properties object propdict = self.properties self.properties = Properties() self.properties.update(propdict, "Upgrade from previous version") self.wasUpgraded = True def upgradeLogfiles(self): # upgrade any LogFiles that need it. This must occur after we've been # attached to our Builder, and after we know about all LogFiles of # all Steps (to get the filenames right). assert self.builder for s in self.steps: for l in s.getLogs(): if l.filename: pass # new-style, log contents are on disk else: logfilename = self.generateLogfileName(s.name, l.name) # let the logfile update its .filename pointer, # transferring its contents onto disk if necessary l.upgrade(logfilename) def checkLogfiles(self): # check that all logfiles exist, and remove references to any that # have been deleted (e.g., by purge()) for s in self.steps: s.checkLogfiles() def saveYourself(self): filename = os.path.join(self.builder.basedir, "%d" % self.number) if os.path.isdir(filename): # leftover from 0.5.0, which stored builds in directories shutil.rmtree(filename, ignore_errors=True) tmpfilename = filename + ".tmp" try: dump(self, open(tmpfilename, "wb"), -1) if runtime.platformType == 'win32': # windows cannot rename a file on top of an existing one, so # fall back to delete-first. There are ways this can fail and # lose the builder's history, so we avoid using it in the # general (non-windows) case if os.path.exists(filename): os.unlink(filename) os.rename(tmpfilename, filename) except: log.msg("unable to save build %s-#%d" % (self.builder.name, self.number)) log.err() def asDict(self): result = {} # Constant result['builderName'] = self.builder.name result['number'] = self.getNumber() result['sourceStamp'] = self.getSourceStamp().asDict() result['reason'] = self.getReason() result['blame'] = self.getResponsibleUsers() # Transient result['properties'] = self.getProperties().asList() result['times'] = self.getTimes() result['text'] = self.getText() result['results'] = self.getResults() result['slave'] = self.getSlavename() # TODO(maruel): Add. #result['test_results'] = self.getTestResults() result['logs'] = [[l.getName(), self.builder.status.getURLForThing(l)] for l in self.getLogs()] result['eta'] = self.getETA() result['steps'] = [bss.asDict() for bss in self.steps] if self.getCurrentStep(): result['currentStep'] = self.getCurrentStep().asDict() else: result['currentStep'] = None return result
class FakeChange: properties = Properties() def __init__(self, number=None): self.number = number self.who = "me"
def upgradeToVersion3(self): # in version 3, self.properties became a Properties object propdict = self.properties self.properties = Properties() self.properties.update(propdict, "Upgrade from previous version") self.wasUpgraded = True
class TestProperties(unittest.TestCase): def setUp(self): self.props = Properties() def testDictBehavior(self): self.props.setProperty("do-tests", 1, "scheduler") self.props.setProperty("do-install", 2, "scheduler") self.assert_(self.props.has_key('do-tests')) self.failUnlessEqual(self.props['do-tests'], 1) self.failUnlessEqual(self.props['do-install'], 2) self.assertRaises(KeyError, lambda: self.props['do-nothing']) self.failUnlessEqual(self.props.getProperty('do-install'), 2) def testUpdate(self): self.props.setProperty("x", 24, "old") newprops = {'a': 1, 'b': 2} self.props.update(newprops, "new") self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') self.failUnlessEqual(self.props.getProperty('a'), 1) self.failUnlessEqual(self.props.getPropertySource('a'), 'new') def testUpdateFromProperties(self): self.props.setProperty("a", 94, "old") self.props.setProperty("x", 24, "old") newprops = Properties() newprops.setProperty('a', 1, "new") newprops.setProperty('b', 2, "new") self.props.updateFromProperties(newprops) self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old') self.failUnlessEqual(self.props.getProperty('a'), 1) self.failUnlessEqual(self.props.getPropertySource('a'), 'new') def testUpdateFromPropertiesNoRuntime(self): self.props.setProperty("a", 94, "old") self.props.setProperty("b", 84, "old") self.props.setProperty("x", 24, "old") newprops = Properties() newprops.setProperty('a', 1, "new", runtime=True) newprops.setProperty('b', 2, "new", runtime=False) newprops.setProperty('c', 3, "new", runtime=True) newprops.setProperty('d', 3, "new", runtime=False) self.props.updateFromPropertiesNoRuntime(newprops) self.failUnlessEqual(self.props.getProperty('a'), 94) self.failUnlessEqual(self.props.getPropertySource('a'), 'old') self.failUnlessEqual(self.props.getProperty('b'), 2) self.failUnlessEqual(self.props.getPropertySource('b'), 'new') self.failUnlessEqual(self.props.getProperty('c'), None) # not updated self.failUnlessEqual(self.props.getProperty('d'), 3) self.failUnlessEqual(self.props.getPropertySource('d'), 'new') self.failUnlessEqual(self.props.getProperty('x'), 24) self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
class AbstractBuildSlave(config.ReconfigurableServiceMixin, pb.Avatar, service.MultiService): """This is the master-side representative for a remote buildbot slave. There is exactly one for each slave described in the config file (the c['slaves'] list). When buildbots connect in (.attach), they get a reference to this instance. The BotMaster object is stashed as the .botmaster attribute. The BotMaster is also our '.parent' Service. I represent a build slave -- a remote machine capable of running builds. I am instantiated by the configuration file, and can be subclassed to add extra functionality.""" implements(IBuildSlave) keepalive_timer = None keepalive_interval = None # reconfig slaves after builders reconfig_priority = 64 def __init__(self, name, password, max_builds=None, notify_on_missing=[], missing_timeout=3600, properties={}, locks=None, keepalive_interval=3600): """ @param name: botname this machine will supply when it connects @param password: password this machine will supply when it connects @param max_builds: maximum number of simultaneous builds that will be run concurrently on this buildslave (the default is None for no limit) @param properties: properties that will be applied to builds run on this slave @type properties: dictionary @param locks: A list of locks that must be acquired before this slave can be used @type locks: dictionary """ service.MultiService.__init__(self) self.slavename = name self.password = password # PB registration self.registration = None self.registered_port = None # these are set when the service is started, and unset when it is # stopped self.botmaster = None self.master = None self.slave_status = SlaveStatus(name) self.slave = None # a RemoteReference to the Bot, when connected self.slave_commands = None self.slavebuilders = {} self.max_builds = max_builds self.access = [] if locks: self.access = locks self.lock_subscriptions = [] self.properties = Properties() self.properties.update(properties, "BuildSlave") self.properties.setProperty("slavename", name, "BuildSlave") self.lastMessageReceived = 0 if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing for i in notify_on_missing: if not isinstance(i, str): config.error('notify_on_missing arg %r is not a string' % (i, )) self.missing_timeout = missing_timeout self.missing_timer = None self.keepalive_interval = keepalive_interval self.detached_subs = None self._old_builder_list = None def __repr__(self): return "<%s %r>" % (self.__class__.__name__, self.slavename) def updateLocks(self): """Convert the L{LockAccess} objects in C{self.locks} into real lock objects, while also maintaining the subscriptions to lock releases.""" # unsubscribe from any old locks for s in self.lock_subscriptions: s.unsubscribe() # convert locks into their real form locks = [(self.botmaster.getLockFromLockAccess(a), a) for a in self.access] self.locks = [(l.getLock(self), la) for l, la in locks] self.lock_subscriptions = [ l.subscribeToReleases(self._lockReleased) for l, la in self.locks ] def locksAvailable(self): """ I am called to see if all the locks I depend on are available, in which I return True, otherwise I return False """ if not self.locks: return True for lock, access in self.locks: if not lock.isAvailable(self, access): return False return True def acquireLocks(self): """ I am called when a build is preparing to run. I try to claim all the locks that are needed for a build to happen. If I can't, then my caller should give up the build and try to get another slave to look at it. """ log.msg("acquireLocks(slave %s, locks %s)" % (self, self.locks)) if not self.locksAvailable(): log.msg("slave %s can't lock, giving up" % (self, )) return False # all locks are available, claim them all for lock, access in self.locks: lock.claim(self, access) return True def releaseLocks(self): """ I am called to release any locks after a build has finished """ log.msg("releaseLocks(%s): %s" % (self, self.locks)) for lock, access in self.locks: lock.release(self, access) def _lockReleased(self): """One of the locks for this slave was released; try scheduling builds.""" if not self.botmaster: return # oh well.. self.botmaster.maybeStartBuildsForSlave(self.slavename) def _applySlaveInfo(self, info): if not info: return self.slave_status.setAdmin(info.get("admin")) self.slave_status.setHost(info.get("host")) self.slave_status.setAccessURI(info.get("access_uri")) self.slave_status.setVersion(info.get("version")) def _saveSlaveInfoDict(self): slaveinfo = { 'admin': self.slave_status.getAdmin(), 'host': self.slave_status.getHost(), 'access_uri': self.slave_status.getAccessURI(), 'version': self.slave_status.getVersion(), } return self.master.db.buildslaves.updateBuildslave( name=self.slavename, slaveinfo=slaveinfo, ) def _getSlaveInfo(self): d = self.master.db.buildslaves.getBuildslaveByName(self.slavename) @d.addCallback def applyInfo(buildslave): if buildslave is None: return self._applySlaveInfo(buildslave.get('slaveinfo')) return d def setServiceParent(self, parent): # botmaster needs to set before setServiceParent which calls startService self.botmaster = parent self.master = parent.master service.MultiService.setServiceParent(self, parent) def startService(self): self.updateLocks() self.startMissingTimer() d = self._getSlaveInfo() d.addCallback(lambda _: service.MultiService.startService(self)) return d @defer.inlineCallbacks def reconfigService(self, new_config): # Given a new BuildSlave, configure this one identically. Because # BuildSlave objects are remotely referenced, we can't replace them # without disconnecting the slave, yet there's no reason to do that. new = self.findNewSlaveInstance(new_config) assert self.slavename == new.slavename # do we need to re-register? if (not self.registration or self.password != new.password or new_config.protocols['pb']['port'] != self.registered_port): if self.registration: yield self.registration.unregister() self.registration = None self.password = new.password self.registered_port = new_config.protocols['pb']['port'] self.registration = self.master.pbmanager.register( self.registered_port, self.slavename, self.password, self.getPerspective) # adopt new instance's configuration parameters self.max_builds = new.max_builds self.access = new.access self.notify_on_missing = new.notify_on_missing self.keepalive_interval = new.keepalive_interval if self.missing_timeout != new.missing_timeout: running_missing_timer = self.missing_timer self.stopMissingTimer() self.missing_timeout = new.missing_timeout if running_missing_timer: self.startMissingTimer() properties = Properties() properties.updateFromProperties(new.properties) self.properties = properties self.updateLocks() # update the attached slave's notion of which builders are attached. # This assumes that the relevant builders have already been configured, # which is why the reconfig_priority is set low in this class. yield self.updateSlave() yield config.ReconfigurableServiceMixin.reconfigService( self, new_config) def stopService(self): if self.registration: self.registration.unregister() self.registration = None self.stopMissingTimer() return service.MultiService.stopService(self) def findNewSlaveInstance(self, new_config): # TODO: called multiple times per reconfig; use 1-element cache? for sl in new_config.slaves: if sl.slavename == self.slavename: return sl assert 0, "no new slave named '%s'" % self.slavename def startMissingTimer(self): if self.notify_on_missing and self.missing_timeout and self.parent: self.stopMissingTimer() # in case it's already running self.missing_timer = reactor.callLater(self.missing_timeout, self._missing_timer_fired) def stopMissingTimer(self): if self.missing_timer: self.missing_timer.cancel() self.missing_timer = None def getPerspective(self, mind, slavename): assert slavename == self.slavename metrics.MetricCountEvent.log("attached_slaves", 1) # record when this connection attempt occurred if self.slave_status: self.slave_status.recordConnectTime() # try to use TCP keepalives try: mind.broker.transport.setTcpKeepAlive(1) except: pass if self.isConnected(): # duplicate slave - send it to arbitration arb = botmaster.DuplicateSlaveArbitrator(self) return arb.getPerspective(mind, slavename) else: log.msg("slave '%s' attaching from %s" % (slavename, mind.broker.transport.getPeer())) return self def doKeepalive(self): self.keepalive_timer = reactor.callLater(self.keepalive_interval, self.doKeepalive) if not self.slave: return d = self.slave.callRemote("print", "Received keepalive from master") d.addErrback(log.msg, "Keepalive failed for '%s'" % (self.slavename, )) def stopKeepaliveTimer(self): if self.keepalive_timer: self.keepalive_timer.cancel() def startKeepaliveTimer(self): assert self.keepalive_interval log.msg("Starting buildslave keepalive timer for '%s'" % (self.slavename, )) self.doKeepalive() def isConnected(self): return self.slave def _missing_timer_fired(self): self.missing_timer = None # notify people, but only if we're still in the config if not self.parent: return buildmaster = self.botmaster.master status = buildmaster.getStatus() text = "The Buildbot working for '%s'\n" % status.getTitle() text += ("has noticed that the buildslave named %s went away\n" % self.slavename) text += "\n" text += ("It last disconnected at %s (buildmaster-local time)\n" % time.ctime(time.time() - self.missing_timeout)) # approx text += "\n" text += "The admin on record (as reported by BUILDSLAVE:info/admin)\n" text += "was '%s'.\n" % self.slave_status.getAdmin() text += "\n" text += "Sincerely,\n" text += " The Buildbot\n" text += " %s\n" % status.getTitleURL() text += "\n" text += "%s\n" % status.getURLForThing(self.slave_status) subject = "Buildbot: buildslave %s was lost" % self.slavename return self._mail_missing_message(subject, text) def updateSlave(self): """Called to add or remove builders after the slave has connected. @return: a Deferred that indicates when an attached slave has accepted the new builders and/or released the old ones.""" if self.slave: return self.sendBuilderList() else: return defer.succeed(None) def updateSlaveStatus(self, buildStarted=None, buildFinished=None): if buildStarted: self.slave_status.buildStarted(buildStarted) if buildFinished: self.slave_status.buildFinished(buildFinished) @metrics.countMethod('AbstractBuildSlave.attached()') def attached(self, bot): """This is called when the slave connects. @return: a Deferred that fires when the attachment is complete """ # the botmaster should ensure this. assert not self.isConnected() metrics.MetricCountEvent.log("AbstractBuildSlave.attached_slaves", 1) # set up the subscription point for eventual detachment self.detached_subs = subscription.SubscriptionPoint("detached") # now we go through a sequence of calls, gathering information, then # tell the Botmaster that it can finally give this slave to all the # Builders that care about it. # we accumulate slave information in this 'state' dictionary, then # set it atomically if we make it far enough through the process state = {} # Reset graceful shutdown status self.slave_status.setGraceful(False) # We want to know when the graceful shutdown flag changes self.slave_status.addGracefulWatcher(self._gracefulChanged) d = defer.succeed(None) @d.addCallback def _log_attachment_on_slave(res): d1 = bot.callRemote("print", "attached") d1.addErrback(lambda why: None) return d1 @d.addCallback def _get_info(res): d1 = bot.callRemote("getSlaveInfo") def _got_info(info): log.msg("Got slaveinfo from '%s'" % self.slavename) # TODO: info{} might have other keys state["admin"] = info.get("admin") state["host"] = info.get("host") state["access_uri"] = info.get("access_uri", None) state["slave_environ"] = info.get("environ", {}) state["slave_basedir"] = info.get("basedir", None) state["slave_system"] = info.get("system", None) def _info_unavailable(why): why.trap(pb.NoSuchMethod) # maybe an old slave, doesn't implement remote_getSlaveInfo log.msg("BuildSlave.info_unavailable") log.err(why) d1.addCallbacks(_got_info, _info_unavailable) return d1 d.addCallback(lambda _: self.startKeepaliveTimer()) @d.addCallback def _get_version(_): d = bot.callRemote("getVersion") def _got_version(version): state["version"] = version def _version_unavailable(why): why.trap(pb.NoSuchMethod) # probably an old slave state["version"] = '(unknown)' d.addCallbacks(_got_version, _version_unavailable) return d @d.addCallback def _get_commands(_): d1 = bot.callRemote("getCommands") def _got_commands(commands): state["slave_commands"] = commands def _commands_unavailable(why): # probably an old slave if why.check(AttributeError): return log.msg("BuildSlave.getCommands is unavailable - ignoring") log.err(why) d1.addCallbacks(_got_commands, _commands_unavailable) return d1 @d.addCallback def _accept_slave(res): self.slave_status.setConnected(True) self._applySlaveInfo(state) self.slave_commands = state.get("slave_commands") self.slave_environ = state.get("slave_environ") self.slave_basedir = state.get("slave_basedir") self.slave_system = state.get("slave_system") self.slave = bot if self.slave_system == "nt": self.path_module = namedModule("ntpath") else: # most everything accepts / as separator, so posix should be a # reasonable fallback self.path_module = namedModule("posixpath") log.msg("bot attached") self.messageReceivedFromSlave() self.stopMissingTimer() self.botmaster.master.status.slaveConnected(self.slavename) d.addCallback(lambda _: self._saveSlaveInfoDict()) d.addCallback(lambda _: self.updateSlave()) d.addCallback( lambda _: self.botmaster.maybeStartBuildsForSlave(self.slavename)) # Finally, the slave gets a reference to this BuildSlave. They # receive this later, after we've started using them. d.addCallback(lambda _: self) return d def messageReceivedFromSlave(self): now = time.time() self.lastMessageReceived = now self.slave_status.setLastMessageReceived(now) def detached(self, mind): metrics.MetricCountEvent.log("AbstractBuildSlave.attached_slaves", -1) self.slave = None self._old_builder_list = [] self.slave_status.removeGracefulWatcher(self._gracefulChanged) self.slave_status.setConnected(False) log.msg("BuildSlave.detached(%s)" % self.slavename) self.botmaster.master.status.slaveDisconnected(self.slavename) self.stopKeepaliveTimer() self.releaseLocks() # notify watchers, but do so in the next reactor iteration so that # any further detached() action by subclasses happens first def notif(): subs = self.detached_subs self.detached_subs = None subs.deliver() eventually(notif) def subscribeToDetach(self, callback): """ Request that C{callable} be invoked with no arguments when the L{detached} method is invoked. @returns: L{Subscription} """ assert self.detached_subs, "detached_subs is only set if attached" return self.detached_subs.subscribe(callback) def disconnect(self): """Forcibly disconnect the slave. This severs the TCP connection and returns a Deferred that will fire (with None) when the connection is probably gone. If the slave is still alive, they will probably try to reconnect again in a moment. This is called in two circumstances. The first is when a slave is removed from the config file. In this case, when they try to reconnect, they will be rejected as an unknown slave. The second is when we wind up with two connections for the same slave, in which case we disconnect the older connection. """ if not self.slave: return defer.succeed(None) log.msg("disconnecting old slave %s now" % self.slavename) # When this Deferred fires, we'll be ready to accept the new slave return self._disconnect(self.slave) def _disconnect(self, slave): # all kinds of teardown will happen as a result of # loseConnection(), but it happens after a reactor iteration or # two. Hook the actual disconnect so we can know when it is safe # to connect the new slave. We have to wait one additional # iteration (with callLater(0)) to make sure the *other* # notifyOnDisconnect handlers have had a chance to run. d = defer.Deferred() # notifyOnDisconnect runs the callback with one argument, the # RemoteReference being disconnected. def _disconnected(rref): eventually(d.callback, None) slave.notifyOnDisconnect(_disconnected) tport = slave.broker.transport # this is the polite way to request that a socket be closed tport.loseConnection() try: # but really we don't want to wait for the transmit queue to # drain. The remote end is unlikely to ACK the data, so we'd # probably have to wait for a (20-minute) TCP timeout. # tport._closeSocket() # however, doing _closeSocket (whether before or after # loseConnection) somehow prevents the notifyOnDisconnect # handlers from being run. Bummer. tport.offset = 0 tport.dataBuffer = "" except: # however, these hacks are pretty internal, so don't blow up if # they fail or are unavailable log.msg("failed to accelerate the shutdown process") log.msg("waiting for slave to finish disconnecting") return d def sendBuilderList(self): our_builders = self.botmaster.getBuildersForSlave(self.slavename) blist = [(b.name, b.config.slavebuilddir) for b in our_builders] if blist == self._old_builder_list: return defer.succeed(None) d = self.slave.callRemote("setBuilderList", blist) def sentBuilderList(ign): self._old_builder_list = blist return ign d.addCallback(sentBuilderList) return d def perspective_keepalive(self): self.messageReceivedFromSlave() def perspective_shutdown(self): log.msg("slave %s wants to shut down" % self.slavename) self.slave_status.setGraceful(True) def addSlaveBuilder(self, sb): self.slavebuilders[sb.builder_name] = sb def removeSlaveBuilder(self, sb): try: del self.slavebuilders[sb.builder_name] except KeyError: pass def buildFinished(self, sb): """This is called when a build on this slave is finished.""" self.botmaster.maybeStartBuildsForSlave(self.slavename) def canStartBuild(self): """ I am called when a build is requested to see if this buildslave can start a build. This function can be used to limit overall concurrency on the buildslave. Note for subclassers: if a slave can become willing to start a build without any action on that slave (for example, by a resource in use on another slave becoming available), then you must arrange for L{maybeStartBuildsForSlave} to be called at that time, or builds on this slave will not start. """ if self.slave_status.isPaused(): return False # If we're waiting to shutdown gracefully, then we shouldn't # accept any new jobs. if self.slave_status.getGraceful(): return False if self.max_builds: active_builders = [ sb for sb in self.slavebuilders.values() if sb.isBusy() ] if len(active_builders) >= self.max_builds: return False if not self.locksAvailable(): return False return True def _mail_missing_message(self, subject, text): # first, see if we have a MailNotifier we can use. This gives us a # fromaddr and a relayhost. buildmaster = self.botmaster.master for st in buildmaster.status: if isinstance(st, MailNotifier): break else: # if not, they get a default MailNotifier, which always uses SMTP # to localhost and uses a dummy fromaddr of "buildbot". log.msg("buildslave-missing msg using default MailNotifier") st = MailNotifier("buildbot") # now construct the mail m = Message() m.set_payload(text) m['Date'] = formatdate(localtime=True) m['Subject'] = subject m['From'] = st.fromaddr recipients = self.notify_on_missing m['To'] = ", ".join(recipients) d = st.sendMessage(m, recipients) # return the Deferred for testing purposes return d def _gracefulChanged(self, graceful): """This is called when our graceful shutdown setting changes""" self.maybeShutdown() @defer.inlineCallbacks def shutdown(self): """Shutdown the slave""" if not self.slave: log.msg("no remote; slave is already shut down") return # First, try the "new" way - calling our own remote's shutdown # method. The method was only added in 0.8.3, so ignore NoSuchMethod # failures. def new_way(): d = self.slave.callRemote('shutdown') d.addCallback(lambda _: True) # successful shutdown request def check_nsm(f): f.trap(pb.NoSuchMethod) return False # fall through to the old way d.addErrback(check_nsm) def check_connlost(f): f.trap(pb.PBConnectionLost) return True # the slave is gone, so call it finished d.addErrback(check_connlost) return d if (yield new_way()): return # done! # Now, the old way. Look for a builder with a remote reference to the # client side slave. If we can find one, then call "shutdown" on the # remote builder, which will cause the slave buildbot process to exit. def old_way(): d = None for b in self.slavebuilders.values(): if b.remote: d = b.remote.callRemote("shutdown") break if d: log.msg("Shutting down (old) slave: %s" % self.slavename) # The remote shutdown call will not complete successfully since the # buildbot process exits almost immediately after getting the # shutdown request. # Here we look at the reason why the remote call failed, and if # it's because the connection was lost, that means the slave # shutdown as expected. def _errback(why): if why.check(pb.PBConnectionLost): log.msg("Lost connection to %s" % self.slavename) else: log.err("Unexpected error when trying to shutdown %s" % self.slavename) d.addErrback(_errback) return d log.err("Couldn't find remote builder to shut down slave") return defer.succeed(None) yield old_way() def maybeShutdown(self): """Shut down this slave if it has been asked to shut down gracefully, and has no active builders.""" if not self.slave_status.getGraceful(): return active_builders = [ sb for sb in self.slavebuilders.values() if sb.isBusy() ] if active_builders: return d = self.shutdown() d.addErrback(log.err, 'error while shutting down slave') def pause(self): """Stop running new builds on the slave.""" self.slave_status.setPaused(True) def unpause(self): """Restart running new builds on the slave.""" self.slave_status.setPaused(False) self.botmaster.maybeStartBuildsForSlave(self.slavename) def isPaused(self): return self.paused
class Change: """I represent a single change to the source tree. This may involve several files, but they are all changed by the same person, and there is a change comment for the group as a whole. If the version control system supports sequential repository- (or branch-) wide change numbers (like SVN, P4, and Arch), then revision= should be set to that number. The highest such number will be used at checkout time to get the correct set of files. If it does not (like CVS), when= should be set to the timestamp (seconds since epoch, as returned by time.time()) when the change was made. when= will be filled in for you (to the current time) if you omit it, which is suitable for ChangeSources which have no way of getting more accurate timestamps. The revision= and branch= values must be ASCII bytestrings, since they will eventually be used in a ShellCommand and passed to os.exec(), which requires bytestrings. These values will also be stored in a database, possibly as unicode, so they must be safely convertable back and forth. This restriction may be relaxed in the future. Changes should be submitted to ChangeMaster.addChange() in chronologically increasing order. Out-of-order changes will probably cause the web status displays to be corrupted.""" implements(interfaces.IStatusEvent) number = None branch = None category = None revision = None # used to create a source-stamp def __init__(self, who, files, comments, isdir=0, links=None, revision=None, when=None, branch=None, category=None, revlink='', properties={}, repository='', project=''): self.who = who self.comments = comments self.isdir = isdir if links is None: links = [] self.links = links def none_or_unicode(x): if x is None: return x return unicode(x) self.revision = none_or_unicode(revision) if when is None: when = util.now() self.when = when self.branch = none_or_unicode(branch) self.category = none_or_unicode(category) self.revlink = revlink self.properties = Properties() self.properties.update(properties, "Change") self.repository = repository self.project = project # keep a sorted list of the files, for easier display self.files = files[:] self.files.sort() def __setstate__(self, dict): self.__dict__ = dict # Older Changes won't have a 'properties' attribute in them if not hasattr(self, 'properties'): self.properties = Properties() if not hasattr(self, 'revlink'): self.revlink = "" def asText(self): data = "" data += self.getFileContents() if self.repository: data += "On: %s\n" % self.repository if self.project: data += "For: %s\n" % self.project data += "At: %s\n" % self.getTime() data += "Changed By: %s\n" % self.who data += "Comments: %s" % self.comments data += "Properties: \n%s\n\n" % self.getProperties() return data def asDict(self): '''returns a dictonary with suitable info for html/mail rendering''' result = {} files = [] for file in self.files: link = filter(lambda s: s.find(file) != -1, self.links) if len(link) == 1: url = link[0] else: url = None files.append(dict(url=url, name=file)) files = sorted(files, cmp=lambda a, b: a['name'] < b['name']) # Constant result['number'] = self.number result['branch'] = self.branch result['category'] = self.category result['who'] = self.getShortAuthor() result['comments'] = self.comments result['revision'] = self.revision result['rev'] = self.revision result['when'] = self.when result['at'] = self.getTime() result['files'] = files result['revlink'] = getattr(self, 'revlink', None) result['properties'] = self.properties.asList() result['repository'] = getattr(self, 'repository', None) result['project'] = getattr(self, 'project', None) return result def getShortAuthor(self): return self.who def getTime(self): if not self.when: return "?" return time.strftime("%a %d %b %Y %H:%M:%S", time.localtime(self.when)) def getTimes(self): return (self.when, None) def getText(self): return [html.escape(self.who)] def getLogs(self): return {} def getFileContents(self): data = "" if len(self.files) == 1: if self.isdir: data += "Directory: %s\n" % self.files[0] else: data += "File: %s\n" % self.files[0] else: data += "Files:\n" for f in self.files: data += " %s\n" % f return data def getProperties(self): data = "" for prop in self.properties.asList(): data += " %s: %s" % (prop[0], prop[1]) return data
def createTriggerProperties(self, properties): # make a new properties object from a dict rendered by the old # properties object trigger_properties = Properties() trigger_properties.update(properties, "Trigger") return trigger_properties
def __init__(self, reason, sources, buildername): self.reason = reason self.sources = sources self.buildername = buildername self.changes = [] self.properties = Properties()
def setUp(self): self.patch(openstack, "client", novaclient) self.patch(openstack, "loading", novaclient) self.patch(openstack, "session", novaclient) self.build = Properties(image=novaclient.TEST_UUIDS['image'])
def getProperties(self): return Properties()
def render_message_subject(self, context): props = Properties.fromDict(context['build']['properties']) props.master = context['master'] body = yield props.render(self.subject) return body
def command_FORCE(self, args, who): errReply = "try 'force build [--branch=BRANCH] [--revision=REVISION] [--props=PROP1=VAL1,PROP2=VAL2...] <WHICH> <REASON>'" args = shlex.split(args) if not args: raise UsageError(errReply) what = args.pop(0) if what != "build": raise UsageError(errReply) opts = ForceOptions() opts.parseOptions(args) which = opts['builder'] branch = opts['branch'] revision = opts['revision'] reason = opts['reason'] props = opts['props'] if which is None: raise UsageError("you must provide a Builder, " + errReply) # keep weird stuff out of the branch, revision, and properties args. branch_validate = self.master.config.validation['branch'] revision_validate = self.master.config.validation['revision'] pname_validate = self.master.config.validation['property_name'] pval_validate = self.master.config.validation['property_value'] if branch and not branch_validate.match(branch): log.msg("bad branch '%s'" % branch) self.send("sorry, bad branch '%s'" % branch) return if revision and not revision_validate.match(revision): log.msg("bad revision '%s'" % revision) self.send("sorry, bad revision '%s'" % revision) return properties = Properties() if props: # split props into name:value dict pdict = {} propertylist = props.split(",") for i in range(0,len(propertylist)): splitproperty = propertylist[i].split("=", 1) pdict[splitproperty[0]] = splitproperty[1] # set properties for prop in pdict: pname = prop pvalue = pdict[prop] if not pname_validate.match(pname) \ or not pval_validate.match(pvalue): log.msg("bad property name='%s', value='%s'" % (pname, pvalue)) self.send("sorry, bad property name='%s', value='%s'" % (pname, pvalue)) return properties.setProperty(pname, pvalue, "Force Build IRC") bc = self.getControl(which) reason = "forced: by %s: %s" % (self.describeUser(who), reason) ss = SourceStamp(branch=branch, revision=revision) d = bc.submitBuildRequest(ss, reason, props=properties.asDict()) def subscribe(buildreq): ireq = IrcBuildRequest(self, self.useRevisions) buildreq.subscribe(ireq.started) d.addCallback(subscribe) d.addErrback(log.err, "while forcing a build")
def send(self, build): props = Properties.fromDict(build['properties']) if build['complete']: state = { SUCCESS: 'success', WARNINGS: 'success', FAILURE: 'failure', SKIPPED: 'success', EXCEPTION: 'error', RETRY: 'pending', CANCELLED: 'error' }.get(build['results'], 'error') description = yield props.render(self.endDescription) else: state = 'pending' description = yield props.render(self.startDescription) context = yield props.render(self.context) sourcestamps = build['buildset'].get('sourcestamps') if not sourcestamps or not sourcestamps[0]: return project = sourcestamps[0]['project'] if project: repoOwner, repoName = project.split('/') else: repo = sourcestamps[0]['repository'].split('/')[-2:] repoOwner = repo[0] repoName = '.'.join(repo[1].split('.')[:-1]) for sourcestamp in sourcestamps: sha = sourcestamp['revision'] try: repo_user = repoOwner.encode('utf-8') repo_user = bytes2NativeString(repo_user, encoding='utf-8') repo_name = repoName.encode('utf-8') repo_name = bytes2NativeString(repo_name, encoding='utf-8') sha = sha.encode('utf-8') sha = bytes2NativeString(sha, encoding='utf-8') state = state.encode('utf-8') state = bytes2NativeString(state, encoding='utf-8') target_url = build['url'].encode('utf-8') target_url = bytes2NativeString(target_url, encoding='utf-8') context = context.encode('utf-8') context = bytes2NativeString(context, encoding='utf-8') description = description.encode('utf-8') description = bytes2NativeString(description, encoding='utf-8') yield self.createStatus(repo_user=repo_user, repo_name=repo_name, sha=sha, state=state, target_url=target_url, context=context, description=description) if self.verbose: log.msg('Status "{state}" sent for ' '{repoOwner}/{repoName} at {sha}.'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha)) except Exception as e: log.err( e, 'Fail to send status "{state}" for ' '{repoOwner}/{repoName} at {sha}'.format( state=state, repoOwner=repoOwner, repoName=repoName, sha=sha))
def __init__(self, mastername, properties=Properties()): self.master = FakeMaster(mastername) self.parent = self self.properties = properties
def command_FORCE(self, args): # FIXME: NEED TO THINK ABOUT! errReply = "try '%s'" % (self.command_FORCE.usage) args = self.splitArgs(args) if not args: raise UsageError(errReply) what = args.pop(0) if what != "build": raise UsageError(errReply) opts = ForceOptions() opts.parseOptions(args) builderName = opts['builder'] builder = yield self.getBuilder(buildername=builderName) branch = opts['branch'] revision = opts['revision'] codebase = opts['codebase'] project = opts['project'] reason = opts['reason'] props = opts['props'] if builderName is None: raise UsageError("you must provide a Builder, " + errReply) # keep weird stuff out of the branch, revision, and properties args. branch_validate = self.master.config.validation['branch'] revision_validate = self.master.config.validation['revision'] pname_validate = self.master.config.validation['property_name'] pval_validate = self.master.config.validation['property_value'] if branch and not branch_validate.match(branch): log.msg("bad branch '%s'" % branch) self.send("sorry, bad branch '%s'" % branch) return if revision and not revision_validate.match(revision): log.msg("bad revision '%s'" % revision) self.send("sorry, bad revision '%s'" % revision) return properties = Properties() properties.master = self.master if props: # split props into name:value dict pdict = {} propertylist = props.split(",") for prop in propertylist: splitproperty = prop.split("=", 1) pdict[splitproperty[0]] = splitproperty[1] # set properties for prop in pdict: pname = prop pvalue = pdict[prop] if not pname_validate.match(pname) \ or not pval_validate.match(pvalue): log.msg("bad property name='%s', value='%s'" % (pname, pvalue)) self.send("sorry, bad property name='%s', value='%s'" % (pname, pvalue)) return properties.setProperty(pname, pvalue, "Force Build chat") reason = "forced: by %s: %s" % (self.describeUser(), reason) try: yield self.master.data.updates.addBuildset(builderids=[builder['builderid']], # For now, we just use # this as the id. scheduler="status.words", sourcestamps=[{ 'codebase': codebase, 'branch': branch, 'revision': revision, 'project': project, 'repository': "null"}], reason=reason, properties=properties.asDict(), waited_for=False) except AssertionError as e: self.send("I can't: " + str(e))
class JhBuildMaster(BuildMaster): jhbuild_config = config def loadConfig(self, f): # modified from parent method to get slaves, projects, change # sources, schedulers, builders and web status ouf of # master.cfg [it would have been cleaner if jhbuild didn't # have to copy all that code.] localDict = {'basedir': os.path.expanduser(self.basedir)} try: exec f in localDict except: log.msg("error while parsing config file") raise jhbuild_config.load() try: config = localDict['BuildmasterConfig'] except KeyError: log.err("missing config dictionary") log.err("config file must define BuildmasterConfig") raise known_keys = ( "bots", "slaves", "sources", "change_source", "schedulers", "builders", "mergeRequests", "slavePortnum", "debugPassword", "logCompressionLimit", "manhole", "status", "projectName", "projectURL", "buildbotURL", "properties", "prioritizeBuilders", "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon", "changeHorizon", "logMaxSize", "logMaxTailSize", "logCompressionMethod", ) for k in config.keys(): if k not in known_keys: log.msg( "unknown key '%s' defined in config dictionary" % k) # the 'slaves' list is read from the 'slaves.csv' file in the # current directory (unless instructed different from command line) # it is a CSV file structured like this: # slavename,password config['slaves'] = [] slaves_csv_file = os.path.join(slaves_dir, 'slaves.csv') if os.path.exists(slaves_csv_file): for x in csv.reader(file(slaves_csv_file)): if not x or x[0].startswith('#'): continue kw = {} build_slave = JhBuildSlave(x[0], x[1]) build_slave.load_extra_configuration(slaves_dir) config['slaves'].append(build_slave) if len(config['slaves']) == 0: log.msg('you must fill slaves.csv with slaves') module_set = jhbuild.moduleset.load(self.jhbuild_config) module_list = module_set.get_module_list( self.jhbuild_config.modules, self.jhbuild_config.skip, include_afters=True) config['projects'] = [x.name for x in module_list \ if not x.name.startswith('meta-')] if self.jhbuild_config.jhbuildbot_svn_commits_box: # trigger builds from mails to svn-commit-list # (note Maildir must be correct, or everything will fail) from jhbuild.buildbot.changes import GnomeMaildirSource config['change_source'] = GnomeMaildirSource( self.jhbuild_config.jhbuildbot_svn_commits_box, modules=module_list, prefix=None) else: # support injection (use 'buildbot sendchange') from buildbot.changes.pb import PBChangeSource config['change_source'] = PBChangeSource() # Schedulers from jhbuild.buildbot.scheduler import SerialScheduler, NightlySerialScheduler, OnCommitScheduler config['schedulers'] = [] for slave in config['slaves']: s = None for project in config['projects']: buildername = str('%s-%s' % (project, slave.slavename)) scheduler_kwargs = {} if slave.scheduler == 'nightly': scheduler_class = NightlySerialScheduler scheduler_kwargs = slave.nightly_kwargs else: scheduler_class = SerialScheduler s = scheduler_class(buildername, project, upstream=s, builderNames=[buildername], **scheduler_kwargs) config['schedulers'].append(s) if self.jhbuild_config.jhbuildbot_svn_commits_box: # schedulers that will launch job when receiving # change notifications s2 = OnCommitScheduler('oc-' + buildername, project, builderNames=[buildername]) config['schedulers'].append(s2) # Builders from jhbuild.buildbot.factory import JHBuildFactory config['builders'] = [] for project in config['projects']: for slave in config['slaves']: f = JHBuildFactory(project, slave) config['builders'].append({ 'name': "%s-%s" % (project, slave.slavename), 'slavename': slave.slavename, 'builddir': 'builddir/%s.%s' % (project, slave.slavename), 'factory': f, 'category': project }) # Status targets if not config.has_key('status'): # let it be possible to define additional status in # master.cfg config['status'] = [] from jhbuild.buildbot.status.web import JHBuildWebStatus config['status'].append( JHBuildWebStatus(self.jhbuild_config.moduleset, config['projects'], [x.slavename for x in config['slaves']], http_port=8080, allowForce=True)) # remaining of the method is a straight copy from buildbot # ... try: # required schedulers = config['schedulers'] builders = config['builders'] slavePortnum = config['slavePortnum'] #slaves = config['slaves'] #change_source = config['change_source'] # optional debugPassword = config.get('debugPassword') manhole = config.get('manhole') status = config.get('status', []) projectName = config.get('projectName') projectURL = config.get('projectURL') buildbotURL = config.get('buildbotURL') properties = config.get('properties', {}) buildCacheSize = config.get('buildCacheSize', None) eventHorizon = config.get('eventHorizon', None) logHorizon = config.get('logHorizon', None) buildHorizon = config.get('buildHorizon', None) logCompressionLimit = config.get('logCompressionLimit', 4 * 1024) if logCompressionLimit is not None and not \ isinstance(logCompressionLimit, int): raise ValueError( "logCompressionLimit needs to be bool or int") logCompressionMethod = config.get('logCompressionMethod', "bz2") if logCompressionMethod not in ('bz2', 'gz'): raise ValueError( "logCompressionMethod needs to be 'bz2', or 'gz'") logMaxSize = config.get('logMaxSize') if logMaxSize is not None and not \ isinstance(logMaxSize, int): raise ValueError("logMaxSize needs to be None or int") logMaxTailSize = config.get('logMaxTailSize') if logMaxTailSize is not None and not \ isinstance(logMaxTailSize, int): raise ValueError( "logMaxTailSize needs to be None or int") mergeRequests = config.get('mergeRequests') if mergeRequests is not None and not callable( mergeRequests): raise ValueError("mergeRequests must be a callable") prioritizeBuilders = config.get('prioritizeBuilders') if prioritizeBuilders is not None and not callable( prioritizeBuilders): raise ValueError("prioritizeBuilders must be callable") changeHorizon = config.get("changeHorizon") if changeHorizon is not None and not isinstance( changeHorizon, int): raise ValueError("changeHorizon needs to be an int") except KeyError, e: log.msg( "config dictionary is missing a required parameter") log.msg("leaving old configuration in place") raise #if "bots" in config: # raise KeyError("c['bots'] is no longer accepted") slaves = config.get('slaves', []) if "bots" in config: m = ("c['bots'] is deprecated as of 0.7.6 and will be " "removed by 0.8.0 . Please use c['slaves'] instead.") log.msg(m) warnings.warn(m, DeprecationWarning) for name, passwd in config['bots']: slaves.append(JhBuildSlave(name, passwd)) if "bots" not in config and "slaves" not in config: log.msg( "config dictionary must have either 'bots' or 'slaves'" ) log.msg("leaving old configuration in place") raise KeyError("must have either 'bots' or 'slaves'") #if "sources" in config: # raise KeyError("c['sources'] is no longer accepted") if changeHorizon is not None: self.change_svc.changeHorizon = changeHorizon change_source = config.get('change_source', []) if isinstance(change_source, (list, tuple)): change_sources = change_source else: change_sources = [change_source] if "sources" in config: m = ( "c['sources'] is deprecated as of 0.7.6 and will be " "removed by 0.8.0 . Please use c['change_source'] instead." ) log.msg(m) warnings.warn(m, DeprecationWarning) for s in config['sources']: change_sources.append(s) # do some validation first for s in slaves: assert interfaces.IBuildSlave.providedBy(s) if s.slavename in ("debug", "change", "status"): raise KeyError("reserved name '%s' used for a bot" % s.slavename) if config.has_key('interlocks'): raise KeyError("c['interlocks'] is no longer accepted") assert isinstance(change_sources, (list, tuple)) for s in change_sources: assert interfaces.IChangeSource(s, None) # this assertion catches c['schedulers'] = Scheduler(), since # Schedulers are service.MultiServices and thus iterable. errmsg = "c['schedulers'] must be a list of Scheduler instances" assert isinstance(schedulers, (list, tuple)), errmsg for s in schedulers: assert interfaces.IScheduler(s, None), errmsg assert isinstance(status, (list, tuple)) for s in status: assert interfaces.IStatusReceiver(s, None) slavenames = [s.slavename for s in slaves] buildernames = [] dirnames = [] # convert builders from objects to config dictionaries builders_dicts = [] for b in builders: if isinstance(b, buildbot.config.BuilderConfig): builders_dicts.append(b.getConfigDict()) elif type(b) is dict: builders_dicts.append(b) else: raise ValueError( "builder %s is not a BuilderConfig object (or a dict)" % b) builders = builders_dicts for b in builders: if b.has_key( 'slavename') and b['slavename'] not in slavenames: raise ValueError("builder %s uses undefined slave %s" \ % (b['name'], b['slavename'])) for n in b.get('slavenames', []): if n not in slavenames: raise ValueError("builder %s uses undefined slave %s" \ % (b['name'], n)) if b['name'] in buildernames: raise ValueError("duplicate builder name %s" % b['name']) buildernames.append(b['name']) # sanity check name (BuilderConfig does this too) if b['name'].startswith("_"): errmsg = ("builder names must not start with an " "underscore: " + b['name']) log.err(errmsg) raise ValueError(errmsg) # Fix the dictionnary with default values, in case this wasn't # specified with a BuilderConfig object (which sets the same defaults) b.setdefault('builddir', buildbot.util.safeTranslate(b['name'])) b.setdefault('slavebuilddir', b['builddir']) if b['builddir'] in dirnames: raise ValueError("builder %s reuses builddir %s" % (b['name'], b['builddir'])) dirnames.append(b['builddir']) unscheduled_buildernames = buildernames[:] schedulernames = [] for s in schedulers: for b in s.listBuilderNames(): assert b in buildernames, \ "%s uses unknown builder %s" % (s, b) if b in unscheduled_buildernames: unscheduled_buildernames.remove(b) if s.name in schedulernames: # TODO: schedulers share a namespace with other Service # children of the BuildMaster node, like status plugins, the # Manhole, the ChangeMaster, and the BotMaster (although most # of these don't have names) msg = ("Schedulers must have unique names, but " "'%s' was a duplicate" % (s.name, )) raise ValueError(msg) schedulernames.append(s.name) if unscheduled_buildernames: log.msg( "Warning: some Builders have no Schedulers to drive them:" " %s" % (unscheduled_buildernames, )) # assert that all locks used by the Builds and their Steps are # uniquely named. lock_dict = {} for b in builders: for l in b.get('locks', []): if isinstance(l, locks.LockAccess ): # User specified access to the lock l = l.lockid if lock_dict.has_key(l.name): if lock_dict[l.name] is not l: raise ValueError( "Two different locks (%s and %s) " "share the name %s" % (l, lock_dict[l.name], l.name)) else: lock_dict[l.name] = l # TODO: this will break with any BuildFactory that doesn't use a # .steps list, but I think the verification step is more # important. for s in b['factory'].steps: for l in s[1].get('locks', []): if isinstance( l, locks.LockAccess ): # User specified access to the lock l = l.lockid if lock_dict.has_key(l.name): if lock_dict[l.name] is not l: raise ValueError( "Two different locks (%s and %s)" " share the name %s" % (l, lock_dict[l.name], l.name)) else: lock_dict[l.name] = l if not isinstance(properties, dict): raise ValueError("c['properties'] must be a dictionary") # slavePortnum supposed to be a strports specification if type(slavePortnum) is int: slavePortnum = "tcp:%d" % slavePortnum # now we're committed to implementing the new configuration, so do # it atomically # TODO: actually, this is spread across a couple of Deferreds, so it # really isn't atomic. d = defer.succeed(None) self.projectName = projectName self.projectURL = projectURL self.buildbotURL = buildbotURL self.properties = Properties() self.properties.update(properties, self.configFileName) self.status.logCompressionLimit = logCompressionLimit self.status.logCompressionMethod = logCompressionMethod self.status.logMaxSize = logMaxSize self.status.logMaxTailSize = logMaxTailSize # Update any of our existing builders with the current log parameters. # This is required so that the new value is picked up after a # reconfig. for builder in self.botmaster.builders.values(): builder.builder_status.setLogCompressionLimit( logCompressionLimit) builder.builder_status.setLogCompressionMethod( logCompressionMethod) builder.builder_status.setLogMaxSize(logMaxSize) builder.builder_status.setLogMaxTailSize(logMaxTailSize) if mergeRequests is not None: self.botmaster.mergeRequests = mergeRequests if prioritizeBuilders is not None: self.botmaster.prioritizeBuilders = prioritizeBuilders self.buildCacheSize = buildCacheSize self.eventHorizon = eventHorizon self.logHorizon = logHorizon self.buildHorizon = buildHorizon # self.slaves: Disconnect any that were attached and removed from the # list. Update self.checker with the new list of passwords, including # debug/change/status. d.addCallback(lambda res: self.loadConfig_Slaves(slaves)) # self.debugPassword if debugPassword: self.checker.addUser("debug", debugPassword) self.debugPassword = debugPassword # self.manhole if manhole != self.manhole: # changing if self.manhole: # disownServiceParent may return a Deferred d.addCallback( lambda res: self.manhole.disownServiceParent()) def _remove(res): self.manhole = None return res d.addCallback(_remove) if manhole: def _add(res): self.manhole = manhole manhole.setServiceParent(self) d.addCallback(_add) # add/remove self.botmaster.builders to match builders. The # botmaster will handle startup/shutdown issues. d.addCallback(lambda res: self.loadConfig_Builders(builders)) d.addCallback(lambda res: self.loadConfig_status(status)) # Schedulers are added after Builders in case they start right away d.addCallback( lambda res: self.loadConfig_Schedulers(schedulers)) # and Sources go after Schedulers for the same reason d.addCallback( lambda res: self.loadConfig_Sources(change_sources)) # self.slavePort if self.slavePortnum != slavePortnum: if self.slavePort: def closeSlavePort(res): d1 = self.slavePort.disownServiceParent() self.slavePort = None return d1 d.addCallback(closeSlavePort) if slavePortnum is not None: def openSlavePort(res): self.slavePort = strports.service( slavePortnum, self.slaveFactory) self.slavePort.setServiceParent(self) d.addCallback(openSlavePort) log.msg("BuildMaster listening on port %s" % slavePortnum) self.slavePortnum = slavePortnum log.msg("configuration update started") def _done(res): self.readConfig = True log.msg("configuration update complete") d.addCallback(_done) d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds()) return d
def __init__(self, number=None): self.properties = Properties() self.number = number self.who = "me"
def addBuildsetForSourceStamps(self, waited_for=False, sourcestamps=None, reason='', external_idstring=None, properties=None, builderNames=None, **kw): if sourcestamps is None: sourcestamps = [] # combine properties if properties: properties.updateFromProperties(self.properties) else: properties = self.properties # make a fresh copy that we actually can modify safely properties = Properties.fromDict(properties.asDict()) # make extra info available from properties.render() properties.master = self.master properties.sourcestamps = [] properties.changes = [] for ss in sourcestamps: if isinstance(ss, integer_types): # fetch actual sourcestamp and changes from data API properties.sourcestamps.append((yield self.master.data.get( ('sourcestamps', ss)))) properties.changes.extend((yield self.master.data.get( ('sourcestamps', ss, 'changes')))) else: # sourcestamp with no change, see addBuildsetForChanges properties.sourcestamps.append(ss) for c in properties.changes: properties.updateFromProperties( Properties.fromDict(c['properties'])) # apply the default builderNames if not builderNames: builderNames = self.builderNames # dynamically get the builder list to schedule builderNames = yield properties.render(builderNames) # Get the builder ids # Note that there is a data.updates.findBuilderId(name) # but that would merely only optimize the single builder case, while # probably the multiple builder case will be severely impacted by the # several db requests needed. builderids = list() for bldr in (yield self.master.data.get(('builders', ))): if bldr['name'] in builderNames: builderids.append(bldr['builderid']) # translate properties object into a dict as required by the # addBuildset method properties_dict = properties.asDict() bsid, brids = yield self.master.data.updates.addBuildset( scheduler=self.name, sourcestamps=sourcestamps, reason=reason, waited_for=waited_for, properties=properties_dict, builderids=builderids, external_idstring=external_idstring, **kw) return (bsid, brids)
def __init__(self, name, builderNames, properties=None, codebases=DEFAULT_CODEBASES): super(BaseScheduler, self).__init__(name=name) ok = True if interfaces.IRenderable.providedBy(builderNames): pass elif isinstance(builderNames, (list, tuple)): for b in builderNames: if not isinstance(b, string_types) and \ not interfaces.IRenderable.providedBy(b): ok = False else: ok = False if not ok: config.error( "The builderNames argument to a scheduler must be a list " "of Builder names or an IRenderable object that will render" "to a list of builder names.") self.builderNames = builderNames if properties is None: properties = {} self.properties = Properties() self.properties.update(properties, "Scheduler") self.properties.setProperty("scheduler", name, "Scheduler") self.objectid = None # Set the codebases that are necessary to process the changes # These codebases will always result in a sourcestamp with or without # changes known_keys = set(['branch', 'repository', 'revision']) if codebases is None: config.error("Codebases cannot be None") elif isinstance(codebases, list): codebases = dict((codebase, {}) for codebase in codebases) elif not isinstance(codebases, dict): config.error( "Codebases must be a dict of dicts, or list of strings") else: for codebase, attrs in codebases.items(): if not isinstance(attrs, dict): config.error("Codebases must be a dict of dicts") else: unk = set(attrs) - known_keys if unk: config.error( "Unknown codebase keys %s for codebase %s" % (', '.join(unk), codebase)) self.codebases = codebases # internal variables self._change_consumer = None self._enable_consumer = None self._change_consumption_lock = defer.DeferredLock() self.enabled = True