def reconfigServiceWithSibling(self, sibling):
# only reconfigure if sibling is configured differently.
# sibling == self is using ComparableMixin's implementation
# only compare compare_attrs
if self.configured and sibling == self:
defer.returnValue(None)
self.configured = True
# render renderables in parallel
# Properties import to resolve cyclic import issue
from buildbot.process.properties import Properties
p = Properties()
p.master = self.master
# render renderables in parallel
secrets = []
kwargs = {}
accumulateClassList(self.__class__, 'secrets', secrets)
for k, v in sibling._config_kwargs.items():
if k in secrets:
value = yield p.render(v)
setattr(self, k, value)
kwargs.update({k: value})
else:
kwargs.update({k: v})
d = yield self.reconfigService(*sibling._config_args,
**sibling._config_kwargs)
defer.returnValue(d)
def getAndCheckProperties(req):
"""
Fetch custom build properties from the HTTP request of a "Force build" or
"Resubmit build" HTML form.
Check the names for valid strings, and return None if a problem is found.
Return a new Properties object containing each property found in req.
"""
master = req.site.buildbot_service.master
pname_validate = master.config.validation['property_name']
pval_validate = master.config.validation['property_value']
properties = Properties()
i = 1
while True:
pname = req.args.get("property%dname" % i, [""])[0]
pvalue = req.args.get("property%dvalue" % i, [""])[0]
if not pname:
break
if not pname_validate.match(pname) \
or not pval_validate.match(pvalue):
log.msg("bad property name='%s', value='%s'" % (pname, pvalue))
return None
properties.setProperty(pname, pvalue, "Force Build Form")
i = i + 1
return properties
def _downloadSshPrivateKeyIfNeeded(self):
if self.sshPrivateKey is None:
defer.returnValue(RC_SUCCESS)
p = Properties()
p.master = self.master
private_key = yield p.render(self.sshPrivateKey)
# not using self.workdir because it may be changed depending on step
# options
workdir = self._getSshDataWorkDir()
rel_key_path = self.build.path_module.relpath(
self._getSshPrivateKeyPath(), workdir)
rel_wrapper_script_path = self.build.path_module.relpath(
self._getSshWrapperScriptPath(), workdir)
yield self.runMkdir(self._getSshDataPath())
if not self.supportsSshPrivateKeyAsEnvOption:
yield self.downloadFileContentToWorker(rel_wrapper_script_path,
self._getSshWrapperScript(),
workdir=workdir, mode=0o700)
yield self.downloadFileContentToWorker(rel_key_path, private_key,
workdir=workdir, mode=0o400)
self.didDownloadSshPrivateKey = True
defer.returnValue(RC_SUCCESS)
class BaseScheduler(service.MultiService, util.ComparableMixin):
"""
A Schduler creates BuildSets and submits them to the BuildMaster.
@ivar name: name of the scheduler
@ivar properties: additional properties specified in this
scheduler's configuration
@type properties: Properties object
"""
implements(interfaces.IScheduler)
def __init__(self, name, properties={}):
"""
@param name: name for this scheduler
@param properties: properties to be propagated from this scheduler
@type properties: dict
"""
service.MultiService.__init__(self)
self.name = name
self.properties = Properties()
self.properties.update(properties, "Scheduler")
self.properties.setProperty("scheduler", name, "Scheduler")
def __repr__(self):
# TODO: why can't id() return a positive number? %d is ugly.
return "<Scheduler '%s' at %d>" % (self.name, id(self))
def submitBuildSet(self, bs):
self.parent.submitBuildSet(bs)
def addChange(self, change):
pass
def _downloadSshPrivateKeyIfNeeded(self):
if self.sshPrivateKey is None:
defer.returnValue(RC_SUCCESS)
p = Properties()
p.master = self.master
private_key = yield p.render(self.sshPrivateKey)
host_key = yield p.render(self.sshHostKey)
# not using self.workdir because it may be changed depending on step
# options
workdir = self._getSshDataWorkDir()
yield self.runMkdir(self._getSshDataPath())
if not self.supportsSshPrivateKeyAsEnvOption:
yield self.downloadFileContentToWorker(self._getSshWrapperScriptPath(),
self._getSshWrapperScript(),
workdir=workdir, mode=0o700)
yield self.downloadFileContentToWorker(self._getSshPrivateKeyPath(),
private_key,
workdir=workdir, mode=0o400)
if self.sshHostKey is not None:
known_hosts_contents = getSshKnownHostsContents(host_key)
yield self.downloadFileContentToWorker(self._getSshHostKeyPath(),
known_hosts_contents,
workdir=workdir, mode=0o400)
self.didDownloadSshPrivateKey = True
defer.returnValue(RC_SUCCESS)
def trigger(self, ssid, set_props=None):
"""Trigger this scheduler with the given sourcestamp ID. Returns a
deferred that will fire when the buildset is finished."""
# properties for this buildset are composed of our own properties,
# potentially overridden by anything from the triggering build
props = Properties()
props.updateFromProperties(self.properties)
if set_props:
props.updateFromProperties(set_props)
# note that this does not use the buildset subscriptions mechanism, as
# the duration of interest to the caller is bounded by the lifetime of
# this process.
if ssid:
d = self.addBuildsetForSourceStamp(reason=self.reason, ssid=ssid,
properties=props)
else:
d = self.addBuildsetForLatest(reason=self.reason, properties=props)
def setup_waiter((bsid,brids)):
d = defer.Deferred()
self._waiters[bsid] = (d, brids)
self._updateWaiters()
return d
d.addCallback(setup_waiter)
return d
def perspective_try(self, branch, revision, patch, builderNames, properties={}):
log.msg("user %s requesting build on builders %s" % (self.username,
builderNames))
for b in builderNames:
if not b in self.parent.builderNames:
log.msg("%s got job with builder %s" % (self, b))
log.msg(" but that wasn't in our list: %s"
% (self.parent.builderNames,))
return
ss = SourceStamp(branch, revision, patch)
reason = "'try' job from user %s" % self.username
# roll the specified props in with our inherited props
combined_props = Properties()
combined_props.updateFromProperties(self.parent.properties)
combined_props.update(properties, "try build")
bs = buildset.BuildSet(builderNames,
ss,
reason=reason,
properties=combined_props)
self.parent.submitBuildSet(bs)
# return a remotely-usable BuildSetStatus object
from buildbot.status.client import makeRemote
return makeRemote(bs.status)
class Try_Userpass(TryBase):
compare_attrs = ( 'name', 'builderNames', 'port', 'userpass', 'properties' )
def __init__(self, name, builderNames, port, userpass,
properties={}):
base.BaseScheduler.__init__(self, name, builderNames, properties)
self.port = port
self.userpass = userpass
self.properties = Properties()
self.properties.update(properties, 'Scheduler')
def startService(self):
TryBase.startService(self)
master = self.parent.parent
# register each user/passwd with the pbmanager
def factory(mind, username):
return Try_Userpass_Perspective(self, username)
self.registrations = []
for user, passwd in self.userpass:
self.registrations.append(
master.pbmanager.register(self.port, user, passwd, factory))
def stopService(self):
d = defer.maybeDeferred(TryBase.stopService, self)
def unreg(_):
return defer.gatherResults(
[ reg.unregister() for reg in self.registrations ])
d.addCallback(unreg)
def testBasic(self):
s = transfer.JSONPropertiesDownload("props.json")
s.build = Mock()
props = Properties()
props.setProperty('key1', 'value1', 'test')
s.build.getProperties.return_value = props
s.build.getSlaveCommandVersion.return_value = 1
ss = Mock()
ss.asDict.return_value = dict(revision="12345")
s.build.getSourceStamp.return_value = ss
s.step_status = Mock()
s.buildslave = Mock()
s.remote = Mock()
s.start()
for c in s.remote.method_calls:
name, command, args = c
commandName = command[3]
kwargs = command[-1]
if commandName == 'downloadFile':
self.assertEquals(kwargs['slavedest'], 'props.json')
reader = kwargs['reader']
data = reader.remote_read(100)
self.assertEquals(data, json.dumps(dict(sourcestamp=ss.asDict(), properties={'key1': 'value1'})))
break
else:
self.assert_(False, "No downloadFile command found")
class TestInterpolatePositional(unittest.TestCase):
def setUp(self):
self.props = Properties()
self.build = FakeBuild(self.props)
def test_string(self):
command = Interpolate("test %s", "one fish")
d = self.build.render(command)
d.addCallback(self.failUnlessEqual,
"test one fish")
def test_twoString(self):
command = Interpolate("test %s, %s", "one fish", "two fish")
d = self.build.render(command)
d.addCallback(self.failUnlessEqual,
"test one fish, two fish")
def test_deferred(self):
renderable = DeferredRenderable()
command = Interpolate("echo '%s'", renderable)
d = self.build.render(command)
d.addCallback(self.failUnlessEqual,
"echo 'red fish'")
renderable.callback("red fish")
return d
def test_renderable(self):
self.props.setProperty("buildername", "blue fish", "test")
command = Interpolate("echo '%s'", Property("buildername"))
d = self.build.render(command)
d.addCallback(self.failUnlessEqual,
"echo 'blue fish'")
return d
def testBasic(self):
s = transfer.JSONPropertiesDownload("props.json")
s.build = Mock()
props = Properties()
props.setProperty('key1', 'value1', 'test')
s.build.getProperties.return_value = props
s.build.getWorkerCommandVersion.return_value = '3.0'
ss = Mock()
ss.asDict.return_value = dict(revision="12345")
s.build.getAllSourceStamps.return_value = [ss]
s.worker = Mock()
s.remote = Mock()
s.start()
for c in s.remote.method_calls:
name, command, args = c
commandName = command[3]
kwargs = command[-1]
if commandName == 'downloadFile':
self.assertEquals(kwargs['workerdest'], 'props.json')
reader = kwargs['reader']
data = reader.remote_read(100)
self.assertEquals(
data, json.dumps(dict(sourcestamps=[ss.asDict()], properties={'key1': 'value1'})))
break
else:
raise ValueError("No downloadFile command found")
def perspective_try(self, branch, revision, patch, repository, project,
builderNames, who="", comment="", properties=None):
log.msg("user %s requesting build on builders %s" % (self.username,
builderNames))
if properties is None:
properties = {}
# build the intersection of the request and our configured list
builderNames = self.scheduler.filterBuilderList(builderNames)
if not builderNames:
return
reason = u"'try' job"
if who:
reason += u" by user %s" % ascii2unicode(who)
if comment:
reason += u" (%s)" % ascii2unicode(comment)
sourcestamp = dict(
branch=branch, revision=revision, repository=repository,
project=project, patch_level=patch[0], patch_body=patch[1],
patch_subdir='', patch_author=who or '',
patch_comment=comment or '', codebase='',
) # note: no way to specify patch subdir - #1769
requested_props = Properties()
requested_props.update(properties, "try build")
(bsid, brids) = yield self.scheduler.addBuildsetForSourceStamps(
sourcestamps=[sourcestamp], reason=reason,
properties=requested_props, builderNames=builderNames)
# return a remotely-usable BuildSetStatus object
bss = RemoteBuildSetStatus(self.scheduler.master, bsid, brids)
defer.returnValue(bss)
def reconfigServiceWithSibling(self, sibling):
# only reconfigure if sibling is configured differently.
# sibling == self is using ComparableMixin's implementation
# only compare compare_attrs
if self.configured and sibling == self:
return None
self.configured = True
# render renderables in parallel
# Properties import to resolve cyclic import issue
from buildbot.process.properties import Properties
p = Properties()
p.master = self.master
# render renderables in parallel
secrets = []
kwargs = {}
accumulateClassList(self.__class__, 'secrets', secrets)
for k, v in sibling._config_kwargs.items():
if k in secrets:
# for non reconfigurable services, we force the attribute
v = yield p.render(v)
setattr(sibling, k, v)
setattr(self, k, v)
kwargs[k] = v
d = yield self.reconfigService(*sibling._config_args,
**kwargs)
return d
def canStartWithWorkerForBuilder(self, workerforbuilder, buildrequests=None):
locks = self.config.locks
if IRenderable.providedBy(locks):
if buildrequests is None:
raise RuntimeError("buildrequests parameter must be specified "
" when using renderable builder locks. Not "
"specifying buildrequests is deprecated")
# collect properties that would be set for a build if we
# started it now and render locks using it
props = Properties()
Build.setupPropertiesKnownBeforeBuildStarts(props, buildrequests,
self, workerforbuilder)
locks = yield props.render(locks)
# Make sure we don't warn and throw an exception at the same time
if buildrequests is None:
warnings.warn(
"Not passing corresponding buildrequests to "
"Builder.canStartWithWorkerForBuilder is deprecated")
locks = [(self.botmaster.getLockFromLockAccess(access), access)
for access in locks]
can_start = Build.canStartWithWorkerForBuilder(locks, workerforbuilder)
defer.returnValue(can_start)
def testBasic(self):
s = transfer.JSONPropertiesDownload("props.json")
s.build = Mock()
props = Properties()
props.setProperty("key1", "value1", "test")
s.build.getProperties.return_value = props
s.build.getWorkerCommandVersion.return_value = 1
ss = Mock()
ss.asDict.return_value = dict(revision="12345")
s.build.getAllSourceStamps.return_value = [ss]
s.worker = Mock()
s.remote = Mock()
s.start()
for c in s.remote.method_calls:
name, command, args = c
commandName = command[3]
kwargs = command[-1]
if commandName == "downloadFile":
self.assertEquals(kwargs["slavedest"], "props.json")
reader = kwargs["reader"]
data = reader.remote_read(100)
self.assertEquals(data, json.dumps(dict(sourcestamps=[ss.asDict()], properties={"key1": "value1"})))
break
else:
raise ValueError("No downloadFile command found")
def buildIDSchedFunc(sched, t, ssid):
"""Generates a unique buildid for this change.
Returns a Properties instance with 'buildid' set to the buildid to use.
scheduler `sched`'s state is modified as a result."""
state = sched.get_state(t)
# Get the last buildid we scheduled from the database
lastid = state.get('last_buildid', '19700101000000')
incrementedid = incrementBuildID(lastid)
nowid = genBuildID()
# Our new buildid will be the highest of the last buildid incremented or
# the buildid based on the current date
newid = str(max(int(nowid), int(incrementedid)))
# Save it in the scheduler's state so we don't generate the same one again.
state['last_buildid'] = newid
sched.set_state(t, state)
props = Properties()
props.setProperty('buildid', newid, 'buildIDSchedFunc')
return props
def testBasicWorker2_16(self):
s = transfer.JSONPropertiesDownload("props.json")
s.build = Mock()
props = Properties()
props.setProperty('key1', 'value1', 'test')
s.build.getProperties.return_value = props
s.build.getWorkerCommandVersion.return_value = '2.16'
ss = Mock()
ss.asDict.return_value = dict(revision="12345")
s.build.getAllSourceStamps.return_value = [ss]
s.worker = Mock()
s.remote = Mock()
s.start()
for c in s.remote.method_calls:
name, command, args = c
commandName = command[3]
kwargs = command[-1]
if commandName == 'downloadFile':
self.assertEqual(kwargs['slavedest'], 'props.json')
reader = kwargs['reader']
data = reader.remote_read(100)
data = bytes2unicode(data)
actualJson = json.loads(data)
expectedJson = dict(sourcestamps=[ss.asDict()], properties={'key1': 'value1'})
self.assertEqual(actualJson, expectedJson)
break
else:
raise ValueError("No downloadFile command found")
def perspective_requestBuild(self, buildername, reason, branch, revision, properties={}):
c = interfaces.IControl(self.master)
bc = c.getBuilder(buildername)
ss = SourceStamp(branch, revision)
bpr = Properties()
bpr.update(properties, "remote requestBuild")
return bc.submitBuildRequest(ss, reason, bpr)
def canStartBuild(self, workerforbuilder, buildrequest):
can_start = True
# check whether the locks that the build will acquire can actually be
# acquired
locks = self.config.locks
if IRenderable.providedBy(locks):
# collect properties that would be set for a build if we
# started it now and render locks using it
props = Properties()
Build.setupPropertiesKnownBeforeBuildStarts(props, [buildrequest],
self, workerforbuilder)
locks = yield props.render(locks)
locks = [(self.botmaster.getLockFromLockAccess(access), access)
for access in locks]
if locks:
can_start = Build._canAcquireLocks(locks, workerforbuilder)
if can_start is False:
defer.returnValue(can_start)
if callable(self.config.canStartBuild):
can_start = yield self.config.canStartBuild(self, workerforbuilder,
buildrequest)
defer.returnValue(can_start)
def trigger(self, waited_for, sourcestamps=None, set_props=None,
parent_buildid=None, parent_relationship=None):
"""Trigger this scheduler with the optional given list of sourcestamps
Returns two deferreds:
idsDeferred -- yields the ids of the buildset and buildrequest, as soon as they are available.
resultsDeferred -- yields the build result(s), when they finish."""
# properties for this buildset are composed of our own properties,
# potentially overridden by anything from the triggering build
props = Properties()
props.updateFromProperties(self.properties)
if set_props:
props.updateFromProperties(set_props)
# note that this does not use the buildset subscriptions mechanism, as
# the duration of interest to the caller is bounded by the lifetime of
# this process.
idsDeferred = self.addBuildsetForSourceStampsWithDefaults(
self.reason,
sourcestamps, waited_for,
properties=props,
parent_buildid=parent_buildid,
parent_relationship=parent_relationship)
resultsDeferred = defer.Deferred()
@idsDeferred.addCallback
def setup_waiter(ids):
bsid, brids = ids
self._waiters[bsid] = (resultsDeferred, brids)
self._updateWaiters()
return ids
return idsDeferred, resultsDeferred
def verifyCode(self, code):
# everything in deferToThread is not counted with trial --coverage :-(
def thd(client_id, client_secret):
url = self.tokenUri
data = {'redirect_uri': self.loginUri, 'code': code,
'grant_type': self.grantType}
auth = None
if self.getTokenUseAuthHeaders:
auth = (client_id, client_secret)
else:
data.update(
{'client_id': client_id, 'client_secret': client_secret})
data.update(self.tokenUriAdditionalParams)
response = requests.post(
url, data=data, auth=auth, verify=self.sslVerify)
response.raise_for_status()
responseContent = bytes2unicode(response.content)
try:
content = json.loads(responseContent)
except ValueError:
content = parse_qs(responseContent)
for k, v in content.items():
content[k] = v[0]
except TypeError:
content = responseContent
session = self.createSessionFromToken(content)
return self.getUserInfoFromOAuthClient(session)
p = Properties()
p.master = self.master
client_id = yield p.render(self.clientId)
client_secret = yield p.render(self.clientSecret)
result = yield threads.deferToThread(thd, client_id, client_secret)
return result
def TryJobBaseGetProps(self, builder, options):
""" Override of try_job_base.TryJobBase.get_props:
http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/master/try_job_base.py?view=markup
We modify it to add "baseurl".
"""
keys = (
############################### Added by borenet ###############################
'baseurl',
################################################################################
'clobber',
'issue',
'patchset',
'requester',
'rietveld',
'root',
'try_job_key',
)
# All these settings have no meaning when False or not set, so don't set
# them in that case.
properties = dict((i, options[i]) for i in keys if options.get(i))
properties['testfilter'] = options['bot'].get(builder, None)
# pylint: disable=W0212
props = Properties()
props.updateFromProperties(self.properties)
props.update(properties, self._PROPERTY_SOURCE)
return props
def perspective_try(self, branch, revision, patch, repository, project,
builderNames, properties={}, ):
db = self.scheduler.master.db
log.msg("user %s requesting build on builders %s" % (self.username,
builderNames))
# build the intersection of the request and our configured list
builderNames = self.scheduler.filterBuilderList(builderNames)
if not builderNames:
return
wfd = defer.waitForDeferred(
db.sourcestamps.createSourceStamp(branch=branch, revision=revision,
repository=repository, project=project, patch_level=patch[0],
patch_body=patch[1], patch_subdir=''))
# note: no way to specify patch subdir - #1769
yield wfd
ssid = wfd.getResult()
reason = "'try' job from user %s" % self.username
requested_props = Properties()
requested_props.update(properties, "try build")
wfd = defer.waitForDeferred(
self.scheduler.addBuildsetForSourceStamp(ssid=ssid,
reason=reason, properties=requested_props,
builderNames=builderNames))
yield wfd
bsid = wfd.getResult()
# return a remotely-usable BuildSetStatus object
bss = BuildSetStatus(bsid, self.scheduler.master.status, db)
from buildbot.status.client import makeRemote
r = makeRemote(bss)
yield r # return value
def perspective_try(self, branch, revision, patch, repository, project,
builderNames, properties={}, ):
log.msg("user %s requesting build on builders %s" % (self.username,
builderNames))
# build the intersection of the request and our configured list
builderNames = self.parent.filterBuilderList(builderNames)
if not builderNames:
return
ss = SourceStamp(branch, revision, patch, repository=repository,
project=project)
reason = "'try' job from user %s" % self.username
# roll the specified props in with our inherited props
combined_props = Properties()
combined_props.updateFromProperties(self.parent.properties)
combined_props.update(properties, "try build")
status = self.parent.parent.parent.status
db = self.parent.parent.db
d = db.runInteraction(self._try, ss, builderNames, reason,
combined_props, db)
def _done(bsid):
# return a remotely-usable BuildSetStatus object
bss = BuildSetStatus(bsid, status, db)
from buildbot.status.client import makeRemote
r = makeRemote(bss)
#self.parent.parent.loop_done() # so it will notify builder loop
return r
d.addCallback(_done)
return d
def MockBuild(my_builder, buildsetup, mastername, slavename, basepath=None,
build_properties=None, slavedir=None):
"""Given a builder object and configuration, mock a Buildbot setup around it.
This sets up a mock BuildMaster, BuildSlave, Build, BuildStatus, and all other
superstructure required for BuildSteps inside the provided builder to render
properly. These BuildSteps are returned to the user in an array. It
additionally returns the build object (in order to get its properties if
desired).
buildsetup is passed straight into the FakeSource's init method and
contains sourcestamp information (revision, branch, etc).
basepath is the directory of the build (what goes under build/slave/, for
example 'Chromium_Linux_Builder'. It is nominally inferred from the builder
name, but it can be overridden. This is useful when pointing the buildrunner
at a different builder than what it's running under.
build_properties will update and override build_properties after all
builder-derived defaults have been set.
"""
my_factory = my_builder['factory']
steplist = ListSteps(my_factory)
build = base.Build([FakeRequest(buildsetup)])
safename = buildbot.util.safeTranslate(my_builder['name'])
if not basepath: basepath = safename
if not slavedir: slavedir = os.path.join(SCRIPT_DIR,
'..', '..', 'slave')
basedir = os.path.join(slavedir, basepath)
build.basedir = basedir
builderstatus = builder.BuilderStatus('test')
builderstatus.nextBuildNumber = 2
builderstatus.basedir = basedir
my_builder['builddir'] = safename
my_builder['slavebuilddir'] = safename
mybuilder = real_builder.Builder(my_builder, builderstatus)
build.setBuilder(mybuilder)
build_status = build_module.BuildStatus(builderstatus, 1)
build_status.setProperty('blamelist', [], 'Build')
build_status.setProperty('mastername', mastername, 'Build')
build_status.setProperty('slavename', slavename, 'Build')
build_status.setProperty('gtest_filter', [], 'Build')
# if build_properties are passed in, overwrite the defaults above:
buildprops = Properties()
if build_properties:
buildprops.update(build_properties, 'Botmaster')
mybuilder.setBotmaster(FakeBotmaster(mastername, buildprops))
buildslave = FakeSlave(safename, slavename)
build.build_status = build_status
build.setupSlaveBuilder(buildslave)
build.setupProperties()
process_steps(steplist, build, buildslave, build_status, basedir)
return steplist, build
def addBuildsetForSourceStamps(self, waited_for=False, sourcestamps=None,
reason='', external_idstring=None, properties=None,
builderNames=None, **kw):
if sourcestamps is None:
sourcestamps = []
# combine properties
if properties:
properties.updateFromProperties(self.properties)
else:
properties = self.properties
# make a fresh copy that we actually can modify safely
properties = Properties.fromDict(properties.asDict())
# make extra info available from properties.render()
properties.master = self.master
properties.sourcestamps = []
properties.changes = []
for ss in sourcestamps:
if isinstance(ss, int):
# fetch actual sourcestamp and changes from data API
properties.sourcestamps.append(
(yield self.master.data.get(('sourcestamps', ss))))
properties.changes.extend(
(yield self.master.data.get(('sourcestamps', ss, 'changes'))))
else:
# sourcestamp with no change, see addBuildsetForChanges
properties.sourcestamps.append(ss)
for c in properties.changes:
properties.updateFromProperties(Properties.fromDict(c['properties']))
# apply the default builderNames
if not builderNames:
builderNames = self.builderNames
# dynamically get the builder list to schedule
builderNames = yield properties.render(builderNames)
# Get the builder ids
# Note that there is a data.updates.findBuilderId(name)
# but that would merely only optimize the single builder case, while
# probably the multiple builder case will be severely impacted by the
# several db requests needed.
builderids = list()
for bldr in (yield self.master.data.get(('builders', ))):
if bldr['name'] in builderNames:
builderids.append(bldr['builderid'])
# translate properties object into a dict as required by the
# addBuildset method
properties_dict = properties.asDict()
bsid, brids = yield self.master.data.updates.addBuildset(
scheduler=self.name, sourcestamps=sourcestamps, reason=reason,
waited_for=waited_for, properties=properties_dict, builderids=builderids,
external_idstring=external_idstring, **kw)
return (bsid, brids)
def createTriggerProperties(self):
# make a new properties object from a dict rendered by the old
# properties object
trigger_properties = Properties()
trigger_properties.update(self.set_properties, "Trigger")
for prop in trigger_properties.asDict():
if prop not in self.set_properties:
self.set_properties[prop] = Property(prop)
return trigger_properties
def getProperties(self, req):
properties = Properties()
for i in itertools.count(0):
pname = req.args.get("property%dname" % i, [""])[0]
pvalue = req.args.get("property%dvalue" % i, [""])[0]
if not pname or not pvalue:
break
properties.setProperty(pname, pvalue, "Custom Force Build Form")
return [properties, None]
def renderSecrets(self, *args):
# Properties import to resolve cyclic import issue
from buildbot.process.properties import Properties
p = Properties()
p.master = self.master
if len(args) == 1:
return p.render(args[0])
return defer.gatherResults([p.render(s) for s in args], consumeErrors=True)
def requestBuild(self, builder, locale):
# returns a Deferred that fires with an IBuildStatus object when the
# build is finished
props = Properties()
props.setProperty('locale', locale, 'scheduler')
props.setProperty('tree', 'app', 'scheduler')
req = BuildRequest("forced build", SourceStamp(), builder,
properties=props)
self.control.getBuilder(builder).requestBuild(req)
return req.waitUntilFinished()
def MockBuild(my_builder,
buildsetup,
mastername,
slavename,
basepath=None,
build_properties=None,
slavedir=None):
"""Given a builder object and configuration, mock a Buildbot setup around it.
This sets up a mock BuildMaster, BuildSlave, Build, BuildStatus, and all other
superstructure required for BuildSteps inside the provided builder to render
properly. These BuildSteps are returned to the user in an array. It
additionally returns the build object (in order to get its properties if
desired).
buildsetup is passed straight into the FakeSource's init method and
contains sourcestamp information (revision, branch, etc).
basepath is the directory of the build (what goes under build/slave/, for
example 'Chromium_Linux_Builder'. It is nominally inferred from the builder
name, but it can be overridden. This is useful when pointing the buildrunner
at a different builder than what it's running under.
build_properties will update and override build_properties after all
builder-derived defaults have been set.
"""
my_factory = my_builder['factory']
steplist = ListSteps(my_factory)
build = base.Build([FakeRequest(buildsetup)])
safename = buildbot.util.safeTranslate(my_builder['name'])
my_builder.setdefault('builddir', safename)
my_builder.setdefault('slavebuilddir', my_builder['builddir'])
workdir_root = None
if not slavedir:
workdir_root = os.path.join(SCRIPT_DIR, '..', '..', 'slave',
my_builder['slavebuilddir'])
if not basepath: basepath = safename
if not slavedir: slavedir = os.path.join(SCRIPT_DIR, '..', '..', 'slave')
basedir = os.path.join(slavedir, basepath)
build.basedir = basedir
if not workdir_root:
workdir_root = basedir
builderstatus = builder.BuilderStatus('test')
builderstatus.basedir = basedir
buildnumber = build_properties.get('buildnumber', 1)
builderstatus.nextBuildNumber = buildnumber + 1
mybuilder = real_builder.Builder(my_builder, builderstatus)
build.setBuilder(mybuilder)
build_status = build_module.BuildStatus(builderstatus, buildnumber)
build_status.setProperty('blamelist', [], 'Build')
build_status.setProperty('mastername', mastername, 'Build')
build_status.setProperty('slavename', slavename, 'Build')
build_status.setProperty('gtest_filter', [], 'Build')
build_status.setProperty('extra_args', [], 'Build')
build_status.setProperty('build_id', buildnumber, 'Build')
# if build_properties are passed in, overwrite the defaults above:
buildprops = Properties()
if build_properties:
buildprops.update(build_properties, 'Botmaster')
mybuilder.setBotmaster(FakeBotmaster(mastername, buildprops))
buildslave = FakeSlave(safename, my_builder.get('slavebuilddir'),
slavename)
build.build_status = build_status
build.setupSlaveBuilder(buildslave)
build.setupProperties()
process_steps(steplist, build, buildslave, build_status, workdir_root)
return steplist, build
class TestWithProperties(unittest.TestCase):
def setUp(self):
self.props = Properties()
def testBasic(self):
# test basic substitution with WithProperties
self.props.setProperty("revision", "47", "test")
command = WithProperties("build-%s.tar.gz", "revision")
self.failUnlessEqual(self.props.render(command), "build-47.tar.gz")
def testDict(self):
# test dict-style substitution with WithProperties
self.props.setProperty("other", "foo", "test")
command = WithProperties("build-%(other)s.tar.gz")
self.failUnlessEqual(self.props.render(command), "build-foo.tar.gz")
def testDictColonMinus(self):
# test dict-style substitution with WithProperties
self.props.setProperty("prop1", "foo", "test")
command = WithProperties(
"build-%(prop1:-empty)s-%(prop2:-empty)s.tar.gz")
self.failUnlessEqual(self.props.render(command),
"build-foo-empty.tar.gz")
def testDictColonPlus(self):
# test dict-style substitution with WithProperties
self.props.setProperty("prop1", "foo", "test")
command = WithProperties(
"build-%(prop1:+exists)s-%(prop2:+exists)s.tar.gz")
self.failUnlessEqual(self.props.render(command),
"build-exists-.tar.gz")
def testEmpty(self):
# None should render as ''
self.props.setProperty("empty", None, "test")
command = WithProperties("build-%(empty)s.tar.gz")
self.failUnlessEqual(self.props.render(command), "build-.tar.gz")
def testRecursiveList(self):
self.props.setProperty("x", 10, "test")
self.props.setProperty("y", 20, "test")
command = [
WithProperties("%(x)s %(y)s"), "and",
WithProperties("%(y)s %(x)s")
]
self.failUnlessEqual(self.props.render(command),
["10 20", "and", "20 10"])
def testRecursiveTuple(self):
self.props.setProperty("x", 10, "test")
self.props.setProperty("y", 20, "test")
command = (WithProperties("%(x)s %(y)s"), "and",
WithProperties("%(y)s %(x)s"))
self.failUnlessEqual(self.props.render(command),
("10 20", "and", "20 10"))
def testRecursiveDict(self):
self.props.setProperty("x", 10, "test")
self.props.setProperty("y", 20, "test")
command = {
WithProperties("%(x)s %(y)s"): WithProperties("%(y)s %(x)s")
}
self.failUnlessEqual(self.props.render(command), {"10 20": "20 10"})
def testLambdaSubst(self):
command = WithProperties('%(foo)s', foo=lambda _: 'bar')
self.failUnlessEqual(self.props.render(command), 'bar')
def testLambdaOverride(self):
self.props.setProperty('x', 10, 'test')
command = WithProperties('%(x)s', x=lambda _: 20)
self.failUnlessEqual(self.props.render(command), '20')
def testLambdaCallable(self):
self.assertRaises(ValueError,
lambda: WithProperties('%(foo)s', foo='bar'))
def testLambdaUseExisting(self):
self.props.setProperty('x', 10, 'test')
self.props.setProperty('y', 20, 'test')
command = WithProperties('%(z)s', z=lambda pmap: pmap['x'] + pmap['y'])
self.failUnlessEqual(self.props.render(command), '30')
def setUp(self):
self.props = Properties()
def createEmail(self,
msgdict,
builderName,
title,
results,
builds=None,
patches=None,
logs=None):
text = msgdict['body'].encode(ENCODING)
type = msgdict['type']
if 'subject' in msgdict:
subject = msgdict['subject'].encode(ENCODING)
else:
subject = self.subject % {
'result': Results[results],
'projectName': title,
'title': title,
'builder': builderName
}
assert '\n' not in subject, \
"Subject cannot contain newlines"
assert type in ('plain', 'html'), \
"'%s' message type must be 'plain' or 'html'." % type
if patches or logs:
m = MIMEMultipart()
txt = MIMEText(text, type, ENCODING)
m.attach(txt)
else:
m = Message()
m.set_payload(text, ENCODING)
m.set_type("text/%s" % type)
m['Date'] = formatdate(localtime=True)
m['Subject'] = subject
m['From'] = self.fromaddr
# m['To'] is added later
if patches:
for (i, patch) in enumerate(patches):
a = self.patch_to_attachment(patch, i)
m.attach(a)
if logs:
for log in logs:
name = "%s.%s" % (log['stepname'], log['name'])
if (self._shouldAttachLog(log['name'])
or self._shouldAttachLog(name)):
# Use distinct filenames for the e-mail summary
if self.buildSetSummary:
filename = "%s.%s" % (log['buildername'], name)
else:
filename = name
text = log['content']['content']
a = MIMEText(text.encode(ENCODING), _charset=ENCODING)
a.add_header('Content-Disposition',
"attachment",
filename=filename)
m.attach(a)
# @todo: is there a better way to do this?
# Add any extra headers that were requested, doing WithProperties
# interpolation if only one build was given
if self.extraHeaders:
extraHeaders = self.extraHeaders
if len(builds) == 1:
props = Properties.fromDict(builds[0]['properties'])
extraHeaders = yield props.render(extraHeaders)
for k, v in iteritems(extraHeaders):
if k in m:
twlog.msg("Warning: Got header " + k +
" in self.extraHeaders "
"but it already exists in the Message - "
"not adding it.")
m[k] = v
defer.returnValue(m)
def __init__(self,
name,
password,
max_builds=None,
notify_on_missing=[],
missing_timeout=3600,
properties={},
locks=None,
keepalive_interval=3600):
"""
@param name: botname this machine will supply when it connects
@param password: password this machine will supply when
it connects
@param max_builds: maximum number of simultaneous builds that will
be run concurrently on this buildslave (the
default is None for no limit)
@param properties: properties that will be applied to builds run on
this slave
@type properties: dictionary
@param locks: A list of locks that must be acquired before this slave
can be used
@type locks: dictionary
"""
service.MultiService.__init__(self)
self.slavename = name
self.password = password
# PB registration
self.registration = None
self.registered_port = None
# these are set when the service is started, and unset when it is
# stopped
self.botmaster = None
self.master = None
self.slave_status = SlaveStatus(name)
self.slave = None # a RemoteReference to the Bot, when connected
self.slave_commands = None
self.slavebuilders = {}
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
self.lock_subscriptions = []
self.properties = Properties()
self.properties.update(properties, "BuildSlave")
self.properties.setProperty("slavename", name, "BuildSlave")
self.lastMessageReceived = 0
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
for i in notify_on_missing:
if not isinstance(i, str):
config.error('notify_on_missing arg %r is not a string' %
(i, ))
self.missing_timeout = missing_timeout
self.missing_timer = None
self.keepalive_interval = keepalive_interval
self.detached_subs = None
self._old_builder_list = None
class BuildRequest:
"""I represent a request to a specific Builder to run a single build.
I have a SourceStamp which specifies what sources I will build. This may
specify a specific revision of the source tree (so source.branch,
source.revision, and source.patch are used). The .patch attribute is
either None or a tuple of (patchlevel, diff), consisting of a number to
use in 'patch -pN', and a unified-format context diff.
Alternatively, the SourceStamp may specify a set of Changes to be built,
contained in source.changes. In this case, I may be mergeable with other
BuildRequests on the same branch.
I may be part of a BuildSet, in which case I will report status results
to it.
I am paired with a BuildRequestStatus object, to which I feed status
information.
@type source: a L{buildbot.sourcestamp.SourceStamp} instance.
@ivar source: the source code that this BuildRequest use
@type reason: string
@ivar reason: the reason this Build is being requested. Schedulers
provide this, but for forced builds the user requesting the
build will provide a string.
@type properties: Properties object
@ivar properties: properties that should be applied to this build
'owner' property is used by Build objects to collect
the list returned by getInterestedUsers
@ivar status: the IBuildStatus object which tracks our status
@ivar submittedAt: a timestamp (seconds since epoch) when this request
was submitted to the Builder. This is used by the CVS
step to compute a checkout timestamp, as well as the
master to prioritize build requests from oldest to
newest.
"""
source = None
builder = None
startCount = 0 # how many times we have tried to start this build
submittedAt = None
implements(interfaces.IBuildRequestControl)
def __init__(self, reason, source, builderName, properties=None):
assert interfaces.ISourceStamp(source, None)
self.reason = reason
self.source = source
self.properties = Properties()
if properties:
self.properties.updateFromProperties(properties)
self.start_watchers = []
self.finish_watchers = []
self.status = BuildRequestStatus(source, builderName)
def canBeMergedWith(self, other):
return self.source.canBeMergedWith(other.source)
def mergeWith(self, others):
return self.source.mergeWith([o.source for o in others])
def mergeReasons(self, others):
"""Return a reason for the merged build request."""
reasons = []
for req in [self] + others:
if req.reason and req.reason not in reasons:
reasons.append(req.reason)
return ", ".join(reasons)
def waitUntilFinished(self):
"""Get a Deferred that will fire (with a
L{buildbot.interfaces.IBuildStatus} instance when the build
finishes."""
d = defer.Deferred()
self.finish_watchers.append(d)
return d
# these are called by the Builder
def requestSubmitted(self, builder):
# the request has been placed on the queue
self.builder = builder
def buildStarted(self, build, buildstatus):
"""This is called by the Builder when a Build has been started in the
hopes of satifying this BuildRequest. It may be called multiple
times, since interrupted builds and lost buildslaves may force
multiple Builds to be run until the fate of the BuildRequest is known
for certain."""
for o in self.start_watchers[:]:
# these observers get the IBuildControl
o(build)
# while these get the IBuildStatus
self.status.buildStarted(buildstatus)
def finished(self, buildstatus):
"""This is called by the Builder when the BuildRequest has been
retired. This happens when its Build has either succeeded (yay!) or
failed (boo!). TODO: If it is halted due to an exception (oops!), or
some other retryable error, C{finished} will not be called yet."""
for w in self.finish_watchers:
w.callback(buildstatus)
self.finish_watchers = []
# IBuildRequestControl
def subscribe(self, observer):
self.start_watchers.append(observer)
def unsubscribe(self, observer):
self.start_watchers.remove(observer)
def cancel(self):
"""Cancel this request. This can only be successful if the Build has
not yet been started.
@return: a boolean indicating if the cancel was successful."""
if self.builder:
return self.builder.cancelBuildRequest(self)
return False
def setSubmitTime(self, t):
self.submittedAt = t
self.status.setSubmitTime(t)
def getSubmitTime(self):
return self.submittedAt
def command_FORCE(self, args):
# FIXME: NEED TO THINK ABOUT!
errReply = "try 'force build [--branch=BRANCH] [--revision=REVISION] [--props=PROP1=VAL1,PROP2=VAL2...] <WHICH> <REASON>'"
args = self.splitArgs(args)
if not args:
raise UsageError(errReply)
what = args.pop(0)
if what != "build":
raise UsageError(errReply)
opts = ForceOptions()
opts.parseOptions(args)
builderName = opts['builder']
branch = opts['branch']
revision = opts['revision']
reason = opts['reason']
props = opts['props']
if builderName is None:
raise UsageError("you must provide a Builder, " + errReply)
# keep weird stuff out of the branch, revision, and properties args.
branch_validate = self.master.config.validation['branch']
revision_validate = self.master.config.validation['revision']
pname_validate = self.master.config.validation['property_name']
pval_validate = self.master.config.validation['property_value']
if branch and not branch_validate.match(branch):
log.msg("bad branch '%s'" % branch)
self.send("sorry, bad branch '%s'" % branch)
return
if revision and not revision_validate.match(revision):
log.msg("bad revision '%s'" % revision)
self.send("sorry, bad revision '%s'" % revision)
return
properties = Properties()
if props:
# split props into name:value dict
pdict = {}
propertylist = props.split(",")
for i in range(0, len(propertylist)):
splitproperty = propertylist[i].split("=", 1)
pdict[splitproperty[0]] = splitproperty[1]
# set properties
for prop in pdict:
pname = prop
pvalue = pdict[prop]
if not pname_validate.match(pname) \
or not pval_validate.match(pvalue):
log.msg("bad property name='%s', value='%s'" %
(pname, pvalue))
self.send("sorry, bad property name='%s', value='%s'" %
(pname, pvalue))
return
properties.setProperty(pname, pvalue, "Force Build chat")
reason = u"forced: by %s: %s" % (self.describeUser(), reason)
d = self.master.data.updates.addBuildset(
builderNames=[builderName],
scheduler=u"status.words", # For now, we just use this as the id.
sourcestamps=[{
'branch': branch,
'revision': revision
}],
reason=reason,
properties=properties.asDict(),
waited_for=False)
@d.addCallback
def subscribe(xxx_todo_changeme):
(bsid, brids) = xxx_todo_changeme
assert 0, "rewrite to not use the status hierarchy" # TODO
# ireq = BuildRequest(self, self.useRevisions)
# buildreq.subscribe(ireq.started)
d.addErrback(log.err, "while forcing a build")
class BaseScheduler(ClusteredBuildbotService, StateMixin):
DEFAULT_CODEBASES = {'': {}}
compare_attrs = ClusteredBuildbotService.compare_attrs + \
('builderNames', 'properties', 'codebases')
def __init__(self,
name,
builderNames,
properties=None,
codebases=DEFAULT_CODEBASES):
super(BaseScheduler, self).__init__(name=name)
ok = True
if interfaces.IRenderable.providedBy(builderNames):
pass
elif isinstance(builderNames, (list, tuple)):
for b in builderNames:
if not isinstance(b, string_types) and \
not interfaces.IRenderable.providedBy(b):
ok = False
else:
ok = False
if not ok:
config.error(
"The builderNames argument to a scheduler must be a list "
"of Builder names or an IRenderable object that will render"
"to a list of builder names.")
self.builderNames = builderNames
if properties is None:
properties = {}
self.properties = Properties()
self.properties.update(properties, "Scheduler")
self.properties.setProperty("scheduler", name, "Scheduler")
self.objectid = None
# Set the codebases that are necessary to process the changes
# These codebases will always result in a sourcestamp with or without
# changes
known_keys = set(['branch', 'repository', 'revision'])
if codebases is None:
config.error("Codebases cannot be None")
elif isinstance(codebases, list):
codebases = dict((codebase, {}) for codebase in codebases)
elif not isinstance(codebases, dict):
config.error(
"Codebases must be a dict of dicts, or list of strings")
else:
for codebase, attrs in codebases.items():
if not isinstance(attrs, dict):
config.error("Codebases must be a dict of dicts")
else:
unk = set(attrs) - known_keys
if unk:
config.error(
"Unknown codebase keys %s for codebase %s" %
(', '.join(unk), codebase))
self.codebases = codebases
# internal variables
self._change_consumer = None
self._enable_consumer = None
self._change_consumption_lock = defer.DeferredLock()
self.enabled = True
def reconfigService(self, *args, **kwargs):
raise NotImplementedError()
# activity handling
@defer.inlineCallbacks
def activate(self):
if not self.enabled:
return None
# even if we aren't called via _activityPoll(), at this point we
# need to ensure the service id is set correctly
if self.serviceid is None:
self.serviceid = yield self._getServiceId()
assert self.serviceid is not None
schedulerData = yield self._getScheduler(self.serviceid)
if schedulerData:
self.enabled = schedulerData['enabled']
if not self._enable_consumer:
yield self.startConsumingEnableEvents()
def _enabledCallback(self, key, msg):
if msg['enabled']:
self.enabled = True
d = self.activate()
else:
d = self.deactivate()
def fn(x):
self.enabled = False
d.addCallback(fn)
return d
@defer.inlineCallbacks
def deactivate(self):
if not self.enabled:
return
yield defer.maybeDeferred(self._stopConsumingChanges)
# service handling
def _getServiceId(self):
return self.master.data.updates.findSchedulerId(self.name)
def _getScheduler(self, sid):
return self.master.db.schedulers.getScheduler(sid)
def _claimService(self):
return self.master.data.updates.trySetSchedulerMaster(
self.serviceid, self.master.masterid)
def _unclaimService(self):
return self.master.data.updates.trySetSchedulerMaster(
self.serviceid, None)
# status queries
# deprecated: these aren't compatible with distributed schedulers
def listBuilderNames(self):
return self.builderNames
# change handling
@defer.inlineCallbacks
def startConsumingChanges(self,
fileIsImportant=None,
change_filter=None,
onlyImportant=False):
assert fileIsImportant is None or callable(fileIsImportant)
# register for changes with the data API
assert not self._change_consumer
self._change_consumer = yield self.master.mq.startConsuming(
lambda k, m: self._changeCallback(k, m, fileIsImportant,
change_filter, onlyImportant),
('changes', None, 'new'))
@defer.inlineCallbacks
def startConsumingEnableEvents(self):
assert not self._enable_consumer
self._enable_consumer = yield self.master.mq.startConsuming(
self._enabledCallback,
('schedulers', str(self.serviceid), 'updated'))
@defer.inlineCallbacks
def _changeCallback(self, key, msg, fileIsImportant, change_filter,
onlyImportant):
# ignore changes delivered while we're not running
if not self._change_consumer:
return
# get a change object, since the API requires it
chdict = yield self.master.db.changes.getChange(msg['changeid'])
change = yield changes.Change.fromChdict(self.master, chdict)
# filter it
if change_filter and not change_filter.filter_change(change):
return
if change.codebase not in self.codebases:
log.msg(format='change contains codebase %(codebase)s that is '
'not processed by scheduler %(name)s',
codebase=change.codebase,
name=self.name)
return
if fileIsImportant:
try:
important = fileIsImportant(change)
if not important and onlyImportant:
return
except Exception:
log.err(failure.Failure(),
'in fileIsImportant check for %s' % change)
return
else:
important = True
# use change_consumption_lock to ensure the service does not stop
# while this change is being processed
d = self._change_consumption_lock.run(self.gotChange, change,
important)
d.addErrback(log.err, 'while processing change')
def _stopConsumingChanges(self):
# (note: called automatically in deactivate)
# acquire the lock change consumption lock to ensure that any change
# consumption is complete before we are done stopping consumption
def stop():
if self._change_consumer:
self._change_consumer.stopConsuming()
self._change_consumer = None
return self._change_consumption_lock.run(stop)
def gotChange(self, change, important):
raise NotImplementedError
# starting builds
@defer.inlineCallbacks
def addBuildsetForSourceStampsWithDefaults(self,
reason,
sourcestamps=None,
waited_for=False,
properties=None,
builderNames=None,
**kw):
if sourcestamps is None:
sourcestamps = []
# convert sourcestamps to a dictionary keyed by codebase
stampsByCodebase = {}
for ss in sourcestamps:
cb = ss['codebase']
if cb in stampsByCodebase:
raise RuntimeError("multiple sourcestamps with same codebase")
stampsByCodebase[cb] = ss
# Merge codebases with the passed list of sourcestamps
# This results in a new sourcestamp for each codebase
stampsWithDefaults = []
for codebase in self.codebases:
cb = yield self.getCodebaseDict(codebase)
ss = {
'codebase': codebase,
'repository': cb.get('repository', ''),
'branch': cb.get('branch', None),
'revision': cb.get('revision', None),
'project': '',
}
# apply info from passed sourcestamps onto the configured default
# sourcestamp attributes for this codebase.
ss.update(stampsByCodebase.get(codebase, {}))
stampsWithDefaults.append(ss)
# fill in any supplied sourcestamps that aren't for a codebase in the
# scheduler's codebase dictionary
for codebase in set(stampsByCodebase) - set(self.codebases):
cb = stampsByCodebase[codebase]
ss = {
'codebase': codebase,
'repository': cb.get('repository', ''),
'branch': cb.get('branch', None),
'revision': cb.get('revision', None),
'project': '',
}
stampsWithDefaults.append(ss)
rv = yield self.addBuildsetForSourceStamps(
sourcestamps=stampsWithDefaults,
reason=reason,
waited_for=waited_for,
properties=properties,
builderNames=builderNames,
**kw)
return rv
def getCodebaseDict(self, codebase):
# Hook for subclasses to change codebase parameters when a codebase does
# not have a change associated with it.
try:
return defer.succeed(self.codebases[codebase])
except KeyError:
return defer.fail()
@defer.inlineCallbacks
def addBuildsetForChanges(self,
waited_for=False,
reason='',
external_idstring=None,
changeids=None,
builderNames=None,
properties=None,
**kw):
if changeids is None:
changeids = []
changesByCodebase = {}
def get_last_change_for_codebase(codebase):
return max(changesByCodebase[codebase],
key=lambda change: change["changeid"])
# Changes are retrieved from database and grouped by their codebase
for changeid in changeids:
chdict = yield self.master.db.changes.getChange(changeid)
changesByCodebase.setdefault(chdict["codebase"], []).append(chdict)
sourcestamps = []
for codebase in sorted(self.codebases):
if codebase not in changesByCodebase:
# codebase has no changes
# create a sourcestamp that has no changes
cb = yield self.getCodebaseDict(codebase)
ss = {
'codebase': codebase,
'repository': cb.get('repository', ''),
'branch': cb.get('branch', None),
'revision': cb.get('revision', None),
'project': '',
}
else:
lastChange = get_last_change_for_codebase(codebase)
ss = lastChange['sourcestampid']
sourcestamps.append(ss)
# add one buildset, using the calculated sourcestamps
bsid, brids = yield self.addBuildsetForSourceStamps(
waited_for,
sourcestamps=sourcestamps,
reason=reason,
external_idstring=external_idstring,
builderNames=builderNames,
properties=properties,
**kw)
return (bsid, brids)
@defer.inlineCallbacks
def addBuildsetForSourceStamps(self,
waited_for=False,
sourcestamps=None,
reason='',
external_idstring=None,
properties=None,
builderNames=None,
**kw):
if sourcestamps is None:
sourcestamps = []
# combine properties
if properties:
properties.updateFromProperties(self.properties)
else:
properties = self.properties
# make a fresh copy that we actually can modify safely
properties = Properties.fromDict(properties.asDict())
# make extra info available from properties.render()
properties.master = self.master
properties.sourcestamps = []
properties.changes = []
for ss in sourcestamps:
if isinstance(ss, integer_types):
# fetch actual sourcestamp and changes from data API
properties.sourcestamps.append((yield self.master.data.get(
('sourcestamps', ss))))
properties.changes.extend((yield self.master.data.get(
('sourcestamps', ss, 'changes'))))
else:
# sourcestamp with no change, see addBuildsetForChanges
properties.sourcestamps.append(ss)
for c in properties.changes:
properties.updateFromProperties(
Properties.fromDict(c['properties']))
# apply the default builderNames
if not builderNames:
builderNames = self.builderNames
# dynamically get the builder list to schedule
builderNames = yield properties.render(builderNames)
# Get the builder ids
# Note that there is a data.updates.findBuilderId(name)
# but that would merely only optimize the single builder case, while
# probably the multiple builder case will be severely impacted by the
# several db requests needed.
builderids = list()
for bldr in (yield self.master.data.get(('builders', ))):
if bldr['name'] in builderNames:
builderids.append(bldr['builderid'])
# translate properties object into a dict as required by the
# addBuildset method
properties_dict = properties.asDict()
bsid, brids = yield self.master.data.updates.addBuildset(
scheduler=self.name,
sourcestamps=sourcestamps,
reason=reason,
waited_for=waited_for,
properties=properties_dict,
builderids=builderids,
external_idstring=external_idstring,
**kw)
return (bsid, brids)
class Change:
"""I represent a single change to the source tree. This may involve several
files, but they are all changed by the same person, and there is a change
comment for the group as a whole."""
implements(interfaces.IStatusEvent)
number = None
branch = None
category = None
revision = None # used to create a source-stamp
links = [] # links are gone, but upgrade code expects this attribute
@classmethod
def fromChdict(cls, master, chdict):
"""
Class method to create a L{Change} from a dictionary as returned
by L{ChangesConnectorComponent.getChange}.
@param master: build master instance
@param ssdict: change dictionary
@returns: L{Change} via Deferred
"""
cache = master.caches.get_cache("Changes", cls._make_ch)
return cache.get(chdict['changeid'], chdict=chdict, master=master)
@classmethod
def _make_ch(cls, changeid, master, chdict):
change = cls(None, None, None, _fromChdict=True)
change.who = chdict['author']
change.comments = chdict['comments']
change.isdir = chdict['is_dir']
change.revision = chdict['revision']
change.branch = chdict['branch']
change.category = chdict['category']
change.revlink = chdict['revlink']
change.repository = chdict['repository']
change.codebase = chdict['codebase']
change.project = chdict['project']
change.number = chdict['changeid']
when = chdict['when_timestamp']
if when:
when = datetime2epoch(when)
change.when = when
change.files = chdict['files'][:]
change.files.sort()
change.properties = Properties()
for n, (v, s) in chdict['properties'].iteritems():
change.properties.setProperty(n, v, s)
return defer.succeed(change)
def __init__(self,
who,
files,
comments,
isdir=0,
revision=None,
when=None,
branch=None,
category=None,
revlink='',
properties={},
repository='',
codebase='',
project='',
_fromChdict=False):
# skip all this madness if we're being built from the database
if _fromChdict:
return
self.who = who
self.comments = comments
self.isdir = isdir
def none_or_unicode(x):
if x is None:
return x
return unicode(x)
self.revision = none_or_unicode(revision)
now = util.now()
if when is None:
self.when = now
elif when > now:
# this happens when the committing system has an incorrect clock, for example.
# handle it gracefully
log.msg(
"received a Change with when > now; assuming the change happened now"
)
self.when = now
else:
self.when = when
self.branch = none_or_unicode(branch)
self.category = none_or_unicode(category)
self.revlink = revlink
self.properties = Properties()
self.properties.update(properties, "Change")
self.repository = repository
self.codebase = codebase
self.project = project
# keep a sorted list of the files, for easier display
self.files = (files or [])[:]
self.files.sort()
def __setstate__(self, dict):
self.__dict__ = dict
# Older Changes won't have a 'properties' attribute in them
if not hasattr(self, 'properties'):
self.properties = Properties()
if not hasattr(self, 'revlink'):
self.revlink = ""
def __str__(self):
return (u"Change(revision=%r, who=%r, branch=%r, comments=%r, " +
u"when=%r, category=%r, project=%r, repository=%r, " +
u"codebase=%r)") % (self.revision, self.who, self.branch,
self.comments, self.when, self.category,
self.project, self.repository,
self.codebase)
def __cmp__(self, other):
return self.number - other.number
def asText(self):
data = ""
data += self.getFileContents()
if self.repository:
data += "On: %s\n" % self.repository
if self.project:
data += "For: %s\n" % self.project
data += "At: %s\n" % self.getTime()
data += "Changed By: %s\n" % self.who
data += "Comments: %s" % self.comments
data += "Properties: \n%s\n\n" % self.getProperties()
return data
def asDict(self):
'''returns a dictonary with suitable info for html/mail rendering'''
result = {}
files = [dict(name=f) for f in self.files]
files.sort(cmp=lambda a, b: a['name'] < b['name'])
# Constant
result['number'] = self.number
result['branch'] = self.branch
result['category'] = self.category
result['who'] = self.getShortAuthor()
result['comments'] = self.comments
result['revision'] = self.revision
result['rev'] = self.revision
result['when'] = self.when
result['at'] = self.getTime()
result['files'] = files
result['revlink'] = getattr(self, 'revlink', None)
result['properties'] = self.properties.asList()
result['repository'] = getattr(self, 'repository', None)
result['codebase'] = getattr(self, 'codebase', '')
result['project'] = getattr(self, 'project', None)
return result
def getShortAuthor(self):
return self.who
def getTime(self):
if not self.when:
return "?"
return time.strftime("%a %d %b %Y %H:%M:%S", time.localtime(self.when))
def getTimes(self):
return (self.when, None)
def getText(self):
return [html.escape(self.who)]
def getLogs(self):
return {}
def getFileContents(self):
data = ""
if len(self.files) == 1:
if self.isdir:
data += "Directory: %s\n" % self.files[0]
else:
data += "File: %s\n" % self.files[0]
else:
data += "Files:\n"
for f in self.files:
data += " %s\n" % f
return data
def getProperties(self):
data = ""
for prop in self.properties.asList():
data += " %s: %s" % (prop[0], prop[1])
return data
def start(self):
properties = self.build.getProperties()
# make a new properties object from a dict rendered by the old
# properties object
props_to_set = Properties()
props_to_set.update(properties.render(self.set_properties), "Trigger")
for p in self.copy_properties:
if p not in properties:
raise RuntimeError(
"copy_property '%s' is not set in the triggering build" %
p)
props_to_set.setProperty(
p, properties[p],
"%s (in triggering build)" % properties.getPropertySource(p))
self.running = True
ss = self.build.getSourceStamp()
if self.updateSourceStamp:
got = properties.getProperty('got_revision')
if got:
ss = ss.getAbsoluteSourceStamp(got)
# (is there an easier way to find the BuildMaster?)
all_schedulers = self.build.builder.botmaster.parent.allSchedulers()
all_schedulers = dict([(sch.name, sch) for sch in all_schedulers])
unknown_schedulers = []
triggered_schedulers = []
# TODO: don't fire any schedulers if we discover an unknown one
for scheduler in self.schedulerNames:
scheduler = properties.render(scheduler)
if all_schedulers.has_key(scheduler):
sch = all_schedulers[scheduler]
if isinstance(sch, Triggerable):
triggered_schedulers.append(scheduler)
else:
unknown_schedulers.append(scheduler)
else:
unknown_schedulers.append(scheduler)
if unknown_schedulers:
self.step_status.setText(['no scheduler:'] + unknown_schedulers)
return self.end(FAILURE)
master = self.build.builder.botmaster.parent # seriously?!
if self.alwaysUseLatest:
d = defer.succeed(None)
else:
d = ss.getSourceStampId(master)
def start_builds(ssid):
dl = []
for scheduler in triggered_schedulers:
sch = all_schedulers[scheduler]
dl.append(sch.trigger(ssid, set_props=props_to_set))
self.step_status.setText(['triggered'] + triggered_schedulers)
d = defer.DeferredList(dl, consumeErrors=1)
if self.waitForFinish:
return d
else:
# do something to handle errors
d.addErrback(
log.err,
'(ignored) while invoking Triggerable schedulers:')
return None
d.addCallback(start_builds)
def cb(rclist):
rc = SUCCESS # (this rc is not the same variable as that above)
for was_cb, results in rclist:
# TODO: make this algo more configurable
if not was_cb:
rc = EXCEPTION
log.err(results)
break
if results == FAILURE:
rc = FAILURE
return self.end(rc)
def eb(why):
return self.end(FAILURE)
if self.waitForFinish:
d.addCallbacks(cb, eb)
d.addErrback(log.err, '(ignored) while triggering builds:')
def __init__(self):
self.sources = []
self.reason = "Because"
self.properties = Properties()
self.id = 9385
class BuildStatus(styles.Versioned):
implements(interfaces.IBuildStatus, interfaces.IStatusEvent)
persistenceVersion = 3
persistenceForgets = ( 'wasUpgraded', )
source = None
reason = None
changes = []
blamelist = []
progress = None
started = None
finished = None
currentStep = None
text = []
results = None
slavename = "???"
# these lists/dicts are defined here so that unserialized instances have
# (empty) values. They are set in __init__ to new objects to make sure
# each instance gets its own copy.
watchers = []
updates = {}
finishedWatchers = []
testResults = {}
def __init__(self, parent, number):
"""
@type parent: L{BuilderStatus}
@type number: int
"""
assert interfaces.IBuilderStatus(parent)
self.builder = parent
self.number = number
self.watchers = []
self.updates = {}
self.finishedWatchers = []
self.steps = []
self.testResults = {}
self.properties = Properties()
def __repr__(self):
return "<%s #%s>" % (self.__class__.__name__, self.number)
# IBuildStatus
def getBuilder(self):
"""
@rtype: L{BuilderStatus}
"""
return self.builder
def getProperty(self, propname):
return self.properties[propname]
def getProperties(self):
return self.properties
def getNumber(self):
return self.number
def getPreviousBuild(self):
if self.number == 0:
return None
return self.builder.getBuild(self.number-1)
def getSourceStamp(self, absolute=False):
if not absolute or not self.properties.has_key('got_revision'):
return self.source
return self.source.getAbsoluteSourceStamp(self.properties['got_revision'])
def getReason(self):
return self.reason
def getChanges(self):
return self.changes
def getResponsibleUsers(self):
return self.blamelist
def getInterestedUsers(self):
# TODO: the Builder should add others: sheriffs, domain-owners
return self.blamelist + self.properties.getProperty('owners', [])
def getSteps(self):
"""Return a list of IBuildStepStatus objects. For invariant builds
(those which always use the same set of Steps), this should be the
complete list, however some of the steps may not have started yet
(step.getTimes()[0] will be None). For variant builds, this may not
be complete (asking again later may give you more of them)."""
return self.steps
def getTimes(self):
return (self.started, self.finished)
_sentinel = [] # used as a sentinel to indicate unspecified initial_value
def getSummaryStatistic(self, name, summary_fn, initial_value=_sentinel):
"""Summarize the named statistic over all steps in which it
exists, using combination_fn and initial_value to combine multiple
results into a single result. This translates to a call to Python's
X{reduce}::
return reduce(summary_fn, step_stats_list, initial_value)
"""
step_stats_list = [
st.getStatistic(name)
for st in self.steps
if st.hasStatistic(name) ]
if initial_value is self._sentinel:
return reduce(summary_fn, step_stats_list)
else:
return reduce(summary_fn, step_stats_list, initial_value)
def isFinished(self):
return (self.finished is not None)
def waitUntilFinished(self):
if self.finished:
d = defer.succeed(self)
else:
d = defer.Deferred()
self.finishedWatchers.append(d)
return d
# while the build is running, the following methods make sense.
# Afterwards they return None
def getETA(self):
if self.finished is not None:
return None
if not self.progress:
return None
eta = self.progress.eta()
if eta is None:
return None
return eta - util.now()
def getCurrentStep(self):
return self.currentStep
# Once you know the build has finished, the following methods are legal.
# Before ths build has finished, they all return None.
def getText(self):
text = []
text.extend(self.text)
for s in self.steps:
text.extend(s.text2)
return text
def getResults(self):
return self.results
def getSlavename(self):
return self.slavename
def getTestResults(self):
return self.testResults
def getTestResultsOrd(self):
trs = self.testResults.keys()
trs.sort()
ret = [ self.testResults[t] for t in trs]
return ret
def getLogs(self):
# TODO: steps should contribute significant logs instead of this
# hack, which returns every log from every step. The logs should get
# names like "compile" and "test" instead of "compile.output"
logs = []
for s in self.steps:
for loog in s.getLogs():
logs.append(loog)
return logs
# subscription interface
def subscribe(self, receiver, updateInterval=None):
# will receive stepStarted and stepFinished messages
# and maybe buildETAUpdate
self.watchers.append(receiver)
if updateInterval is not None:
self.sendETAUpdate(receiver, updateInterval)
def sendETAUpdate(self, receiver, updateInterval):
self.updates[receiver] = None
ETA = self.getETA()
if ETA is not None:
receiver.buildETAUpdate(self, self.getETA())
# they might have unsubscribed during buildETAUpdate
if receiver in self.watchers:
self.updates[receiver] = reactor.callLater(updateInterval,
self.sendETAUpdate,
receiver,
updateInterval)
def unsubscribe(self, receiver):
if receiver in self.watchers:
self.watchers.remove(receiver)
if receiver in self.updates:
if self.updates[receiver] is not None:
self.updates[receiver].cancel()
del self.updates[receiver]
# methods for the base.Build to invoke
def addStepWithName(self, name):
"""The Build is setting up, and has added a new BuildStep to its
list. Create a BuildStepStatus object to which it can send status
updates."""
s = BuildStepStatus(self, len(self.steps))
s.setName(name)
self.steps.append(s)
return s
def setProperty(self, propname, value, source, runtime=True):
self.properties.setProperty(propname, value, source, runtime)
def addTestResult(self, result):
self.testResults[result.getName()] = result
def setSourceStamp(self, sourceStamp):
self.source = sourceStamp
self.changes = self.source.changes
def setReason(self, reason):
self.reason = reason
def setBlamelist(self, blamelist):
self.blamelist = blamelist
def setProgress(self, progress):
self.progress = progress
def buildStarted(self, build):
"""The Build has been set up and is about to be started. It can now
be safely queried, so it is time to announce the new build."""
self.started = util.now()
# now that we're ready to report status, let the BuilderStatus tell
# the world about us
self.builder.buildStarted(self)
def setSlavename(self, slavename):
self.slavename = slavename
def setText(self, text):
assert isinstance(text, (list, tuple))
self.text = text
def setResults(self, results):
self.results = results
def buildFinished(self):
self.currentStep = None
self.finished = util.now()
for r in self.updates.keys():
if self.updates[r] is not None:
self.updates[r].cancel()
del self.updates[r]
watchers = self.finishedWatchers
self.finishedWatchers = []
for w in watchers:
w.callback(self)
# methods called by our BuildStepStatus children
def stepStarted(self, step):
self.currentStep = step
for w in self.watchers:
receiver = w.stepStarted(self, step)
if receiver:
if type(receiver) == type(()):
step.subscribe(receiver[0], receiver[1])
else:
step.subscribe(receiver)
d = step.waitUntilFinished()
d.addCallback(lambda step: step.unsubscribe(receiver))
step.waitUntilFinished().addCallback(self._stepFinished)
def _stepFinished(self, step):
results = step.getResults()
for w in self.watchers:
w.stepFinished(self, step, results)
# methods called by our BuilderStatus parent
def pruneSteps(self):
# this build is very old: remove the build steps too
self.steps = []
# persistence stuff
def generateLogfileName(self, stepname, logname):
"""Return a filename (relative to the Builder's base directory) where
the logfile's contents can be stored uniquely.
The base filename is made by combining our build number, the Step's
name, and the log's name, then removing unsuitable characters. The
filename is then made unique by appending _0, _1, etc, until it does
not collide with any other logfile.
These files are kept in the Builder's basedir (rather than a
per-Build subdirectory) because that makes cleanup easier: cron and
find will help get rid of the old logs, but the empty directories are
more of a hassle to remove."""
starting_filename = "%d-log-%s-%s" % (self.number, stepname, logname)
starting_filename = re.sub(r'[^\w\.\-]', '_', starting_filename)
# now make it unique
unique_counter = 0
filename = starting_filename
while filename in [l.filename
for step in self.steps
for l in step.getLogs()
if l.filename]:
filename = "%s_%d" % (starting_filename, unique_counter)
unique_counter += 1
return filename
def __getstate__(self):
d = styles.Versioned.__getstate__(self)
# for now, a serialized Build is always "finished". We will never
# save unfinished builds.
if not self.finished:
d['finished'] = util.now()
# TODO: push an "interrupted" step so it is clear that the build
# was interrupted. The builder will have a 'shutdown' event, but
# someone looking at just this build will be confused as to why
# the last log is truncated.
for k in 'builder', 'watchers', 'updates', 'finishedWatchers':
if k in d: del d[k]
return d
def __setstate__(self, d):
styles.Versioned.__setstate__(self, d)
# self.builder must be filled in by our parent when loading
for step in self.steps:
step.build = self
self.watchers = []
self.updates = {}
self.finishedWatchers = []
def upgradeToVersion1(self):
if hasattr(self, "sourceStamp"):
# the old .sourceStamp attribute wasn't actually very useful
maxChangeNumber, patch = self.sourceStamp
changes = getattr(self, 'changes', [])
source = sourcestamp.SourceStamp(branch=None,
revision=None,
patch=patch,
changes=changes)
self.source = source
self.changes = source.changes
del self.sourceStamp
self.wasUpgraded = True
def upgradeToVersion2(self):
self.properties = {}
self.wasUpgraded = True
def upgradeToVersion3(self):
# in version 3, self.properties became a Properties object
propdict = self.properties
self.properties = Properties()
self.properties.update(propdict, "Upgrade from previous version")
self.wasUpgraded = True
def upgradeLogfiles(self):
# upgrade any LogFiles that need it. This must occur after we've been
# attached to our Builder, and after we know about all LogFiles of
# all Steps (to get the filenames right).
assert self.builder
for s in self.steps:
for l in s.getLogs():
if l.filename:
pass # new-style, log contents are on disk
else:
logfilename = self.generateLogfileName(s.name, l.name)
# let the logfile update its .filename pointer,
# transferring its contents onto disk if necessary
l.upgrade(logfilename)
def checkLogfiles(self):
# check that all logfiles exist, and remove references to any that
# have been deleted (e.g., by purge())
for s in self.steps:
s.checkLogfiles()
def saveYourself(self):
filename = os.path.join(self.builder.basedir, "%d" % self.number)
if os.path.isdir(filename):
# leftover from 0.5.0, which stored builds in directories
shutil.rmtree(filename, ignore_errors=True)
tmpfilename = filename + ".tmp"
try:
dump(self, open(tmpfilename, "wb"), -1)
if runtime.platformType == 'win32':
# windows cannot rename a file on top of an existing one, so
# fall back to delete-first. There are ways this can fail and
# lose the builder's history, so we avoid using it in the
# general (non-windows) case
if os.path.exists(filename):
os.unlink(filename)
os.rename(tmpfilename, filename)
except:
log.msg("unable to save build %s-#%d" % (self.builder.name,
self.number))
log.err()
def asDict(self):
result = {}
# Constant
result['builderName'] = self.builder.name
result['number'] = self.getNumber()
result['sourceStamp'] = self.getSourceStamp().asDict()
result['reason'] = self.getReason()
result['blame'] = self.getResponsibleUsers()
# Transient
result['properties'] = self.getProperties().asList()
result['times'] = self.getTimes()
result['text'] = self.getText()
result['results'] = self.getResults()
result['slave'] = self.getSlavename()
# TODO(maruel): Add.
#result['test_results'] = self.getTestResults()
result['logs'] = [[l.getName(),
self.builder.status.getURLForThing(l)] for l in self.getLogs()]
result['eta'] = self.getETA()
result['steps'] = [bss.asDict() for bss in self.steps]
if self.getCurrentStep():
result['currentStep'] = self.getCurrentStep().asDict()
else:
result['currentStep'] = None
return result
class FakeChange:
properties = Properties()
def __init__(self, number=None):
self.number = number
self.who = "me"
def upgradeToVersion3(self):
# in version 3, self.properties became a Properties object
propdict = self.properties
self.properties = Properties()
self.properties.update(propdict, "Upgrade from previous version")
self.wasUpgraded = True
class TestProperties(unittest.TestCase):
def setUp(self):
self.props = Properties()
def testDictBehavior(self):
self.props.setProperty("do-tests", 1, "scheduler")
self.props.setProperty("do-install", 2, "scheduler")
self.assert_(self.props.has_key('do-tests'))
self.failUnlessEqual(self.props['do-tests'], 1)
self.failUnlessEqual(self.props['do-install'], 2)
self.assertRaises(KeyError, lambda: self.props['do-nothing'])
self.failUnlessEqual(self.props.getProperty('do-install'), 2)
def testUpdate(self):
self.props.setProperty("x", 24, "old")
newprops = {'a': 1, 'b': 2}
self.props.update(newprops, "new")
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateFromProperties(self):
self.props.setProperty("a", 94, "old")
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new")
newprops.setProperty('b', 2, "new")
self.props.updateFromProperties(newprops)
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateFromPropertiesNoRuntime(self):
self.props.setProperty("a", 94, "old")
self.props.setProperty("b", 84, "old")
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new", runtime=True)
newprops.setProperty('b', 2, "new", runtime=False)
newprops.setProperty('c', 3, "new", runtime=True)
newprops.setProperty('d', 3, "new", runtime=False)
self.props.updateFromPropertiesNoRuntime(newprops)
self.failUnlessEqual(self.props.getProperty('a'), 94)
self.failUnlessEqual(self.props.getPropertySource('a'), 'old')
self.failUnlessEqual(self.props.getProperty('b'), 2)
self.failUnlessEqual(self.props.getPropertySource('b'), 'new')
self.failUnlessEqual(self.props.getProperty('c'), None) # not updated
self.failUnlessEqual(self.props.getProperty('d'), 3)
self.failUnlessEqual(self.props.getPropertySource('d'), 'new')
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
class AbstractBuildSlave(config.ReconfigurableServiceMixin, pb.Avatar,
service.MultiService):
"""This is the master-side representative for a remote buildbot slave.
There is exactly one for each slave described in the config file (the
c['slaves'] list). When buildbots connect in (.attach), they get a
reference to this instance. The BotMaster object is stashed as the
.botmaster attribute. The BotMaster is also our '.parent' Service.
I represent a build slave -- a remote machine capable of
running builds. I am instantiated by the configuration file, and can be
subclassed to add extra functionality."""
implements(IBuildSlave)
keepalive_timer = None
keepalive_interval = None
# reconfig slaves after builders
reconfig_priority = 64
def __init__(self,
name,
password,
max_builds=None,
notify_on_missing=[],
missing_timeout=3600,
properties={},
locks=None,
keepalive_interval=3600):
"""
@param name: botname this machine will supply when it connects
@param password: password this machine will supply when
it connects
@param max_builds: maximum number of simultaneous builds that will
be run concurrently on this buildslave (the
default is None for no limit)
@param properties: properties that will be applied to builds run on
this slave
@type properties: dictionary
@param locks: A list of locks that must be acquired before this slave
can be used
@type locks: dictionary
"""
service.MultiService.__init__(self)
self.slavename = name
self.password = password
# PB registration
self.registration = None
self.registered_port = None
# these are set when the service is started, and unset when it is
# stopped
self.botmaster = None
self.master = None
self.slave_status = SlaveStatus(name)
self.slave = None # a RemoteReference to the Bot, when connected
self.slave_commands = None
self.slavebuilders = {}
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
self.lock_subscriptions = []
self.properties = Properties()
self.properties.update(properties, "BuildSlave")
self.properties.setProperty("slavename", name, "BuildSlave")
self.lastMessageReceived = 0
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
for i in notify_on_missing:
if not isinstance(i, str):
config.error('notify_on_missing arg %r is not a string' %
(i, ))
self.missing_timeout = missing_timeout
self.missing_timer = None
self.keepalive_interval = keepalive_interval
self.detached_subs = None
self._old_builder_list = None
def __repr__(self):
return "<%s %r>" % (self.__class__.__name__, self.slavename)
def updateLocks(self):
"""Convert the L{LockAccess} objects in C{self.locks} into real lock
objects, while also maintaining the subscriptions to lock releases."""
# unsubscribe from any old locks
for s in self.lock_subscriptions:
s.unsubscribe()
# convert locks into their real form
locks = [(self.botmaster.getLockFromLockAccess(a), a)
for a in self.access]
self.locks = [(l.getLock(self), la) for l, la in locks]
self.lock_subscriptions = [
l.subscribeToReleases(self._lockReleased) for l, la in self.locks
]
def locksAvailable(self):
"""
I am called to see if all the locks I depend on are available,
in which I return True, otherwise I return False
"""
if not self.locks:
return True
for lock, access in self.locks:
if not lock.isAvailable(self, access):
return False
return True
def acquireLocks(self):
"""
I am called when a build is preparing to run. I try to claim all
the locks that are needed for a build to happen. If I can't, then
my caller should give up the build and try to get another slave
to look at it.
"""
log.msg("acquireLocks(slave %s, locks %s)" % (self, self.locks))
if not self.locksAvailable():
log.msg("slave %s can't lock, giving up" % (self, ))
return False
# all locks are available, claim them all
for lock, access in self.locks:
lock.claim(self, access)
return True
def releaseLocks(self):
"""
I am called to release any locks after a build has finished
"""
log.msg("releaseLocks(%s): %s" % (self, self.locks))
for lock, access in self.locks:
lock.release(self, access)
def _lockReleased(self):
"""One of the locks for this slave was released; try scheduling
builds."""
if not self.botmaster:
return # oh well..
self.botmaster.maybeStartBuildsForSlave(self.slavename)
def _applySlaveInfo(self, info):
if not info:
return
self.slave_status.setAdmin(info.get("admin"))
self.slave_status.setHost(info.get("host"))
self.slave_status.setAccessURI(info.get("access_uri"))
self.slave_status.setVersion(info.get("version"))
def _saveSlaveInfoDict(self):
slaveinfo = {
'admin': self.slave_status.getAdmin(),
'host': self.slave_status.getHost(),
'access_uri': self.slave_status.getAccessURI(),
'version': self.slave_status.getVersion(),
}
return self.master.db.buildslaves.updateBuildslave(
name=self.slavename,
slaveinfo=slaveinfo,
)
def _getSlaveInfo(self):
d = self.master.db.buildslaves.getBuildslaveByName(self.slavename)
@d.addCallback
def applyInfo(buildslave):
if buildslave is None:
return
self._applySlaveInfo(buildslave.get('slaveinfo'))
return d
def setServiceParent(self, parent):
# botmaster needs to set before setServiceParent which calls startService
self.botmaster = parent
self.master = parent.master
service.MultiService.setServiceParent(self, parent)
def startService(self):
self.updateLocks()
self.startMissingTimer()
d = self._getSlaveInfo()
d.addCallback(lambda _: service.MultiService.startService(self))
return d
@defer.inlineCallbacks
def reconfigService(self, new_config):
# Given a new BuildSlave, configure this one identically. Because
# BuildSlave objects are remotely referenced, we can't replace them
# without disconnecting the slave, yet there's no reason to do that.
new = self.findNewSlaveInstance(new_config)
assert self.slavename == new.slavename
# do we need to re-register?
if (not self.registration or self.password != new.password
or new_config.protocols['pb']['port'] != self.registered_port):
if self.registration:
yield self.registration.unregister()
self.registration = None
self.password = new.password
self.registered_port = new_config.protocols['pb']['port']
self.registration = self.master.pbmanager.register(
self.registered_port, self.slavename, self.password,
self.getPerspective)
# adopt new instance's configuration parameters
self.max_builds = new.max_builds
self.access = new.access
self.notify_on_missing = new.notify_on_missing
self.keepalive_interval = new.keepalive_interval
if self.missing_timeout != new.missing_timeout:
running_missing_timer = self.missing_timer
self.stopMissingTimer()
self.missing_timeout = new.missing_timeout
if running_missing_timer:
self.startMissingTimer()
properties = Properties()
properties.updateFromProperties(new.properties)
self.properties = properties
self.updateLocks()
# update the attached slave's notion of which builders are attached.
# This assumes that the relevant builders have already been configured,
# which is why the reconfig_priority is set low in this class.
yield self.updateSlave()
yield config.ReconfigurableServiceMixin.reconfigService(
self, new_config)
def stopService(self):
if self.registration:
self.registration.unregister()
self.registration = None
self.stopMissingTimer()
return service.MultiService.stopService(self)
def findNewSlaveInstance(self, new_config):
# TODO: called multiple times per reconfig; use 1-element cache?
for sl in new_config.slaves:
if sl.slavename == self.slavename:
return sl
assert 0, "no new slave named '%s'" % self.slavename
def startMissingTimer(self):
if self.notify_on_missing and self.missing_timeout and self.parent:
self.stopMissingTimer() # in case it's already running
self.missing_timer = reactor.callLater(self.missing_timeout,
self._missing_timer_fired)
def stopMissingTimer(self):
if self.missing_timer:
self.missing_timer.cancel()
self.missing_timer = None
def getPerspective(self, mind, slavename):
assert slavename == self.slavename
metrics.MetricCountEvent.log("attached_slaves", 1)
# record when this connection attempt occurred
if self.slave_status:
self.slave_status.recordConnectTime()
# try to use TCP keepalives
try:
mind.broker.transport.setTcpKeepAlive(1)
except:
pass
if self.isConnected():
# duplicate slave - send it to arbitration
arb = botmaster.DuplicateSlaveArbitrator(self)
return arb.getPerspective(mind, slavename)
else:
log.msg("slave '%s' attaching from %s" %
(slavename, mind.broker.transport.getPeer()))
return self
def doKeepalive(self):
self.keepalive_timer = reactor.callLater(self.keepalive_interval,
self.doKeepalive)
if not self.slave:
return
d = self.slave.callRemote("print", "Received keepalive from master")
d.addErrback(log.msg, "Keepalive failed for '%s'" % (self.slavename, ))
def stopKeepaliveTimer(self):
if self.keepalive_timer:
self.keepalive_timer.cancel()
def startKeepaliveTimer(self):
assert self.keepalive_interval
log.msg("Starting buildslave keepalive timer for '%s'" %
(self.slavename, ))
self.doKeepalive()
def isConnected(self):
return self.slave
def _missing_timer_fired(self):
self.missing_timer = None
# notify people, but only if we're still in the config
if not self.parent:
return
buildmaster = self.botmaster.master
status = buildmaster.getStatus()
text = "The Buildbot working for '%s'\n" % status.getTitle()
text += ("has noticed that the buildslave named %s went away\n" %
self.slavename)
text += "\n"
text += ("It last disconnected at %s (buildmaster-local time)\n" %
time.ctime(time.time() - self.missing_timeout)) # approx
text += "\n"
text += "The admin on record (as reported by BUILDSLAVE:info/admin)\n"
text += "was '%s'.\n" % self.slave_status.getAdmin()
text += "\n"
text += "Sincerely,\n"
text += " The Buildbot\n"
text += " %s\n" % status.getTitleURL()
text += "\n"
text += "%s\n" % status.getURLForThing(self.slave_status)
subject = "Buildbot: buildslave %s was lost" % self.slavename
return self._mail_missing_message(subject, text)
def updateSlave(self):
"""Called to add or remove builders after the slave has connected.
@return: a Deferred that indicates when an attached slave has
accepted the new builders and/or released the old ones."""
if self.slave:
return self.sendBuilderList()
else:
return defer.succeed(None)
def updateSlaveStatus(self, buildStarted=None, buildFinished=None):
if buildStarted:
self.slave_status.buildStarted(buildStarted)
if buildFinished:
self.slave_status.buildFinished(buildFinished)
@metrics.countMethod('AbstractBuildSlave.attached()')
def attached(self, bot):
"""This is called when the slave connects.
@return: a Deferred that fires when the attachment is complete
"""
# the botmaster should ensure this.
assert not self.isConnected()
metrics.MetricCountEvent.log("AbstractBuildSlave.attached_slaves", 1)
# set up the subscription point for eventual detachment
self.detached_subs = subscription.SubscriptionPoint("detached")
# now we go through a sequence of calls, gathering information, then
# tell the Botmaster that it can finally give this slave to all the
# Builders that care about it.
# we accumulate slave information in this 'state' dictionary, then
# set it atomically if we make it far enough through the process
state = {}
# Reset graceful shutdown status
self.slave_status.setGraceful(False)
# We want to know when the graceful shutdown flag changes
self.slave_status.addGracefulWatcher(self._gracefulChanged)
d = defer.succeed(None)
@d.addCallback
def _log_attachment_on_slave(res):
d1 = bot.callRemote("print", "attached")
d1.addErrback(lambda why: None)
return d1
@d.addCallback
def _get_info(res):
d1 = bot.callRemote("getSlaveInfo")
def _got_info(info):
log.msg("Got slaveinfo from '%s'" % self.slavename)
# TODO: info{} might have other keys
state["admin"] = info.get("admin")
state["host"] = info.get("host")
state["access_uri"] = info.get("access_uri", None)
state["slave_environ"] = info.get("environ", {})
state["slave_basedir"] = info.get("basedir", None)
state["slave_system"] = info.get("system", None)
def _info_unavailable(why):
why.trap(pb.NoSuchMethod)
# maybe an old slave, doesn't implement remote_getSlaveInfo
log.msg("BuildSlave.info_unavailable")
log.err(why)
d1.addCallbacks(_got_info, _info_unavailable)
return d1
d.addCallback(lambda _: self.startKeepaliveTimer())
@d.addCallback
def _get_version(_):
d = bot.callRemote("getVersion")
def _got_version(version):
state["version"] = version
def _version_unavailable(why):
why.trap(pb.NoSuchMethod)
# probably an old slave
state["version"] = '(unknown)'
d.addCallbacks(_got_version, _version_unavailable)
return d
@d.addCallback
def _get_commands(_):
d1 = bot.callRemote("getCommands")
def _got_commands(commands):
state["slave_commands"] = commands
def _commands_unavailable(why):
# probably an old slave
if why.check(AttributeError):
return
log.msg("BuildSlave.getCommands is unavailable - ignoring")
log.err(why)
d1.addCallbacks(_got_commands, _commands_unavailable)
return d1
@d.addCallback
def _accept_slave(res):
self.slave_status.setConnected(True)
self._applySlaveInfo(state)
self.slave_commands = state.get("slave_commands")
self.slave_environ = state.get("slave_environ")
self.slave_basedir = state.get("slave_basedir")
self.slave_system = state.get("slave_system")
self.slave = bot
if self.slave_system == "nt":
self.path_module = namedModule("ntpath")
else:
# most everything accepts / as separator, so posix should be a
# reasonable fallback
self.path_module = namedModule("posixpath")
log.msg("bot attached")
self.messageReceivedFromSlave()
self.stopMissingTimer()
self.botmaster.master.status.slaveConnected(self.slavename)
d.addCallback(lambda _: self._saveSlaveInfoDict())
d.addCallback(lambda _: self.updateSlave())
d.addCallback(
lambda _: self.botmaster.maybeStartBuildsForSlave(self.slavename))
# Finally, the slave gets a reference to this BuildSlave. They
# receive this later, after we've started using them.
d.addCallback(lambda _: self)
return d
def messageReceivedFromSlave(self):
now = time.time()
self.lastMessageReceived = now
self.slave_status.setLastMessageReceived(now)
def detached(self, mind):
metrics.MetricCountEvent.log("AbstractBuildSlave.attached_slaves", -1)
self.slave = None
self._old_builder_list = []
self.slave_status.removeGracefulWatcher(self._gracefulChanged)
self.slave_status.setConnected(False)
log.msg("BuildSlave.detached(%s)" % self.slavename)
self.botmaster.master.status.slaveDisconnected(self.slavename)
self.stopKeepaliveTimer()
self.releaseLocks()
# notify watchers, but do so in the next reactor iteration so that
# any further detached() action by subclasses happens first
def notif():
subs = self.detached_subs
self.detached_subs = None
subs.deliver()
eventually(notif)
def subscribeToDetach(self, callback):
"""
Request that C{callable} be invoked with no arguments when the
L{detached} method is invoked.
@returns: L{Subscription}
"""
assert self.detached_subs, "detached_subs is only set if attached"
return self.detached_subs.subscribe(callback)
def disconnect(self):
"""Forcibly disconnect the slave.
This severs the TCP connection and returns a Deferred that will fire
(with None) when the connection is probably gone.
If the slave is still alive, they will probably try to reconnect
again in a moment.
This is called in two circumstances. The first is when a slave is
removed from the config file. In this case, when they try to
reconnect, they will be rejected as an unknown slave. The second is
when we wind up with two connections for the same slave, in which
case we disconnect the older connection.
"""
if not self.slave:
return defer.succeed(None)
log.msg("disconnecting old slave %s now" % self.slavename)
# When this Deferred fires, we'll be ready to accept the new slave
return self._disconnect(self.slave)
def _disconnect(self, slave):
# all kinds of teardown will happen as a result of
# loseConnection(), but it happens after a reactor iteration or
# two. Hook the actual disconnect so we can know when it is safe
# to connect the new slave. We have to wait one additional
# iteration (with callLater(0)) to make sure the *other*
# notifyOnDisconnect handlers have had a chance to run.
d = defer.Deferred()
# notifyOnDisconnect runs the callback with one argument, the
# RemoteReference being disconnected.
def _disconnected(rref):
eventually(d.callback, None)
slave.notifyOnDisconnect(_disconnected)
tport = slave.broker.transport
# this is the polite way to request that a socket be closed
tport.loseConnection()
try:
# but really we don't want to wait for the transmit queue to
# drain. The remote end is unlikely to ACK the data, so we'd
# probably have to wait for a (20-minute) TCP timeout.
# tport._closeSocket()
# however, doing _closeSocket (whether before or after
# loseConnection) somehow prevents the notifyOnDisconnect
# handlers from being run. Bummer.
tport.offset = 0
tport.dataBuffer = ""
except:
# however, these hacks are pretty internal, so don't blow up if
# they fail or are unavailable
log.msg("failed to accelerate the shutdown process")
log.msg("waiting for slave to finish disconnecting")
return d
def sendBuilderList(self):
our_builders = self.botmaster.getBuildersForSlave(self.slavename)
blist = [(b.name, b.config.slavebuilddir) for b in our_builders]
if blist == self._old_builder_list:
return defer.succeed(None)
d = self.slave.callRemote("setBuilderList", blist)
def sentBuilderList(ign):
self._old_builder_list = blist
return ign
d.addCallback(sentBuilderList)
return d
def perspective_keepalive(self):
self.messageReceivedFromSlave()
def perspective_shutdown(self):
log.msg("slave %s wants to shut down" % self.slavename)
self.slave_status.setGraceful(True)
def addSlaveBuilder(self, sb):
self.slavebuilders[sb.builder_name] = sb
def removeSlaveBuilder(self, sb):
try:
del self.slavebuilders[sb.builder_name]
except KeyError:
pass
def buildFinished(self, sb):
"""This is called when a build on this slave is finished."""
self.botmaster.maybeStartBuildsForSlave(self.slavename)
def canStartBuild(self):
"""
I am called when a build is requested to see if this buildslave
can start a build. This function can be used to limit overall
concurrency on the buildslave.
Note for subclassers: if a slave can become willing to start a build
without any action on that slave (for example, by a resource in use on
another slave becoming available), then you must arrange for
L{maybeStartBuildsForSlave} to be called at that time, or builds on
this slave will not start.
"""
if self.slave_status.isPaused():
return False
# If we're waiting to shutdown gracefully, then we shouldn't
# accept any new jobs.
if self.slave_status.getGraceful():
return False
if self.max_builds:
active_builders = [
sb for sb in self.slavebuilders.values() if sb.isBusy()
]
if len(active_builders) >= self.max_builds:
return False
if not self.locksAvailable():
return False
return True
def _mail_missing_message(self, subject, text):
# first, see if we have a MailNotifier we can use. This gives us a
# fromaddr and a relayhost.
buildmaster = self.botmaster.master
for st in buildmaster.status:
if isinstance(st, MailNotifier):
break
else:
# if not, they get a default MailNotifier, which always uses SMTP
# to localhost and uses a dummy fromaddr of "buildbot".
log.msg("buildslave-missing msg using default MailNotifier")
st = MailNotifier("buildbot")
# now construct the mail
m = Message()
m.set_payload(text)
m['Date'] = formatdate(localtime=True)
m['Subject'] = subject
m['From'] = st.fromaddr
recipients = self.notify_on_missing
m['To'] = ", ".join(recipients)
d = st.sendMessage(m, recipients)
# return the Deferred for testing purposes
return d
def _gracefulChanged(self, graceful):
"""This is called when our graceful shutdown setting changes"""
self.maybeShutdown()
@defer.inlineCallbacks
def shutdown(self):
"""Shutdown the slave"""
if not self.slave:
log.msg("no remote; slave is already shut down")
return
# First, try the "new" way - calling our own remote's shutdown
# method. The method was only added in 0.8.3, so ignore NoSuchMethod
# failures.
def new_way():
d = self.slave.callRemote('shutdown')
d.addCallback(lambda _: True) # successful shutdown request
def check_nsm(f):
f.trap(pb.NoSuchMethod)
return False # fall through to the old way
d.addErrback(check_nsm)
def check_connlost(f):
f.trap(pb.PBConnectionLost)
return True # the slave is gone, so call it finished
d.addErrback(check_connlost)
return d
if (yield new_way()):
return # done!
# Now, the old way. Look for a builder with a remote reference to the
# client side slave. If we can find one, then call "shutdown" on the
# remote builder, which will cause the slave buildbot process to exit.
def old_way():
d = None
for b in self.slavebuilders.values():
if b.remote:
d = b.remote.callRemote("shutdown")
break
if d:
log.msg("Shutting down (old) slave: %s" % self.slavename)
# The remote shutdown call will not complete successfully since the
# buildbot process exits almost immediately after getting the
# shutdown request.
# Here we look at the reason why the remote call failed, and if
# it's because the connection was lost, that means the slave
# shutdown as expected.
def _errback(why):
if why.check(pb.PBConnectionLost):
log.msg("Lost connection to %s" % self.slavename)
else:
log.err("Unexpected error when trying to shutdown %s" %
self.slavename)
d.addErrback(_errback)
return d
log.err("Couldn't find remote builder to shut down slave")
return defer.succeed(None)
yield old_way()
def maybeShutdown(self):
"""Shut down this slave if it has been asked to shut down gracefully,
and has no active builders."""
if not self.slave_status.getGraceful():
return
active_builders = [
sb for sb in self.slavebuilders.values() if sb.isBusy()
]
if active_builders:
return
d = self.shutdown()
d.addErrback(log.err, 'error while shutting down slave')
def pause(self):
"""Stop running new builds on the slave."""
self.slave_status.setPaused(True)
def unpause(self):
"""Restart running new builds on the slave."""
self.slave_status.setPaused(False)
self.botmaster.maybeStartBuildsForSlave(self.slavename)
def isPaused(self):
return self.paused
class Change:
"""I represent a single change to the source tree. This may involve
several files, but they are all changed by the same person, and there is
a change comment for the group as a whole.
If the version control system supports sequential repository- (or
branch-) wide change numbers (like SVN, P4, and Arch), then revision=
should be set to that number. The highest such number will be used at
checkout time to get the correct set of files.
If it does not (like CVS), when= should be set to the timestamp (seconds
since epoch, as returned by time.time()) when the change was made. when=
will be filled in for you (to the current time) if you omit it, which is
suitable for ChangeSources which have no way of getting more accurate
timestamps.
The revision= and branch= values must be ASCII bytestrings, since they
will eventually be used in a ShellCommand and passed to os.exec(), which
requires bytestrings. These values will also be stored in a database,
possibly as unicode, so they must be safely convertable back and forth.
This restriction may be relaxed in the future.
Changes should be submitted to ChangeMaster.addChange() in
chronologically increasing order. Out-of-order changes will probably
cause the web status displays to be corrupted."""
implements(interfaces.IStatusEvent)
number = None
branch = None
category = None
revision = None # used to create a source-stamp
def __init__(self,
who,
files,
comments,
isdir=0,
links=None,
revision=None,
when=None,
branch=None,
category=None,
revlink='',
properties={},
repository='',
project=''):
self.who = who
self.comments = comments
self.isdir = isdir
if links is None:
links = []
self.links = links
def none_or_unicode(x):
if x is None: return x
return unicode(x)
self.revision = none_or_unicode(revision)
if when is None:
when = util.now()
self.when = when
self.branch = none_or_unicode(branch)
self.category = none_or_unicode(category)
self.revlink = revlink
self.properties = Properties()
self.properties.update(properties, "Change")
self.repository = repository
self.project = project
# keep a sorted list of the files, for easier display
self.files = files[:]
self.files.sort()
def __setstate__(self, dict):
self.__dict__ = dict
# Older Changes won't have a 'properties' attribute in them
if not hasattr(self, 'properties'):
self.properties = Properties()
if not hasattr(self, 'revlink'):
self.revlink = ""
def asText(self):
data = ""
data += self.getFileContents()
if self.repository:
data += "On: %s\n" % self.repository
if self.project:
data += "For: %s\n" % self.project
data += "At: %s\n" % self.getTime()
data += "Changed By: %s\n" % self.who
data += "Comments: %s" % self.comments
data += "Properties: \n%s\n\n" % self.getProperties()
return data
def asDict(self):
'''returns a dictonary with suitable info for html/mail rendering'''
result = {}
files = []
for file in self.files:
link = filter(lambda s: s.find(file) != -1, self.links)
if len(link) == 1:
url = link[0]
else:
url = None
files.append(dict(url=url, name=file))
files = sorted(files, cmp=lambda a, b: a['name'] < b['name'])
# Constant
result['number'] = self.number
result['branch'] = self.branch
result['category'] = self.category
result['who'] = self.getShortAuthor()
result['comments'] = self.comments
result['revision'] = self.revision
result['rev'] = self.revision
result['when'] = self.when
result['at'] = self.getTime()
result['files'] = files
result['revlink'] = getattr(self, 'revlink', None)
result['properties'] = self.properties.asList()
result['repository'] = getattr(self, 'repository', None)
result['project'] = getattr(self, 'project', None)
return result
def getShortAuthor(self):
return self.who
def getTime(self):
if not self.when:
return "?"
return time.strftime("%a %d %b %Y %H:%M:%S", time.localtime(self.when))
def getTimes(self):
return (self.when, None)
def getText(self):
return [html.escape(self.who)]
def getLogs(self):
return {}
def getFileContents(self):
data = ""
if len(self.files) == 1:
if self.isdir:
data += "Directory: %s\n" % self.files[0]
else:
data += "File: %s\n" % self.files[0]
else:
data += "Files:\n"
for f in self.files:
data += " %s\n" % f
return data
def getProperties(self):
data = ""
for prop in self.properties.asList():
data += " %s: %s" % (prop[0], prop[1])
return data
def createTriggerProperties(self, properties):
# make a new properties object from a dict rendered by the old
# properties object
trigger_properties = Properties()
trigger_properties.update(properties, "Trigger")
return trigger_properties
def __init__(self, reason, sources, buildername):
self.reason = reason
self.sources = sources
self.buildername = buildername
self.changes = []
self.properties = Properties()
def setUp(self):
self.patch(openstack, "client", novaclient)
self.patch(openstack, "loading", novaclient)
self.patch(openstack, "session", novaclient)
self.build = Properties(image=novaclient.TEST_UUIDS['image'])
def getProperties(self):
return Properties()
def render_message_subject(self, context):
props = Properties.fromDict(context['build']['properties'])
props.master = context['master']
body = yield props.render(self.subject)
return body
def command_FORCE(self, args, who):
errReply = "try 'force build [--branch=BRANCH] [--revision=REVISION] [--props=PROP1=VAL1,PROP2=VAL2...] <WHICH> <REASON>'"
args = shlex.split(args)
if not args:
raise UsageError(errReply)
what = args.pop(0)
if what != "build":
raise UsageError(errReply)
opts = ForceOptions()
opts.parseOptions(args)
which = opts['builder']
branch = opts['branch']
revision = opts['revision']
reason = opts['reason']
props = opts['props']
if which is None:
raise UsageError("you must provide a Builder, " + errReply)
# keep weird stuff out of the branch, revision, and properties args.
branch_validate = self.master.config.validation['branch']
revision_validate = self.master.config.validation['revision']
pname_validate = self.master.config.validation['property_name']
pval_validate = self.master.config.validation['property_value']
if branch and not branch_validate.match(branch):
log.msg("bad branch '%s'" % branch)
self.send("sorry, bad branch '%s'" % branch)
return
if revision and not revision_validate.match(revision):
log.msg("bad revision '%s'" % revision)
self.send("sorry, bad revision '%s'" % revision)
return
properties = Properties()
if props:
# split props into name:value dict
pdict = {}
propertylist = props.split(",")
for i in range(0,len(propertylist)):
splitproperty = propertylist[i].split("=", 1)
pdict[splitproperty[0]] = splitproperty[1]
# set properties
for prop in pdict:
pname = prop
pvalue = pdict[prop]
if not pname_validate.match(pname) \
or not pval_validate.match(pvalue):
log.msg("bad property name='%s', value='%s'" % (pname, pvalue))
self.send("sorry, bad property name='%s', value='%s'" %
(pname, pvalue))
return
properties.setProperty(pname, pvalue, "Force Build IRC")
bc = self.getControl(which)
reason = "forced: by %s: %s" % (self.describeUser(who), reason)
ss = SourceStamp(branch=branch, revision=revision)
d = bc.submitBuildRequest(ss, reason, props=properties.asDict())
def subscribe(buildreq):
ireq = IrcBuildRequest(self, self.useRevisions)
buildreq.subscribe(ireq.started)
d.addCallback(subscribe)
d.addErrback(log.err, "while forcing a build")
def send(self, build):
props = Properties.fromDict(build['properties'])
if build['complete']:
state = {
SUCCESS: 'success',
WARNINGS: 'success',
FAILURE: 'failure',
SKIPPED: 'success',
EXCEPTION: 'error',
RETRY: 'pending',
CANCELLED: 'error'
}.get(build['results'], 'error')
description = yield props.render(self.endDescription)
else:
state = 'pending'
description = yield props.render(self.startDescription)
context = yield props.render(self.context)
sourcestamps = build['buildset'].get('sourcestamps')
if not sourcestamps or not sourcestamps[0]:
return
project = sourcestamps[0]['project']
if project:
repoOwner, repoName = project.split('/')
else:
repo = sourcestamps[0]['repository'].split('/')[-2:]
repoOwner = repo[0]
repoName = '.'.join(repo[1].split('.')[:-1])
for sourcestamp in sourcestamps:
sha = sourcestamp['revision']
try:
repo_user = repoOwner.encode('utf-8')
repo_user = bytes2NativeString(repo_user, encoding='utf-8')
repo_name = repoName.encode('utf-8')
repo_name = bytes2NativeString(repo_name, encoding='utf-8')
sha = sha.encode('utf-8')
sha = bytes2NativeString(sha, encoding='utf-8')
state = state.encode('utf-8')
state = bytes2NativeString(state, encoding='utf-8')
target_url = build['url'].encode('utf-8')
target_url = bytes2NativeString(target_url, encoding='utf-8')
context = context.encode('utf-8')
context = bytes2NativeString(context, encoding='utf-8')
description = description.encode('utf-8')
description = bytes2NativeString(description, encoding='utf-8')
yield self.createStatus(repo_user=repo_user,
repo_name=repo_name,
sha=sha,
state=state,
target_url=target_url,
context=context,
description=description)
if self.verbose:
log.msg('Status "{state}" sent for '
'{repoOwner}/{repoName} at {sha}.'.format(
state=state,
repoOwner=repoOwner,
repoName=repoName,
sha=sha))
except Exception as e:
log.err(
e, 'Fail to send status "{state}" for '
'{repoOwner}/{repoName} at {sha}'.format(
state=state,
repoOwner=repoOwner,
repoName=repoName,
sha=sha))
def __init__(self, mastername, properties=Properties()):
self.master = FakeMaster(mastername)
self.parent = self
self.properties = properties
def command_FORCE(self, args):
# FIXME: NEED TO THINK ABOUT!
errReply = "try '%s'" % (self.command_FORCE.usage)
args = self.splitArgs(args)
if not args:
raise UsageError(errReply)
what = args.pop(0)
if what != "build":
raise UsageError(errReply)
opts = ForceOptions()
opts.parseOptions(args)
builderName = opts['builder']
builder = yield self.getBuilder(buildername=builderName)
branch = opts['branch']
revision = opts['revision']
codebase = opts['codebase']
project = opts['project']
reason = opts['reason']
props = opts['props']
if builderName is None:
raise UsageError("you must provide a Builder, " + errReply)
# keep weird stuff out of the branch, revision, and properties args.
branch_validate = self.master.config.validation['branch']
revision_validate = self.master.config.validation['revision']
pname_validate = self.master.config.validation['property_name']
pval_validate = self.master.config.validation['property_value']
if branch and not branch_validate.match(branch):
log.msg("bad branch '%s'" % branch)
self.send("sorry, bad branch '%s'" % branch)
return
if revision and not revision_validate.match(revision):
log.msg("bad revision '%s'" % revision)
self.send("sorry, bad revision '%s'" % revision)
return
properties = Properties()
properties.master = self.master
if props:
# split props into name:value dict
pdict = {}
propertylist = props.split(",")
for prop in propertylist:
splitproperty = prop.split("=", 1)
pdict[splitproperty[0]] = splitproperty[1]
# set properties
for prop in pdict:
pname = prop
pvalue = pdict[prop]
if not pname_validate.match(pname) \
or not pval_validate.match(pvalue):
log.msg("bad property name='%s', value='%s'" %
(pname, pvalue))
self.send("sorry, bad property name='%s', value='%s'" %
(pname, pvalue))
return
properties.setProperty(pname, pvalue, "Force Build chat")
reason = "forced: by %s: %s" % (self.describeUser(), reason)
try:
yield self.master.data.updates.addBuildset(builderids=[builder['builderid']],
# For now, we just use
# this as the id.
scheduler="status.words",
sourcestamps=[{
'codebase': codebase, 'branch': branch,
'revision': revision, 'project': project,
'repository': "null"}],
reason=reason,
properties=properties.asDict(),
waited_for=False)
except AssertionError as e:
self.send("I can't: " + str(e))
class JhBuildMaster(BuildMaster):
jhbuild_config = config
def loadConfig(self, f):
# modified from parent method to get slaves, projects, change
# sources, schedulers, builders and web status ouf of
# master.cfg [it would have been cleaner if jhbuild didn't
# have to copy all that code.]
localDict = {'basedir': os.path.expanduser(self.basedir)}
try:
exec f in localDict
except:
log.msg("error while parsing config file")
raise
jhbuild_config.load()
try:
config = localDict['BuildmasterConfig']
except KeyError:
log.err("missing config dictionary")
log.err("config file must define BuildmasterConfig")
raise
known_keys = (
"bots",
"slaves",
"sources",
"change_source",
"schedulers",
"builders",
"mergeRequests",
"slavePortnum",
"debugPassword",
"logCompressionLimit",
"manhole",
"status",
"projectName",
"projectURL",
"buildbotURL",
"properties",
"prioritizeBuilders",
"eventHorizon",
"buildCacheSize",
"logHorizon",
"buildHorizon",
"changeHorizon",
"logMaxSize",
"logMaxTailSize",
"logCompressionMethod",
)
for k in config.keys():
if k not in known_keys:
log.msg(
"unknown key '%s' defined in config dictionary" %
k)
# the 'slaves' list is read from the 'slaves.csv' file in the
# current directory (unless instructed different from command line)
# it is a CSV file structured like this:
# slavename,password
config['slaves'] = []
slaves_csv_file = os.path.join(slaves_dir, 'slaves.csv')
if os.path.exists(slaves_csv_file):
for x in csv.reader(file(slaves_csv_file)):
if not x or x[0].startswith('#'):
continue
kw = {}
build_slave = JhBuildSlave(x[0], x[1])
build_slave.load_extra_configuration(slaves_dir)
config['slaves'].append(build_slave)
if len(config['slaves']) == 0:
log.msg('you must fill slaves.csv with slaves')
module_set = jhbuild.moduleset.load(self.jhbuild_config)
module_list = module_set.get_module_list(
self.jhbuild_config.modules,
self.jhbuild_config.skip,
include_afters=True)
config['projects'] = [x.name for x in module_list \
if not x.name.startswith('meta-')]
if self.jhbuild_config.jhbuildbot_svn_commits_box:
# trigger builds from mails to svn-commit-list
# (note Maildir must be correct, or everything will fail)
from jhbuild.buildbot.changes import GnomeMaildirSource
config['change_source'] = GnomeMaildirSource(
self.jhbuild_config.jhbuildbot_svn_commits_box,
modules=module_list,
prefix=None)
else:
# support injection (use 'buildbot sendchange')
from buildbot.changes.pb import PBChangeSource
config['change_source'] = PBChangeSource()
# Schedulers
from jhbuild.buildbot.scheduler import SerialScheduler, NightlySerialScheduler, OnCommitScheduler
config['schedulers'] = []
for slave in config['slaves']:
s = None
for project in config['projects']:
buildername = str('%s-%s' % (project, slave.slavename))
scheduler_kwargs = {}
if slave.scheduler == 'nightly':
scheduler_class = NightlySerialScheduler
scheduler_kwargs = slave.nightly_kwargs
else:
scheduler_class = SerialScheduler
s = scheduler_class(buildername,
project,
upstream=s,
builderNames=[buildername],
**scheduler_kwargs)
config['schedulers'].append(s)
if self.jhbuild_config.jhbuildbot_svn_commits_box:
# schedulers that will launch job when receiving
# change notifications
s2 = OnCommitScheduler('oc-' + buildername,
project,
builderNames=[buildername])
config['schedulers'].append(s2)
# Builders
from jhbuild.buildbot.factory import JHBuildFactory
config['builders'] = []
for project in config['projects']:
for slave in config['slaves']:
f = JHBuildFactory(project, slave)
config['builders'].append({
'name':
"%s-%s" % (project, slave.slavename),
'slavename':
slave.slavename,
'builddir':
'builddir/%s.%s' % (project, slave.slavename),
'factory':
f,
'category':
project
})
# Status targets
if not config.has_key('status'):
# let it be possible to define additional status in
# master.cfg
config['status'] = []
from jhbuild.buildbot.status.web import JHBuildWebStatus
config['status'].append(
JHBuildWebStatus(self.jhbuild_config.moduleset,
config['projects'],
[x.slavename for x in config['slaves']],
http_port=8080,
allowForce=True))
# remaining of the method is a straight copy from buildbot
# ...
try:
# required
schedulers = config['schedulers']
builders = config['builders']
slavePortnum = config['slavePortnum']
#slaves = config['slaves']
#change_source = config['change_source']
# optional
debugPassword = config.get('debugPassword')
manhole = config.get('manhole')
status = config.get('status', [])
projectName = config.get('projectName')
projectURL = config.get('projectURL')
buildbotURL = config.get('buildbotURL')
properties = config.get('properties', {})
buildCacheSize = config.get('buildCacheSize', None)
eventHorizon = config.get('eventHorizon', None)
logHorizon = config.get('logHorizon', None)
buildHorizon = config.get('buildHorizon', None)
logCompressionLimit = config.get('logCompressionLimit',
4 * 1024)
if logCompressionLimit is not None and not \
isinstance(logCompressionLimit, int):
raise ValueError(
"logCompressionLimit needs to be bool or int")
logCompressionMethod = config.get('logCompressionMethod',
"bz2")
if logCompressionMethod not in ('bz2', 'gz'):
raise ValueError(
"logCompressionMethod needs to be 'bz2', or 'gz'")
logMaxSize = config.get('logMaxSize')
if logMaxSize is not None and not \
isinstance(logMaxSize, int):
raise ValueError("logMaxSize needs to be None or int")
logMaxTailSize = config.get('logMaxTailSize')
if logMaxTailSize is not None and not \
isinstance(logMaxTailSize, int):
raise ValueError(
"logMaxTailSize needs to be None or int")
mergeRequests = config.get('mergeRequests')
if mergeRequests is not None and not callable(
mergeRequests):
raise ValueError("mergeRequests must be a callable")
prioritizeBuilders = config.get('prioritizeBuilders')
if prioritizeBuilders is not None and not callable(
prioritizeBuilders):
raise ValueError("prioritizeBuilders must be callable")
changeHorizon = config.get("changeHorizon")
if changeHorizon is not None and not isinstance(
changeHorizon, int):
raise ValueError("changeHorizon needs to be an int")
except KeyError, e:
log.msg(
"config dictionary is missing a required parameter")
log.msg("leaving old configuration in place")
raise
#if "bots" in config:
# raise KeyError("c['bots'] is no longer accepted")
slaves = config.get('slaves', [])
if "bots" in config:
m = ("c['bots'] is deprecated as of 0.7.6 and will be "
"removed by 0.8.0 . Please use c['slaves'] instead.")
log.msg(m)
warnings.warn(m, DeprecationWarning)
for name, passwd in config['bots']:
slaves.append(JhBuildSlave(name, passwd))
if "bots" not in config and "slaves" not in config:
log.msg(
"config dictionary must have either 'bots' or 'slaves'"
)
log.msg("leaving old configuration in place")
raise KeyError("must have either 'bots' or 'slaves'")
#if "sources" in config:
# raise KeyError("c['sources'] is no longer accepted")
if changeHorizon is not None:
self.change_svc.changeHorizon = changeHorizon
change_source = config.get('change_source', [])
if isinstance(change_source, (list, tuple)):
change_sources = change_source
else:
change_sources = [change_source]
if "sources" in config:
m = (
"c['sources'] is deprecated as of 0.7.6 and will be "
"removed by 0.8.0 . Please use c['change_source'] instead."
)
log.msg(m)
warnings.warn(m, DeprecationWarning)
for s in config['sources']:
change_sources.append(s)
# do some validation first
for s in slaves:
assert interfaces.IBuildSlave.providedBy(s)
if s.slavename in ("debug", "change", "status"):
raise KeyError("reserved name '%s' used for a bot" %
s.slavename)
if config.has_key('interlocks'):
raise KeyError("c['interlocks'] is no longer accepted")
assert isinstance(change_sources, (list, tuple))
for s in change_sources:
assert interfaces.IChangeSource(s, None)
# this assertion catches c['schedulers'] = Scheduler(), since
# Schedulers are service.MultiServices and thus iterable.
errmsg = "c['schedulers'] must be a list of Scheduler instances"
assert isinstance(schedulers, (list, tuple)), errmsg
for s in schedulers:
assert interfaces.IScheduler(s, None), errmsg
assert isinstance(status, (list, tuple))
for s in status:
assert interfaces.IStatusReceiver(s, None)
slavenames = [s.slavename for s in slaves]
buildernames = []
dirnames = []
# convert builders from objects to config dictionaries
builders_dicts = []
for b in builders:
if isinstance(b, buildbot.config.BuilderConfig):
builders_dicts.append(b.getConfigDict())
elif type(b) is dict:
builders_dicts.append(b)
else:
raise ValueError(
"builder %s is not a BuilderConfig object (or a dict)"
% b)
builders = builders_dicts
for b in builders:
if b.has_key(
'slavename') and b['slavename'] not in slavenames:
raise ValueError("builder %s uses undefined slave %s" \
% (b['name'], b['slavename']))
for n in b.get('slavenames', []):
if n not in slavenames:
raise ValueError("builder %s uses undefined slave %s" \
% (b['name'], n))
if b['name'] in buildernames:
raise ValueError("duplicate builder name %s" %
b['name'])
buildernames.append(b['name'])
# sanity check name (BuilderConfig does this too)
if b['name'].startswith("_"):
errmsg = ("builder names must not start with an "
"underscore: " + b['name'])
log.err(errmsg)
raise ValueError(errmsg)
# Fix the dictionnary with default values, in case this wasn't
# specified with a BuilderConfig object (which sets the same defaults)
b.setdefault('builddir',
buildbot.util.safeTranslate(b['name']))
b.setdefault('slavebuilddir', b['builddir'])
if b['builddir'] in dirnames:
raise ValueError("builder %s reuses builddir %s" %
(b['name'], b['builddir']))
dirnames.append(b['builddir'])
unscheduled_buildernames = buildernames[:]
schedulernames = []
for s in schedulers:
for b in s.listBuilderNames():
assert b in buildernames, \
"%s uses unknown builder %s" % (s, b)
if b in unscheduled_buildernames:
unscheduled_buildernames.remove(b)
if s.name in schedulernames:
# TODO: schedulers share a namespace with other Service
# children of the BuildMaster node, like status plugins, the
# Manhole, the ChangeMaster, and the BotMaster (although most
# of these don't have names)
msg = ("Schedulers must have unique names, but "
"'%s' was a duplicate" % (s.name, ))
raise ValueError(msg)
schedulernames.append(s.name)
if unscheduled_buildernames:
log.msg(
"Warning: some Builders have no Schedulers to drive them:"
" %s" % (unscheduled_buildernames, ))
# assert that all locks used by the Builds and their Steps are
# uniquely named.
lock_dict = {}
for b in builders:
for l in b.get('locks', []):
if isinstance(l, locks.LockAccess
): # User specified access to the lock
l = l.lockid
if lock_dict.has_key(l.name):
if lock_dict[l.name] is not l:
raise ValueError(
"Two different locks (%s and %s) "
"share the name %s" %
(l, lock_dict[l.name], l.name))
else:
lock_dict[l.name] = l
# TODO: this will break with any BuildFactory that doesn't use a
# .steps list, but I think the verification step is more
# important.
for s in b['factory'].steps:
for l in s[1].get('locks', []):
if isinstance(
l, locks.LockAccess
): # User specified access to the lock
l = l.lockid
if lock_dict.has_key(l.name):
if lock_dict[l.name] is not l:
raise ValueError(
"Two different locks (%s and %s)"
" share the name %s" %
(l, lock_dict[l.name], l.name))
else:
lock_dict[l.name] = l
if not isinstance(properties, dict):
raise ValueError("c['properties'] must be a dictionary")
# slavePortnum supposed to be a strports specification
if type(slavePortnum) is int:
slavePortnum = "tcp:%d" % slavePortnum
# now we're committed to implementing the new configuration, so do
# it atomically
# TODO: actually, this is spread across a couple of Deferreds, so it
# really isn't atomic.
d = defer.succeed(None)
self.projectName = projectName
self.projectURL = projectURL
self.buildbotURL = buildbotURL
self.properties = Properties()
self.properties.update(properties, self.configFileName)
self.status.logCompressionLimit = logCompressionLimit
self.status.logCompressionMethod = logCompressionMethod
self.status.logMaxSize = logMaxSize
self.status.logMaxTailSize = logMaxTailSize
# Update any of our existing builders with the current log parameters.
# This is required so that the new value is picked up after a
# reconfig.
for builder in self.botmaster.builders.values():
builder.builder_status.setLogCompressionLimit(
logCompressionLimit)
builder.builder_status.setLogCompressionMethod(
logCompressionMethod)
builder.builder_status.setLogMaxSize(logMaxSize)
builder.builder_status.setLogMaxTailSize(logMaxTailSize)
if mergeRequests is not None:
self.botmaster.mergeRequests = mergeRequests
if prioritizeBuilders is not None:
self.botmaster.prioritizeBuilders = prioritizeBuilders
self.buildCacheSize = buildCacheSize
self.eventHorizon = eventHorizon
self.logHorizon = logHorizon
self.buildHorizon = buildHorizon
# self.slaves: Disconnect any that were attached and removed from the
# list. Update self.checker with the new list of passwords, including
# debug/change/status.
d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
# self.debugPassword
if debugPassword:
self.checker.addUser("debug", debugPassword)
self.debugPassword = debugPassword
# self.manhole
if manhole != self.manhole:
# changing
if self.manhole:
# disownServiceParent may return a Deferred
d.addCallback(
lambda res: self.manhole.disownServiceParent())
def _remove(res):
self.manhole = None
return res
d.addCallback(_remove)
if manhole:
def _add(res):
self.manhole = manhole
manhole.setServiceParent(self)
d.addCallback(_add)
# add/remove self.botmaster.builders to match builders. The
# botmaster will handle startup/shutdown issues.
d.addCallback(lambda res: self.loadConfig_Builders(builders))
d.addCallback(lambda res: self.loadConfig_status(status))
# Schedulers are added after Builders in case they start right away
d.addCallback(
lambda res: self.loadConfig_Schedulers(schedulers))
# and Sources go after Schedulers for the same reason
d.addCallback(
lambda res: self.loadConfig_Sources(change_sources))
# self.slavePort
if self.slavePortnum != slavePortnum:
if self.slavePort:
def closeSlavePort(res):
d1 = self.slavePort.disownServiceParent()
self.slavePort = None
return d1
d.addCallback(closeSlavePort)
if slavePortnum is not None:
def openSlavePort(res):
self.slavePort = strports.service(
slavePortnum, self.slaveFactory)
self.slavePort.setServiceParent(self)
d.addCallback(openSlavePort)
log.msg("BuildMaster listening on port %s" %
slavePortnum)
self.slavePortnum = slavePortnum
log.msg("configuration update started")
def _done(res):
self.readConfig = True
log.msg("configuration update complete")
d.addCallback(_done)
d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
return d
def __init__(self, number=None):
self.properties = Properties()
self.number = number
self.who = "me"
def addBuildsetForSourceStamps(self,
waited_for=False,
sourcestamps=None,
reason='',
external_idstring=None,
properties=None,
builderNames=None,
**kw):
if sourcestamps is None:
sourcestamps = []
# combine properties
if properties:
properties.updateFromProperties(self.properties)
else:
properties = self.properties
# make a fresh copy that we actually can modify safely
properties = Properties.fromDict(properties.asDict())
# make extra info available from properties.render()
properties.master = self.master
properties.sourcestamps = []
properties.changes = []
for ss in sourcestamps:
if isinstance(ss, integer_types):
# fetch actual sourcestamp and changes from data API
properties.sourcestamps.append((yield self.master.data.get(
('sourcestamps', ss))))
properties.changes.extend((yield self.master.data.get(
('sourcestamps', ss, 'changes'))))
else:
# sourcestamp with no change, see addBuildsetForChanges
properties.sourcestamps.append(ss)
for c in properties.changes:
properties.updateFromProperties(
Properties.fromDict(c['properties']))
# apply the default builderNames
if not builderNames:
builderNames = self.builderNames
# dynamically get the builder list to schedule
builderNames = yield properties.render(builderNames)
# Get the builder ids
# Note that there is a data.updates.findBuilderId(name)
# but that would merely only optimize the single builder case, while
# probably the multiple builder case will be severely impacted by the
# several db requests needed.
builderids = list()
for bldr in (yield self.master.data.get(('builders', ))):
if bldr['name'] in builderNames:
builderids.append(bldr['builderid'])
# translate properties object into a dict as required by the
# addBuildset method
properties_dict = properties.asDict()
bsid, brids = yield self.master.data.updates.addBuildset(
scheduler=self.name,
sourcestamps=sourcestamps,
reason=reason,
waited_for=waited_for,
properties=properties_dict,
builderids=builderids,
external_idstring=external_idstring,
**kw)
return (bsid, brids)
def __init__(self,
name,
builderNames,
properties=None,
codebases=DEFAULT_CODEBASES):
super(BaseScheduler, self).__init__(name=name)
ok = True
if interfaces.IRenderable.providedBy(builderNames):
pass
elif isinstance(builderNames, (list, tuple)):
for b in builderNames:
if not isinstance(b, string_types) and \
not interfaces.IRenderable.providedBy(b):
ok = False
else:
ok = False
if not ok:
config.error(
"The builderNames argument to a scheduler must be a list "
"of Builder names or an IRenderable object that will render"
"to a list of builder names.")
self.builderNames = builderNames
if properties is None:
properties = {}
self.properties = Properties()
self.properties.update(properties, "Scheduler")
self.properties.setProperty("scheduler", name, "Scheduler")
self.objectid = None
# Set the codebases that are necessary to process the changes
# These codebases will always result in a sourcestamp with or without
# changes
known_keys = set(['branch', 'repository', 'revision'])
if codebases is None:
config.error("Codebases cannot be None")
elif isinstance(codebases, list):
codebases = dict((codebase, {}) for codebase in codebases)
elif not isinstance(codebases, dict):
config.error(
"Codebases must be a dict of dicts, or list of strings")
else:
for codebase, attrs in codebases.items():
if not isinstance(attrs, dict):
config.error("Codebases must be a dict of dicts")
else:
unk = set(attrs) - known_keys
if unk:
config.error(
"Unknown codebase keys %s for codebase %s" %
(', '.join(unk), codebase))
self.codebases = codebases
# internal variables
self._change_consumer = None
self._enable_consumer = None
self._change_consumption_lock = defer.DeferredLock()
self.enabled = True
