class TestProperties(unittest.TestCase):
def setUp(self):
self.props = Properties()
def testDictBehavior(self):
self.props.setProperty("do-tests", 1, "scheduler")
self.props.setProperty("do-install", 2, "scheduler")
self.assert_(self.props.has_key('do-tests'))
self.failUnlessEqual(self.props['do-tests'], 1)
self.failUnlessEqual(self.props['do-install'], 2)
self.assertRaises(KeyError, lambda: self.props['do-nothing'])
self.failUnlessEqual(self.props.getProperty('do-install'), 2)
def testUpdate(self):
self.props.setProperty("x", 24, "old")
newprops = {'a': 1, 'b': 2}
self.props.update(newprops, "new")
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateFromProperties(self):
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new")
newprops.setProperty('b', 2, "new")
self.props.updateFromProperties(newprops)
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
class TestProperties(unittest.TestCase):
def setUp(self):
self.props = Properties()
def testDictBehavior(self):
self.props.setProperty("do-tests", 1, "scheduler")
self.props.setProperty("do-install", 2, "scheduler")
self.assert_(self.props.has_key('do-tests'))
self.failUnlessEqual(self.props['do-tests'], 1)
self.failUnlessEqual(self.props['do-install'], 2)
self.assertRaises(KeyError, lambda : self.props['do-nothing'])
self.failUnlessEqual(self.props.getProperty('do-install'), 2)
def testUpdate(self):
self.props.setProperty("x", 24, "old")
newprops = { 'a' : 1, 'b' : 2 }
self.props.update(newprops, "new")
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateFromProperties(self):
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new")
newprops.setProperty('b', 2, "new")
self.props.updateFromProperties(newprops)
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def test_defaultProperties(self):
props = Properties()
props.setProperty('foo', 1, 'Scheduler')
props.setProperty('bar', 'bleh', 'Change')
yield self.makeBuilder(defaultProperties={'bar': 'onoes', 'cuckoo': 42})
self.bldr.setupProperties(props)
self.assertEquals(props.getProperty('bar'), 'bleh')
self.assertEquals(props.getProperty('cuckoo'), 42)
def test_defaultProperties(self):
props = Properties()
props.setProperty('foo', 1, 'Scheduler')
props.setProperty('bar', 'bleh', 'Change')
yield self.makeBuilder(defaultProperties={'bar': 'onoes', 'cuckoo': 42})
self.bldr.setupProperties(props)
self.assertEquals(props.getProperty('bar'), 'bleh')
self.assertEquals(props.getProperty('cuckoo'), 42)
def trigger(self, ss, set_props=None):
self.ss = ss
self.set_props = set_props
props = Properties()
props.updateFromProperties(self.properties)
if set_props:
props.updateFromProperties(set_props)
self.script_repo_revision = props.getProperty('script_repo_revision')
assert self.script_repo_revision, 'script_repo_revision should be set'
self.release_config = props.getProperty('release_config')
assert self.release_config, 'release_config should be set'
def _run_loop(_):
self.loop = LoopingCall(self.poll)
reactor.callLater(0, self.loop.start, self.pollInterval)
reactor.callLater(self.pollTimeout, self.stopLoop,
'Timeout after %s' % self.pollTimeout)
d = self.getReleaseConfig()
d.addCallback(_run_loop)
def trigger(self, ss, set_props=None):
self.ss = ss
self.set_props = set_props
props = Properties()
props.updateFromProperties(self.properties)
if set_props:
props.updateFromProperties(set_props)
self.script_repo_revision = props.getProperty('script_repo_revision')
assert self.script_repo_revision, 'script_repo_revision should be set'
self.release_config = props.getProperty('release_config')
assert self.release_config, 'release_config should be set'
def _run_loop(_):
self.loop = LoopingCall(self.poll)
reactor.callLater(0, self.loop.start, self.pollInterval)
reactor.callLater(self.pollTimeout, self.stopLoop,
'Timeout after %s' % self.pollTimeout)
d = self.getReleaseConfig()
d.addCallback(_run_loop)
class AbstractWorker(service.BuildbotService):
"""This is the master-side representative for a remote buildbot worker.
There is exactly one for each worker described in the config file (the
c['workers'] list). When buildbots connect in (.attach), they get a
reference to this instance. The BotMaster object is stashed as the
.botmaster attribute. The BotMaster is also our '.parent' Service.
I represent a worker -- a remote machine capable of
running builds. I am instantiated by the configuration file, and can be
subclassed to add extra functionality."""
# reconfig workers after builders
reconfig_priority = 64
quarantine_timer = None
quarantine_timeout = quarantine_initial_timeout = 10
quarantine_max_timeout = 60 * 60
start_missing_on_startup = True
DEFAULT_MISSING_TIMEOUT = 3600
DEFAULT_KEEPALIVE_INTERVAL = 3600
# override to True if isCompatibleWithBuild may return False
builds_may_be_incompatible = False
def checkConfig(self,
name,
password,
max_builds=None,
notify_on_missing=None,
missing_timeout=None,
properties=None,
defaultProperties=None,
locks=None,
keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL,
machine_name=None):
"""
@param name: botname this machine will supply when it connects
@param password: password this machine will supply when
it connects
@param max_builds: maximum number of simultaneous builds that will
be run concurrently on this worker (the
default is None for no limit)
@param properties: properties that will be applied to builds run on
this worker
@type properties: dictionary
@param defaultProperties: properties that will be applied to builds
run on this worker only if the property
has not been set by another source
@type defaultProperties: dictionary
@param locks: A list of locks that must be acquired before this worker
can be used
@type locks: dictionary
@param machine_name: The name of the machine to associate with the
worker.
"""
self.name = name = bytes2unicode(name)
self.machine_name = machine_name
self.password = password
# protocol registration
self.registration = None
self._graceful = False
self._paused = False
# these are set when the service is started
self.manager = None
self.workerid = None
self.worker_status = WorkerStatus(name)
self.worker_commands = None
self.workerforbuilders = {}
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
self.lock_subscriptions = []
self.properties = Properties()
self.properties.update(properties or {}, "Worker")
self.properties.setProperty("workername", name, "Worker")
self.defaultProperties = Properties()
self.defaultProperties.update(defaultProperties or {}, "Worker")
if self.machine_name is not None:
self.properties.setProperty('machine_name', self.machine_name,
'Worker')
self.machine = None
self.lastMessageReceived = 0
if notify_on_missing is None:
notify_on_missing = []
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
for i in notify_on_missing:
if not isinstance(i, str):
config.error('notify_on_missing arg %r is not a string' %
(i, ))
self.missing_timeout = missing_timeout
self.missing_timer = None
# a protocol connection, if we're currently connected
self.conn = None
# during disconnection self.conn will be set to None before all disconnection notifications
# are delivered. During that period _pending_disconnection_delivery_notifier will be set to
# a notifier and allows interested users to wait until all disconnection notifications are
# delivered.
self._pending_disconnection_delivery_notifier = None
self._old_builder_list = None
self._configured_builderid_list = None
def __repr__(self):
return "<{} {}>".format(self.__class__.__name__, repr(self.name))
@property
def workername(self):
# workername is now an alias to twisted.Service's name
return self.name
@property
def botmaster(self):
if self.master is None:
return None
return self.master.botmaster
@defer.inlineCallbacks
def updateLocks(self):
"""Convert the L{LockAccess} objects in C{self.locks} into real lock
objects, while also maintaining the subscriptions to lock releases."""
# unsubscribe from any old locks
for s in self.lock_subscriptions:
s.unsubscribe()
# convert locks into their real form
locks = yield self.botmaster.getLockFromLockAccesses(
self.access, self.config_version)
self.locks = [(l.getLockForWorker(self.workername), la)
for l, la in locks]
self.lock_subscriptions = [
l.subscribeToReleases(self._lockReleased) for l, la in self.locks
]
def locksAvailable(self):
"""
I am called to see if all the locks I depend on are available,
in which I return True, otherwise I return False
"""
if not self.locks:
return True
for lock, access in self.locks:
if not lock.isAvailable(self, access):
return False
return True
def acquireLocks(self):
"""
I am called when a build is preparing to run. I try to claim all
the locks that are needed for a build to happen. If I can't, then
my caller should give up the build and try to get another worker
to look at it.
"""
log.msg("acquireLocks(worker {}, locks {})".format(self, self.locks))
if not self.locksAvailable():
log.msg("worker {} can't lock, giving up".format(self))
return False
# all locks are available, claim them all
for lock, access in self.locks:
lock.claim(self, access)
return True
def releaseLocks(self):
"""
I am called to release any locks after a build has finished
"""
log.msg("releaseLocks({}): {}".format(self, self.locks))
for lock, access in self.locks:
lock.release(self, access)
def _lockReleased(self):
"""One of the locks for this worker was released; try scheduling
builds."""
if not self.botmaster:
return # oh well..
self.botmaster.maybeStartBuildsForWorker(self.name)
def _applyWorkerInfo(self, info):
if not info:
return
self.worker_status.setAdmin(info.get("admin"))
self.worker_status.setHost(info.get("host"))
self.worker_status.setAccessURI(info.get("access_uri", None))
self.worker_status.setVersion(info.get("version", "(unknown)"))
# store everything as Properties
for k, v in info.items():
if k in ('environ', 'worker_commands'):
continue
self.worker_status.info.setProperty(k, v, "Worker")
@defer.inlineCallbacks
def _getWorkerInfo(self):
worker = yield self.master.data.get(('workers', self.workerid))
self._applyWorkerInfo(worker['workerinfo'])
def setServiceParent(self, parent):
# botmaster needs to set before setServiceParent which calls
# startService
self.manager = parent
return super().setServiceParent(parent)
@defer.inlineCallbacks
def startService(self):
# tracks config version for locks
self.config_version = self.master.config_version
self.updateLocks()
self.workerid = yield self.master.data.updates.findWorkerId(self.name)
self.workerActionConsumer = yield self.master.mq.startConsuming(
self.controlWorker,
("control", "worker", str(self.workerid), None))
yield self._getWorkerInfo()
yield super().startService()
# startMissingTimer wants the service to be running to really start
if self.start_missing_on_startup:
self.startMissingTimer()
@defer.inlineCallbacks
def reconfigService(self,
name,
password,
max_builds=None,
notify_on_missing=None,
missing_timeout=DEFAULT_MISSING_TIMEOUT,
properties=None,
defaultProperties=None,
locks=None,
keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL,
machine_name=None):
# Given a Worker config arguments, configure this one identically.
# Because Worker objects are remotely referenced, we can't replace them
# without disconnecting the worker, yet there's no reason to do that.
assert self.name == name
self.password = password
# adopt new instance's configuration parameters
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
if notify_on_missing is None:
notify_on_missing = []
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
if self.missing_timeout != missing_timeout:
running_missing_timer = self.missing_timer
self.stopMissingTimer()
self.missing_timeout = missing_timeout
if running_missing_timer:
self.startMissingTimer()
self.properties = Properties()
self.properties.update(properties or {}, "Worker")
self.properties.setProperty("workername", name, "Worker")
self.defaultProperties = Properties()
self.defaultProperties.update(defaultProperties or {}, "Worker")
# Note that before first reconfig self.machine will always be None and
# out of sync with self.machine_name, thus more complex logic is needed.
if self.machine is not None and self.machine_name != machine_name:
self.machine.unregisterWorker(self)
self.machine = None
self.machine_name = machine_name
if self.machine is None and self.machine_name is not None:
self.machine = self.master.machine_manager.getMachineByName(
self.machine_name)
if self.machine is not None:
self.machine.registerWorker(self)
self.properties.setProperty("machine_name", self.machine_name,
"Worker")
else:
log.err("Unknown machine '{}' for worker '{}'".format(
self.machine_name, self.name))
# update our records with the worker manager
if not self.registration:
self.registration = yield self.master.workers.register(self)
yield self.registration.update(self, self.master.config)
# tracks config version for locks
self.config_version = self.master.config_version
self.updateLocks()
@defer.inlineCallbacks
def reconfigServiceWithSibling(self, sibling):
# reconfigServiceWithSibling will only reconfigure the worker when it is configured
# differently.
# However, the worker configuration depends on which builder it is configured
yield super().reconfigServiceWithSibling(sibling)
# update the attached worker's notion of which builders are attached.
# This assumes that the relevant builders have already been configured,
# which is why the reconfig_priority is set low in this class.
bids = [
b.getBuilderId()
for b in self.botmaster.getBuildersForWorker(self.name)
]
bids = yield defer.gatherResults(bids, consumeErrors=True)
if self._configured_builderid_list != bids:
yield self.master.data.updates.workerConfigured(
self.workerid, self.master.masterid, bids)
yield self.updateWorker()
self._configured_builderid_list = bids
@defer.inlineCallbacks
def stopService(self):
if self.registration:
yield self.registration.unregister()
self.registration = None
self.workerActionConsumer.stopConsuming()
self.stopMissingTimer()
self.stopQuarantineTimer()
# mark this worker as configured for zero builders in this master
yield self.master.data.updates.workerConfigured(
self.workerid, self.master.masterid, [])
# during master shutdown we need to wait until the disconnection notification deliveries
# are completed, otherwise some of the events may still be firing long after the master
# is completely shut down.
yield self.disconnect()
yield self.waitForCompleteShutdown()
yield super().stopService()
def isCompatibleWithBuild(self, build_props):
# given a build properties object, determines whether the build is
# compatible with the currently running worker or not. This is most
# often useful for latent workers where it's possible to request
# different kinds of workers.
return defer.succeed(True)
def startMissingTimer(self):
if self.missing_timeout and self.parent and self.running:
self.stopMissingTimer() # in case it's already running
self.missing_timer = self.master.reactor.callLater(
self.missing_timeout, self._missing_timer_fired)
def stopMissingTimer(self):
if self.missing_timer:
if self.missing_timer.active():
self.missing_timer.cancel()
self.missing_timer = None
def isConnected(self):
return self.conn
def _missing_timer_fired(self):
self.missing_timer = None
# notify people, but only if we're still in the config
if not self.parent:
return
last_connection = time.ctime(time.time() - self.missing_timeout)
self.master.data.updates.workerMissing(workerid=self.workerid,
masterid=self.master.masterid,
last_connection=last_connection,
notify=self.notify_on_missing)
def updateWorker(self):
"""Called to add or remove builders after the worker has connected.
@return: a Deferred that indicates when an attached worker has
accepted the new builders and/or released the old ones."""
if self.conn:
return self.sendBuilderList()
# else:
return defer.succeed(None)
@defer.inlineCallbacks
def attached(self, conn):
"""This is called when the worker connects."""
assert self.conn is None
metrics.MetricCountEvent.log("AbstractWorker.attached_workers", 1)
# now we go through a sequence of calls, gathering information, then
# tell the Botmaster that it can finally give this worker to all the
# Builders that care about it.
# Reset graceful shutdown status
self._graceful = False
self.conn = conn
self._old_builder_list = None # clear builder list before proceed
self.worker_status.setConnected(True)
self._applyWorkerInfo(conn.info)
self.worker_commands = conn.info.get("worker_commands", {})
self.worker_environ = conn.info.get("environ", {})
self.worker_basedir = conn.info.get("basedir", None)
self.worker_system = conn.info.get("system", None)
self.conn.notifyOnDisconnect(self.detached)
workerinfo = {
'admin': conn.info.get('admin'),
'host': conn.info.get('host'),
'access_uri': conn.info.get('access_uri'),
'version': conn.info.get('version')
}
yield self.master.data.updates.workerConnected(
workerid=self.workerid,
masterid=self.master.masterid,
workerinfo=workerinfo)
if self.worker_system == "nt":
self.path_module = namedModule("ntpath")
else:
# most everything accepts / as separator, so posix should be a
# reasonable fallback
self.path_module = namedModule("posixpath")
log.msg("bot attached")
self.messageReceivedFromWorker()
self.stopMissingTimer()
yield self.updateWorker()
yield self.botmaster.maybeStartBuildsForWorker(self.name)
self.updateState()
def messageReceivedFromWorker(self):
now = time.time()
self.lastMessageReceived = now
self.worker_status.setLastMessageReceived(now)
def setupProperties(self, props):
for name in self.properties.properties:
props.setProperty(name, self.properties.getProperty(name),
"Worker")
for name in self.defaultProperties.properties:
if name not in props:
props.setProperty(name,
self.defaultProperties.getProperty(name),
"Worker")
@defer.inlineCallbacks
def _handle_disconnection_delivery_notifier(self):
self._pending_disconnection_delivery_notifier = Notifier()
yield self.conn.waitForNotifyDisconnectedDelivered()
self._pending_disconnection_delivery_notifier.notify(None)
self._pending_disconnection_delivery_notifier = None
@defer.inlineCallbacks
def detached(self):
# protect against race conditions in conn disconnect path and someone
# calling detached directly. At the moment the null worker does that.
if self.conn is None:
return
metrics.MetricCountEvent.log("AbstractWorker.attached_workers", -1)
self._handle_disconnection_delivery_notifier()
yield self.conn.waitShutdown()
self.conn = None
self._old_builder_list = []
self.worker_status.setConnected(False)
log.msg("Worker.detached({})".format(self.name))
self.releaseLocks()
yield self.master.data.updates.workerDisconnected(
workerid=self.workerid,
masterid=self.master.masterid,
)
def disconnect(self):
"""Forcibly disconnect the worker.
This severs the TCP connection and returns a Deferred that will fire
(with None) when the connection is probably gone.
If the worker is still alive, they will probably try to reconnect
again in a moment.
This is called in two circumstances. The first is when a worker is
removed from the config file. In this case, when they try to
reconnect, they will be rejected as an unknown worker. The second is
when we wind up with two connections for the same worker, in which
case we disconnect the older connection.
"""
if self.conn is None:
return defer.succeed(None)
log.msg("disconnecting old worker {} now".format(self.name))
# When this Deferred fires, we'll be ready to accept the new worker
return self._disconnect(self.conn)
def waitForCompleteShutdown(self):
# This function waits until the disconnection to happen and the disconnection
# notifications have been delivered and acted upon.
return self._waitForCompleteShutdownImpl(self.conn)
@defer.inlineCallbacks
def _waitForCompleteShutdownImpl(self, conn):
if conn:
d = defer.Deferred()
def _disconnected():
eventually(d.callback, None)
conn.notifyOnDisconnect(_disconnected)
yield d
yield conn.waitForNotifyDisconnectedDelivered()
elif self._pending_disconnection_delivery_notifier is not None:
yield self._pending_disconnection_delivery_notifier.wait()
@defer.inlineCallbacks
def _disconnect(self, conn):
# This function waits until the disconnection to happen and the disconnection
# notifications have been delivered and acted upon
d = self._waitForCompleteShutdownImpl(conn)
conn.loseConnection()
log.msg("waiting for worker to finish disconnecting")
yield d
@defer.inlineCallbacks
def sendBuilderList(self):
our_builders = self.botmaster.getBuildersForWorker(self.name)
blist = [(b.name, b.config.workerbuilddir) for b in our_builders]
if blist == self._old_builder_list:
return
slist = yield self.conn.remoteSetBuilderList(builders=blist)
self._old_builder_list = blist
# Nothing has changed, so don't need to re-attach to everything
if not slist:
return
dl = []
for name in slist:
# use get() since we might have changed our mind since then
b = self.botmaster.builders.get(name)
if b:
d1 = self.attachBuilder(b)
dl.append(d1)
yield defer.DeferredList(dl)
def attachBuilder(self, builder):
return builder.attached(self, self.worker_commands)
def controlWorker(self, key, params):
log.msg("worker {} wants to {}: {}".format(self.name, key[-1], params))
if key[-1] == "stop":
return self.shutdownRequested()
if key[-1] == "pause":
self.pause()
if key[-1] == "unpause":
self.unpause()
if key[-1] == "kill":
self.shutdown()
def shutdownRequested(self):
self._graceful = True
self.maybeShutdown()
self.updateState()
def addWorkerForBuilder(self, wfb):
self.workerforbuilders[wfb.builder_name] = wfb
def removeWorkerForBuilder(self, wfb):
try:
del self.workerforbuilders[wfb.builder_name]
except KeyError:
pass
def buildFinished(self, wfb):
"""This is called when a build on this worker is finished."""
self.botmaster.maybeStartBuildsForWorker(self.name)
def canStartBuild(self):
"""
I am called when a build is requested to see if this worker
can start a build. This function can be used to limit overall
concurrency on the worker.
Note for subclassers: if a worker can become willing to start a build
without any action on that worker (for example, by a resource in use on
another worker becoming available), then you must arrange for
L{maybeStartBuildsForWorker} to be called at that time, or builds on
this worker will not start.
"""
# If we're waiting to shutdown gracefully or paused, then we shouldn't
# accept any new jobs.
if self._graceful or self._paused:
return False
if self.max_builds:
active_builders = [
wfb for wfb in self.workerforbuilders.values() if wfb.isBusy()
]
if len(active_builders) >= self.max_builds:
return False
if not self.locksAvailable():
return False
return True
@defer.inlineCallbacks
def shutdown(self):
"""Shutdown the worker"""
if not self.conn:
log.msg("no remote; worker is already shut down")
return
yield self.conn.remoteShutdown()
def maybeShutdown(self):
"""Shut down this worker if it has been asked to shut down gracefully,
and has no active builders."""
if not self._graceful:
return
active_builders = [
wfb for wfb in self.workerforbuilders.values() if wfb.isBusy()
]
if active_builders:
return
d = self.shutdown()
d.addErrback(log.err, 'error while shutting down worker')
def updateState(self):
self.master.data.updates.setWorkerState(self.workerid, self._paused,
self._graceful)
def pause(self):
"""Stop running new builds on the worker."""
self._paused = True
self.updateState()
def unpause(self):
"""Restart running new builds on the worker."""
self._paused = False
self.botmaster.maybeStartBuildsForWorker(self.name)
self.updateState()
def isPaused(self):
return self._paused
def resetQuarantine(self):
self.quarantine_timeout = self.quarantine_initial_timeout
def putInQuarantine(self):
if self.quarantine_timer: # already in quarantine
return
self.pause()
self.quarantine_timer = self.master.reactor.callLater(
self.quarantine_timeout, self.exitQuarantine)
log.msg("{} has been put in quarantine for {}s".format(
self.name, self.quarantine_timeout))
# next we will wait twice as long
self.quarantine_timeout *= 2
if self.quarantine_timeout > self.quarantine_max_timeout:
# unless we hit the max timeout
self.quarantine_timeout = self.quarantine_max_timeout
def exitQuarantine(self):
self.quarantine_timer = None
self.unpause()
def stopQuarantineTimer(self):
if self.quarantine_timer is not None:
self.quarantine_timer.cancel()
self.quarantine_timer = None
self.unpause()
class TestProperties(unittest.TestCase):
def setUp(self):
self.props = Properties()
def testDictBehavior(self):
# note that dictionary-like behavior is deprecated and not exposed to
# users!
self.props.setProperty("do-tests", 1, "scheduler")
self.props.setProperty("do-install", 2, "scheduler")
self.assert_(self.props.has_key('do-tests'))
self.failUnlessEqual(self.props['do-tests'], 1)
self.failUnlessEqual(self.props['do-install'], 2)
self.assertRaises(KeyError, lambda: self.props['do-nothing'])
self.failUnlessEqual(self.props.getProperty('do-install'), 2)
self.assertIn('do-tests', self.props)
self.assertNotIn('missing-do-tests', self.props)
def testAsList(self):
self.props.setProperty("happiness", 7, "builder")
self.props.setProperty("flames", True, "tester")
self.assertEqual(sorted(self.props.asList()),
[('flames', True, 'tester'),
('happiness', 7, 'builder')])
def testAsDict(self):
self.props.setProperty("msi_filename", "product.msi", 'packager')
self.props.setProperty("dmg_filename", "product.dmg", 'packager')
self.assertEqual(
self.props.asDict(),
dict(msi_filename=('product.msi', 'packager'),
dmg_filename=('product.dmg', 'packager')))
def testUpdate(self):
self.props.setProperty("x", 24, "old")
newprops = {'a': 1, 'b': 2}
self.props.update(newprops, "new")
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateRuntime(self):
self.props.setProperty("x", 24, "old")
newprops = {'a': 1, 'b': 2}
self.props.update(newprops, "new", runtime=True)
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
self.assertEqual(self.props.runtime, set(['a', 'b']))
def testUpdateFromProperties(self):
self.props.setProperty("a", 94, "old")
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new")
newprops.setProperty('b', 2, "new")
self.props.updateFromProperties(newprops)
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateFromPropertiesNoRuntime(self):
self.props.setProperty("a", 94, "old")
self.props.setProperty("b", 84, "old")
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new", runtime=True)
newprops.setProperty('b', 2, "new", runtime=False)
newprops.setProperty('c', 3, "new", runtime=True)
newprops.setProperty('d', 3, "new", runtime=False)
self.props.updateFromPropertiesNoRuntime(newprops)
self.failUnlessEqual(self.props.getProperty('a'), 94)
self.failUnlessEqual(self.props.getPropertySource('a'), 'old')
self.failUnlessEqual(self.props.getProperty('b'), 2)
self.failUnlessEqual(self.props.getPropertySource('b'), 'new')
self.failUnlessEqual(self.props.getProperty('c'), None) # not updated
self.failUnlessEqual(self.props.getProperty('d'), 3)
self.failUnlessEqual(self.props.getPropertySource('d'), 'new')
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
# IProperties methods
def test_getProperty(self):
self.props.properties['p1'] = (['p', 1], 'test')
self.assertEqual(self.props.getProperty('p1'), ['p', 1])
def test_getProperty_default_None(self):
self.assertEqual(self.props.getProperty('p1'), None)
def test_getProperty_default(self):
self.assertEqual(self.props.getProperty('p1', 2), 2)
def test_hasProperty_false(self):
self.assertFalse(self.props.hasProperty('x'))
def test_hasProperty_true(self):
self.props.properties['x'] = (False, 'test')
self.assertTrue(self.props.hasProperty('x'))
def test_has_key_false(self):
self.assertFalse(self.props.has_key('x'))
def test_setProperty(self):
self.props.setProperty('x', 'y', 'test')
self.assertEqual(self.props.properties['x'], ('y', 'test'))
self.assertNotIn('x', self.props.runtime)
def test_setProperty_runtime(self):
self.props.setProperty('x', 'y', 'test', runtime=True)
self.assertEqual(self.props.properties['x'], ('y', 'test'))
self.assertIn('x', self.props.runtime)
def test_setProperty_no_source(self):
self.assertRaises(TypeError, lambda: self.props.setProperty('x', 'y'))
def test_getProperties(self):
self.assertIdentical(self.props.getProperties(), self.props)
def test_getBuild(self):
self.assertIdentical(self.props.getBuild(), self.props.build)
def test_render(self):
class FakeRenderable(object):
implements(IRenderable)
def getRenderingFor(self, props):
return props.getProperty('x') + 'z'
self.props.setProperty('x', 'y', 'test')
self.assertEqual(self.props.render(FakeRenderable()), 'yz')
class TestProperties(unittest.TestCase):
def setUp(self):
self.props = Properties()
def testDictBehavior(self):
# note that dictionary-like behavior is deprecated and not exposed to
# users!
self.props.setProperty("do-tests", 1, "scheduler")
self.props.setProperty("do-install", 2, "scheduler")
self.assert_(self.props.has_key('do-tests'))
self.failUnlessEqual(self.props['do-tests'], 1)
self.failUnlessEqual(self.props['do-install'], 2)
self.assertRaises(KeyError, lambda : self.props['do-nothing'])
self.failUnlessEqual(self.props.getProperty('do-install'), 2)
self.assertIn('do-tests', self.props)
self.assertNotIn('missing-do-tests', self.props)
def testAsList(self):
self.props.setProperty("happiness", 7, "builder")
self.props.setProperty("flames", True, "tester")
self.assertEqual(sorted(self.props.asList()),
[ ('flames', True, 'tester'), ('happiness', 7, 'builder') ])
def testAsDict(self):
self.props.setProperty("msi_filename", "product.msi", 'packager')
self.props.setProperty("dmg_filename", "product.dmg", 'packager')
self.assertEqual(self.props.asDict(),
dict(msi_filename=('product.msi', 'packager'), dmg_filename=('product.dmg', 'packager')))
def testUpdate(self):
self.props.setProperty("x", 24, "old")
newprops = { 'a' : 1, 'b' : 2 }
self.props.update(newprops, "new")
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateRuntime(self):
self.props.setProperty("x", 24, "old")
newprops = { 'a' : 1, 'b' : 2 }
self.props.update(newprops, "new", runtime=True)
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
self.assertEqual(self.props.runtime, set(['a', 'b']))
def testUpdateFromProperties(self):
self.props.setProperty("a", 94, "old")
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new")
newprops.setProperty('b', 2, "new")
self.props.updateFromProperties(newprops)
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateFromPropertiesNoRuntime(self):
self.props.setProperty("a", 94, "old")
self.props.setProperty("b", 84, "old")
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new", runtime=True)
newprops.setProperty('b', 2, "new", runtime=False)
newprops.setProperty('c', 3, "new", runtime=True)
newprops.setProperty('d', 3, "new", runtime=False)
self.props.updateFromPropertiesNoRuntime(newprops)
self.failUnlessEqual(self.props.getProperty('a'), 94)
self.failUnlessEqual(self.props.getPropertySource('a'), 'old')
self.failUnlessEqual(self.props.getProperty('b'), 2)
self.failUnlessEqual(self.props.getPropertySource('b'), 'new')
self.failUnlessEqual(self.props.getProperty('c'), None) # not updated
self.failUnlessEqual(self.props.getProperty('d'), 3)
self.failUnlessEqual(self.props.getPropertySource('d'), 'new')
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
# IProperties methods
def test_getProperty(self):
self.props.properties['p1'] = (['p', 1], 'test')
self.assertEqual(self.props.getProperty('p1'), ['p', 1])
def test_getProperty_default_None(self):
self.assertEqual(self.props.getProperty('p1'), None)
def test_getProperty_default(self):
self.assertEqual(self.props.getProperty('p1', 2), 2)
def test_hasProperty_false(self):
self.assertFalse(self.props.hasProperty('x'))
def test_hasProperty_true(self):
self.props.properties['x'] = (False, 'test')
self.assertTrue(self.props.hasProperty('x'))
def test_has_key_false(self):
self.assertFalse(self.props.has_key('x'))
def test_setProperty(self):
self.props.setProperty('x', 'y', 'test')
self.assertEqual(self.props.properties['x'], ('y', 'test'))
self.assertNotIn('x', self.props.runtime)
def test_setProperty_runtime(self):
self.props.setProperty('x', 'y', 'test', runtime=True)
self.assertEqual(self.props.properties['x'], ('y', 'test'))
self.assertIn('x', self.props.runtime)
def test_setProperty_no_source(self):
self.assertRaises(TypeError, lambda :
self.props.setProperty('x', 'y'))
def test_getProperties(self):
self.assertIdentical(self.props.getProperties(), self.props)
def test_getBuild(self):
self.assertIdentical(self.props.getBuild(), self.props.build)
def test_render(self):
class FakeRenderable(object):
implements(IRenderable)
def getRenderingFor(self, props):
return props.getProperty('x') + 'z'
self.props.setProperty('x', 'y', 'test')
self.assertEqual(self.props.render(FakeRenderable()), 'yz')
class BuildStatus(styles.Versioned):
implements(interfaces.IBuildStatus, interfaces.IStatusEvent)
persistenceVersion = 3
persistenceForgets = ( 'wasUpgraded', )
source = None
reason = None
changes = []
blamelist = []
progress = None
started = None
finished = None
currentStep = None
text = []
results = None
slavename = "???"
# these lists/dicts are defined here so that unserialized instances have
# (empty) values. They are set in __init__ to new objects to make sure
# each instance gets its own copy.
watchers = []
updates = {}
finishedWatchers = []
testResults = {}
def __init__(self, parent, number):
"""
@type parent: L{BuilderStatus}
@type number: int
"""
assert interfaces.IBuilderStatus(parent)
self.builder = parent
self.number = number
self.watchers = []
self.updates = {}
self.finishedWatchers = []
self.steps = []
self.testResults = {}
self.properties = Properties()
def __repr__(self):
return "<%s #%s>" % (self.__class__.__name__, self.number)
# IBuildStatus
def getBuilder(self):
"""
@rtype: L{BuilderStatus}
"""
return self.builder
def getProperty(self, propname):
return self.properties[propname]
def getProperties(self):
return self.properties
def getNumber(self):
return self.number
def getPreviousBuild(self):
if self.number == 0:
return None
return self.builder.getBuild(self.number-1)
def getSourceStamp(self, absolute=False):
if not absolute or not self.properties.has_key('got_revision'):
return self.source
return self.source.getAbsoluteSourceStamp(self.properties['got_revision'])
def getReason(self):
return self.reason
def getChanges(self):
return self.changes
def getResponsibleUsers(self):
return self.blamelist
def getInterestedUsers(self):
# TODO: the Builder should add others: sheriffs, domain-owners
return self.blamelist + self.properties.getProperty('owners', [])
def getSteps(self):
"""Return a list of IBuildStepStatus objects. For invariant builds
(those which always use the same set of Steps), this should be the
complete list, however some of the steps may not have started yet
(step.getTimes()[0] will be None). For variant builds, this may not
be complete (asking again later may give you more of them)."""
return self.steps
def getTimes(self):
return (self.started, self.finished)
_sentinel = [] # used as a sentinel to indicate unspecified initial_value
def getSummaryStatistic(self, name, summary_fn, initial_value=_sentinel):
"""Summarize the named statistic over all steps in which it
exists, using combination_fn and initial_value to combine multiple
results into a single result. This translates to a call to Python's
X{reduce}::
return reduce(summary_fn, step_stats_list, initial_value)
"""
step_stats_list = [
st.getStatistic(name)
for st in self.steps
if st.hasStatistic(name) ]
if initial_value is self._sentinel:
return reduce(summary_fn, step_stats_list)
else:
return reduce(summary_fn, step_stats_list, initial_value)
def isFinished(self):
return (self.finished is not None)
def waitUntilFinished(self):
if self.finished:
d = defer.succeed(self)
else:
d = defer.Deferred()
self.finishedWatchers.append(d)
return d
# while the build is running, the following methods make sense.
# Afterwards they return None
def getETA(self):
if self.finished is not None:
return None
if not self.progress:
return None
eta = self.progress.eta()
if eta is None:
return None
return eta - util.now()
def getCurrentStep(self):
return self.currentStep
# Once you know the build has finished, the following methods are legal.
# Before ths build has finished, they all return None.
def getText(self):
text = []
text.extend(self.text)
for s in self.steps:
text.extend(s.text2)
return text
def getResults(self):
return self.results
def getSlavename(self):
return self.slavename
def getTestResults(self):
return self.testResults
def getTestResultsOrd(self):
trs = self.testResults.keys()
trs.sort()
ret = [ self.testResults[t] for t in trs]
return ret
def getLogs(self):
# TODO: steps should contribute significant logs instead of this
# hack, which returns every log from every step. The logs should get
# names like "compile" and "test" instead of "compile.output"
logs = []
for s in self.steps:
for loog in s.getLogs():
logs.append(loog)
return logs
# subscription interface
def subscribe(self, receiver, updateInterval=None):
# will receive stepStarted and stepFinished messages
# and maybe buildETAUpdate
self.watchers.append(receiver)
if updateInterval is not None:
self.sendETAUpdate(receiver, updateInterval)
def sendETAUpdate(self, receiver, updateInterval):
self.updates[receiver] = None
ETA = self.getETA()
if ETA is not None:
receiver.buildETAUpdate(self, self.getETA())
# they might have unsubscribed during buildETAUpdate
if receiver in self.watchers:
self.updates[receiver] = reactor.callLater(updateInterval,
self.sendETAUpdate,
receiver,
updateInterval)
def unsubscribe(self, receiver):
if receiver in self.watchers:
self.watchers.remove(receiver)
if receiver in self.updates:
if self.updates[receiver] is not None:
self.updates[receiver].cancel()
del self.updates[receiver]
# methods for the base.Build to invoke
def addStepWithName(self, name):
"""The Build is setting up, and has added a new BuildStep to its
list. Create a BuildStepStatus object to which it can send status
updates."""
s = BuildStepStatus(self, len(self.steps))
s.setName(name)
self.steps.append(s)
return s
def setProperty(self, propname, value, source, runtime=True):
self.properties.setProperty(propname, value, source, runtime)
def addTestResult(self, result):
self.testResults[result.getName()] = result
def setSourceStamp(self, sourceStamp):
self.source = sourceStamp
self.changes = self.source.changes
def setReason(self, reason):
self.reason = reason
def setBlamelist(self, blamelist):
self.blamelist = blamelist
def setProgress(self, progress):
self.progress = progress
def buildStarted(self, build):
"""The Build has been set up and is about to be started. It can now
be safely queried, so it is time to announce the new build."""
self.started = util.now()
# now that we're ready to report status, let the BuilderStatus tell
# the world about us
self.builder.buildStarted(self)
def setSlavename(self, slavename):
self.slavename = slavename
def setText(self, text):
assert isinstance(text, (list, tuple))
self.text = text
def setResults(self, results):
self.results = results
def buildFinished(self):
self.currentStep = None
self.finished = util.now()
for r in self.updates.keys():
if self.updates[r] is not None:
self.updates[r].cancel()
del self.updates[r]
watchers = self.finishedWatchers
self.finishedWatchers = []
for w in watchers:
w.callback(self)
# methods called by our BuildStepStatus children
def stepStarted(self, step):
self.currentStep = step
for w in self.watchers:
receiver = w.stepStarted(self, step)
if receiver:
if type(receiver) == type(()):
step.subscribe(receiver[0], receiver[1])
else:
step.subscribe(receiver)
d = step.waitUntilFinished()
d.addCallback(lambda step: step.unsubscribe(receiver))
step.waitUntilFinished().addCallback(self._stepFinished)
def _stepFinished(self, step):
results = step.getResults()
for w in self.watchers:
w.stepFinished(self, step, results)
# methods called by our BuilderStatus parent
def pruneSteps(self):
# this build is very old: remove the build steps too
self.steps = []
# persistence stuff
def generateLogfileName(self, stepname, logname):
"""Return a filename (relative to the Builder's base directory) where
the logfile's contents can be stored uniquely.
The base filename is made by combining our build number, the Step's
name, and the log's name, then removing unsuitable characters. The
filename is then made unique by appending _0, _1, etc, until it does
not collide with any other logfile.
These files are kept in the Builder's basedir (rather than a
per-Build subdirectory) because that makes cleanup easier: cron and
find will help get rid of the old logs, but the empty directories are
more of a hassle to remove."""
starting_filename = "%d-log-%s-%s" % (self.number, stepname, logname)
starting_filename = re.sub(r'[^\w\.\-]', '_', starting_filename)
# now make it unique
unique_counter = 0
filename = starting_filename
while filename in [l.filename
for step in self.steps
for l in step.getLogs()
if l.filename]:
filename = "%s_%d" % (starting_filename, unique_counter)
unique_counter += 1
return filename
def __getstate__(self):
d = styles.Versioned.__getstate__(self)
# for now, a serialized Build is always "finished". We will never
# save unfinished builds.
if not self.finished:
d['finished'] = util.now()
# TODO: push an "interrupted" step so it is clear that the build
# was interrupted. The builder will have a 'shutdown' event, but
# someone looking at just this build will be confused as to why
# the last log is truncated.
for k in 'builder', 'watchers', 'updates', 'finishedWatchers':
if k in d: del d[k]
return d
def __setstate__(self, d):
styles.Versioned.__setstate__(self, d)
# self.builder must be filled in by our parent when loading
for step in self.steps:
step.build = weakref.ref(self)
self.watchers = []
self.updates = {}
self.finishedWatchers = []
def upgradeToVersion1(self):
if hasattr(self, "sourceStamp"):
# the old .sourceStamp attribute wasn't actually very useful
maxChangeNumber, patch = self.sourceStamp
changes = getattr(self, 'changes', [])
source = sourcestamp.SourceStamp(branch=None,
revision=None,
patch=patch,
changes=changes)
self.source = source
self.changes = source.changes
del self.sourceStamp
self.wasUpgraded = True
def upgradeToVersion2(self):
self.properties = {}
self.wasUpgraded = True
def upgradeToVersion3(self):
# in version 3, self.properties became a Properties object
propdict = self.properties
self.properties = Properties()
self.properties.update(propdict, "Upgrade from previous version")
self.wasUpgraded = True
def upgradeLogfiles(self):
# upgrade any LogFiles that need it. This must occur after we've been
# attached to our Builder, and after we know about all LogFiles of
# all Steps (to get the filenames right).
assert self.builder
for s in self.steps:
for l in s.getLogs():
if l.filename:
pass # new-style, log contents are on disk
else:
logfilename = self.generateLogfileName(s.name, l.name)
# let the logfile update its .filename pointer,
# transferring its contents onto disk if necessary
l.upgrade(logfilename)
def checkLogfiles(self):
# check that all logfiles exist, and remove references to any that
# have been deleted (e.g., by purge())
for s in self.steps:
s.checkLogfiles()
def saveYourself(self):
filename = os.path.join(self.builder.basedir, "%d" % self.number)
if os.path.isdir(filename):
# leftover from 0.5.0, which stored builds in directories
shutil.rmtree(filename, ignore_errors=True)
tmpfilename = filename + ".tmp"
try:
dump(self, open(tmpfilename, "wb"), -1)
if runtime.platformType == 'win32':
# windows cannot rename a file on top of an existing one, so
# fall back to delete-first. There are ways this can fail and
# lose the builder's history, so we avoid using it in the
# general (non-windows) case
if os.path.exists(filename):
os.unlink(filename)
os.rename(tmpfilename, filename)
except:
log.msg("unable to save build %s-#%d" % (self.builder.name,
self.number))
log.err()
def asDict(self):
result = {}
# Constant
result['builderName'] = self.builder.name
result['number'] = self.getNumber()
result['sourceStamp'] = self.getSourceStamp().asDict()
result['reason'] = self.getReason()
result['blame'] = self.getResponsibleUsers()
# Transient
result['properties'] = self.getProperties().asList()
result['times'] = self.getTimes()
result['text'] = self.getText()
result['results'] = self.getResults()
result['slave'] = self.getSlavename()
# TODO(maruel): Add.
#result['test_results'] = self.getTestResults()
result['logs'] = [[l.getName(),
self.builder.status.getURLForThing(l)] for l in self.getLogs()]
result['eta'] = self.getETA()
result['steps'] = [bss.asDict() for bss in self.steps]
if self.getCurrentStep():
result['currentStep'] = self.getCurrentStep().asDict()
else:
result['currentStep'] = None
return result
class TestProperties(unittest.TestCase):
def setUp(self):
self.props = Properties()
def testDictBehavior(self):
self.props.setProperty("do-tests", 1, "scheduler")
self.props.setProperty("do-install", 2, "scheduler")
self.assert_(self.props.has_key('do-tests'))
self.failUnlessEqual(self.props['do-tests'], 1)
self.failUnlessEqual(self.props['do-install'], 2)
self.assertRaises(KeyError, lambda : self.props['do-nothing'])
self.failUnlessEqual(self.props.getProperty('do-install'), 2)
def testAsList(self):
self.props.setProperty("happiness", 7, "builder")
self.props.setProperty("flames", True, "tester")
self.assertEqual(sorted(self.props.asList()),
[ ('flames', True, 'tester'), ('happiness', 7, 'builder') ])
def testAsDict(self):
self.props.setProperty("msi_filename", "product.msi", 'packager')
self.props.setProperty("dmg_filename", "product.dmg", 'packager')
self.assertEqual(self.props.asDict(),
dict(msi_filename='product.msi', dmg_filename='product.dmg'))
def testUpdate(self):
self.props.setProperty("x", 24, "old")
newprops = { 'a' : 1, 'b' : 2 }
self.props.update(newprops, "new")
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateFromProperties(self):
self.props.setProperty("a", 94, "old")
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new")
newprops.setProperty('b', 2, "new")
self.props.updateFromProperties(newprops)
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateFromPropertiesNoRuntime(self):
self.props.setProperty("a", 94, "old")
self.props.setProperty("b", 84, "old")
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new", runtime=True)
newprops.setProperty('b', 2, "new", runtime=False)
newprops.setProperty('c', 3, "new", runtime=True)
newprops.setProperty('d', 3, "new", runtime=False)
self.props.updateFromPropertiesNoRuntime(newprops)
self.failUnlessEqual(self.props.getProperty('a'), 94)
self.failUnlessEqual(self.props.getPropertySource('a'), 'old')
self.failUnlessEqual(self.props.getProperty('b'), 2)
self.failUnlessEqual(self.props.getPropertySource('b'), 'new')
self.failUnlessEqual(self.props.getProperty('c'), None) # not updated
self.failUnlessEqual(self.props.getProperty('d'), 3)
self.failUnlessEqual(self.props.getPropertySource('d'), 'new')
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
class TestProperties(unittest.TestCase):
def setUp(self):
self.props = Properties()
def testDictBehavior(self):
self.props.setProperty("do-tests", 1, "scheduler")
self.props.setProperty("do-install", 2, "scheduler")
self.assert_(self.props.has_key('do-tests'))
self.failUnlessEqual(self.props['do-tests'], 1)
self.failUnlessEqual(self.props['do-install'], 2)
self.assertRaises(KeyError, lambda : self.props['do-nothing'])
self.failUnlessEqual(self.props.getProperty('do-install'), 2)
def testAsList(self):
self.props.setProperty("happiness", 7, "builder")
self.props.setProperty("flames", True, "tester")
self.assertEqual(sorted(self.props.asList()),
[ ('flames', True, 'tester'), ('happiness', 7, 'builder') ])
def testAsDict(self):
self.props.setProperty("msi_filename", "product.msi", 'packager')
self.props.setProperty("dmg_filename", "product.dmg", 'packager')
self.assertEqual(self.props.asDict(),
dict(msi_filename=('product.msi', 'packager'), dmg_filename=('product.dmg', 'packager')))
def testUpdate(self):
self.props.setProperty("x", 24, "old")
newprops = { 'a' : 1, 'b' : 2 }
self.props.update(newprops, "new")
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateFromProperties(self):
self.props.setProperty("a", 94, "old")
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new")
newprops.setProperty('b', 2, "new")
self.props.updateFromProperties(newprops)
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
self.failUnlessEqual(self.props.getProperty('a'), 1)
self.failUnlessEqual(self.props.getPropertySource('a'), 'new')
def testUpdateFromPropertiesNoRuntime(self):
self.props.setProperty("a", 94, "old")
self.props.setProperty("b", 84, "old")
self.props.setProperty("x", 24, "old")
newprops = Properties()
newprops.setProperty('a', 1, "new", runtime=True)
newprops.setProperty('b', 2, "new", runtime=False)
newprops.setProperty('c', 3, "new", runtime=True)
newprops.setProperty('d', 3, "new", runtime=False)
self.props.updateFromPropertiesNoRuntime(newprops)
self.failUnlessEqual(self.props.getProperty('a'), 94)
self.failUnlessEqual(self.props.getPropertySource('a'), 'old')
self.failUnlessEqual(self.props.getProperty('b'), 2)
self.failUnlessEqual(self.props.getPropertySource('b'), 'new')
self.failUnlessEqual(self.props.getProperty('c'), None) # not updated
self.failUnlessEqual(self.props.getProperty('d'), 3)
self.failUnlessEqual(self.props.getPropertySource('d'), 'new')
self.failUnlessEqual(self.props.getProperty('x'), 24)
self.failUnlessEqual(self.props.getPropertySource('x'), 'old')
class AbstractWorker(service.BuildbotService):
"""This is the master-side representative for a remote buildbot worker.
There is exactly one for each worker described in the config file (the
c['workers'] list). When buildbots connect in (.attach), they get a
reference to this instance. The BotMaster object is stashed as the
.botmaster attribute. The BotMaster is also our '.parent' Service.
I represent a worker -- a remote machine capable of
running builds. I am instantiated by the configuration file, and can be
subclassed to add extra functionality."""
# reconfig workers after builders
reconfig_priority = 64
quarantine_timer = None
quarantine_timeout = quarantine_initial_timeout = 10
quarantine_max_timeout = 60 * 60
start_missing_on_startup = True
DEFAULT_MISSING_TIMEOUT = 3600
DEFAULT_KEEPALIVE_INTERVAL = 3600
# override to True if isCompatibleWithBuild may return False
builds_may_be_incompatible = False
def checkConfig(self, name, password, max_builds=None,
notify_on_missing=None,
missing_timeout=None,
properties=None, defaultProperties=None,
locks=None, keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL):
"""
@param name: botname this machine will supply when it connects
@param password: password this machine will supply when
it connects
@param max_builds: maximum number of simultaneous builds that will
be run concurrently on this worker (the
default is None for no limit)
@param properties: properties that will be applied to builds run on
this worker
@type properties: dictionary
@param defaultProperties: properties that will be applied to builds
run on this worker only if the property
has not been set by another source
@type defaultProperties: dictionary
@param locks: A list of locks that must be acquired before this worker
can be used
@type locks: dictionary
"""
self.name = name = bytes2unicode(name)
self.password = password
# protocol registration
self.registration = None
self._graceful = False
self._paused = False
# these are set when the service is started
self.manager = None
self.workerid = None
self.worker_status = WorkerStatus(name)
self.worker_commands = None
self.workerforbuilders = {}
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
self.lock_subscriptions = []
self.properties = Properties()
self.properties.update(properties or {}, "Worker")
self.properties.setProperty("workername", name, "Worker")
self.defaultProperties = Properties()
self.defaultProperties.update(defaultProperties or {}, "Worker")
self.lastMessageReceived = 0
if notify_on_missing is None:
notify_on_missing = []
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
for i in notify_on_missing:
if not isinstance(i, str):
config.error(
'notify_on_missing arg %r is not a string' % (i,))
self.missing_timeout = missing_timeout
self.missing_timer = None
# a protocol connection, if we're currently connected
self.conn = None
self._old_builder_list = None
self._configured_builderid_list = None
def __repr__(self):
return "<%s %r>" % (self.__class__.__name__, self.name)
@property
def workername(self):
# workername is now an alias to twisted.Service's name
return self.name
@property
def botmaster(self):
if self.master is None:
return None
return self.master.botmaster
def updateLocks(self):
"""Convert the L{LockAccess} objects in C{self.locks} into real lock
objects, while also maintaining the subscriptions to lock releases."""
# unsubscribe from any old locks
for s in self.lock_subscriptions:
s.unsubscribe()
# convert locks into their real form
locks = [(self.botmaster.getLockFromLockAccess(a), a)
for a in self.access]
self.locks = [(l.getLockForWorker(self.workername), la)
for l, la in locks]
self.lock_subscriptions = [l.subscribeToReleases(self._lockReleased)
for l, la in self.locks]
def locksAvailable(self):
"""
I am called to see if all the locks I depend on are available,
in which I return True, otherwise I return False
"""
if not self.locks:
return True
for lock, access in self.locks:
if not lock.isAvailable(self, access):
return False
return True
def acquireLocks(self):
"""
I am called when a build is preparing to run. I try to claim all
the locks that are needed for a build to happen. If I can't, then
my caller should give up the build and try to get another worker
to look at it.
"""
log.msg("acquireLocks(worker %s, locks %s)" % (self, self.locks))
if not self.locksAvailable():
log.msg("worker %s can't lock, giving up" % (self, ))
return False
# all locks are available, claim them all
for lock, access in self.locks:
lock.claim(self, access)
return True
def releaseLocks(self):
"""
I am called to release any locks after a build has finished
"""
log.msg("releaseLocks(%s): %s" % (self, self.locks))
for lock, access in self.locks:
lock.release(self, access)
def _lockReleased(self):
"""One of the locks for this worker was released; try scheduling
builds."""
if not self.botmaster:
return # oh well..
self.botmaster.maybeStartBuildsForWorker(self.name)
def _applyWorkerInfo(self, info):
if not info:
return
self.worker_status.setAdmin(info.get("admin"))
self.worker_status.setHost(info.get("host"))
self.worker_status.setAccessURI(info.get("access_uri", None))
self.worker_status.setVersion(info.get("version", "(unknown)"))
@defer.inlineCallbacks
def _getWorkerInfo(self):
worker = yield self.master.data.get(
('workers', self.workerid))
self._applyWorkerInfo(worker['workerinfo'])
def setServiceParent(self, parent):
# botmaster needs to set before setServiceParent which calls
# startService
self.manager = parent
return super().setServiceParent(parent)
@defer.inlineCallbacks
def startService(self):
self.updateLocks()
self.workerid = yield self.master.data.updates.findWorkerId(
self.name)
self.workerActionConsumer = yield self.master.mq.startConsuming(self.controlWorker,
("control", "worker",
str(self.workerid),
None))
yield self._getWorkerInfo()
yield super().startService()
# startMissingTimer wants the service to be running to really start
if self.start_missing_on_startup:
self.startMissingTimer()
@defer.inlineCallbacks
def reconfigService(self, name, password, max_builds=None,
notify_on_missing=None, missing_timeout=DEFAULT_MISSING_TIMEOUT,
properties=None, defaultProperties=None,
locks=None, keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL):
# Given a Worker config arguments, configure this one identically.
# Because Worker objects are remotely referenced, we can't replace them
# without disconnecting the worker, yet there's no reason to do that.
assert self.name == name
self.password = password
# adopt new instance's configuration parameters
self.max_builds = max_builds
self.access = []
if locks:
self.access = locks
if notify_on_missing is None:
notify_on_missing = []
if isinstance(notify_on_missing, str):
notify_on_missing = [notify_on_missing]
self.notify_on_missing = notify_on_missing
if self.missing_timeout != missing_timeout:
running_missing_timer = self.missing_timer
self.stopMissingTimer()
self.missing_timeout = missing_timeout
if running_missing_timer:
self.startMissingTimer()
self.properties = Properties()
self.properties.update(properties or {}, "Worker")
self.properties.setProperty("workername", name, "Worker")
self.defaultProperties = Properties()
self.defaultProperties.update(defaultProperties or {}, "Worker")
# update our records with the worker manager
if not self.registration:
self.registration = yield self.master.workers.register(self)
yield self.registration.update(self, self.master.config)
self.updateLocks()
@defer.inlineCallbacks
def reconfigServiceWithSibling(self, sibling):
# reconfigServiceWithSibling will only reconfigure the worker when it is configured differently.
# However, the worker configuration depends on which builder it is configured
yield super().reconfigServiceWithSibling(sibling)
# update the attached worker's notion of which builders are attached.
# This assumes that the relevant builders have already been configured,
# which is why the reconfig_priority is set low in this class.
bids = [
b.getBuilderId() for b in self.botmaster.getBuildersForWorker(self.name)]
bids = yield defer.gatherResults(bids, consumeErrors=True)
if self._configured_builderid_list != bids:
yield self.master.data.updates.workerConfigured(self.workerid, self.master.masterid, bids)
yield self.updateWorker()
self._configured_builderid_list = bids
@defer.inlineCallbacks
def stopService(self):
if self.registration:
yield self.registration.unregister()
self.registration = None
self.workerActionConsumer.stopConsuming()
self.stopMissingTimer()
self.stopQuarantineTimer()
# mark this worker as configured for zero builders in this master
yield self.master.data.updates.workerConfigured(self.workerid, self.master.masterid, [])
yield super().stopService()
def isCompatibleWithBuild(self, build_props):
# given a build properties object, determines whether the build is
# compatible with the currently running worker or not. This is most
# often useful for latent workers where it's possible to request
# different kinds of workers.
return defer.succeed(True)
def startMissingTimer(self):
if self.missing_timeout and self.parent and self.running:
self.stopMissingTimer() # in case it's already running
self.missing_timer = self.master.reactor.callLater(self.missing_timeout,
self._missing_timer_fired)
def stopMissingTimer(self):
if self.missing_timer:
if self.missing_timer.active():
self.missing_timer.cancel()
self.missing_timer = None
def isConnected(self):
return self.conn
def _missing_timer_fired(self):
self.missing_timer = None
# notify people, but only if we're still in the config
if not self.parent:
return
last_connection = time.ctime(time.time() - self.missing_timeout)
self.master.data.updates.workerMissing(
workerid=self.workerid,
masterid=self.master.masterid,
last_connection=last_connection,
notify=self.notify_on_missing
)
def updateWorker(self):
"""Called to add or remove builders after the worker has connected.
@return: a Deferred that indicates when an attached worker has
accepted the new builders and/or released the old ones."""
if self.conn:
return self.sendBuilderList()
# else:
return defer.succeed(None)
@defer.inlineCallbacks
def attached(self, conn):
"""This is called when the worker connects."""
assert self.conn is None
metrics.MetricCountEvent.log("AbstractWorker.attached_workers", 1)
# now we go through a sequence of calls, gathering information, then
# tell the Botmaster that it can finally give this worker to all the
# Builders that care about it.
# Reset graceful shutdown status
self._graceful = False
self.conn = conn
self._old_builder_list = None # clear builder list before proceed
self.worker_status.setConnected(True)
self._applyWorkerInfo(conn.info)
self.worker_commands = conn.info.get("worker_commands", {})
self.worker_environ = conn.info.get("environ", {})
self.worker_basedir = conn.info.get("basedir", None)
self.worker_system = conn.info.get("system", None)
self.conn.notifyOnDisconnect(self.detached)
workerinfo = {
'admin': conn.info.get('admin'),
'host': conn.info.get('host'),
'access_uri': conn.info.get('access_uri'),
'version': conn.info.get('version')
}
yield self.master.data.updates.workerConnected(
workerid=self.workerid,
masterid=self.master.masterid,
workerinfo=workerinfo
)
if self.worker_system == "nt":
self.path_module = namedModule("ntpath")
else:
# most everything accepts / as separator, so posix should be a
# reasonable fallback
self.path_module = namedModule("posixpath")
log.msg("bot attached")
self.messageReceivedFromWorker()
self.stopMissingTimer()
yield self.updateWorker()
yield self.botmaster.maybeStartBuildsForWorker(self.name)
self.updateState()
def messageReceivedFromWorker(self):
now = time.time()
self.lastMessageReceived = now
self.worker_status.setLastMessageReceived(now)
def setupProperties(self, props):
for name in self.properties.properties:
props.setProperty(
name, self.properties.getProperty(name), "Worker")
for name in self.defaultProperties.properties:
if name not in props:
props.setProperty(
name, self.defaultProperties.getProperty(name), "Worker")
@defer.inlineCallbacks
def detached(self):
# protect against race conditions in conn disconnect path and someone
# calling detached directly. At the moment the null worker does that.
if self.conn is None:
return
metrics.MetricCountEvent.log("AbstractWorker.attached_workers", -1)
self.conn = None
self._old_builder_list = []
self.worker_status.setConnected(False)
log.msg("Worker.detached(%s)" % (self.name,))
self.releaseLocks()
yield self.master.data.updates.workerDisconnected(
workerid=self.workerid,
masterid=self.master.masterid,
)
def disconnect(self):
"""Forcibly disconnect the worker.
This severs the TCP connection and returns a Deferred that will fire
(with None) when the connection is probably gone.
If the worker is still alive, they will probably try to reconnect
again in a moment.
This is called in two circumstances. The first is when a worker is
removed from the config file. In this case, when they try to
reconnect, they will be rejected as an unknown worker. The second is
when we wind up with two connections for the same worker, in which
case we disconnect the older connection.
"""
if self.conn is None:
return defer.succeed(None)
log.msg("disconnecting old worker %s now" % (self.name,))
# When this Deferred fires, we'll be ready to accept the new worker
return self._disconnect(self.conn)
def _disconnect(self, conn):
# all kinds of teardown will happen as a result of
# loseConnection(), but it happens after a reactor iteration or
# two. Hook the actual disconnect so we can know when it is safe
# to connect the new worker. We have to wait one additional
# iteration (with callLater(0)) to make sure the *other*
# notifyOnDisconnect handlers have had a chance to run.
d = defer.Deferred()
# notifyOnDisconnect runs the callback
def _disconnected():
eventually(d.callback, None)
conn.notifyOnDisconnect(_disconnected)
conn.loseConnection()
log.msg("waiting for worker to finish disconnecting")
return d
@defer.inlineCallbacks
def sendBuilderList(self):
our_builders = self.botmaster.getBuildersForWorker(self.name)
blist = [(b.name, b.config.workerbuilddir) for b in our_builders]
if blist == self._old_builder_list:
return
slist = yield self.conn.remoteSetBuilderList(builders=blist)
self._old_builder_list = blist
# Nothing has changed, so don't need to re-attach to everything
if not slist:
return
dl = []
for name in slist:
# use get() since we might have changed our mind since then
b = self.botmaster.builders.get(name)
if b:
d1 = self.attachBuilder(b)
dl.append(d1)
yield defer.DeferredList(dl)
def attachBuilder(self, builder):
return builder.attached(self, self.worker_commands)
def controlWorker(self, key, params):
log.msg("worker {} wants to {}: {}".format(self.name, key[-1], params))
if key[-1] == "stop":
return self.shutdownRequested()
if key[-1] == "pause":
self.pause()
if key[-1] == "unpause":
self.unpause()
if key[-1] == "kill":
self.shutdown()
def shutdownRequested(self):
self._graceful = True
self.maybeShutdown()
self.updateState()
def addWorkerForBuilder(self, wfb):
self.workerforbuilders[wfb.builder_name] = wfb
def removeWorkerForBuilder(self, wfb):
try:
del self.workerforbuilders[wfb.builder_name]
except KeyError:
pass
def buildFinished(self, wfb):
"""This is called when a build on this worker is finished."""
self.botmaster.maybeStartBuildsForWorker(self.name)
def canStartBuild(self):
"""
I am called when a build is requested to see if this worker
can start a build. This function can be used to limit overall
concurrency on the worker.
Note for subclassers: if a worker can become willing to start a build
without any action on that worker (for example, by a resource in use on
another worker becoming available), then you must arrange for
L{maybeStartBuildsForWorker} to be called at that time, or builds on
this worker will not start.
"""
# If we're waiting to shutdown gracefully or paused, then we shouldn't
# accept any new jobs.
if self._graceful or self._paused:
return False
if self.max_builds:
active_builders = [wfb for wfb in self.workerforbuilders.values()
if wfb.isBusy()]
if len(active_builders) >= self.max_builds:
return False
if not self.locksAvailable():
return False
return True
@defer.inlineCallbacks
def shutdown(self):
"""Shutdown the worker"""
if not self.conn:
log.msg("no remote; worker is already shut down")
return
yield self.conn.remoteShutdown()
def maybeShutdown(self):
"""Shut down this worker if it has been asked to shut down gracefully,
and has no active builders."""
if not self._graceful:
return
active_builders = [wfb for wfb in self.workerforbuilders.values()
if wfb.isBusy()]
if active_builders:
return
d = self.shutdown()
d.addErrback(log.err, 'error while shutting down worker')
def updateState(self):
self.master.data.updates.setWorkerState(self.workerid, self._paused, self._graceful)
def pause(self):
"""Stop running new builds on the worker."""
self._paused = True
self.updateState()
def unpause(self):
"""Restart running new builds on the worker."""
self._paused = False
self.botmaster.maybeStartBuildsForWorker(self.name)
self.updateState()
def isPaused(self):
return self._paused
def resetQuarantine(self):
self.quarantine_timeout = self.quarantine_initial_timeout
def putInQuarantine(self):
if self.quarantine_timer: # already in quarantine
return
self.pause()
self.quarantine_timer = self.master.reactor.callLater(
self.quarantine_timeout, self.exitQuarantine)
log.msg("{} has been put in quarantine for {}s".format(
self.name, self.quarantine_timeout))
# next we will wait twice as long
self.quarantine_timeout *= 2
if self.quarantine_timeout > self.quarantine_max_timeout:
# unless we hit the max timeout
self.quarantine_timeout = self.quarantine_max_timeout
def exitQuarantine(self):
self.quarantine_timer = None
self.unpause()
def stopQuarantineTimer(self):
if self.quarantine_timer is not None:
self.quarantine_timer.cancel()
self.quarantine_timer = None
self.unpause()
