def _startBuildFailed(self, why, build, slavebuilder):
# put the build back on the buildable list
log.msg("I tried to tell the slave that the build %s started, but "
"remote_startBuild failed: %s" % (build, why))
# release the slave. This will queue a call to maybeStartBuild, which
# will fire after other notifyOnDisconnect handlers have marked the
# slave as disconnected (so we don't try to use it again).
slavebuilder.buildFinished()
log.msg("re-queueing the BuildRequest")
self.building.remove(build)
self._resubmit_buildreqs(build).addErrback(log.err)
def _startBuildFailed(self, why, build, slavebuilder):
# put the build back on the buildable list
log.msg("I tried to tell the slave that the build %s started, but "
"remote_startBuild failed: %s" % (build, why))
# release the slave. This will queue a call to maybeStartBuild, which
# will fire after other notifyOnDisconnect handlers have marked the
# slave as disconnected (so we don't try to use it again).
slavebuilder.buildFinished()
log.msg("re-queueing the BuildRequest")
self.building.remove(build)
self._resubmit_buildreqs(build).addErrback(log.err)
def _startBuildFor(self, slavebuilder, buildrequests):
"""Start a build on the given slave.
@param build: the L{base.Build} to start
@param sb: the L{SlaveBuilder} which will host this build
@return: (via Deferred) boolean indicating that the build was
succesfully started.
"""
# as of the Python versions supported now, try/finally can't be used
# with a generator expression. So instead, we push cleanup functions
# into a list so that, at any point, we can abort this operation.
cleanups = []
def run_cleanups():
try:
while cleanups:
fn = cleanups.pop()
fn()
except:
log.err(failure.Failure(), "while running %r" % (run_cleanups,))
# the last cleanup we want to perform is to update the big
# status based on any other cleanup
cleanups.append(lambda : self.updateBigStatus())
build = self.config.factory.newBuild(buildrequests)
build.setBuilder(self)
log.msg("starting build %s using slave %s" % (build, slavebuilder))
# set up locks
build.setLocks(self.config.locks)
cleanups.append(lambda : slavebuilder.slave.releaseLocks())
if len(self.config.env) > 0:
build.setSlaveEnvironment(self.config.env)
# append the build to self.building
self.building.append(build)
cleanups.append(lambda : self.building.remove(build))
# update the big status accordingly
self.updateBigStatus()
try:
ready = yield slavebuilder.prepare(self.builder_status, build)
except:
log.err(failure.Failure(), 'while preparing slavebuilder:')
ready = False
# If prepare returns True then it is ready and we start a build
# If it returns false then we don't start a new build.
if not ready:
log.msg("slave %s can't build %s after all; re-queueing the "
"request" % (build, slavebuilder))
run_cleanups()
defer.returnValue(False)
return
# ping the slave to make sure they're still there. If they've
# fallen off the map (due to a NAT timeout or something), this
# will fail in a couple of minutes, depending upon the TCP
# timeout.
#
# TODO: This can unnecessarily suspend the starting of a build, in
# situations where the slave is live but is pushing lots of data to
# us in a build.
log.msg("starting build %s.. pinging the slave %s"
% (build, slavebuilder))
try:
ping_success = yield slavebuilder.ping()
except:
log.err(failure.Failure(), 'while pinging slave before build:')
ping_success = False
if not ping_success:
log.msg("slave ping failed; re-queueing the request")
run_cleanups()
defer.returnValue(False)
return
# The buildslave is ready to go. slavebuilder.buildStarted() sets its
# state to BUILDING (so we won't try to use it for any other builds).
# This gets set back to IDLE by the Build itself when it finishes.
slavebuilder.buildStarted()
cleanups.append(lambda : slavebuilder.buildFinished())
# tell the remote that it's starting a build, too
try:
yield slavebuilder.remote.callRemote("startBuild")
except:
log.err(failure.Failure(), 'while calling remote startBuild:')
run_cleanups()
defer.returnValue(False)
return
# create the BuildStatus object that goes with the Build
bs = self.builder_status.newBuild()
# record the build in the db - one row per buildrequest
try:
bids = []
for req in build.requests:
bid = yield self.master.db.builds.addBuild(req.id, bs.number)
bids.append(bid)
except:
log.err(failure.Failure(), 'while adding rows to build table:')
run_cleanups()
defer.returnValue(False)
return
# let status know
self.master.status.build_started(req.id, self.name, bs)
# start the build. This will first set up the steps, then tell the
# BuildStatus that it has started, which will announce it to the world
# (through our BuilderStatus object, which is its parent). Finally it
# will start the actual build process. This is done with a fresh
# Deferred since _startBuildFor should not wait until the build is
# finished.
d = build.startBuild(bs, self.expectations, slavebuilder)
d.addCallback(self.buildFinished, slavebuilder, bids)
# this shouldn't happen. if it does, the slave will be wedged
d.addErrback(log.err)
# make sure the builder's status is represented correctly
self.updateBigStatus()
defer.returnValue(True)
def _startBuildFor(self, slavebuilder, buildrequests):
# Build a stack of cleanup functions so that, at any point, we can
# abort this operation and unwind the commitments made so far.
cleanups = []
def run_cleanups():
try:
while cleanups:
fn = cleanups.pop()
fn()
except Exception:
log.err(failure.Failure(), "while running %r" % (run_cleanups,))
# the last cleanup we want to perform is to update the big
# status based on any other cleanup
cleanups.append(lambda: self.updateBigStatus())
build = self.config.factory.newBuild(buildrequests)
build.setBuilder(self)
log.msg("starting build %s using slave %s" % (build, slavebuilder))
# set up locks
build.setLocks(self.config.locks)
cleanups.append(slavebuilder.slave.releaseLocks)
if len(self.config.env) > 0:
build.setSlaveEnvironment(self.config.env)
# append the build to self.building
self.building.append(build)
cleanups.append(lambda: self.building.remove(build))
# update the big status accordingly
self.updateBigStatus()
try:
ready = yield slavebuilder.prepare(self.builder_status, build)
except Exception:
log.err(failure.Failure(), 'while preparing slavebuilder:')
ready = False
# If prepare returns True then it is ready and we start a build
# If it returns false then we don't start a new build.
if not ready:
log.msg("slave %s can't build %s after all; re-queueing the "
"request" % (build, slavebuilder))
run_cleanups()
defer.returnValue(False)
return
# ping the slave to make sure they're still there. If they've
# fallen off the map (due to a NAT timeout or something), this
# will fail in a couple of minutes, depending upon the TCP
# timeout.
#
# TODO: This can unnecessarily suspend the starting of a build, in
# situations where the slave is live but is pushing lots of data to
# us in a build.
log.msg("starting build %s.. pinging the slave %s"
% (build, slavebuilder))
try:
ping_success = yield slavebuilder.ping()
except Exception:
log.err(failure.Failure(), 'while pinging slave before build:')
ping_success = False
if not ping_success:
log.msg("slave ping failed; re-queueing the request")
run_cleanups()
defer.returnValue(False)
return
# The buildslave is ready to go. slavebuilder.buildStarted() sets its
# state to BUILDING (so we won't try to use it for any other builds).
# This gets set back to IDLE by the Build itself when it finishes.
slavebuilder.buildStarted()
cleanups.append(lambda: slavebuilder.buildFinished())
# tell the remote that it's starting a build, too
try:
yield slavebuilder.slave.conn.remoteStartBuild(build.builder.name)
except Exception:
log.err(failure.Failure(), 'while calling remote startBuild:')
run_cleanups()
defer.returnValue(False)
return
# create the BuildStatus object that goes with the Build
bs = self.builder_status.newBuild()
# IMPORTANT: no yielding is allowed from here to the startBuild call!
# it's possible that we lost the slave remote between the ping above
# and now. If so, bail out. The build.startBuild call below transfers
# responsibility for monitoring this connection to the Build instance,
# so this check ensures we hand off a working connection.
if not slavebuilder.slave.conn: # TODO: replace with isConnected()
log.msg("slave disappeared before build could start")
run_cleanups()
defer.returnValue(False)
return
# let status know
self.master.status.build_started(buildrequests[0].id, self.name, bs)
# start the build. This will first set up the steps, then tell the
# BuildStatus that it has started, which will announce it to the world
# (through our BuilderStatus object, which is its parent). Finally it
# will start the actual build process. This is done with a fresh
# Deferred since _startBuildFor should not wait until the build is
# finished. This uses `maybeDeferred` to ensure that any exceptions
# raised by startBuild are treated as deferred errbacks (see
# http://trac.buildbot.net/ticket/2428).
d = defer.maybeDeferred(build.startBuild,
bs, self.expectations, slavebuilder)
d.addCallback(lambda _: self.buildFinished(build, slavebuilder))
# this shouldn't happen. if it does, the slave will be wedged
d.addErrback(log.err, 'from a running build; this is a '
'serious error - please file a bug at http://buildbot.net')
# make sure the builder's status is represented correctly
self.updateBigStatus()
defer.returnValue(True)
def _startBuildFor(self, slavebuilder, buildrequests):
"""Start a build on the given slave.
@param build: the L{base.Build} to start
@param sb: the L{SlaveBuilder} which will host this build
@return: (via Deferred) boolean indicating that the build was
succesfully started.
"""
# as of the Python versions supported now, try/finally can't be used
# with a generator expression. So instead, we push cleanup functions
# into a list so that, at any point, we can abort this operation.
cleanups = []
def run_cleanups():
while cleanups:
fn = cleanups.pop()
fn()
# the last cleanup we want to perform is to update the big
# status based on any other cleanup
cleanups.append(lambda: self.updateBigStatus())
build = self.config.factory.newBuild(buildrequests)
build.setBuilder(self)
log.msg("starting build %s using slave %s" % (build, slavebuilder))
# set up locks
build.setLocks(self.config.locks)
cleanups.append(lambda: slavebuilder.slave.releaseLocks())
if len(self.config.env) > 0:
build.setSlaveEnvironment(self.config.env)
# append the build to self.building
self.building.append(build)
cleanups.append(lambda: self.building.remove(build))
# update the big status accordingly
self.updateBigStatus()
try:
wfd = defer.waitForDeferred(
slavebuilder.prepare(self.builder_status, build))
yield wfd
ready = wfd.getResult()
except:
log.err(failure.Failure(), 'while preparing slavebuilder:')
ready = False
# If prepare returns True then it is ready and we start a build
# If it returns false then we don't start a new build.
if not ready:
log.msg("slave %s can't build %s after all; re-queueing the "
"request" % (build, slavebuilder))
run_cleanups()
yield False
return
# ping the slave to make sure they're still there. If they've
# fallen off the map (due to a NAT timeout or something), this
# will fail in a couple of minutes, depending upon the TCP
# timeout.
#
# TODO: This can unnecessarily suspend the starting of a build, in
# situations where the slave is live but is pushing lots of data to
# us in a build.
log.msg("starting build %s.. pinging the slave %s" %
(build, slavebuilder))
try:
wfd = defer.waitForDeferred(slavebuilder.ping())
yield wfd
ping_success = wfd.getResult()
except:
log.err(failure.Failure(), 'while pinging slave before build:')
ping_success = False
if not ping_success:
log.msg("slave ping failed; re-queueing the request")
run_cleanups()
yield False
return
# The buildslave is ready to go. slavebuilder.buildStarted() sets its
# state to BUILDING (so we won't try to use it for any other builds).
# This gets set back to IDLE by the Build itself when it finishes.
slavebuilder.buildStarted()
cleanups.append(lambda: slavebuilder.buildFinished())
# tell the remote that it's starting a build, too
try:
wfd = defer.waitForDeferred(
slavebuilder.remote.callRemote("startBuild"))
yield wfd
wfd.getResult()
except:
log.err(failure.Failure(), 'while calling remote startBuild:')
run_cleanups()
yield False
return
# create the BuildStatus object that goes with the Build
bs = self.builder_status.newBuild()
# record the build in the db - one row per buildrequest
try:
bids = []
for req in build.requests:
wfd = defer.waitForDeferred(
self.master.db.builds.addBuild(req.id, bs.number))
yield wfd
bids.append(wfd.getResult())
except:
log.err(failure.Failure(), 'while adding rows to build table:')
run_cleanups()
yield False
return
# let status know
self.master.status.build_started(req.id, self.name, bs)
# start the build. This will first set up the steps, then tell the
# BuildStatus that it has started, which will announce it to the world
# (through our BuilderStatus object, which is its parent). Finally it
# will start the actual build process. This is done with a fresh
# Deferred since _startBuildFor should not wait until the build is
# finished.
d = build.startBuild(bs, self.expectations, slavebuilder)
d.addCallback(self.buildFinished, slavebuilder, bids)
# this shouldn't happen. if it does, the slave will be wedged
d.addErrback(log.err)
# make sure the builder's status is represented correctly
self.updateBigStatus()
yield True
def _startBuildFor(self, slavebuilder, buildrequests):
# Build a stack of cleanup functions so that, at any point, we can
# abort this operation and unwind the commitments made so far.
cleanups = []
def run_cleanups():
try:
while cleanups:
fn = cleanups.pop()
fn()
except:
log.err(failure.Failure(),
"while running %r" % (run_cleanups, ))
# the last cleanup we want to perform is to update the big
# status based on any other cleanup
cleanups.append(lambda: self.updateBigStatus())
build = self.config.factory.newBuild(buildrequests)
build.setBuilder(self)
log.msg("starting build %s using slave %s" % (build, slavebuilder))
# set up locks
build.setLocks(self.config.locks)
cleanups.append(lambda: slavebuilder.slave.releaseLocks())
if len(self.config.env) > 0:
build.setSlaveEnvironment(self.config.env)
# append the build to self.building
self.building.append(build)
cleanups.append(lambda: self.building.remove(build))
# update the big status accordingly
self.updateBigStatus()
try:
ready = yield slavebuilder.prepare(self.builder_status, build)
except:
log.err(failure.Failure(), 'while preparing slavebuilder:')
ready = False
# If prepare returns True then it is ready and we start a build
# If it returns false then we don't start a new build.
if not ready:
log.msg("slave %s can't build %s after all; re-queueing the "
"request" % (build, slavebuilder))
run_cleanups()
defer.returnValue(False)
return
# ping the slave to make sure they're still there. If they've
# fallen off the map (due to a NAT timeout or something), this
# will fail in a couple of minutes, depending upon the TCP
# timeout.
#
# TODO: This can unnecessarily suspend the starting of a build, in
# situations where the slave is live but is pushing lots of data to
# us in a build.
log.msg("starting build %s.. pinging the slave %s" %
(build, slavebuilder))
try:
ping_success = yield slavebuilder.ping()
except:
log.err(failure.Failure(), 'while pinging slave before build:')
ping_success = False
if not ping_success:
log.msg("slave ping failed; re-queueing the request")
run_cleanups()
defer.returnValue(False)
return
# The buildslave is ready to go. slavebuilder.buildStarted() sets its
# state to BUILDING (so we won't try to use it for any other builds).
# This gets set back to IDLE by the Build itself when it finishes.
slavebuilder.buildStarted()
cleanups.append(lambda: slavebuilder.buildFinished())
# tell the remote that it's starting a build, too
try:
yield slavebuilder.slave.conn.remoteStartBuild(build.builder.name)
except:
log.err(failure.Failure(), 'while calling remote startBuild:')
run_cleanups()
defer.returnValue(False)
return
# create the BuildStatus object that goes with the Build
bs = self.builder_status.newBuild()
# IMPORTANT: no yielding is allowed from here to the startBuild call!
# it's possible that we lost the slave remote between the ping above
# and now. If so, bail out. The build.startBuild call below transfers
# responsibility for monitoring this connection to the Build instance,
# so this check ensures we hand off a working connection.
if not slavebuilder.slave.conn: # TODO: replace with isConnected()
log.msg("slave disappeared before build could start")
run_cleanups()
defer.returnValue(False)
return
# let status know
self.master.status.build_started(buildrequests[0].id, self.name, bs)
# start the build. This will first set up the steps, then tell the
# BuildStatus that it has started, which will announce it to the world
# (through our BuilderStatus object, which is its parent). Finally it
# will start the actual build process. This is done with a fresh
# Deferred since _startBuildFor should not wait until the build is
# finished. This uses `maybeDeferred` to ensure that any exceptions
# raised by startBuild are treated as deferred errbacks (see
# http://trac.buildbot.net/ticket/2428).
d = defer.maybeDeferred(build.startBuild, bs, self.expectations,
slavebuilder)
d.addCallback(lambda _: self.buildFinished(build, slavebuilder))
# this shouldn't happen. if it does, the slave will be wedged
d.addErrback(
log.err, 'from a running build; this is a '
'serious error - please file a bug at http://buildbot.net')
# make sure the builder's status is represented correctly
self.updateBigStatus()
defer.returnValue(True)
def _startBuildFor(self, slavebuilder, buildrequests, build_status=None):
"""Start a build on the given slave.
@param build: the L{base.Build} to start
@param sb: the L{SlaveBuilder} which will host this build
@return: (via Deferred) boolean indicating that the build was
succesfully started.
"""
# as of the Python versions supported now, try/finally can't be used
# with a generator expression. So instead, we push cleanup functions
# into a list so that, at any point, we can abort this operation.
cleanups = []
def run_cleanups():
try:
while cleanups:
fn = cleanups.pop()
fn()
except:
log.err(failure.Failure(),
"while running %r" % (run_cleanups, ))
# the last cleanup we want to perform is to update the big
# status based on any other cleanup
cleanups.append(lambda: self.updateBigStatus())
build = self.config.factory.newBuild(buildrequests)
build.setBuilder(self)
log.msg("starting build %s using slave %s" % (build, slavebuilder))
# set up locks
build.setLocks(self.config.locks)
cleanups.append(lambda: slavebuilder.slave.releaseLocks()
if slavebuilder.slave else None)
if len(self.config.env) > 0:
build.setSlaveEnvironment(self.config.env)
# append the build to self.building
self.building.append(build)
cleanups.append(lambda: self.building.remove(build))
# update the big status accordingly
self.updateBigStatus()
# ping the slave to make sure they're still there. If they've
# fallen off the map (due to a NAT timeout or something), this
# will fail in a couple of minutes, depending upon the TCP
# timeout.
#
# TODO: This can unnecessarily suspend the starting of a build, in
# situations where the slave is live but is pushing lots of data to
# us in a build.
log.msg("starting build %s.. pinging the slave %s" %
(build, slavebuilder))
try:
ping_success = yield slavebuilder.ping(
timeout=self.master.config.remoteCallTimeout)
except:
log.err(failure.Failure(), 'while pinging slave before build:')
raise
if not ping_success:
log.msg("build %s slave %s ping failed; re-queueing the request" %
(build, slavebuilder))
run_cleanups()
raise Exception("Ping failed")
#check slave is still available
ready = slavebuilder.isAvailable()
if ready:
try:
ready = yield slavebuilder.prepare(self.builder_status, build)
except:
log.err(failure.Failure(), 'while preparing slavebuilder:')
raise
# If prepare returns True then it is ready and we start a build
# If it returns false then we don't start a new build.
if not ready:
log.msg("slave %s can't build %s after all; re-queueing the "
"request" % (build, slavebuilder))
run_cleanups()
raise Exception("Unknown")
# The buildslave is ready to go. slavebuilder.buildStarted() sets its
# state to BUILDING (so we won't try to use it for any other builds).
# This gets set back to IDLE by the Build itself when it finishes.
if slavebuilder.buildStarted():
cleanups.append(lambda: slavebuilder.buildFinished())
else:
log.msg("slave %s can't build %s after all; re-queueing the "
"request" % (build, slavebuilder))
run_cleanups()
raise Exception("Unknown")
# create the BuildStatus object that goes with the Build
if build_status is None:
bs = self.builder_status.newBuild()
else:
bs = build_status
bs.builder = self.builder_status
bs.slavename = slavebuilder.slave.slavename
bs.waitUntilFinished().addCallback(
self.builder_status._buildFinished)
# update the steps to use finished steps
# record the build in the db - one row per buildrequest
try:
bids = []
if len(build.requests) > 0:
main_br = build.requests[0]
bid = yield self.master.db.builds.addBuild(
main_br.id, bs.number, slavebuilder.slave.slavename)
bids.append(bid)
# add build information to merged br
for req in build.requests[1:]:
bid = yield self.master.db.builds.addBuild(
req.id, bs.number)
self.master.status.build_started(req.id, self.name, bs)
bids.append(bid)
except:
log.err(failure.Failure(), 'while adding rows to build table:')
run_cleanups()
raise
# IMPORTANT: no yielding is allowed from here to the startBuild call!
# it's possible that we lost the slave remote between the ping above
# and now. If so, bail out. The build.startBuild call below transfers
# responsibility for monitoring this connection to the Build instance,
# so this check ensures we hand off a working connection.
if not slavebuilder.remote:
log.msg("slave disappeared before build could start")
run_cleanups()
raise Exception("Slave seems to have disappered")
# let status know
self.master.status.build_started(main_br.id, self.name, bs)
# start the build. This will first set up the steps, then tell the
# BuildStatus that it has started, which will announce it to the world
# (through our BuilderStatus object, which is its parent). Finally it
# will start the actual build process. This is done with a fresh
# Deferred since _startBuildFor should not wait until the build is
# finished. This uses `maybeDeferred` to ensure that any exceptions
# raised by startBuild are treated as deferred errbacks (see
# http://trac.buildbot.net/ticket/2428).
d = defer.maybeDeferred(build.startBuild, bs, self.expectations,
slavebuilder)
d.addCallback(self.buildFinished, slavebuilder, bids)
# this shouldn't happen. if it does, the slave will be wedged
d.addErrback(
log.err, 'from a running build; this is a '
'serious error - please file a bug at http://buildbot.net')
# make sure the builder's status is represented correctly
self.updateBigStatus()
defer.returnValue(True)
