def start_instance(self, build):
"""
I start a new instance of a VM.
If a base_image is specified, I will make a clone of that otherwise i will
use image directly.
If i'm not given libvirt domain definition XML, I will look for my name
in the list of defined virtual machines and start that.
"""
if self.domain is not None:
log.msg("Cannot start_instance '%s' as already active" % self.name)
defer.returnValue(False)
yield self._prepare_base_image()
try:
if self.xml:
self.domain = yield self.connection.create(self.xml)
else:
self.domain = yield self.connection.lookupByName(self.name)
yield self.domain.create()
except:
klog.err_json(
failure.Failure(),
"Cannot start a VM (%s), failing gracefully and triggering"
"a new build check" % self.name)
self.domain = None
defer.returnValue(False)
defer.returnValue(True)
def changeCallback(change):
# ignore changes delivered while we're not running
if not self._change_subscription:
return
if change_filter and not change_filter.filter_change(change):
return
if change.codebase not in self.codebases:
log.msg(format='change contains codebase %(codebase)s that is'
'not processed by scheduler %(name)s',
codebase=change.codebase, name=self.name)
return
if fileIsImportant:
try:
important = fileIsImportant(change)
if not important and onlyImportant:
return
except:
klog.err_json(failure.Failure(),
'in fileIsImportant check for %s' % change)
return
else:
important = True
# use change_consumption_lock to ensure the service does not stop
# while this change is being processed
d = self._change_consumption_lock.run(self.gotChange, change, important)
d.addErrback(klog.err_json, 'while processing change')
def old_way():
d = None
for b in self.slavebuilders.values():
if b.remote:
d = b.remote.callRemote("shutdown")
break
if d:
log.msg("Shutting down (old) slave: %s" % self.slavename)
# The remote shutdown call will not complete successfully since the
# buildbot process exits almost immediately after getting the
# shutdown request.
# Here we look at the reason why the remote call failed, and if
# it's because the connection was lost, that means the slave
# shutdown as expected.
def _errback(why):
if why.check(pb.PBConnectionLost):
log.msg("Lost connection to %s" % self.slavename)
else:
klog.err_json(
"Unexpected error when trying to shutdown %s" %
self.slavename)
d.addErrback(_errback)
return d
klog.err_json("Couldn't find remote builder to shut down slave")
return defer.succeed(None)
def rmdirRecursive(dir):
"""This is a replacement for shutil.rmtree that works better under
windows. Thanks to Bear at the OSAF for the code."""
if not os.path.exists(dir):
return
if os.path.islink(dir) or os.path.isfile(dir):
os.remove(dir)
return
# Verify the directory is read/write/execute for the current user
os.chmod(dir, 0700)
# os.listdir below only returns a list of unicode filenames if the parameter is unicode
# Thus, if a non-unicode-named dir contains a unicode filename, that filename will get garbled.
# So force dir to be unicode.
if not isinstance(dir, unicode):
try:
dir = unicode(dir, "utf-8")
except:
klog.err_json(
"rmdirRecursive: decoding from UTF-8 failed (ignoring)")
try:
list = os.listdir(dir)
except WindowsError, e:
msg = ("rmdirRecursive: unable to listdir %s (%s). Trying to "
"remove like a dir" % (dir, e.strerror.decode('mbcs')))
log.msg(msg.encode('utf-8'))
os.rmdir(dir)
return
def _not_attached(self, why, slave):
# already log.err'ed by SlaveBuilder._attachFailure
# TODO: remove from self.slaves (except that detached() should get
# run first, right?)
klog.err_json(why, 'slave failed to attach')
self.builder_status.addPointEvent(
['failed', 'connect', slave.slavename])
def _errback(why):
if why.check(pb.PBConnectionLost):
log.msg("Lost connection to %s" % self.slavename)
else:
klog.err_json(
"Unexpected error when trying to shutdown %s" %
self.slavename)
def error_thd(err):
msg = "There was an error in createBuildUserStatus. Project: {proj}; Builder: {builder}"
klog.err_json(err,
_why=msg.format(
proj=step.builder.project,
builder=step.builder.friendly_name,
))
def buildStarted(self, s):
"""Now the BuildStatus object is ready to go (it knows all of its
Steps, its ETA, etc), so it is safe to notify our watchers."""
assert s.builder is self # paranoia
assert s not in self.currentBuilds
self.currentBuilds.append(s)
self.buildCache.get(s.number, val=s)
# now that the BuildStatus is prepared to answer queries, we can
# announce the new build to all our watchers
for w in self.watchers: # TODO: maybe do this later? callLater(0)?
try:
receiver = w.buildStarted(self.getName(), s)
if receiver:
if type(receiver) == type(()):
s.subscribe(receiver[0], receiver[1])
else:
s.subscribe(receiver)
d = s.waitUntilFinished()
d.addCallback(lambda s: s.unsubscribe(receiver))
except:
log.msg("Exception caught notifying %r of buildStarted event" %
w)
klog.err_json()
def handleJobFile(self, filename, f):
try:
parsed_job = self.parseJob(f)
builderNames = parsed_job['builderNames']
except BadJobfile:
log.msg("%s reports a bad jobfile in %s" % (self, filename))
klog.err_json()
return defer.succeed(None)
# Validate/fixup the builder names.
builderNames = self.filterBuilderList(builderNames)
if not builderNames:
log.msg(
"incoming Try job did not specify any allowed builder names")
return defer.succeed(None)
who = ""
if parsed_job['who']:
who = parsed_job['who']
comment = ""
if parsed_job['comment']:
comment = parsed_job['comment']
d = self.master.db.sourcestampsets.addSourceStampSet()
def addsourcestamp(setid):
self.master.db.sourcestamps.addSourceStamp(
sourcestampsetid=setid,
branch=parsed_job['branch'],
revision=parsed_job['baserev'],
patch_body=parsed_job['patch_body'],
patch_level=parsed_job['patch_level'],
patch_author=who,
patch_comment=comment,
patch_subdir='', # TODO: can't set this remotely - #1769
project=parsed_job['project'],
repository=parsed_job['repository'])
return setid
d.addCallback(addsourcestamp)
def create_buildset(setid):
reason = "'try' job"
if parsed_job['who']:
reason += " by user %s" % parsed_job['who']
properties = parsed_job['properties']
requested_props = Properties()
requested_props.update(properties, "try build")
return self.addBuildsetForSourceStamp(
ssid=None,
setid=setid,
reason=reason,
external_idstring=parsed_job['jobid'],
builderNames=builderNames,
properties=requested_props)
d.addCallback(create_buildset)
return d
def builderAdded(self, name, basedir, category=None, friendly_name=None, description=None, project=None):
"""
@rtype: L{BuilderStatus}
"""
filename = os.path.join(self.basedir, basedir, "builder")
log.msg("trying to load status pickle from %s" % filename)
builder_status = None
if friendly_name is None:
friendly_name = name
try:
with open(filename, "rb") as f:
builder_status = load(f)
builder_status.master = self.master
builder_status.basedir = os.path.join(self.basedir, basedir)
# (bug #1068) if we need to upgrade, we probably need to rewrite
# this pickle, too. We determine this by looking at the list of
# Versioned objects that have been unpickled, and (after doUpgrade)
# checking to see if any of them set wasUpgraded. The Versioneds'
# upgradeToVersionNN methods all set this.
versioneds = styles.versionedsToUpgrade
styles.doUpgrade()
if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]:
log.msg("re-writing upgraded builder pickle")
builder_status.saveYourself()
except IOError:
log.msg("no saved status pickle, creating a new one")
except:
log.msg("error while loading status pickle, creating a new one")
log.msg("error follows:")
klog.err_json()
if not builder_status:
builder_status = builder.BuilderStatus(name, category, self.master, friendly_name,
description, project=project)
builder_status.addPointEvent(["builder", "created"])
log.msg("added builder %s in category %s" % (name, category))
# an unpickled object might not have category set from before,
# so set it here to make sure
builder_status.category = category
builder_status.description = description
builder_status.master = self.master
builder_status.basedir = os.path.join(self.basedir, basedir)
builder_status.name = name # it might have been updated
builder_status.setStatus(self)
builder_status.friendly_name = friendly_name
if not os.path.isdir(builder_status.basedir):
os.makedirs(builder_status.basedir)
builder_status.determineNextBuildNumber()
builder_status.setBigState("offline")
for t in self.watchers:
self.announceNewBuilder(t, name, builder_status)
return builder_status
def __init__(self,
svnurl,
split_file=None,
svnuser=None,
svnpasswd=None,
pollInterval=10 * 60,
histmax=100,
svnbin='svn',
revlinktmpl='',
category=None,
project='',
cachepath=None,
pollinterval=-2,
extra_args=None):
# for backward compatibility; the parameter used to be spelled with 'i'
if pollinterval != -2:
pollInterval = pollinterval
base.PollingChangeSource.__init__(self,
name=svnurl,
pollInterval=pollInterval)
if svnurl.endswith("/"):
svnurl = svnurl[:-1] # strip the trailing slash
self.svnurl = svnurl
self.extra_args = extra_args
self.split_file = split_file or split_file_alwaystrunk
self.svnuser = svnuser
self.svnpasswd = svnpasswd
self.revlinktmpl = revlinktmpl
self.environ = os.environ.copy() # include environment variables
# required for ssh-agent auth
self.svnbin = svnbin
self.histmax = histmax
self._prefix = None
self.category = category
self.project = project
self.cachepath = cachepath
if self.cachepath and os.path.exists(self.cachepath):
try:
with open(self.cachepath, "r") as f:
self.last_change = int(f.read().strip())
log.msg(
"SVNPoller: SVNPoller(%s) setting last_change to %s" %
(self.svnurl, self.last_change))
# try writing it, too
with open(self.cachepath, "w") as f:
f.write(str(self.last_change))
except:
self.cachepath = None
log.msg((
"SVNPoller: SVNPoller(%s) cache file corrupt or unwriteable; "
+ "skipping and not using") % self.svnurl)
klog.err_json()
def run_cleanups():
try:
while cleanups:
fn = cleanups.pop()
fn()
except:
klog.err_json(failure.Failure(),
"while running %r" % (run_cleanups, ))
def handleFileRotation(self):
if self.shouldRotate():
try:
self.flush()
self.rotate()
except:
klog.err_json(failure.Failure(), 'Rotate build log file failed')
self._checkFileIsOpen()
def _maybe_retry(e):
klog.err_json()
if attempt < self.max_attempts:
reactor.callLater(attempt * self.retry_multiplier,
self._retrying_fetch, u, data, event,
attempt + 1)
else:
return e
def deliver(self, *args, **kwargs):
for sub in list(self.subscriptions):
try:
sub.callback(*args, **kwargs)
except:
klog.err_json(
failure.Failure(),
'while invoking callback %s to %s' % (sub.callback, self))
def queueNextServerPush(self):
"""Queue the next push or call it immediately.
Called to signal new items are available to be sent or on shutdown.
A timer should be queued to trigger a network request or the callback
should be called immediately. If a status push is already queued, ignore
the current call."""
# Determine the delay.
if self.wasLastPushSuccessful():
if self.stopped:
# Shutting down.
delay = 0
else:
# Normal case.
delay = self.bufferDelay
else:
if self.stopped:
# Too bad, we can't do anything now, we're shutting down and the
# receiver is also down. We'll just save the objects to disk.
return
else:
# The server is inaccessible, retry less often.
delay = self.retryDelay
# Cleanup a previously queued task if necessary.
if self.task:
# Warning: we could be running inside the task.
if self.task.active():
# There was already a task queue, don't requeue it, just let it
# go.
return
else:
if self.task.active():
# There was a task queued but it is requested to call it
# *right now* so cancel it.
self.task.cancel()
# Otherwise, it was just a stray object.
self.task = None
# Do the queue/direct call.
if delay:
# Call in delay seconds.
self.task = reactor.callLater(delay, self.serverPushCb)
elif self.stopped:
if not self.queue.nbItems():
return
# Call right now, we're shutting down.
@defer.inlineCallbacks
def BlockForEverythingBeingSent():
yield self.serverPushCb()
return BlockForEverythingBeingSent()
else:
# delay should never be 0. That can cause Buildbot to spin tightly
# trying to push events that may not be received well by a status
# listener.
klog.err_json('Did not expect delay to be 0, but it is.')
return
def test_err_json_for_exception_with_why(self, log):
try:
0 / 0
except Exception as ex:
err_json(ex, _why="Very ugly exception")
fail_dict = json.loads(log.msg.call_args[0][0])
self.check_exception_with_why(fail_dict)
def test_err_json_for_exception(self, log):
try:
0 / 0
except Exception as ex:
err_json(ex)
fail_dict = json.loads(log.msg.call_args[0][0])
self.check_exception(fail_dict)
def buildException(self, why):
log.msg("%s.buildException" % self)
klog.err_json(why)
# try to finish the build, but since we've already faced an exception,
# this may not work well.
try:
self.buildFinished(["Build Caught Exception"], DEPENDENCY_FAILURE)
except:
klog.err_json(Failure(), 'while finishing a build with an exception')
def test_err_json_for_failure(self, log):
try:
0 / 0
except:
err_json()
fail_dict = json.loads(log.msg.call_args[0][0])
self.check_failure(fail_dict, "test_err_json_for_failure")
def report(self):
try:
for interface, handler in self.handlers.iteritems():
report = handler.report()
if not report:
continue
for line in report.split("\n"):
log.msg(line)
except:
klog.err_json(None, "generating metric report")
def _describe(self, done=False):
"""Return a list of short strings to describe this step, for the
status display. This uses the first few words of the shell command.
You can replace this by setting .description in your subclass, or by
overriding this method to describe the step better.
@type done: boolean
@param done: whether the command is complete or not, to improve the
way the command is described. C{done=False} is used
while the command is still running, so a single
imperfect-tense verb is appropriate ('compiling',
'testing', ...) C{done=True} is used when the command
has finished, and the default getText() method adds some
text, so a simple noun is appropriate ('compile',
'tests' ...)
"""
try:
if done and self.descriptionDone is not None:
return self.descriptionDone
if self.description is not None:
return self.description
# we may have no command if this is a step that sets its command
# name late in the game (e.g., in start())
if not self.command:
return ["???"]
words = self.command
if isinstance(words, (str, unicode)):
words = words.split()
try:
len(words)
except AttributeError:
# WithProperties and Property don't have __len__
return ["???"]
# flatten any nested lists
words = flatten(words, (list, tuple))
# strip instances and other detritus (which can happen if a
# description is requested before rendering)
words = [w for w in words if isinstance(w, (str, unicode))]
if len(words) < 1:
return ["???"]
desc = words[:]
desc[0] = "'" + desc[0]
desc[-1] = desc[-1] + "'"
return desc
except:
klog.err_json(failure.Failure(), "Error describing step")
return ["???"]
def stop(self):
try:
self.poll()
if self.poller is not None:
self.poller.stop()
except:
klog.err_json(failure.Failure(),
'LogFileWatcher failed to stop poller')
if self.started:
self.f.close()
def failedToGetPerspective(self, why):
"""The login process failed, most likely because of an authorization
failure (bad password), but it is also possible that we lost the new
connection before we managed to send our credentials.
"""
log.msg("ReconnectingPBClientFactory.failedToGetPerspective")
if why.check(pb.PBConnectionLost):
log.msg("we lost the brand-new connection")
# retrying might help here, let clientConnectionLost decide
return
# probably authorization
self.stopTrying() # logging in harder won't help
klog.err_json(why)
def publishState(self, target=None):
state = self.currentBigState
if target is not None:
# unicast
target.builderChangedState(self.name, state)
return
for w in self.watchers:
try:
w.builderChangedState(self.name, state)
except:
log.msg("Exception caught publishing state to %r" % w)
klog.err_json()
def saveYourself(self):
filename = "sessions"
tmpfilename = filename + ".tmp"
try:
with open(tmpfilename, "wb") as f:
dump(self, f, -1)
if runtime.platformType == 'win32':
# windows cannot rename a file on top of an existing one
if os.path.exists(filename):
os.unlink(filename)
os.rename(tmpfilename, filename)
except:
log.msg("unable to save builder %s" % self.name)
klog.err_json()
def saveYourself(self):
filename = os.path.join(self.basedir, "changes.pck")
tmpfilename = filename + ".tmp"
try:
with open(tmpfilename, "wb") as f:
dump(self, f)
if runtime.platformType == 'win32':
# windows cannot rename a file on top of an existing one
if os.path.exists(filename):
os.unlink(filename)
os.rename(tmpfilename, filename)
except Exception:
log.msg("unable to save changes")
klog.err_json()
def _set_failed(why):
log.msg("BuildSlave.sendBuilderList (%s) failed" % self)
klog.err_json(why)
# TODO: hang up on them?, without setBuilderList we can't use
# them
if self.substantiation_deferred:
d = self.substantiation_deferred
self.substantiation_deferred = None
self.substantiation_build = None
d.errback(why)
if self.missing_timer:
self.missing_timer.cancel()
self.missing_timer = None
# TODO: maybe log? send an email?
return why
def updateBigStatus(self):
try:
# Catch exceptions here, since this is called in a LoopingCall.
if not self.builder_status:
return
if not self.slaves:
self.builder_status.setBigState("offline")
elif self.building or self.old_building:
self.builder_status.setBigState("building")
else:
self.builder_status.setBigState("idle")
except Exception:
klog.err_json(
None, "while trying to update status of builder '%s'" %
(self.name, ))
def results(heads):
if not heads:
return
if len(heads.split()) > 1:
klog.err_json(
("hgpoller: caught several heads in branch %r "
"from repository %r. Staying at previous revision"
"You should wait until the situation is normal again "
"due to a merge or directly strip if remote repo "
"gets stripped later.") % (self.branch, self.repourl))
return
# in case of whole reconstruction, are we sure that we'll get the
# same node -> rev assignations ?
return int(heads.strip())
