def _soft_disconnect(self, fast=False):
if self.building:
# wait until build finished
return
# a negative build_wait_timeout means the worker should never be shut
# down, so just disconnect.
if self.build_wait_timeout < 0:
yield AbstractWorker.disconnect(self)
return
if self.missing_timer:
self.missing_timer.cancel()
self.missing_timer = None
# if master is stopping, we will never achieve consistent state, as workermanager
# wont accept new connection
if self._substantiation_notifier and self.master.running:
log.msg("Weird: Got request to stop before started. Allowing "
"worker to start cleanly to avoid inconsistent state")
yield self._substantiation_notifier.wait()
self.substantiation_build = None
log.msg("Substantiation complete, immediately terminating.")
if self.conn is not None:
yield defer.DeferredList([
AbstractWorker.disconnect(self),
self.insubstantiate(fast)
], consumeErrors=True, fireOnOneErrback=True)
else:
yield AbstractWorker.disconnect(self)
yield self.stop_instance(fast)
def _soft_disconnect(self, fast=False):
if self.building:
# wait until build finished
return
# a negative build_wait_timeout means the worker should never be shut
# down, so just disconnect.
if self.build_wait_timeout < 0:
yield AbstractWorker.disconnect(self)
return
self.stopMissingTimer()
# if master is stopping, we will never achieve consistent state, as workermanager
# won't accept new connection
if self._substantiation_notifier and self.master.running:
log.msg("Weird: Got request to stop before started. Allowing "
"worker to start cleanly to avoid inconsistent state")
yield self._substantiation_notifier.wait()
self.substantiation_build = None
log.msg("Substantiation complete, immediately terminating.")
if self.conn is not None:
yield defer.DeferredList(
[AbstractWorker.disconnect(self),
self.insubstantiate(fast)],
consumeErrors=True,
fireOnOneErrback=True)
else:
yield AbstractWorker.disconnect(self)
yield self.stop_instance(fast)
def checkConfig(self,
name,
password,
build_wait_timeout=60 * 10,
**kwargs):
AbstractWorker.checkConfig(self, name, password, **kwargs)
self.build_wait_timeout = build_wait_timeout
self._substantiation_notifier = Notifier()
def buildFinished(self, sb):
assert not sb.isBusy()
if not self.building:
if self.build_wait_timeout == 0:
# we insubstantiate asynchronously to trigger more bugs with the fake reactor
self.master.reactor.callLater(0, self.insubstantiate)
# insubstantiate will automatically retry to create build for this worker
else:
self._setBuildWaitTimer()
# AbstractWorker.buildFinished() will try to start the next build for that worker
AbstractWorker.buildFinished(self, sb)
def buildFinished(self, sb):
AbstractWorker.buildFinished(self, sb)
if not self.building:
if self.build_wait_timeout == 0:
d = self.insubstantiate()
# try starting builds for this worker after insubstantiating;
# this will cause the worker to re-substantiate immediately if
# there are pending build requests.
d.addCallback(lambda _:
self.botmaster.maybeStartBuildsForWorker(self.workername))
else:
self._setBuildWaitTimer()
def buildFinished(self, wfb):
assert not wfb.isBusy()
if not self.building:
if self.build_wait_timeout == 0:
# we insubstantiate asynchronously to trigger more bugs with
# the fake reactor
self.master.reactor.callLater(0, self._soft_disconnect)
# insubstantiate will automatically retry to create build for
# this worker
else:
self._setBuildWaitTimer()
# AbstractWorker.buildFinished() will try to start the next build for
# that worker
AbstractWorker.buildFinished(self, wfb)
def reconfigService(self,
name,
password,
build_wait_timeout=60 * 10,
**kwargs):
self.build_wait_timeout = build_wait_timeout
return AbstractWorker.reconfigService(self, name, password, **kwargs)
def canStartBuild(self):
# we were disconnected, but all the builds are not yet cleaned up.
if self.conn is None and self.building:
return False
if self.insubstantiating:
return False
return AbstractWorker.canStartBuild(self)
def canStartBuild(self):
# we were disconnected, but all the builds are not yet cleaned up.
if self.conn is None and self.building:
return False
if self.insubstantiating:
return False
return AbstractWorker.canStartBuild(self)
def reconfigService(self,
name,
password,
build_wait_timeout=60 * 10,
**kwargs):
self._substantiation_notifier = Notifier()
self.build_wait_timeout = build_wait_timeout
return AbstractWorker.reconfigService(self, name, password, **kwargs)
def stopService(self):
# the worker might be insubstantiating from buildWaitTimeout
while self.insubstantiating:
yield asyncSleep(0.1)
if self.conn is not None or self._substantiation_notifier:
yield self._soft_disconnect()
self._clearBuildWaitTimer()
res = yield AbstractWorker.stopService(self)
defer.returnValue(res)
def updateWorker(self):
"""Called to add or remove builders after the worker has connected.
Also called after botmaster's builders are initially set.
@return: a Deferred that indicates when an attached worker has
accepted the new builders and/or released the old ones."""
for b in self.botmaster.getBuildersForWorker(self.name):
if b.name not in self.workerforbuilders:
b.addLatentWorker(self)
return AbstractWorker.updateWorker(self)
def updateWorker(self):
"""Called to add or remove builders after the worker has connected.
Also called after botmaster's builders are initially set.
@return: a Deferred that indicates when an attached worker has
accepted the new builders and/or released the old ones."""
for b in self.botmaster.getBuildersForWorker(self.name):
if b.name not in self.workerforbuilders:
b.addLatentWorker(self)
return AbstractWorker.updateWorker(self)
def stopService(self):
# the worker might be insubstantiating from buildWaitTimeout
if self.state in [
States.INSUBSTANTIATING, States.INSUBSTANTIATING_SUBSTANTIATING
]:
yield self._insubstantiation_notifier.wait()
if self.conn is not None or self.state in [
States.SUBSTANTIATING, States.SUBSTANTIATED
]:
yield self._soft_disconnect()
self._clearBuildWaitTimer()
res = yield AbstractWorker.stopService(self)
return res
def attached(self, bot):
if not self._substantiation_notifier and self.build_wait_timeout >= 0:
msg = 'Worker %s received connection while not trying to ' \
'substantiate. Disconnecting.' % (self.name,)
log.msg(msg)
self._disconnect(bot)
raise RuntimeError(msg)
try:
yield AbstractWorker.attached(self, bot)
except Exception:
self._substantiation_failed(failure.Failure())
return
log.msg(r"Worker %s substantiated \o/" % (self.name, ))
if not self._substantiation_notifier:
log.msg("No substantiation deferred for %s" % (self.name, ))
else:
log.msg("Firing %s substantiation deferred with success" %
(self.name, ))
self.substantiation_build = None
self._substantiation_notifier.notify(True)
def attached(self, bot):
if not self._substantiation_notifier and self.build_wait_timeout >= 0:
msg = 'Worker %s received connection while not trying to ' \
'substantiate. Disconnecting.' % (self.name,)
log.msg(msg)
self._disconnect(bot)
raise RuntimeError(msg)
try:
yield AbstractWorker.attached(self, bot)
except Exception:
self._substantiation_failed(failure.Failure())
return
log.msg(r"Worker %s substantiated \o/" % (self.name,))
if not self._substantiation_notifier:
log.msg("No substantiation deferred for %s" % (self.name,))
else:
log.msg(
"Firing %s substantiation deferred with success" % (self.name,))
self.substantiation_build = None
self._substantiation_notifier.notify(True)
def attached(self, bot):
if self.state != States.SUBSTANTIATING and \
self.build_wait_timeout >= 0:
msg = 'Worker %s received connection while not trying to ' \
'substantiate. Disconnecting.' % (self.name,)
log.msg(msg)
self._disconnect(bot)
raise RuntimeError(msg)
try:
yield AbstractWorker.attached(self, bot)
except Exception:
self._substantiation_failed(failure.Failure())
return
log.msg(r"Worker %s substantiated \o/" % (self.name, ))
# only change state when we are actually substantiating. We could
# end up at this point in different state than SUBSTANTIATING if
# build_wait_timeout is negative. When build_wait_timeout is not
# negative, we throw an error (see above)
if self.state == States.SUBSTANTIATING:
self.state = States.SUBSTANTIATED
self._fireSubstantiationNotifier(True)
def checkConfig(self, name, password,
build_wait_timeout=60 * 10,
**kwargs):
AbstractWorker.checkConfig(self, name, password, **kwargs)
self.build_wait_timeout = build_wait_timeout
self._substantiation_notifier = Notifier()
def checkConfig(self, name, password,
build_wait_timeout=60 * 10,
**kwargs):
AbstractWorker.checkConfig(self, name, password, **kwargs)
def checkConfig(self,
name,
password,
build_wait_timeout=60 * 10,
**kwargs):
AbstractWorker.checkConfig(self, name, password, **kwargs)
def detached(self):
AbstractWorker.detached(self)
if self._substantiation_notifier:
d = self._substantiate(self.substantiation_build)
d.addErrback(log.err, 'while re-substantiating')
def reconfigService(self, name, password,
build_wait_timeout=60 * 10,
**kwargs):
self._substantiation_notifier = Notifier()
self.build_wait_timeout = build_wait_timeout
return AbstractWorker.reconfigService(self, name, password, **kwargs)
def detached(self):
AbstractWorker.detached(self)
if self._substantiation_notifier:
d = self._substantiate(self.substantiation_build)
d.addErrback(log.err, 'while re-substantiating')
def canStartBuild(self):
if self.insubstantiating:
return False
return AbstractWorker.canStartBuild(self)
def stopService(self):
if self.conn is not None or self._substantiation_notifier:
yield self._soft_disconnect()
res = yield AbstractWorker.stopService(self)
defer.returnValue(res)
def _disconnect(res):
log.msg("VM destroyed (%s): Forcing its connection closed." %
self.workername)
return AbstractWorker.disconnect(self)
def reconfigService(self, name, password,
build_wait_timeout=60 * 10,
**kwargs):
self.build_wait_timeout = build_wait_timeout
return AbstractWorker.reconfigService(self, name, password, **kwargs)
def canStartBuild(self):
if self.insubstantiating:
return False
return AbstractWorker.canStartBuild(self)
def stopService(self):
if self.conn is not None or self._substantiation_notifier:
yield self._soft_disconnect()
res = yield AbstractWorker.stopService(self)
defer.returnValue(res)
