def _work(self):
self._busy = False
label, sizeGB, callback = self._queue.get()
self._busy = True
with globallock.lock():
callback(None, "Localizing label %s" % label)
logging.info("Localizing label '%(label)s'", dict(label=label))
try:
sh.run(["solvent", "localize", "--label", label])
except Exception as e:
logging.exception("Unable to localize label '%(label)s'", dict(label=label))
with globallock.lock():
callback(False, "Unable to localize label '%s': '%s'" % (label, str(e)))
return
with globallock.lock():
callback(None, "Done localizing label %s, Building image using inaugurator" % label)
logging.info("Done localizing label '%(label)s', building image using inaugurator", dict(
label=label))
vmInstance, stateMachine = self._startInauguratorVM(label, sizeGB)
self._event.wait()
self._event.clear()
with globallock.lock():
assert stateMachine.state() in [
hoststatemachine.STATE_INAUGURATION_DONE, hoststatemachine.STATE_DESTROYED]
if stateMachine.state() == hoststatemachine.STATE_DESTROYED:
logging.error("Unable to build image using inaugurator")
callback(False, "Unable to build image using inaugurator. Review rackattack provider logs")
return
self._imageStore.put(filename=vmInstance.disk1Image(), imageLabel=label, sizeGB=sizeGB)
stateMachine.unassign()
stateMachine.destroy()
callback(True, "Done building image using inaugurator (label %s)" % label)
logging.info("Done building image using inaugurator (label %(label)s)", dict(label=label))
def test_unregister(self):
with globallock.lock():
self.tested.register('awesome-server', self.checkIn, self.done, self.progress, self.failed)
self.assertEquals(self.checkIn.call_count, 0)
self.tested._checkIn('awesome-server')
self.checkIn.assert_called_once_with()
with globallock.lock():
self.tested.unregister('awesome-server')
self.tested._server.stopListeningOnID.assert_called_once_with('awesome-server')
self.checkIn.assert_called_once_with()
self.tested._checkIn('awesome-server')
self.checkIn.assert_called_once_with()
with globallock.lock():
self.assertRaises(AssertionError, self.tested.unregister, 'awesome-server')
self.tested._server.stopListeningOnID.assert_called_once_with('awesome-server')
def _done(self, id):
logging.info("%(id)s done", dict(id=id))
with globallock.lock():
if id not in self._registered:
logging.error("Unknown Inaugurator done: %(id)s", dict(id=id))
return
self._registered[id]['doneCallback']()
def _checkIn(self, id):
logging.info("%(id)s inaugurator check in", dict(id=id))
with globallock.lock():
if id not in self._registered:
logging.error("Unknown Inaugurator checked in: %(id)s", dict(id=id))
return
self._registered[id]['checkInCallback']()
def _progress(self, id, progress):
if u'state' in progress and progress[u'state'] == 'digesting':
return
with globallock.lock():
if id not in self._registered:
logging.error("Unknown Inaugurator progress: %(id)s", dict(id=id))
return
self._registered[id]['progressCallback'](progress)
def _reload(self):
logging.info("Reloading configuration")
rack = self._loadRackYAML()
with globallock.lock():
for hostData in rack["HOSTS"]:
self._normalizeStateCase(hostData)
if self._registeredHost(hostData["id"]):
self._registeredHostConfiguration(hostData)
else:
self._newHostInConfiguration(hostData)
def _work(self):
incoming, peer, respondCallback, transaction = self._queue.get()
transaction.reportState('dequeued (%d left in queue)' % self._queue.qsize())
try:
handler = getattr(self, "cmd_" + incoming['cmd'])
with globallock.lock():
response = handler(peer=peer, ** incoming['arguments'])
except Exception, e:
logging.exception('Handling')
response = dict(exceptionString=str(e), exceptionType=e.__class__.__name__)
def _startInauguratorVM(self, label, sizeGB):
with globallock.lock():
requirement = api.Requirement(
imageLabel=label, imageHint="build", hardwareConstraints=dict(
minimumDisk1SizeGB=sizeGB, minimumDisk2SizeGB=1)).__dict__
vmInstance = vm.VM.createFromNewImage(config.IMAGE_BUILDING_VM_INDEX, requirement)
stateMachine = hoststatemachine.HostStateMachine(
vmInstance, self._inaugurate, self._tftpboot, self._dnsmasq, self._reclaimHost)
stateMachine.assign(self._vmChangedState, imageLabel=label, imageHint="build")
stateMachine.setDestroyCallback(self._vmCommitedSuicide)
return vmInstance, stateMachine
def test_register(self):
with globallock.lock():
self.tested.register('awesome-server', self.checkIn, self.done, self.progress)
self.tested._server.listenOnID.assert_called_once_with('awesome-server')
self.assertEquals(self.checkIn.call_count, 0)
self.assertEquals(self.done.call_count, 0)
self.assertEquals(self.progress.call_count, 0)
self.tested._checkIn('awesome-server')
self.tested._checkIn('non-awesome-server')
self.checkIn.assert_called_once_with()
self.tested._done('awesome-server')
self.tested._done('non-awesome-server')
self.done.assert_called_once_with()
self.tested._progress('awesome-server', 'some progress')
self.tested._progress('non-awesome-server', 'some other progress')
self.progress.assert_called_once_with('some progress')
with globallock.lock():
self.assertRaises(AssertionError, self.tested.register, 'awesome-server', None, None, None)
self.done.assert_called_once_with()
self.progress.assert_called_once_with('some progress')
self.tested._server.listenOnID.assert_called_once_with('awesome-server')
def run(self):
try:
timeout = None
while True:
self._event.wait(timeout=timeout)
self._event.clear()
with globallock.lock():
self._runOne()
timeout = self._nextTimeout()
except:
logging.exception("Timers thread died")
suicide.killSelf()
raise
def _handleSoftReclamationFailedMsg(self):
hostsIDs = os.read(self._softReclaimFailedFd, self._READ_BUF_SIZE)
hostsIDs = hostsIDs.split(",")
for hostID in hostsIDs:
if not hostID:
continue
with globallock.lock():
try:
host = self._hosts.byID(hostID)
except:
logging.warn("A soft reclamation failure notification was received for a non-existent "
"host %(hostID)s", dict(hostID=hostID))
continue
try:
host.softReclaimFailed()
except Exception as e:
logging.error("Error handling soft reclamation failure for host %(host)s: %(message)s",
dict(host=hostID, message=e.message))
dnsmasq.DNSMasq.eraseLeasesFile()
dnsmasq.DNSMasq.killAllPrevious()
dnsmasqInstance = dnsmasq.DNSMasq(tftpboot=tftpbootInstance,
serverIP=network.myIP(),
netmask=network.netmask(),
firstIP=args.ipAddress,
lastIP=args.ipAddress,
gateway=network.gateway(),
nameserver=network.myIP())
logging.info(
"Sleeping 1 second to let dnsmasq go up, so it can receive SIGHUP")
time.sleep(1)
logging.info(
"Done Sleeping 1 second to let dnsmasq go up, so it can receive SIGHUP")
inaugurateInstance = inaugurate.Inaugurate(bindHostname=network.myIP())
with globallock.lock():
dnsmasqInstance.add(args.macAddress, args.ipAddress)
inaugurateInstance.register(ipAddress=args.ipAddress,
checkInCallback=inaugurateCheckIn,
doneCallback=inaugurateDone)
tftpbootInstance.configureForInaugurator(args.macAddress,
args.ipAddress,
clearDisk=True)
sol = serialoverlan.SerialOverLan(args.ipmiHost, args.ipmiUsername,
args.ipmiPassword, args.hostID)
ipmiInstance = ipmi.IPMI(args.ipmiHost, args.ipmiUsername, args.ipmiPassword)
ipmiInstance.powerCycle()
try:
logging.info("Waiting for inaugurator to check in")
checkInEvent.wait(6 * 60)
if not checkInEvent.isSet():
def serialLogFilename(vmID):
with globallock.lock():
return hostsInstance.byID(vmID).hostImplementation().serialLogFilename()
def test_filterDigesting(self):
with globallock.lock():
self.tested.register('awesome-server', self.checkIn, self.done, self.progress)
self.tested._progress('awesome-server', dict(state='digesting'))
self.assertEquals(self.progress.call_count, 0)
def _progress(self, id, progress):
with globallock.lock():
if id not in self._registered:
logging.error("Unknown Inaugurator progress: %(id)s", dict(id=id))
return
self._registered[id]['progressCallback'](progress)
def createPostMortemPackForAllocationID(allocationID):
with globallock.lock():
return allocationsInstance.byIndex(int(allocationID)).createPostMortemPack()
def _failure(self, id, message):
with globallock.lock():
if id not in self._registered:
logging.error("Unknown Inaugurator failure: %(id)s", dict(id=id))
return
self._registered[id]['failureCallback'](message)
withLocalObjectStore=True)
dnsmasq.DNSMasq.eraseLeasesFile()
dnsmasq.DNSMasq.killAllPrevious()
dnsmasqInstance = dnsmasq.DNSMasq(
tftpboot=tftpbootInstance,
serverIP=network.myIP(),
netmask=network.netmask(),
firstIP=args.ipAddress,
lastIP=args.ipAddress,
gateway=network.gateway(),
nameserver=network.myIP())
logging.info("Sleeping 1 second to let dnsmasq go up, so it can receive SIGHUP")
time.sleep(1)
logging.info("Done Sleeping 1 second to let dnsmasq go up, so it can receive SIGHUP")
inaugurateInstance = inaugurate.Inaugurate(bindHostname=network.myIP())
with globallock.lock():
dnsmasqInstance.add(args.macAddress, args.ipAddress)
inaugurateInstance.register(
ipAddress=args.ipAddress,
checkInCallback=inaugurateCheckIn,
doneCallback=inaugurateDone)
tftpbootInstance.configureForInaugurator(args.macAddress, args.ipAddress, clearDisk=True)
sol = serialoverlan.SerialOverLan(args.ipmiHost, args.ipmiUsername, args.ipmiPassword, args.hostID)
ipmiInstance = ipmi.IPMI(args.ipmiHost, args.ipmiUsername, args.ipmiPassword)
ipmiInstance.powerCycle()
try:
logging.info("Waiting for inaugurator to check in")
checkInEvent.wait(6 * 60)
if not checkInEvent.isSet():
raise Exception("Timeout waiting for inaugurator to checkin")
logging.info("Inaugurator checked in, waiting for inaugurator to complete")
