def _work(self): self._busy = False label, sizeGB, callback = self._queue.get() self._busy = True with globallock.lock(): callback(None, "Localizing label %s" % label) logging.info("Localizing label '%(label)s'", dict(label=label)) try: sh.run(["solvent", "localize", "--label", label]) except Exception as e: logging.exception("Unable to localize label '%(label)s'", dict(label=label)) with globallock.lock(): callback(False, "Unable to localize label '%s': '%s'" % (label, str(e))) return with globallock.lock(): callback(None, "Done localizing label %s, Building image using inaugurator" % label) logging.info("Done localizing label '%(label)s', building image using inaugurator", dict( label=label)) vmInstance, stateMachine = self._startInauguratorVM(label, sizeGB) self._event.wait() self._event.clear() with globallock.lock(): assert stateMachine.state() in [ hoststatemachine.STATE_INAUGURATION_DONE, hoststatemachine.STATE_DESTROYED] if stateMachine.state() == hoststatemachine.STATE_DESTROYED: logging.error("Unable to build image using inaugurator") callback(False, "Unable to build image using inaugurator. Review rackattack provider logs") return self._imageStore.put(filename=vmInstance.disk1Image(), imageLabel=label, sizeGB=sizeGB) stateMachine.unassign() stateMachine.destroy() callback(True, "Done building image using inaugurator (label %s)" % label) logging.info("Done building image using inaugurator (label %(label)s)", dict(label=label))
def test_unregister(self): with globallock.lock(): self.tested.register('awesome-server', self.checkIn, self.done, self.progress, self.failed) self.assertEquals(self.checkIn.call_count, 0) self.tested._checkIn('awesome-server') self.checkIn.assert_called_once_with() with globallock.lock(): self.tested.unregister('awesome-server') self.tested._server.stopListeningOnID.assert_called_once_with('awesome-server') self.checkIn.assert_called_once_with() self.tested._checkIn('awesome-server') self.checkIn.assert_called_once_with() with globallock.lock(): self.assertRaises(AssertionError, self.tested.unregister, 'awesome-server') self.tested._server.stopListeningOnID.assert_called_once_with('awesome-server')
def _done(self, id): logging.info("%(id)s done", dict(id=id)) with globallock.lock(): if id not in self._registered: logging.error("Unknown Inaugurator done: %(id)s", dict(id=id)) return self._registered[id]['doneCallback']()
def _checkIn(self, id): logging.info("%(id)s inaugurator check in", dict(id=id)) with globallock.lock(): if id not in self._registered: logging.error("Unknown Inaugurator checked in: %(id)s", dict(id=id)) return self._registered[id]['checkInCallback']()
def _progress(self, id, progress): if u'state' in progress and progress[u'state'] == 'digesting': return with globallock.lock(): if id not in self._registered: logging.error("Unknown Inaugurator progress: %(id)s", dict(id=id)) return self._registered[id]['progressCallback'](progress)
def _reload(self): logging.info("Reloading configuration") rack = self._loadRackYAML() with globallock.lock(): for hostData in rack["HOSTS"]: self._normalizeStateCase(hostData) if self._registeredHost(hostData["id"]): self._registeredHostConfiguration(hostData) else: self._newHostInConfiguration(hostData)
def _work(self): incoming, peer, respondCallback, transaction = self._queue.get() transaction.reportState('dequeued (%d left in queue)' % self._queue.qsize()) try: handler = getattr(self, "cmd_" + incoming['cmd']) with globallock.lock(): response = handler(peer=peer, ** incoming['arguments']) except Exception, e: logging.exception('Handling') response = dict(exceptionString=str(e), exceptionType=e.__class__.__name__)
def _startInauguratorVM(self, label, sizeGB): with globallock.lock(): requirement = api.Requirement( imageLabel=label, imageHint="build", hardwareConstraints=dict( minimumDisk1SizeGB=sizeGB, minimumDisk2SizeGB=1)).__dict__ vmInstance = vm.VM.createFromNewImage(config.IMAGE_BUILDING_VM_INDEX, requirement) stateMachine = hoststatemachine.HostStateMachine( vmInstance, self._inaugurate, self._tftpboot, self._dnsmasq, self._reclaimHost) stateMachine.assign(self._vmChangedState, imageLabel=label, imageHint="build") stateMachine.setDestroyCallback(self._vmCommitedSuicide) return vmInstance, stateMachine
def test_register(self): with globallock.lock(): self.tested.register('awesome-server', self.checkIn, self.done, self.progress) self.tested._server.listenOnID.assert_called_once_with('awesome-server') self.assertEquals(self.checkIn.call_count, 0) self.assertEquals(self.done.call_count, 0) self.assertEquals(self.progress.call_count, 0) self.tested._checkIn('awesome-server') self.tested._checkIn('non-awesome-server') self.checkIn.assert_called_once_with() self.tested._done('awesome-server') self.tested._done('non-awesome-server') self.done.assert_called_once_with() self.tested._progress('awesome-server', 'some progress') self.tested._progress('non-awesome-server', 'some other progress') self.progress.assert_called_once_with('some progress') with globallock.lock(): self.assertRaises(AssertionError, self.tested.register, 'awesome-server', None, None, None) self.done.assert_called_once_with() self.progress.assert_called_once_with('some progress') self.tested._server.listenOnID.assert_called_once_with('awesome-server')
def run(self): try: timeout = None while True: self._event.wait(timeout=timeout) self._event.clear() with globallock.lock(): self._runOne() timeout = self._nextTimeout() except: logging.exception("Timers thread died") suicide.killSelf() raise
def _handleSoftReclamationFailedMsg(self): hostsIDs = os.read(self._softReclaimFailedFd, self._READ_BUF_SIZE) hostsIDs = hostsIDs.split(",") for hostID in hostsIDs: if not hostID: continue with globallock.lock(): try: host = self._hosts.byID(hostID) except: logging.warn("A soft reclamation failure notification was received for a non-existent " "host %(hostID)s", dict(hostID=hostID)) continue try: host.softReclaimFailed() except Exception as e: logging.error("Error handling soft reclamation failure for host %(host)s: %(message)s", dict(host=hostID, message=e.message))
dnsmasq.DNSMasq.eraseLeasesFile() dnsmasq.DNSMasq.killAllPrevious() dnsmasqInstance = dnsmasq.DNSMasq(tftpboot=tftpbootInstance, serverIP=network.myIP(), netmask=network.netmask(), firstIP=args.ipAddress, lastIP=args.ipAddress, gateway=network.gateway(), nameserver=network.myIP()) logging.info( "Sleeping 1 second to let dnsmasq go up, so it can receive SIGHUP") time.sleep(1) logging.info( "Done Sleeping 1 second to let dnsmasq go up, so it can receive SIGHUP") inaugurateInstance = inaugurate.Inaugurate(bindHostname=network.myIP()) with globallock.lock(): dnsmasqInstance.add(args.macAddress, args.ipAddress) inaugurateInstance.register(ipAddress=args.ipAddress, checkInCallback=inaugurateCheckIn, doneCallback=inaugurateDone) tftpbootInstance.configureForInaugurator(args.macAddress, args.ipAddress, clearDisk=True) sol = serialoverlan.SerialOverLan(args.ipmiHost, args.ipmiUsername, args.ipmiPassword, args.hostID) ipmiInstance = ipmi.IPMI(args.ipmiHost, args.ipmiUsername, args.ipmiPassword) ipmiInstance.powerCycle() try: logging.info("Waiting for inaugurator to check in") checkInEvent.wait(6 * 60) if not checkInEvent.isSet():
def serialLogFilename(vmID): with globallock.lock(): return hostsInstance.byID(vmID).hostImplementation().serialLogFilename()
def test_filterDigesting(self): with globallock.lock(): self.tested.register('awesome-server', self.checkIn, self.done, self.progress) self.tested._progress('awesome-server', dict(state='digesting')) self.assertEquals(self.progress.call_count, 0)
def _progress(self, id, progress): with globallock.lock(): if id not in self._registered: logging.error("Unknown Inaugurator progress: %(id)s", dict(id=id)) return self._registered[id]['progressCallback'](progress)
def createPostMortemPackForAllocationID(allocationID): with globallock.lock(): return allocationsInstance.byIndex(int(allocationID)).createPostMortemPack()
def _failure(self, id, message): with globallock.lock(): if id not in self._registered: logging.error("Unknown Inaugurator failure: %(id)s", dict(id=id)) return self._registered[id]['failureCallback'](message)
withLocalObjectStore=True) dnsmasq.DNSMasq.eraseLeasesFile() dnsmasq.DNSMasq.killAllPrevious() dnsmasqInstance = dnsmasq.DNSMasq( tftpboot=tftpbootInstance, serverIP=network.myIP(), netmask=network.netmask(), firstIP=args.ipAddress, lastIP=args.ipAddress, gateway=network.gateway(), nameserver=network.myIP()) logging.info("Sleeping 1 second to let dnsmasq go up, so it can receive SIGHUP") time.sleep(1) logging.info("Done Sleeping 1 second to let dnsmasq go up, so it can receive SIGHUP") inaugurateInstance = inaugurate.Inaugurate(bindHostname=network.myIP()) with globallock.lock(): dnsmasqInstance.add(args.macAddress, args.ipAddress) inaugurateInstance.register( ipAddress=args.ipAddress, checkInCallback=inaugurateCheckIn, doneCallback=inaugurateDone) tftpbootInstance.configureForInaugurator(args.macAddress, args.ipAddress, clearDisk=True) sol = serialoverlan.SerialOverLan(args.ipmiHost, args.ipmiUsername, args.ipmiPassword, args.hostID) ipmiInstance = ipmi.IPMI(args.ipmiHost, args.ipmiUsername, args.ipmiPassword) ipmiInstance.powerCycle() try: logging.info("Waiting for inaugurator to check in") checkInEvent.wait(6 * 60) if not checkInEvent.isSet(): raise Exception("Timeout waiting for inaugurator to checkin") logging.info("Inaugurator checked in, waiting for inaugurator to complete")