def __init__(self, preallocator): """ Here we maintain several data structures used to keep track of the jobs present for the autograder. Live jobs contains: - jobs that are yet to be assigned and run - jobs that are currently running Dead jobs contains: - jobs that have been completed, or have been 'deleted' when in the live jobs queue Unassigned jobs: This is a FIFO queue of jobs that are pending assignment. - We enforce the invariant that all jobs in this queue must be present in live jobs queueLock protects all the internal data structure of JobQueue. This is needed since there are multiple worker threads and they might be using the makeUnassigned api. """ self.liveJobs = TangoDictionary("liveJobs") self.deadJobs = TangoDictionary("deadJobs") self.unassignedJobs = TangoQueue("unassignedLiveJobs") self.queueLock = threading.Lock() self.preallocator = preallocator self.log = logging.getLogger("JobQueue") self.nextID = 1
def update(self, vm, num): """ update - Updates the number of machines of a certain type to be preallocated. This function is called via the TangoServer HTTP interface. It will validate the request,update the machine list, and then spawn child threads to do the creation and destruction of machines as necessary. """ self.lock.acquire() if vm.name not in self.machines: self.machines.set(vm.name, [[], TangoQueue(vm.name)]) self.log.debug("Creating empty pool of %s instances" % (vm.name)) self.lock.release() delta = num - len(self.machines.get(vm.name)[0]) if delta > 0: # We need more self.machines, spin them up. self.log.debug("update: Creating %d new %s instances" % (delta, vm.name)) threading.Thread(target=self.__create(vm, delta)).start() elif delta < 0: # We have too many self.machines, remove them from the pool self.log.debug("update: Destroying %d preallocated %s instances" % (-delta, vm.name)) for i in range(-1 * delta): threading.Thread(target=self.__destroy(vm)).start()
def addVM(self, vm): """ addVM - add a particular VM instance to the pool """ self.lock.acquire() # REUSEV_VMS=False code path does not call Preallcator::update to # create machine, so manually handle it here. if vm.name not in self.machines.keys(): self.machines.set(vm.name, [[], TangoQueue(vm.name)]) self.log.debug("Creating empty pool of %s instances" % (vm.name)) machine = self.machines.get(vm.name) machine[0].append(vm.id) self.machines.set(vm.name, machine) self.lock.release()
def incrementPoolSize(self, vm, delta): """ Called by jobQueue to create the pool and allcoate given number of vms """ self.log.debug("incrementPoolSize| acquiring lock on preallocator") with self.lock: self.log.debug("incrementPoolSize| acquired lock on preallocator") if vm.name not in self.machines.keys(): self.machines.set(vm.name, [[], TangoQueue(vm.name)]) # see comments in jobManager.py for the same call self.machines.get(vm.name)[1].make_empty() self.log.debug("Creating empty pool of %s instances" % (vm.name)) self.log.debug("incrementPoolSize| released lock on preallocator") self.log.debug("incrementPoolSize: add %d new %s instances" % (delta, vm.name)) threading.Thread(target=self.__create(vm, delta)).start()
def runQueueTests(self): self.testQueue = TangoQueue("self.testQueue") self.expectedSize = 0 self.assertEqual(self.testQueue.qsize(), self.expectedSize) self.assertTrue(self.testQueue.empty()) self.addAllToQueue() # Test the blocking get for x in self.test_entries: item = self.testQueue.get() self.expectedSize -= 1 self.assertEqual(self.testQueue.qsize(), self.expectedSize) self.assertEqual(item, x) self.addAllToQueue() # Test the blocking get for x in self.test_entries: item = self.testQueue.get_nowait() self.expectedSize -= 1 self.assertEqual(self.testQueue.qsize(), self.expectedSize) self.assertEqual(item, x) self.addAllToQueue() # Remove all the even entries for x in self.test_entries: if (x % 2 == 0): self.testQueue.remove(x) self.expectedSize -= 1 self.assertEqual(self.testQueue.qsize(), self.expectedSize) # Test that get only returns odd keys in order for x in self.test_entries: if (x % 2 == 1): item = self.testQueue.get_nowait() self.expectedSize -= 1 self.assertEqual(self.testQueue.qsize(), self.expectedSize) self.assertEqual(item, x)
"Unable to pre-allocate a vm for job job %s:%d [try %d]" % (job.name, job.id, job.retries)) job.appendTrace( "%s|Dispatched job %s:%d [try %d]" % (datetime.utcnow().ctime(), job.name, job.id, job.retries)) Worker(job, vmms, self.jobQueue, self.preallocator, preVM).start() except Exception as err: self.jobQueue.makeDead(job.id, str(err)) if __name__ == "__main__": if not Config.USE_REDIS: print( "You need to have Redis running to be able to initiate stand-alone\ JobManager") else: tango = TangoServer() tango.log.debug("Resetting Tango VMs") tango.resetTango(tango.preallocator.vmms) for key in tango.preallocator.machines.keys(): tango.preallocator.machines.set(key, [[], TangoQueue(key)]) jobs = JobManager(tango.jobQueue) print("Starting the stand-alone Tango JobManager") jobs.run()
def destroyRedisPools(): for key in server.preallocator.machines.keys(): print "clean up pool", key server.preallocator.machines.set(key, [[], TangoQueue(key)]) server.preallocator.machines.get(key)[1].make_empty()