def __init__(self, preallocator):
"""
Here we maintain several data structures used to keep track of the
jobs present for the autograder.
Live jobs contains:
- jobs that are yet to be assigned and run
- jobs that are currently running
Dead jobs contains:
- jobs that have been completed, or have been 'deleted' when in
the live jobs queue
Unassigned jobs:
This is a FIFO queue of jobs that are pending assignment.
- We enforce the invariant that all jobs in this queue must be
present in live jobs
queueLock protects all the internal data structure of JobQueue. This
is needed since there are multiple worker threads and they might be
using the makeUnassigned api.
"""
self.liveJobs = TangoDictionary("liveJobs")
self.deadJobs = TangoDictionary("deadJobs")
self.unassignedJobs = TangoQueue("unassignedLiveJobs")
self.queueLock = threading.Lock()
self.preallocator = preallocator
self.log = logging.getLogger("JobQueue")
self.nextID = 1
def update(self, vm, num):
""" update - Updates the number of machines of a certain type
to be preallocated.
This function is called via the TangoServer HTTP interface.
It will validate the request,update the machine list, and
then spawn child threads to do the creation and destruction
of machines as necessary.
"""
self.lock.acquire()
if vm.name not in self.machines:
self.machines.set(vm.name, [[], TangoQueue(vm.name)])
self.log.debug("Creating empty pool of %s instances" % (vm.name))
self.lock.release()
delta = num - len(self.machines.get(vm.name)[0])
if delta > 0:
# We need more self.machines, spin them up.
self.log.debug("update: Creating %d new %s instances" %
(delta, vm.name))
threading.Thread(target=self.__create(vm, delta)).start()
elif delta < 0:
# We have too many self.machines, remove them from the pool
self.log.debug("update: Destroying %d preallocated %s instances" %
(-delta, vm.name))
for i in range(-1 * delta):
threading.Thread(target=self.__destroy(vm)).start()
def addVM(self, vm):
""" addVM - add a particular VM instance to the pool
"""
self.lock.acquire()
# REUSEV_VMS=False code path does not call Preallcator::update to
# create machine, so manually handle it here.
if vm.name not in self.machines.keys():
self.machines.set(vm.name, [[], TangoQueue(vm.name)])
self.log.debug("Creating empty pool of %s instances" % (vm.name))
machine = self.machines.get(vm.name)
machine[0].append(vm.id)
self.machines.set(vm.name, machine)
self.lock.release()
def incrementPoolSize(self, vm, delta):
"""
Called by jobQueue to create the pool and allcoate given number of vms
"""
self.log.debug("incrementPoolSize| acquiring lock on preallocator")
with self.lock:
self.log.debug("incrementPoolSize| acquired lock on preallocator")
if vm.name not in self.machines.keys():
self.machines.set(vm.name, [[], TangoQueue(vm.name)])
# see comments in jobManager.py for the same call
self.machines.get(vm.name)[1].make_empty()
self.log.debug("Creating empty pool of %s instances" %
(vm.name))
self.log.debug("incrementPoolSize| released lock on preallocator")
self.log.debug("incrementPoolSize: add %d new %s instances" %
(delta, vm.name))
threading.Thread(target=self.__create(vm, delta)).start()
def runQueueTests(self):
self.testQueue = TangoQueue("self.testQueue")
self.expectedSize = 0
self.assertEqual(self.testQueue.qsize(), self.expectedSize)
self.assertTrue(self.testQueue.empty())
self.addAllToQueue()
# Test the blocking get
for x in self.test_entries:
item = self.testQueue.get()
self.expectedSize -= 1
self.assertEqual(self.testQueue.qsize(), self.expectedSize)
self.assertEqual(item, x)
self.addAllToQueue()
# Test the blocking get
for x in self.test_entries:
item = self.testQueue.get_nowait()
self.expectedSize -= 1
self.assertEqual(self.testQueue.qsize(), self.expectedSize)
self.assertEqual(item, x)
self.addAllToQueue()
# Remove all the even entries
for x in self.test_entries:
if (x % 2 == 0):
self.testQueue.remove(x)
self.expectedSize -= 1
self.assertEqual(self.testQueue.qsize(), self.expectedSize)
# Test that get only returns odd keys in order
for x in self.test_entries:
if (x % 2 == 1):
item = self.testQueue.get_nowait()
self.expectedSize -= 1
self.assertEqual(self.testQueue.qsize(), self.expectedSize)
self.assertEqual(item, x)
"Unable to pre-allocate a vm for job job %s:%d [try %d]"
% (job.name, job.id, job.retries))
job.appendTrace(
"%s|Dispatched job %s:%d [try %d]" %
(datetime.utcnow().ctime(), job.name, job.id, job.retries))
Worker(job, vmms, self.jobQueue, self.preallocator,
preVM).start()
except Exception as err:
self.jobQueue.makeDead(job.id, str(err))
if __name__ == "__main__":
if not Config.USE_REDIS:
print(
"You need to have Redis running to be able to initiate stand-alone\
JobManager")
else:
tango = TangoServer()
tango.log.debug("Resetting Tango VMs")
tango.resetTango(tango.preallocator.vmms)
for key in tango.preallocator.machines.keys():
tango.preallocator.machines.set(key, [[], TangoQueue(key)])
jobs = JobManager(tango.jobQueue)
print("Starting the stand-alone Tango JobManager")
jobs.run()
def destroyRedisPools():
for key in server.preallocator.machines.keys():
print "clean up pool", key
server.preallocator.machines.set(key, [[], TangoQueue(key)])
server.preallocator.machines.get(key)[1].make_empty()
