def addDead(self, job: TangoJob) -> int:
""" addDead - add a job to the dead queue.
Called by validateJob when a job validation fails.
Returns -1 on failure and the job id on success
"""
if (not isinstance(job, TangoJob)):
return -1
# Get an id for the new job
self.log.debug("add|Getting next ID")
nextId = self._getNextID()
if (nextId == -1):
self.log.info("add|JobQueue is full")
return -1
job.setId(nextId)
self.log.debug("addDead|Gotten next ID: " + str(job.id))
self.log.info("addDead|Unassigning job %s" % str(job.id))
job.makeUnassigned()
job.retries = 0
self.log.debug("addDead|Acquiring lock to job queue.")
self.queueLock.acquire()
self.log.debug("addDead|Acquired lock to job queue.")
# We add the job into the dead jobs dictionary
self.deadJobs.set(job.id, job)
self.queueLock.release()
self.log.debug("addDead|Released lock to job queue.")
return job.id
def add(self, job: TangoJob) -> Union[int, str]:
"""add - add job to live queue
This function assigns an ID number to a *new* job and then adds it
to the queue of live jobs.
Returns the job id on success, -1 otherwise
"""
if (not isinstance(job, TangoJob)):
return -1
# Get an id for the new job
self.log.debug("add|Getting next ID")
nextId = self._getNextID()
if (nextId == -1):
self.log.info("add|JobQueue is full")
return -1
job.setId(nextId)
self.log.debug("add|Gotten next ID: " + str(job.id))
self.log.info("add|Unassigning job ID: %d" % (job.id))
# Make the job unassigned
job.makeUnassigned()
# Since we assume that the job is new, we set the number of retries
# of this job to 0
job.retries = 0
# Add the job to the queue. Careful not to append the trace until we
# know the job has actually been added to the queue.
self.log.debug("add|Acquiring lock to job queue.")
self.queueLock.acquire()
self.log.debug("add| Acquired lock to job queue.")
# Adds the job to the live jobs dictionary
self.liveJobs.set(job.id, job)
# Add this to the unassigned job queue too
self.unassignedJobs.put(int(job.id))
job.appendTrace("%s|Added job %s:%d to queue" %
(datetime.utcnow().ctime(), job.name, job.id))
self.log.debug("Ref: " + str(job._remoteLocation))
self.log.debug("job_id: " + str(job.id))
self.log.debug("job_name: " + str(job.name))
self.queueLock.release()
self.log.debug("add|Releasing lock to job queue.")
self.log.info("Added job %s:%s to queue, details = %s" %
(job.name, job.id, str(job.__dict__)))
return str(job.id)
class TestJobQueue(unittest.TestCase):
def setUp(self):
if Config.USE_REDIS:
__db = redis.StrictRedis(Config.REDIS_HOSTNAME,
Config.REDIS_PORT,
db=0)
__db.flushall()
self.job1 = TangoJob(
name="sample_job_1",
vm="ilter.img",
outputFile="sample_job_1_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096,
)
self.job2 = TangoJob(
name="sample_job_2",
vm="ilter.img",
outputFile="sample_job_2_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096,
)
self.jobQueue = JobQueue(None)
self.jobQueue.reset()
self.jobId1 = self.jobQueue.add(self.job1)
self.jobId2 = self.jobQueue.add(self.job2)
def test_sharedInt(self):
if Config.USE_REDIS:
num1 = TangoIntValue("nextID", 1000)
num2 = TangoIntValue("nextID", 3000)
self.assertEqual(num1.get(), 1000)
self.assertEqual(num1.get(), num2.get())
else:
return
def test_job(self):
self.job1.makeUnassigned()
self.assertTrue(self.job1.isNotAssigned())
job = self.jobQueue.get(self.jobId1)
self.assertTrue(job.isNotAssigned())
self.job1.makeAssigned()
print("Checkout:")
self.assertFalse(self.job1.isNotAssigned())
self.assertFalse(job.isNotAssigned())
def test_add(self):
info = self.jobQueue.getInfo()
self.assertEqual(info["size"], 2)
def test_addToUnassigned(self):
info = self.jobQueue.getInfo()
self.assertEqual(info["size_unassignedjobs"], 2)
def test_addDead(self):
return self.assertEqual(1, 1)
def test_delJob(self):
self.jobQueue.delJob(self.jobId1, 0)
info = self.jobQueue.getInfo()
self.assertEqual(info["size"], 1)
self.assertEqual(info["size_deadjobs"], 1)
self.assertEqual(info["size_unassignedjobs"], 1)
self.jobQueue.delJob(self.jobId1, 1)
info = self.jobQueue.getInfo()
self.assertEqual(info["size_deadjobs"], 0)
self.assertEqual(info["size"], 1)
self.assertEqual(info["size_unassignedjobs"], 1)
return False
def test_get(self):
ret_job_1 = self.jobQueue.get(self.jobId1)
self.assertEqual(str(ret_job_1.id), self.jobId1)
ret_job_2 = self.jobQueue.get(self.jobId2)
self.assertEqual(str(ret_job_2.id), self.jobId2)
def test_getNextPendingJob(self):
self.jobQueue.assignJob(self.jobId2)
# job 2 should have been removed from unassigned queue
info = self.jobQueue.getInfo()
self.assertEqual(info["size_unassignedjobs"], 1)
self.jobQueue.assignJob(self.jobId1)
info = self.jobQueue.getInfo()
self.assertEqual(info["size_unassignedjobs"], 0)
self.jobQueue.unassignJob(self.jobId1)
info = self.jobQueue.getInfo()
self.assertEqual(info["size_unassignedjobs"], 1)
job = self.jobQueue.getNextPendingJob()
self.assertMultiLineEqual(str(job.id), self.jobId1)
def test_getNextPendingJob2(self):
job = self.jobQueue.getNextPendingJob()
self.assertMultiLineEqual(str(job.id), self.jobId1)
job = self.jobQueue.getNextPendingJob()
self.assertMultiLineEqual(str(job.id), self.jobId2)
def test_assignJob(self):
self.jobQueue.assignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
self.assertFalse(job.isNotAssigned())
def test_unassignJob(self):
self.jobQueue.assignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
self.assertTrue(job.assigned)
self.jobQueue.unassignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
return self.assertEqual(job.assigned, False)
def test_makeDead(self):
info = self.jobQueue.getInfo()
self.assertEqual(info["size_deadjobs"], 0)
self.assertEqual(info["size_unassignedjobs"], 2)
self.jobQueue.makeDead(self.jobId1, "test")
info = self.jobQueue.getInfo()
self.assertEqual(info["size_deadjobs"], 1)
self.assertEqual(info["size_unassignedjobs"], 1)
def test__getNextID(self):
init_id = self.jobQueue.nextID
for i in range(1, Config.MAX_JOBID + 100):
id = self.jobQueue._getNextID()
self.assertNotEqual(str(id), self.jobId1)
self.jobQueue.nextID = init_id
class TestJobQueue(unittest.TestCase):
def setUp(self):
if Config.USE_REDIS:
__db = redis.StrictRedis(
Config.REDIS_HOSTNAME, Config.REDIS_PORT, db=0)
__db.flushall()
self.job1 = TangoJob(
name="sample_job_1",
vm="ilter.img",
outputFile="sample_job_1_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096)
self.job2 = TangoJob(
name="sample_job_2",
vm="ilter.img",
outputFile="sample_job_2_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096)
self.jobQueue = JobQueue(None)
self.jobQueue.reset()
self.jobId1 = self.jobQueue.add(self.job1)
self.jobId2 = self.jobQueue.add(self.job2)
def test_sharedInt(self):
if Config.USE_REDIS:
num1 = TangoIntValue("nextID", 1000)
num2 = TangoIntValue("nextID", 3000)
self.assertEqual(num1.get(), 1000)
self.assertEqual(num1.get(), num2.get())
else:
return
def test_job(self):
self.job1.makeUnassigned()
self.assertTrue(self.job1.isNotAssigned())
job = self.jobQueue.get(self.jobId1)
self.assertTrue(job.isNotAssigned())
self.job1.makeAssigned()
print "Checkout:"
self.assertFalse(self.job1.isNotAssigned())
self.assertFalse(job.isNotAssigned())
def test_add(self):
info = self.jobQueue.getInfo()
self.assertEqual(info['size'], 2)
def test_addDead(self):
return self.assertEqual(1, 1)
def test_remove(self):
self.jobQueue.remove(self.jobId1)
info = self.jobQueue.getInfo()
self.assertEqual(info['size'], 1)
self.jobQueue.remove(self.jobId2)
info = self.jobQueue.getInfo()
self.assertEqual(info['size'], 0)
def test_delJob(self):
self.jobQueue.delJob(self.jobId1, 0)
info = self.jobQueue.getInfo()
self.assertEqual(info['size'], 1)
self.assertEqual(info['size_deadjobs'], 1)
self.jobQueue.delJob(self.jobId1, 1)
info = self.jobQueue.getInfo()
self.assertEqual(info['size_deadjobs'], 0)
return False
def test_get(self):
ret_job_1 = self.jobQueue.get(self.jobId1)
self.assertEqual(str(ret_job_1.id), self.jobId1)
ret_job_2 = self.jobQueue.get(self.jobId2)
self.assertEqual(str(ret_job_2.id), self.jobId2)
def test_getNextPendingJob(self):
self.jobQueue.assignJob(self.jobId2)
self.jobQueue.unassignJob(self.jobId1)
exp_id = self.jobQueue.getNextPendingJob()
self.assertMultiLineEqual(exp_id, self.jobId1)
def test_getNextPendingJobReuse(self):
return False
def test_assignJob(self):
self.jobQueue.assignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
self.assertFalse(job.isNotAssigned())
def test_unassignJob(self):
self.jobQueue.assignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
self.assertTrue(job.assigned)
self.jobQueue.unassignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
return self.assertEqual(job.assigned, False)
def test_makeDead(self):
info = self.jobQueue.getInfo()
self.assertEqual(info['size_deadjobs'], 0)
self.jobQueue.makeDead(self.jobId1, "test")
info = self.jobQueue.getInfo()
self.assertEqual(info['size_deadjobs'], 1)
def test__getNextID(self):
init_id = self.jobQueue.nextID
for i in xrange(1, Config.MAX_JOBID + 100):
id = self.jobQueue._getNextID()
self.assertNotEqual(str(id), self.jobId1)
self.jobQueue.nextID = init_id