def setUp(self):
if Config.USE_REDIS:
__db = redis.StrictRedis(Config.REDIS_HOSTNAME,
Config.REDIS_PORT,
db=0)
__db.flushall()
self.job1 = TangoJob(
name="sample_job_1",
vm="ilter.img",
outputFile="sample_job_1_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096,
)
self.job2 = TangoJob(
name="sample_job_2",
vm="ilter.img",
outputFile="sample_job_2_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096,
)
self.jobQueue = JobQueue(None)
self.jobQueue.reset()
self.jobId1 = self.jobQueue.add(self.job1)
self.jobId2 = self.jobQueue.add(self.job2)
def addDead(self, job: TangoJob) -> int:
""" addDead - add a job to the dead queue.
Called by validateJob when a job validation fails.
Returns -1 on failure and the job id on success
"""
if (not isinstance(job, TangoJob)):
return -1
# Get an id for the new job
self.log.debug("add|Getting next ID")
nextId = self._getNextID()
if (nextId == -1):
self.log.info("add|JobQueue is full")
return -1
job.setId(nextId)
self.log.debug("addDead|Gotten next ID: " + str(job.id))
self.log.info("addDead|Unassigning job %s" % str(job.id))
job.makeUnassigned()
job.retries = 0
self.log.debug("addDead|Acquiring lock to job queue.")
self.queueLock.acquire()
self.log.debug("addDead|Acquired lock to job queue.")
# We add the job into the dead jobs dictionary
self.deadJobs.set(job.id, job)
self.queueLock.release()
self.log.debug("addDead|Released lock to job queue.")
return job.id
def setUp(self):
if Config.USE_REDIS:
__db = redis.StrictRedis(
Config.REDIS_HOSTNAME, Config.REDIS_PORT, db=0)
__db.flushall()
self.job1 = TangoJob(
name="sample_job_1",
vm="ilter.img",
outputFile="sample_job_1_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096)
self.job2 = TangoJob(
name="sample_job_2",
vm="ilter.img",
outputFile="sample_job_2_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096)
self.jobQueue = JobQueue(None)
self.jobQueue.reset()
self.jobId1 = self.jobQueue.add(self.job1)
self.jobId2 = self.jobQueue.add(self.job2)
def convertJobObj(self, key, courselab, jobObj):
""" convertJobObj - Converts a dictionary into a TangoJob object
"""
name = jobObj['jobName']
limitingKey = jobObj['limitingKey']
dirPath = self.getDirPath(key, courselab)
outputFile = self.getOutFilePath(key, courselab, jobObj['output_file'])
timeout = jobObj['timeout']
notifyURL = None
maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE
if 'callback_url' in jobObj:
notifyURL = jobObj['callback_url']
# List of input files
input = []
for file in jobObj['files']:
inFile = file['localFile']
vmFile = file['destFile']
handinfile = InputFile(
localFile="%s/%s" % (dirPath, inFile),
destFile=vmFile)
input.append(handinfile)
# VM object
if "vm" in jobObj:
vm = self.createTangoMachine(jobObj["image"], vmObj=jobObj["vm"])
else:
vm = self.createTangoMachine(jobObj["image"])
# for backward compatibility
accessKeyId = None
accessKey = None
if "accessKey" in jobObj and len(jobObj["accessKey"]) > 0:
accessKeyId = jobObj["accessKeyId"]
accessKey = jobObj["accessKey"]
job = TangoJob(
name=name,
limitingKey=limitingKey,
vm=vm,
outputFile=outputFile,
input=input,
timeout=timeout,
notifyURL=notifyURL,
maxOutputFileSize=maxOutputFileSize,
accessKey=accessKey,
accessKeyId=accessKeyId
)
self.log.debug("inputFiles: %s" % [file.localFile for file in input])
self.log.debug("outputFile: %s" % outputFile)
return job
def convertJobObj(self, dirName, jobObj):
"""convertJobObj - Converts a dictionary into a TangoJob object"""
name = jobObj["jobName"]
outputFile = "%s/%s/%s/%s" % (
self.COURSELABS,
dirName,
self.OUTPUT_FOLDER,
jobObj["output_file"],
)
timeout = jobObj["timeout"]
notifyURL = None
maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE
if "callback_url" in jobObj:
notifyURL = jobObj["callback_url"]
# List of input files
input = []
for file in jobObj["files"]:
inFile = file["localFile"]
vmFile = file["destFile"]
handinfile = InputFile(
localFile="%s/%s/%s" % (self.COURSELABS, dirName, inFile),
destFile=vmFile,
)
input.append(handinfile)
# VM object
vm = self.createTangoMachine(jobObj["image"])
# for backward compatibility
accessKeyId = None
accessKey = None
if "accessKey" in jobObj and len(jobObj["accessKey"]) > 0:
accessKeyId = jobObj["accessKeyId"]
accessKey = jobObj["accessKey"]
job = TangoJob(
name=name,
vm=vm,
outputFile=outputFile,
input=input,
timeout=timeout,
notifyURL=notifyURL,
maxOutputFileSize=maxOutputFileSize,
accessKey=accessKey,
accessKeyId=accessKeyId,
)
self.log.debug("inputFiles: %s" % [file.localFile for file in input])
self.log.debug("outputFile: %s" % outputFile)
return job
def convertJobObj(self, dirName, jobObj):
""" convertJobObj - Converts a dictionary into a TangoJob object
"""
name = jobObj['jobName']
outputFile = "%s/%s/%s/%s" % (self.COURSELABS, dirName,
self.OUTPUT_FOLDER,
jobObj['output_file'])
timeout = jobObj['timeout']
notifyURL = None
maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE
if 'callback_url' in jobObj:
notifyURL = jobObj['callback_url']
# List of input files
input = []
for file in jobObj['files']:
inFile = file['localFile']
vmFile = file['destFile']
handinfile = InputFile(localFile="%s/%s/%s" %
(self.COURSELABS, dirName, inFile),
destFile=vmFile)
input.append(handinfile)
# VM object
vm = self.createTangoMachine(jobObj["image"])
# for backward compatibility
accessKeyId = None
accessKey = None
if "accessKey" in jobObj and len(jobObj["accessKey"]) > 0:
accessKeyId = jobObj["accessKeyId"]
accessKey = jobObj["accessKey"]
# argument of job from rest request
arg = jobObj.get('arg', None)
job = TangoJob(name=name,
vm=vm,
outputFile=outputFile,
input=input,
timeout=timeout,
notifyURL=notifyURL,
maxOutputFileSize=maxOutputFileSize,
accessKey=accessKey,
accessKeyId=accessKeyId,
arg=arg)
self.log.debug("inputFiles: %s" % [file.localFile for file in input])
self.log.debug("outputFile: %s" % outputFile)
return job
def add(self, job: TangoJob) -> Union[int, str]:
"""add - add job to live queue
This function assigns an ID number to a *new* job and then adds it
to the queue of live jobs.
Returns the job id on success, -1 otherwise
"""
if (not isinstance(job, TangoJob)):
return -1
# Get an id for the new job
self.log.debug("add|Getting next ID")
nextId = self._getNextID()
if (nextId == -1):
self.log.info("add|JobQueue is full")
return -1
job.setId(nextId)
self.log.debug("add|Gotten next ID: " + str(job.id))
self.log.info("add|Unassigning job ID: %d" % (job.id))
# Make the job unassigned
job.makeUnassigned()
# Since we assume that the job is new, we set the number of retries
# of this job to 0
job.retries = 0
# Add the job to the queue. Careful not to append the trace until we
# know the job has actually been added to the queue.
self.log.debug("add|Acquiring lock to job queue.")
self.queueLock.acquire()
self.log.debug("add| Acquired lock to job queue.")
# Adds the job to the live jobs dictionary
self.liveJobs.set(job.id, job)
# Add this to the unassigned job queue too
self.unassignedJobs.put(int(job.id))
job.appendTrace("%s|Added job %s:%d to queue" %
(datetime.utcnow().ctime(), job.name, job.id))
self.log.debug("Ref: " + str(job._remoteLocation))
self.log.debug("job_id: " + str(job.id))
self.log.debug("job_name: " + str(job.name))
self.queueLock.release()
self.log.debug("add|Releasing lock to job queue.")
self.log.info("Added job %s:%s to queue, details = %s" %
(job.name, job.id, str(job.__dict__)))
return str(job.id)
def convertJobObj(self, dirName, jobObj):
""" convertJobObj - Converts a dictionary into a TangoJob object
"""
name = jobObj['jobName']
outputFile = "%s/%s/%s/%s" % (self.COURSELABS,
dirName,
self.OUTPUT_FOLDER,
jobObj['output_file'])
timeout = jobObj['timeout']
notifyURL = None
maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE
if 'callback_url' in jobObj:
notifyURL = jobObj['callback_url']
# List of input files
input = []
for file in jobObj['files']:
inFile = file['localFile']
vmFile = file['destFile']
handinfile = InputFile(
localFile="%s/%s/%s" % (self.COURSELABS, dirName, inFile),
destFile=vmFile)
input.append(handinfile)
# VM object
vm = self.createTangoMachine(jobObj["image"])
job = TangoJob(
name=name,
vm=vm,
outputFile=outputFile,
input=input,
timeout=timeout,
notifyURL=notifyURL,
maxOutputFileSize=maxOutputFileSize)
self.log.debug("inputFiles: %s" % [file.localFile for file in input])
self.log.debug("outputFile: %s" % outputFile)
return job
def reuseVM(self, job: TangoJob) -> Optional['vm']:
"""Helps a job reuse a vm. This is called if CONFIG.REUSE_VM is
set to true.
"""
# Create a pool if necessary
# This is when there is no existing pool for the vm name required.
if self.preallocator.poolSize(job.vm.name) == 0:
self.preallocator.update(job.vm, Config.POOL_SIZE)
# If the job hasn't been assigned to a worker yet, we try to
# allocate a new vm for this job
if (job.isNotAssigned()):
# Note: This could return None, when all VMs are being used
return self.preallocator.allocVM(job.vm.name)
else:
# In the case where a job is already assigned, it should have
# a vm, and we just return that vm here
if job.vm:
return job.vm
else:
raise Exception("Job assigned without vm")
class TestJobQueue(unittest.TestCase):
def setUp(self):
if Config.USE_REDIS:
__db = redis.StrictRedis(Config.REDIS_HOSTNAME,
Config.REDIS_PORT,
db=0)
__db.flushall()
self.job1 = TangoJob(
name="sample_job_1",
vm="ilter.img",
outputFile="sample_job_1_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096,
)
self.job2 = TangoJob(
name="sample_job_2",
vm="ilter.img",
outputFile="sample_job_2_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096,
)
self.jobQueue = JobQueue(None)
self.jobQueue.reset()
self.jobId1 = self.jobQueue.add(self.job1)
self.jobId2 = self.jobQueue.add(self.job2)
def test_sharedInt(self):
if Config.USE_REDIS:
num1 = TangoIntValue("nextID", 1000)
num2 = TangoIntValue("nextID", 3000)
self.assertEqual(num1.get(), 1000)
self.assertEqual(num1.get(), num2.get())
else:
return
def test_job(self):
self.job1.makeUnassigned()
self.assertTrue(self.job1.isNotAssigned())
job = self.jobQueue.get(self.jobId1)
self.assertTrue(job.isNotAssigned())
self.job1.makeAssigned()
print("Checkout:")
self.assertFalse(self.job1.isNotAssigned())
self.assertFalse(job.isNotAssigned())
def test_add(self):
info = self.jobQueue.getInfo()
self.assertEqual(info["size"], 2)
def test_addToUnassigned(self):
info = self.jobQueue.getInfo()
self.assertEqual(info["size_unassignedjobs"], 2)
def test_addDead(self):
return self.assertEqual(1, 1)
def test_delJob(self):
self.jobQueue.delJob(self.jobId1, 0)
info = self.jobQueue.getInfo()
self.assertEqual(info["size"], 1)
self.assertEqual(info["size_deadjobs"], 1)
self.assertEqual(info["size_unassignedjobs"], 1)
self.jobQueue.delJob(self.jobId1, 1)
info = self.jobQueue.getInfo()
self.assertEqual(info["size_deadjobs"], 0)
self.assertEqual(info["size"], 1)
self.assertEqual(info["size_unassignedjobs"], 1)
return False
def test_get(self):
ret_job_1 = self.jobQueue.get(self.jobId1)
self.assertEqual(str(ret_job_1.id), self.jobId1)
ret_job_2 = self.jobQueue.get(self.jobId2)
self.assertEqual(str(ret_job_2.id), self.jobId2)
def test_getNextPendingJob(self):
self.jobQueue.assignJob(self.jobId2)
# job 2 should have been removed from unassigned queue
info = self.jobQueue.getInfo()
self.assertEqual(info["size_unassignedjobs"], 1)
self.jobQueue.assignJob(self.jobId1)
info = self.jobQueue.getInfo()
self.assertEqual(info["size_unassignedjobs"], 0)
self.jobQueue.unassignJob(self.jobId1)
info = self.jobQueue.getInfo()
self.assertEqual(info["size_unassignedjobs"], 1)
job = self.jobQueue.getNextPendingJob()
self.assertMultiLineEqual(str(job.id), self.jobId1)
def test_getNextPendingJob2(self):
job = self.jobQueue.getNextPendingJob()
self.assertMultiLineEqual(str(job.id), self.jobId1)
job = self.jobQueue.getNextPendingJob()
self.assertMultiLineEqual(str(job.id), self.jobId2)
def test_assignJob(self):
self.jobQueue.assignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
self.assertFalse(job.isNotAssigned())
def test_unassignJob(self):
self.jobQueue.assignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
self.assertTrue(job.assigned)
self.jobQueue.unassignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
return self.assertEqual(job.assigned, False)
def test_makeDead(self):
info = self.jobQueue.getInfo()
self.assertEqual(info["size_deadjobs"], 0)
self.assertEqual(info["size_unassignedjobs"], 2)
self.jobQueue.makeDead(self.jobId1, "test")
info = self.jobQueue.getInfo()
self.assertEqual(info["size_deadjobs"], 1)
self.assertEqual(info["size_unassignedjobs"], 1)
def test__getNextID(self):
init_id = self.jobQueue.nextID
for i in range(1, Config.MAX_JOBID + 100):
id = self.jobQueue._getNextID()
self.assertNotEqual(str(id), self.jobId1)
self.jobQueue.nextID = init_id
class TestJobQueue(unittest.TestCase):
def setUp(self):
if Config.USE_REDIS:
__db = redis.StrictRedis(
Config.REDIS_HOSTNAME, Config.REDIS_PORT, db=0)
__db.flushall()
self.job1 = TangoJob(
name="sample_job_1",
vm="ilter.img",
outputFile="sample_job_1_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096)
self.job2 = TangoJob(
name="sample_job_2",
vm="ilter.img",
outputFile="sample_job_2_output",
input=[],
timeout=30,
notifyURL="notifyMeUrl",
maxOutputFileSize=4096)
self.jobQueue = JobQueue(None)
self.jobQueue.reset()
self.jobId1 = self.jobQueue.add(self.job1)
self.jobId2 = self.jobQueue.add(self.job2)
def test_sharedInt(self):
if Config.USE_REDIS:
num1 = TangoIntValue("nextID", 1000)
num2 = TangoIntValue("nextID", 3000)
self.assertEqual(num1.get(), 1000)
self.assertEqual(num1.get(), num2.get())
else:
return
def test_job(self):
self.job1.makeUnassigned()
self.assertTrue(self.job1.isNotAssigned())
job = self.jobQueue.get(self.jobId1)
self.assertTrue(job.isNotAssigned())
self.job1.makeAssigned()
print "Checkout:"
self.assertFalse(self.job1.isNotAssigned())
self.assertFalse(job.isNotAssigned())
def test_add(self):
info = self.jobQueue.getInfo()
self.assertEqual(info['size'], 2)
def test_addDead(self):
return self.assertEqual(1, 1)
def test_remove(self):
self.jobQueue.remove(self.jobId1)
info = self.jobQueue.getInfo()
self.assertEqual(info['size'], 1)
self.jobQueue.remove(self.jobId2)
info = self.jobQueue.getInfo()
self.assertEqual(info['size'], 0)
def test_delJob(self):
self.jobQueue.delJob(self.jobId1, 0)
info = self.jobQueue.getInfo()
self.assertEqual(info['size'], 1)
self.assertEqual(info['size_deadjobs'], 1)
self.jobQueue.delJob(self.jobId1, 1)
info = self.jobQueue.getInfo()
self.assertEqual(info['size_deadjobs'], 0)
return False
def test_get(self):
ret_job_1 = self.jobQueue.get(self.jobId1)
self.assertEqual(str(ret_job_1.id), self.jobId1)
ret_job_2 = self.jobQueue.get(self.jobId2)
self.assertEqual(str(ret_job_2.id), self.jobId2)
def test_getNextPendingJob(self):
self.jobQueue.assignJob(self.jobId2)
self.jobQueue.unassignJob(self.jobId1)
exp_id = self.jobQueue.getNextPendingJob()
self.assertMultiLineEqual(exp_id, self.jobId1)
def test_getNextPendingJobReuse(self):
return False
def test_assignJob(self):
self.jobQueue.assignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
self.assertFalse(job.isNotAssigned())
def test_unassignJob(self):
self.jobQueue.assignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
self.assertTrue(job.assigned)
self.jobQueue.unassignJob(self.jobId1)
job = self.jobQueue.get(self.jobId1)
return self.assertEqual(job.assigned, False)
def test_makeDead(self):
info = self.jobQueue.getInfo()
self.assertEqual(info['size_deadjobs'], 0)
self.jobQueue.makeDead(self.jobId1, "test")
info = self.jobQueue.getInfo()
self.assertEqual(info['size_deadjobs'], 1)
def test__getNextID(self):
init_id = self.jobQueue.nextID
for i in xrange(1, Config.MAX_JOBID + 100):
id = self.jobQueue._getNextID()
self.assertNotEqual(str(id), self.jobId1)
self.jobQueue.nextID = init_id
def __validateJob(self, job: TangoJob, vmms: List[Any]) -> int:
""" validateJob - validate the input arguments in an addJob request.
"""
errors = 0
# If this isn't a Tango job then bail with an error
if (not isinstance(job, TangoJob)):
return -1
# Every job must have a name
if not job.name:
self.log.error("validateJob: Missing job.name")
job.appendTrace("%s|validateJob: Missing job.name" %
(datetime.utcnow().ctime()))
errors += 1
# Check the virtual machine field
if not job.vm:
self.log.error("validateJob: Missing job.vm")
job.appendTrace("%s|validateJob: Missing job.vm" %
(datetime.utcnow().ctime()))
errors += 1
else:
if not job.vm.image:
self.log.error("validateJob: Missing job.vm.image")
job.appendTrace("%s|validateJob: Missing job.vm.image" %
(datetime.utcnow().ctime()))
errors += 1
else:
vobj = vmms[Config.VMMS_NAME]
imgList = vobj.getImages()
if job.vm.image not in imgList:
self.log.error("validateJob: Image not found: %s" %
job.vm.image)
job.appendTrace("%s|validateJob: Image not found: %s" %
(datetime.utcnow().ctime(), job.vm.image))
errors += 1
else:
(name, ext) = os.path.splitext(job.vm.image)
job.vm.name = name
if not job.vm.vmms:
self.log.error("validateJob: Missing job.vm.vmms")
job.appendTrace("%s|validateJob: Missing job.vm.vmms" %
(datetime.utcnow().ctime()))
errors += 1
else:
if job.vm.vmms not in vmms:
self.log.error("validateJob: Invalid vmms name: %s" %
job.vm.vmms)
job.appendTrace("%s|validateJob: Invalid vmms name: %s" %
(datetime.utcnow().ctime(), job.vm.vmms))
errors += 1
# Check the output file
if not job.outputFile:
self.log.error("validateJob: Missing job.outputFile")
job.appendTrace("%s|validateJob: Missing job.outputFile" %
(datetime.utcnow().ctime()))
errors += 1
else:
if not os.path.exists(os.path.dirname(job.outputFile)):
self.log.error("validateJob: Bad output path: %s",
job.outputFile)
job.appendTrace("%s|validateJob: Bad output path: %s" %
(datetime.utcnow().ctime(), job.outputFile))
errors += 1
# Check for max output file size parameter
if not job.maxOutputFileSize:
self.log.debug(
"validateJob: Setting job.maxOutputFileSize "
"to default value: %d bytes", Config.MAX_OUTPUT_FILE_SIZE)
job.maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE
# Check the list of input files
hasMakefile = False
for inputFile in job.input:
if not inputFile.localFile:
self.log.error("validateJob: Missing inputFile.localFile")
job.appendTrace("%s|validateJob: Missing inputFile.localFile" %
(datetime.utcnow().ctime()))
errors += 1
else:
if not os.path.exists(os.path.dirname(job.outputFile)):
self.log.error("validateJob: Bad output path: %s",
job.outputFile)
job.appendTrace(
"%s|validateJob: Bad output path: %s" %
(datetime.utcnow().ctime(), job.outputFile))
errors += 1
if inputFile.destFile == 'Makefile':
hasMakefile = True
# Check if input files include a Makefile
if not hasMakefile:
self.log.error("validateJob: Missing Makefile in input files.")
job.appendTrace(
"%s|validateJob: Missing Makefile in input files." %
(datetime.utcnow().ctime()))
errors += 1
# Check if job timeout has been set; If not set timeout to default
if not job.timeout or job.timeout <= 0:
self.log.debug(
"validateJob: Setting job.timeout to"
" default config value: %d secs", Config.RUNJOB_TIMEOUT)
job.timeout = Config.RUNJOB_TIMEOUT
# Any problems, return an error status
if errors > 0:
self.log.error("validateJob: Job rejected: %d errors" % errors)
job.appendTrace("%s|validateJob: Job rejected: %d errors" %
(datetime.utcnow().ctime(), errors))
return -1
else:
return 0
