def setUp(self): if Config.USE_REDIS: __db = redis.StrictRedis(Config.REDIS_HOSTNAME, Config.REDIS_PORT, db=0) __db.flushall() self.job1 = TangoJob( name="sample_job_1", vm="ilter.img", outputFile="sample_job_1_output", input=[], timeout=30, notifyURL="notifyMeUrl", maxOutputFileSize=4096, ) self.job2 = TangoJob( name="sample_job_2", vm="ilter.img", outputFile="sample_job_2_output", input=[], timeout=30, notifyURL="notifyMeUrl", maxOutputFileSize=4096, ) self.jobQueue = JobQueue(None) self.jobQueue.reset() self.jobId1 = self.jobQueue.add(self.job1) self.jobId2 = self.jobQueue.add(self.job2)
def addDead(self, job: TangoJob) -> int: """ addDead - add a job to the dead queue. Called by validateJob when a job validation fails. Returns -1 on failure and the job id on success """ if (not isinstance(job, TangoJob)): return -1 # Get an id for the new job self.log.debug("add|Getting next ID") nextId = self._getNextID() if (nextId == -1): self.log.info("add|JobQueue is full") return -1 job.setId(nextId) self.log.debug("addDead|Gotten next ID: " + str(job.id)) self.log.info("addDead|Unassigning job %s" % str(job.id)) job.makeUnassigned() job.retries = 0 self.log.debug("addDead|Acquiring lock to job queue.") self.queueLock.acquire() self.log.debug("addDead|Acquired lock to job queue.") # We add the job into the dead jobs dictionary self.deadJobs.set(job.id, job) self.queueLock.release() self.log.debug("addDead|Released lock to job queue.") return job.id
def setUp(self): if Config.USE_REDIS: __db = redis.StrictRedis( Config.REDIS_HOSTNAME, Config.REDIS_PORT, db=0) __db.flushall() self.job1 = TangoJob( name="sample_job_1", vm="ilter.img", outputFile="sample_job_1_output", input=[], timeout=30, notifyURL="notifyMeUrl", maxOutputFileSize=4096) self.job2 = TangoJob( name="sample_job_2", vm="ilter.img", outputFile="sample_job_2_output", input=[], timeout=30, notifyURL="notifyMeUrl", maxOutputFileSize=4096) self.jobQueue = JobQueue(None) self.jobQueue.reset() self.jobId1 = self.jobQueue.add(self.job1) self.jobId2 = self.jobQueue.add(self.job2)
def convertJobObj(self, key, courselab, jobObj): """ convertJobObj - Converts a dictionary into a TangoJob object """ name = jobObj['jobName'] limitingKey = jobObj['limitingKey'] dirPath = self.getDirPath(key, courselab) outputFile = self.getOutFilePath(key, courselab, jobObj['output_file']) timeout = jobObj['timeout'] notifyURL = None maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE if 'callback_url' in jobObj: notifyURL = jobObj['callback_url'] # List of input files input = [] for file in jobObj['files']: inFile = file['localFile'] vmFile = file['destFile'] handinfile = InputFile( localFile="%s/%s" % (dirPath, inFile), destFile=vmFile) input.append(handinfile) # VM object if "vm" in jobObj: vm = self.createTangoMachine(jobObj["image"], vmObj=jobObj["vm"]) else: vm = self.createTangoMachine(jobObj["image"]) # for backward compatibility accessKeyId = None accessKey = None if "accessKey" in jobObj and len(jobObj["accessKey"]) > 0: accessKeyId = jobObj["accessKeyId"] accessKey = jobObj["accessKey"] job = TangoJob( name=name, limitingKey=limitingKey, vm=vm, outputFile=outputFile, input=input, timeout=timeout, notifyURL=notifyURL, maxOutputFileSize=maxOutputFileSize, accessKey=accessKey, accessKeyId=accessKeyId ) self.log.debug("inputFiles: %s" % [file.localFile for file in input]) self.log.debug("outputFile: %s" % outputFile) return job
def convertJobObj(self, dirName, jobObj): """convertJobObj - Converts a dictionary into a TangoJob object""" name = jobObj["jobName"] outputFile = "%s/%s/%s/%s" % ( self.COURSELABS, dirName, self.OUTPUT_FOLDER, jobObj["output_file"], ) timeout = jobObj["timeout"] notifyURL = None maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE if "callback_url" in jobObj: notifyURL = jobObj["callback_url"] # List of input files input = [] for file in jobObj["files"]: inFile = file["localFile"] vmFile = file["destFile"] handinfile = InputFile( localFile="%s/%s/%s" % (self.COURSELABS, dirName, inFile), destFile=vmFile, ) input.append(handinfile) # VM object vm = self.createTangoMachine(jobObj["image"]) # for backward compatibility accessKeyId = None accessKey = None if "accessKey" in jobObj and len(jobObj["accessKey"]) > 0: accessKeyId = jobObj["accessKeyId"] accessKey = jobObj["accessKey"] job = TangoJob( name=name, vm=vm, outputFile=outputFile, input=input, timeout=timeout, notifyURL=notifyURL, maxOutputFileSize=maxOutputFileSize, accessKey=accessKey, accessKeyId=accessKeyId, ) self.log.debug("inputFiles: %s" % [file.localFile for file in input]) self.log.debug("outputFile: %s" % outputFile) return job
def convertJobObj(self, dirName, jobObj): """ convertJobObj - Converts a dictionary into a TangoJob object """ name = jobObj['jobName'] outputFile = "%s/%s/%s/%s" % (self.COURSELABS, dirName, self.OUTPUT_FOLDER, jobObj['output_file']) timeout = jobObj['timeout'] notifyURL = None maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE if 'callback_url' in jobObj: notifyURL = jobObj['callback_url'] # List of input files input = [] for file in jobObj['files']: inFile = file['localFile'] vmFile = file['destFile'] handinfile = InputFile(localFile="%s/%s/%s" % (self.COURSELABS, dirName, inFile), destFile=vmFile) input.append(handinfile) # VM object vm = self.createTangoMachine(jobObj["image"]) # for backward compatibility accessKeyId = None accessKey = None if "accessKey" in jobObj and len(jobObj["accessKey"]) > 0: accessKeyId = jobObj["accessKeyId"] accessKey = jobObj["accessKey"] # argument of job from rest request arg = jobObj.get('arg', None) job = TangoJob(name=name, vm=vm, outputFile=outputFile, input=input, timeout=timeout, notifyURL=notifyURL, maxOutputFileSize=maxOutputFileSize, accessKey=accessKey, accessKeyId=accessKeyId, arg=arg) self.log.debug("inputFiles: %s" % [file.localFile for file in input]) self.log.debug("outputFile: %s" % outputFile) return job
def add(self, job: TangoJob) -> Union[int, str]: """add - add job to live queue This function assigns an ID number to a *new* job and then adds it to the queue of live jobs. Returns the job id on success, -1 otherwise """ if (not isinstance(job, TangoJob)): return -1 # Get an id for the new job self.log.debug("add|Getting next ID") nextId = self._getNextID() if (nextId == -1): self.log.info("add|JobQueue is full") return -1 job.setId(nextId) self.log.debug("add|Gotten next ID: " + str(job.id)) self.log.info("add|Unassigning job ID: %d" % (job.id)) # Make the job unassigned job.makeUnassigned() # Since we assume that the job is new, we set the number of retries # of this job to 0 job.retries = 0 # Add the job to the queue. Careful not to append the trace until we # know the job has actually been added to the queue. self.log.debug("add|Acquiring lock to job queue.") self.queueLock.acquire() self.log.debug("add| Acquired lock to job queue.") # Adds the job to the live jobs dictionary self.liveJobs.set(job.id, job) # Add this to the unassigned job queue too self.unassignedJobs.put(int(job.id)) job.appendTrace("%s|Added job %s:%d to queue" % (datetime.utcnow().ctime(), job.name, job.id)) self.log.debug("Ref: " + str(job._remoteLocation)) self.log.debug("job_id: " + str(job.id)) self.log.debug("job_name: " + str(job.name)) self.queueLock.release() self.log.debug("add|Releasing lock to job queue.") self.log.info("Added job %s:%s to queue, details = %s" % (job.name, job.id, str(job.__dict__))) return str(job.id)
def convertJobObj(self, dirName, jobObj): """ convertJobObj - Converts a dictionary into a TangoJob object """ name = jobObj['jobName'] outputFile = "%s/%s/%s/%s" % (self.COURSELABS, dirName, self.OUTPUT_FOLDER, jobObj['output_file']) timeout = jobObj['timeout'] notifyURL = None maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE if 'callback_url' in jobObj: notifyURL = jobObj['callback_url'] # List of input files input = [] for file in jobObj['files']: inFile = file['localFile'] vmFile = file['destFile'] handinfile = InputFile( localFile="%s/%s/%s" % (self.COURSELABS, dirName, inFile), destFile=vmFile) input.append(handinfile) # VM object vm = self.createTangoMachine(jobObj["image"]) job = TangoJob( name=name, vm=vm, outputFile=outputFile, input=input, timeout=timeout, notifyURL=notifyURL, maxOutputFileSize=maxOutputFileSize) self.log.debug("inputFiles: %s" % [file.localFile for file in input]) self.log.debug("outputFile: %s" % outputFile) return job
def reuseVM(self, job: TangoJob) -> Optional['vm']: """Helps a job reuse a vm. This is called if CONFIG.REUSE_VM is set to true. """ # Create a pool if necessary # This is when there is no existing pool for the vm name required. if self.preallocator.poolSize(job.vm.name) == 0: self.preallocator.update(job.vm, Config.POOL_SIZE) # If the job hasn't been assigned to a worker yet, we try to # allocate a new vm for this job if (job.isNotAssigned()): # Note: This could return None, when all VMs are being used return self.preallocator.allocVM(job.vm.name) else: # In the case where a job is already assigned, it should have # a vm, and we just return that vm here if job.vm: return job.vm else: raise Exception("Job assigned without vm")
class TestJobQueue(unittest.TestCase): def setUp(self): if Config.USE_REDIS: __db = redis.StrictRedis(Config.REDIS_HOSTNAME, Config.REDIS_PORT, db=0) __db.flushall() self.job1 = TangoJob( name="sample_job_1", vm="ilter.img", outputFile="sample_job_1_output", input=[], timeout=30, notifyURL="notifyMeUrl", maxOutputFileSize=4096, ) self.job2 = TangoJob( name="sample_job_2", vm="ilter.img", outputFile="sample_job_2_output", input=[], timeout=30, notifyURL="notifyMeUrl", maxOutputFileSize=4096, ) self.jobQueue = JobQueue(None) self.jobQueue.reset() self.jobId1 = self.jobQueue.add(self.job1) self.jobId2 = self.jobQueue.add(self.job2) def test_sharedInt(self): if Config.USE_REDIS: num1 = TangoIntValue("nextID", 1000) num2 = TangoIntValue("nextID", 3000) self.assertEqual(num1.get(), 1000) self.assertEqual(num1.get(), num2.get()) else: return def test_job(self): self.job1.makeUnassigned() self.assertTrue(self.job1.isNotAssigned()) job = self.jobQueue.get(self.jobId1) self.assertTrue(job.isNotAssigned()) self.job1.makeAssigned() print("Checkout:") self.assertFalse(self.job1.isNotAssigned()) self.assertFalse(job.isNotAssigned()) def test_add(self): info = self.jobQueue.getInfo() self.assertEqual(info["size"], 2) def test_addToUnassigned(self): info = self.jobQueue.getInfo() self.assertEqual(info["size_unassignedjobs"], 2) def test_addDead(self): return self.assertEqual(1, 1) def test_delJob(self): self.jobQueue.delJob(self.jobId1, 0) info = self.jobQueue.getInfo() self.assertEqual(info["size"], 1) self.assertEqual(info["size_deadjobs"], 1) self.assertEqual(info["size_unassignedjobs"], 1) self.jobQueue.delJob(self.jobId1, 1) info = self.jobQueue.getInfo() self.assertEqual(info["size_deadjobs"], 0) self.assertEqual(info["size"], 1) self.assertEqual(info["size_unassignedjobs"], 1) return False def test_get(self): ret_job_1 = self.jobQueue.get(self.jobId1) self.assertEqual(str(ret_job_1.id), self.jobId1) ret_job_2 = self.jobQueue.get(self.jobId2) self.assertEqual(str(ret_job_2.id), self.jobId2) def test_getNextPendingJob(self): self.jobQueue.assignJob(self.jobId2) # job 2 should have been removed from unassigned queue info = self.jobQueue.getInfo() self.assertEqual(info["size_unassignedjobs"], 1) self.jobQueue.assignJob(self.jobId1) info = self.jobQueue.getInfo() self.assertEqual(info["size_unassignedjobs"], 0) self.jobQueue.unassignJob(self.jobId1) info = self.jobQueue.getInfo() self.assertEqual(info["size_unassignedjobs"], 1) job = self.jobQueue.getNextPendingJob() self.assertMultiLineEqual(str(job.id), self.jobId1) def test_getNextPendingJob2(self): job = self.jobQueue.getNextPendingJob() self.assertMultiLineEqual(str(job.id), self.jobId1) job = self.jobQueue.getNextPendingJob() self.assertMultiLineEqual(str(job.id), self.jobId2) def test_assignJob(self): self.jobQueue.assignJob(self.jobId1) job = self.jobQueue.get(self.jobId1) self.assertFalse(job.isNotAssigned()) def test_unassignJob(self): self.jobQueue.assignJob(self.jobId1) job = self.jobQueue.get(self.jobId1) self.assertTrue(job.assigned) self.jobQueue.unassignJob(self.jobId1) job = self.jobQueue.get(self.jobId1) return self.assertEqual(job.assigned, False) def test_makeDead(self): info = self.jobQueue.getInfo() self.assertEqual(info["size_deadjobs"], 0) self.assertEqual(info["size_unassignedjobs"], 2) self.jobQueue.makeDead(self.jobId1, "test") info = self.jobQueue.getInfo() self.assertEqual(info["size_deadjobs"], 1) self.assertEqual(info["size_unassignedjobs"], 1) def test__getNextID(self): init_id = self.jobQueue.nextID for i in range(1, Config.MAX_JOBID + 100): id = self.jobQueue._getNextID() self.assertNotEqual(str(id), self.jobId1) self.jobQueue.nextID = init_id
class TestJobQueue(unittest.TestCase): def setUp(self): if Config.USE_REDIS: __db = redis.StrictRedis( Config.REDIS_HOSTNAME, Config.REDIS_PORT, db=0) __db.flushall() self.job1 = TangoJob( name="sample_job_1", vm="ilter.img", outputFile="sample_job_1_output", input=[], timeout=30, notifyURL="notifyMeUrl", maxOutputFileSize=4096) self.job2 = TangoJob( name="sample_job_2", vm="ilter.img", outputFile="sample_job_2_output", input=[], timeout=30, notifyURL="notifyMeUrl", maxOutputFileSize=4096) self.jobQueue = JobQueue(None) self.jobQueue.reset() self.jobId1 = self.jobQueue.add(self.job1) self.jobId2 = self.jobQueue.add(self.job2) def test_sharedInt(self): if Config.USE_REDIS: num1 = TangoIntValue("nextID", 1000) num2 = TangoIntValue("nextID", 3000) self.assertEqual(num1.get(), 1000) self.assertEqual(num1.get(), num2.get()) else: return def test_job(self): self.job1.makeUnassigned() self.assertTrue(self.job1.isNotAssigned()) job = self.jobQueue.get(self.jobId1) self.assertTrue(job.isNotAssigned()) self.job1.makeAssigned() print "Checkout:" self.assertFalse(self.job1.isNotAssigned()) self.assertFalse(job.isNotAssigned()) def test_add(self): info = self.jobQueue.getInfo() self.assertEqual(info['size'], 2) def test_addDead(self): return self.assertEqual(1, 1) def test_remove(self): self.jobQueue.remove(self.jobId1) info = self.jobQueue.getInfo() self.assertEqual(info['size'], 1) self.jobQueue.remove(self.jobId2) info = self.jobQueue.getInfo() self.assertEqual(info['size'], 0) def test_delJob(self): self.jobQueue.delJob(self.jobId1, 0) info = self.jobQueue.getInfo() self.assertEqual(info['size'], 1) self.assertEqual(info['size_deadjobs'], 1) self.jobQueue.delJob(self.jobId1, 1) info = self.jobQueue.getInfo() self.assertEqual(info['size_deadjobs'], 0) return False def test_get(self): ret_job_1 = self.jobQueue.get(self.jobId1) self.assertEqual(str(ret_job_1.id), self.jobId1) ret_job_2 = self.jobQueue.get(self.jobId2) self.assertEqual(str(ret_job_2.id), self.jobId2) def test_getNextPendingJob(self): self.jobQueue.assignJob(self.jobId2) self.jobQueue.unassignJob(self.jobId1) exp_id = self.jobQueue.getNextPendingJob() self.assertMultiLineEqual(exp_id, self.jobId1) def test_getNextPendingJobReuse(self): return False def test_assignJob(self): self.jobQueue.assignJob(self.jobId1) job = self.jobQueue.get(self.jobId1) self.assertFalse(job.isNotAssigned()) def test_unassignJob(self): self.jobQueue.assignJob(self.jobId1) job = self.jobQueue.get(self.jobId1) self.assertTrue(job.assigned) self.jobQueue.unassignJob(self.jobId1) job = self.jobQueue.get(self.jobId1) return self.assertEqual(job.assigned, False) def test_makeDead(self): info = self.jobQueue.getInfo() self.assertEqual(info['size_deadjobs'], 0) self.jobQueue.makeDead(self.jobId1, "test") info = self.jobQueue.getInfo() self.assertEqual(info['size_deadjobs'], 1) def test__getNextID(self): init_id = self.jobQueue.nextID for i in xrange(1, Config.MAX_JOBID + 100): id = self.jobQueue._getNextID() self.assertNotEqual(str(id), self.jobId1) self.jobQueue.nextID = init_id
def __validateJob(self, job: TangoJob, vmms: List[Any]) -> int: """ validateJob - validate the input arguments in an addJob request. """ errors = 0 # If this isn't a Tango job then bail with an error if (not isinstance(job, TangoJob)): return -1 # Every job must have a name if not job.name: self.log.error("validateJob: Missing job.name") job.appendTrace("%s|validateJob: Missing job.name" % (datetime.utcnow().ctime())) errors += 1 # Check the virtual machine field if not job.vm: self.log.error("validateJob: Missing job.vm") job.appendTrace("%s|validateJob: Missing job.vm" % (datetime.utcnow().ctime())) errors += 1 else: if not job.vm.image: self.log.error("validateJob: Missing job.vm.image") job.appendTrace("%s|validateJob: Missing job.vm.image" % (datetime.utcnow().ctime())) errors += 1 else: vobj = vmms[Config.VMMS_NAME] imgList = vobj.getImages() if job.vm.image not in imgList: self.log.error("validateJob: Image not found: %s" % job.vm.image) job.appendTrace("%s|validateJob: Image not found: %s" % (datetime.utcnow().ctime(), job.vm.image)) errors += 1 else: (name, ext) = os.path.splitext(job.vm.image) job.vm.name = name if not job.vm.vmms: self.log.error("validateJob: Missing job.vm.vmms") job.appendTrace("%s|validateJob: Missing job.vm.vmms" % (datetime.utcnow().ctime())) errors += 1 else: if job.vm.vmms not in vmms: self.log.error("validateJob: Invalid vmms name: %s" % job.vm.vmms) job.appendTrace("%s|validateJob: Invalid vmms name: %s" % (datetime.utcnow().ctime(), job.vm.vmms)) errors += 1 # Check the output file if not job.outputFile: self.log.error("validateJob: Missing job.outputFile") job.appendTrace("%s|validateJob: Missing job.outputFile" % (datetime.utcnow().ctime())) errors += 1 else: if not os.path.exists(os.path.dirname(job.outputFile)): self.log.error("validateJob: Bad output path: %s", job.outputFile) job.appendTrace("%s|validateJob: Bad output path: %s" % (datetime.utcnow().ctime(), job.outputFile)) errors += 1 # Check for max output file size parameter if not job.maxOutputFileSize: self.log.debug( "validateJob: Setting job.maxOutputFileSize " "to default value: %d bytes", Config.MAX_OUTPUT_FILE_SIZE) job.maxOutputFileSize = Config.MAX_OUTPUT_FILE_SIZE # Check the list of input files hasMakefile = False for inputFile in job.input: if not inputFile.localFile: self.log.error("validateJob: Missing inputFile.localFile") job.appendTrace("%s|validateJob: Missing inputFile.localFile" % (datetime.utcnow().ctime())) errors += 1 else: if not os.path.exists(os.path.dirname(job.outputFile)): self.log.error("validateJob: Bad output path: %s", job.outputFile) job.appendTrace( "%s|validateJob: Bad output path: %s" % (datetime.utcnow().ctime(), job.outputFile)) errors += 1 if inputFile.destFile == 'Makefile': hasMakefile = True # Check if input files include a Makefile if not hasMakefile: self.log.error("validateJob: Missing Makefile in input files.") job.appendTrace( "%s|validateJob: Missing Makefile in input files." % (datetime.utcnow().ctime())) errors += 1 # Check if job timeout has been set; If not set timeout to default if not job.timeout or job.timeout <= 0: self.log.debug( "validateJob: Setting job.timeout to" " default config value: %d secs", Config.RUNJOB_TIMEOUT) job.timeout = Config.RUNJOB_TIMEOUT # Any problems, return an error status if errors > 0: self.log.error("validateJob: Job rejected: %d errors" % errors) job.appendTrace("%s|validateJob: Job rejected: %d errors" % (datetime.utcnow().ctime(), errors)) return -1 else: return 0