def test_pop_and_delete(self): for i in range(N): j = JobQueue.pop() self.assertIsInstance(j, Job) JobQueue.delete(j) self.assertEqual(JobQueue.length(), 0) return
def cl2s_submit(ad, verbose=False): """ Submit the input job ClassAd to CL2S. The important thing to notice is that we split job cluster into their N component jobs and submit those as independent jobs to the job queue. Return the exit code: 0 if success >0 otherwise. """ # Determine the log level. if(verbose): logutils.logger.setLevel(logging.DEBUG) # Parse the raw ad. Remember that the parsing code stores the raw ClassAd # attributes both as they are and as lowercase and uppercase. classAd = ClassAd.ClassAd(ad) logutils.logger.debug('Parsed input ClassAd:\n%s' % (classAd)) # See if we have a Job ad. If not specified, we assume it is a Job ad. ad_type = getattr(classAd, 'MyType', 'Job') if(ad_type.lower() != 'job'): logutils.logger.critical('Expecting a Job ClassAd, got a %s ad.' \ % (ad_type)) return(1) # Remember though that Cmd is the parsed value for Executable... if(hasattr(classAd, 'Executable'.lower())): classAd.Cmd = classAd.executable # Job ads need a few attributes: Cmd, JobUniverse and Owner. required_attrs = ('Cmd', 'JobUniverse', 'Owner') for attr in required_attrs: if(not hasattr(classAd, attr.lower())): logutils.logger.critical('The Job ClassAds needs a %s attribute.' \ % (attr)) return(2) # JobUniverse needs to be a valid Condor universe ID but cannot be Standard # Universe. # UNIVERSE = {"VANILLA": 5, "SCHEDULER": 7, "GRID": 9, "JAVA": 10, # "PARALLEL": 11, "LOCAL": 12, "VM": 13} valid_universe_ids = (5, 7, 9, 10, 11, 12, 13) if(classAd.jobuniverse not in valid_universe_ids): logutils.logger.critical('The Job ClassAds universe is not valid: %d.' \ % (classAd.jobuniverse)) return(3) # See if we need to split a cluster job. In that case, remember to replace # $(Process) with range(classAd.CL2S_INSTANCES) num_instances = classAd.CL2S_INSTANCES # Now set classAd.CL2S_INSTANCES to 1 print('Submitting job(s).') classAd.CL2S_INSTANCES = 1 new_ad_text = str(classAd) for i in range(num_instances): # Replace $(Process) with i, create a JOb instance and queue it. JobQueue.push(Job.Job(new_ad_text.replace('$(Process)', str(i)))) print('%d job(s) submitted to cluster -1.' % (num_instances)) return
def test_push(self): n = 0 for i in range(N): j = Job(CLASS_AD.replace('j9am01070', 'j9am0%04d' % (i))) JobQueue.push(j) n += 1 self.assertEqual(JobQueue.length(), n) return
# Split ads into slot and job ClassAds. try: (job_ad, slot_ad) = ads.split(SEPARATOR, 1) except: logutils.logger.critical('Unable to separate slot and job ads: %s' % (ads)) sys.exit(2) logutils.logger.debug('Worker slot ClassAd %s' % (slot_ad)) logutils.logger.debug('Job ClassAd %s' % (job_ad)) logutils.logger.debug('Job was %sed' % (response)) # If the job was accepted, remove it from the queue. Otherwise re-insert it. job = Job.Job(job_ad) if(response.lower() == 'accept'): logutils.logger.debug('Deleting the job from the queue.') try: JobQueue.delete(job) except Exception, e: logutils.logger.critical('Exception in dequeuing job: %s' % (e)) sys.exit(3) else: logutils.logger.debug('Re-inserting the job in the queue.') try: JobQueue.reinsert(job) except Exception, e: logutils.logger.critical('Exception in enqueuing job: %s' % (e)) sys.exit(4) logutils.logger.debug('reply_fetch.py: quitting') sys.exit(0)
from cl2s import Job # Log the fact that we started up. logutils.logger.debug('fetch_job.py: started') # Read the raw Slot ClassAd from STDIN slot_ad = sys.stdin.read() logutils.logger.debug('Worker slot ClassAd:\n%s' % (slot_ad)) # Fetch a Job instance from the queue. This returns a Job instance if it managed # to fetch one; None if there aren't any; an exception if soe error occurred. try: job = JobQueue.pop() except Exception, e: logutils.logger.critical('Exception in fetching work: %s' % (e)) sys.exit(1) if(job is None): logutils.logger.debug('Noting to do...') sys.exit(0) if(not isinstance(job, Job.Job)): logutils.logger.critical('Was expecting a Job instance, got %s instead.' \ % (job.__class__.__name__)) sys.exit(2) # If we got a Job, print it out to STDOUT and quit. sys.stdout.write('%s\n' % (job)) logutils.logger.debug('fetch_job.py: quitting') sys.exit(0)