def test_pop_and_delete(self):
for i in range(N):
j = JobQueue.pop()
self.assertIsInstance(j, Job)
JobQueue.delete(j)
self.assertEqual(JobQueue.length(), 0)
return
def cl2s_submit(ad, verbose=False):
"""
Submit the input job ClassAd to CL2S. The important thing to notice is that
we split job cluster into their N component jobs and submit those as
independent jobs to the job queue.
Return the exit code: 0 if success >0 otherwise.
"""
# Determine the log level.
if(verbose):
logutils.logger.setLevel(logging.DEBUG)
# Parse the raw ad. Remember that the parsing code stores the raw ClassAd
# attributes both as they are and as lowercase and uppercase.
classAd = ClassAd.ClassAd(ad)
logutils.logger.debug('Parsed input ClassAd:\n%s' % (classAd))
# See if we have a Job ad. If not specified, we assume it is a Job ad.
ad_type = getattr(classAd, 'MyType', 'Job')
if(ad_type.lower() != 'job'):
logutils.logger.critical('Expecting a Job ClassAd, got a %s ad.' \
% (ad_type))
return(1)
# Remember though that Cmd is the parsed value for Executable...
if(hasattr(classAd, 'Executable'.lower())):
classAd.Cmd = classAd.executable
# Job ads need a few attributes: Cmd, JobUniverse and Owner.
required_attrs = ('Cmd', 'JobUniverse', 'Owner')
for attr in required_attrs:
if(not hasattr(classAd, attr.lower())):
logutils.logger.critical('The Job ClassAds needs a %s attribute.' \
% (attr))
return(2)
# JobUniverse needs to be a valid Condor universe ID but cannot be Standard
# Universe.
# UNIVERSE = {"VANILLA": 5, "SCHEDULER": 7, "GRID": 9, "JAVA": 10,
# "PARALLEL": 11, "LOCAL": 12, "VM": 13}
valid_universe_ids = (5, 7, 9, 10, 11, 12, 13)
if(classAd.jobuniverse not in valid_universe_ids):
logutils.logger.critical('The Job ClassAds universe is not valid: %d.' \
% (classAd.jobuniverse))
return(3)
# See if we need to split a cluster job. In that case, remember to replace
# $(Process) with range(classAd.CL2S_INSTANCES)
num_instances = classAd.CL2S_INSTANCES
# Now set classAd.CL2S_INSTANCES to 1
print('Submitting job(s).')
classAd.CL2S_INSTANCES = 1
new_ad_text = str(classAd)
for i in range(num_instances):
# Replace $(Process) with i, create a JOb instance and queue it.
JobQueue.push(Job.Job(new_ad_text.replace('$(Process)', str(i))))
print('%d job(s) submitted to cluster -1.' % (num_instances))
return
def test_push(self):
n = 0
for i in range(N):
j = Job(CLASS_AD.replace('j9am01070', 'j9am0%04d' % (i)))
JobQueue.push(j)
n += 1
self.assertEqual(JobQueue.length(), n)
return
# Split ads into slot and job ClassAds.
try:
(job_ad, slot_ad) = ads.split(SEPARATOR, 1)
except:
logutils.logger.critical('Unable to separate slot and job ads: %s' % (ads))
sys.exit(2)
logutils.logger.debug('Worker slot ClassAd %s' % (slot_ad))
logutils.logger.debug('Job ClassAd %s' % (job_ad))
logutils.logger.debug('Job was %sed' % (response))
# If the job was accepted, remove it from the queue. Otherwise re-insert it.
job = Job.Job(job_ad)
if(response.lower() == 'accept'):
logutils.logger.debug('Deleting the job from the queue.')
try:
JobQueue.delete(job)
except Exception, e:
logutils.logger.critical('Exception in dequeuing job: %s' % (e))
sys.exit(3)
else:
logutils.logger.debug('Re-inserting the job in the queue.')
try:
JobQueue.reinsert(job)
except Exception, e:
logutils.logger.critical('Exception in enqueuing job: %s' % (e))
sys.exit(4)
logutils.logger.debug('reply_fetch.py: quitting')
sys.exit(0)
from cl2s import Job
# Log the fact that we started up.
logutils.logger.debug('fetch_job.py: started')
# Read the raw Slot ClassAd from STDIN
slot_ad = sys.stdin.read()
logutils.logger.debug('Worker slot ClassAd:\n%s' % (slot_ad))
# Fetch a Job instance from the queue. This returns a Job instance if it managed
# to fetch one; None if there aren't any; an exception if soe error occurred.
try:
job = JobQueue.pop()
except Exception, e:
logutils.logger.critical('Exception in fetching work: %s' % (e))
sys.exit(1)
if(job is None):
logutils.logger.debug('Noting to do...')
sys.exit(0)
if(not isinstance(job, Job.Job)):
logutils.logger.critical('Was expecting a Job instance, got %s instead.' \
% (job.__class__.__name__))
sys.exit(2)
# If we got a Job, print it out to STDOUT and quit.
sys.stdout.write('%s\n' % (job))
logutils.logger.debug('fetch_job.py: quitting')
sys.exit(0)
