def testAddCommit(self):
"""
_testAddCommit_
Test the add() and commit() methods of the JobGroup class. Verify that
jobs are not returned from getJobs() until commit() has been called.
"""
testJob = Job()
testJobGroup = JobGroup()
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs before jobs have been added."
testJobGroup.add(testJob)
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs commit() was called."
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 1, \
"ERROR: JobGroup has wrong number of jobs."
assert testJob in testJobGroup.getJobs(), \
"ERROR: JobGroup has unknown jobs."
return
def loadData(self):
"""
_loadData_
Load all data that is associated with the jobgroup. This includes
loading all the subscription information, the output fileset
information and all the jobs that are associated with the group.
"""
existingTransaction = self.beginTransaction()
if self.id < 0 or self.uid == None:
self.load()
self.subscription.loadData()
self.output.loadData()
loadAction = self.daofactory(classname = "JobGroup.LoadJobs")
result = loadAction.execute(self.id, conn = self.getDBConn(),
transaction = self.existingTransaction())
self.jobs = []
self.newjobs = []
for jobID in result:
newJob = Job(id = jobID["id"])
newJob.loadData()
self.add(newJob)
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def testLen(self):
"""
__testLen__
Test that the __len__ function will actualy return the correct length.
"""
#This is simple. It should just have a length equal to the number of committed
#And yet to be committed jobs
testJobA = Job()
testJobB = Job()
testJobC = Job()
testJobD = Job()
testJobGroup = JobGroup(jobs = [testJobA, testJobB])
testJobGroup.commit()
self.assertEqual(len(testJobGroup), 2)
testJobGroup.add(testJobC)
self.assertEqual(len(testJobGroup), 3)
testJobGroup.commit()
testJobGroup.add(testJobD)
self.assertEqual(len(testJobGroup), 4)
return
def loadData(self):
"""
_loadData_
Load all data that is associated with the jobgroup. This includes
loading all the subscription information, the output fileset
information and all the jobs that are associated with the group.
"""
existingTransaction = self.beginTransaction()
if self.id < 0 or self.uid == None:
self.load()
self.subscription.loadData()
self.output.loadData()
loadAction = self.daofactory(classname="JobGroup.LoadJobs")
result = loadAction.execute(self.id,
conn=self.getDBConn(),
transaction=self.existingTransaction())
self.jobs = []
self.newjobs = []
for jobID in result:
newJob = Job(id=jobID["id"])
newJob.loadData()
self.add(newJob)
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def testLen(self):
"""
__testLen__
Test that the __len__ function will actualy return the correct length.
"""
# This is simple. It should just have a length equal to the number of committed
# And yet to be committed jobs
testJobA = Job()
testJobB = Job()
testJobC = Job()
testJobD = Job()
testJobGroup = JobGroup(jobs=[testJobA, testJobB])
testJobGroup.commit()
self.assertEqual(len(testJobGroup), 2)
testJobGroup.add(testJobC)
self.assertEqual(len(testJobGroup), 3)
testJobGroup.commit()
testJobGroup.add(testJobD)
self.assertEqual(len(testJobGroup), 4)
return
def testAddCommit(self):
"""
_testAddCommit_
Test the add() and commit() methods of the JobGroup class. Verify that
jobs are not returned from getJobs() until commit() has been called.
"""
testJob = Job()
testJobGroup = JobGroup()
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs before jobs have been added."
testJobGroup.add(testJob)
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs commit() was called."
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 1, \
"ERROR: JobGroup has wrong number of jobs."
assert testJob in testJobGroup.getJobs(), \
"ERROR: JobGroup has unknown jobs."
return
def commitBulk(self):
"""
Creates jobs in a group instead of singly, as is done in jobGroup.commit()
"""
myThread = threading.currentThread()
if self.id == -1:
myThread.transaction.begin()
#existingTransaction = self.beginTransaction()
self.create()
#self.commitTransaction(existingTransaction)
myThread.transaction.commit()
existingTransaction = self.beginTransaction()
listOfJobs = []
for job in self.newjobs:
#First do all the header stuff
if job["id"] != None:
continue
job["jobgroup"] = self.id
if job["name"] == None:
job["name"] = makeUUID()
listOfJobs.append(job)
bulkAction = self.daofactory(classname = "Jobs.New")
result = bulkAction.execute(jobList = listOfJobs)
#Use the results of the bulk commit to get the jobIDs
fileDict = {}
for job in listOfJobs:
job['id'] = result[job['name']]
fileDict[job['id']] = []
for file in job['input_files']:
fileDict[job['id']].append(file['id'])
maskAction = self.daofactory(classname = "Masks.New")
maskAction.execute(jobList = listOfJobs, conn = self.getDBConn(), \
transaction = self.existingTransaction())
fileAction = self.daofactory(classname = "Jobs.AddFiles")
fileAction.execute(jobDict = fileDict, conn = self.getDBConn(), \
transaction = self.existingTransaction())
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def __init__(self, subscription = None, jobs = None, id = -1, uid = None, location = None):
WMBSBase.__init__(self)
WMJobGroup.__init__(self, subscription=subscription, jobs = jobs)
self.id = id
self.lastUpdate = None
self.uid = uid
if location != None:
self.setSite(location)
return
def commitBulk(self):
"""
Creates jobs in a group instead of singly, as is done in jobGroup.commit()
"""
myThread = threading.currentThread()
if self.id == -1:
myThread.transaction.begin()
#existingTransaction = self.beginTransaction()
self.create()
#self.commitTransaction(existingTransaction)
myThread.transaction.commit()
existingTransaction = self.beginTransaction()
listOfJobs = []
for job in self.newjobs:
#First do all the header stuff
if job["id"] != None:
continue
job["jobgroup"] = self.id
if job["name"] == None:
job["name"] = makeUUID()
listOfJobs.append(job)
bulkAction = self.daofactory(classname="Jobs.New")
result = bulkAction.execute(jobList=listOfJobs)
#Use the results of the bulk commit to get the jobIDs
fileDict = {}
for job in listOfJobs:
job['id'] = result[job['name']]
fileDict[job['id']] = []
for file in job['input_files']:
fileDict[job['id']].append(file['id'])
maskAction = self.daofactory(classname="Masks.New")
maskAction.execute(jobList = listOfJobs, conn = self.getDBConn(), \
transaction = self.existingTransaction())
fileAction = self.daofactory(classname="Jobs.AddFiles")
fileAction.execute(jobDict = fileDict, conn = self.getDBConn(), \
transaction = self.existingTransaction())
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def execute(self, *args, **kwargs):
self.logger.debug("Transforming old specs into jobs.")
# mapping to cache job def - blocks association
blocks = {}
regroupjobs = {}
## grouping in a dictionary can happen here
for job in args[0]:
if job.jobDefinitionID in regroupjobs:
regroupjobs[job.jobDefinitionID].append(job)
else:
regroupjobs[job.jobDefinitionID] = [job]
jobgroups = []
## here converting the grouping into proper JobGroup-Jobs
for jobdef in regroupjobs:
jobgroup = blocks.get(jobdef, None)
if jobgroup is None:
configreq = {'subresource': 'jobgroup',
'subjobdef': jobdef,
'subuser': kwargs['task']['tm_user_dn']}
self.logger.debug("Retrieving %d jobdef information from task manager db: %s" %(jobdef, str(configreq)))
jobgroup = self.server.get(self.resturl, data = configreq)
self.logger.debug("Jobgroup information in task manager: %s" % str(jobgroup))
jobgroup = jobgroup[0]['result'][0]
blocks[jobdef] = jobgroup['tm_data_blocks']
jg = WMJobGroup()
for job in regroupjobs[jobdef]:
parser = PassThroughOptionParser()
parser.add_option('--inputFile', dest='inputfiles', type='string')
parser.add_option('--runAndLumis', dest='runlumis', type='string')
parser.add_option('--availableSites', dest='allsites', type='string')
parser.add_option('--jobNumber', dest='jobnum', type='int')
(options, args) = parser.parse_args(shlex.split(job.jobParameters))
jj = WMJob()
jj['input_files'] = []
for infile in literal_eval(options.inputfiles):
jj['input_files'].append({'lfn': infile,
'block': blocks[jobdef],
'locations': [ss for ss in literal_eval(options.allsites)]})
if options.runlumis:
jj['mask']['runAndLumis'] = literal_eval(options.runlumis)
jj['panda_oldjobid'] = job.PandaID
jj['jobnum'] = options.jobnum
jg.add(jj)
setattr(jg, 'blocks', blocks[jobdef])
jg.commit()
jobgroups.append(jg)
return Result(task=kwargs['task'], result=jobgroups)
def testCreate(self):
"""
_testCreate_
Test the JobGroup constructor and passing different job containers
into it.
"""
testSubscription = Subscription()
testJobGroupA = JobGroup(subscription = testSubscription)
assert testJobGroupA.subscription == testSubscription, \
"ERROR: Failed to pass subscription in constructor"
assert len(testJobGroupA.jobs) == 0 and len(testJobGroupA.newjobs) == 0, \
"ERROR: JobGroup not empty on creation"
testJobA = Job()
testJobB = Job()
testJobGroupB = JobGroup(jobs = [testJobA, testJobB])
assert testJobGroupB.jobs == [], \
"ERROR: Jobs committed to jobgroup too soon."
jobGroupJobs = testJobGroupB.newjobs
goldenJobs = [testJobA, testJobB]
for job in jobGroupJobs:
assert job in goldenJobs, \
"ERROR: Extra job in job group"
goldenJobs.remove(job)
assert len(goldenJobs) == 0, \
"ERROR: Job missing from job group"
testJobGroupC = JobGroup(jobs = testJobA)
assert testJobGroupC.jobs == [], \
"ERROR: Jobs committed to jobgroup too soon."
jobGroupJobs = testJobGroupC.newjobs
assert len(jobGroupJobs) == 1, \
"ERROR: Wrong number of jobs in jobgroup."
assert testJobA in jobGroupJobs, \
"ERROR: Wrong job in jobgroup."
return
def __init__(self,
subscription=None,
jobs=None,
id=-1,
uid=None,
location=None):
WMBSBase.__init__(self)
WMJobGroup.__init__(self, subscription=subscription, jobs=jobs)
self.id = id
self.lastUpdate = None
self.uid = uid
if location != None:
self.setSite(location)
return
def commit(self):
"""
_commit_
Write any new jobs to the database, creating them in the database if
necessary.
"""
existingTransaction = self.beginTransaction()
if self.id == -1:
self.create()
for j in self.newjobs:
j.create(group = self)
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def commit(self):
"""
_commit_
Write any new jobs to the database, creating them in the database if
necessary.
"""
existingTransaction = self.beginTransaction()
if self.id == -1:
self.create()
for j in self.newjobs:
j.create(group=self)
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def testGetJobs(self):
"""
_testGetJobs_
Verify that the getJobs() method of the JobGroup class returns the
correct output for each output container type it supports.
"""
testJobA = Job()
testJobB = Job()
testJobGroup = JobGroup(jobs=[testJobA, testJobB])
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 2, "ERROR: Wrong number of jobs in job group"
goldenJobs = [testJobA, testJobB]
for job in testJobGroup.getJobs():
assert job in goldenJobs, "ERROR: Unknown Job in JobGroup."
goldenJobs.remove(job)
assert len(goldenJobs) == 0, "ERROR: Jobs are missing from the JobGroup."
goldenIDs = []
goldenIDs.append(testJobA["id"])
goldenIDs.append(testJobB["id"])
for jobID in testJobGroup.getJobs(type="id"):
assert jobID in goldenIDs, "ERROR: Unknown JobID in JobGroup"
goldenIDs.remove(jobID)
assert len(goldenIDs) == 0, "ERROR: Job IDs are missing from the JobGroup."
return
def testAddOutput(self):
"""
_testAddOutput_
Test the JobGroup's addOutput() method. Verify that files are committed
to the output fileset immediately and are available from the getOutput()
method.
"""
testFile = File()
testJobGroup = JobGroup()
assert len(testJobGroup.getOutput()) == 0, \
"ERROR: Files in the output fileset before anything has been added."
testJobGroup.addOutput(testFile)
assert len(testJobGroup.getOutput()) == 1, \
"ERROR: Unknown number of files in JobGroup output fileset."
assert testFile in testJobGroup.getOutput(), \
"ERROR: Unknown file in the JobGroup output fileset."
return
def testAddOutput(self):
"""
_testAddOutput_
Test the JobGroup's addOutput() method. Verify that files are committed
to the output fileset immediately and are available from the getOutput()
method.
"""
testFile = File()
testJobGroup = JobGroup()
assert len(testJobGroup.getOutput()) == 0, "ERROR: Files in the output fileset before anything has been added."
testJobGroup.addOutput(testFile)
assert len(testJobGroup.getOutput()) == 1, "ERROR: Unknown number of files in JobGroup output fileset."
assert testFile in testJobGroup.getOutput(), "ERROR: Unknown file in the JobGroup output fileset."
return
def testGetJobs(self):
"""
_testGetJobs_
Verify that the getJobs() method of the JobGroup class returns the
correct output for each output container type it supports.
"""
testJobA = Job()
testJobB = Job()
testJobGroup = JobGroup(jobs = [testJobA, testJobB])
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 2, \
"ERROR: Wrong number of jobs in job group"
goldenJobs = [testJobA, testJobB]
for job in testJobGroup.getJobs():
assert job in goldenJobs, \
"ERROR: Unknown Job in JobGroup."
goldenJobs.remove(job)
assert len(goldenJobs) == 0, \
"ERROR: Jobs are missing from the JobGroup."
goldenIDs = []
goldenIDs.append(testJobA["id"])
goldenIDs.append(testJobB["id"])
for jobID in testJobGroup.getJobs(type = "id"):
assert jobID in goldenIDs, \
"ERROR: Unknown JobID in JobGroup"
goldenIDs.remove(jobID)
assert len(goldenIDs) == 0, \
"ERROR: Job IDs are missing from the JobGroup."
return
def execute(self, *args, **kwargs):
self.logger.debug("Transforming old specs into jobs.")
# mapping to cache job def - blocks association
blocks = {}
regroupjobs = {}
## grouping in a dictionary can happen here
for job in args[0]:
if job.jobDefinitionID in regroupjobs:
regroupjobs[job.jobDefinitionID].append(job)
else:
regroupjobs[job.jobDefinitionID] = [job]
jobgroups = []
## here converting the grouping into proper JobGroup-Jobs
for jobdef in regroupjobs:
jobgroup = blocks.get(jobdef, None)
if jobgroup is None:
configreq = {
'subresource': 'jobgroup',
'subjobdef': jobdef,
'subuser': kwargs['task']['tm_user_dn']
}
self.logger.debug(
"Retrieving %d jobdef information from task manager db: %s"
% (jobdef, str(configreq)))
jobgroup = self.server.get(self.resturl, data=configreq)
self.logger.debug("Jobgroup information in task manager: %s" %
str(jobgroup))
jobgroup = jobgroup[0]['result'][0]
blocks[jobdef] = jobgroup['tm_data_blocks']
jg = WMJobGroup()
for job in regroupjobs[jobdef]:
parser = PassThroughOptionParser()
parser.add_option('--inputFile',
dest='inputfiles',
type='string')
parser.add_option('--runAndLumis',
dest='runlumis',
type='string')
parser.add_option('--availableSites',
dest='allsites',
type='string')
parser.add_option('--jobNumber', dest='jobnum', type='int')
(options,
args) = parser.parse_args(shlex.split(job.jobParameters))
jj = WMJob()
jj['input_files'] = []
for infile in literal_eval(options.inputfiles):
jj['input_files'].append({
'lfn':
infile,
'block':
blocks[jobdef],
'locations':
[ss for ss in literal_eval(options.allsites)]
})
if options.runlumis:
jj['mask']['runAndLumis'] = literal_eval(options.runlumis)
jj['panda_oldjobid'] = job.PandaID
jj['jobnum'] = options.jobnum
jg.add(jj)
setattr(jg, 'blocks', blocks[jobdef])
jg.commit()
jobgroups.append(jg)
return Result(task=kwargs['task'], result=jobgroups)
