def loadData(self):
"""
_loadData_
Load all data that is associated with the jobgroup. This includes
loading all the subscription information, the output fileset
information and all the jobs that are associated with the group.
"""
existingTransaction = self.beginTransaction()
if self.id < 0 or self.uid == None:
self.load()
self.subscription.loadData()
self.output.loadData()
loadAction = self.daofactory(classname="JobGroup.LoadJobs")
result = loadAction.execute(self.id,
conn=self.getDBConn(),
transaction=self.existingTransaction())
self.jobs = []
self.newjobs = []
for jobID in result:
newJob = Job(id=jobID["id"])
newJob.loadData()
self.add(newJob)
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def loadData(self):
"""
_loadData_
Load all data that is associated with the jobgroup. This includes
loading all the subscription information, the output fileset
information and all the jobs that are associated with the group.
"""
existingTransaction = self.beginTransaction()
if self.id < 0 or self.uid == None:
self.load()
self.subscription.loadData()
self.output.loadData()
loadAction = self.daofactory(classname = "JobGroup.LoadJobs")
result = loadAction.execute(self.id, conn = self.getDBConn(),
transaction = self.existingTransaction())
self.jobs = []
self.newjobs = []
for jobID in result:
newJob = Job(id = jobID["id"])
newJob.loadData()
self.add(newJob)
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def testGetJobs(self):
"""
_testGetJobs_
Verify that the getJobs() method of the JobGroup class returns the
correct output for each output container type it supports.
"""
testJobA = Job()
testJobB = Job()
testJobGroup = JobGroup(jobs=[testJobA, testJobB])
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 2, "ERROR: Wrong number of jobs in job group"
goldenJobs = [testJobA, testJobB]
for job in testJobGroup.getJobs():
assert job in goldenJobs, "ERROR: Unknown Job in JobGroup."
goldenJobs.remove(job)
assert len(goldenJobs) == 0, "ERROR: Jobs are missing from the JobGroup."
goldenIDs = []
goldenIDs.append(testJobA["id"])
goldenIDs.append(testJobB["id"])
for jobID in testJobGroup.getJobs(type="id"):
assert jobID in goldenIDs, "ERROR: Unknown JobID in JobGroup"
goldenIDs.remove(jobID)
assert len(goldenIDs) == 0, "ERROR: Job IDs are missing from the JobGroup."
return
def testLen(self):
"""
__testLen__
Test that the __len__ function will actualy return the correct length.
"""
# This is simple. It should just have a length equal to the number of committed
# And yet to be committed jobs
testJobA = Job()
testJobB = Job()
testJobC = Job()
testJobD = Job()
testJobGroup = JobGroup(jobs=[testJobA, testJobB])
testJobGroup.commit()
self.assertEqual(len(testJobGroup), 2)
testJobGroup.add(testJobC)
self.assertEqual(len(testJobGroup), 3)
testJobGroup.commit()
testJobGroup.add(testJobD)
self.assertEqual(len(testJobGroup), 4)
return
def testAddCommit(self):
"""
_testAddCommit_
Test the add() and commit() methods of the JobGroup class. Verify that
jobs are not returned from getJobs() until commit() has been called.
"""
testJob = Job()
testJobGroup = JobGroup()
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs before jobs have been added."
testJobGroup.add(testJob)
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs commit() was called."
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 1, \
"ERROR: JobGroup has wrong number of jobs."
assert testJob in testJobGroup.getJobs(), \
"ERROR: JobGroup has unknown jobs."
return
def testLen(self):
"""
__testLen__
Test that the __len__ function will actualy return the correct length.
"""
#This is simple. It should just have a length equal to the number of committed
#And yet to be committed jobs
testJobA = Job()
testJobB = Job()
testJobC = Job()
testJobD = Job()
testJobGroup = JobGroup(jobs = [testJobA, testJobB])
testJobGroup.commit()
self.assertEqual(len(testJobGroup), 2)
testJobGroup.add(testJobC)
self.assertEqual(len(testJobGroup), 3)
testJobGroup.commit()
testJobGroup.add(testJobD)
self.assertEqual(len(testJobGroup), 4)
return
def testAddCommit(self):
"""
_testAddCommit_
Test the add() and commit() methods of the JobGroup class. Verify that
jobs are not returned from getJobs() until commit() has been called.
"""
testJob = Job()
testJobGroup = JobGroup()
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs before jobs have been added."
testJobGroup.add(testJob)
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs commit() was called."
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 1, \
"ERROR: JobGroup has wrong number of jobs."
assert testJob in testJobGroup.getJobs(), \
"ERROR: JobGroup has unknown jobs."
return
def commitBulk(self):
"""
Creates jobs in a group instead of singly, as is done in jobGroup.commit()
"""
myThread = threading.currentThread()
if self.id == -1:
myThread.transaction.begin()
#existingTransaction = self.beginTransaction()
self.create()
#self.commitTransaction(existingTransaction)
myThread.transaction.commit()
existingTransaction = self.beginTransaction()
listOfJobs = []
for job in self.newjobs:
#First do all the header stuff
if job["id"] != None:
continue
job["jobgroup"] = self.id
if job["name"] == None:
job["name"] = makeUUID()
listOfJobs.append(job)
bulkAction = self.daofactory(classname = "Jobs.New")
result = bulkAction.execute(jobList = listOfJobs)
#Use the results of the bulk commit to get the jobIDs
fileDict = {}
for job in listOfJobs:
job['id'] = result[job['name']]
fileDict[job['id']] = []
for file in job['input_files']:
fileDict[job['id']].append(file['id'])
maskAction = self.daofactory(classname = "Masks.New")
maskAction.execute(jobList = listOfJobs, conn = self.getDBConn(), \
transaction = self.existingTransaction())
fileAction = self.daofactory(classname = "Jobs.AddFiles")
fileAction.execute(jobDict = fileDict, conn = self.getDBConn(), \
transaction = self.existingTransaction())
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def commitBulk(self):
"""
Creates jobs in a group instead of singly, as is done in jobGroup.commit()
"""
myThread = threading.currentThread()
if self.id == -1:
myThread.transaction.begin()
#existingTransaction = self.beginTransaction()
self.create()
#self.commitTransaction(existingTransaction)
myThread.transaction.commit()
existingTransaction = self.beginTransaction()
listOfJobs = []
for job in self.newjobs:
#First do all the header stuff
if job["id"] != None:
continue
job["jobgroup"] = self.id
if job["name"] == None:
job["name"] = makeUUID()
listOfJobs.append(job)
bulkAction = self.daofactory(classname="Jobs.New")
result = bulkAction.execute(jobList=listOfJobs)
#Use the results of the bulk commit to get the jobIDs
fileDict = {}
for job in listOfJobs:
job['id'] = result[job['name']]
fileDict[job['id']] = []
for file in job['input_files']:
fileDict[job['id']].append(file['id'])
maskAction = self.daofactory(classname="Masks.New")
maskAction.execute(jobList = listOfJobs, conn = self.getDBConn(), \
transaction = self.existingTransaction())
fileAction = self.daofactory(classname="Jobs.AddFiles")
fileAction.execute(jobDict = fileDict, conn = self.getDBConn(), \
transaction = self.existingTransaction())
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def execute(self, *args, **kwargs):
self.logger.debug("Transforming old specs into jobs.")
# mapping to cache job def - blocks association
blocks = {}
regroupjobs = {}
## grouping in a dictionary can happen here
for job in args[0]:
if job.jobDefinitionID in regroupjobs:
regroupjobs[job.jobDefinitionID].append(job)
else:
regroupjobs[job.jobDefinitionID] = [job]
jobgroups = []
## here converting the grouping into proper JobGroup-Jobs
for jobdef in regroupjobs:
jobgroup = blocks.get(jobdef, None)
if jobgroup is None:
configreq = {'subresource': 'jobgroup',
'subjobdef': jobdef,
'subuser': kwargs['task']['tm_user_dn']}
self.logger.debug("Retrieving %d jobdef information from task manager db: %s" %(jobdef, str(configreq)))
jobgroup = self.server.get(self.resturl, data = configreq)
self.logger.debug("Jobgroup information in task manager: %s" % str(jobgroup))
jobgroup = jobgroup[0]['result'][0]
blocks[jobdef] = jobgroup['tm_data_blocks']
jg = WMJobGroup()
for job in regroupjobs[jobdef]:
parser = PassThroughOptionParser()
parser.add_option('--inputFile', dest='inputfiles', type='string')
parser.add_option('--runAndLumis', dest='runlumis', type='string')
parser.add_option('--availableSites', dest='allsites', type='string')
parser.add_option('--jobNumber', dest='jobnum', type='int')
(options, args) = parser.parse_args(shlex.split(job.jobParameters))
jj = WMJob()
jj['input_files'] = []
for infile in literal_eval(options.inputfiles):
jj['input_files'].append({'lfn': infile,
'block': blocks[jobdef],
'locations': [ss for ss in literal_eval(options.allsites)]})
if options.runlumis:
jj['mask']['runAndLumis'] = literal_eval(options.runlumis)
jj['panda_oldjobid'] = job.PandaID
jj['jobnum'] = options.jobnum
jg.add(jj)
setattr(jg, 'blocks', blocks[jobdef])
jg.commit()
jobgroups.append(jg)
return Result(task=kwargs['task'], result=jobgroups)
def commit(self):
"""
_commit_
Write any new jobs to the database, creating them in the database if
necessary.
"""
existingTransaction = self.beginTransaction()
if self.id == -1:
self.create()
for j in self.newjobs:
j.create(group = self)
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def commit(self):
"""
_commit_
Write any new jobs to the database, creating them in the database if
necessary.
"""
existingTransaction = self.beginTransaction()
if self.id == -1:
self.create()
for j in self.newjobs:
j.create(group=self)
WMJobGroup.commit(self)
self.commitTransaction(existingTransaction)
return
def testGetJobs(self):
"""
_testGetJobs_
Verify that the getJobs() method of the JobGroup class returns the
correct output for each output container type it supports.
"""
testJobA = Job()
testJobB = Job()
testJobGroup = JobGroup(jobs = [testJobA, testJobB])
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 2, \
"ERROR: Wrong number of jobs in job group"
goldenJobs = [testJobA, testJobB]
for job in testJobGroup.getJobs():
assert job in goldenJobs, \
"ERROR: Unknown Job in JobGroup."
goldenJobs.remove(job)
assert len(goldenJobs) == 0, \
"ERROR: Jobs are missing from the JobGroup."
goldenIDs = []
goldenIDs.append(testJobA["id"])
goldenIDs.append(testJobB["id"])
for jobID in testJobGroup.getJobs(type = "id"):
assert jobID in goldenIDs, \
"ERROR: Unknown JobID in JobGroup"
goldenIDs.remove(jobID)
assert len(goldenIDs) == 0, \
"ERROR: Job IDs are missing from the JobGroup."
return
def execute(self, *args, **kwargs):
self.logger.debug("Transforming old specs into jobs.")
# mapping to cache job def - blocks association
blocks = {}
regroupjobs = {}
## grouping in a dictionary can happen here
for job in args[0]:
if job.jobDefinitionID in regroupjobs:
regroupjobs[job.jobDefinitionID].append(job)
else:
regroupjobs[job.jobDefinitionID] = [job]
jobgroups = []
## here converting the grouping into proper JobGroup-Jobs
for jobdef in regroupjobs:
jobgroup = blocks.get(jobdef, None)
if jobgroup is None:
configreq = {
'subresource': 'jobgroup',
'subjobdef': jobdef,
'subuser': kwargs['task']['tm_user_dn']
}
self.logger.debug(
"Retrieving %d jobdef information from task manager db: %s"
% (jobdef, str(configreq)))
jobgroup = self.server.get(self.resturl, data=configreq)
self.logger.debug("Jobgroup information in task manager: %s" %
str(jobgroup))
jobgroup = jobgroup[0]['result'][0]
blocks[jobdef] = jobgroup['tm_data_blocks']
jg = WMJobGroup()
for job in regroupjobs[jobdef]:
parser = PassThroughOptionParser()
parser.add_option('--inputFile',
dest='inputfiles',
type='string')
parser.add_option('--runAndLumis',
dest='runlumis',
type='string')
parser.add_option('--availableSites',
dest='allsites',
type='string')
parser.add_option('--jobNumber', dest='jobnum', type='int')
(options,
args) = parser.parse_args(shlex.split(job.jobParameters))
jj = WMJob()
jj['input_files'] = []
for infile in literal_eval(options.inputfiles):
jj['input_files'].append({
'lfn':
infile,
'block':
blocks[jobdef],
'locations':
[ss for ss in literal_eval(options.allsites)]
})
if options.runlumis:
jj['mask']['runAndLumis'] = literal_eval(options.runlumis)
jj['panda_oldjobid'] = job.PandaID
jj['jobnum'] = options.jobnum
jg.add(jj)
setattr(jg, 'blocks', blocks[jobdef])
jg.commit()
jobgroups.append(jg)
return Result(task=kwargs['task'], result=jobgroups)