def testCommitBulk(self):
"""
_testCommitBulk_
Exactly the same as testCommit, but using commitBulk() instead of commit()
"""
myThread = threading.currentThread()
testJobGroupA = self.createLargerTestJobGroup(commitFlag=False)
testJobGroupB = JobGroup(id=testJobGroupA.id)
testJobGroupB.loadData()
assert len(testJobGroupA.getJobs()) == 0, \
"ERROR: Original object commited too early"
assert len(testJobGroupB.getJobs()) == 0, \
"ERROR: Loaded JobGroup has too many jobs"
testJobGroupA.commitBulk()
testJobGroupA.loadData()
self.assertEqual(len(testJobGroupA.getJobs()), 102)
testJobGroupC = JobGroup(id=testJobGroupA.id)
testJobGroupC.loadData()
self.assertEqual(len(testJobGroupC.getJobs()), 102)
self.assertEqual(testJobGroupC.jobs[0].getFiles()[0]['lfn'], '/this/is/a/lfnC')
self.assertEqual(testJobGroupC.jobs[1].getFiles()[0]['lfn'], '/this/is/a/lfnD')
return
def testCommit(self):
"""
_testCommit_
Verify that jobs are not added to a job group until commit() is called
on the JobGroup. Also verify that commit() correctly commits the jobs
to the database.
"""
testJobGroupA = self.createTestJobGroup(commitFlag=False)
testJobGroupB = JobGroup(id=testJobGroupA.id)
testJobGroupB.loadData()
assert len(testJobGroupA.getJobs()) == 0, \
"ERROR: Original object commited too early"
assert len(testJobGroupB.getJobs()) == 0, \
"ERROR: Loaded JobGroup has too many jobs"
testJobGroupA.commit()
testJobGroupA.loadData()
assert len(testJobGroupA.getJobs()) == 2, \
"ERROR: Original object did not commit jobs"
testJobGroupC = JobGroup(id=testJobGroupA.id)
testJobGroupC.loadData()
assert len(testJobGroupC.getJobs()) == 2, \
"ERROR: Loaded object has too few jobs."
return
def testCommitBulk(self):
"""
_testCommitBulk_
Exactly the same as testCommit, but using commitBulk() instead of commit()
"""
myThread = threading.currentThread()
testJobGroupA = self.createLargerTestJobGroup(commitFlag=False)
testJobGroupB = JobGroup(id=testJobGroupA.id)
testJobGroupB.loadData()
assert len(testJobGroupA.getJobs()) == 0, \
"ERROR: Original object commited too early"
assert len(testJobGroupB.getJobs()) == 0, \
"ERROR: Loaded JobGroup has too many jobs"
testJobGroupA.commitBulk()
testJobGroupA.loadData()
self.assertEqual(len(testJobGroupA.getJobs()), 102)
testJobGroupC = JobGroup(id=testJobGroupA.id)
testJobGroupC.loadData()
self.assertEqual(len(testJobGroupC.getJobs()), 102)
self.assertEqual(testJobGroupC.jobs[0].getFiles()[0]['lfn'],
'/this/is/a/lfnC')
self.assertEqual(testJobGroupC.jobs[1].getFiles()[0]['lfn'],
'/this/is/a/lfnD')
return
def testLoadData(self):
"""
_testLoadData_
Test loading the JobGroup, it's meta data and any data associated with
its output fileset and jobs from the database.
"""
testJobGroupA = self.createTestJobGroup()
testJobGroupB = JobGroup(id=testJobGroupA.id)
testJobGroupB.loadData()
assert testJobGroupB.subscription["id"] == \
testJobGroupA.subscription["id"], \
"ERROR: Job group did not load subscription correctly"
goldenJobs = testJobGroupA.getJobs(type="list")
for job in testJobGroupB.getJobs(type="list"):
assert job in goldenJobs, \
"ERROR: JobGroup loaded an unknown job: %s, %s" % (job, goldenJobs)
goldenJobs.remove(job)
assert len(goldenJobs) == 0, \
"ERROR: JobGroup didn't load all jobs"
assert testJobGroupB.output.id == testJobGroupA.output.id, \
"ERROR: Output fileset didn't load properly"
return
def testCommit(self):
"""
_testCommit_
Verify that jobs are not added to a job group until commit() is called
on the JobGroup. Also verify that commit() correctly commits the jobs
to the database.
"""
testJobGroupA = self.createTestJobGroup(commitFlag=False)
testJobGroupB = JobGroup(id=testJobGroupA.id)
testJobGroupB.loadData()
assert len(testJobGroupA.getJobs()) == 0, \
"ERROR: Original object commited too early"
assert len(testJobGroupB.getJobs()) == 0, \
"ERROR: Loaded JobGroup has too many jobs"
testJobGroupA.commit()
testJobGroupA.loadData()
assert len(testJobGroupA.getJobs()) == 2, \
"ERROR: Original object did not commit jobs"
testJobGroupC = JobGroup(id=testJobGroupA.id)
testJobGroupC.loadData()
assert len(testJobGroupC.getJobs()) == 2, \
"ERROR: Loaded object has too few jobs."
return
def testLoadData(self):
"""
_testLoadData_
Test loading the JobGroup, it's meta data and any data associated with
its output fileset and jobs from the database.
"""
testJobGroupA = self.createTestJobGroup()
testJobGroupB = JobGroup(id=testJobGroupA.id)
testJobGroupB.loadData()
assert testJobGroupB.subscription["id"] == \
testJobGroupA.subscription["id"], \
"ERROR: Job group did not load subscription correctly"
goldenJobs = testJobGroupA.getJobs(type="list")
for job in testJobGroupB.getJobs(type="list"):
assert job in goldenJobs, \
"ERROR: JobGroup loaded an unknown job: %s, %s" % (job, goldenJobs)
goldenJobs.remove(job)
assert len(goldenJobs) == 0, \
"ERROR: JobGroup didn't load all jobs"
assert testJobGroupB.output.id == testJobGroupA.output.id, \
"ERROR: Output fileset didn't load properly"
return
def testLoadData(self):
"""
_testLoadData_
Test loading the JobGroup, it's meta data and any data associated with
its output fileset and jobs from the database.
"""
testJobGroupA = self.createTestJobGroup()
testJobGroupB = JobGroup(id=testJobGroupA.id)
testJobGroupB.loadData()
self.assertEqual(testJobGroupB.subscription["id"], testJobGroupA.subscription["id"],
"Job group did not load subscription correctly")
# Build a tuple with important job information for each job group
goldenJobs = [(job['id'], job['name'],
sorted([inp_file['lfn'] for inp_file in job['input_files']]))
for job in testJobGroupA.getJobs(type="list")]
loadedJobs = [(job['id'], job['name'],
sorted([inp_file['lfn'] for inp_file in job['input_files']]))
for job in testJobGroupB.getJobs(type="list")]
# Make sure each job from one group is in the other list
for job in loadedJobs:
self.assertIn(job, goldenJobs, "JobGroup loaded an unknown job: \n%s \nis not in \n%s" %
(pretty(job), pretty(goldenJobs)))
goldenJobs.remove(job)
self.assertFalse(goldenJobs, "JobGroup didn't load all jobs")
self.assertEqual(testJobGroupB.output.id, testJobGroupA.output.id, "Output fileset didn't load properly")
return
def testCommitTransaction(self):
"""
_testCommitTransaction_
Create a JobGroup and then add some jobs to it. Begin a transaction
and then call commit() on the JobGroup. Verify that the newly committed
jobs can be loaded from the database. Rollback the transaction and then
verify that the jobs that were committed before are no longer associated
with the JobGroup.
"""
testJobGroupA = self.createTestJobGroup(commitFlag=False)
testJobGroupB = JobGroup(id=testJobGroupA.id)
testJobGroupB.loadData()
assert len(testJobGroupA.getJobs()) == 0, \
"ERROR: Original object commited too early"
assert len(testJobGroupB.getJobs()) == 0, \
"ERROR: Loaded JobGroup has too many jobs"
myThread = threading.currentThread()
myThread.transaction.begin()
testJobGroupA.commit()
assert len(testJobGroupA.getJobs()) == 2, \
"ERROR: Original object did not commit jobs"
testJobGroupC = JobGroup(id=testJobGroupA.id)
testJobGroupC.loadData()
assert len(testJobGroupC.getJobs()) == 2, \
"ERROR: Loaded object has too few jobs."
myThread.transaction.rollback()
testJobGroupD = JobGroup(id=testJobGroupA.id)
testJobGroupD.loadData()
assert len(testJobGroupD.getJobs()) == 0, \
"ERROR: Loaded object has too many jobs."
return
def testCommitTransaction(self):
"""
_testCommitTransaction_
Create a JobGroup and then add some jobs to it. Begin a transaction
and then call commit() on the JobGroup. Verify that the newly committed
jobs can be loaded from the database. Rollback the transaction and then
verify that the jobs that were committed before are no longer associated
with the JobGroup.
"""
testJobGroupA = self.createTestJobGroup(commitFlag=False)
testJobGroupB = JobGroup(id=testJobGroupA.id)
testJobGroupB.loadData()
assert len(testJobGroupA.getJobs()) == 0, \
"ERROR: Original object commited too early"
assert len(testJobGroupB.getJobs()) == 0, \
"ERROR: Loaded JobGroup has too many jobs"
myThread = threading.currentThread()
myThread.transaction.begin()
testJobGroupA.commit()
assert len(testJobGroupA.getJobs()) == 2, \
"ERROR: Original object did not commit jobs"
testJobGroupC = JobGroup(id=testJobGroupA.id)
testJobGroupC.loadData()
assert len(testJobGroupC.getJobs()) == 2, \
"ERROR: Loaded object has too few jobs."
myThread.transaction.rollback()
testJobGroupD = JobGroup(id=testJobGroupA.id)
testJobGroupD.loadData()
assert len(testJobGroupD.getJobs()) == 0, \
"ERROR: Loaded object has too many jobs."
return
def verifyFileMetaData(self, jobID, fwkJobReportFiles):
"""
_verifyFileMetaData_
Verify that all the files that were output by a job made it into WMBS
correctly. Compare the contents of WMBS to the files in the frameworks
job report.
Note that fwkJobReportFiles is a list of DataStructs File objects.
"""
testJob = Job(id = jobID)
testJob.loadData()
inputLFNs = []
for inputFile in testJob["input_files"]:
inputLFNs.append(inputFile["lfn"])
for fwkJobReportFile in fwkJobReportFiles:
outputFile = File(lfn = fwkJobReportFile["lfn"])
outputFile.loadData(parentage = 1)
assert outputFile["events"] == int(fwkJobReportFile["events"]), \
"Error: Output file has wrong events: %s, %s" % \
(outputFile["events"], fwkJobReportFile["events"])
assert outputFile["size"] == int(fwkJobReportFile["size"]), \
"Error: Output file has wrong size: %s, %s" % \
(outputFile["size"], fwkJobReportFile["size"])
for ckType in fwkJobReportFile["checksums"]:
assert ckType in outputFile["checksums"], \
"Error: Output file is missing checksums: %s" % ckType
assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \
"Error: Checksums don't match."
assert len(fwkJobReportFile["checksums"]) == \
len(outputFile["checksums"]), \
"Error: Wrong number of checksums."
jobType = self.getJobTypeAction.execute(jobID = jobID)
if jobType == "Merge":
assert str(outputFile["merged"]) == "True", \
"Error: Merge jobs should output merged files."
else:
assert outputFile["merged"] == fwkJobReportFile["merged"], \
"Error: Output file merged output is wrong: %s, %s" % \
(outputFile["merged"], fwkJobReportFile["merged"])
assert len(outputFile["locations"]) == 1, \
"Error: outputfile should have one location: %s" % outputFile["locations"]
assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \
"Error: wrong location for file."
assert len(outputFile["parents"]) == len(inputLFNs), \
"Error: Output file has wrong number of parents."
for outputParent in outputFile["parents"]:
assert outputParent["lfn"] in inputLFNs, \
"Error: Unknown parent file: %s" % outputParent["lfn"]
fwjrRuns = {}
for run in fwkJobReportFile["runs"]:
fwjrRuns[run.run] = run.lumis
for run in outputFile["runs"]:
assert run.run in fwjrRuns, \
"Error: Extra run in output: %s" % run.run
for lumi in run:
assert lumi in fwjrRuns[run.run], \
"Error: Extra lumi: %s" % lumi
fwjrRuns[run.run].remove(lumi)
if len(fwjrRuns[run.run]) == 0:
del fwjrRuns[run.run]
assert len(fwjrRuns) == 0, \
"Error: Missing runs, lumis: %s" % fwjrRuns
testJobGroup = JobGroup(id = testJob["jobgroup"])
testJobGroup.loadData()
jobGroupFileset = testJobGroup.output
jobGroupFileset.loadData()
assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \
"Error: output file not in jobgroup fileset."
if testJob["mask"]["FirstEvent"] == None:
assert outputFile["first_event"] == 0, \
"Error: first event not set correctly: 0, %s" % \
outputFile["first_event"]
else:
assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \
"Error: last event not set correctly: %s, %s" % \
(testJob["mask"]["FirstEvent"], outputFile["first_event"])
return
def verifyFileMetaData(self, jobID, fwkJobReportFiles):
"""
_verifyFileMetaData_
Verify that all the files that were output by a job made it into WMBS
correctly. Compare the contents of WMBS to the files in the frameworks
job report.
Note that fwkJobReportFiles is a list of DataStructs File objects.
"""
testJob = Job(id = jobID)
testJob.loadData()
inputLFNs = []
for inputFile in testJob["input_files"]:
inputLFNs.append(inputFile["lfn"])
for fwkJobReportFile in fwkJobReportFiles:
outputFile = File(lfn = fwkJobReportFile["lfn"])
outputFile.loadData(parentage = 1)
assert outputFile["events"] == int(fwkJobReportFile["events"]), \
"Error: Output file has wrong events: %s, %s" % \
(outputFile["events"], fwkJobReportFile["events"])
assert outputFile["size"] == int(fwkJobReportFile["size"]), \
"Error: Output file has wrong size: %s, %s" % \
(outputFile["size"], fwkJobReportFile["size"])
for ckType in fwkJobReportFile["checksums"].keys():
assert ckType in outputFile["checksums"].keys(), \
"Error: Output file is missing checksums: %s" % ckType
assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \
"Error: Checksums don't match."
assert len(fwkJobReportFile["checksums"].keys()) == \
len(outputFile["checksums"].keys()), \
"Error: Wrong number of checksums."
jobType = self.getJobTypeAction.execute(jobID = jobID)
if jobType == "Merge":
assert str(outputFile["merged"]) == "True", \
"Error: Merge jobs should output merged files."
else:
assert outputFile["merged"] == fwkJobReportFile["merged"], \
"Error: Output file merged output is wrong: %s, %s" % \
(outputFile["merged"], fwkJobReportFile["merged"])
assert len(outputFile["locations"]) == 1, \
"Error: outputfile should have one location: %s" % outputFile["locations"]
assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \
"Error: wrong location for file."
assert len(outputFile["parents"]) == len(inputLFNs), \
"Error: Output file has wrong number of parents."
for outputParent in outputFile["parents"]:
assert outputParent["lfn"] in inputLFNs, \
"Error: Unknown parent file: %s" % outputParent["lfn"]
fwjrRuns = {}
for run in fwkJobReportFile["runs"]:
fwjrRuns[run.run] = run.lumis
for run in outputFile["runs"]:
assert run.run in fwjrRuns, \
"Error: Extra run in output: %s" % run.run
for lumi in run:
assert lumi in fwjrRuns[run.run], \
"Error: Extra lumi: %s" % lumi
fwjrRuns[run.run].remove(lumi)
if len(fwjrRuns[run.run]) == 0:
del fwjrRuns[run.run]
assert len(fwjrRuns.keys()) == 0, \
"Error: Missing runs, lumis: %s" % fwjrRuns
testJobGroup = JobGroup(id = testJob["jobgroup"])
testJobGroup.loadData()
jobGroupFileset = testJobGroup.output
jobGroupFileset.loadData()
assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \
"Error: output file not in jobgroup fileset."
if testJob["mask"]["FirstEvent"] == None:
assert outputFile["first_event"] == 0, \
"Error: first event not set correctly: 0, %s" % \
outputFile["first_event"]
else:
assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \
"Error: last event not set correctly: %s, %s" % \
(testJob["mask"]["FirstEvent"], outputFile["first_event"])
return
