def testLen(self):
"""
__testLen__
Test that the __len__ function will actualy return the correct length.
"""
#This is simple. It should just have a length equal to the number of committed
#And yet to be committed jobs
testJobA = Job()
testJobB = Job()
testJobC = Job()
testJobD = Job()
testJobGroup = JobGroup(jobs = [testJobA, testJobB])
testJobGroup.commit()
self.assertEqual(len(testJobGroup), 2)
testJobGroup.add(testJobC)
self.assertEqual(len(testJobGroup), 3)
testJobGroup.commit()
testJobGroup.add(testJobD)
self.assertEqual(len(testJobGroup), 4)
return
def setUp(self):
"""
_setUp_
Initial Setup for the Job Testcase
"""
self.inputFiles = []
for i in range(1, 1000):
lfn = "/store/data/%s/%s/file.root" % (random.randint(
1000, 9999), random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn=lfn,
size=size,
events=events,
checksums={"cksum": "1"})
file.addRun(Run(run, *[lumi]))
self.inputFiles.append(file)
self.dummyJob = Job(files=self.inputFiles)
return
def testA(self):
"""instantiate"""
document = Document()
document[u'pset_tweak_details'] = {}
document[u'pset_tweak_details'][u'process'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'][u'seed1'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'][u'seed2'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'][u'seed3'] = {}
document = self.database.commitOne(document)[0]
seeder = ReproducibleSeeding(CouchUrl=self.testInit.couchUrl,
CouchDBName=self.testInit.couchDbName,
ConfigCacheDoc=document[u'id'])
job = Job("testjob")
seeder(job)
baggage = job.getBaggage()
seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1",
None)
self.assertTrue(seed1 != None)
def main():
"""main functionf for testing"""
from WMCore.DataStructs.Job import Job
from WMCore.DataStructs.File import File
from WMCore.DataStructs.Run import Run
from WMCore.DataStructs.JobPackage import JobPackage
from WMCore.Services.UUID import makeUUID
from WMCore.WMSpec.Makers.TaskMaker import TaskMaker
factory = HarvestingWorkloadFactory()
workload = factory("derp", getTestArguments())
task = workload.getTask('Harvesting')
job = Job("SampleJob")
job["id"] = makeUUID()
job["task"] = task.getPathName()
job["workflow"] = workload.name()
file = File(lfn="/store/relval/CMSSW_3_8_2/RelValMinBias/GEN-SIM-RECO/MC_38Y_V9-v1/0019/FEC5BB4D-BFAF-DF11-A52A-001A92810AD2.root")
job.addFile(file)
jpackage = JobPackage()
jpackage[1] = job
import pickle
handle = open("%s/JobPackage.pkl" % os.getcwd(), 'w')
pickle.dump(jpackage, handle)
handle.close()
taskMaker = TaskMaker(workload, os.getcwd())
taskMaker.skipSubscription = True
taskMaker.processWorkload()
task.build(os.getcwd())
def testBaggage(self):
"""
_testBaggage_
Verify that job baggage is persisted with the package.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
baggage = newJob.getBaggage()
setattr(baggage, "thisJob", newJob["name"])
setattr(baggage, "seed1", 11111111)
setattr(baggage, "seed2", 22222222)
setattr(baggage, "seed3", 33333333)
setattr(baggage, "seed4", 44444444)
setattr(baggage, "seed5", 55555555)
package[i] = newJob
package.save(self.persistFile)
assert os.path.exists(self.persistFile), \
"Error: Package file was never created."
newPackage = JobPackage()
newPackage.load(self.persistFile)
# There is an extra key for the directory the package is stored in.
assert len(newPackage.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = newPackage[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
jobBaggage = job.getBaggage()
assert jobBaggage.thisJob == "Job%d" % i, \
"Error: Job baggage has wrong name."
assert jobBaggage.seed1 == 11111111, \
"Error: Job baggee has wrong value for seed1."
assert jobBaggage.seed2 == 22222222, \
"Error: Job baggee has wrong value for seed2."
assert jobBaggage.seed3 == 33333333, \
"Error: Job baggee has wrong value for seed3."
assert jobBaggage.seed4 == 44444444, \
"Error: Job baggee has wrong value for seed4."
assert jobBaggage.seed5 == 55555555, \
"Error: Job baggee has wrong value for seed5."
return
def testBaggage(self):
"""
_testBaggage_
Verify that job baggage is persisted with the package.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
baggage = newJob.getBaggage()
setattr(baggage, "thisJob", newJob["name"])
setattr(baggage, "seed1", 11111111)
setattr(baggage, "seed2", 22222222)
setattr(baggage, "seed3", 33333333)
setattr(baggage, "seed4", 44444444)
setattr(baggage, "seed5", 55555555)
package[i] = newJob
package.save(self.persistFile)
assert os.path.exists(self.persistFile), \
"Error: Package file was never created."
newPackage = JobPackage()
newPackage.load(self.persistFile)
# There is an extra key for the directory the package is stored in.
assert len(newPackage.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = newPackage[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
jobBaggage = job.getBaggage()
assert jobBaggage.thisJob == "Job%d" % i, \
"Error: Job baggage has wrong name."
assert jobBaggage.seed1 == 11111111, \
"Error: Job baggee has wrong value for seed1."
assert jobBaggage.seed2 == 22222222, \
"Error: Job baggee has wrong value for seed2."
assert jobBaggage.seed3 == 33333333, \
"Error: Job baggee has wrong value for seed3."
assert jobBaggage.seed4 == 44444444, \
"Error: Job baggee has wrong value for seed4."
assert jobBaggage.seed5 == 55555555, \
"Error: Job baggee has wrong value for seed5."
return
def testFirstEvent(self):
"""
_testFirstEvent_
Verify that we set the firstEvent parameter whenever the FirstEvent
field in the job mask is a positive integer. And the job is
not production.
"""
job = Job()
job["input_files"] = [{"lfn": "bogusFile", "parents": []}]
job["mask"] = Mask()
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertFalse(hasattr(tweak.process.source, "skipEvents"),
"Error: There should be no skipEvents tweak.")
self.assertFalse(hasattr(tweak.process.source, "firstEvent"),
"Error: There should be no firstEvent tweak.")
job["mask"]["FirstEvent"] = 0
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertTrue(hasattr(tweak.process.source, "skipEvents"),
"Error: There should be a skipEvents tweak.")
self.assertEqual(tweak.process.source.skipEvents,
'customTypeCms.untracked.uint32(0)',
"Error: The skipEvents tweak should be 0.")
return
def testFirstRun(self):
"""
_testFirstRun_
Verify that when we set the FirstRun in the mask, it is set in the
process but when it is not, then no firstRun appears in the process.
This for jobs with real input files.
"""
job = Job()
job["input_files"] = [{"lfn": "bogusFile", "parents": []}]
job["mask"] = Mask()
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertFalse(hasattr(tweak.process.source, "firstRun"),
"Error: There should be no firstRun tweak.")
job["mask"]["FirstRun"] = 93
tweak = WMTweaks.makeJobTweak(job, tweak)
self.assertTrue(hasattr(tweak.process.source, "firstRun"),
"Error: There should be a firstRun tweak.")
self.assertEqual(tweak.process.source.firstRun,
'customTypeCms.untracked.uint32(93)',
"Error: The firstRun tweak should be 93.")
return
def testFirstEventMC(self):
"""
_testFirstEventMC_
Verify that we set the firstEvent parameter whenever the FirstEvent
field in the job mask is a positive integer and the job is a production
one. Otherwise we get a number based on the counter (modulo 2^32 - 1)
"""
job = Job()
job["input_files"] = [{"lfn": "MCFakeFile", "parents": []}]
job["mask"] = Mask()
job["counter"] = 5
job["mask"]["FirstLumi"] = 200
try:
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertRaises(WMTweakMaskError, WMTweaks.makeJobTweak, job)
except WMTweakMaskError:
pass
job["mask"]["FirstEvent"] = 100
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertFalse(
hasattr(tweak.process.source, "skipEvents"),
"Error: There should be no skipEvents tweak, it's MC.")
self.assertTrue(hasattr(tweak.process.source, "firstEvent"),
"Error: There should be a first event tweak")
self.assertEqual(tweak.process.source.firstEvent,
'customTypeCms.untracked.uint32(100)',
"Error: The firstEvent tweak should be 100.")
return
def testMask3(self):
mymask = Mask()
mymask['FirstEvent'] = 9999
mymask['LastEvent'] = 999
myjob = Job()
myjob["mask"] = mymask
self.roundTrip(myjob)
def testAddCommit(self):
"""
_testAddCommit_
Test the add() and commit() methods of the JobGroup class. Verify that
jobs are not returned from getJobs() until commit() has been called.
"""
testJob = Job()
testJobGroup = JobGroup()
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs before jobs have been added."
testJobGroup.add(testJob)
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs commit() was called."
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 1, \
"ERROR: JobGroup has wrong number of jobs."
assert testJob in testJobGroup.getJobs(), \
"ERROR: JobGroup has unknown jobs."
return
def testFirstRunMC(self):
"""
_testFirstRunMC_
Verify that we set the lumi in a MC job and it gets into
process.source.firstRun parameter.
"""
job = Job()
job["input_files"] = [{"lfn": "MCFakeFile", "parents": []}]
job["mask"] = Mask()
job["mask"]["FirstLumi"] = 200
job["mask"]["FirstEvent"] = 100
job["counter"] = 5
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertTrue(hasattr(tweak.process.source, "firstRun"),
"Error: There should be a first run tweak")
self.assertEqual(tweak.process.source.firstRun,
'customTypeCms.untracked.uint32(1)',
"Error: The first run should be 1")
job["mask"]["FirstRun"] = 5
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertTrue(hasattr(tweak.process.source, "firstRun"),
"Error: There should be a first run tweak")
self.assertEqual(tweak.process.source.firstRun,
'customTypeCms.untracked.uint32(5)',
"Error: The first run should be 5")
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
stepHelper = step = self.task.makeStep("DeleteTest")
self.step = stepHelper.data
self.actualStep = stepHelper
template = DeleteTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.insert(0, self.testDir)
sys.path.insert(0, self.sandboxDir)
self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def testAddingJobs(self):
"""
_testAddingJobs_
Verify that adding jobs to the package works as expected.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
package[i] = newJob
# There is an extra key for the directory the package is stored in.
assert len(package.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = package[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
return
def testPersist(self):
"""
_testPersist_
Verify that we're able to save and load the job package.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
package[i] = newJob
package.save(self.persistFile)
assert os.path.exists(self.persistFile), \
"Error: Package file was never created."
newPackage = JobPackage()
newPackage.load(self.persistFile)
# There is an extra key for the directory the package is stored in.
assert len(newPackage.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = newPackage[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
return
def testFirstLumiMC(self):
"""
_testFirstLumiMC_
Verify that we set the lumi in a MC job and it gets into
process.source.firstRun parameter, and if we don't at least we
get the counter there.
"""
job = Job()
job["input_files"] = [{"lfn": "MCFakeFile", "parents": []}]
job["mask"] = Mask()
job["counter"] = 5
job["mask"]["FirstEvent"] = 100
try:
tweak = WMTweaks.makeJobTweak(job)
self.assertRaises(WMTweakMaskError, WMTweaks.makeJobTweak, job)
except WMTweakMaskError:
pass
job["mask"]["FirstLumi"] = 200
tweak = WMTweaks.makeJobTweak(job)
self.assertTrue(hasattr(tweak.process.source, "firstLuminosityBlock"),
"Error: There should be a first lumi tweak")
self.assertEqual(tweak.process.source.firstLuminosityBlock, 200,
"Error: The first luminosity block should be 5")
job["mask"]["FirstLumi"] = 10
tweak = WMTweaks.makeJobTweak(job)
self.assertTrue(hasattr(tweak.process.source, "firstLuminosityBlock"),
"Error: There should be a first lumi tweak")
self.assertEqual(tweak.process.source.firstLuminosityBlock, 10,
"Error: The first luminosity block should be 10")
def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def testCreate(self):
"""
_testCreate_
Test the JobGroup constructor and passing different job containers
into it.
"""
testSubscription = Subscription()
testJobGroupA = JobGroup(subscription = testSubscription)
assert testJobGroupA.subscription == testSubscription, \
"ERROR: Failed to pass subscription in constructor"
assert len(testJobGroupA.jobs) == 0 and len(testJobGroupA.newjobs) == 0, \
"ERROR: JobGroup not empty on creation"
testJobA = Job()
testJobB = Job()
testJobGroupB = JobGroup(jobs = [testJobA, testJobB])
assert testJobGroupB.jobs == [], \
"ERROR: Jobs committed to jobgroup too soon."
jobGroupJobs = testJobGroupB.newjobs
goldenJobs = [testJobA, testJobB]
for job in jobGroupJobs:
assert job in goldenJobs, \
"ERROR: Extra job in job group"
goldenJobs.remove(job)
assert len(goldenJobs) == 0, \
"ERROR: Job missing from job group"
testJobGroupC = JobGroup(jobs = testJobA)
assert testJobGroupC.jobs == [], \
"ERROR: Jobs committed to jobgroup too soon."
jobGroupJobs = testJobGroupC.newjobs
assert len(jobGroupJobs) == 1, \
"ERROR: Wrong number of jobs in jobgroup."
assert testJobA in jobGroupJobs, \
"ERROR: Wrong job in jobgroup."
return
def createTestJob(self):
"""
Create a test job to pass to the DashboardInterface
"""
job = Job(name = "ThisIsASillyName")
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
testFileA.addRun(Run(1, *[45]))
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(1, *[46]))
job.addFile(testFileA)
job.addFile(testFileB)
job['id'] = 1
return job
def associateWorkUnits(self):
"""
_associateWorkUnits_
Add the WorkUnits that this job requires
Returns: N/A
"""
existsAction = self.daofactory(
classname='WorkUnit.ExistsByTaskFileLumi')
addAction = self.daofactory(classname='WorkUnit.Add')
assocAction = self.daofactory(classname='Jobs.AddWorkUnits')
files = WMJob.getFiles(self)
jobMask = self['mask']
workflow = self.getWorkflow()
wfid = workflow['taskid']
lumisInJob = 0
for wmfile in files:
fileMask = jobMask.filterRunLumisByMask(runs=wmfile['runs'])
for runObj in fileMask:
lumisInJob += len(runObj.lumis)
for wmfile in files:
fileid = wmfile['id']
fileMask = jobMask.filterRunLumisByMask(runs=wmfile['runs'])
for runObj in fileMask:
run = runObj.run
lumis = runObj.lumis
for lumi in lumis:
if not existsAction.execute(
taskid=wfid,
fileid=fileid,
run_lumi=runObj,
conn=self.getDBConn(),
transaction=self.existingTransaction()):
addAction.execute(
taskid=wfid,
last_unit_count=lumisInJob,
fileid=fileid,
run=run,
lumi=lumi,
conn=self.getDBConn(),
transaction=self.existingTransaction())
assocAction.execute(jobid=self["id"],
fileid=fileid,
run=run,
lumi=lumi,
conn=self.getDBConn(),
transaction=self.existingTransaction())
def __init__(self, name = None, files = None, id = None):
"""
___init___
jobgroup object is used to determine the workflow.
inputFiles is a list of files that the job will process.
"""
WMBSBase.__init__(self)
WMJob.__init__(self, name = name, files = files)
self["id"] = id
self["jobgroup"] = None
self["couch_record"] = None
self["attachments"] = {}
self["cache_dir"] = None
self["sandbox"] = None
self['fwjr'] = None
self["mask"] = Mask()
self['custom'] = {} # For local add-ons that we want to send to JSON
return
def __init__(self, name=None, files=None, id=None):
"""
___init___
jobgroup object is used to determine the workflow.
inputFiles is a list of files that the job will process.
"""
WMBSBase.__init__(self)
WMJob.__init__(self, name=name, files=files)
self["id"] = id
self["jobgroup"] = None
self["couch_record"] = None
self["attachments"] = {}
self["cache_dir"] = None
self["sandbox"] = None
self['fwjr'] = None
self["mask"] = Mask()
self['custom'] = {} # For local add-ons that we want to send to JSON
return
def getFiles(self, type="list"):
"""
_getFiles_
Retrieve a list of files that are associated with the job.
"""
if self["id"] == None:
return WMJob.getFiles(self, type)
existingTransaction = self.beginTransaction()
idAction = self.daofactory(classname="Jobs.LoadFiles")
fileIDs = idAction.execute(self["id"], conn=self.getDBConn(), transaction=self.existingTransaction())
currentFileIDs = WMJob.getFiles(self, type="id")
for fileID in fileIDs:
if fileID["id"] not in currentFileIDs:
self.loadData()
break
self.commitTransaction(existingTransaction)
return WMJob.getFiles(self, type)
def createTestJob(self, fwjr):
"""
_createTestJob_
Creates a minimal job to report
"""
job = Job('finishedJob')
job['retry_count'] = 1
job['workflow'] = 'testWorkflow'
job['fwjr'] = fwjr
return job
def testA(self):
"""instantiate"""
document = Document()
document[u'pset_tweak_details'] = {}
document[u'pset_tweak_details'][u'process'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed1'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed2'] = {}
document[u'pset_tweak_details'][u'process'][u'RandomNumberGeneratorService'][u'seed3'] = {}
document = self.database.commitOne(document)[0]
seeder = ReproducibleSeeding(CouchUrl = self.testInit.couchUrl,
CouchDBName = self.testInit.couchDbName,
ConfigCacheDoc = document[u'id'])
job = Job("testjob")
seeder(job)
baggage = job.getBaggage()
seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1", None)
self.failUnless(seed1 != None)
def getFiles(self, type = "list"):
"""
_getFiles_
Retrieve a list of files that are associated with the job.
"""
if self["id"] == None:
return WMJob.getFiles(self, type)
existingTransaction = self.beginTransaction()
idAction = self.daofactory(classname = "Jobs.LoadFiles")
fileIDs = idAction.execute(self["id"], conn = self.getDBConn(),
transaction = self.existingTransaction())
currentFileIDs = WMJob.getFiles(self, type = "id")
for fileID in fileIDs:
if fileID["id"] not in currentFileIDs:
self.loadData()
break
self.commitTransaction(existingTransaction)
return WMJob.getFiles(self, type)
def testC(self):
"""test building a tweak from the seeds"""
job = Job("TestJob")
seeder = AutomaticSeeding()
job.addBaggageParameter(
"process.RandomNumberGeneratorService.seed1.initialSeed", 123445)
job.addBaggageParameter(
"process.RandomNumberGeneratorService.seed2.initialSeed", 123445)
job.addBaggageParameter(
"process.RandomNumberGeneratorService.seed3.initialSeed", 7464738)
job.addBaggageParameter(
"process.RandomNumberGeneratorService.seed44.initialSeed",
98273762)
seeder(job)
tweak = PSetTweak()
for x in job.baggage.process.RandomNumberGeneratorService:
parameter = "process.RandomNumberGeneratorService.%s.initialSeed" % x._internal_name
tweak.addParameter(parameter, x.initialSeed)
print tweak
def testGetJobs(self):
"""
_testGetJobs_
Verify that the getJobs() method of the JobGroup class returns the
correct output for each output container type it supports.
"""
testJobA = Job()
testJobB = Job()
testJobGroup = JobGroup(jobs = [testJobA, testJobB])
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 2, \
"ERROR: Wrong number of jobs in job group"
goldenJobs = [testJobA, testJobB]
for job in testJobGroup.getJobs():
assert job in goldenJobs, \
"ERROR: Unknown Job in JobGroup."
goldenJobs.remove(job)
assert len(goldenJobs) == 0, \
"ERROR: Jobs are missing from the JobGroup."
goldenIDs = []
goldenIDs.append(testJobA["id"])
goldenIDs.append(testJobB["id"])
for jobID in testJobGroup.getJobs(type = "id"):
assert jobID in goldenIDs, \
"ERROR: Unknown JobID in JobGroup"
goldenIDs.remove(jobID)
assert len(goldenIDs) == 0, \
"ERROR: Job IDs are missing from the JobGroup."
return
def associateFiles(self):
"""
_associateFiles_
Update the wmbs_job_assoc table with the files in the inputFiles for the
job.
"""
files = WMJob.getFiles(self, type="id")
if len(files) > 0:
addAction = self.daofactory(classname="Jobs.AddFiles")
addAction.execute(self["id"], files, conn=self.getDBConn(), transaction=self.existingTransaction())
return
def getDataStructsJob(self):
"""
_getDataStructsJob_
Returns the DataStructs version of this job
"""
job = WMJob(name=self['name'])
# Transfer all simple keys
for key in self:
if isinstance(self.get(key), (str, bytes, int, float)):
job[key] = self[key]
for fileObj in self['input_files']:
job['input_files'].append(fileObj.returnDataStructsFile())
job['mask'] = WMMask()
for key in self["mask"]:
job["mask"][key] = self["mask"][key]
job.baggage = self.baggage
return job
def getDataStructsJob(self):
"""
_getDataStructsJob_
Returns the DataStructs version of this job
"""
job = WMJob(name = self['name'])
# Transfer all simple keys
for key in self.keys():
keyType = type(self.get(key))
if keyType in [str, long, int, float]:
job[key] = self[key]
for file in self['input_files']:
job['input_files'].append(file.returnDataStructsFile())
job['mask'] = WMMask()
for key in self["mask"].keys():
job["mask"][key] = self["mask"][key]
job.baggage = self.baggage
return job
def associateFiles(self):
"""
_associateFiles_
Update the wmbs_job_assoc table with the files in the inputFiles for the
job.
"""
files = WMJob.getFiles(self, type = "id")
if len(files) > 0:
addAction = self.daofactory(classname = "Jobs.AddFiles")
addAction.execute(self["id"], files, conn = self.getDBConn(),
transaction = self.existingTransaction())
return
def getDataStructsJob(self):
"""
_getDataStructsJob_
Returns the DataStructs version of this job
"""
job = WMJob(name=self['name'])
# Transfer all simple keys
for key in self.keys():
keyType = type(self.get(key))
if keyType in [str, long, int, float]:
job[key] = self[key]
for file in self['input_files']:
job['input_files'].append(file.returnDataStructsFile())
job['mask'] = WMMask()
for key in self["mask"].keys():
job["mask"][key] = self["mask"][key]
job.baggage = self.baggage
return job
def createTestJob(self):
"""
_createTestJob_
Create a test job that has parents for each input file.
"""
newJob = Job(name = "TestJob")
newJob.addFile(File(lfn = "/some/file/one",
parents = set([File(lfn = "/some/parent/one")])))
newJob.addFile(File(lfn = "/some/file/two",
parents = set([File(lfn = "/some/parent/two")])))
return newJob
def createJobs(totalJobs=100):
"""
_createJobs_
"""
global lastJobID
newJobs = []
for i in range(totalJobs):
newJob = Job(name=makeUUID(), files=[createFile(), createFile()])
newJob["task"] = "SomeTask"
newJob["workflow"] = "SomeWorkflow"
newJob["owner"] = "*****@*****.**"
newJob["id"] = lastJobID
newJob["couch_record"] = None
newJobs.append(newJob)
lastJobID += 1
return newJobs
def testC(self):
"""test building a tweak from the seeds"""
job = Job("TestJob")
seeder = AutomaticSeeding()
job.addBaggageParameter("process.RandomNumberGeneratorService.seed1.initialSeed", 123445)
job.addBaggageParameter("process.RandomNumberGeneratorService.seed2.initialSeed", 123445)
job.addBaggageParameter("process.RandomNumberGeneratorService.seed3.initialSeed", 7464738)
job.addBaggageParameter("process.RandomNumberGeneratorService.seed44.initialSeed", 98273762)
seeder(job)
tweak = PSetTweak()
for x in job.baggage.process.RandomNumberGeneratorService:
parameter = "process.RandomNumberGeneratorService.%s.initialSeed" % x._internal_name
tweak.addParameter(parameter, x.initialSeed)
print(tweak)
def associateWorkUnits(self):
"""
_associateWorkUnits_
Add the WorkUnits that this job requires
Returns: N/A
"""
existsAction = self.daofactory(classname='WorkUnit.ExistsByTaskFileLumi')
addAction = self.daofactory(classname='WorkUnit.Add')
assocAction = self.daofactory(classname='Jobs.AddWorkUnits')
files = WMJob.getFiles(self)
jobMask = self['mask']
workflow = self.getWorkflow()
wfid = workflow['taskid']
lumisInJob = 0
for wmfile in files:
fileMask = jobMask.filterRunLumisByMask(runs=wmfile['runs'])
for runObj in fileMask:
lumisInJob += len(runObj.lumis)
for wmfile in files:
fileid = wmfile['id']
fileMask = jobMask.filterRunLumisByMask(runs=wmfile['runs'])
for runObj in fileMask:
run = runObj.run
lumis = runObj.lumis
for lumi in lumis:
if not existsAction.execute(taskid=wfid, fileid=fileid, run_lumi=runObj,
conn=self.getDBConn(), transaction=self.existingTransaction()):
addAction.execute(taskid=wfid, last_unit_count=lumisInJob, fileid=fileid, run=run,
lumi=lumi,
conn=self.getDBConn(), transaction=self.existingTransaction())
assocAction.execute(jobid=self["id"], fileid=fileid, run=run, lumi=lumi,
conn=self.getDBConn(), transaction=self.existingTransaction())
def setUp(self):
"""
build a step for testing purposes
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = step = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def setUp(self):
"""
_setUp_
Initial Setup for the Job Testcase
"""
self.inputFiles = []
for i in range(1,1000):
lfn = "/store/data/%s/%s/file.root" % (random.randint(1000, 9999),
random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"})
file.addRun(Run(run, *[lumi]))
self.inputFiles.append(file)
self.dummyJob = Job(files = self.inputFiles)
return
class JobTest(unittest.TestCase):
"""
_JobTest_
Testcase for the Job class
Instantiate a dummy Job object with a dummy Subscription
and a dummy Fileset full of random files as input
"""
def setUp(self):
"""
_setUp_
Initial Setup for the Job Testcase
"""
self.inputFiles = []
for i in range(1, 1000):
lfn = "/store/data/%s/%s/file.root" % (random.randint(
1000, 9999), random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn=lfn,
size=size,
events=events,
checksums={"cksum": "1"})
file.addRun(Run(run, *[lumi]))
self.inputFiles.append(file)
self.dummyJob = Job(files=self.inputFiles)
return
def tearDown(self):
"""
No tearDown method for this Testcase
"""
pass
def testGetFilesList(self):
"""
_testGetFilesList_
Verify that the Job::getFiles(type = "list") method returns the same
files in the same order that they were passed in.
"""
assert self.dummyJob.getFiles() == self.inputFiles, \
"ERROR: Initial fileset does not match Job fileset"
return
def testGetFilesSet(self):
"""
_testGetFilesSet_
Verify that the Job::getFiles(type = "set") method returns the correct
input files in the form of a set.
"""
assert self.dummyJob.getFiles(type = "set") == set(self.inputFiles), \
"ERROR: getFiles(type = 'set') does not work correctly."
return
def testGetFilesLFN(self):
"""
_testGetFilesLFN_
Verify that the Job::getFiles(type = "lfn") method returns the same
files in the same order that they were passed in.
"""
jobLFNs = self.dummyJob.getFiles(type="lfn")
goldenLFNs = []
for file in self.inputFiles:
goldenLFNs.append(file["lfn"])
assert len(goldenLFNs) == len(jobLFNs), \
"ERROR: Job has different number of files than input"
for jobLFN in jobLFNs:
assert jobLFN in goldenLFNs, \
"ERROR: LFN missing from job."
return
def testAddFile(self):
"""
_testAddFile_
Verify that the Job::addFile() method works properly.
"""
dummyFileAddFile = File("/tmp/dummyFileAddFileTest", 1234, 1, 2)
self.dummyJob.addFile(dummyFileAddFile)
assert dummyFileAddFile in self.dummyJob.getFiles(), \
"ERROR: Couldn't add file to Job - addFile method error"
return
def testChangeState(self):
"""
_testChangeState_
Verify that the Job::changeState() method updates the state and the
state time.
"""
currentTime = time.time()
self.dummyJob.changeState("created")
assert self.dummyJob["state_time"] > currentTime - 1 and \
self.dummyJob["state_time"] < currentTime + 1, \
"ERROR: State time not updated on state change"
assert self.dummyJob["state"] == "created", \
"ERROR: Couldn't change Job state - changeState method error"
def testChangeOutcome(self):
"""
_testChangeOutcome_
Verify that the Job::changeOutcome() method changes the final outcome
of the job.
"""
self.dummyJob.changeOutcome("success")
assert self.dummyJob["outcome"] == "success", \
"ERROR: Job outcome failed to update."
return
def testGetBaggage(self):
"""
test that setting/accessing the Job Baggage ConfigSection works
"""
setattr(self.dummyJob.baggage, "baggageContents", {"key": "value"})
try:
baggage = self.dummyJob.getBaggage()
except Exception, ex:
msg = "Error calling Job.getBaggage()\n"
msg += str(ex)
self.fail(msg)
self.failUnless(hasattr(baggage, "baggageContents"))
class JobTest(unittest.TestCase):
"""
_JobTest_
Testcase for the Job class
Instantiate a dummy Job object with a dummy Subscription
and a dummy Fileset full of random files as input
"""
def setUp(self):
"""
_setUp_
Initial Setup for the Job Testcase
"""
self.inputFiles = []
for i in range(1,1000):
lfn = "/store/data/%s/%s/file.root" % (random.randint(1000, 9999),
random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn = lfn, size = size, events = events, checksums = {"cksum": "1"})
file.addRun(Run(run, *[lumi]))
self.inputFiles.append(file)
self.dummyJob = Job(files = self.inputFiles)
return
def tearDown(self):
"""
No tearDown method for this Testcase
"""
pass
def testGetFilesList(self):
"""
_testGetFilesList_
Verify that the Job::getFiles(type = "list") method returns the same
files in the same order that they were passed in.
"""
assert self.dummyJob.getFiles() == self.inputFiles, \
"ERROR: Initial fileset does not match Job fileset"
return
def testGetFilesSet(self):
"""
_testGetFilesSet_
Verify that the Job::getFiles(type = "set") method returns the correct
input files in the form of a set.
"""
assert self.dummyJob.getFiles(type = "set") == set(self.inputFiles), \
"ERROR: getFiles(type = 'set') does not work correctly."
return
def testGetFilesLFN(self):
"""
_testGetFilesLFN_
Verify that the Job::getFiles(type = "lfn") method returns the same
files in the same order that they were passed in.
"""
jobLFNs = self.dummyJob.getFiles(type = "lfn")
goldenLFNs = []
for file in self.inputFiles:
goldenLFNs.append(file["lfn"])
assert len(goldenLFNs) == len(jobLFNs), \
"ERROR: Job has different number of files than input"
for jobLFN in jobLFNs:
assert jobLFN in goldenLFNs, \
"ERROR: LFN missing from job."
return
def testAddFile(self):
"""
_testAddFile_
Verify that the Job::addFile() method works properly.
"""
dummyFileAddFile = File("/tmp/dummyFileAddFileTest", 1234, 1, 2)
self.dummyJob.addFile(dummyFileAddFile)
assert dummyFileAddFile in self.dummyJob.getFiles(), \
"ERROR: Couldn't add file to Job - addFile method error"
return
def testChangeState(self):
"""
_testChangeState_
Verify that the Job::changeState() method updates the state and the
state time.
"""
currentTime = time.time()
self.dummyJob.changeState("created")
assert self.dummyJob["state_time"] > currentTime - 1 and \
self.dummyJob["state_time"] < currentTime + 1, \
"ERROR: State time not updated on state change"
assert self.dummyJob["state"] == "created", \
"ERROR: Couldn't change Job state - changeState method error"
def testChangeOutcome(self):
"""
_testChangeOutcome_
Verify that the Job::changeOutcome() method changes the final outcome
of the job.
"""
self.dummyJob.changeOutcome("success")
assert self.dummyJob["outcome"] == "success", \
"ERROR: Job outcome failed to update."
return
def testGetBaggage(self):
"""
test that setting/accessing the Job Baggage ConfigSection works
"""
self.dummyJob.addBaggageParameter("baggageContents", True)
self.dummyJob.addBaggageParameter("trustPUSitelists", False)
self.dummyJob.addBaggageParameter("skipPileupEvents", 20000)
baggage = self.dummyJob.getBaggage()
self.assertTrue(getattr(baggage, "baggageContents"))
self.assertFalse(getattr(baggage, "trustPUSitelists"))
self.assertEqual(getattr(baggage, "skipPileupEvents"), 20000)
self.assertFalse(hasattr(baggage, "IDontExist"))
