def testInstantiation(self):
"""
_testInstantiation_
Verify that the WMTask and the WMTaskHelper classes can be
instantiated.
"""
WMTask("task1")
makeWMTask("task2")
return
def testTrustSitelists(self):
"""
_testTrustSitelists_
Verify that we can set/get the proper TrustSitelists and TrustPUSitelists
flag.
Also make sure they are retrievable through job splitting parameters.
"""
testTask = makeWMTask("TestTask")
testTask.setJobResourceInformation(timePerEvent=50, memoryReq=4000,
sizePerEvent=10)
splitArgs = testTask.jobSplittingParameters(performance=False)
self.assertFalse(splitArgs['trustSitelists'])
self.assertFalse(splitArgs['trustPUSitelists'])
testTask.setTrustSitelists(True, True)
trustlists = testTask.getTrustSitelists()
self.assertTrue(trustlists['trustlists'])
self.assertTrue(trustlists['trustPUlists'])
splitArgs = testTask.jobSplittingParameters(performance=False)
self.assertTrue(splitArgs['trustSitelists'])
self.assertTrue(splitArgs['trustPUSitelists'])
testTask.setTrustSitelists(False, False)
trustlists = testTask.getTrustSitelists()
self.assertFalse(trustlists['trustlists'])
self.assertFalse(trustlists['trustPUlists'])
return
def testSetGetPriority(self):
"""
_testSetGetPriority_
Test and see whether we can set and get the
priorities in a task.
"""
testTask = makeWMTask("TestTask")
testTask.setTaskType("Processing")
# Should be null to start
self.assertEqual(testTask.getTaskPriority(), None)
# Should not accept bad values
testTask.setTaskPriority(priority = 'fail')
self.assertEqual(testTask.getTaskPriority(), None)
# Should cast strings
testTask.setTaskPriority(priority = '1')
self.assertEqual(testTask.getTaskPriority(), 1)
# Should handle ints
testTask.setTaskPriority(priority = 2)
self.assertEqual(testTask.getTaskPriority(), 2)
return
def testJobSplittingArgs(self):
"""
_testJobSplittingArgs_
Verify that setting job splitting arguments for a task works correctly.
When pulling the job splitting arguments out of a task the site white
list and black list should also be included as they need to be passed to
the job splitting code.
"""
testTask = makeWMTask("TestTask")
testTask.setTaskType("Processing")
assert testTask.taskType() == "Processing", "Error: Wrong task type."
testTask.setSplittingAlgorithm("MadeUpAlgo", events_per_job=100, max_job_size=24, one_more_param="Hello")
testTask.setSiteWhitelist(["T1_US_FNAL", "T1_CH_CERN"])
testTask.setSiteBlacklist(["T2_US_PERDUE", "T2_US_UCSD", "T1_TW_ASGC"])
assert testTask.jobSplittingAlgorithm() == "MadeUpAlgo", "Error: Wrong job splitting algorithm name."
algoParams = testTask.jobSplittingParameters()
assert len(algoParams.keys()) == 6, "Error: Wrong number of algo parameters."
assert "algorithm" in algoParams.keys(), "Error: Missing algo parameter."
assert algoParams["algorithm"] == "MadeUpAlgo", "Error: Parameter has wrong value."
assert "events_per_job" in algoParams.keys(), "Error: Missing algo parameter."
assert algoParams["events_per_job"] == 100, "Error: Parameter has wrong value."
assert "max_job_size" in algoParams.keys(), "Error: Missing algo parameter."
assert algoParams["max_job_size"] == 24, "Error: Parameter has wrong value."
assert "one_more_param" in algoParams.keys(), "Error: Missing algo parameter."
assert algoParams["one_more_param"] == "Hello", "Error: Parameter has wrong value."
return
def testPresetSeeder(self):
"""
_testPresetSeeder_
Test whether the PresetSeeder works
"""
task1 = makeWMTask("task2")
seederDict = {"generator.initialSeed": 1001, "evtgenproducer.initialSeed": 1001}
task1.addGenerator("PresetSeeder", **seederDict)
manager = GeneratorManager(task = task1)
jobs = self.oneHundredFiles()
for jobGrp in jobs:
manager(jobGrp)
for jobGrp in jobs:
count = 0
for job in jobGrp.jobs:
conf = job.getBaggage()
self.assertEqual(conf.PresetSeeder.evtgenproducer.initialSeed, 1001)
self.assertEqual(conf.PresetSeeder.generator.initialSeed, 1001)
return
def testSiteWhiteBlacklists(self):
"""
_testSiteWhiteBlacklists_
Verify that methods for setting and retrieving site white and black
lists work correctly.
"""
testTask = makeWMTask("TestTask")
testTask.setSiteWhitelist(["T1_US_FNAL", "T1_DE_KIT"])
testTask.setSiteBlacklist(
["T1_IT_IN2P3", "T1_CH_CERN", "T2_US_NEBRASKA"])
taskWhitelist = testTask.siteWhitelist()
assert len(taskWhitelist) == 2, \
"Error: Wrong number of sites in white list."
assert "T1_US_FNAL" in taskWhitelist, \
"Error: Site missing from white list."
assert "T1_DE_KIT" in taskWhitelist, \
"Error: Site missing from white list."
taskBlacklist = testTask.siteBlacklist()
assert len(taskBlacklist) == 3, \
"Error: Wrong number if sites in black list."
assert "T1_IT_IN2P3" in taskBlacklist, \
"Error: Site missing from black list."
assert "T1_CH_CERN" in taskBlacklist, \
"Error: Site missing from black list."
assert "T2_US_NEBRASKA" in taskBlacklist, \
"Error: Site missing from black list."
return
def testSubscriptionInformation(self):
"""
_testSubscriptionInformation_
Check the three methods related to the subscription information in a task
Make sure that we can set the subscription information for all datasets produced by this task
and we can select only some primaryDatasets/DataTiers, and check that we can update the dataset
in a subscription information section.
"""
testTask = makeWMTask("TestTask")
cmsswStep = testTask.makeStep("cmsRun1")
cmsswStep.setStepType("CMSSW")
testTask.applyTemplates()
cmsswHelper = cmsswStep.getTypeHelper()
cmsswHelper.addOutputModule(
"outputRECO", primaryDataset="OneParticle", processedDataset="DawnOfAnEra-v1", dataTier="RECO"
)
cmsswHelper.addOutputModule(
"outputDQM", primaryDataset="TwoParticles", processedDataset="DawnOfAnEra-v1", dataTier="DQM"
)
cmsswHelper.addOutputModule(
"outputAOD", primaryDataset="OneParticle", processedDataset="DawnOfAnEra-v1", dataTier="AOD"
)
self.assertEqual(testTask.getSubscriptionInformation(), {}, "There should not be any subscription info")
testTask.setSubscriptionInformation(["mercury"], ["mars", "earth"], ["earth"], "High", "Replica", "OneParticle")
subInfo = testTask.getSubscriptionInformation()
outputRecoSubInfo = {
"CustodialSites": ["mercury"],
"NonCustodialSites": ["mars", "earth"],
"AutoApproveSites": ["earth"],
"Priority": "High",
"CustodialSubType": "Replica",
}
self.assertEqual(
subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong"
)
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertFalse("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
testTask.setSubscriptionInformation(["jupiter"], primaryDataset="TwoParticles")
subInfo = testTask.getSubscriptionInformation()
self.assertEqual(
subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong"
)
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
recoutOutputModule = cmsswHelper.getOutputModule("outputRECO")
setattr(recoutOutputModule, "primaryDataset", "ThreeParticles")
testTask.updateSubscriptionDataset("outputRECO", recoutOutputModule)
subInfo = testTask.getSubscriptionInformation()
self.assertEqual(
subInfo["/ThreeParticles/DawnOfAnEra-v1/RECO"],
outputRecoSubInfo,
"The RECO subscription information is wrong",
)
self.assertFalse("/OneParticle/DawnOfAnEra-v1/RECO" in subInfo, "The RECO subscription information is wrong")
def testGetSwVersionAndScramArch(self):
"""
_testGetSwVersionAndScramArch_
Test whether we can fetch the CMSSW release and ScramArch
being used in a task
"""
testTask = makeWMTask("MultiTask")
taskCmssw = testTask.makeStep("cmsRun1")
taskCmssw.setStepType("CMSSW")
taskCmsswStageOut = taskCmssw.addStep("stageOut1")
taskCmsswStageOut.setStepType("StageOut")
taskCmsswLogArch = taskCmsswStageOut.addStep("logArch1")
taskCmsswLogArch.setStepType("LogArchive")
testTask.applyTemplates()
taskCmsswHelper = taskCmssw.getTypeHelper()
taskCmsswHelper.cmsswSetup("CMSSW_1_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc123")
self.assertEqual(testTask.getSwVersion(), "CMSSW_1_2_3")
self.assertEqual(testTask.getSwVersion(allSteps=True), ["CMSSW_1_2_3"])
self.assertEqual(testTask.getScramArch(), "slc7_amd64_gcc123")
self.assertEqual(testTask.getScramArch(allSteps=True), ["slc7_amd64_gcc123"])
return
def testJobSplittingArgs(self):
"""
_testJobSplittingArgs_
Verify that setting job splitting arguments for a task works correctly.
When pulling the job splitting arguments out of a task the site white
list and black list should also be included as they need to be passed to
the job splitting code.
"""
testTask = makeWMTask("TestTask")
testTask.setTaskType("Processing")
assert testTask.taskType() == "Processing", \
"Error: Wrong task type."
testTask.setSplittingAlgorithm("MadeUpAlgo", events_per_job = 100,
max_job_size = 24,
one_more_param = "Hello")
testTask.setSiteWhitelist(["T1_US_FNAL", "T1_CH_CERN"])
testTask.setSiteBlacklist(["T2_US_PERDUE", "T2_US_UCSD", "T1_TW_ASGC"])
testTask.addInputDataset(primary = "PrimaryDataset",
processed = "ProcessedDataset",
tier = "DataTier",
dbsurl = "DBSURL",
block_whitelist = ["Block1", "Block2"],
block_blacklist = ["Block3", "Block4", "Block5"],
run_whitelist = [1, 2, 3],
run_blacklist = [4, 5])
assert testTask.jobSplittingAlgorithm() == "MadeUpAlgo", \
"Error: Wrong job splitting algorithm name."
algoParams = testTask.jobSplittingParameters()
assert len(algoParams.keys()) == 8, \
"Error: Wrong number of algo parameters."
assert "algorithm" in algoParams.keys(), \
"Error: Missing algo parameter."
assert algoParams["algorithm"] == "MadeUpAlgo", \
"Error: Parameter has wrong value."
assert "events_per_job" in algoParams.keys(), \
"Error: Missing algo parameter."
assert algoParams["events_per_job"] == 100, \
"Error: Parameter has wrong value."
assert "max_job_size" in algoParams.keys(), \
"Error: Missing algo parameter."
assert algoParams["max_job_size"] == 24, \
"Error: Parameter has wrong value."
assert "one_more_param" in algoParams.keys(), \
"Error: Missing algo parameter."
assert algoParams["one_more_param"] == "Hello", \
"Error: Parameter has wrong value."
assert len(algoParams["runWhitelist"]) == 3, \
"Error: Wrong number of runs in whitelist."
return
def testAddingSteps(self):
"""
_testAddingSteps_
Verify that adding steps to a task works correctly.
"""
task1 = makeWMTask("task1")
task2a = task1.addTask("task2a")
task2b = task1.addTask("task2b")
task2c = task1.addTask("task2c")
task3 = task2a.addTask("task3")
step1 = makeWMStep("step1")
step1.addStep("step1a")
step1.addStep("step1b")
step1.addStep("step1c")
step2 = makeWMStep("step2")
step2.addStep("step2a")
step2.addStep("step2b")
step2.addStep("step2c")
step3 = makeWMStep("step3")
step3.addStep("step3a")
step3.addStep("step3b")
step3.addStep("step3c")
step4 = makeWMStep("step4")
step4.addStep("step4a")
step4.addStep("step4b")
step4.addStep("step4c")
task1.setStep(step1)
task2a.setStep(step2)
task2b.setStep(step3)
task2c.setStep(step4)
self.assertEqual(task1.getStep("step1a").name(), "step1a")
self.assertEqual(task2a.getStep("step2b").name(), "step2b")
self.assertEqual(task2b.getStep("step3c").name(), "step3c")
self.assertEqual(task2c.getStep("step4a").name(), "step4a")
self.assertEqual(task1.getStep("step2"), None)
self.assertEqual(task2a.getStep("step4"), None)
self.assertEqual(task2b.getStep("step2"), None)
self.assertEqual(task2c.getStep("step1"), None)
self.assertEqual(task1.listNodes(),
['task1', 'task2a', 'task3', 'task2b', 'task2c'])
return
def testMaxRSS(self):
"""
_testMaxRSS_
Test whether we can properly add MaxRSS performance monitor
to this task.
"""
testTask = makeWMTask("TestTask")
testTask.setMaxRSS(123)
self.assertEqual(testTask.data.watchdog.monitors, ['PerformanceMonitor'])
self.assertEqual(testTask.data.watchdog.PerformanceMonitor.maxRSS, 123)
return
def testProcessingVerAndAcquisitionEra(self):
"""
_testProcessingVerAndAcquisitionEra_
Test that we can add a processing version and acquisition era,
and then get it back.
"""
testTask = makeWMTask("TestTask")
testTask.setAcquisitionEra("StoneAge")
testTask.setProcessingVersion("vLast")
self.assertEqual(testTask.getAcquisitionEra(), "StoneAge")
self.assertEqual(testTask.getProcessingVersion(), "vLast")
return
def testGetSwVersionAndScramArchMulti(self):
"""
_testGetSwVersionAndScramArchMulti_
Test whether we can fetch the CMSSW release and ScramArch
being used in a task
"""
testTask = makeWMTask("MultiTask")
taskCmssw = testTask.makeStep("cmsRun1")
taskCmssw.setStepType("CMSSW")
taskCmsswStageOut = taskCmssw.addStep("stageOut1")
taskCmsswStageOut.setStepType("StageOut")
taskCmsswLogArch = taskCmsswStageOut.addStep("logArch1")
taskCmsswLogArch.setStepType("LogArchive")
testTask.applyTemplates()
taskCmsswHelper = taskCmssw.getTypeHelper()
taskCmsswHelper.cmsswSetup("CMSSW_1_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc123")
# setup step2/cmsRun2
step1Cmssw = testTask.getStep("cmsRun1")
step2Cmssw = step1Cmssw.addTopStep("cmsRun2")
step2Cmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(step2Cmssw.data)
step2CmsswHelper = step2Cmssw.getTypeHelper()
step2CmsswHelper.setupChainedProcessing("cmsRun1", "RAWSIMoutput")
step2CmsswHelper.cmsswSetup("CMSSW_2_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc223")
# setup step3/cmsRun3 --> duplicate CMSSW and ScramArch
step3Cmssw = step2Cmssw.addTopStep("cmsRun3")
step3Cmssw.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(step3Cmssw.data)
step3CmsswHelper = step3Cmssw.getTypeHelper()
step3CmsswHelper.setupChainedProcessing("cmsRun2", "AODoutput")
step3CmsswHelper.cmsswSetup("CMSSW_1_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc123")
self.assertEqual(testTask.getSwVersion(), "CMSSW_1_2_3")
self.assertEqual(testTask.getSwVersion(allSteps=True), ["CMSSW_1_2_3", "CMSSW_2_2_3", "CMSSW_1_2_3"])
self.assertEqual(testTask.getScramArch(), "slc7_amd64_gcc123")
self.assertEqual(testTask.getScramArch(allSteps=True), ["slc7_amd64_gcc123", "slc7_amd64_gcc223", "slc7_amd64_gcc123"])
return
def testPerformanceMonitor(self):
"""
_testPerformanceMonitor_
Test automated adding of performanceMonitor
Really you shouldn't be using this, so don't add it.
"""
testTask = makeWMTask("TestTask")
testTask.setPerformanceMonitor(maxRSS = 100, maxVSize = 101)
self.assertEqual(testTask.data.watchdog.monitors, ['PerformanceMonitor'])
self.assertEqual(testTask.data.watchdog.PerformanceMonitor.maxRSS, 100)
self.assertEqual(testTask.data.watchdog.PerformanceMonitor.maxVSize, 101)
return
def testAddNotifications(self):
"""
_testAddNotifications_
Test whether we can add notification addresses
"""
testTask = makeWMTask("TestTask")
testTask.setTaskType("Processing")
targetList = ['*****@*****.**', '*****@*****.**']
self.assertEqual(testTask.getNotifications(), [])
for x in targetList:
testTask.addNotification(target=x)
self.assertEqual(testTask.getNotifications(), targetList)
def testProcessedDatasetElements(self):
"""
_testProcessedDatasetElements_
Test that we can add a processing version and acquisition era,
and then get it back.
"""
testTask = makeWMTask("TestTask")
testTask.setAcquisitionEra("StoneAge")
testTask.setProcessingVersion(2)
testTask.setProcessingString("Test")
self.assertEqual(testTask.getAcquisitionEra(), "StoneAge", "Wrong acquisition era in the task")
self.assertEqual(testTask.getProcessingVersion(), 2, "Wrong processing version in the task")
self.assertEqual(testTask.getProcessingString(), "Test", "Wrong processing string in the task")
return
def testTreeBuilding(self):
"""
_testTreeBuilding_
Verify that tasks can be created and arranged in a hierarchy.
"""
task1 = makeWMTask("task1")
task2a = task1.addTask("task2a")
task2b = task1.addTask("task2b")
task2c = task1.addTask("task2c")
goldenTasks = ["task2a", "task2b", "task2c"]
for childTask in task1.childTaskIterator():
assert childTask.name() in goldenTasks, "Error: Unknown child task: %s" % childTask.name()
goldenTasks.remove(childTask.name())
assert len(goldenTasks) == 0, "Error: Missing tasks."
return
def testPerformanceMonitor(self):
"""
_testPerformanceMonitor_
Test automated adding of performanceMonitor
Really you shouldn't be using this, so don't add it.
"""
testTask = makeWMTask("TestTask")
testTask.setPerformanceMonitor(softTimeout=100,
gracePeriod=1)
self.assertEqual(testTask.data.watchdog.monitors, ['PerformanceMonitor'])
self.assertFalse(hasattr(testTask.data.watchdog.PerformanceMonitor, "maxRSS"))
self.assertFalse(hasattr(testTask.data.watchdog.PerformanceMonitor, "maxVSize"))
self.assertEqual(testTask.data.watchdog.PerformanceMonitor.softTimeout, 100)
self.assertEqual(testTask.data.watchdog.PerformanceMonitor.hardTimeout, 101)
return
def testAddLumiMask(self):
"""
_testAddLumiMask_
Verify that setting and getting the lumiMask objects for a task works correctly.
Do a round trip of a typical lumi mask
"""
testTask = makeWMTask("TestTask")
lumiMask = LumiList(compactList = {
'1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]],
'2':[[1,45]],
'3':[[1,45],[50,80]],
})
testTask.setLumiMask(lumiMask = lumiMask.getCompactList())
outMask = LumiList(compactList = testTask.getLumiMask())
self.assertEqual(lumiMask.getCMSSWString(), outMask.getCMSSWString())
return
def testPerformanceMonitor(self):
"""
_testPerformanceMonitor_
Test automated adding of performanceMonitor
Really you shouldn't be using this, so don't add it.
"""
testTask = makeWMTask("TestTask")
testTask.setPerformanceMonitor(softTimeout=100, gracePeriod=1)
self.assertEqual(testTask.data.watchdog.monitors,
['PerformanceMonitor'])
self.assertFalse(
hasattr(testTask.data.watchdog.PerformanceMonitor, "maxPSS"))
self.assertEqual(testTask.data.watchdog.PerformanceMonitor.softTimeout,
100)
self.assertEqual(testTask.data.watchdog.PerformanceMonitor.hardTimeout,
101)
return
def testInputPileup(self):
"""
_testInputPileup_
Verify that the input pileup dataset getter/setter methods work correctly
"""
testTask = makeWMTask("TestTask")
self.assertEqual(testTask.getInputPileupDatasets(), [])
pileupConfig = parsePileupConfig("/MC/ProcessedDataset/DataTier",
"/Data/ProcessedDataset/DataTier")
# then mimic the setupPileup method
thesePU = []
for puType, puList in viewitems(pileupConfig):
# there should be only one type and one PU dataset
testTask.setInputPileupDatasets(puList)
thesePU.extend(puList)
self.assertItemsEqual(testTask.getInputPileupDatasets(), thesePU)
with self.assertRaises(ValueError):
testTask.setInputPileupDatasets(None)
def testProcessedDatasetElements(self):
"""
_testProcessedDatasetElements_
Test that we can add a processing version and acquisition era,
and then get it back.
"""
testTask = makeWMTask("TestTask")
testTask.setAcquisitionEra("StoneAge")
testTask.setProcessingVersion(2)
testTask.setProcessingString("Test")
self.assertEqual(testTask.getAcquisitionEra(), "StoneAge",
"Wrong acquisition era in the task")
self.assertEqual(testTask.getProcessingVersion(), 2,
"Wrong processing version in the task")
self.assertEqual(testTask.getProcessingString(), "Test",
"Wrong processing string in the task")
return
def testAddEnvironmentVariables(self):
"""
_testAddEnvironmentVariables_
Verify that methods for setting and retrieving task environment variables.
"""
testTask = makeWMTask("TestTask")
testDict = {
"VAR0": "Value0",
"VAR1": "Value1",
"VAR2": "Value2",
"VAR3": "Value3",
}
testTask.addEnvironmentVariables(testDict)
retrievedDict = testTask.getEnvironmentVariables()
self.assertEqual(
retrievedDict, testDict,
"Error: Env variables dict doesn't match with test dict")
return
def testTreeBuilding(self):
"""
_testTreeBuilding_
Verify that tasks can be created and arranged in a hierarchy.
"""
task1 = makeWMTask("task1")
task1.addTask("task2a")
task1.addTask("task2b")
task1.addTask("task2c")
goldenTasks = ["task2a", "task2b", "task2c"]
for childTask in task1.childTaskIterator():
assert childTask.name() in goldenTasks, \
"Error: Unknown child task: %s" % childTask.name()
goldenTasks.remove(childTask.name())
assert len(goldenTasks) == 0, \
"Error: Missing tasks."
return
def testParameters(self):
"""
_testParameters_
Test any random junk that we throw into parameters
"""
testTask = makeWMTask("TestTask")
# Test the primarySubType first
# Before we set anything, the subType should be none
self.assertEqual(testTask.getPrimarySubType(), None)
# After we set the task Type, but before the subType, it
# should be the taskType
testTask.setTaskType("SillyTask")
self.assertEqual(testTask.getPrimarySubType(), "SillyTask")
testTask.setPrimarySubType(subType="subType")
self.assertEqual(testTask.getPrimarySubType(), "subType")
return
def testParameters(self):
"""
_testParameters_
Test any random junk that we throw into parameters
"""
testTask = makeWMTask("TestTask")
# Test the primarySubType first
# Before we set anything, the subType should be none
self.assertEqual(testTask.getPrimarySubType(), None)
# After we set the task Type, but before the subType, it
# should be the taskType
testTask.setTaskType("SillyTask")
self.assertEqual(testTask.getPrimarySubType(), "SillyTask")
testTask.setPrimarySubType(subType="subType")
self.assertEqual(testTask.getPrimarySubType(), "subType")
return
def testAddLumiMask(self):
"""
_testAddLumiMask_
Verify that setting and getting the lumiMask objects for a task works correctly.
Do a round trip of a typical lumi mask
"""
testTask = makeWMTask("TestTask")
lumiMask = LumiList(
compactList={
'1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130],
[133, 136]],
'2': [[1, 45]],
'3': [[1, 45], [50, 80]],
})
testTask.setLumiMask(lumiMask=lumiMask.getCompactList())
outMask = testTask.getLumiMask()
self.assertEqual(lumiMask.getCMSSWString(), outMask.getCMSSWString())
return
def testDeleteChild(self):
"""
_testDeleteChild_
Test that we can remove all reference from a child
and other children are left intact
"""
task1 = makeWMTask("task1")
task1.addTask("task2a")
task1.addTask("task2b")
task1.addTask("task2c")
task1.deleteChild("task2a")
childrenNumber = 0
for childTask in task1.childTaskIterator():
if childTask.name() == "task2a":
self.fail("Error: It was possible to find the deleted child")
childrenNumber += 1
self.assertEqual(childrenNumber, 2, "Error: Wrong number of children tasks")
return
def testDeleteChild(self):
"""
_testDeleteChild_
Test that we can remove all reference from a child
and other children are left intact
"""
task1 = makeWMTask("task1")
task1.addTask("task2a")
task1.addTask("task2b")
task1.addTask("task2c")
task1.deleteChild("task2a")
childrenNumber = 0
for childTask in task1.childTaskIterator():
if childTask.name() == "task2a":
self.fail("Error: It was possible to find the deleted child")
childrenNumber += 1
self.assertEqual(childrenNumber, 2, "Error: Wrong number of children tasks")
return
def testWMTask(self):
"""
_testWMTask_
Test whether or not we can read the seeder parameters out of a WMTask.
Also tests RandomSeeder and RunAndLumiSeeder
"""
task1 = makeWMTask("task1")
randomDict = {"generator.initialSeed": None, "evtgenproducer.initialSeed": None, "MAXINT": 1}
lumiDict = {"lumi_per_run": 5}
task1.addGenerator("RandomSeeder", **randomDict)
task1.addGenerator("RunAndLumiSeeder", **lumiDict)
manager = GeneratorManager(task = task1)
jobs = self.oneHundredFiles()
for jobGrp in jobs:
manager(jobGrp)
for jobGrp in jobs:
count = 0
for job in jobGrp.jobs:
conf = job.getBaggage()
self.assertTrue(hasattr(conf.RandomSeeder.evtgenproducer, 'initialSeed'))
self.assertTrue(hasattr(conf.RandomSeeder.generator, 'initialSeed'))
#self.assertEqual(job["mask"]["FirstLumi"], count%6)
#self.assertEqual(job["mask"]["FirstRun"], (count/6) + 1)
x = conf.RandomSeeder.generator.initialSeed
self.assertTrue( x > 0, "ERROR: producing negative random numbers")
self.assertTrue( x <= 1, "ERROR: MAXINT tag failed; producing bad random number %i" %(x))
count += 1
return
def createMultiTaskObject(self):
"""
Creates a multi task object in the following structure:
Task1:
cmsRun1
stageOut1
logArch1
Task2:
cmsRun1
cmsRun2
stageOut1
logArch1
:return: a task helper object for the first task
"""
task1 = makeWMTask("Taskname_1")
task1.setTaskType("Production")
task1Cmssw = task1.makeStep("cmsRun1")
task1Cmssw.setStepType("CMSSW")
taskCmsswStageOut = task1Cmssw.addStep("stageOut1")
taskCmsswStageOut.setStepType("StageOut")
taskCmsswLogArch = taskCmsswStageOut.addStep("logArch1")
taskCmsswLogArch.setStepType("LogArchive")
task1.applyTemplates()
task2 = task1.addTask("Taskname_2")
task1.setTaskType("Processing")
task2ParentCmssw = task2.makeStep("cmsRun1")
task2ParentCmssw.setStepType("CMSSW")
task2Cmssw = task2ParentCmssw.addTopStep("cmsRun2")
task2Cmssw.setStepType("CMSSW")
taskCmsswStageOut = task2Cmssw.addStep("stageOut1")
taskCmsswStageOut.setStepType("StageOut")
taskCmsswLogArch = taskCmsswStageOut.addStep("logArch1")
taskCmsswLogArch.setStepType("LogArchive")
task2.applyTemplates()
return task1
def testSubscriptionInformation(self):
"""
_testSubscriptionInformation_
Check the three methods related to the subscription information in a task
Make sure that we can set the subscription information for all datasets produced by this task
and we can select only some primaryDatasets/DataTiers.
Since subscriptions are defined during request assignment, there is no more need to update
them, they are set once only.
"""
testTask = makeWMTask("TestTask")
cmsswStep = testTask.makeStep("cmsRun1")
cmsswStep.setStepType("CMSSW")
testTask.applyTemplates()
cmsswHelper = cmsswStep.getTypeHelper()
cmsswHelper.addOutputModule("outputRECO", primaryDataset="OneParticle",
processedDataset="DawnOfAnEra-v1", dataTier="RECO")
cmsswHelper.addOutputModule("outputDQM", primaryDataset="TwoParticles",
processedDataset="DawnOfAnEra-v1", dataTier="DQM")
cmsswHelper.addOutputModule("outputAOD", primaryDataset="OneParticle",
processedDataset="DawnOfAnEra-v1", dataTier="AOD")
childStep = cmsswHelper.addTopStep("cmsRun2")
childStep.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(childStep.data)
childStep = childStep.getTypeHelper()
childStep.addOutputModule("outputAOD", primaryDataset="ThreeParticles",
processedDataset="DawnOfAnEra-v1", dataTier="MINIAOD")
self.assertEqual(testTask.getSubscriptionInformation(), {}, "There should not be any subscription info")
testTask.setSubscriptionInformation(custodialSites=["mercury"],
nonCustodialSites=["mars", "earth"],
autoApproveSites=["earth"],
custodialSubType="Replica",
nonCustodialSubType="Move",
custodialGroup="DataOps",
nonCustodialGroup="AnalysisOps",
priority="Normal",
deleteFromSource=True,
primaryDataset="OneParticle")
subInfo = testTask.getSubscriptionInformation()
outputRecoSubInfo = {"CustodialSites": ["mercury"],
"NonCustodialSites": ["mars", "earth"],
"AutoApproveSites": ["earth"],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Move",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "AnalysisOps",
"Priority": "Normal",
"DeleteFromSource": True}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertFalse("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertFalse("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
testTask.setSubscriptionInformation(custodialSites=["jupiter"],
primaryDataset="TwoParticles")
subInfo = testTask.getSubscriptionInformation()
outputDQMSubInfo = {"CustodialSites": ["jupiter"],
"NonCustodialSites": [],
"AutoApproveSites": [],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Replica",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "DataOps",
"Priority": "Low",
"DeleteFromSource": False}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertEqual(subInfo["/TwoParticles/DawnOfAnEra-v1/DQM"], outputDQMSubInfo,
"The DQM subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertFalse("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
testTask.setSubscriptionInformation(nonCustodialSites=["jupiter"],
primaryDataset="ThreeParticles")
subInfo = testTask.getSubscriptionInformation()
outputAODSubInfo = {"CustodialSites": [],
"NonCustodialSites": ["jupiter"],
"AutoApproveSites": [],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Replica",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "DataOps",
"Priority": "Low",
"DeleteFromSource": False}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertEqual(subInfo["/TwoParticles/DawnOfAnEra-v1/DQM"], outputDQMSubInfo,
"The DQM subscription information is wrong")
self.assertEqual(subInfo["/ThreeParticles/DawnOfAnEra-v1/MINIAOD"], outputAODSubInfo,
"The AOD subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertTrue("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
def testInputDataset(self):
"""
_testInputDataset_
Verify that the addInputDataset() method works correctly and that the
run/block black and white lists can be changed after calling
addInputDataset().
"""
testTask = makeWMTask("TestTask")
assert testTask.getInputDatasetPath() == None, \
"Error: Input dataset path should be None."
assert testTask.inputDatasetDBSURL() == None, \
"Error: Input DBS URL should be None."
assert testTask.inputBlockWhitelist() == None, \
"Error: Input block white list should be None."
assert testTask.inputBlockBlacklist() == None, \
"Error: Input block black list should be None."
assert testTask.inputRunWhitelist() == None, \
"Error: Input run white list should be None."
assert testTask.inputRunBlacklist() == None, \
"Error: Input run black list should be None."
testTask.addInputDataset(
primary="PrimaryDataset",
processed="ProcessedDataset",
tier="DataTier",
dbsurl="DBSURL",
block_whitelist=["Block1", "Block2"],
block_blacklist=["Block3", "Block4", "Block5"],
run_whitelist=[1, 2, 3],
run_blacklist=[4, 5])
assert testTask.getInputDatasetPath() == "/PrimaryDataset/ProcessedDataset/DataTier", \
"Error: Input dataset path is wrong."
assert testTask.inputDatasetDBSURL() == "DBSURL", \
"Error: Input DBS URL is wrong."
assert len(testTask.inputBlockWhitelist()) == 2, \
"Error: Wrong number of blocks in white list."
assert "Block1" in testTask.inputBlockWhitelist(), \
"Error: Block missing from white list."
assert "Block2" in testTask.inputBlockWhitelist(), \
"Error: Block missing from white list."
assert len(testTask.inputBlockBlacklist()) == 3, \
"Error: Wrong number of blocks in black list."
assert "Block3" in testTask.inputBlockBlacklist(), \
"Error: Block missing from black list."
assert "Block4" in testTask.inputBlockBlacklist(), \
"Error: Block missing from black list."
assert "Block5" in testTask.inputBlockBlacklist(), \
"Error: Block missing from black list."
assert len(testTask.inputRunWhitelist()) == 3, \
"Error: Wrong number of runs in white list."
assert 1 in testTask.inputRunWhitelist(), \
"Error: Run is missing from white list."
assert 2 in testTask.inputRunWhitelist(), \
"Error: Run is missing from white list."
assert 3 in testTask.inputRunWhitelist(), \
"Error: Run is missing from white list."
assert len(testTask.inputRunBlacklist()) == 2, \
"Error: Wrong number of runs in black list."
assert 4 in testTask.inputRunBlacklist(), \
"Error: Run is missing from black list."
assert 5 in testTask.inputRunBlacklist(), \
"Error: Run is missing from black list."
testTask.setInputBlockWhitelist(["Block6"])
assert len(testTask.inputBlockWhitelist()) == 1, \
"Error: Wrong number of blocks in white list."
assert "Block6" in testTask.inputBlockWhitelist(), \
"Error: Block missing from white list."
testTask.setInputBlockBlacklist(["Block7", "Block8"])
assert len(testTask.inputBlockBlacklist()) == 2, \
"Error: Wrong number of blocks in black list."
assert "Block7" in testTask.inputBlockBlacklist(), \
"Error: Block missing from black list."
assert "Block8" in testTask.inputBlockBlacklist(), \
"Error: Block missing from black list."
testTask.setInputRunWhitelist([6])
assert len(testTask.inputRunWhitelist()) == 1, \
"Error: Wrong number of runs in white list."
assert 6 in testTask.inputRunWhitelist(), \
"Error: Run missing from white list."
testTask.setInputRunBlacklist([7, 8])
assert len(testTask.inputRunBlacklist()) == 2, \
"Error: Wrong number of runs in black list."
assert 7 in testTask.inputRunBlacklist(), \
"Error: Run missing from black list."
assert 8 in testTask.inputRunBlacklist(), \
"Error: Run missing from black list."
return
def testJobSplittingArgs(self):
"""
_testJobSplittingArgs_
Verify that setting job splitting arguments for a task works correctly.
When pulling the job splitting arguments out of a task the site white
list and black list should also be included as they need to be passed to
the job splitting code.
"""
testTask = makeWMTask("TestTask")
testTask.setTaskType("Processing")
self.assertEqual(testTask.taskType(), "Processing",
"Error: Wrong task type.")
testTask.setJobResourceInformation(
timePerEvent=12, memoryReq=2300000, sizePerEvent=512)
testTask.setSplittingAlgorithm(
"MadeUpAlgo",
events_per_job=100,
max_job_size=24,
one_more_param="Hello")
testTask.setSiteWhitelist(["T1_US_FNAL", "T1_CH_CERN"])
testTask.setSiteBlacklist(["T2_US_PERDUE", "T2_US_UCSD", "T1_TW_ASGC"])
testTask.addInputDataset(
primary="PrimaryDataset",
processed="ProcessedDataset",
tier="DataTier",
dbsurl="DBSURL",
block_whitelist=["Block1", "Block2"],
block_blacklist=["Block3", "Block4", "Block5"],
run_whitelist=[1, 2, 3],
run_blacklist=[4, 5])
# Make sure we can set individual performance parameters without affecting the others
testTask.setJobResourceInformation(timePerEvent=14)
self.assertEqual(testTask.jobSplittingAlgorithm(), "MadeUpAlgo",
"Error: Wrong job splitting algorithm name.")
algoParams = testTask.jobSplittingParameters(performance=False)
self.assertEqual(
len(algoParams), 9, "Error: Wrong number of algo parameters.")
algoParams = testTask.jobSplittingParameters()
self.assertEqual(
len(algoParams), 10, "Error: Wrong number of algo parameters.")
self.assertTrue("algorithm" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(algoParams["algorithm"], "MadeUpAlgo",
"Error: Parameter has wrong value.")
self.assertTrue("events_per_job" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(algoParams["events_per_job"], 100,
"Error: Parameter has wrong value.")
self.assertTrue("max_job_size" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(algoParams["max_job_size"], 24,
"Error: Parameter has wrong value.")
self.assertTrue("one_more_param" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(algoParams["one_more_param"], "Hello",
"Error: Parameter has wrong value.")
self.assertTrue("runWhitelist" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(
len(algoParams["runWhitelist"]), 3,
"Error: Wrong number of runs in whitelist.")
self.assertTrue("performance" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(algoParams["performance"]["timePerEvent"], 14,
"Error: Wrong time per event")
self.assertEqual(algoParams["performance"]["memoryRequirement"],
2300000, "Error: Wrong memory requirement")
self.assertEqual(algoParams["performance"]["sizePerEvent"], 512,
"Error: Wrong size per event")
return
def executeCMSSWStack(opts, scram):
def getOutputModules():
pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\
"config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\
"tweakJson = makeTweak(config.process).jsondictionary();"+\
"print tweakJson[\"process\"][\"outputModules_\"]"
with tempSetLogLevel(logger=logging.getLogger(), level=logging.ERROR):
ret = scram("python -c '%s'" % pythonScript,
runtimeDir=os.getcwd())
if ret > 0:
msg = 'Error getting output modules from the pset.\n\tScram Diagnostic %s' % scram.diagnostic(
)
handleException("FAILED", EC_CMSRunWrapper, msg)
mintime()
sys.exit(EC_CMSRunWrapper)
output = literal_eval(scram.getStdout())
return output
cmssw = CMSSW()
cmssw.stepName = "cmsRun"
cmssw.step = WMStep(cmssw.stepName)
CMSSWTemplate().install(cmssw.step)
cmssw.task = makeWMTask(cmssw.stepName)
cmssw.workload = newWorkload(cmssw.stepName)
cmssw.step.application.setup.softwareEnvironment = ''
cmssw.step.application.setup.scramArch = opts.scramArch
cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion
cmssw.step.application.configuration.section_("arguments")
cmssw.step.application.configuration.arguments.globalTag = ""
for output in getOutputModules():
cmssw.step.output.modules.section_(output)
getattr(cmssw.step.output.modules, output).primaryDataset = ''
getattr(cmssw.step.output.modules, output).processedDataset = ''
getattr(cmssw.step.output.modules, output).dataTier = ''
#cmssw.step.application.command.arguments = '' #TODO
cmssw.step.user.inputSandboxes = [opts.archiveJob]
cmssw.step.user.userFiles = opts.userFiles or ''
#Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles.
#Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime..
cmssw.job = {'counter': 0, 'workflow': 'unused'}
cmssw.step.user.lfnBase = '/store/temp/user/'
cmssw.step.section_("builder")
cmssw.step.builder.workingDir = os.getcwd()
cmssw.step.runtime.invokeCommand = 'python'
cmssw.step.runtime.scramPreDir = os.getcwd()
cmssw.step.runtime.preScripts = []
cmssw.step.runtime.scramPreScripts = [
('%s/TweakPSet.py --location=%s ' + '--inputFile=\'%s\' ' +
'--runAndLumis=\'%s\' ' + '--firstEvent=%s ' + '--lastEvent=%s ' +
'--firstLumi=%s ' + '--firstRun=%s ' + '--seeding=%s ' +
'--lheInputFiles=%s ' + '--oneEventMode=%s ' + '--eventsPerLumi=%s ' +
'--maxRuntime=%s') %
(os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis,
opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun,
opts.seeding, opts.lheInputFiles, opts.oneEventMode,
opts.eventsPerLumi, opts.maxRuntime)
]
cmssw.step.section_("execution") #exitStatus of cmsRun is set here
cmssw.report = Report("cmsRun") #report is loaded and put here
cmssw.execute()
return cmssw
def testJobSplittingArgs(self):
"""
_testJobSplittingArgs_
Verify that setting job splitting arguments for a task works correctly.
When pulling the job splitting arguments out of a task the site white
list and black list should also be included as they need to be passed to
the job splitting code.
"""
testTask = makeWMTask("TestTask")
testTask.setTaskType("Processing")
self.assertEqual(testTask.taskType(), "Processing",
"Error: Wrong task type.")
testTask.setJobResourceInformation(timePerEvent=12, memoryReq=2300000,
sizePerEvent=512)
testTask.setSplittingAlgorithm("MadeUpAlgo", events_per_job=100,
max_job_size=24,
one_more_param="Hello")
testTask.setSiteWhitelist(["T1_US_FNAL", "T1_CH_CERN"])
testTask.setSiteBlacklist(["T2_US_PERDUE", "T2_US_UCSD", "T1_TW_ASGC"])
testTask.addInputDataset(name="/PrimaryDataset/ProcessedDataset/DataTier",
primary="PrimaryDataset",
processed="ProcessedDataset",
tier="DataTier",
dbsurl="DBSURL",
block_whitelist=["Block1", "Block2"],
block_blacklist=["Block3", "Block4", "Block5"],
run_whitelist=[1, 2, 3],
run_blacklist=[4, 5])
# Make sure we can set individual performance parameters without affecting the others
testTask.setJobResourceInformation(timePerEvent=14)
self.assertEqual(testTask.jobSplittingAlgorithm(), "MadeUpAlgo",
"Error: Wrong job splitting algorithm name.")
algoParams = testTask.jobSplittingParameters(performance=False)
self.assertEqual(len(algoParams), 10,
"Error: Wrong number of algo parameters.")
algoParams = testTask.jobSplittingParameters()
self.assertEqual(len(algoParams), 11,
"Error: Wrong number of algo parameters.")
self.assertTrue("algorithm" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(algoParams["algorithm"], "MadeUpAlgo",
"Error: Parameter has wrong value.")
self.assertTrue("events_per_job" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(algoParams["events_per_job"], 100,
"Error: Parameter has wrong value.")
self.assertTrue("max_job_size" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(algoParams["max_job_size"], 24,
"Error: Parameter has wrong value.")
self.assertTrue("one_more_param" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(algoParams["one_more_param"], "Hello",
"Error: Parameter has wrong value.")
self.assertTrue("runWhitelist" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(len(algoParams["runWhitelist"]), 3,
"Error: Wrong number of runs in whitelist.")
self.assertTrue("performance" in algoParams,
"Error: Missing algo parameter.")
self.assertEqual(algoParams["performance"]["timePerEvent"], 14,
"Error: Wrong time per event")
self.assertEqual(algoParams["performance"]["memoryRequirement"], 2300000,
"Error: Wrong memory requirement")
self.assertEqual(algoParams["performance"]["sizePerEvent"], 512,
"Error: Wrong size per event")
return
def executeCMSSWStack(opts, scram):
def getOutputModules():
pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\
"config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\
"tweakJson = makeTweak(config.process).jsondictionary();"+\
"print tweakJson[\"process\"][\"outputModules_\"]"
ret = scram("python -c '%s'" % pythonScript, logName=subprocess.PIPE, runtimeDir=os.getcwd())
if ret > 0:
msg = scram.diagnostic()
handleException("FAILED", EC_CMSRunWrapper, 'Error getting output modules from the pset.\n\tScram Env %s\n\tCommand:%s' % (msg, pythonScript))
mintime()
sys.exit(EC_CMSRunWrapper)
return literal_eval(scram.stdout)
cmssw = CMSSW()
cmssw.stepName = "cmsRun"
cmssw.step = WMStep(cmssw.stepName)
CMSSWTemplate().install(cmssw.step)
cmssw.task = makeWMTask(cmssw.stepName)
cmssw.workload = newWorkload(cmssw.stepName)
cmssw.step.application.setup.softwareEnvironment = ''
cmssw.step.application.setup.scramArch = opts.scramArch
cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion
cmssw.step.application.configuration.section_("arguments")
cmssw.step.application.configuration.arguments.globalTag = ""
for output in getOutputModules():
cmssw.step.output.modules.section_(output)
getattr(cmssw.step.output.modules, output).primaryDataset = ''
getattr(cmssw.step.output.modules, output).processedDataset = ''
getattr(cmssw.step.output.modules, output).dataTier = ''
#cmssw.step.application.command.arguments = '' #TODO
cmssw.step.user.inputSandboxes = [opts.archiveJob]
cmssw.step.user.userFiles = opts.userFiles or ''
#Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles.
#Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime..
cmssw.job = {'counter' : 0, 'workflow' : 'unused'}
cmssw.step.user.lfnBase = '/store/temp/user/'
cmssw.step.section_("builder")
cmssw.step.builder.workingDir = os.getcwd()
cmssw.step.runtime.invokeCommand = 'python'
cmssw.step.runtime.scramPreDir = os.getcwd()
cmssw.step.runtime.preScripts = []
cmssw.step.runtime.scramPreScripts = [('%s/TweakPSet.py --location=%s '+
'--inputFile=\'%s\' '+
'--runAndLumis=\'%s\' '+
'--firstEvent=%s '+
'--lastEvent=%s '+
'--firstLumi=%s '+
'--firstRun=%s '+
'--seeding=%s '+
'--lheInputFiles=%s '+
'--oneEventMode=%s ' +
'--eventsPerLumi=%s') %
(os.getcwd(), os.getcwd(),
opts.inputFile,
opts.runAndLumis,
opts.firstEvent,
opts.lastEvent,
opts.firstLumi,
opts.firstRun,
opts.seeding,
opts.lheInputFiles,
opts.oneEventMode,
opts.eventsPerLumi)]
cmssw.step.section_("execution") #exitStatus of cmsRun is set here
cmssw.report = Report("cmsRun") #report is loaded and put here
cmssw.execute()
return cmssw
def testSubscriptionInformation(self):
"""
_testSubscriptionInformation_
Check the three methods related to the subscription information in a task
Make sure that we can set the subscription information for all datasets produced by this task
and we can select only some primaryDatasets/DataTiers, and check that we can update the dataset
in a subscription information section.
"""
testTask = makeWMTask("TestTask")
cmsswStep = testTask.makeStep("cmsRun1")
cmsswStep.setStepType("CMSSW")
testTask.applyTemplates()
cmsswHelper = cmsswStep.getTypeHelper()
cmsswHelper.addOutputModule("outputRECO",
primaryDataset="OneParticle",
processedDataset="DawnOfAnEra-v1",
dataTier="RECO")
cmsswHelper.addOutputModule("outputDQM",
primaryDataset="TwoParticles",
processedDataset="DawnOfAnEra-v1",
dataTier="DQM")
cmsswHelper.addOutputModule("outputAOD",
primaryDataset="OneParticle",
processedDataset="DawnOfAnEra-v1",
dataTier="AOD")
self.assertEqual(testTask.getSubscriptionInformation(), {},
"There should not be any subscription info")
testTask.setSubscriptionInformation(["mercury"], ["mars", "earth"],
["earth"], "High", "OneParticle")
subInfo = testTask.getSubscriptionInformation()
outputRecoSubInfo = {
"CustodialSites": ["mercury"],
"NonCustodialSites": ["mars", "earth"],
"AutoApproveSites": ["earth"],
"Priority": "High"
}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"],
outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo,
"The AOD subscription information is wrong")
self.assertFalse("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo,
"The DQM subscription information is wrong")
testTask.setSubscriptionInformation(["jupiter"],
primaryDataset="TwoParticles")
subInfo = testTask.getSubscriptionInformation()
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"],
outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo,
"The AOD subscription information is wrong")
self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo,
"The DQM subscription information is wrong")
recoutOutputModule = cmsswHelper.getOutputModule("outputRECO")
setattr(recoutOutputModule, "primaryDataset", "ThreeParticles")
testTask.updateSubscriptionDataset("outputRECO", recoutOutputModule)
subInfo = testTask.getSubscriptionInformation()
self.assertEqual(subInfo["/ThreeParticles/DawnOfAnEra-v1/RECO"],
outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertFalse("/OneParticle/DawnOfAnEra-v1/RECO" in subInfo,
"The RECO subscription information is wrong")
def testInputDataset(self):
"""
_testInputDataset_
Verify that the addInputDataset() method works correctly and that the
run/block black and white lists can be changed after calling
addInputDataset().
"""
testTask = makeWMTask("TestTask")
assert testTask.getInputDatasetPath() is None, \
"Error: Input dataset path should be None."
assert testTask.dbsUrl() is None, \
"Error: Input DBS URL should be None."
assert testTask.inputBlockWhitelist() is None, \
"Error: Input block white list should be None."
assert testTask.inputBlockBlacklist() is None, \
"Error: Input block black list should be None."
assert testTask.inputRunWhitelist() is None, \
"Error: Input run white list should be None."
assert testTask.inputRunBlacklist() is None, \
"Error: Input run black list should be None."
testTask.addInputDataset(name="/PrimaryDataset/ProcessedDataset/DataTier",
primary="PrimaryDataset",
processed="ProcessedDataset",
tier="DataTier",
dbsurl="DBSURL",
block_whitelist=["Block1", "Block2"],
block_blacklist=["Block3", "Block4", "Block5"],
run_whitelist=[1, 2, 3],
run_blacklist=[4, 5])
assert testTask.getInputDatasetPath() == "/PrimaryDataset/ProcessedDataset/DataTier", \
"Error: Input dataset path is wrong."
assert testTask.dbsUrl() == "DBSURL", \
"Error: Input DBS URL is wrong."
assert len(testTask.inputBlockWhitelist()) == 2, \
"Error: Wrong number of blocks in white list."
assert "Block1" in testTask.inputBlockWhitelist(), \
"Error: Block missing from white list."
assert "Block2" in testTask.inputBlockWhitelist(), \
"Error: Block missing from white list."
assert len(testTask.inputBlockBlacklist()) == 3, \
"Error: Wrong number of blocks in black list."
assert "Block3" in testTask.inputBlockBlacklist(), \
"Error: Block missing from black list."
assert "Block4" in testTask.inputBlockBlacklist(), \
"Error: Block missing from black list."
assert "Block5" in testTask.inputBlockBlacklist(), \
"Error: Block missing from black list."
assert len(testTask.inputRunWhitelist()) == 3, \
"Error: Wrong number of runs in white list."
assert 1 in testTask.inputRunWhitelist(), \
"Error: Run is missing from white list."
assert 2 in testTask.inputRunWhitelist(), \
"Error: Run is missing from white list."
assert 3 in testTask.inputRunWhitelist(), \
"Error: Run is missing from white list."
assert len(testTask.inputRunBlacklist()) == 2, \
"Error: Wrong number of runs in black list."
assert 4 in testTask.inputRunBlacklist(), \
"Error: Run is missing from black list."
assert 5 in testTask.inputRunBlacklist(), \
"Error: Run is missing from black list."
testTask.setInputBlockWhitelist(["Block6"])
assert len(testTask.inputBlockWhitelist()) == 1, \
"Error: Wrong number of blocks in white list."
assert "Block6" in testTask.inputBlockWhitelist(), \
"Error: Block missing from white list."
testTask.setInputBlockBlacklist(["Block7", "Block8"])
assert len(testTask.inputBlockBlacklist()) == 2, \
"Error: Wrong number of blocks in black list."
assert "Block7" in testTask.inputBlockBlacklist(), \
"Error: Block missing from black list."
assert "Block8" in testTask.inputBlockBlacklist(), \
"Error: Block missing from black list."
testTask.setInputRunWhitelist([6])
assert len(testTask.inputRunWhitelist()) == 1, \
"Error: Wrong number of runs in white list."
assert 6 in testTask.inputRunWhitelist(), \
"Error: Run missing from white list."
testTask.setInputRunBlacklist([7, 8])
assert len(testTask.inputRunBlacklist()) == 2, \
"Error: Wrong number of runs in black list."
assert 7 in testTask.inputRunBlacklist(), \
"Error: Run missing from black list."
assert 8 in testTask.inputRunBlacklist(), \
"Error: Run missing from black list."
return
def testSubscriptionInformation(self):
"""
_testSubscriptionInformation_
Check the three methods related to the subscription information in a task
Make sure that we can set the subscription information for all datasets produced by this task
and we can select only some primaryDatasets/DataTiers.
Since subscriptions are defined during request assignment, there is no more need to update
them, they are set once only.
"""
testTask = makeWMTask("TestTask")
cmsswStep = testTask.makeStep("cmsRun1")
cmsswStep.setStepType("CMSSW")
testTask.applyTemplates()
cmsswHelper = cmsswStep.getTypeHelper()
cmsswHelper.addOutputModule("outputRECO", primaryDataset="OneParticle",
processedDataset="DawnOfAnEra-v1", dataTier="RECO")
cmsswHelper.addOutputModule("outputDQM", primaryDataset="TwoParticles",
processedDataset="DawnOfAnEra-v1", dataTier="DQM")
cmsswHelper.addOutputModule("outputAOD", primaryDataset="OneParticle",
processedDataset="DawnOfAnEra-v1", dataTier="AOD")
childStep = cmsswHelper.addTopStep("cmsRun2")
childStep.setStepType("CMSSW")
template = StepFactory.getStepTemplate("CMSSW")
template(childStep.data)
childStep = childStep.getTypeHelper()
childStep.addOutputModule("outputAOD", primaryDataset="ThreeParticles",
processedDataset="DawnOfAnEra-v1", dataTier="MINIAOD")
self.assertEqual(testTask.getSubscriptionInformation(), {}, "There should not be any subscription info")
testTask.setSubscriptionInformation(custodialSites=["mercury"],
nonCustodialSites=["mars", "earth"],
autoApproveSites=["earth"],
custodialSubType="Replica",
nonCustodialSubType="Move",
custodialGroup="DataOps",
nonCustodialGroup="AnalysisOps",
priority="Normal",
deleteFromSource=True,
primaryDataset="OneParticle")
subInfo = testTask.getSubscriptionInformation()
outputRecoSubInfo = {"CustodialSites": ["mercury"],
"NonCustodialSites": ["mars", "earth"],
"AutoApproveSites": ["earth"],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Move",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "AnalysisOps",
"Priority": "Normal",
"DeleteFromSource": True}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertFalse("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertFalse("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
testTask.setSubscriptionInformation(custodialSites=["jupiter"],
primaryDataset="TwoParticles")
subInfo = testTask.getSubscriptionInformation()
outputDQMSubInfo = {"CustodialSites": ["jupiter"],
"NonCustodialSites": [],
"AutoApproveSites": [],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Replica",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "DataOps",
"Priority": "Low",
"DeleteFromSource": False}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertEqual(subInfo["/TwoParticles/DawnOfAnEra-v1/DQM"], outputDQMSubInfo,
"The DQM subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertFalse("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
testTask.setSubscriptionInformation(nonCustodialSites=["jupiter"],
primaryDataset="ThreeParticles")
subInfo = testTask.getSubscriptionInformation()
outputAODSubInfo = {"CustodialSites": [],
"NonCustodialSites": ["jupiter"],
"AutoApproveSites": [],
"CustodialSubType": "Replica",
"NonCustodialSubType": "Replica",
"CustodialGroup": "DataOps",
"NonCustodialGroup": "DataOps",
"Priority": "Low",
"DeleteFromSource": False}
self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo,
"The RECO subscription information is wrong")
self.assertEqual(subInfo["/TwoParticles/DawnOfAnEra-v1/DQM"], outputDQMSubInfo,
"The DQM subscription information is wrong")
self.assertEqual(subInfo["/ThreeParticles/DawnOfAnEra-v1/MINIAOD"], outputAODSubInfo,
"The AOD subscription information is wrong")
self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong")
self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong")
self.assertTrue("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)