def test3StepsMemCoresSettingsB(self):
"""
_test3StepsMemCoresSettingsB_
Mix Multicore settings at both step and request level and make sure they
are properly propagated to each step. Three steps in the task.
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
configDocs = injectStepChainConfigMC(self.configDatabase)
testArguments.update(getThreeStepsOverride())
for s in ['Step1', 'Step2', 'Step3']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
testArguments['Step2']['KeepOutput'] = False
# Test Multicore/Memory settings at step level **only**
testArguments['Step1']["Multicore"] = 2
testArguments['Step3']["Multicore"] = 4
testArguments['Step1']["Memory"] = 2200.0
testArguments['Step3']["Memory"] = 4400.0
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('cmsRun2', 'stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1, "%s should have 1 core" % step)
self.assertEqual(prodTask.getStepHelper('cmsRun1').getNumberOfCores(), testArguments['Step1']["Multicore"])
self.assertEqual(prodTask.getStepHelper('cmsRun3').getNumberOfCores(), testArguments['Step3']["Multicore"])
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], testArguments["Memory"])
# Test mix of Multicore/Memory settings at both top and step level
testArguments["Multicore"] = 3
testArguments['Step1']["Multicore"] = 2
testArguments['Step3']["Multicore"] = 4
testArguments["Memory"] = 3300.0
testArguments['Step1']["Memory"] = 2200.0
testArguments['Step3']["Memory"] = 4400.0
testWorkload = factory.factoryWorkloadConstruction("TestWorkload2", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1, "%s should have 1 core" % step)
self.assertEqual(prodTask.getStepHelper('cmsRun1').getNumberOfCores(), testArguments['Step1']["Multicore"])
self.assertEqual(prodTask.getStepHelper('cmsRun2').getNumberOfCores(), testArguments["Multicore"])
self.assertEqual(prodTask.getStepHelper('cmsRun3').getNumberOfCores(), testArguments['Step3']["Multicore"])
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], testArguments["Memory"])
return
def test1StepMemCoresSettings(self):
"""
_test1StepMemCoresSettings_
Make sure the multicore and memory setings are properly propagated to
all steps. Single step in a task.
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
testArguments.update(getSingleStepOverride())
testArguments['Step1']['ConfigCacheID'] = injectStepChainConfigSingle(
self.configDatabase),
if isinstance(testArguments['Step1']['ConfigCacheID'], tuple):
testArguments['Step1']['ConfigCacheID'] = testArguments['Step1'][
'ConfigCacheID'][0]
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
# then test Memory requirements
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
testArguments["Multicore"] = 6
testArguments["Memory"] = 4600.0
testArguments["EventStreams"] = 3
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload2", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
if step in ('stageOut1', 'logArch1'):
self.assertEqual(stepHelper.getNumberOfCores(), 1,
"%s should have 1 core" % step)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
else:
self.assertEqual(stepHelper.getNumberOfCores(),
testArguments["Multicore"])
self.assertEqual(stepHelper.getNumberOfStreams(),
testArguments["EventStreams"])
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'],
testArguments["Memory"])
return
def test3StepsMemCoresSettingsA(self):
"""
_test3StepsMemCoresSettingsA_
Make sure the multicore and memory setings are properly propagated to
all steps. Three steps in the task.
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
configDocs = injectStepChainConfigMC(self.configDatabase)
testArguments.update(getThreeStepsOverride())
for s in ['Step1', 'Step2', 'Step3']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
testArguments['Step2']['KeepOutput'] = False
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('cmsRun1', 'cmsRun2', 'cmsRun3', 'stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
# then test Memory requirements
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
# Test Multicore/Memory settings at TOP level **only**
testArguments["Multicore"] = 6
testArguments["Memory"] = 4600.0
testArguments["EventStreams"] = 3
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload2", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('cmsRun1', 'cmsRun2', 'cmsRun3', 'stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
if step in ('stageOut1', 'logArch1'):
self.assertEqual(stepHelper.getNumberOfCores(), 1,
"%s should have 1 core" % step)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
else:
self.assertEqual(stepHelper.getNumberOfCores(),
testArguments["Multicore"])
self.assertEqual(stepHelper.getNumberOfStreams(),
testArguments["EventStreams"])
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'],
testArguments["Memory"])
def test1StepMemCoresSettings(self):
"""
_test1StepMemCoresSettings_
Make sure the multicore and memory setings are properly propagated to
all steps. Single step in a task.
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
testArguments.update(getSingleStepOverride())
testArguments['Step1']['ConfigCacheID'] = injectStepChainConfigSingle(self.configDatabase),
if isinstance(testArguments['Step1']['ConfigCacheID'], tuple):
testArguments['Step1']['ConfigCacheID'] = testArguments['Step1']['ConfigCacheID'][0]
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1)
# then test Memory requirements
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
testArguments["Multicore"] = 6
testArguments["Memory"] = 4600.0
testWorkload = factory.factoryWorkloadConstruction("TestWorkload2", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
if step in ('stageOut1', 'logArch1'):
self.assertEqual(stepHelper.getNumberOfCores(), 1, "%s should have 1 core" % step)
else:
self.assertEqual(stepHelper.getNumberOfCores(), testArguments["Multicore"])
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], testArguments["Memory"])
return
def test3StepsMemCoresSettingsA(self):
"""
_test3StepsMemCoresSettingsA_
Make sure the multicore and memory setings are properly propagated to
all steps. Three steps in the task.
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
configDocs = injectStepChainConfigMC(self.configDatabase)
testArguments.update(getThreeStepsOverride())
for s in ['Step1', 'Step2', 'Step3']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
testArguments['Step2']['KeepOutput'] = False
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('cmsRun1', 'cmsRun2', 'cmsRun3', 'stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1)
# then test Memory requirements
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
# Test Multicore/Memory settings at TOP level **only**
testArguments["Multicore"] = 6
testArguments["Memory"] = 4600.0
testWorkload = factory.factoryWorkloadConstruction("TestWorkload2", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('cmsRun1', 'cmsRun2', 'cmsRun3', 'stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
if step in ('stageOut1', 'logArch1'):
self.assertEqual(stepHelper.getNumberOfCores(), 1, "%s should have 1 core" % step)
else:
self.assertEqual(stepHelper.getNumberOfCores(), testArguments["Multicore"])
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], testArguments["Memory"])
def testStepChainMC(self):
"""
Build a StepChain workload starting from scratch
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments.update(request['createRequest'])
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "stepchain_t"
})
configDocs = injectStepChainConfigMC(self.configDatabase)
for s in ['Step1', 'Step2', 'Step3']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload, "ProdMinBias", "MCFakeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "production")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_0_0_pre11")
self.assertEqual(testWorkload.getProcessingString(), "START70_V4")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertEqual(testWorkload.getPrepID(), "Step-00")
self.assertEqual(sorted(testWorkload.getCMSSWVersions()), ['CMSSW_7_0_0_pre11', 'CMSSW_7_0_0_pre12'])
# test workload attributes
self.assertEqual(testWorkload.processingString, "START70_V4")
self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_0_0_pre11")
self.assertEqual(testWorkload.processingVersion, 1)
self.assertFalse(testWorkload.lumiList, "Wrong lumiList")
self.assertEqual(testWorkload.data.policies.start.policyName, "MonteCarlo")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 8)
for t in ['ProdMinBias', 'ProdMinBiasMergeAODSIMoutput', 'ProdMinBiasMergeRECOSIMoutput']:
self.assertTrue(t in tasks, "Wrong task name")
self.assertFalse('ProdMinBiasMergeRAWSIMoutput' in tasks, "Wrong task name")
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "ProdMinBias")
self.assertEqual(task.getPathName(), "/TestWorkload/ProdMinBias")
self.assertEqual(task.taskType(), "Production", "Wrong task type")
splitParams = task.jobSplittingParameters()
self.assertEqual(splitParams['algorithm'], "EventBased", "Wrong job splitting algo")
self.assertEqual(splitParams['events_per_job'], 150)
self.assertEqual(splitParams['events_per_lumi'], 50)
self.assertFalse(splitParams['lheInputFiles'], "Wrong LHE flag")
self.assertTrue(splitParams['performance']['timePerEvent'] > 4.75)
self.assertTrue(splitParams['performance']['sizePerEvent'] > 1233)
self.assertTrue(splitParams['performance']['memoryRequirement'] == 2400)
self.assertFalse(task.getTrustSitelists(), "Wrong input location flag")
self.assertFalse(task.inputRunWhitelist(), "Wrong run white list")
# test workload step stuff
self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'cmsRun2', 'cmsRun3', 'logArch1', 'stageOut1'])
self.assertEqual(task.getTopStepName(), 'cmsRun1')
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertFalse(task.getInputStep(), "Wrong input step")
outModsAndDsets = task.listOutputDatasetsAndModules()
outMods = sorted([elem['outputModule'] for elem in outModsAndDsets])
outDsets = sorted([elem['outputDataset'] for elem in outModsAndDsets])
self.assertEqual(outMods, ['AODSIMoutput', 'RECOSIMoutput'], "Wrong output modules")
self.assertTrue('/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterD-START70_V4-v1/AODSIM' in outDsets)
self.assertTrue('/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterC-START70_V4-v1/GEN-SIM-RECO' in outDsets)
self.assertEqual(task.getSwVersion(), 'CMSSW_7_0_0_pre12')
self.assertEqual(task.getScramArch(), 'slc5_amd64_gcc481')
step = task.getStep("cmsRun1")
self.assertFalse(step.data.tree.parent)
self.assertFalse(getattr(step.data.input, 'inputStepName', None))
self.assertFalse(getattr(step.data.input, 'inputOutputModule', None))
self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName, 'FilterA')
self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier, 'GEN-SIM')
self.assertFalse(step.data.output.keep)
self.assertEqual(sorted(step.data.tree.childNames), ['cmsRun2', 'logArch1', 'stageOut1'])
self.assertEqual(step.data.application.setup.cmsswVersion, 'CMSSW_7_0_0_pre12')
self.assertEqual(step.data.application.setup.scramArch, 'slc5_amd64_gcc481')
self.assertEqual(step.data.application.configuration.arguments.globalTag, 'START70_V4::All')
step = task.getStep("cmsRun2")
self.assertEqual(step.data.tree.parent, "cmsRun1")
self.assertEqual(step.data.input.inputStepName, 'cmsRun1')
self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName, 'FilterB')
self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier, 'GEN-SIM-RAW')
self.assertFalse(step.data.output.keep)
self.assertEqual(step.data.tree.childNames, ["cmsRun3"])
self.assertEqual(step.data.application.setup.cmsswVersion, 'CMSSW_7_0_0_pre11')
self.assertEqual(step.data.application.setup.scramArch, 'slc5_amd64_gcc481')
self.assertEqual(step.data.application.configuration.arguments.globalTag, 'START70_V4::All')
step = task.getStep("cmsRun3")
self.assertEqual(step.data.tree.parent, "cmsRun2")
self.assertEqual(step.data.input.inputStepName, 'cmsRun2')
self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
self.assertEqual(step.data.output.modules.RECOSIMoutput.filterName, 'FilterC')
self.assertEqual(step.data.output.modules.AODSIMoutput.filterName, 'FilterD')
self.assertEqual(step.data.output.modules.RECOSIMoutput.dataTier, 'GEN-SIM-RECO')
self.assertEqual(step.data.output.modules.AODSIMoutput.dataTier, 'AODSIM')
self.assertTrue(step.data.output.keep)
self.assertFalse(step.data.tree.childNames)
self.assertEqual(step.data.application.setup.cmsswVersion, 'CMSSW_7_0_0_pre11')
self.assertEqual(step.data.application.setup.scramArch, 'slc5_amd64_gcc481')
self.assertEqual(step.data.application.configuration.arguments.globalTag, 'START70_V4::All')
return
def testStepChainSingleStep(self):
"""
Build a StepChain single step, reading AODSIM and producing MINIAODSIM
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
request = {
"Campaign": "TaskForceUnitTest",
"CMSSWVersion": "CMSSW_7_5_0",
"ScramArch": "slc6_amd64_gcc491",
"DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
"GlobalTag": "PHYS14_25_V3",
"AcquisitionEra": "SingleStep",
"ProcessingString": "UnitTest_StepChain",
"ProcessingVersion": 3,
"PrepID": "MainStep",
"CouchURL": os.environ["COUCHURL"],
"CouchDBName": "stepchain_t",
"Memory": 3500,
"SizePerEvent": 2600,
"TimePerEvent": 26.5,
"Step1": {
"ConfigCacheID": injectStepChainConfigSingle(self.configDatabase),
"GlobalTag": "PHYS14_25_V44",
"InputDataset": "/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/RunIISpring15DR74-Asympt50ns_MCRUN2_74_V9A-v1/AODSIM",
"SplittingAlgo": "EventAwareLumiBased",
"EventsPerJob": 500,
"StepName": "StepMini"},
"StepChain": 1
}
testArguments.update(request)
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload, "StepMini", "GravWhatever", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "processing")
self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
self.assertEqual(testWorkload.getAcquisitionEra(), "SingleStep")
self.assertEqual(testWorkload.getProcessingString(), "UnitTest_StepChain")
self.assertEqual(testWorkload.getProcessingVersion(), 3)
self.assertEqual(testWorkload.getPrepID(), "MainStep")
self.assertEqual(sorted(testWorkload.getCMSSWVersions()), ['CMSSW_7_5_0'])
self.assertEqual(testWorkload.data.policies.start.policyName, "Block")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 2)
self.assertEqual(sorted(tasks), ['LogCollectForStepMini', 'StepMini'])
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.taskType(), "Processing", "Wrong task type")
splitParams = task.jobSplittingParameters()
self.assertEqual(splitParams['algorithm'], "EventAwareLumiBased", "Wrong job splitting algo")
self.assertEqual(splitParams['events_per_job'], 500)
self.assertTrue(splitParams['performance']['timePerEvent'] > 26.4)
self.assertTrue(splitParams['performance']['sizePerEvent'] > 2599)
self.assertTrue(splitParams['performance']['memoryRequirement'] == 3500)
# test workload step stuff
self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'stageOut1'])
self.assertEqual(task.getTopStepName(), 'cmsRun1')
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertFalse(task.getInputStep(), "Wrong input step")
outModsAndDsets = task.listOutputDatasetsAndModules()[0]
self.assertEqual(outModsAndDsets['outputModule'], 'MINIAODSIMoutput')
self.assertEqual(outModsAndDsets['outputDataset'],
'/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/SingleStep-UnitTest_StepChain-v3/MINIAODSIM')
self.assertEqual(task.getSwVersion(), 'CMSSW_7_5_0')
self.assertEqual(task.getScramArch(), 'slc6_amd64_gcc491')
step = task.getStep("cmsRun1")
self.assertEqual(step.data.application.configuration.arguments.globalTag, 'PHYS14_25_V44')
return
def testStepMapping(self):
"""
Build a mapping of steps, input and output modules
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
testArguments.update(deepcopy(REQUEST))
testArguments['StepChain'] = 1
testArguments['Step1']['ConfigCacheID'] = injectStepChainConfigSingle(
self.configDatabase)
testArguments.update({
"CMSSWVersion": "CMSSW_8_0_17",
"ScramArch": "slc6_amd64_gcc530",
"StepChain": 4
})
# Create a new DIGI step in Step3 and shift Step3 to Step4
testArguments['Step4'] = deepcopy(testArguments['Step3'])
testArguments['Step3'] = {
"GlobalTag": "GT-Step3",
"InputFromOutputModule": "RAWSIMoutput",
"InputStep": "GENSIM",
"StepName": "DIGI2"
}
configDocs = injectStepChainConfigMC(self.configDatabase)
for s in ['Step1', 'Step2', 'Step3', 'Step4']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
testArguments[s]['KeepOutput'] = False
# these are the inverse...
testArguments['Step3']['ConfigCacheID'] = configDocs['Step4']
testArguments['Step4']['ConfigCacheID'] = configDocs['Step3']
expectedTasks = set([
'GENSIM', 'RECOMergeAODSIMoutput', 'RECOMergeRECOSIMoutput',
'RECOAODSIMoutputMergeLogCollect',
'RECORECOSIMoutputMergeLogCollect',
'RECOCleanupUnmergedAODSIMoutput',
'RECOCleanupUnmergedRECOSIMoutput'
])
expectedSteps = set([
'cmsRun1', 'cmsRun2', 'cmsRun3', 'cmsRun4', 'stageOut1', 'logArch1'
])
factory = StepChainWorkloadFactory()
# Raise exception because the last Step cannot have KeepOutput=False
self.assertRaises(WMSpecFactoryException,
factory.factoryWorkloadConstruction, "TestWorkload",
testArguments)
testArguments['Step4']['KeepOutput'] = True
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
self.assertEqual(len(testWorkload.listAllTaskNames()),
len(expectedTasks))
self.assertEqual(set(testWorkload.listAllTaskNames()), expectedTasks)
task = testWorkload.getTask('GENSIM')
self.assertEqual(set(task.listAllStepNames()), expectedSteps)
step1 = task.getStep('cmsRun1')
stepInputSection = step1.data.input.dictionary_()
self.assertFalse('inputStepName' in stepInputSection)
self.assertEqual(set(step1.data.output.modules.dictionary_().keys()),
set(['RAWSIMoutput']))
self.assertEqual(
step1.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'],
'GEN-SIM')
step2 = task.getStep('cmsRun2')
stepInputSection = step2.data.input.dictionary_()
self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun1')
self.assertTrue(set(stepInputSection['inputOutputModule']),
'RAWSIMoutput')
self.assertEqual(set(step2.data.output.modules.dictionary_().keys()),
set(['RAWSIMoutput']))
self.assertEqual(
step2.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'],
'GEN-SIM-RAW')
step3 = task.getStep('cmsRun3')
stepInputSection = step3.data.input.dictionary_()
self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun1')
self.assertTrue(set(stepInputSection['inputOutputModule']),
'RAWSIMoutput')
self.assertEqual(set(step3.data.output.modules.dictionary_().keys()),
set(['RAWSIMoutput']))
self.assertEqual(
step3.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'],
'GEN-SIM-RAW')
step4 = task.getStep('cmsRun4')
stepInputSection = step4.data.input.dictionary_()
self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun2')
self.assertTrue(set(stepInputSection['inputOutputModule']),
'RAWSIMoutput')
self.assertEqual(set(step4.data.output.modules.dictionary_().keys()),
set(['AODSIMoutput', 'RECOSIMoutput']))
self.assertEqual(
step4.data.output.modules.AODSIMoutput.dictionary_()['dataTier'],
'AODSIM')
def testStepChainMC(self):
"""
Build a StepChain workload starting from scratch
"""
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments = request['createRequest']
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "stepchain_t"
})
configDocs = injectStepChainConfigMC(self.configDatabase)
for s in ['Step1', 'Step2', 'Step3']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
factory = StepChainWorkloadFactory()
# test that we cannot stage out different samples with the same output module
self.assertRaises(WMSpecFactoryException,
factory.factoryWorkloadConstruction, "TestWorkload",
testArguments)
testArguments['Step2']['KeepOutput'] = False
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"ProdMinBias",
"MCFakeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "production")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_0_0_pre11")
self.assertEqual(testWorkload.getProcessingString(), "START70_V4")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertEqual(testWorkload.getPrepID(), "Step-00")
self.assertEqual(sorted(testWorkload.getCMSSWVersions()),
['CMSSW_7_0_0_pre11', 'CMSSW_7_0_0_pre12'])
# test workload attributes
self.assertEqual(testWorkload.processingString, "START70_V4")
self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_0_0_pre11")
self.assertEqual(testWorkload.processingVersion, 1)
self.assertFalse(testWorkload.lumiList, "Wrong lumiList")
self.assertEqual(testWorkload.data.policies.start.policyName,
"MonteCarlo")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 10)
for t in [
'ProdMinBias', 'ProdMinBiasMergeRAWSIMoutput',
'RECOPROD1MergeAODSIMoutput', 'RECOPROD1MergeRECOSIMoutput'
]:
self.assertTrue(t in tasks, "Wrong task name")
self.assertFalse('ProdMinBiasMergeAODSIMoutput' in tasks,
"Wrong task name")
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "ProdMinBias")
self.assertEqual(task.getPathName(), "/TestWorkload/ProdMinBias")
self.assertEqual(task.taskType(), "Production", "Wrong task type")
splitParams = task.jobSplittingParameters()
self.assertEqual(splitParams['algorithm'], "EventBased",
"Wrong job splitting algo")
self.assertEqual(splitParams['events_per_job'], 150)
self.assertEqual(splitParams['events_per_lumi'], 50)
self.assertFalse(splitParams['lheInputFiles'], "Wrong LHE flag")
self.assertTrue(splitParams['performance']['timePerEvent'] > 4.75)
self.assertTrue(splitParams['performance']['sizePerEvent'] > 1233)
self.assertTrue(
splitParams['performance']['memoryRequirement'] == 2400)
self.assertFalse(task.getTrustSitelists().get('trustlists'),
"Wrong input location flag")
self.assertFalse(task.inputRunWhitelist(), "Wrong run white list")
# test workload step stuff
self.assertEqual(
sorted(task.listAllStepNames()),
['cmsRun1', 'cmsRun2', 'cmsRun3', 'logArch1', 'stageOut1'])
self.assertEqual(task.getTopStepName(), 'cmsRun1')
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertFalse(task.getInputStep(), "Wrong input step")
outModsAndDsets = task.listOutputDatasetsAndModules()
outMods = set([elem['outputModule'] for elem in outModsAndDsets])
outDsets = [elem['outputDataset'] for elem in outModsAndDsets]
self.assertEqual(
outMods, set(['RAWSIMoutput', 'AODSIMoutput', 'RECOSIMoutput']),
"Wrong output modules")
self.assertTrue(
'/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterA-START70_V4-v1/GEN-SIM'
in outDsets)
self.assertTrue(
'/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterD-START70_V4-v1/AODSIM'
in outDsets)
self.assertTrue(
'/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterC-START70_V4-v1/GEN-SIM-RECO'
in outDsets)
self.assertEqual(task.getSwVersion(), 'CMSSW_7_0_0_pre12')
self.assertEqual(task.getScramArch(), 'slc5_amd64_gcc481')
step = task.getStep("cmsRun1")
self.assertFalse(step.data.tree.parent)
self.assertFalse(getattr(step.data.input, 'inputStepName', None))
self.assertFalse(getattr(step.data.input, 'inputOutputModule', None))
self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName,
'FilterA')
self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier,
'GEN-SIM')
self.assertTrue(step.data.output.keep)
self.assertEqual(sorted(step.data.tree.childNames),
['cmsRun2', 'logArch1', 'stageOut1'])
self.assertEqual(step.data.application.setup.cmsswVersion,
'CMSSW_7_0_0_pre12')
self.assertEqual(step.data.application.setup.scramArch,
'slc5_amd64_gcc481')
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
'START70_V4::All')
step = task.getStep("cmsRun2")
self.assertEqual(step.data.tree.parent, "cmsRun1")
self.assertEqual(step.data.input.inputStepName, 'cmsRun1')
self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName,
'FilterB')
self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier,
'GEN-SIM-RAW')
self.assertFalse(step.data.output.keep)
self.assertEqual(step.data.tree.childNames, ["cmsRun3"])
self.assertEqual(step.data.application.setup.cmsswVersion,
'CMSSW_7_0_0_pre11')
self.assertEqual(step.data.application.setup.scramArch,
'slc5_amd64_gcc481')
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
'START70_V4::All')
step = task.getStep("cmsRun3")
self.assertEqual(step.data.tree.parent, "cmsRun2")
self.assertEqual(step.data.input.inputStepName, 'cmsRun2')
self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
self.assertEqual(step.data.output.modules.RECOSIMoutput.filterName,
'FilterC')
self.assertEqual(step.data.output.modules.AODSIMoutput.filterName,
'FilterD')
self.assertEqual(step.data.output.modules.RECOSIMoutput.dataTier,
'GEN-SIM-RECO')
self.assertEqual(step.data.output.modules.AODSIMoutput.dataTier,
'AODSIM')
self.assertTrue(step.data.output.keep)
self.assertFalse(step.data.tree.childNames)
self.assertEqual(step.data.application.setup.cmsswVersion,
'CMSSW_7_0_0_pre11')
self.assertEqual(step.data.application.setup.scramArch,
'slc5_amd64_gcc481')
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
'START70_V4::All')
return
def testStepChainMC(self):
"""
Build a StepChain workload starting from scratch
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
testArguments.update(deepcopy(REQUEST))
configDocs = injectStepChainConfigMC(self.configDatabase)
for s in ['Step1', 'Step2', 'Step3']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
testArguments[s]['KeepOutput'] = True
factory = StepChainWorkloadFactory()
# test that we cannot stage out different samples with the same output module
self.assertRaises(WMSpecFactoryException,
factory.factoryWorkloadConstruction, "TestWorkload",
testArguments)
testArguments['Step2']['KeepOutput'] = False
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"GENSIM",
"MCFakeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "production")
self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
self.assertEqual(testWorkload.getAcquisitionEra(),
"AcquisitionEra_StepChain")
self.assertEqual(testWorkload.getProcessingString(),
"ProcessingString_StepChain")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertEqual(testWorkload.getPrepID(), "PREP-StepChain")
self.assertItemsEqual(testWorkload.getCMSSWVersions(),
['CMSSW_7_1_25_patch2', 'CMSSW_8_0_21'])
self.assertFalse(testWorkload.getLumiList(), "Wrong lumiList")
self.assertEqual(testWorkload.data.policies.start.policyName,
"MonteCarlo")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 10)
for t in [
'GENSIM', 'GENSIMMergeRAWSIMoutput', 'RECOMergeAODSIMoutput',
'RECOMergeRECOSIMoutput', 'GENSIMRAWSIMoutputMergeLogCollect',
'RECOAODSIMoutputMergeLogCollect',
'RECORECOSIMoutputMergeLogCollect'
]:
self.assertTrue(t in tasks, "Wrong task name")
self.assertFalse('ProdMinBiasMergeAODSIMoutput' in tasks,
"Wrong task name")
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "GENSIM")
self.assertEqual(task.getPathName(), "/TestWorkload/GENSIM")
self.assertEqual(task.taskType(), "Production", "Wrong task type")
splitParams = task.jobSplittingParameters()
self.assertEqual(splitParams['algorithm'], "EventBased",
"Wrong job splitting algo")
self.assertEqual(splitParams['events_per_job'], 200)
self.assertEqual(splitParams['events_per_lumi'], 100)
self.assertFalse(splitParams['lheInputFiles'], "Wrong LHE flag")
self.assertTrue(splitParams['performance']['timePerEvent'] >= 144)
self.assertTrue(splitParams['performance']['sizePerEvent'] >= 512)
self.assertTrue(
splitParams['performance']['memoryRequirement'] == 3500)
self.assertFalse(task.getTrustSitelists().get('trustlists'),
"Wrong input location flag")
self.assertFalse(task.inputRunWhitelist(), "Wrong run white list")
# test workload step stuff
self.assertEqual(
sorted(task.listAllStepNames()),
['cmsRun1', 'cmsRun2', 'cmsRun3', 'logArch1', 'stageOut1'])
self.assertEqual(task.getTopStepName(), 'cmsRun1')
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertFalse(task.getInputStep(), "Wrong input step")
outModsAndDsets = task.listOutputDatasetsAndModules()
outMods = set([elem['outputModule'] for elem in outModsAndDsets])
outDsets = [elem['outputDataset'] for elem in outModsAndDsets]
self.assertEqual(
outMods, set(['RAWSIMoutput', 'AODSIMoutput', 'RECOSIMoutput']),
"Wrong output modules")
self.assertTrue(
'/PrimaryDataset-StepChain/AcquisitionEra_StepChain-FilterA-ProcessingString_StepChain-v1/GEN-SIM'
in outDsets)
self.assertTrue(
'/PrimaryDataset-StepChain/AcquisitionEra_StepChain-FilterD-ProcessingString_StepChain-v1/AODSIM'
in outDsets)
self.assertTrue(
'/PrimaryDataset-StepChain/AcquisitionEra_StepChain-FilterC-ProcessingString_StepChain-v1/GEN-SIM-RECO'
in outDsets)
self.assertEqual(task.getSwVersion(),
testArguments['Step1']["CMSSWVersion"])
self.assertEqual(task.getScramArch(),
testArguments['Step1']["ScramArch"])
step = task.getStep("cmsRun1")
self.assertFalse(step.data.tree.parent)
self.assertFalse(getattr(step.data.input, 'inputStepName', None))
self.assertFalse(getattr(step.data.input, 'inputOutputModule', None))
self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName,
'FilterA')
self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier,
'GEN-SIM')
self.assertTrue(step.data.output.keep)
self.assertEqual(sorted(step.data.tree.childNames),
['cmsRun2', 'logArch1', 'stageOut1'])
self.assertEqual(step.data.application.setup.cmsswVersion,
testArguments['Step1']["CMSSWVersion"])
self.assertEqual(step.data.application.setup.scramArch,
testArguments['Step1']["ScramArch"])
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
testArguments['Step1']["GlobalTag"])
step = task.getStep("cmsRun2")
self.assertEqual(step.data.tree.parent, "cmsRun1")
self.assertEqual(step.data.input.inputStepName, 'cmsRun1')
self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName,
'FilterB')
self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier,
'GEN-SIM-RAW')
self.assertFalse(step.data.output.keep)
self.assertEqual(step.data.tree.childNames, ["cmsRun3"])
self.assertEqual(step.data.application.setup.cmsswVersion,
testArguments['Step2']["CMSSWVersion"])
self.assertEqual(step.data.application.setup.scramArch,
testArguments['Step2']["ScramArch"])
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
testArguments['Step2']["GlobalTag"])
step = task.getStep("cmsRun3")
self.assertEqual(step.data.tree.parent, "cmsRun2")
self.assertEqual(step.data.input.inputStepName, 'cmsRun2')
self.assertEqual(step.data.input.inputOutputModule, 'PREMIXRAWoutput')
self.assertEqual(step.data.output.modules.RECOSIMoutput.filterName,
'FilterC')
self.assertEqual(step.data.output.modules.AODSIMoutput.filterName,
'FilterD')
self.assertEqual(step.data.output.modules.RECOSIMoutput.dataTier,
'GEN-SIM-RECO')
self.assertEqual(step.data.output.modules.AODSIMoutput.dataTier,
'AODSIM')
self.assertTrue(step.data.output.keep)
self.assertFalse(step.data.tree.childNames)
self.assertEqual(step.data.application.setup.cmsswVersion,
testArguments['Step3']["CMSSWVersion"])
self.assertEqual(step.data.application.setup.scramArch,
testArguments['Step3']["ScramArch"])
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
testArguments['Step3']["GlobalTag"])
# test merge stuff
task = testWorkload.getTaskByPath(
'/TestWorkload/GENSIM/GENSIMMergeRAWSIMoutput')
self.assertEqual(task.taskType(), "Merge")
self.assertEqual(task.getSwVersion(),
testArguments['Step1']["CMSSWVersion"])
self.assertEqual(task.getScramArch(),
testArguments['Step1']["ScramArch"])
task = testWorkload.getTaskByPath(
'/TestWorkload/GENSIM/RECOMergeAODSIMoutput')
self.assertEqual(task.taskType(), "Merge")
self.assertEqual(task.getSwVersion(),
testArguments['Step3']["CMSSWVersion"])
self.assertEqual(task.getScramArch(),
testArguments['Step3']["ScramArch"])
# test logCollect stuff
task = testWorkload.getTaskByPath(
'/TestWorkload/GENSIM/GENSIMMergeRAWSIMoutput/GENSIMRAWSIMoutputMergeLogCollect'
)
self.assertEqual(task.taskType(), "LogCollect")
step = task.getStep("logCollect1")
self.assertEqual(step.data.application.setup.cmsswVersion,
testArguments['Step1']["CMSSWVersion"])
self.assertEqual(step.data.application.setup.scramArch,
testArguments['Step1']["ScramArch"])
task = testWorkload.getTaskByPath(
'/TestWorkload/GENSIM/RECOMergeAODSIMoutput/RECOAODSIMoutputMergeLogCollect'
)
self.assertEqual(task.taskType(), "LogCollect")
step = task.getStep("logCollect1")
self.assertEqual(step.data.application.setup.cmsswVersion,
testArguments['Step3']["CMSSWVersion"])
self.assertEqual(step.data.application.setup.scramArch,
testArguments['Step3']["ScramArch"])
return
def testStepChainSingleStep(self):
"""
Build a StepChain with a single step and no input dataset
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
testArguments.update(deepcopy(REQUEST))
testArguments.pop("Step2")
testArguments.pop("Step3")
testArguments['StepChain'] = 1
testArguments['Step1']['ConfigCacheID'] = injectStepChainConfigSingle(
self.configDatabase)
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"GENSIM",
"GravWhatever",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "production")
self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
self.assertEqual(testWorkload.getAcquisitionEra(),
"AcquisitionEra_StepChain")
self.assertEqual(testWorkload.getProcessingString(),
"ProcessingString_StepChain")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertEqual(testWorkload.getPrepID(), "PREP-StepChain")
self.assertEqual(sorted(testWorkload.getCMSSWVersions()),
['CMSSW_7_1_25_patch2'])
self.assertEqual(testWorkload.data.policies.start.policyName,
"MonteCarlo")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 4)
self.assertTrue('GENSIMMergeMINIAODSIMoutput' in tasks)
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.taskType(), "Production", "Wrong task type")
splitParams = task.jobSplittingParameters()
self.assertEqual(splitParams['algorithm'], "EventBased",
"Wrong job splitting algo")
self.assertEqual(splitParams['events_per_job'], 200)
self.assertEqual(splitParams['events_per_lumi'], 100)
self.assertTrue(splitParams['performance']['timePerEvent'] >= 144)
self.assertTrue(splitParams['performance']['sizePerEvent'] >= 512)
self.assertTrue(
splitParams['performance']['memoryRequirement'] == 3500)
# test workload step stuff
self.assertEqual(sorted(task.listAllStepNames()),
['cmsRun1', 'logArch1', 'stageOut1'])
self.assertEqual(task.getTopStepName(), 'cmsRun1')
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertFalse(task.getInputStep(), "Wrong input step")
outModsAndDsets = task.listOutputDatasetsAndModules()[0]
self.assertEqual(outModsAndDsets['outputModule'], 'MINIAODSIMoutput')
self.assertEqual(
outModsAndDsets['outputDataset'],
'/PrimaryDataset-StepChain/AcquisitionEra_StepChain-ProcessingString_StepChain-v1/MINIAODSIM'
)
self.assertEqual(task.getSwVersion(), 'CMSSW_7_1_25_patch2')
self.assertItemsEqual(task.getScramArch(), ['slc6_amd64_gcc481'])
step = task.getStep("cmsRun1")
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
'GT-Step1')
return
def test3StepsMemCoresSettingsB(self):
"""
_test3StepsMemCoresSettingsB_
Mix Multicore settings at both step and request level and make sure they
are properly propagated to each step. Three steps in the task.
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
configDocs = injectStepChainConfigMC(self.configDatabase)
testArguments.update(getThreeStepsOverride())
for s in ['Step1', 'Step2', 'Step3']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
testArguments['Step2']['KeepOutput'] = False
# Test Multicore/Memory settings at step level **only**
testArguments['Step1']["Multicore"] = 2
testArguments['Step3']["Multicore"] = 4
testArguments['Step1']["Memory"] = 2200.0
testArguments['Step3']["Memory"] = 4400.0
testArguments['Step1']["EventStreams"] = 2
testArguments['Step3']["EventStreams"] = 4
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('cmsRun2', 'stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1,
"%s should have 1 core" % step)
self.assertEqual(
prodTask.getStepHelper('cmsRun1').getNumberOfCores(),
testArguments['Step1']["Multicore"])
self.assertEqual(
prodTask.getStepHelper('cmsRun3').getNumberOfCores(),
testArguments['Step3']["Multicore"])
self.assertEqual(
prodTask.getStepHelper('cmsRun1').getNumberOfStreams(),
testArguments['Step1']["EventStreams"])
self.assertEqual(
prodTask.getStepHelper('cmsRun3').getNumberOfStreams(),
testArguments['Step3']["EventStreams"])
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'],
testArguments["Memory"])
# Test mix of Multicore/Memory settings at both top and step level
testArguments["Multicore"] = 3
testArguments['Step1']["Multicore"] = 2
testArguments['Step3']["Multicore"] = 4
testArguments["Memory"] = 3300.0
testArguments['Step1']["Memory"] = 2200.0
testArguments['Step3']["Memory"] = 4400.0
testArguments["EventStreams"] = 6
testArguments['Step1']["EventStreams"] = 4
testArguments['Step3']["EventStreams"] = 8
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload2", testArguments)
prodTask = testWorkload.getTask('StepOne')
for step in ('stageOut1', 'logArch1'):
stepHelper = prodTask.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1,
"%s should have 1 core" % step)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
self.assertEqual(
prodTask.getStepHelper('cmsRun1').getNumberOfCores(),
testArguments['Step1']["Multicore"])
self.assertEqual(
prodTask.getStepHelper('cmsRun2').getNumberOfCores(),
testArguments["Multicore"])
self.assertEqual(
prodTask.getStepHelper('cmsRun3').getNumberOfCores(),
testArguments['Step3']["Multicore"])
self.assertEqual(
prodTask.getStepHelper('cmsRun1').getNumberOfStreams(),
testArguments['Step1']["EventStreams"])
self.assertEqual(
prodTask.getStepHelper('cmsRun2').getNumberOfStreams(),
testArguments["EventStreams"])
self.assertEqual(
prodTask.getStepHelper('cmsRun3').getNumberOfStreams(),
testArguments['Step3']["EventStreams"])
perfParams = prodTask.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'],
testArguments["Memory"])
return
def testStepMapping(self):
"""
Build a mapping of steps, input and output modules
"""
factory = StepChainWorkloadFactory()
request = json.load(open(self.jsonTemplate))
testArguments = request['createRequest']
# Create a new DIGI step in Step3 and shift Step3 to Step4
testArguments['Step4'] = copy(testArguments['Step3'])
testArguments['Step3'] = {"GlobalTag": "START70_V4::All",
"InputFromOutputModule": "RAWSIMoutput",
"InputStep": "ProdMinBias",
"StepName": "DIGIPROD2"}
testArguments['StepChain'] = 4
testArguments.update({"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "stepchain_t"})
configDocs = injectStepChainConfigMC(self.configDatabase)
for s in ['Step1', 'Step2', 'Step3', 'Step4']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
testArguments[s]['KeepOutput'] = False
# docs are in the wrong order for this case
testArguments['Step3']['ConfigCacheID'] = configDocs['Step4']
testArguments['Step4']['ConfigCacheID'] = configDocs['Step3']
expectedTasks = set(['ProdMinBias', 'RECOPROD1MergeAODSIMoutput', 'RECOPROD1MergeRECOSIMoutput',
'RECOPROD1AODSIMoutputMergeLogCollect', 'RECOPROD1RECOSIMoutputMergeLogCollect',
'RECOPROD1CleanupUnmergedAODSIMoutput', 'RECOPROD1CleanupUnmergedRECOSIMoutput'])
expectedSteps = set(['cmsRun1', 'cmsRun2', 'cmsRun3', 'cmsRun4', 'stageOut1', 'logArch1'])
self.assertRaises(WMSpecFactoryException, factory.factoryWorkloadConstruction,
"TestWorkload", testArguments)
testArguments['Step4']['KeepOutput'] = True
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
self.assertEqual(len(testWorkload.listAllTaskNames()), len(expectedTasks))
self.assertEqual(set(testWorkload.listAllTaskNames()), expectedTasks)
task = testWorkload.getTask('ProdMinBias')
self.assertEqual(set(task.listAllStepNames()), expectedSteps)
step1 = task.getStep('cmsRun1')
stepInputSection = step1.data.input.dictionary_()
self.assertFalse('inputStepName' in stepInputSection)
self.assertEqual(set(step1.data.output.modules.dictionary_().keys()), set(['RAWSIMoutput']))
self.assertEqual(step1.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'], 'GEN-SIM')
step2 = task.getStep('cmsRun2')
stepInputSection = step2.data.input.dictionary_()
self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun1')
self.assertTrue(set(stepInputSection['inputOutputModule']), 'RAWSIMoutput')
self.assertEqual(set(step2.data.output.modules.dictionary_().keys()), set(['RAWSIMoutput']))
self.assertEqual(step2.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'], 'GEN-SIM-RAW')
step3 = task.getStep('cmsRun3')
stepInputSection = step3.data.input.dictionary_()
self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun1')
self.assertTrue(set(stepInputSection['inputOutputModule']), 'RAWSIMoutput')
self.assertEqual(set(step3.data.output.modules.dictionary_().keys()), set(['RAWSIMoutput']))
self.assertEqual(step3.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'], 'GEN-SIM-RAW')
step4 = task.getStep('cmsRun4')
stepInputSection = step4.data.input.dictionary_()
self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun2')
self.assertTrue(set(stepInputSection['inputOutputModule']), 'RAWSIMoutput')
self.assertEqual(set(step4.data.output.modules.dictionary_().keys()), set(['AODSIMoutput', 'RECOSIMoutput']))
self.assertEqual(step4.data.output.modules.AODSIMoutput.dictionary_()['dataTier'], 'AODSIM')
from __future__ import division, print_function
import unittest
from copy import deepcopy
from Utils.PythonVersion import PY3
from WMCore.ReqMgr.DataStructs.Request import initialize_clone, RequestInfo
from WMCore.WMSpec.StdSpecs.ReReco import ReRecoWorkloadFactory
from WMCore.WMSpec.StdSpecs.StepChain import StepChainWorkloadFactory
from WMCore.WMSpec.StdSpecs.TaskChain import TaskChainWorkloadFactory
from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException
### Spec arguments definition with only key and its default value
RERECO_ARGS = ReRecoWorkloadFactory.getWorkloadCreateArgs()
STEPCHAIN_ARGS = StepChainWorkloadFactory.getWorkloadCreateArgs()
TASKCHAIN_ARGS = TaskChainWorkloadFactory.getWorkloadCreateArgs()
# inner Task/Step definition
STEP_ARGS = StepChainWorkloadFactory.getChainCreateArgs()
TASK_ARGS = TaskChainWorkloadFactory.getChainCreateArgs()
### Some original request dicts (ones to be cloned from)
rerecoOriginalArgs = {
'Memory': 234,
'SkimName1': 'skim_2017',
'SkimInput1': 'RECOoutput',
'Skim1ConfigCacheID': 'abcdef',
'IncludeParents': True,
'TimePerEvent': 1.2,
'RequestType': 'ReReco',
'RequestName': 'test_rereco'
def testStepChainReDigi(self):
"""
Build a StepChain workload with input dataset in the first step
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
testArguments.update(deepcopy(REQUEST))
configDocs = injectStepChainConfigMC(self.configDatabase)
for s in ['Step1', 'Step2', 'Step3']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
testArguments['Step1'] = deepcopy(testArguments.pop('Step2'))
testArguments['Step2'] = deepcopy(testArguments.pop('Step3'))
testArguments['StepChain'] = 2
testArguments['Step1'].pop('InputFromOutputModule')
testArguments['Step1'].pop('InputStep')
testArguments['Step1'].update({
'KeepOutput':
False,
'InputDataset':
'/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM',
'BlockWhitelist': [
"/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM#3a09df90-5593-11e4-bd05-003048f0e3f4",
"/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM#3a8b15e6-54e0-11e4-afc7-003048f0e3f4"
]
})
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"DIGI",
"Block",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "processing")
self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
self.assertEqual(testWorkload.getAcquisitionEra(),
"AcquisitionEra_StepChain")
self.assertEqual(testWorkload.getProcessingString(),
"ProcessingString_StepChain")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertEqual(testWorkload.getPrepID(), "PREP-StepChain")
self.assertEqual(sorted(testWorkload.getCMSSWVersions()),
['CMSSW_8_0_21'])
self.assertFalse(testWorkload.getLumiList(), "Wrong lumiList")
self.assertEqual(testWorkload.data.policies.start.policyName, "Block")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 7)
for t in [
'DIGI', 'RECOMergeAODSIMoutput', 'RECOMergeRECOSIMoutput',
'RECOAODSIMoutputMergeLogCollect',
'RECORECOSIMoutputMergeLogCollect',
'RECOCleanupUnmergedAODSIMoutput',
'RECOCleanupUnmergedRECOSIMoutput'
]:
self.assertTrue(t in tasks, "Wrong task name")
self.assertFalse('ProdMinBiasMergeAODSIMoutput' in tasks,
"Wrong task name")
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "DIGI")
self.assertEqual(task.getPathName(), "/TestWorkload/DIGI")
self.assertEqual(task.taskType(), "Processing", "Wrong task type")
splitParams = task.jobSplittingParameters()
self.assertEqual(splitParams['algorithm'], "EventAwareLumiBased",
"Wrong job splitting algo")
self.assertEqual(splitParams['events_per_job'], 200)
self.assertTrue(splitParams['performance']['timePerEvent'] >= 144)
self.assertTrue(splitParams['performance']['sizePerEvent'] >= 512)
self.assertTrue(
splitParams['performance']['memoryRequirement'] == 3500)
self.assertEqual(
task.getInputDatasetPath(),
'/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM'
)
self.assertTrue(
'/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM#3a09df90-5593-11e4-bd05-003048f0e3f4'
in task.inputBlockWhitelist())
self.assertTrue(
'/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM#3a8b15e6-54e0-11e4-afc7-003048f0e3f4'
in task.inputBlockWhitelist())
# test workload step stuff
self.assertEqual(sorted(task.listAllStepNames()),
['cmsRun1', 'cmsRun2', 'logArch1', 'stageOut1'])
self.assertEqual(task.getTopStepName(), 'cmsRun1')
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertFalse(task.getInputStep(), "Wrong input step")
outModsAndDsets = task.listOutputDatasetsAndModules()
outMods = set([elem['outputModule'] for elem in outModsAndDsets])
outDsets = [elem['outputDataset'] for elem in outModsAndDsets]
self.assertEqual(outMods, set(['AODSIMoutput', 'RECOSIMoutput']),
"Wrong output modules")
self.assertTrue(
'/PrimaryDataset-StepChain/AcquisitionEra_StepChain-FilterD-ProcessingString_StepChain-v1/AODSIM'
in outDsets)
self.assertTrue(
'/PrimaryDataset-StepChain/AcquisitionEra_StepChain-FilterC-ProcessingString_StepChain-v1/GEN-SIM-RECO'
in outDsets)
return
def testStepChainSingleStep(self):
"""
Build a StepChain single step, reading AODSIM and producing MINIAODSIM
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
request = {
"Campaign": "TaskForceUnitTest",
"CMSSWVersion": "CMSSW_7_5_0",
"ScramArch": "slc6_amd64_gcc491",
"DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
"GlobalTag": "PHYS14_25_V3",
"AcquisitionEra": "SingleStep",
"ProcessingString": "UnitTest_StepChain",
"ProcessingVersion": 3,
"PrepID": "MainStep",
"CouchURL": os.environ["COUCHURL"],
"CouchDBName": "stepchain_t",
"Memory": 3500,
"SizePerEvent": 2600,
"TimePerEvent": 26.5,
"Step1": {
"ConfigCacheID":
injectStepChainConfigSingle(self.configDatabase),
"GlobalTag": "PHYS14_25_V44",
"InputDataset":
"/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/RunIISpring15DR74-Asympt50ns_MCRUN2_74_V9A-v1/AODSIM",
"SplittingAlgo": "EventAwareLumiBased",
"EventsPerJob": 500,
"StepName": "StepMini"
},
"StepChain": 1
}
testArguments.update(request)
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"StepMini",
"GravWhatever",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "processing")
self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
self.assertEqual(testWorkload.getAcquisitionEra(), "SingleStep")
self.assertEqual(testWorkload.getProcessingString(),
"UnitTest_StepChain")
self.assertEqual(testWorkload.getProcessingVersion(), 3)
self.assertEqual(testWorkload.getPrepID(), "MainStep")
self.assertEqual(sorted(testWorkload.getCMSSWVersions()),
['CMSSW_7_5_0'])
self.assertEqual(testWorkload.data.policies.start.policyName, "Block")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 4)
self.assertTrue('StepMiniMergeMINIAODSIMoutput' in tasks)
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.taskType(), "Processing", "Wrong task type")
splitParams = task.jobSplittingParameters()
self.assertEqual(splitParams['algorithm'], "EventAwareLumiBased",
"Wrong job splitting algo")
self.assertEqual(splitParams['events_per_job'], 500)
self.assertTrue(splitParams['performance']['timePerEvent'] > 26.4)
self.assertTrue(splitParams['performance']['sizePerEvent'] > 2599)
self.assertTrue(
splitParams['performance']['memoryRequirement'] == 3500)
# test workload step stuff
self.assertEqual(sorted(task.listAllStepNames()),
['cmsRun1', 'logArch1', 'stageOut1'])
self.assertEqual(task.getTopStepName(), 'cmsRun1')
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertFalse(task.getInputStep(), "Wrong input step")
outModsAndDsets = task.listOutputDatasetsAndModules()[0]
self.assertEqual(outModsAndDsets['outputModule'], 'MINIAODSIMoutput')
self.assertEqual(
outModsAndDsets['outputDataset'],
'/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/SingleStep-UnitTest_StepChain-v3/MINIAODSIM'
)
self.assertEqual(task.getSwVersion(), 'CMSSW_7_5_0')
self.assertEqual(task.getScramArch(), 'slc6_amd64_gcc491')
step = task.getStep("cmsRun1")
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
'PHYS14_25_V44')
return
def testStepChainOutput(self):
"""
Build a StepChain workload defining different processed dataset name among the steps
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
testArguments.update(deepcopy(REQUEST))
configDocs = injectStepChainConfigMC(self.configDatabase)
for s in ['Step1', 'Step2', 'Step3']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
testArguments[s]['AcquisitionEra'] = 'AcqEra_' + s
testArguments[s]['ProcessingString'] = 'ProcStr_' + s
testArguments['Step2']['KeepOutput'] = False
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
# test workload properties
self.assertEqual(testWorkload.getAcquisitionEra(),
"AcquisitionEra_StepChain")
self.assertEqual(testWorkload.getProcessingString(),
"ProcessingString_StepChain")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 10)
for t in [
'GENSIM', 'GENSIMMergeRAWSIMoutput', 'RECOMergeAODSIMoutput',
'RECOMergeRECOSIMoutput', 'GENSIMRAWSIMoutputMergeLogCollect',
'RECOAODSIMoutputMergeLogCollect',
'RECORECOSIMoutputMergeLogCollect'
]:
self.assertTrue(t in tasks, "Wrong task name")
self.assertFalse('ProdMinBiasMergeAODSIMoutput' in tasks,
"Wrong task name")
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "GENSIM")
self.assertEqual(task.getPathName(), "/TestWorkload/GENSIM")
self.assertEqual(task.taskType(), "Production", "Wrong task type")
# test workload and task output data stuff
self.assertEqual(
sorted(task.listAllStepNames()),
['cmsRun1', 'cmsRun2', 'cmsRun3', 'logArch1', 'stageOut1'])
outModsAndDsets = task.listOutputDatasetsAndModules()
outMods = [elem['outputModule'] for elem in outModsAndDsets]
outDsets = [elem['outputDataset'] for elem in outModsAndDsets]
self.assertItemsEqual(
outMods, ['RAWSIMoutput', 'AODSIMoutput', 'RECOSIMoutput'],
"Wrong output modules")
self.assertTrue(
'/PrimaryDataset-StepChain/AcqEra_Step1-FilterA-ProcStr_Step1-v1/GEN-SIM'
in outDsets)
self.assertTrue(
'/PrimaryDataset-StepChain/AcqEra_Step3-FilterD-ProcStr_Step3-v1/AODSIM'
in outDsets)
self.assertTrue(
'/PrimaryDataset-StepChain/AcqEra_Step3-FilterC-ProcStr_Step3-v1/GEN-SIM-RECO'
in outDsets)
step = task.getStepHelper("cmsRun1")
self.assertEqual(step.listOutputModules(), [u'RAWSIMoutput'])
outputData = step.getOutputModule(u'RAWSIMoutput')
self.assertEqual(outputData.dataTier, 'GEN-SIM')
self.assertTrue(outputData.transient)
self.assertTrue(outputData.lfnBase.startswith('/store/unmerged'))
self.assertEqual(outputData.processedDataset,
'AcqEra_Step1-FilterA-ProcStr_Step1-v1')
step = task.getStepHelper("cmsRun2")
self.assertEqual(step.listOutputModules(), [u'RAWSIMoutput'])
outputData = step.getOutputModule(u'RAWSIMoutput')
self.assertEqual(outputData.dataTier, 'GEN-SIM-RAW')
self.assertTrue(outputData.transient)
self.assertTrue(outputData.lfnBase.startswith('/store/unmerged'))
self.assertEqual(outputData.processedDataset,
'AcqEra_Step2-FilterB-ProcStr_Step2-v1')
step = task.getStepHelper("cmsRun3")
self.assertEqual(step.listOutputModules(),
['AODSIMoutput', 'RECOSIMoutput'])
outputData = step.getOutputModule('AODSIMoutput')
self.assertEqual(outputData.dataTier, 'AODSIM')
self.assertTrue(outputData.transient)
self.assertTrue(outputData.lfnBase.startswith('/store/unmerged'))
self.assertEqual(outputData.processedDataset,
'AcqEra_Step3-FilterD-ProcStr_Step3-v1')
outputData = step.getOutputModule('RECOSIMoutput')
self.assertEqual(outputData.dataTier, 'GEN-SIM-RECO')
self.assertTrue(outputData.transient)
self.assertTrue(outputData.lfnBase.startswith('/store/unmerged'))
self.assertEqual(outputData.processedDataset,
'AcqEra_Step3-FilterC-ProcStr_Step3-v1')
# test merge tasks now
task = testWorkload.getTaskByPath(
'/TestWorkload/GENSIM/GENSIMMergeRAWSIMoutput')
self.assertEqual(task.taskType(), "Merge")
step = task.getStepHelper("cmsRun1")
self.assertEqual(step.listOutputModules(), ['Merged'])
outputData = step.getOutputModule('Merged')
self.assertEqual(outputData.dataTier, 'GEN-SIM')
self.assertFalse(outputData.transient)
self.assertTrue(outputData.lfnBase.startswith('/store/data'))
self.assertEqual(outputData.processedDataset,
'AcqEra_Step1-FilterA-ProcStr_Step1-v1')
task = testWorkload.getTaskByPath(
'/TestWorkload/GENSIM/RECOMergeAODSIMoutput')
self.assertEqual(task.taskType(), "Merge")
step = task.getStepHelper("cmsRun1")
self.assertEqual(step.listOutputModules(), ['Merged'])
outputData = step.getOutputModule('Merged')
self.assertEqual(outputData.dataTier, 'AODSIM')
self.assertFalse(outputData.transient)
self.assertTrue(outputData.lfnBase.startswith('/store/data'))
self.assertEqual(outputData.processedDataset,
'AcqEra_Step3-FilterD-ProcStr_Step3-v1')
task = testWorkload.getTaskByPath(
'/TestWorkload/GENSIM/RECOMergeRECOSIMoutput')
self.assertEqual(task.taskType(), "Merge")
step = task.getStepHelper("cmsRun1")
self.assertEqual(step.listOutputModules(), ['Merged'])
outputData = step.getOutputModule('Merged')
self.assertEqual(outputData.dataTier, 'GEN-SIM-RECO')
self.assertFalse(outputData.transient)
self.assertTrue(outputData.lfnBase.startswith('/store/data'))
self.assertEqual(outputData.processedDataset,
'AcqEra_Step3-FilterC-ProcStr_Step3-v1')
return
def testStepMapping(self):
"""
Build a mapping of steps, input and output modules
"""
factory = StepChainWorkloadFactory()
request = json.load(open(self.jsonTemplate))
testArguments = request['createRequest']
# Create a new DIGI step in Step3 and shift Step3 to Step4
testArguments['Step4'] = copy(testArguments['Step3'])
testArguments['Step3'] = {
"GlobalTag": "START70_V4::All",
"InputFromOutputModule": "RAWSIMoutput",
"InputStep": "ProdMinBias",
"StepName": "DIGIPROD2"
}
testArguments['StepChain'] = 4
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "stepchain_t"
})
configDocs = injectStepChainConfigMC(self.configDatabase)
for s in ['Step1', 'Step2', 'Step3', 'Step4']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
testArguments[s]['KeepOutput'] = False
# docs are in the wrong order for this case
testArguments['Step3']['ConfigCacheID'] = configDocs['Step4']
testArguments['Step4']['ConfigCacheID'] = configDocs['Step3']
expectedTasks = set([
'ProdMinBias', 'RECOPROD1MergeAODSIMoutput',
'RECOPROD1MergeRECOSIMoutput',
'RECOPROD1AODSIMoutputMergeLogCollect',
'RECOPROD1RECOSIMoutputMergeLogCollect',
'RECOPROD1CleanupUnmergedAODSIMoutput',
'RECOPROD1CleanupUnmergedRECOSIMoutput'
])
expectedSteps = set([
'cmsRun1', 'cmsRun2', 'cmsRun3', 'cmsRun4', 'stageOut1', 'logArch1'
])
self.assertRaises(WMSpecFactoryException,
factory.factoryWorkloadConstruction, "TestWorkload",
testArguments)
testArguments['Step4']['KeepOutput'] = True
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
self.assertEqual(len(testWorkload.listAllTaskNames()),
len(expectedTasks))
self.assertEqual(set(testWorkload.listAllTaskNames()), expectedTasks)
task = testWorkload.getTask('ProdMinBias')
self.assertEqual(set(task.listAllStepNames()), expectedSteps)
step1 = task.getStep('cmsRun1')
stepInputSection = step1.data.input.dictionary_()
self.assertFalse('inputStepName' in stepInputSection)
self.assertEqual(set(step1.data.output.modules.dictionary_().keys()),
set(['RAWSIMoutput']))
self.assertEqual(
step1.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'],
'GEN-SIM')
step2 = task.getStep('cmsRun2')
stepInputSection = step2.data.input.dictionary_()
self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun1')
self.assertTrue(set(stepInputSection['inputOutputModule']),
'RAWSIMoutput')
self.assertEqual(set(step2.data.output.modules.dictionary_().keys()),
set(['RAWSIMoutput']))
self.assertEqual(
step2.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'],
'GEN-SIM-RAW')
step3 = task.getStep('cmsRun3')
stepInputSection = step3.data.input.dictionary_()
self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun1')
self.assertTrue(set(stepInputSection['inputOutputModule']),
'RAWSIMoutput')
self.assertEqual(set(step3.data.output.modules.dictionary_().keys()),
set(['RAWSIMoutput']))
self.assertEqual(
step3.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'],
'GEN-SIM-RAW')
step4 = task.getStep('cmsRun4')
stepInputSection = step4.data.input.dictionary_()
self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun2')
self.assertTrue(set(stepInputSection['inputOutputModule']),
'RAWSIMoutput')
self.assertEqual(set(step4.data.output.modules.dictionary_().keys()),
set(['AODSIMoutput', 'RECOSIMoutput']))
self.assertEqual(
step4.data.output.modules.AODSIMoutput.dictionary_()['dataTier'],
'AODSIM')
"""
from __future__ import division, print_function
import unittest
from copy import deepcopy
from WMCore.ReqMgr.DataStructs.Request import initialize_clone
from WMCore.WMSpec.StdSpecs.ReReco import ReRecoWorkloadFactory
from WMCore.WMSpec.StdSpecs.StepChain import StepChainWorkloadFactory
from WMCore.WMSpec.StdSpecs.TaskChain import TaskChainWorkloadFactory
from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException
### Spec arguments definition with only key and its default value
RERECO_ARGS = ReRecoWorkloadFactory.getWorkloadCreateArgs()
STEPCHAIN_ARGS = StepChainWorkloadFactory.getWorkloadCreateArgs()
TASKCHAIN_ARGS = TaskChainWorkloadFactory.getWorkloadCreateArgs()
# inner Task/Step definition
STEP_ARGS = StepChainWorkloadFactory.getChainCreateArgs()
TASK_ARGS = TaskChainWorkloadFactory.getChainCreateArgs()
### Some original request dicts (ones to be cloned from)
rerecoOriginalArgs = {'Memory': 234, 'SkimName1': 'skim_2017', 'SkimInput1': 'RECOoutput',
'Skim1ConfigCacheID': 'abcdef',
'TimePerEvent': 1.2, 'RequestType': 'ReReco', 'RequestName': 'test_rereco'}
stepChainOriginalArgs = {'Memory': 1234, 'TimePerEvent': 1.2, 'RequestType': 'StepChain',
"ScramArch": ["slc6_amd64_gcc481"], "RequestName": "test_stepchain",
"Step1": {"ConfigCacheID": "blah", "GlobalTag": "PHYS18", "BlockWhitelist": ["A", "B"]},
"Step2": {"AcquisitionEra": "ACQERA", "ProcessingVersion": 3, "LumisPerJob": 10},
"StepChain": 2, "ConfigCacheID": None}
taskChainOriginalArgs = {'PrepID': None, 'Campaign': "MainTask", 'RequestType': 'TaskChain',
