def testInvalidKeys_initialize_clone(self):
"""
Test some undefined/assignment user arguments in initialize_clone
"""
# test with unknown key
requestArgs = {'Multicore': 1, 'WRONG': 'Alan'}
originalArgs = deepcopy(taskChainOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
with self.assertRaises(WMSpecFactoryException):
TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)
# test with assignment key
requestArgs = {'Multicore': 1, 'TrustSitelists': False}
originalArgs = deepcopy(taskChainOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
with self.assertRaises(WMSpecFactoryException):
TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)
# test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception)
requestArgs = {'SubRequestType': "RelVal", "Step2": {"LumisPerJob": 5, "Campaign": "Step2Camp"}}
originalArgs = deepcopy(taskChainOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
with self.assertRaises(WMSpecFactoryException):
TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)
# make sure unsupported keys are remove from both top and inner dicts
requestArgs = {}
originalArgs = deepcopy(taskChainOriginalArgs)
originalArgs['BAD'] = 'KEY'
originalArgs['Task1']['SiteWhitelist'] = ['This_key_should_not_be_here']
originalArgs['Task1']['Alan'] = False
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
self.assertFalse('BAD' in cloneArgs)
self.assertFalse('SiteWhitelist' in cloneArgs['Task1'])
self.assertFalse('Alan' in cloneArgs['Task1'])
def testPileupFetcherOnMC(self):
pileupMcArgs = TaskChainWorkloadFactory.getTestArguments()
pileupMcArgs['Task1']["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO"
pileupMcArgs['Task1']["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW"
pileupMcArgs['Task1']["ConfigCacheID"] = self.injectGenerationConfig()
pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs)
# now that the workload was created and args validated, we can add this PileupConfig
pileupMcArgs["PileupConfig"] = parsePileupConfig(pileupMcArgs['Task1']["MCPileup"],
pileupMcArgs['Task1']["DataPileup"])
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % (self.testDir, testWorkload.name())
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(pileupMcArgs["PileupConfig"], task, taskPath)
def testMultithreadedTaskChain(self):
"""
Test for multithreaded task chains where all steps run with the same
number of cores
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
arguments['Task1']['Multicore'] = 4
arguments['Task2']['Multicore'] = 4
arguments['Task3']['Multicore'] = 4
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction("MultiChain", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
hlt = self.workload.getTaskByPath('/MultiChain/HLTD')
reco = self.workload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = self.workload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
hltStep = hlt.getStepHelper("cmsRun1")
recoStep = reco.getStepHelper("cmsRun1")
miniAODStep = miniAOD.getStepHelper("cmsRun1")
self.assertEqual(hltStep.getNumberOfCores(), 4)
self.assertEqual(recoStep.getNumberOfCores(), 4)
self.assertEqual(miniAODStep.getNumberOfCores(), 4)
return
def testPileupFetcherOnMC(self):
pileupMcArgs = TaskChainWorkloadFactory.getTestArguments()
pileupMcArgs['Task1']["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO"
pileupMcArgs['Task1']["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW"
pileupMcArgs['Task1']["ConfigCacheID"] = self.injectGenerationConfig()
pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs)
# now that the workload was created and args validated, we can add this PileupConfig
pileupMcArgs["PileupConfig"] = parsePileupConfig(pileupMcArgs['Task1']["MCPileup"],
pileupMcArgs['Task1']["DataPileup"])
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % (self.testDir, testWorkload.name())
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def testMultithreadedTaskChain(self):
"""
Test for multithreaded task chains where all steps run with the same
number of cores
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
arguments['Task1']['Multicore'] = 4
arguments['Task2']['Multicore'] = 4
arguments['Task3']['Multicore'] = 4
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction(
"MultiChain", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
hlt = self.workload.getTaskByPath('/MultiChain/HLTD')
reco = self.workload.getTaskByPath(
'/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = self.workload.getTaskByPath(
'/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT'
)
hltStep = hlt.getStepHelper("cmsRun1")
recoStep = reco.getStepHelper("cmsRun1")
miniAODStep = miniAOD.getStepHelper("cmsRun1")
self.assertEqual(hltStep.getNumberOfCores(), 4)
self.assertEqual(recoStep.getNumberOfCores(), 4)
self.assertEqual(miniAODStep.getNumberOfCores(), 4)
return
def testMultithreadedTaskChain(self):
"""
Test multi-task TaskChain with default and multicore settings
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
arguments.pop('Multicore', None)
arguments.pop('Memory', None)
arguments.pop('EventStreams', None)
arguments['Task1'].pop('Multicore', None)
arguments['Task1'].pop('Memory', None)
arguments['Task2'].pop('Multicore', None)
arguments['Task2'].pop('Memory', None)
arguments['Task2'].pop('EventStreams', None)
arguments['Task3'].pop('Multicore', None)
arguments['Task3'].pop('Memory', None)
arguments['Task3'].pop('EventStreams', None)
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("MultiChain", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath(
'/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(hlt.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
self.assertEqual(reco.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
self.assertEqual(miniAOD.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
# now all with 16 cores, 16 event streams and 8GB of memory inherited from the top level
arguments['Multicore'] = 16
arguments['Memory'] = 8000
arguments['EventStreams'] = 16
testWorkload = factory.factoryWorkloadConstruction("MultiChain", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath(
'/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(hlt.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
self.assertEqual(reco.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
self.assertEqual(miniAOD.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
return
def testMultithreadedTaskChain(self):
"""
Test multi-task TaskChain with default and multicore settings
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
arguments.pop('Multicore', None)
arguments.pop('Memory', None)
arguments['Task1'].pop('Multicore', None)
arguments['Task1'].pop('Memory', None)
arguments['Task2'].pop('Multicore', None)
arguments['Task2'].pop('Memory', None)
arguments['Task3'].pop('Multicore', None)
arguments['Task3'].pop('Memory', None)
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("MultiChain", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath(
'/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(hlt.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
self.assertEqual(reco.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
self.assertEqual(miniAOD.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
# now all with 16 cores, 16 event streams and 8GB of memory inherited from the top level
arguments['Multicore'] = 16
arguments['Memory'] = 8000
arguments['EventStreams'] = 16
testWorkload = factory.factoryWorkloadConstruction("MultiChain", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath(
'/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(hlt.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
self.assertEqual(reco.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
self.assertEqual(miniAOD.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
return
def testMultithreadedTasksTaskChain(self):
"""
Test for multithreaded task chains where each step
may run with a different number of cores
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("MultiChain2", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain2/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
hltMemory = hlt.jobSplittingParameters()['performance']['memoryRequirement']
recoMemory = reco.jobSplittingParameters()['performance']['memoryRequirement']
aodMemory = miniAOD.jobSplittingParameters()['performance']['memoryRequirement']
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task2']['Multicore'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task3']['Multicore'])
self.assertEqual(hltMemory, arguments['Memory'])
self.assertEqual(recoMemory, arguments['Task2']['Memory'])
self.assertEqual(aodMemory, arguments['Task3']['Memory'])
# and test another mix of cores and memory
arguments['Task1']['Multicore'] = arguments.pop('Multicore', None)
arguments['Task1']['Memory'] = arguments.pop('Memory', None)
arguments['Task2'].pop('Multicore', None)
arguments['Task2'].pop('Memory', None)
arguments['Task3']['Multicore'] = 2
testWorkload = factory.factoryWorkloadConstruction("MultiChain2", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain2/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
hltMemory = hlt.jobSplittingParameters()['performance']['memoryRequirement']
recoMemory = reco.jobSplittingParameters()['performance']['memoryRequirement']
aodMemory = miniAOD.jobSplittingParameters()['performance']['memoryRequirement']
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task1']['Multicore'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task3']['Multicore'])
self.assertEqual(hltMemory, arguments['Task1']['Memory'])
self.assertEqual(recoMemory, 2300.0)
self.assertEqual(aodMemory, arguments['Task3']['Memory'])
return
def createMCWMSpec(self, name='MonteCarloWorkload'):
mcArgs = TaskChainWorkloadFactory.getTestArguments()
mcArgs["CouchDBName"] = rerecoArgs["CouchDBName"]
mcArgs["Task1"]["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"])
wmspec = taskChainWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents=10000)
return wmspec
def createMCWMSpec(self, name='MonteCarloWorkload'):
mcArgs = TaskChainWorkloadFactory.getTestArguments()
mcArgs["CouchDBName"] = rerecoArgs["CouchDBName"]
mcArgs["Task1"]["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"])
wmspec = taskChainWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents=10000)
return wmspec
def testTrustFlags(self):
"""
_testTrustFlags_
Given a taskChain with 4 tasks, test whether TrustSitelists is set for
the top level tasks and TrustPUSitelists is properly set to all tasks.
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
testArguments.update(createMultiGTArgs())
testArguments["ConfigCacheUrl"] = self.testInit.couchUrl
testArguments["CouchDBName"] = self.testInit.couchDbName
testArguments["Task1"]["ConfigCacheID"] = processorDocs['DigiHLT']
testArguments["Task2"]["ConfigCacheID"] = processorDocs['Reco']
testArguments["Task3"]["ConfigCacheID"] = processorDocs['ALCAReco']
testArguments["Task4"]["ConfigCacheID"] = processorDocs['Skims']
arguments = testArguments
factory = TaskChainWorkloadFactory()
workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)
for task in workload.getAllTasks():
flags = task.getTrustSitelists().values()
self.assertEqual(flags, [False, False])
# set both flags to true now
workload.setTrustLocationFlag(True, True)
for task in workload.getAllTasks():
flags = task.getTrustSitelists()
if task.isTopOfTree():
self.assertEqual(flags.values(), [True, True])
elif task.taskType() in ["Cleanup", "LogCollect"]:
self.assertEqual(flags.values(), [False, False])
else:
self.assertFalse(flags['trustlists'])
self.assertTrue(flags['trustPUlists'])
# set both to false now
workload.setTrustLocationFlag(False, False)
for task in workload.getAllTasks(cpuOnly=True):
flags = task.getTrustSitelists().values()
self.assertEqual(flags, [False, False])
return
def getProdArgs():
mcArgs = TaskChainWorkloadFactory.getTestArguments()
mcArgs.update({
"CouchURL": None,
"CouchDBName": None,
"ConfigCacheDoc" : None
})
mcArgs.pop('ConfigCacheDoc')
return mcArgs
def getProdArgs():
mcArgs = TaskChainWorkloadFactory.getTestArguments()
mcArgs.update({
"CouchURL": None,
"CouchDBName": None,
"ConfigCacheDoc": None
})
mcArgs.pop('ConfigCacheDoc')
return mcArgs
def testMultithreadedTasksTaskChain(self):
"""
Test for multithreaded task chains where each step
may run with a different number of cores
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction(
"MultiChain2", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
hlt = self.workload.getTaskByPath('/MultiChain2/HLTD')
reco = self.workload.getTaskByPath(
'/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = self.workload.getTaskByPath(
'/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT'
)
hltMemory = hlt.jobSplittingParameters(
)['performance']['memoryRequirement']
recoMemory = reco.jobSplittingParameters(
)['performance']['memoryRequirement']
aodMemory = miniAOD.jobSplittingParameters(
)['performance']['memoryRequirement']
hltStep = hlt.getStepHelper("cmsRun1")
recoStep = reco.getStepHelper("cmsRun1")
miniAODStep = miniAOD.getStepHelper("cmsRun1")
self.assertEqual(hltStep.getNumberOfCores(), 4)
self.assertEqual(recoStep.getNumberOfCores(), 8)
self.assertEqual(miniAODStep.getNumberOfCores(), 1)
self.assertEqual(recoMemory, 8000.0)
self.assertEqual(aodMemory, 2000.0)
self.assertEqual(hltMemory, 4000.0)
return
def testTrustFlags(self):
"""
_testTrustFlags_
Given a taskChain with 4 tasks, test whether TrustSitelists is set for
the top level tasks and TrustPUSitelists is properly set to all tasks.
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
testArguments.update(createMultiGTArgs())
testArguments["CouchURL"] = self.testInit.couchUrl
testArguments["CouchDBName"] = self.testInit.couchDbName
testArguments["Task1"]["ConfigCacheID"] = processorDocs['DigiHLT']
testArguments["Task2"]["ConfigCacheID"] = processorDocs['Reco']
testArguments["Task3"]["ConfigCacheID"] = processorDocs['ALCAReco']
testArguments["Task4"]["ConfigCacheID"] = processorDocs['Skims']
arguments = testArguments
factory = TaskChainWorkloadFactory()
workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)
for task in workload.getAllTasks():
flags = task.getTrustSitelists().values()
self.assertEqual(flags, [False, False])
# set both flags to true now
workload.setTrustLocationFlag(True, True)
for task in workload.getAllTasks():
flags = task.getTrustSitelists()
if task.isTopOfTree():
self.assertEqual(flags.values(), [True, True])
elif task.taskType() in ["Merge", "Harvesting", "Cleanup", "LogCollect"]:
self.assertEqual(flags.values(), [False, False])
else:
self.assertFalse(flags['trustlists'])
self.assertTrue(flags['trustPUlists'])
# set both to false now
workload.setTrustLocationFlag(False, False)
for task in workload.getAllTasks(cpuOnly=True):
flags = task.getTrustSitelists().values()
self.assertEqual(flags, [False, False])
return
def testMultithreadedTasksTaskChain(self):
"""
Test for multithreaded task chains where each step
may run with a different number of cores
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction("MultiChain2", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
hlt = self.workload.getTaskByPath("/MultiChain2/HLTD")
reco = self.workload.getTaskByPath("/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT")
miniAOD = self.workload.getTaskByPath(
"/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT"
)
hltMemory = hlt.jobSplittingParameters()["performance"]["memoryRequirement"]
recoMemory = reco.jobSplittingParameters()["performance"]["memoryRequirement"]
aodMemory = miniAOD.jobSplittingParameters()["performance"]["memoryRequirement"]
hltStep = hlt.getStepHelper("cmsRun1")
recoStep = reco.getStepHelper("cmsRun1")
miniAODStep = miniAOD.getStepHelper("cmsRun1")
self.assertEqual(hltStep.getNumberOfCores(), 4)
self.assertEqual(recoStep.getNumberOfCores(), 8)
self.assertEqual(miniAODStep.getNumberOfCores(), 1)
self.assertEqual(recoMemory, 3200.0)
self.assertEqual(aodMemory, 2000.0)
self.assertEqual(hltMemory, 2400.0)
return
def buildMultithreadedTaskChain(self, filename):
""" d
Build a TaskChain from several sources and customization
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(filename))
# Construct args from the pieces starting with test args ...
arguments = testArguments
# ... continuing with the request
for key in [
"CMSSWVersion",
"ScramArch",
"GlobalTag",
"ProcessingVersion",
"Multicore",
"Memory",
"TaskChain",
"Task1",
"Task2",
"Task3",
]:
arguments.update({key: request["createRequest"][key]})
for key in ["SiteBlacklist"]:
arguments.update({key: request["assignRequest"][key]})
# ... then some local overrides
del arguments["ConfigCacheID"]
del arguments["ConfigCacheUrl"]
arguments.update({"CouchURL": self.testInit.couchUrl, "CouchDBName": self.testInit.couchDbName})
# ... now fill in the ConfigCache documents created and override the inputs to link them up
arguments["Task1"]["ConfigCacheID"] = processorDocs["DigiHLT"]
arguments["Task2"]["ConfigCacheID"] = processorDocs["Reco"]
arguments["Task2"]["InputFromOutputModule"] = "writeRAWDIGI"
arguments["Task3"]["ConfigCacheID"] = processorDocs["ALCAReco"]
arguments["Task3"]["InputFromOutputModule"] = "writeALCA"
return arguments
def buildMultithreadedTaskChain(self, filename):
""" d
Build a TaskChain from several sources and customization
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(filename))
# Construct args from the pieces starting with test args ...
arguments = testArguments
# ... continuing with the request
for key in [
'CMSSWVersion', 'ScramArch', 'GlobalTag', 'ProcessingVersion',
'Multicore', 'Memory', 'TaskChain', 'Task1', 'Task2', 'Task3'
]:
arguments.update({key: request['createRequest'][key]})
for key in ['SiteBlacklist']:
arguments.update({key: request['assignRequest'][key]})
# ... then some local overrides
del arguments['ConfigCacheID']
del arguments['ConfigCacheUrl']
arguments.update({
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
})
# ... now fill in the ConfigCache documents created and override the inputs to link them up
arguments['Task1']['ConfigCacheID'] = processorDocs['DigiHLT']
arguments['Task2']['ConfigCacheID'] = processorDocs['Reco']
arguments['Task2']['InputFromOutputModule'] = 'writeRAWDIGI'
arguments['Task3']['ConfigCacheID'] = processorDocs['ALCAReco']
arguments['Task3']['InputFromOutputModule'] = 'writeALCA'
return arguments
def buildMultithreadedTaskChain(self, filename):
""" d
Build a TaskChain from several sources and customization
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
# Read in the request
request = json.load(open(filename))
# Construct args from the pieces starting with test args ...
arguments = testArguments
# ... continuing with the request
for key in ['CMSSWVersion', 'ScramArch', 'GlobalTag', 'ProcessingVersion',
'Multicore', 'Memory',
'TaskChain', 'Task1', 'Task2', 'Task3']:
arguments.update({key : request['createRequest'][key]})
for key in ['SiteBlacklist']:
arguments.update({key : request['assignRequest'][key]})
# ... then some local overrides
del arguments['ConfigCacheID']
del arguments['ConfigCacheUrl']
arguments.update({
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
})
# ... now fill in the ConfigCache documents created and override the inputs to link them up
arguments['Task1']['ConfigCacheID'] = processorDocs['DigiHLT']
arguments['Task2']['ConfigCacheID'] = processorDocs['Reco']
arguments['Task2']['InputFromOutputModule'] = 'writeRAWDIGI'
arguments['Task3']['ConfigCacheID'] = processorDocs['ALCAReco']
arguments['Task3']['InputFromOutputModule'] = 'writeALCA'
return arguments
def testInvalidKeys_initialize_clone(self):
"""
Test some undefined/assignment user arguments in initialize_clone
"""
# test with unknown key
requestArgs = {'Multicore': 1, 'WRONG': 'Alan', 'ReqMgr2Only': False}
originalArgs = deepcopy(mcOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
with self.assertRaises(WMSpecFactoryException):
MonteCarloWorkloadFactory().factoryWorkloadConstruction(
cloneArgs["RequestName"], cloneArgs)
# test with assignment key
requestArgs = {
'Multicore': 1,
'TrustSitelists': False,
'ReqMgr2Only': False
}
originalArgs = deepcopy(mcOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
with self.assertRaises(WMSpecFactoryException):
MonteCarloWorkloadFactory().factoryWorkloadConstruction(
cloneArgs["RequestName"], cloneArgs)
# test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception)
requestArgs = {
'SubRequestType': "RelVal",
"Step2": {
"LumisPerJob": 5,
"Campaign": "Step2Camp"
}
}
originalArgs = deepcopy(taskChainOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS,
TASK_ARGS)
with self.assertRaises(WMSpecFactoryException):
TaskChainWorkloadFactory().factoryWorkloadConstruction(
cloneArgs["RequestName"], cloneArgs)
def testGeneratorWorkflow(self):
"""
_testGeneratorWorkflow_
Test creating a request with an initial generator task
it mocks a request where there are 2 similar paths starting
from the generator, each one with a different PrimaryDataset, CMSSW configuration
and processed dataset. Dropping the RAW output as well.
Also include an ignored output module to keep things interesting...
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 6,
"IgnoredOutputModules": ["writeSkim2", "writeRAWDEBUGDIGI"],
"Task1": {
"TaskName": "GenSim",
"ConfigCacheID": generatorDoc,
"SplittingAlgo": "EventBased",
"RequestNumEvents": 10000,
"Seeding": "AutomaticSeeding",
"PrimaryDataset": "RelValTTBar",
},
"Task2": {
"TaskName": "DigiHLT_new",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "LumiBased",
"CMSSWVersion": "CMSSW_5_2_6",
"GlobalTag": "GR_39_P_V5:All",
"PrimaryDataset": "PURelValTTBar",
"KeepOutput": False
},
"Task3": {
"TaskName": "DigiHLT_ref",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "EventBased",
"CMSSWVersion": "CMSSW_5_2_7",
"GlobalTag": "GR_40_P_V5:All",
"AcquisitionEra": "ReleaseValidationNewConditions",
"ProcessingVersion": 3,
"ProcessingString": "Test",
"KeepOutput": False
},
"Task4": {
"TaskName": "Reco",
"InputTask": "DigiHLT_new",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgo": "FileBased",
"TransientOutputModules": ["writeRECO"]
},
"Task5": {
"TaskName": "ALCAReco",
"InputTask": "DigiHLT_ref",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgo": "LumiBased",
},
"Task6": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgo": "LumiBased",
}
}
testArguments.update(arguments)
arguments = testArguments
print arguments
factory = TaskChainWorkloadFactory()
# Test a malformed task chain definition
arguments['Task4']['TransientOutputModules'].append('writeAOD')
self.assertRaises(WMSpecFactoryException, factory.validateSchema,
arguments)
arguments['Task4']['TransientOutputModules'].remove('writeAOD')
try:
self.workload = factory.factoryWorkloadConstruction(
"PullingTheChain", arguments)
except Exception, ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
import traceback
traceback.print_exc()
self.fail(msg)
def testMultipleGlobalTags(self):
"""
_testMultipleGlobalTags_
Test creating a workload that starts in a processing task
with an input dataset, and has different globalTags
and CMSSW versions (with corresponding scramArch) in
each task
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
lumiDict = {"1":[[2,4], [8,50]], "2":[[100,200], [210,210]]}
lumiDict2 = {"1":[[2,4], [8,40]], "2":[[100,150], [210,210]]}
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "DefaultGlobalTag",
"LumiList": lumiDict,
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain" : 4,
"Task1" :{
"TaskName" : "DigiHLT",
"ConfigCacheID" : processorDocs['DigiHLT'],
"InputDataset" : "/MinimumBias/Commissioning10-v4/GEN-SIM",
"SplittingAlgo" : "FileBased",
},
"Task2" : {
"TaskName" : "Reco",
"InputTask" : "DigiHLT",
"InputFromOutputModule" : "writeRAWDIGI",
"ConfigCacheID" : processorDocs['Reco'],
"SplittingAlgo" : "FileBased",
"GlobalTag" : "GlobalTagForReco",
"CMSSWVersion" : "CMSSW_3_1_2",
"ScramArch" : "CompatibleRECOArch",
"PrimaryDataset" : "ZeroBias",
"LumiList": lumiDict2,
},
"Task3" : {
"TaskName" : "ALCAReco",
"InputTask" : "Reco",
"InputFromOutputModule" : "writeALCA",
"ConfigCacheID" : processorDocs['ALCAReco'],
"SplittingAlgo" : "FileBased",
"GlobalTag" : "GlobalTagForALCAReco",
"CMSSWVersion" : "CMSSW_3_1_3",
"ScramArch" : "CompatibleALCAArch",
},
"Task4" : {
"TaskName" : "Skims",
"InputTask" : "Reco",
"InputFromOutputModule" : "writeRECO",
"ConfigCacheID" : processorDocs['Skims'],
"SplittingAlgo" : "FileBased",
}
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)
except Exception, ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
import unittest
from copy import deepcopy
from Utils.PythonVersion import PY3
from WMCore.ReqMgr.DataStructs.Request import initialize_clone, RequestInfo
from WMCore.WMSpec.StdSpecs.ReReco import ReRecoWorkloadFactory
from WMCore.WMSpec.StdSpecs.StepChain import StepChainWorkloadFactory
from WMCore.WMSpec.StdSpecs.TaskChain import TaskChainWorkloadFactory
from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException
### Spec arguments definition with only key and its default value
RERECO_ARGS = ReRecoWorkloadFactory.getWorkloadCreateArgs()
STEPCHAIN_ARGS = StepChainWorkloadFactory.getWorkloadCreateArgs()
TASKCHAIN_ARGS = TaskChainWorkloadFactory.getWorkloadCreateArgs()
# inner Task/Step definition
STEP_ARGS = StepChainWorkloadFactory.getChainCreateArgs()
TASK_ARGS = TaskChainWorkloadFactory.getChainCreateArgs()
### Some original request dicts (ones to be cloned from)
rerecoOriginalArgs = {
'Memory': 234,
'SkimName1': 'skim_2017',
'SkimInput1': 'RECOoutput',
'Skim1ConfigCacheID': 'abcdef',
'IncludeParents': True,
'TimePerEvent': 1.2,
'RequestType': 'ReReco',
'RequestName': 'test_rereco'
}
def testPileupTaskChain(self):
"""
Test for multithreaded task chains where each step
may run with a different number of cores
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain" : 2,
"Task1" :{
"InputDataset" : "/cosmics/whatever-input-v1/GEN-SIM",
"TaskName" : "DIGI",
"ConfigCacheID" : processorDocs['DigiHLT'],
"SplittingAlgo" : "LumiBased",
"LumisPerJob": 4,
"MCPileup": "/some/cosmics-mc-v1/GEN-SIM",
"DeterministicPileup": True,
"CMSSWVersion" : "CMSSW_5_2_6",
"GlobalTag" : "GR_39_P_V5:All",
"PrimaryDataset" : "PURelValTTBar",
"AcquisitionEra": "CMSSW_5_2_6",
"ProcessingString": "ProcStr_Task1"
},
"Task2" : {
"TaskName" : "RECO",
"InputTask" : "DIGI",
"InputFromOutputModule" : "writeRAWDIGI",
"ConfigCacheID" : processorDocs['Reco'],
"DataPileup": "/some/minbias-data-v1/GEN-SIM",
"SplittingAlgo" : "LumiBased",
"LumisPerJob": 2,
"GlobalTag": "GR_R_62_V3::All",
"AcquisitionEra": "CMSSW_5_2_7",
"ProcessingString": "ProcStr_Task2"
},
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
self.workload = factory.factoryWorkloadConstruction("PullingTheChain", arguments)
firstTask = self.workload.getTaskByPath("/PullingTheChain/DIGI")
cmsRunStep = firstTask.getStep("cmsRun1").getTypeHelper()
pileupData = cmsRunStep.getPileup()
self.assertFalse(hasattr(pileupData, "data"))
self.assertEqual(pileupData.mc.dataset, ["/some/cosmics-mc-v1/GEN-SIM"])
splitting = firstTask.jobSplittingParameters()
self.assertTrue(splitting["deterministicPileup"])
secondTask = self.workload.getTaskByPath("/PullingTheChain/DIGI/DIGIMergewriteRAWDIGI/RECO")
cmsRunStep = secondTask.getStep("cmsRun1").getTypeHelper()
pileupData = cmsRunStep.getPileup()
self.assertFalse(hasattr(pileupData, "mc"))
self.assertEqual(pileupData.data.dataset, ["/some/minbias-data-v1/GEN-SIM"])
splitting = secondTask.jobSplittingParameters()
self.assertFalse(splitting["deterministicPileup"])
def testMultipleGlobalTags(self):
"""
_testMultipleGlobalTags_
Test creating a workload that starts in a processing task
with an input dataset, and has different globalTags
and CMSSW versions (with corresponding scramArch) in
each task
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
lumiDict = {"1": [[2, 4], [8, 50]], "2": [[100, 200], [210, 210]]}
lumiDict2 = {"1": [[2, 4], [8, 40]], "2": [[100, 150], [210, 210]]}
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "DefaultGlobalTag",
"LumiList": lumiDict,
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 4,
"Task1": {
"TaskName": "DigiHLT",
"ConfigCacheID": processorDocs['DigiHLT'],
"InputDataset": "/MinimumBias/Commissioning10-v4/GEN-SIM",
"SplittingAlgo": "FileBased",
},
"Task2": {
"TaskName": "Reco",
"InputTask": "DigiHLT",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgo": "FileBased",
"GlobalTag": "GlobalTagForReco",
"CMSSWVersion": "CMSSW_3_1_2",
"ScramArch": "CompatibleRECOArch",
"PrimaryDataset": "ZeroBias",
"LumiList": lumiDict2,
},
"Task3": {
"TaskName": "ALCAReco",
"InputTask": "Reco",
"InputFromOutputModule": "writeALCA",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgo": "FileBased",
"GlobalTag": "GlobalTagForALCAReco",
"CMSSWVersion": "CMSSW_3_1_3",
"ScramArch": "CompatibleALCAArch",
},
"Task4": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgo": "FileBased",
}
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction(
"YankingTheChain", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
testWMBSHelper = WMBSHelper(self.workload,
"DigiHLT",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._checkTask(
self.workload.getTaskByPath("/YankingTheChain/DigiHLT"),
arguments['Task1'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"),
arguments['Task2'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"
), arguments['Task3'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"
), arguments['Task4'], arguments)
digi = self.workload.getTaskByPath("/YankingTheChain/DigiHLT")
self.assertEqual(lumiDict, digi.getLumiMask())
digiStep = digi.getStepHelper("cmsRun1")
self.assertEqual(digiStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])
# Make sure this task has a different lumilist than the global one
reco = self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
self.assertEqual(lumiDict2, reco.getLumiMask())
recoStep = reco.getStepHelper("cmsRun1")
self.assertEqual(recoStep.getGlobalTag(),
arguments['Task2']['GlobalTag'])
self.assertEqual(recoStep.getCMSSWVersion(),
arguments['Task2']['CMSSWVersion'])
self.assertEqual(recoStep.getScramArch(),
arguments['Task2']['ScramArch'])
alca = self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"
)
self.assertEqual(lumiDict, alca.getLumiMask())
alcaStep = alca.getStepHelper("cmsRun1")
self.assertEqual(alcaStep.getGlobalTag(),
arguments['Task3']['GlobalTag'])
self.assertEqual(alcaStep.getCMSSWVersion(),
arguments['Task3']['CMSSWVersion'])
self.assertEqual(alcaStep.getScramArch(),
arguments['Task3']['ScramArch'])
skim = self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"
)
skimStep = skim.getStepHelper("cmsRun1")
self.assertEqual(skimStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(skimStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(skimStep.getScramArch(), arguments['ScramArch'])
# Verify the output datasets
outputDatasets = self.workload.listOutputDatasets()
self.assertEqual(len(outputDatasets), 14,
"Number of output datasets doesn't match")
self.assertTrue(
"/MinimumBias/ReleaseValidation-RawDigiFilter-v1/RAW-DIGI"
in outputDatasets,
"/MinimumBias/ReleaseValidation-RawDigiFilter-v1/RAW-DIGI not in output datasets"
)
self.assertTrue(
"/MinimumBias/ReleaseValidation-RawDebugDigiFilter-v1/RAW-DEBUG-DIGI"
in outputDatasets,
"/MinimumBias/ReleaseValidation-RawDebugDigiFilter-v1/RAW-DEBUG-DIGI not in output datasets"
)
self.assertTrue(
"/ZeroBias/ReleaseValidation-reco-v1/RECO" in outputDatasets,
"/ZeroBias/ReleaseValidation-reco-v1/RECO not in output datasets")
self.assertTrue(
"/ZeroBias/ReleaseValidation-AOD-v1/AOD" in outputDatasets,
"/ZeroBias/ReleaseValidation-AOD-v1/AOD not in output datasets")
self.assertTrue(
"/ZeroBias/ReleaseValidation-alca-v1/ALCARECO" in outputDatasets,
"/ZeroBias/ReleaseValidation-alca-v1/ALCARECO not in output datasets"
)
for i in range(1, 5):
self.assertTrue(
"/MinimumBias/ReleaseValidation-alca%d-v1/ALCARECO" % i
in outputDatasets,
"/MinimumBias/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets"
% i)
for i in range(1, 6):
self.assertTrue(
"/MinimumBias/ReleaseValidation-skim%d-v1/RECO-AOD" % i
in outputDatasets,
"/MinimumBias/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets"
% i)
return
def testProcessingWithScenarios(self):
"""
_testProcessingWithScenarios_
Test creating a processing workload with an input dataset that
uses scenarios instead of configuration files
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 4,
"Task1": {
"TaskName": "DigiHLT",
"ConfigCacheID": processorDocs['DigiHLT'],
"InputDataset": "/MinimumBias/Commissioning10-v4/GEN-SIM",
"SplittingAlgorithm": "FileBased",
"SplittingArguments": {
"files_per_job": 1
},
},
"Task2": {
"TaskName": "Reco",
"InputTask": "DigiHLT",
"InputFromOutputModule": "writeRAWDIGI",
"ProcScenario": "pp",
"ScenarioMethod": "promptReco",
"ScenarioArguments": {
'outputs': [{
'dataTier': "RECO",
'moduleLabel': "RECOoutput"
}, {
'dataTier': "AOD",
'moduleLabel': "AODoutput"
}, {
'dataTier': "ALCARECO",
'moduleLabel': "ALCARECOoutput"
}]
},
"SplittingAlgorithm": "FileBased",
"SplittingArguments": {
"files_per_job": 1
},
},
"Task3": {
"TaskName": "ALCAReco",
"InputTask": "Reco",
"InputFromOutputModule": "ALCARECOoutput",
"ProcScenario": "pp",
"ScenarioMethod": "alcaReco",
"ScenarioArguments": {
'outputs': [{
'dataTier': "ALCARECO",
'moduleLabel': "ALCARECOoutput"
}]
},
"SplittingAlgorithm": "FileBased",
"SplittingArguments": {
"files_per_job": 1
},
},
"Task4": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "RECOoutput",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgorithm": "FileBased",
"SplittingArguments": {
"files_per_job": 10
},
}
}
factory = TaskChainWorkloadFactory()
try:
self.workload = factory("YankingTheChain", arguments)
except Exception, ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
def testMultipleGlobalTags(self):
"""
_testMultipleGlobalTags_
Test creating a workload that starts in a processing task
with an input dataset, and has different globalTags
and CMSSW versions (with corresponding scramArch) in
each task
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
testArguments.update(createMultiGTArgs())
lumiDict = {"1": [[2, 4], [8, 50]], "2": [[100, 200], [210, 210]]}
testArguments["ConfigCacheUrl"] = self.testInit.couchUrl
testArguments["CouchDBName"] = self.testInit.couchDbName
testArguments["Task1"]["LumiList"] = lumiDict
testArguments["Task1"]["ConfigCacheID"] = processorDocs['DigiHLT']
testArguments["Task2"]["ConfigCacheID"] = processorDocs['Reco']
testArguments["Task3"]["ConfigCacheID"] = processorDocs['ALCAReco']
testArguments["Task4"]["ConfigCacheID"] = processorDocs['Skims']
arguments = testArguments
factory = TaskChainWorkloadFactory()
self.workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)
testWMBSHelper = WMBSHelper(self.workload, "DigiHLT", "SomeBlock", cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT"), arguments['Task1'], arguments)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"),
arguments['Task2'],
arguments)
self._checkTask(self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"),
arguments['Task3'], arguments)
self._checkTask(self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"),
arguments['Task4'], arguments)
digi = self.workload.getTaskByPath("/YankingTheChain/DigiHLT")
self.assertEqual(lumiDict, digi.getLumiMask().getCompactList())
digiStep = digi.getStepHelper("cmsRun1")
self.assertEqual(digiStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])
# Make sure this task has a different lumilist than the global one
reco = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
recoStep = reco.getStepHelper("cmsRun1")
self.assertEqual(recoStep.getGlobalTag(), arguments['Task2']['GlobalTag'])
self.assertEqual(recoStep.getCMSSWVersion(), arguments['Task2']['CMSSWVersion'])
self.assertEqual(recoStep.getScramArch(), arguments['Task2']['ScramArch'])
alca = self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco")
alcaStep = alca.getStepHelper("cmsRun1")
self.assertEqual(alcaStep.getGlobalTag(), arguments['Task3']['GlobalTag'])
self.assertEqual(alcaStep.getCMSSWVersion(), arguments['Task3']['CMSSWVersion'])
self.assertEqual(alcaStep.getScramArch(), arguments['Task3']['ScramArch'])
skim = self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims")
skimStep = skim.getStepHelper("cmsRun1")
self.assertEqual(skimStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(skimStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(skimStep.getScramArch(), arguments['ScramArch'])
# Verify the output datasets
outputDatasets = self.workload.listOutputDatasets()
self.assertEqual(len(outputDatasets), 14, "Number of output datasets doesn't match")
self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-RawDigiFilter-FAKE-v1/RAW-DIGI" in outputDatasets)
self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-RawDebugDigiFilter-FAKE-v1/RAW-DEBUG-DIGI" in outputDatasets)
self.assertTrue("/ZeroBias/ReleaseValidation-reco-FAKE-v1/RECO" in outputDatasets)
self.assertTrue("/ZeroBias/ReleaseValidation-AOD-FAKE-v1/AOD" in outputDatasets)
self.assertTrue("/ZeroBias/ReleaseValidation-alca-FAKE-v1/ALCARECO" in outputDatasets)
for i in range(1, 5):
self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-alca%d-FAKE-v1/ALCARECO" % i in outputDatasets)
for i in range(1, 6):
self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-skim%d-FAKE-v1/RECO-AOD" % i in outputDatasets)
return
def test1TaskMemCoresSettings(self):
"""
_test1TaskMemCoresSettings_
Make sure the multicore and memory setings are properly propagated to
all steps. Single step in a task.
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"ConfigCacheUrl": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"TaskChain": 1,
"Task1": {
"TaskName": "TaskOne",
"ConfigCacheID": generatorDoc,
"RequestNumEvents": 10000,
"PrimaryDataset": "RelValTTBar",
},
}
testArguments.update(arguments)
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)
taskPaths = ['/TestTaskChainWorkload/TaskOne',
'/TestTaskChainWorkload/TaskOne/LogCollectForTaskOne',
'/TestTaskChainWorkload/TaskOne/TaskOneMergewriteGENSIM',
'/TestTaskChainWorkload/TaskOne/TaskOneMergewriteGENSIM/TaskOnewriteGENSIMMergeLogCollect',
'/TestTaskChainWorkload/TaskOne/TaskOneCleanupUnmergedwriteGENSIM']
for task in taskPaths:
taskObj = testWorkload.getTaskByPath(task)
if taskObj.taskType() in ('Production', 'Processing'):
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
elif taskObj.taskType() == 'LogCollect':
stepHelper = taskObj.getStepHelper('logCollect1')
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
# now play with cores at top level
testArguments['Multicore'] = 2
testArguments['EventStreams'] = 2
testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)
for task in taskPaths:
taskObj = testWorkload.getTaskByPath(task)
if taskObj.taskType() in ('Production', 'Processing'):
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
if step == 'cmsRun1':
self.assertEqual(stepHelper.getNumberOfCores(), testArguments['Multicore'])
self.assertEqual(stepHelper.getNumberOfStreams(), testArguments["EventStreams"])
else:
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
elif taskObj.taskType() == 'LogCollect':
stepHelper = taskObj.getStepHelper('logCollect1')
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
# last but not least, play with cores at task level
testArguments['Task1']['Multicore'] = 2
testArguments['Task1']['EventStreams'] = 2
testArguments.pop('Multicore', None)
testArguments.pop('EventStreams', None)
testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)
for task in taskPaths:
taskObj = testWorkload.getTaskByPath(task)
if taskObj.taskType() in ('Production', 'Processing'):
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
if step == 'cmsRun1':
self.assertEqual(stepHelper.getNumberOfCores(), testArguments['Task1']['Multicore'])
self.assertEqual(stepHelper.getNumberOfStreams(), testArguments['Task1']['EventStreams'])
else:
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
elif taskObj.taskType() == 'LogCollect':
stepHelper = taskObj.getStepHelper('logCollect1')
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
return
def test1TaskMemCoresSettings(self):
"""
_test1StepMemCoresSettings_
Make sure the multicore and memory setings are properly propagated to
all steps. Single step in a task.
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"TaskChain": 1,
"Task1": {
"TaskName": "TaskOne",
"ConfigCacheID": generatorDoc,
"RequestNumEvents": 10000,
"PrimaryDataset": "RelValTTBar",
},
}
testArguments.update(arguments)
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)
taskPaths = ['/TestTaskChainWorkload/TaskOne',
'/TestTaskChainWorkload/TaskOne/LogCollectForTaskOne',
'/TestTaskChainWorkload/TaskOne/TaskOneMergewriteGENSIM',
'/TestTaskChainWorkload/TaskOne/TaskOneMergewriteGENSIM/TaskOnewriteGENSIMMergeLogCollect',
'/TestTaskChainWorkload/TaskOne/TaskOneCleanupUnmergedwriteGENSIM']
for task in taskPaths:
taskObj = testWorkload.getTaskByPath(task)
if taskObj.taskType() in ('Production', 'Processing'):
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1)
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
elif taskObj.taskType() in ('LogCollect'):
stepHelper = taskObj.getStepHelper('logCollect1')
self.assertEqual(stepHelper.getNumberOfCores(), 1)
# now play with cores at top level
testArguments['Multicore'] = 2
testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)
for task in taskPaths:
taskObj = testWorkload.getTaskByPath(task)
if taskObj.taskType() in ('Production', 'Processing'):
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
if step == 'cmsRun1':
self.assertEqual(stepHelper.getNumberOfCores(), testArguments['Multicore'])
else:
self.assertEqual(stepHelper.getNumberOfCores(), 1)
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
elif taskObj.taskType() in ('LogCollect'):
stepHelper = taskObj.getStepHelper('logCollect1')
self.assertEqual(stepHelper.getNumberOfCores(), 1)
# last but not least, play with cores at task level
testArguments['Task1']['Multicore'] = 2
testArguments.pop('Multicore', None)
testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)
for task in taskPaths:
taskObj = testWorkload.getTaskByPath(task)
if taskObj.taskType() in ('Production', 'Processing'):
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
if step == 'cmsRun1':
self.assertEqual(stepHelper.getNumberOfCores(), testArguments['Task1']['Multicore'])
else:
self.assertEqual(stepHelper.getNumberOfCores(), 1)
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
elif taskObj.taskType() in ('LogCollect'):
stepHelper = taskObj.getStepHelper('logCollect1')
self.assertEqual(stepHelper.getNumberOfCores(), 1)
return
def testMultipleGlobalTags(self):
"""
_testMultipleGlobalTags_
Test creating a workload that starts in a processing task
with an input dataset, and has different globalTags
and CMSSW versions (with corresponding scramArch) in
each task
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
testArguments.update(createMultiGTArgs())
lumiDict = {"1":[[2,4], [8,50]], "2":[[100,200], [210,210]]}
testArguments["CouchURL"] = self.testInit.couchUrl
testArguments["CouchDBName"] = self.testInit.couchDbName
testArguments["Task1"]["LumiList"] = lumiDict
testArguments["Task1"]["ConfigCacheID"] = processorDocs['DigiHLT']
testArguments["Task2"]["ConfigCacheID"] = processorDocs['Reco']
testArguments["Task3"]["ConfigCacheID"] = processorDocs['ALCAReco']
testArguments["Task4"]["ConfigCacheID"] = processorDocs['Skims']
arguments = testArguments
factory = TaskChainWorkloadFactory()
self.workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)
testWMBSHelper = WMBSHelper(self.workload, "DigiHLT", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT"), arguments['Task1'], arguments)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"), arguments['Task2'],
arguments)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"),
arguments['Task3'], arguments)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"),
arguments['Task4'], arguments)
digi = self.workload.getTaskByPath("/YankingTheChain/DigiHLT")
self.assertEqual(lumiDict, digi.getLumiMask().getCompactList())
digiStep = digi.getStepHelper("cmsRun1")
self.assertEqual(digiStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])
# Make sure this task has a different lumilist than the global one
reco = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
recoStep = reco.getStepHelper("cmsRun1")
self.assertEqual(recoStep.getGlobalTag(), arguments['Task2']['GlobalTag'])
self.assertEqual(recoStep.getCMSSWVersion(), arguments['Task2']['CMSSWVersion'])
self.assertEqual(recoStep.getScramArch(), arguments['Task2']['ScramArch'])
alca = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco")
alcaStep = alca.getStepHelper("cmsRun1")
self.assertEqual(alcaStep.getGlobalTag(), arguments['Task3']['GlobalTag'])
self.assertEqual(alcaStep.getCMSSWVersion(), arguments['Task3']['CMSSWVersion'])
self.assertEqual(alcaStep.getScramArch(), arguments['Task3']['ScramArch'])
skim = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims")
skimStep = skim.getStepHelper("cmsRun1")
self.assertEqual(skimStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(skimStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(skimStep.getScramArch(), arguments['ScramArch'])
# Verify the output datasets
outputDatasets = self.workload.listOutputDatasets()
self.assertEqual(len(outputDatasets), 14, "Number of output datasets doesn't match")
self.assertTrue("/MinimumBias/ReleaseValidation-RawDigiFilter-FAKE-v1/RAW-DIGI" in outputDatasets)
self.assertTrue("/MinimumBias/ReleaseValidation-RawDebugDigiFilter-FAKE-v1/RAW-DEBUG-DIGI" in outputDatasets)
self.assertTrue("/ZeroBias/ReleaseValidation-reco-FAKE-v1/RECO" in outputDatasets)
self.assertTrue("/ZeroBias/ReleaseValidation-AOD-FAKE-v1/AOD" in outputDatasets)
self.assertTrue("/ZeroBias/ReleaseValidation-alca-FAKE-v1/ALCARECO" in outputDatasets)
for i in range(1, 5):
self.assertTrue("/MinimumBias/ReleaseValidation-alca%d-FAKE-v1/ALCARECO" % i in outputDatasets)
for i in range(1, 6):
self.assertTrue("/MinimumBias/ReleaseValidation-skim%d-FAKE-v1/RECO-AOD" % i in outputDatasets)
return
def testMultithreadedTasksTaskChain(self):
"""
Test for multithreaded task chains where each step
may run with a different number of cores
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("MultiChain2", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain2/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath(
'/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
hltMemory = hlt.jobSplittingParameters()['performance']['memoryRequirement']
recoMemory = reco.jobSplittingParameters()['performance']['memoryRequirement']
aodMemory = miniAOD.jobSplittingParameters()['performance']['memoryRequirement']
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task2']['Multicore'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task3']['Multicore'])
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task2']['EventStreams'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task3']['EventStreams'])
self.assertEqual(hltMemory, arguments['Memory'])
self.assertEqual(recoMemory, arguments['Task2']['Memory'])
self.assertEqual(aodMemory, arguments['Task3']['Memory'])
# and test another mix of cores, event streams and memory
arguments['Task1']['Multicore'] = arguments.pop('Multicore', None)
arguments['Task1']['Memory'] = arguments.pop('Memory', None)
arguments['Task2'].pop('Multicore', None)
arguments['Task2'].pop('Memory', None)
arguments['Task3']['Multicore'] = 2
arguments['EventStreams'] = 32
arguments['Task2']['EventStreams'] = 8
arguments['Task3']['EventStreams'] = 0
testWorkload = factory.factoryWorkloadConstruction("MultiChain2", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain2/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath(
'/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
hltMemory = hlt.jobSplittingParameters()['performance']['memoryRequirement']
recoMemory = reco.jobSplittingParameters()['performance']['memoryRequirement']
aodMemory = miniAOD.jobSplittingParameters()['performance']['memoryRequirement']
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task1']['Multicore'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task3']['Multicore'])
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task2']['EventStreams'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task3']['EventStreams'])
self.assertEqual(hltMemory, arguments['Task1']['Memory'])
self.assertEqual(recoMemory, 2300.0)
self.assertEqual(aodMemory, arguments['Task3']['Memory'])
return
def testMultipleGlobalTags(self):
"""
_testMultipleGlobalTags_
Test creating a workload that starts in a processing task
with an input dataset, and has different globalTags
and CMSSW versions (with corresponding scramArch) in
each task
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
lumiDict = {"1": [[2, 4], [8, 50]], "2": [[100, 200], [210, 210]]}
lumiDict2 = {"1": [[2, 4], [8, 40]], "2": [[100, 150], [210, 210]]}
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "DefaultGlobalTag",
"LumiList": lumiDict,
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 4,
"Task1": {
"TaskName": "DigiHLT",
"ConfigCacheID": processorDocs['DigiHLT'],
"InputDataset": "/MinimumBias/Commissioning10-v4/GEN-SIM",
"SplittingAlgo": "FileBased",
},
"Task2": {
"TaskName": "Reco",
"InputTask": "DigiHLT",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgo": "FileBased",
"GlobalTag": "GlobalTagForReco",
"CMSSWVersion": "CMSSW_3_1_2",
"ScramArch": "CompatibleRECOArch",
"PrimaryDataset": "ZeroBias",
"LumiList": lumiDict2,
},
"Task3": {
"TaskName": "ALCAReco",
"InputTask": "Reco",
"InputFromOutputModule": "writeALCA",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgo": "FileBased",
"GlobalTag": "GlobalTagForALCAReco",
"CMSSWVersion": "CMSSW_3_1_3",
"ScramArch": "CompatibleALCAArch",
},
"Task4": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgo": "FileBased",
}
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction(
"YankingTheChain", arguments)
except Exception, ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
def testMultipleGlobalTags(self):
"""
_testMultipleGlobalTags_
Test creating a workload that starts in a processing task
with an input dataset, and has different globalTags
and CMSSW versions (with corresponding scramArch) in
each task
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
lumiDict = {"1":[[2,4], [8,50]], "2":[[100,200], [210,210]]}
lumiDict2 = {"1":[[2,4], [8,40]], "2":[[100,150], [210,210]]}
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "DefaultGlobalTag",
"LumiList": lumiDict,
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain" : 4,
"Task1" :{
"TaskName" : "DigiHLT",
"ConfigCacheID" : processorDocs['DigiHLT'],
"InputDataset" : "/MinimumBias/Commissioning10-v4/GEN-SIM",
"SplittingAlgo" : "FileBased",
},
"Task2" : {
"TaskName" : "Reco",
"InputTask" : "DigiHLT",
"InputFromOutputModule" : "writeRAWDIGI",
"ConfigCacheID" : processorDocs['Reco'],
"SplittingAlgo" : "FileBased",
"GlobalTag" : "GlobalTagForReco",
"CMSSWVersion" : "CMSSW_3_1_2",
"ScramArch" : "CompatibleRECOArch",
"PrimaryDataset" : "ZeroBias",
"LumiList": lumiDict2,
},
"Task3" : {
"TaskName" : "ALCAReco",
"InputTask" : "Reco",
"InputFromOutputModule" : "writeALCA",
"ConfigCacheID" : processorDocs['ALCAReco'],
"SplittingAlgo" : "FileBased",
"GlobalTag" : "GlobalTagForALCAReco",
"CMSSWVersion" : "CMSSW_3_1_3",
"ScramArch" : "CompatibleALCAArch",
},
"Task4" : {
"TaskName" : "Skims",
"InputTask" : "Reco",
"InputFromOutputModule" : "writeRECO",
"ConfigCacheID" : processorDocs['Skims'],
"SplittingAlgo" : "FileBased",
}
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
testWMBSHelper = WMBSHelper(self.workload, "DigiHLT", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT"), arguments['Task1'], arguments)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"), arguments['Task2'],
arguments)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"),
arguments['Task3'], arguments)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"),
arguments['Task4'], arguments)
digi = self.workload.getTaskByPath("/YankingTheChain/DigiHLT")
self.assertEqual(lumiDict, digi.getLumiMask())
digiStep = digi.getStepHelper("cmsRun1")
self.assertEqual(digiStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])
# Make sure this task has a different lumilist than the global one
reco = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
self.assertEqual(lumiDict2, reco.getLumiMask())
recoStep = reco.getStepHelper("cmsRun1")
self.assertEqual(recoStep.getGlobalTag(), arguments['Task2']['GlobalTag'])
self.assertEqual(recoStep.getCMSSWVersion(), arguments['Task2']['CMSSWVersion'])
self.assertEqual(recoStep.getScramArch(), arguments['Task2']['ScramArch'])
alca = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco")
self.assertEqual(lumiDict, alca.getLumiMask())
alcaStep = alca.getStepHelper("cmsRun1")
self.assertEqual(alcaStep.getGlobalTag(), arguments['Task3']['GlobalTag'])
self.assertEqual(alcaStep.getCMSSWVersion(), arguments['Task3']['CMSSWVersion'])
self.assertEqual(alcaStep.getScramArch(), arguments['Task3']['ScramArch'])
skim = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims")
skimStep = skim.getStepHelper("cmsRun1")
self.assertEqual(skimStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(skimStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(skimStep.getScramArch(), arguments['ScramArch'])
# Verify the output datasets
outputDatasets = self.workload.listOutputDatasets()
self.assertEqual(len(outputDatasets), 14, "Number of output datasets doesn't match")
self.assertTrue("/MinimumBias/ReleaseValidation-RawDigiFilter-v1/RAW-DIGI" in outputDatasets,
"/MinimumBias/ReleaseValidation-RawDigiFilter-v1/RAW-DIGI not in output datasets")
self.assertTrue("/MinimumBias/ReleaseValidation-RawDebugDigiFilter-v1/RAW-DEBUG-DIGI" in outputDatasets,
"/MinimumBias/ReleaseValidation-RawDebugDigiFilter-v1/RAW-DEBUG-DIGI not in output datasets")
self.assertTrue("/ZeroBias/ReleaseValidation-reco-v1/RECO" in outputDatasets,
"/ZeroBias/ReleaseValidation-reco-v1/RECO not in output datasets")
self.assertTrue("/ZeroBias/ReleaseValidation-AOD-v1/AOD" in outputDatasets,
"/ZeroBias/ReleaseValidation-AOD-v1/AOD not in output datasets")
self.assertTrue("/ZeroBias/ReleaseValidation-alca-v1/ALCARECO" in outputDatasets,
"/ZeroBias/ReleaseValidation-alca-v1/ALCARECO not in output datasets")
for i in range(1, 5):
self.assertTrue("/MinimumBias/ReleaseValidation-alca%d-v1/ALCARECO" % i in outputDatasets,
"/MinimumBias/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets" % i)
for i in range(1, 6):
self.assertTrue("/MinimumBias/ReleaseValidation-skim%d-v1/RECO-AOD" % i in outputDatasets,
"/MinimumBias/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets" % i)
return
def testGeneratorWorkflow(self):
"""
_testGeneratorWorkflow_
Test creating a request with an initial generator task
it mocks a request where there are 2 similar paths starting
from the generator, each one with a different PrimaryDataset, CMSSW configuration
and processed dataset. Dropping the RAW output as well.
Also include an ignored output module to keep things interesting...
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 6,
"IgnoredOutputModules": ["writeSkim2", "writeRAWDEBUGDIGI"],
"Task1": {
"TaskName": "GenSim",
"ConfigCacheID": generatorDoc,
"SplittingAlgo": "EventBased",
"RequestNumEvents": 10000,
"Seeding": "AutomaticSeeding",
"PrimaryDataset": "RelValTTBar",
},
"Task2": {
"TaskName": "DigiHLT_new",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "LumiBased",
"CMSSWVersion": "CMSSW_5_2_6",
"GlobalTag": "GR_39_P_V5:All",
"PrimaryDataset": "PURelValTTBar",
"KeepOutput": False
},
"Task3": {
"TaskName": "DigiHLT_ref",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "EventBased",
"CMSSWVersion": "CMSSW_5_2_7",
"GlobalTag": "GR_40_P_V5:All",
"AcquisitionEra": "ReleaseValidationNewConditions",
"ProcessingVersion": 3,
"ProcessingString": "Test",
"KeepOutput": False
},
"Task4": {
"TaskName": "Reco",
"InputTask": "DigiHLT_new",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgo": "FileBased",
"TransientOutputModules": ["writeRECO"]
},
"Task5": {
"TaskName": "ALCAReco",
"InputTask": "DigiHLT_ref",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgo": "LumiBased",
},
"Task6": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgo": "LumiBased",
}
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
# Test a malformed task chain definition
arguments['Task4']['TransientOutputModules'].append('writeAOD')
self.assertRaises(WMSpecFactoryException, factory.validateSchema,
arguments)
arguments['Task4']['TransientOutputModules'].remove('writeAOD')
try:
self.workload = factory.factoryWorkloadConstruction(
"PullingTheChain", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
import traceback
traceback.print_exc()
self.fail(msg)
testWMBSHelper = WMBSHelper(self.workload,
"GenSim",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
firstTask = self.workload.getTaskByPath("/PullingTheChain/GenSim")
self._checkTask(firstTask, arguments['Task1'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new"),
arguments['Task2'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref"),
arguments['Task3'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco"
), arguments['Task4'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref/ALCAReco"
), arguments['Task5'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco/Skims"
), arguments['Task6'], arguments)
# Verify the output datasets
outputDatasets = self.workload.listOutputDatasets()
self.assertEqual(len(outputDatasets), 11,
"Number of output datasets doesn't match")
self.assertTrue(
"/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM"
in outputDatasets,
"/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM not in output datasets"
)
self.assertFalse(
"/RelValTTBar/ReleaseValidation-reco-v1/RECO" in outputDatasets,
"/RelValTTBar/ReleaseValidation-reco-v1/RECO in output datasets")
self.assertTrue(
"/RelValTTBar/ReleaseValidation-AOD-v1/AOD" in outputDatasets,
"/RelValTTBar/ReleaseValidation-AOD-v1/AOD not in output datasets")
self.assertTrue(
"/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO"
in outputDatasets,
"/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO not in output datasets"
)
for i in range(1, 5):
self.assertTrue(
"/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO" % i
in outputDatasets,
"/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets"
% i)
for i in range(1, 6):
if i == 2:
continue
self.assertTrue(
"/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD" % i
in outputDatasets,
"/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets"
% i)
return
def __call__(self, workflowName, args):
workload = TaskChainWorkloadFactory.__call__(self, workflowName, args)
#delattr(workload.taskIterator().next().steps().data.application.configuration,
# 'configCacheUrl')
return workload
def testGeneratorWorkflow(self):
"""
_testGeneratorWorkflow_
Test creating a request with an initial generator task
it mocks a request where there are 2 similar paths starting
from the generator, each one with a different PrimaryDataset, CMSSW configuration
and processed dataset. Dropping the RAW output as well.
Also include an ignored output module to keep things interesting...
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain" : 6,
"IgnoredOutputModules" : ["writeSkim2", "writeRAWDEBUGDIGI"],
"Task1" :{
"TaskName" : "GenSim",
"ConfigCacheID" : generatorDoc,
"SplittingAlgo" : "EventBased",
"RequestNumEvents" : 10000,
"Seeding" : "AutomaticSeeding",
"PrimaryDataset" : "RelValTTBar",
},
"Task2" : {
"TaskName" : "DigiHLT_new",
"InputTask" : "GenSim",
"InputFromOutputModule" : "writeGENSIM",
"ConfigCacheID" : processorDocs['DigiHLT'],
"SplittingAlgo" : "LumiBased",
"CMSSWVersion" : "CMSSW_5_2_6",
"GlobalTag" : "GR_39_P_V5:All",
"PrimaryDataset" : "PURelValTTBar",
"KeepOutput" : False
},
"Task3" : {
"TaskName" : "DigiHLT_ref",
"InputTask" : "GenSim",
"InputFromOutputModule" : "writeGENSIM",
"ConfigCacheID" : processorDocs['DigiHLT'],
"SplittingAlgo" : "EventBased",
"CMSSWVersion" : "CMSSW_5_2_7",
"GlobalTag" : "GR_40_P_V5:All",
"AcquisitionEra" : "ReleaseValidationNewConditions",
"ProcessingVersion" : 3,
"ProcessingString" : "Test",
"KeepOutput" : False
},
"Task4" : {
"TaskName" : "Reco",
"InputTask" : "DigiHLT_new",
"InputFromOutputModule" : "writeRAWDIGI",
"ConfigCacheID" : processorDocs['Reco'],
"SplittingAlgo" : "FileBased",
"TransientOutputModules" : ["writeRECO"]
},
"Task5" : {
"TaskName" : "ALCAReco",
"InputTask" : "DigiHLT_ref",
"InputFromOutputModule" : "writeRAWDIGI",
"ConfigCacheID" : processorDocs['ALCAReco'],
"SplittingAlgo" : "LumiBased",
},
"Task6" : {
"TaskName" : "Skims",
"InputTask" : "Reco",
"InputFromOutputModule" : "writeRECO",
"ConfigCacheID" : processorDocs['Skims'],
"SplittingAlgo" : "LumiBased",
}
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
# Test a malformed task chain definition
arguments['Task4']['TransientOutputModules'].append('writeAOD')
self.assertRaises(WMSpecFactoryException, factory.validateSchema, arguments)
arguments['Task4']['TransientOutputModules'].remove('writeAOD')
try:
self.workload = factory.factoryWorkloadConstruction("PullingTheChain", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
import traceback
traceback.print_exc()
self.fail(msg)
testWMBSHelper = WMBSHelper(self.workload, "GenSim", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
firstTask = self.workload.getTaskByPath("/PullingTheChain/GenSim")
self._checkTask(firstTask, arguments['Task1'], arguments)
self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new"), arguments['Task2'], arguments)
self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref"), arguments['Task3'], arguments)
self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco"),
arguments['Task4'], arguments)
self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref/ALCAReco"),
arguments['Task5'], arguments)
self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco/Skims"),
arguments['Task6'], arguments)
# Verify the output datasets
outputDatasets = self.workload.listOutputDatasets()
self.assertEqual(len(outputDatasets), 11, "Number of output datasets doesn't match")
self.assertTrue("/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM" in outputDatasets,
"/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM not in output datasets")
self.assertFalse("/RelValTTBar/ReleaseValidation-reco-v1/RECO" in outputDatasets,
"/RelValTTBar/ReleaseValidation-reco-v1/RECO in output datasets")
self.assertTrue("/RelValTTBar/ReleaseValidation-AOD-v1/AOD" in outputDatasets,
"/RelValTTBar/ReleaseValidation-AOD-v1/AOD not in output datasets")
self.assertTrue("/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO" in outputDatasets,
"/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO not in output datasets")
for i in range(1, 5):
self.assertTrue("/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO" % i in outputDatasets,
"/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets" % i)
for i in range(1, 6):
if i == 2:
continue
self.assertTrue("/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD" % i in outputDatasets,
"/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets" % i)
return
def testGeneratorWorkflow(self):
"""
_testGeneratorWorkflow_
Test creating a request with an initial generator task
it mocks a request where there are 2 similar paths starting
from the generator, each one with a different PrimaryDataset, CMSSW configuration
and processed dataset. Dropping the RAW output as well.
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
processorDocs = makeProcessingConfigs(self.configDatabase)
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 6,
"Task1": {
"TaskName": "GenSim",
"ConfigCacheID": generatorDoc,
"SplittingAlgorithm": "EventBased",
"SplittingArguments": {
"events_per_job": 250
},
"RequestNumEvents": 10000,
"Seeding": "Automatic",
"PrimaryDataset": "RelValTTBar",
},
"Task2": {
"TaskName": "DigiHLT_new",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgorithm": "LumiBased",
"SplittingArguments": {
"lumis_per_job": 2
},
"CMSSWVersion": "CMSSW_5_2_6",
"GlobalTag": "GR_39_P_V5:All",
"PrimaryDataset": "PURelValTTBar",
"KeepOutput": False
},
"Task3": {
"TaskName": "DigiHLT_ref",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgorithm": "EventBased",
"SplittingArguments": {
"events_per_job": 100
},
"CMSSWVersion": "CMSSW_5_2_7",
"GlobalTag": "GR_40_P_V5:All",
"AcquisitionEra": "ReleaseValidationNewConditions",
"ProcessingVersion": 3,
"ProcessingString": "Test",
"KeepOutput": False
},
"Task4": {
"TaskName": "Reco",
"InputTask": "DigiHLT_new",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgorithm": "FileBased",
"SplittingArguments": {
"files_per_job": 1
},
},
"Task5": {
"TaskName": "ALCAReco",
"InputTask": "DigiHLT_ref",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgorithm": "LumiBased",
"SplittingArguments": {
"lumis_per_job": 8
},
},
"Task6": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgorithm": "LumiBased",
"SplittingArguments": {
"lumis_per_job": 10
},
}
}
factory = TaskChainWorkloadFactory()
try:
self.workload = factory("PullingTheChain", arguments)
except Exception, ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
def testPileupTaskChain(self):
"""
Test for multithreaded task chains where each step
may run with a different number of cores
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 2,
"Task1": {
"InputDataset": "/cosmics/whatever-input-v1/GEN-SIM",
"TaskName": "DIGI",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "LumiBased",
"LumisPerJob": 4,
"MCPileup": "/some/cosmics-mc-v1/GEN-SIM",
"DeterministicPileup": True,
"CMSSWVersion": "CMSSW_5_2_6",
"GlobalTag": "GR_39_P_V5:All",
"PrimaryDataset": "PURelValTTBar",
"AcquisitionEra": "CMSSW_5_2_6",
"ProcessingString": "ProcStr_Task1"
},
"Task2": {
"TaskName": "RECO",
"InputTask": "DIGI",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"DataPileup": "/some/minbias-data-v1/GEN-SIM",
"SplittingAlgo": "LumiBased",
"LumisPerJob": 2,
"GlobalTag": "GR_R_62_V3::All",
"AcquisitionEra": "CMSSW_5_2_7",
"ProcessingString": "ProcStr_Task2"
},
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
self.workload = factory.factoryWorkloadConstruction(
"PullingTheChain", arguments)
firstTask = self.workload.getTaskByPath("/PullingTheChain/DIGI")
cmsRunStep = firstTask.getStep("cmsRun1").getTypeHelper()
pileupData = cmsRunStep.getPileup()
self.assertFalse(hasattr(pileupData, "data"))
self.assertEqual(pileupData.mc.dataset,
["/some/cosmics-mc-v1/GEN-SIM"])
splitting = firstTask.jobSplittingParameters()
self.assertTrue(splitting["deterministicPileup"])
secondTask = self.workload.getTaskByPath(
"/PullingTheChain/DIGI/DIGIMergewriteRAWDIGI/RECO")
cmsRunStep = secondTask.getStep("cmsRun1").getTypeHelper()
pileupData = cmsRunStep.getPileup()
self.assertFalse(hasattr(pileupData, "mc"))
self.assertEqual(pileupData.data.dataset,
["/some/minbias-data-v1/GEN-SIM"])
splitting = secondTask.jobSplittingParameters()
self.assertFalse(splitting["deterministicPileup"])
def __call__(self, workflowName, args):
workload = TaskChainWorkloadFactory.__call__(self, workflowName, args)
#delattr(workload.taskIterator().next().steps().data.application.configuration,
# 'configCacheUrl')
return workload
def testGeneratorWorkflow(self):
"""
_testGeneratorWorkflow_
Test creating a request with an initial generator task
it mocks a request where there are 2 similar paths starting
from the generator, each one with a different PrimaryDataset, CMSSW configuration
and processed dataset. Dropping the RAW output as well.
Also include an ignored output module to keep things interesting...
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
processorDocs = makeProcessingConfigs(self.configDatabase)
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain" : 6,
"IgnoredOutputModules" : ["writeSkim2", "writeRAWDEBUGDIGI"],
"Task1" :{
"TaskName" : "GenSim",
"ConfigCacheID" : generatorDoc,
"SplittingAlgorithm" : "EventBased",
"SplittingArguments" : {"events_per_job" : 250},
"RequestNumEvents" : 10000,
"Seeding" : "Automatic",
"PrimaryDataset" : "RelValTTBar",
},
"Task2" : {
"TaskName" : "DigiHLT_new",
"InputTask" : "GenSim",
"InputFromOutputModule" : "writeGENSIM",
"ConfigCacheID" : processorDocs['DigiHLT'],
"SplittingAlgorithm" : "LumiBased",
"SplittingArguments" : {"lumis_per_job" : 2 },
"CMSSWVersion" : "CMSSW_5_2_6",
"GlobalTag" : "GR_39_P_V5:All",
"PrimaryDataset" : "PURelValTTBar",
"KeepOutput" : False
},
"Task3" : {
"TaskName" : "DigiHLT_ref",
"InputTask" : "GenSim",
"InputFromOutputModule" : "writeGENSIM",
"ConfigCacheID" : processorDocs['DigiHLT'],
"SplittingAlgorithm" : "EventBased",
"SplittingArguments" : {"events_per_job" : 100 },
"CMSSWVersion" : "CMSSW_5_2_7",
"GlobalTag" : "GR_40_P_V5:All",
"AcquisitionEra" : "ReleaseValidationNewConditions",
"ProcessingVersion" : 3,
"ProcessingString" : "Test",
"KeepOutput" : False
},
"Task4" : {
"TaskName" : "Reco",
"InputTask" : "DigiHLT_new",
"InputFromOutputModule" : "writeRAWDIGI",
"ConfigCacheID" : processorDocs['Reco'],
"SplittingAlgorithm" : "FileBased",
"SplittingArguments" : {"files_per_job" : 1 },
"TransientOutputModules" : ["writeRECO"]
},
"Task5" : {
"TaskName" : "ALCAReco",
"InputTask" : "DigiHLT_ref",
"InputFromOutputModule" : "writeRAWDIGI",
"ConfigCacheID" : processorDocs['ALCAReco'],
"SplittingAlgorithm" : "LumiBased",
"SplittingArguments" : {"lumis_per_job" : 8 },
},
"Task6" : {
"TaskName" : "Skims",
"InputTask" : "Reco",
"InputFromOutputModule" : "writeRECO",
"ConfigCacheID" : processorDocs['Skims'],
"SplittingAlgorithm" : "LumiBased",
"SplittingArguments" : {"lumis_per_job" : 10 },
}
}
factory = TaskChainWorkloadFactory()
# Test a malformed task chain definition
arguments['Task4']['TransientOutputModules'].append('writeAOD')
self.assertRaises(WMSpecFactoryException, factory.validateSchema, arguments)
arguments['Task4']['TransientOutputModules'].remove('writeAOD')
try:
factory.validateSchema(arguments)
self.workload = factory("PullingTheChain", arguments)
except Exception, ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
from __future__ import division, print_function
import unittest
from copy import deepcopy
from WMCore.ReqMgr.DataStructs.Request import initialize_clone
from WMCore.WMSpec.StdSpecs.ReReco import ReRecoWorkloadFactory
from WMCore.WMSpec.StdSpecs.StepChain import StepChainWorkloadFactory
from WMCore.WMSpec.StdSpecs.TaskChain import TaskChainWorkloadFactory
from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException
### Spec arguments definition with only key and its default value
RERECO_ARGS = ReRecoWorkloadFactory.getWorkloadCreateArgs()
STEPCHAIN_ARGS = StepChainWorkloadFactory.getWorkloadCreateArgs()
TASKCHAIN_ARGS = TaskChainWorkloadFactory.getWorkloadCreateArgs()
# inner Task/Step definition
STEP_ARGS = StepChainWorkloadFactory.getChainCreateArgs()
TASK_ARGS = TaskChainWorkloadFactory.getChainCreateArgs()
### Some original request dicts (ones to be cloned from)
rerecoOriginalArgs = {'Memory': 234, 'SkimName1': 'skim_2017', 'SkimInput1': 'RECOoutput',
'Skim1ConfigCacheID': 'abcdef',
'TimePerEvent': 1.2, 'RequestType': 'ReReco', 'RequestName': 'test_rereco'}
stepChainOriginalArgs = {'Memory': 1234, 'TimePerEvent': 1.2, 'RequestType': 'StepChain',
"ScramArch": ["slc6_amd64_gcc481"], "RequestName": "test_stepchain",
"Step1": {"ConfigCacheID": "blah", "GlobalTag": "PHYS18", "BlockWhitelist": ["A", "B"]},
"Step2": {"AcquisitionEra": "ACQERA", "ProcessingVersion": 3, "LumisPerJob": 10},
"StepChain": 2, "ConfigCacheID": None}
taskChainOriginalArgs = {'PrepID': None, 'Campaign': "MainTask", 'RequestType': 'TaskChain',
"ScramArch": ["slc6_amd64_gcc481", "slc7_amd64_gcc630"], "RequestName": "test_taskchain",
