def testInvalidKeys_initialize_clone(self):
"""
Test some undefined/assignment user arguments in initialize_clone
"""
# test with unknown key
requestArgs = {'Multicore': 1, 'WRONG': 'Alan'}
originalArgs = deepcopy(taskChainOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
with self.assertRaises(WMSpecFactoryException):
TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)
# test with assignment key
requestArgs = {'Multicore': 1, 'TrustSitelists': False}
originalArgs = deepcopy(taskChainOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
with self.assertRaises(WMSpecFactoryException):
TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)
# test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception)
requestArgs = {'SubRequestType': "RelVal", "Step2": {"LumisPerJob": 5, "Campaign": "Step2Camp"}}
originalArgs = deepcopy(taskChainOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
with self.assertRaises(WMSpecFactoryException):
TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)
# make sure unsupported keys are remove from both top and inner dicts
requestArgs = {}
originalArgs = deepcopy(taskChainOriginalArgs)
originalArgs['BAD'] = 'KEY'
originalArgs['Task1']['SiteWhitelist'] = ['This_key_should_not_be_here']
originalArgs['Task1']['Alan'] = False
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
self.assertFalse('BAD' in cloneArgs)
self.assertFalse('SiteWhitelist' in cloneArgs['Task1'])
self.assertFalse('Alan' in cloneArgs['Task1'])
def testPileupFetcherOnMC(self):
pileupMcArgs = TaskChainWorkloadFactory.getTestArguments()
pileupMcArgs['Task1']["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO"
pileupMcArgs['Task1']["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW"
pileupMcArgs['Task1']["ConfigCacheID"] = self.injectGenerationConfig()
pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs)
# now that the workload was created and args validated, we can add this PileupConfig
pileupMcArgs["PileupConfig"] = parsePileupConfig(pileupMcArgs['Task1']["MCPileup"],
pileupMcArgs['Task1']["DataPileup"])
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % (self.testDir, testWorkload.name())
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(pileupMcArgs["PileupConfig"], task, taskPath)
def testMultithreadedTaskChain(self):
"""
Test for multithreaded task chains where all steps run with the same
number of cores
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
arguments['Task1']['Multicore'] = 4
arguments['Task2']['Multicore'] = 4
arguments['Task3']['Multicore'] = 4
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction(
"MultiChain", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
hlt = self.workload.getTaskByPath('/MultiChain/HLTD')
reco = self.workload.getTaskByPath(
'/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = self.workload.getTaskByPath(
'/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT'
)
hltStep = hlt.getStepHelper("cmsRun1")
recoStep = reco.getStepHelper("cmsRun1")
miniAODStep = miniAOD.getStepHelper("cmsRun1")
self.assertEqual(hltStep.getNumberOfCores(), 4)
self.assertEqual(recoStep.getNumberOfCores(), 4)
self.assertEqual(miniAODStep.getNumberOfCores(), 4)
return
def testMultithreadedTaskChain(self):
"""
Test multi-task TaskChain with default and multicore settings
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
arguments.pop('Multicore', None)
arguments.pop('Memory', None)
arguments.pop('EventStreams', None)
arguments['Task1'].pop('Multicore', None)
arguments['Task1'].pop('Memory', None)
arguments['Task2'].pop('Multicore', None)
arguments['Task2'].pop('Memory', None)
arguments['Task2'].pop('EventStreams', None)
arguments['Task3'].pop('Multicore', None)
arguments['Task3'].pop('Memory', None)
arguments['Task3'].pop('EventStreams', None)
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("MultiChain", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath(
'/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(hlt.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
self.assertEqual(reco.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
self.assertEqual(miniAOD.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
# now all with 16 cores, 16 event streams and 8GB of memory inherited from the top level
arguments['Multicore'] = 16
arguments['Memory'] = 8000
arguments['EventStreams'] = 16
testWorkload = factory.factoryWorkloadConstruction("MultiChain", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath(
'/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(hlt.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
self.assertEqual(reco.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
self.assertEqual(miniAOD.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
return
def testMultithreadedTasksTaskChain(self):
"""
Test for multithreaded task chains where each step
may run with a different number of cores
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction(
"MultiChain2", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
hlt = self.workload.getTaskByPath('/MultiChain2/HLTD')
reco = self.workload.getTaskByPath(
'/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = self.workload.getTaskByPath(
'/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT'
)
hltMemory = hlt.jobSplittingParameters(
)['performance']['memoryRequirement']
recoMemory = reco.jobSplittingParameters(
)['performance']['memoryRequirement']
aodMemory = miniAOD.jobSplittingParameters(
)['performance']['memoryRequirement']
hltStep = hlt.getStepHelper("cmsRun1")
recoStep = reco.getStepHelper("cmsRun1")
miniAODStep = miniAOD.getStepHelper("cmsRun1")
self.assertEqual(hltStep.getNumberOfCores(), 4)
self.assertEqual(recoStep.getNumberOfCores(), 8)
self.assertEqual(miniAODStep.getNumberOfCores(), 1)
self.assertEqual(recoMemory, 8000.0)
self.assertEqual(aodMemory, 2000.0)
self.assertEqual(hltMemory, 4000.0)
return
def testTrustFlags(self):
"""
_testTrustFlags_
Given a taskChain with 4 tasks, test whether TrustSitelists is set for
the top level tasks and TrustPUSitelists is properly set to all tasks.
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
testArguments.update(createMultiGTArgs())
testArguments["ConfigCacheUrl"] = self.testInit.couchUrl
testArguments["CouchDBName"] = self.testInit.couchDbName
testArguments["Task1"]["ConfigCacheID"] = processorDocs['DigiHLT']
testArguments["Task2"]["ConfigCacheID"] = processorDocs['Reco']
testArguments["Task3"]["ConfigCacheID"] = processorDocs['ALCAReco']
testArguments["Task4"]["ConfigCacheID"] = processorDocs['Skims']
arguments = testArguments
factory = TaskChainWorkloadFactory()
workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)
for task in workload.getAllTasks():
flags = task.getTrustSitelists().values()
self.assertEqual(flags, [False, False])
# set both flags to true now
workload.setTrustLocationFlag(True, True)
for task in workload.getAllTasks():
flags = task.getTrustSitelists()
if task.isTopOfTree():
self.assertEqual(flags.values(), [True, True])
elif task.taskType() in ["Cleanup", "LogCollect"]:
self.assertEqual(flags.values(), [False, False])
else:
self.assertFalse(flags['trustlists'])
self.assertTrue(flags['trustPUlists'])
# set both to false now
workload.setTrustLocationFlag(False, False)
for task in workload.getAllTasks(cpuOnly=True):
flags = task.getTrustSitelists().values()
self.assertEqual(flags, [False, False])
return
def testInvalidKeys_initialize_clone(self):
"""
Test some undefined/assignment user arguments in initialize_clone
"""
# test with unknown key
requestArgs = {'Multicore': 1, 'WRONG': 'Alan', 'ReqMgr2Only': False}
originalArgs = deepcopy(mcOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
with self.assertRaises(WMSpecFactoryException):
MonteCarloWorkloadFactory().factoryWorkloadConstruction(
cloneArgs["RequestName"], cloneArgs)
# test with assignment key
requestArgs = {
'Multicore': 1,
'TrustSitelists': False,
'ReqMgr2Only': False
}
originalArgs = deepcopy(mcOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
with self.assertRaises(WMSpecFactoryException):
MonteCarloWorkloadFactory().factoryWorkloadConstruction(
cloneArgs["RequestName"], cloneArgs)
# test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception)
requestArgs = {
'SubRequestType': "RelVal",
"Step2": {
"LumisPerJob": 5,
"Campaign": "Step2Camp"
}
}
originalArgs = deepcopy(taskChainOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS,
TASK_ARGS)
with self.assertRaises(WMSpecFactoryException):
TaskChainWorkloadFactory().factoryWorkloadConstruction(
cloneArgs["RequestName"], cloneArgs)
def testPileupTaskChain(self):
"""
Test for multithreaded task chains where each step
may run with a different number of cores
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 2,
"Task1": {
"InputDataset": "/cosmics/whatever-input-v1/GEN-SIM",
"TaskName": "DIGI",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "LumiBased",
"LumisPerJob": 4,
"MCPileup": "/some/cosmics-mc-v1/GEN-SIM",
"DeterministicPileup": True,
"CMSSWVersion": "CMSSW_5_2_6",
"GlobalTag": "GR_39_P_V5:All",
"PrimaryDataset": "PURelValTTBar",
"AcquisitionEra": "CMSSW_5_2_6",
"ProcessingString": "ProcStr_Task1"
},
"Task2": {
"TaskName": "RECO",
"InputTask": "DIGI",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"DataPileup": "/some/minbias-data-v1/GEN-SIM",
"SplittingAlgo": "LumiBased",
"LumisPerJob": 2,
"GlobalTag": "GR_R_62_V3::All",
"AcquisitionEra": "CMSSW_5_2_7",
"ProcessingString": "ProcStr_Task2"
},
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
self.workload = factory.factoryWorkloadConstruction(
"PullingTheChain", arguments)
firstTask = self.workload.getTaskByPath("/PullingTheChain/DIGI")
cmsRunStep = firstTask.getStep("cmsRun1").getTypeHelper()
pileupData = cmsRunStep.getPileup()
self.assertFalse(hasattr(pileupData, "data"))
self.assertEqual(pileupData.mc.dataset,
["/some/cosmics-mc-v1/GEN-SIM"])
splitting = firstTask.jobSplittingParameters()
self.assertTrue(splitting["deterministicPileup"])
secondTask = self.workload.getTaskByPath(
"/PullingTheChain/DIGI/DIGIMergewriteRAWDIGI/RECO")
cmsRunStep = secondTask.getStep("cmsRun1").getTypeHelper()
pileupData = cmsRunStep.getPileup()
self.assertFalse(hasattr(pileupData, "mc"))
self.assertEqual(pileupData.data.dataset,
["/some/minbias-data-v1/GEN-SIM"])
splitting = secondTask.jobSplittingParameters()
self.assertFalse(splitting["deterministicPileup"])
def testMultithreadedTasksTaskChain(self):
"""
Test for multithreaded task chains where each step
may run with a different number of cores
"""
arguments = self.buildMultithreadedTaskChain(self.differentNCores)
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("MultiChain2", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain2/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath(
'/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
hltMemory = hlt.jobSplittingParameters()['performance']['memoryRequirement']
recoMemory = reco.jobSplittingParameters()['performance']['memoryRequirement']
aodMemory = miniAOD.jobSplittingParameters()['performance']['memoryRequirement']
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task2']['Multicore'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task3']['Multicore'])
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task2']['EventStreams'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task3']['EventStreams'])
self.assertEqual(hltMemory, arguments['Memory'])
self.assertEqual(recoMemory, arguments['Task2']['Memory'])
self.assertEqual(aodMemory, arguments['Task3']['Memory'])
# and test another mix of cores, event streams and memory
arguments['Task1']['Multicore'] = arguments.pop('Multicore', None)
arguments['Task1']['Memory'] = arguments.pop('Memory', None)
arguments['Task2'].pop('Multicore', None)
arguments['Task2'].pop('Memory', None)
arguments['Task3']['Multicore'] = 2
arguments['EventStreams'] = 32
arguments['Task2']['EventStreams'] = 8
arguments['Task3']['EventStreams'] = 0
testWorkload = factory.factoryWorkloadConstruction("MultiChain2", arguments)
hlt = testWorkload.getTaskByPath('/MultiChain2/HLTD')
reco = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
miniAOD = testWorkload.getTaskByPath(
'/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')
hltMemory = hlt.jobSplittingParameters()['performance']['memoryRequirement']
recoMemory = reco.jobSplittingParameters()['performance']['memoryRequirement']
aodMemory = miniAOD.jobSplittingParameters()['performance']['memoryRequirement']
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task1']['Multicore'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), 1)
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task3']['Multicore'])
self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task2']['EventStreams'])
self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task3']['EventStreams'])
self.assertEqual(hltMemory, arguments['Task1']['Memory'])
self.assertEqual(recoMemory, 2300.0)
self.assertEqual(aodMemory, arguments['Task3']['Memory'])
return
def test1TaskMemCoresSettings(self):
"""
_test1TaskMemCoresSettings_
Make sure the multicore and memory setings are properly propagated to
all steps. Single step in a task.
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"ConfigCacheUrl": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"TaskChain": 1,
"Task1": {
"TaskName": "TaskOne",
"ConfigCacheID": generatorDoc,
"RequestNumEvents": 10000,
"PrimaryDataset": "RelValTTBar",
},
}
testArguments.update(arguments)
factory = TaskChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)
taskPaths = ['/TestTaskChainWorkload/TaskOne',
'/TestTaskChainWorkload/TaskOne/LogCollectForTaskOne',
'/TestTaskChainWorkload/TaskOne/TaskOneMergewriteGENSIM',
'/TestTaskChainWorkload/TaskOne/TaskOneMergewriteGENSIM/TaskOnewriteGENSIMMergeLogCollect',
'/TestTaskChainWorkload/TaskOne/TaskOneCleanupUnmergedwriteGENSIM']
for task in taskPaths:
taskObj = testWorkload.getTaskByPath(task)
if taskObj.taskType() in ('Production', 'Processing'):
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
elif taskObj.taskType() == 'LogCollect':
stepHelper = taskObj.getStepHelper('logCollect1')
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
# now play with cores at top level
testArguments['Multicore'] = 2
testArguments['EventStreams'] = 2
testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)
for task in taskPaths:
taskObj = testWorkload.getTaskByPath(task)
if taskObj.taskType() in ('Production', 'Processing'):
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
if step == 'cmsRun1':
self.assertEqual(stepHelper.getNumberOfCores(), testArguments['Multicore'])
self.assertEqual(stepHelper.getNumberOfStreams(), testArguments["EventStreams"])
else:
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
elif taskObj.taskType() == 'LogCollect':
stepHelper = taskObj.getStepHelper('logCollect1')
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
# last but not least, play with cores at task level
testArguments['Task1']['Multicore'] = 2
testArguments['Task1']['EventStreams'] = 2
testArguments.pop('Multicore', None)
testArguments.pop('EventStreams', None)
testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)
for task in taskPaths:
taskObj = testWorkload.getTaskByPath(task)
if taskObj.taskType() in ('Production', 'Processing'):
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
if step == 'cmsRun1':
self.assertEqual(stepHelper.getNumberOfCores(), testArguments['Task1']['Multicore'])
self.assertEqual(stepHelper.getNumberOfStreams(), testArguments['Task1']['EventStreams'])
else:
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
elif taskObj.taskType() == 'LogCollect':
stepHelper = taskObj.getStepHelper('logCollect1')
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
return
def testGeneratorWorkflow(self):
"""
_testGeneratorWorkflow_
Test creating a request with an initial generator task
it mocks a request where there are 2 similar paths starting
from the generator, each one with a different PrimaryDataset, CMSSW configuration
and processed dataset. Dropping the RAW output as well.
Also include an ignored output module to keep things interesting...
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 6,
"IgnoredOutputModules": ["writeSkim2", "writeRAWDEBUGDIGI"],
"Task1": {
"TaskName": "GenSim",
"ConfigCacheID": generatorDoc,
"SplittingAlgo": "EventBased",
"RequestNumEvents": 10000,
"Seeding": "AutomaticSeeding",
"PrimaryDataset": "RelValTTBar",
},
"Task2": {
"TaskName": "DigiHLT_new",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "LumiBased",
"CMSSWVersion": "CMSSW_5_2_6",
"GlobalTag": "GR_39_P_V5:All",
"PrimaryDataset": "PURelValTTBar",
"KeepOutput": False
},
"Task3": {
"TaskName": "DigiHLT_ref",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "EventBased",
"CMSSWVersion": "CMSSW_5_2_7",
"GlobalTag": "GR_40_P_V5:All",
"AcquisitionEra": "ReleaseValidationNewConditions",
"ProcessingVersion": 3,
"ProcessingString": "Test",
"KeepOutput": False
},
"Task4": {
"TaskName": "Reco",
"InputTask": "DigiHLT_new",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgo": "FileBased",
"TransientOutputModules": ["writeRECO"]
},
"Task5": {
"TaskName": "ALCAReco",
"InputTask": "DigiHLT_ref",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgo": "LumiBased",
},
"Task6": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgo": "LumiBased",
}
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
# Test a malformed task chain definition
arguments['Task4']['TransientOutputModules'].append('writeAOD')
self.assertRaises(WMSpecFactoryException, factory.validateSchema,
arguments)
arguments['Task4']['TransientOutputModules'].remove('writeAOD')
try:
self.workload = factory.factoryWorkloadConstruction(
"PullingTheChain", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
import traceback
traceback.print_exc()
self.fail(msg)
testWMBSHelper = WMBSHelper(self.workload,
"GenSim",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
firstTask = self.workload.getTaskByPath("/PullingTheChain/GenSim")
self._checkTask(firstTask, arguments['Task1'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new"),
arguments['Task2'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref"),
arguments['Task3'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco"
), arguments['Task4'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref/ALCAReco"
), arguments['Task5'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco/Skims"
), arguments['Task6'], arguments)
# Verify the output datasets
outputDatasets = self.workload.listOutputDatasets()
self.assertEqual(len(outputDatasets), 11,
"Number of output datasets doesn't match")
self.assertTrue(
"/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM"
in outputDatasets,
"/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM not in output datasets"
)
self.assertFalse(
"/RelValTTBar/ReleaseValidation-reco-v1/RECO" in outputDatasets,
"/RelValTTBar/ReleaseValidation-reco-v1/RECO in output datasets")
self.assertTrue(
"/RelValTTBar/ReleaseValidation-AOD-v1/AOD" in outputDatasets,
"/RelValTTBar/ReleaseValidation-AOD-v1/AOD not in output datasets")
self.assertTrue(
"/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO"
in outputDatasets,
"/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO not in output datasets"
)
for i in range(1, 5):
self.assertTrue(
"/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO" % i
in outputDatasets,
"/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets"
% i)
for i in range(1, 6):
if i == 2:
continue
self.assertTrue(
"/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD" % i
in outputDatasets,
"/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets"
% i)
return
def testMultipleGlobalTags(self):
"""
_testMultipleGlobalTags_
Test creating a workload that starts in a processing task
with an input dataset, and has different globalTags
and CMSSW versions (with corresponding scramArch) in
each task
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
lumiDict = {"1": [[2, 4], [8, 50]], "2": [[100, 200], [210, 210]]}
lumiDict2 = {"1": [[2, 4], [8, 40]], "2": [[100, 150], [210, 210]]}
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "DefaultGlobalTag",
"LumiList": lumiDict,
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 4,
"Task1": {
"TaskName": "DigiHLT",
"ConfigCacheID": processorDocs['DigiHLT'],
"InputDataset": "/MinimumBias/Commissioning10-v4/GEN-SIM",
"SplittingAlgo": "FileBased",
},
"Task2": {
"TaskName": "Reco",
"InputTask": "DigiHLT",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgo": "FileBased",
"GlobalTag": "GlobalTagForReco",
"CMSSWVersion": "CMSSW_3_1_2",
"ScramArch": "CompatibleRECOArch",
"PrimaryDataset": "ZeroBias",
"LumiList": lumiDict2,
},
"Task3": {
"TaskName": "ALCAReco",
"InputTask": "Reco",
"InputFromOutputModule": "writeALCA",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgo": "FileBased",
"GlobalTag": "GlobalTagForALCAReco",
"CMSSWVersion": "CMSSW_3_1_3",
"ScramArch": "CompatibleALCAArch",
},
"Task4": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgo": "FileBased",
}
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction(
"YankingTheChain", arguments)
except Exception as ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
testWMBSHelper = WMBSHelper(self.workload,
"DigiHLT",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._checkTask(
self.workload.getTaskByPath("/YankingTheChain/DigiHLT"),
arguments['Task1'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"),
arguments['Task2'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"
), arguments['Task3'], arguments)
self._checkTask(
self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"
), arguments['Task4'], arguments)
digi = self.workload.getTaskByPath("/YankingTheChain/DigiHLT")
self.assertEqual(lumiDict, digi.getLumiMask())
digiStep = digi.getStepHelper("cmsRun1")
self.assertEqual(digiStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])
# Make sure this task has a different lumilist than the global one
reco = self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
self.assertEqual(lumiDict2, reco.getLumiMask())
recoStep = reco.getStepHelper("cmsRun1")
self.assertEqual(recoStep.getGlobalTag(),
arguments['Task2']['GlobalTag'])
self.assertEqual(recoStep.getCMSSWVersion(),
arguments['Task2']['CMSSWVersion'])
self.assertEqual(recoStep.getScramArch(),
arguments['Task2']['ScramArch'])
alca = self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"
)
self.assertEqual(lumiDict, alca.getLumiMask())
alcaStep = alca.getStepHelper("cmsRun1")
self.assertEqual(alcaStep.getGlobalTag(),
arguments['Task3']['GlobalTag'])
self.assertEqual(alcaStep.getCMSSWVersion(),
arguments['Task3']['CMSSWVersion'])
self.assertEqual(alcaStep.getScramArch(),
arguments['Task3']['ScramArch'])
skim = self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"
)
skimStep = skim.getStepHelper("cmsRun1")
self.assertEqual(skimStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(skimStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(skimStep.getScramArch(), arguments['ScramArch'])
# Verify the output datasets
outputDatasets = self.workload.listOutputDatasets()
self.assertEqual(len(outputDatasets), 14,
"Number of output datasets doesn't match")
self.assertTrue(
"/MinimumBias/ReleaseValidation-RawDigiFilter-v1/RAW-DIGI"
in outputDatasets,
"/MinimumBias/ReleaseValidation-RawDigiFilter-v1/RAW-DIGI not in output datasets"
)
self.assertTrue(
"/MinimumBias/ReleaseValidation-RawDebugDigiFilter-v1/RAW-DEBUG-DIGI"
in outputDatasets,
"/MinimumBias/ReleaseValidation-RawDebugDigiFilter-v1/RAW-DEBUG-DIGI not in output datasets"
)
self.assertTrue(
"/ZeroBias/ReleaseValidation-reco-v1/RECO" in outputDatasets,
"/ZeroBias/ReleaseValidation-reco-v1/RECO not in output datasets")
self.assertTrue(
"/ZeroBias/ReleaseValidation-AOD-v1/AOD" in outputDatasets,
"/ZeroBias/ReleaseValidation-AOD-v1/AOD not in output datasets")
self.assertTrue(
"/ZeroBias/ReleaseValidation-alca-v1/ALCARECO" in outputDatasets,
"/ZeroBias/ReleaseValidation-alca-v1/ALCARECO not in output datasets"
)
for i in range(1, 5):
self.assertTrue(
"/MinimumBias/ReleaseValidation-alca%d-v1/ALCARECO" % i
in outputDatasets,
"/MinimumBias/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets"
% i)
for i in range(1, 6):
self.assertTrue(
"/MinimumBias/ReleaseValidation-skim%d-v1/RECO-AOD" % i
in outputDatasets,
"/MinimumBias/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets"
% i)
return
def testMultipleGlobalTags(self):
"""
_testMultipleGlobalTags_
Test creating a workload that starts in a processing task
with an input dataset, and has different globalTags
and CMSSW versions (with corresponding scramArch) in
each task
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
lumiDict = {"1": [[2, 4], [8, 50]], "2": [[100, 200], [210, 210]]}
lumiDict2 = {"1": [[2, 4], [8, 40]], "2": [[100, 150], [210, 210]]}
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "DefaultGlobalTag",
"LumiList": lumiDict,
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 4,
"Task1": {
"TaskName": "DigiHLT",
"ConfigCacheID": processorDocs['DigiHLT'],
"InputDataset": "/MinimumBias/Commissioning10-v4/GEN-SIM",
"SplittingAlgo": "FileBased",
},
"Task2": {
"TaskName": "Reco",
"InputTask": "DigiHLT",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgo": "FileBased",
"GlobalTag": "GlobalTagForReco",
"CMSSWVersion": "CMSSW_3_1_2",
"ScramArch": "CompatibleRECOArch",
"PrimaryDataset": "ZeroBias",
"LumiList": lumiDict2,
},
"Task3": {
"TaskName": "ALCAReco",
"InputTask": "Reco",
"InputFromOutputModule": "writeALCA",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgo": "FileBased",
"GlobalTag": "GlobalTagForALCAReco",
"CMSSWVersion": "CMSSW_3_1_3",
"ScramArch": "CompatibleALCAArch",
},
"Task4": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgo": "FileBased",
}
}
testArguments.update(arguments)
arguments = testArguments
factory = TaskChainWorkloadFactory()
try:
self.workload = factory.factoryWorkloadConstruction(
"YankingTheChain", arguments)
except Exception, ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
def testGeneratorWorkflow(self):
"""
_testGeneratorWorkflow_
Test creating a request with an initial generator task
it mocks a request where there are 2 similar paths starting
from the generator, each one with a different PrimaryDataset, CMSSW configuration
and processed dataset. Dropping the RAW output as well.
Also include an ignored output module to keep things interesting...
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 6,
"IgnoredOutputModules": ["writeSkim2", "writeRAWDEBUGDIGI"],
"Task1": {
"TaskName": "GenSim",
"ConfigCacheID": generatorDoc,
"SplittingAlgo": "EventBased",
"RequestNumEvents": 10000,
"Seeding": "AutomaticSeeding",
"PrimaryDataset": "RelValTTBar",
},
"Task2": {
"TaskName": "DigiHLT_new",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "LumiBased",
"CMSSWVersion": "CMSSW_5_2_6",
"GlobalTag": "GR_39_P_V5:All",
"PrimaryDataset": "PURelValTTBar",
"KeepOutput": False
},
"Task3": {
"TaskName": "DigiHLT_ref",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgo": "EventBased",
"CMSSWVersion": "CMSSW_5_2_7",
"GlobalTag": "GR_40_P_V5:All",
"AcquisitionEra": "ReleaseValidationNewConditions",
"ProcessingVersion": 3,
"ProcessingString": "Test",
"KeepOutput": False
},
"Task4": {
"TaskName": "Reco",
"InputTask": "DigiHLT_new",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgo": "FileBased",
"TransientOutputModules": ["writeRECO"]
},
"Task5": {
"TaskName": "ALCAReco",
"InputTask": "DigiHLT_ref",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgo": "LumiBased",
},
"Task6": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgo": "LumiBased",
}
}
testArguments.update(arguments)
arguments = testArguments
print arguments
factory = TaskChainWorkloadFactory()
# Test a malformed task chain definition
arguments['Task4']['TransientOutputModules'].append('writeAOD')
self.assertRaises(WMSpecFactoryException, factory.validateSchema,
arguments)
arguments['Task4']['TransientOutputModules'].remove('writeAOD')
try:
self.workload = factory.factoryWorkloadConstruction(
"PullingTheChain", arguments)
except Exception, ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
import traceback
traceback.print_exc()
self.fail(msg)
def testGeneratorWorkflow(self):
"""
_testGeneratorWorkflow_
Test creating a request with an initial generator task
it mocks a request where there are 2 similar paths starting
from the generator, each one with a different PrimaryDataset, CMSSW configuration
and processed dataset. Dropping the RAW output as well.
"""
generatorDoc = makeGeneratorConfig(self.configDatabase)
processorDocs = makeProcessingConfigs(self.configDatabase)
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 6,
"Task1": {
"TaskName": "GenSim",
"ConfigCacheID": generatorDoc,
"SplittingAlgorithm": "EventBased",
"SplittingArguments": {
"events_per_job": 250
},
"RequestNumEvents": 10000,
"Seeding": "Automatic",
"PrimaryDataset": "RelValTTBar",
},
"Task2": {
"TaskName": "DigiHLT_new",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgorithm": "LumiBased",
"SplittingArguments": {
"lumis_per_job": 2
},
"CMSSWVersion": "CMSSW_5_2_6",
"GlobalTag": "GR_39_P_V5:All",
"PrimaryDataset": "PURelValTTBar",
"KeepOutput": False
},
"Task3": {
"TaskName": "DigiHLT_ref",
"InputTask": "GenSim",
"InputFromOutputModule": "writeGENSIM",
"ConfigCacheID": processorDocs['DigiHLT'],
"SplittingAlgorithm": "EventBased",
"SplittingArguments": {
"events_per_job": 100
},
"CMSSWVersion": "CMSSW_5_2_7",
"GlobalTag": "GR_40_P_V5:All",
"AcquisitionEra": "ReleaseValidationNewConditions",
"ProcessingVersion": 3,
"ProcessingString": "Test",
"KeepOutput": False
},
"Task4": {
"TaskName": "Reco",
"InputTask": "DigiHLT_new",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['Reco'],
"SplittingAlgorithm": "FileBased",
"SplittingArguments": {
"files_per_job": 1
},
},
"Task5": {
"TaskName": "ALCAReco",
"InputTask": "DigiHLT_ref",
"InputFromOutputModule": "writeRAWDIGI",
"ConfigCacheID": processorDocs['ALCAReco'],
"SplittingAlgorithm": "LumiBased",
"SplittingArguments": {
"lumis_per_job": 8
},
},
"Task6": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "writeRECO",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgorithm": "LumiBased",
"SplittingArguments": {
"lumis_per_job": 10
},
}
}
factory = TaskChainWorkloadFactory()
try:
self.workload = factory("PullingTheChain", arguments)
except Exception, ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
def testMultipleGlobalTags(self):
"""
_testMultipleGlobalTags_
Test creating a workload that starts in a processing task
with an input dataset, and has different globalTags
and CMSSW versions (with corresponding scramArch) in
each task
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
testArguments = TaskChainWorkloadFactory.getTestArguments()
testArguments.update(createMultiGTArgs())
lumiDict = {"1": [[2, 4], [8, 50]], "2": [[100, 200], [210, 210]]}
testArguments["ConfigCacheUrl"] = self.testInit.couchUrl
testArguments["CouchDBName"] = self.testInit.couchDbName
testArguments["Task1"]["LumiList"] = lumiDict
testArguments["Task1"]["ConfigCacheID"] = processorDocs['DigiHLT']
testArguments["Task2"]["ConfigCacheID"] = processorDocs['Reco']
testArguments["Task3"]["ConfigCacheID"] = processorDocs['ALCAReco']
testArguments["Task4"]["ConfigCacheID"] = processorDocs['Skims']
arguments = testArguments
factory = TaskChainWorkloadFactory()
self.workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)
testWMBSHelper = WMBSHelper(self.workload, "DigiHLT", "SomeBlock", cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT"), arguments['Task1'], arguments)
self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"),
arguments['Task2'],
arguments)
self._checkTask(self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"),
arguments['Task3'], arguments)
self._checkTask(self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"),
arguments['Task4'], arguments)
digi = self.workload.getTaskByPath("/YankingTheChain/DigiHLT")
self.assertEqual(lumiDict, digi.getLumiMask().getCompactList())
digiStep = digi.getStepHelper("cmsRun1")
self.assertEqual(digiStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])
# Make sure this task has a different lumilist than the global one
reco = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
recoStep = reco.getStepHelper("cmsRun1")
self.assertEqual(recoStep.getGlobalTag(), arguments['Task2']['GlobalTag'])
self.assertEqual(recoStep.getCMSSWVersion(), arguments['Task2']['CMSSWVersion'])
self.assertEqual(recoStep.getScramArch(), arguments['Task2']['ScramArch'])
alca = self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco")
alcaStep = alca.getStepHelper("cmsRun1")
self.assertEqual(alcaStep.getGlobalTag(), arguments['Task3']['GlobalTag'])
self.assertEqual(alcaStep.getCMSSWVersion(), arguments['Task3']['CMSSWVersion'])
self.assertEqual(alcaStep.getScramArch(), arguments['Task3']['ScramArch'])
skim = self.workload.getTaskByPath(
"/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims")
skimStep = skim.getStepHelper("cmsRun1")
self.assertEqual(skimStep.getGlobalTag(), arguments['GlobalTag'])
self.assertEqual(skimStep.getCMSSWVersion(), arguments['CMSSWVersion'])
self.assertEqual(skimStep.getScramArch(), arguments['ScramArch'])
# Verify the output datasets
outputDatasets = self.workload.listOutputDatasets()
self.assertEqual(len(outputDatasets), 14, "Number of output datasets doesn't match")
self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-RawDigiFilter-FAKE-v1/RAW-DIGI" in outputDatasets)
self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-RawDebugDigiFilter-FAKE-v1/RAW-DEBUG-DIGI" in outputDatasets)
self.assertTrue("/ZeroBias/ReleaseValidation-reco-FAKE-v1/RECO" in outputDatasets)
self.assertTrue("/ZeroBias/ReleaseValidation-AOD-FAKE-v1/AOD" in outputDatasets)
self.assertTrue("/ZeroBias/ReleaseValidation-alca-FAKE-v1/ALCARECO" in outputDatasets)
for i in range(1, 5):
self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-alca%d-FAKE-v1/ALCARECO" % i in outputDatasets)
for i in range(1, 6):
self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-skim%d-FAKE-v1/RECO-AOD" % i in outputDatasets)
return
def testProcessingWithScenarios(self):
"""
_testProcessingWithScenarios_
Test creating a processing workload with an input dataset that
uses scenarios instead of configuration files
"""
processorDocs = makeProcessingConfigs(self.configDatabase)
arguments = {
"AcquisitionEra": "ReleaseValidation",
"Requestor": "*****@*****.**",
"CMSSWVersion": "CMSSW_3_5_8",
"ScramArch": "slc5_ia32_gcc434",
"ProcessingVersion": 1,
"GlobalTag": "GR10_P_v4::All",
"CouchURL": self.testInit.couchUrl,
"CouchDBName": self.testInit.couchDbName,
"SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
"DashboardHost": "127.0.0.1",
"DashboardPort": 8884,
"TaskChain": 4,
"Task1": {
"TaskName": "DigiHLT",
"ConfigCacheID": processorDocs['DigiHLT'],
"InputDataset": "/MinimumBias/Commissioning10-v4/GEN-SIM",
"SplittingAlgorithm": "FileBased",
"SplittingArguments": {
"files_per_job": 1
},
},
"Task2": {
"TaskName": "Reco",
"InputTask": "DigiHLT",
"InputFromOutputModule": "writeRAWDIGI",
"ProcScenario": "pp",
"ScenarioMethod": "promptReco",
"ScenarioArguments": {
'outputs': [{
'dataTier': "RECO",
'moduleLabel': "RECOoutput"
}, {
'dataTier': "AOD",
'moduleLabel': "AODoutput"
}, {
'dataTier': "ALCARECO",
'moduleLabel': "ALCARECOoutput"
}]
},
"SplittingAlgorithm": "FileBased",
"SplittingArguments": {
"files_per_job": 1
},
},
"Task3": {
"TaskName": "ALCAReco",
"InputTask": "Reco",
"InputFromOutputModule": "ALCARECOoutput",
"ProcScenario": "pp",
"ScenarioMethod": "alcaReco",
"ScenarioArguments": {
'outputs': [{
'dataTier': "ALCARECO",
'moduleLabel': "ALCARECOoutput"
}]
},
"SplittingAlgorithm": "FileBased",
"SplittingArguments": {
"files_per_job": 1
},
},
"Task4": {
"TaskName": "Skims",
"InputTask": "Reco",
"InputFromOutputModule": "RECOoutput",
"ConfigCacheID": processorDocs['Skims'],
"SplittingAlgorithm": "FileBased",
"SplittingArguments": {
"files_per_job": 10
},
}
}
factory = TaskChainWorkloadFactory()
try:
self.workload = factory("YankingTheChain", arguments)
except Exception, ex:
msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
self.fail(msg)
