def testMCWithLHE(self):
"""
_testMCWithLHE_
Create a MonteCarlo workflow with a variation on the type of work
done, this refers to the previous LHEStepZero where the input
can be .lhe files and there is more than one lumi per job.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
defaultArguments["LheInputFiles"] = "True"
defaultArguments["EventsPerJob"] = 200
defaultArguments["EventsPerLumi"] = 50
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
splitting = productionTask.jobSplittingParameters()
self.assertEqual(splitting["events_per_job"], 200)
self.assertEqual(splitting["events_per_lumi"], 50)
self.assertEqual(splitting["lheInputFiles"], True)
self.assertFalse(splitting["deterministicPileup"])
return
def testPileupFetcherOnMC(self):
pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments()
pileupMcArgs["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO"
pileupMcArgs["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW"
pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs)
# now that the workload was created and args validated, we can add this PileupConfig
pileupMcArgs["PileupConfig"] = parsePileupConfig(pileupMcArgs["MCPileup"], pileupMcArgs["DataPileup"])
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % (self.testDir, testWorkload.name())
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def testInvalidKeys_initialize_clone(self):
"""
Test some undefined/assignment user arguments in initialize_clone
"""
# test with unknown key
requestArgs = {'Multicore': 1, 'WRONG': 'Alan'}
originalArgs = deepcopy(mcOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
with self.assertRaises(WMSpecFactoryException):
MonteCarloWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)
# test with assignment key
requestArgs = {'Multicore': 1, 'TrustSitelists': False}
originalArgs = deepcopy(mcOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
with self.assertRaises(WMSpecFactoryException):
MonteCarloWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)
# test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception)
requestArgs = {'SubRequestType': "RelVal", "Step2": {"LumisPerJob": 5, "Campaign": "Step2Camp"}}
originalArgs = deepcopy(taskChainOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
with self.assertRaises(WMSpecFactoryException):
TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)
# make sure unsupported keys are remove from both top and inner dicts
requestArgs = {}
originalArgs = deepcopy(taskChainOriginalArgs)
originalArgs['BAD'] = 'KEY'
originalArgs['Task1']['SiteWhitelist'] = ['This_key_should_not_be_here']
originalArgs['Task1']['Alan'] = False
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
self.assertFalse('BAD' in cloneArgs)
self.assertFalse('SiteWhitelist' in cloneArgs['Task1'])
self.assertFalse('Alan' in cloneArgs['Task1'])
def testMonteCarlo(self):
"""
_testMonteCarlo_
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
return
def testPileupFetcherOnMC(self):
pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments()
pileupMcArgs["PileupConfig"] = {"cosmics": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"],
"minbias": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]}
pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs)
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % (self.testDir, testWorkload.name())
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def testPileupFetcherOnMC(self):
pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments()
pileupMcArgs["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO"
pileupMcArgs["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW"
pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", pileupMcArgs)
# now that the workload was created and args validated, we can add this PileupConfig
pileupMcArgs["PileupConfig"] = parsePileupConfig(
pileupMcArgs["MCPileup"], pileupMcArgs["DataPileup"])
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % (self.testDir, testWorkload.name())
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def testMCWithPileup(self):
"""
_testMCWithPileup_
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
The input configuration includes pileup input files.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
# Add pileup inputs
defaultArguments["MCPileup"] = COSMICS_PU
defaultArguments["DataPileup"] = DATA_PU
defaultArguments["DeterministicPileup"] = True
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
cmsRunStep = productionTask.getStep("cmsRun1").getTypeHelper()
pileupData = cmsRunStep.getPileup()
self.assertEqual(pileupData.data.dataset, [DATA_PU])
self.assertEqual(pileupData.mc.dataset, [COSMICS_PU])
splitting = productionTask.jobSplittingParameters()
self.assertTrue(splitting["deterministicPileup"])
return
def testPileupFetcherOnMC(self):
pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments()
pileupMcArgs["PileupConfig"] = {
"cosmics": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"minbias": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", pileupMcArgs)
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % (self.testDir, testWorkload.name())
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
def validateSchema(self, schema):
"""
_validateSchema_
Check for required fields, and some skim facts
"""
MonteCarloWorkloadFactory.validateSchema(self, schema)
#Now let's validate new stuff
self.validateEventsPerLumi(schema)
def testMonteCarloExtension(self):
"""
_testMonteCarloExtension_
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test. This uses a non-zero first
lumi. Check that the splitting arguments are correctly
set for the lfn counter.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
defaultArguments["FirstLumi"] = 10001
defaultArguments["EventsPerJob"] = 100
defaultArguments["FirstEvent"] = 10001
#defaultArguments["FirstEvent"] = 10001
initial_lfn_counter = 100 # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
productionSplitting = productionTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in productionSplitting,
"No initial lfn counter was stored")
self.assertEqual(productionSplitting["initial_lfn_counter"],
initial_lfn_counter, "Wrong initial LFN counter")
for outputMod in ["OutputA", "OutputB"]:
mergeTask = testWorkload.getTaskByPath(
'/TestWorkload/Production/ProductionMerge%s' % outputMod)
mergeSplitting = mergeTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in mergeSplitting,
"No initial lfn counter was stored")
self.assertEqual(mergeSplitting["initial_lfn_counter"],
initial_lfn_counter, "Wrong initial LFN counter")
return
def testMemCoresSettings(self):
"""
_testMemCoresSettings_
Make sure the multicore and memory setings are properly propagated to
all tasks and steps.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", defaultArguments)
# test default values
taskObj = testWorkload.getTask('Production')
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
# then test Memory requirements
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
# now test case where args are provided
defaultArguments["Multicore"] = 6
defaultArguments["Memory"] = 4600.0
defaultArguments["EventStreams"] = 3
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", defaultArguments)
taskObj = testWorkload.getTask('Production')
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
if step == 'cmsRun1':
self.assertEqual(stepHelper.getNumberOfCores(),
defaultArguments["Multicore"])
self.assertEqual(stepHelper.getNumberOfStreams(),
defaultArguments["EventStreams"])
else:
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
# then test Memory requirements
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'],
defaultArguments["Memory"])
return
def __call__(self, workloadName, arguments):
"""
Create a workload instance for a LHE Step0 request
Just pass it down to MonteCarlo
"""
#Override splitting arguments
# Splitting arguments
timePerEvent = int(arguments.get('TimePerEvent', 60))
filterEfficiency = float(arguments.get('FilterEfficiency', 1.0))
totalTime = int(arguments.get('TotalTime', 9 * 3600))
self.totalEvents = int(int(arguments['RequestNumEvents']) / filterEfficiency)
if arguments.get("LheInputFiles", False) == True \
or arguments.get("LheInputFiles", False) == "True":
self.lheInputFiles = True
# These are mostly place holders because the job splitting algo and
# parameters will be updated after the workflow has been created.
eventsPerJob = int(totalTime/timePerEvent/filterEfficiency)
self.prodJobSplitAlgo = arguments.setdefault("ProdJobSplitAlgo", 'EventBased')
self.prodJobSplitArgs = arguments.setdefault("ProdJobSplitArgs",
{"events_per_job": eventsPerJob,
"events_per_lumi": arguments['EventsPerLumi']})
self.prodJobSplitArgs.setdefault("lheInputFiles", self.lheInputFiles)
mcWorkload = MonteCarloWorkloadFactory.__call__(self, workloadName, arguments)
mcWorkload.setBlockCloseSettings(mcWorkload.getBlockCloseMaxWaitTime(), 5,
250000000, mcWorkload.getBlockCloseMaxSize())
return mcWorkload
def testMemCoresSettings(self):
"""
_testMemCoresSettings_
Make sure the multicore and memory setings are properly propagated to
all tasks and steps.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
# test default values
taskObj = testWorkload.getTask('Production')
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
# then test Memory requirements
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], 2300.0)
# now test case where args are provided
defaultArguments["Multicore"] = 6
defaultArguments["Memory"] = 4600.0
defaultArguments["EventStreams"] = 3
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
taskObj = testWorkload.getTask('Production')
for step in ('cmsRun1', 'stageOut1', 'logArch1'):
stepHelper = taskObj.getStepHelper(step)
if step == 'cmsRun1':
self.assertEqual(stepHelper.getNumberOfCores(), defaultArguments["Multicore"])
self.assertEqual(stepHelper.getNumberOfStreams(), defaultArguments["EventStreams"])
else:
self.assertEqual(stepHelper.getNumberOfCores(), 1)
self.assertEqual(stepHelper.getNumberOfStreams(), 0)
# then test Memory requirements
perfParams = taskObj.jobSplittingParameters()['performance']
self.assertEqual(perfParams['memoryRequirement'], defaultArguments["Memory"])
return
def getMCArgs():
mcArgs = MonteCarloWorkloadFactory.getTestArguments()
mcArgs.update({
"CouchURL": None,
"CouchDBName": None,
"ConfigCacheDoc" : None
})
mcArgs.pop('ConfigCacheDoc')
return mcArgs
def getMCArgs():
mcArgs = MonteCarloWorkloadFactory.getTestArguments()
mcArgs.update({
"CouchURL": None,
"CouchDBName": None,
"ConfigCacheDoc": None
})
mcArgs.pop('ConfigCacheDoc')
return mcArgs
def testMonteCarloExtension(self):
"""
_testMonteCarloExtension_
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test. This uses a non-zero first
lumi. Check that the splitting arguments are correctly
set for the lfn counter.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
defaultArguments["FirstLumi"] = 10001
defaultArguments["EventsPerJob"] = 100
defaultArguments["FirstEvent"] = 10001
# defaultArguments["FirstEvent"] = 10001
initial_lfn_counter = (
100
) # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath("/TestWorkload/Production")
productionSplitting = productionTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored")
self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter")
for outputMod in ["OutputA", "OutputB"]:
mergeTask = testWorkload.getTaskByPath("/TestWorkload/Production/ProductionMerge%s" % outputMod)
mergeSplitting = mergeTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored")
self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter")
return
def testMonteCarlo(self):
"""
_testMonteCarlo_
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
return
def testInvalidKeys_initialize_clone(self):
"""
Test some undefined/assignment user arguments in initialize_clone
"""
# test with unknown key
requestArgs = {'Multicore': 1, 'WRONG': 'Alan', 'ReqMgr2Only': False}
originalArgs = deepcopy(mcOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
with self.assertRaises(WMSpecFactoryException):
MonteCarloWorkloadFactory().factoryWorkloadConstruction(
cloneArgs["RequestName"], cloneArgs)
# test with assignment key
requestArgs = {
'Multicore': 1,
'TrustSitelists': False,
'ReqMgr2Only': False
}
originalArgs = deepcopy(mcOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
with self.assertRaises(WMSpecFactoryException):
MonteCarloWorkloadFactory().factoryWorkloadConstruction(
cloneArgs["RequestName"], cloneArgs)
# test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception)
requestArgs = {
'SubRequestType': "RelVal",
"Step2": {
"LumisPerJob": 5,
"Campaign": "Step2Camp"
}
}
originalArgs = deepcopy(taskChainOriginalArgs)
cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS,
TASK_ARGS)
with self.assertRaises(WMSpecFactoryException):
TaskChainWorkloadFactory().factoryWorkloadConstruction(
cloneArgs["RequestName"], cloneArgs)
def __call__(self, workloadName, arguments):
"""
Create a workload instance for a LHE Step0 request
Just pass it down to MonteCarlo
"""
#Override splitting arguments
# Splitting arguments
timePerEvent = int(arguments.get('TimePerEvent', 60))
filterEfficiency = float(arguments.get('FilterEfficiency', 1.0))
totalTime = int(arguments.get('TotalTime', 9 * 3600))
self.totalEvents = int(int(arguments['RequestNumEvents']) / filterEfficiency)
# These are mostly place holders because the job splitting algo and
# parameters will be updated after the workflow has been created.
eventsPerJob = int(totalTime/timePerEvent/filterEfficiency)
self.prodJobSplitAlgo = arguments.setdefault("ProdJobSplitAlgo", 'EventBased')
self.prodJobSplitArgs = arguments.setdefault("ProdJobSplitArgs",
{"events_per_job": eventsPerJob,
"events_per_lumi": arguments['EventsPerLumi']})
return MonteCarloWorkloadFactory.__call__(self, workloadName, arguments)
def __call__(self, workflowName, args):
workload = MonteCarloWorkloadFactory.__call__(self, workflowName, args)
delattr(workload.taskIterator().next().steps().data.application.configuration,
'configCacheUrl')
return workload
if len(sys.argv) != 3:
print("Usage:")
print("./injectMonteCarloWorkflow.py PROCESSING_VERSION NUM_EVENTS")
sys.exit(1)
else:
arguments["ProcessingVersion"] = sys.argv[1]
numEvents = int(sys.argv[2])
connectToDB()
workloadName = "MonteCarlo-%s" % arguments["ProcessingVersion"]
workloadFile = "MonteCarlo-%s.pkl" % arguments["ProcessingVersion"]
os.mkdir(workloadName)
factory = MonteCarloWorkloadFactory()
workload = factory(workloadName, arguments)
workloadPath = os.path.join(workloadName, workloadFile)
workload.setOwner("*****@*****.**")
workload.setSpecUrl(workloadPath)
# Build a sandbox using TaskMaker
taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadPath)
myThread = threading.currentThread()
myThread.transaction.begin()
for workloadTask in workload.taskIterator():
def __init__(self):
MonteCarloWorkloadFactory.__init__(self)
def makeWorkload(self, schema):
factory = MonteCarloWorkloadFactory()
return factory(schema['RequestName'], schema).data
Unittests for IteratorTools functions
"""
from __future__ import division, print_function
from pprint import pprint
import unittest
from copy import deepcopy
from WMCore.ReqMgr.DataStructs.Request import initialize_clone
from WMCore.WMSpec.StdSpecs.MonteCarlo import MonteCarloWorkloadFactory
from WMCore.WMSpec.StdSpecs.ReReco import ReRecoWorkloadFactory
from WMCore.WMSpec.StdSpecs.StepChain import StepChainWorkloadFactory
from WMCore.WMSpec.StdSpecs.TaskChain import TaskChainWorkloadFactory
from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException
### Spec arguments definition with only key and its default value
MC_ARGS = MonteCarloWorkloadFactory.getWorkloadCreateArgs()
RERECO_ARGS = ReRecoWorkloadFactory.getWorkloadCreateArgs()
STEPCHAIN_ARGS = StepChainWorkloadFactory.getWorkloadCreateArgs()
TASKCHAIN_ARGS = TaskChainWorkloadFactory.getWorkloadCreateArgs()
# inner Task/Step definition
STEP_ARGS = StepChainWorkloadFactory.getChainCreateArgs()
TASK_ARGS = TaskChainWorkloadFactory.getChainCreateArgs()
### Some original request dicts (ones to be cloned from)
mcOriginalArgs = {
'Memory': 1234,
'TimePerEvent': 1.2,
'RequestType': 'MonteCarlo',
"LheInputFiles": True,
"ScramArch": ["slc6_amd64_gcc481"],
"RequestName": "test_mc"
def __call__(self, workflowName, args):
workload = MonteCarloWorkloadFactory.__call__(self, workflowName, args)
#delattr(workload.taskIterator().next().steps().data.application.configuration,
# 'configCacheUrl')
return workload
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = ['/TestWorkload/Production',
'/TestWorkload/Production/ProductionMergeOutputB',
'/TestWorkload/Production/ProductionMergeOutputA']
expWfTasks = ['/TestWorkload/Production',
'/TestWorkload/Production/LogCollect',
'/TestWorkload/Production/ProductionCleanupUnmergedOutputA',
'/TestWorkload/Production/ProductionCleanupUnmergedOutputB',
'/TestWorkload/Production/ProductionMergeOutputA',
'/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect',
'/TestWorkload/Production/ProductionMergeOutputB',
'/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect']
expFsets = ['FILESET_DEFINED_DURING_RUNTIME',
'/TestWorkload/Production/unmerged-OutputBUSER',
'/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive',
'/TestWorkload/Production/ProductionMergeOutputA/merged-MergedRECO',
'/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive',
'/TestWorkload/Production/ProductionMergeOutputB/merged-MergedUSER',
'/TestWorkload/Production/unmerged-logArchive',
'/TestWorkload/Production/unmerged-OutputARECO']
subMaps = ['FILESET_DEFINED_DURING_RUNTIME',
(6,
'/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive',
'/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect',
'MinFileBased',
'LogCollect'),
(3,
'/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive',
'/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect',
'MinFileBased',
'LogCollect'),
(8,
'/TestWorkload/Production/unmerged-logArchive',
'/TestWorkload/Production/LogCollect',
'MinFileBased',
'LogCollect'),
(7,
'/TestWorkload/Production/unmerged-OutputARECO',
'/TestWorkload/Production/ProductionCleanupUnmergedOutputA',
'SiblingProcessingBased',
'Cleanup'),
(5,
'/TestWorkload/Production/unmerged-OutputARECO',
'/TestWorkload/Production/ProductionMergeOutputA',
'ParentlessMergeBySize',
'Merge'),
(4,
'/TestWorkload/Production/unmerged-OutputBUSER',
'/TestWorkload/Production/ProductionCleanupUnmergedOutputB',
'SiblingProcessingBased',
'Cleanup'),
(2,
'/TestWorkload/Production/unmerged-OutputBUSER',
'/TestWorkload/Production/ProductionMergeOutputB',
'ParentlessMergeBySize',
'Merge')]
testArguments = MonteCarloWorkloadFactory.getTestArguments()
testArguments["CouchURL"] = os.environ["COUCHURL"]
testArguments["CouchDBName"] = TEST_DB_NAME
testArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
myMask = Mask(FirstRun=1, FirstLumi=1, FirstEvent=1, LastRun=1, LastLumi=10, LastEvent=1000)
testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask,
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# same function as in WMBSHelper, otherwise we cannot know which fileset name is
maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)])
topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest()
expFsets[0] = topFilesetName
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subMaps[0] = (1, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production')
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
### create another top level subscription
myMask = Mask(FirstRun=1, FirstLumi=11, FirstEvent=1001, LastRun=1, LastLumi=20, LastEvent=2000)
testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask,
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# same function as in WMBSHelper, otherwise we cannot know which fileset name is
maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)])
topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest()
expFsets.append(topFilesetName)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subMaps.append((9, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production'))
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
def __init__(self):
MonteCarloWorkloadFactory.__init__(self)
self.lheInputFiles = False
