def testRelValMCWithPileup(self):
"""
Configure, instantiate, install into WMBS and check that the
subscriptions in WMBS are setup correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "relvalmc_t"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {
"cosmics": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"],
"minbias": ["/some/minbias/dataset3"],
}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock")
testWMBSHelper.createSubscription()
# now run the tests on single workload instance installed into WMBS
# each of the subtests is dealing with specific tasks
self._generationTaskTest()
self._stepOneTaskTest()
self._stepTwoTaskTest()
def testRelValMC(self):
"""
Configure, instantiate, install into WMBS and check that the
subscriptions in WMBS are setup correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "relvalmc_t"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Generation", "SomeBlock")
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# now run the tests on single workload instance installed into WMBS
# each of the subtests is dealing with specific tasks
self._generationTaskTest()
self._stepTwoTaskTest()
self._stepOneTaskTest()
return
def testRelValMCWithPileup(self):
"""
Configure, instantiate, install into WMBS and check that the
subscriptions in WMBS are setup correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "relvalmc_t"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {"cosmics": ["/some/cosmics/dataset1","/some/cosmics/dataset2"],
"minbias": ["/some/minbias/dataset3"]}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Generation", "SomeBlock")
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# now run the tests on single workload instance installed into WMBS
# each of the subtests is dealing with specific tasks
self._generationTaskTest()
self._stepOneTaskTest()
self._stepTwoTaskTest()
def testPileupFetcherOnRelValMC(self):
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "pileupfetcher_t"
# in this test, try not to define generation task datatier (first output module
# should be automatically picked up)
#defaultArguments["GenDataTier"] = "GEN-SIM-RAW"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments[
"RecoConfigCacheID"] = self.injectReconstructionConfig()
defaultArguments["AlcaRecoConfigCacheID"] = self.injectAlcaRecoConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {
"cosmics": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"minbias": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % ("/tmp", testWorkload.name())
self.dirsToErase.append(pathBase)
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(defaultArguments, task, taskPath)
def testPileupFetcherOnRelValMC(self):
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "pileupfetcher_t"
# in this test, try not to define generation task datatier (first output module
# should be automatically picked up)
# defaultArguments["GenDataTier"] = "GEN-SIM-RAW"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["RecoConfigCacheID"] = self.injectReconstructionConfig()
defaultArguments["AlcaRecoConfigCacheID"] = self.injectAlcaRecoConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {
"cosmics": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"minbias": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % ("/tmp", testWorkload.name())
self.dirsToErase.append(pathBase)
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(defaultArguments, task, taskPath)
from WMCore.WMBS.Subscription import Subscription
from WMCore.WMBS.Workflow import Workflow
from WMComponent.DBSBuffer.Database.Interface.DBSBufferFile import DBSBufferFile
from WMCore.DataStructs.Run import Run
from WMCore.WMSpec.StdSpecs.RelValMC import relValMCWorkload, getTestArguments
from DBSAPI.dbsApi import DbsApi
from WMCore.WMSpec.Makers.TaskMaker import TaskMaker
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
numEvents = None
# The default arguments are set in:
# WMCore/WMSpec/StdSpecs/RelValMC.py
arguments = getTestArguments()
#arguments["GenConfigCacheID"] will be taken from CLI
arguments["GenJobSplitAlgo"] = "EventBased"
arguments["GenJobSplitArgs"] = {"events_per_job": 1000}
#arguments["RecoConfigCacheID"] will be taken from CLI
arguments["RecoJobSplitAlgo"] = "FileBased"
arguments["RecoJobSplitArgs"] = {"files_per_job": 1}
#arguments["AlcaRecoConfigCacheID"] will be taken from CLI
arguments["AlcaRecoJobSplitAlgo"] = "FileBased"
arguments["AlcaRecoJobSplitArgs"] = {"files_per_job": 1}
if len(sys.argv) != 6:
print "Usage:"
print sys.argv[
0], "PROCESSING_VERSION NUM_EVENTS GenConfigCacheID RecoConfigCacheID AlcaRecoConfigCacheID"
sys.exit(1)
from WMCore.WMBS.Subscription import Subscription
from WMCore.WMBS.Workflow import Workflow
from WMComponent.DBSBuffer.Database.Interface.DBSBufferFile import DBSBufferFile
from WMCore.DataStructs.Run import Run
from WMCore.WMSpec.StdSpecs.RelValMC import relValMCWorkload, getTestArguments
from DBSAPI.dbsApi import DbsApi
from WMCore.WMSpec.Makers.TaskMaker import TaskMaker
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
numEvents = None
# The default arguments are set in:
# WMCore/WMSpec/StdSpecs/RelValMC.py
arguments = getTestArguments()
#arguments["GenConfigCacheID"] will be taken from CLI
arguments["GenJobSplitAlgo"] = "EventBased"
arguments["GenJobSplitArgs"] = {"events_per_job": 1000}
#arguments["RecoConfigCacheID"] will be taken from CLI
arguments["RecoJobSplitAlgo"] = "FileBased"
arguments["RecoJobSplitArgs"] = {"files_per_job": 1}
#arguments["AlcaRecoConfigCacheID"] will be taken from CLI
arguments["AlcaRecoJobSplitAlgo"] = "FileBased"
arguments["AlcaRecoJobSplitArgs"] = {"files_per_job": 1}
if len(sys.argv) != 6:
print "Usage:"
print sys.argv[0], "PROCESSING_VERSION NUM_EVENTS GenConfigCacheID RecoConfigCacheID AlcaRecoConfigCacheID"
sys.exit(1)