def testRelValMCWithPileup(self):
"""
Configure, instantiate, install into WMBS and check that the
subscriptions in WMBS are setup correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "relvalmc_t"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {
"cosmics": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"],
"minbias": ["/some/minbias/dataset3"],
}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock")
testWMBSHelper.createSubscription()
# now run the tests on single workload instance installed into WMBS
# each of the subtests is dealing with specific tasks
self._generationTaskTest()
self._stepOneTaskTest()
self._stepTwoTaskTest()
def testRelValMC(self):
"""
Configure, instantiate, install into WMBS and check that the
subscriptions in WMBS are setup correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "relvalmc_t"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Generation", "SomeBlock")
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# now run the tests on single workload instance installed into WMBS
# each of the subtests is dealing with specific tasks
self._generationTaskTest()
self._stepTwoTaskTest()
self._stepOneTaskTest()
return
def testRelValMCWithPileup(self):
"""
Configure, instantiate, install into WMBS and check that the
subscriptions in WMBS are setup correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "relvalmc_t"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {"cosmics": ["/some/cosmics/dataset1","/some/cosmics/dataset2"],
"minbias": ["/some/minbias/dataset3"]}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Generation", "SomeBlock")
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# now run the tests on single workload instance installed into WMBS
# each of the subtests is dealing with specific tasks
self._generationTaskTest()
self._stepOneTaskTest()
self._stepTwoTaskTest()
def testPileupFetcherOnRelValMC(self):
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "pileupfetcher_t"
# in this test, try not to define generation task datatier (first output module
# should be automatically picked up)
#defaultArguments["GenDataTier"] = "GEN-SIM-RAW"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments[
"RecoConfigCacheID"] = self.injectReconstructionConfig()
defaultArguments["AlcaRecoConfigCacheID"] = self.injectAlcaRecoConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {
"cosmics": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"minbias": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
]
}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % ("/tmp", testWorkload.name())
self.dirsToErase.append(pathBase)
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(defaultArguments, task, taskPath)
def testPileupFetcherOnRelValMC(self):
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "pileupfetcher_t"
# in this test, try not to define generation task datatier (first output module
# should be automatically picked up)
# defaultArguments["GenDataTier"] = "GEN-SIM-RAW"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["RecoConfigCacheID"] = self.injectReconstructionConfig()
defaultArguments["AlcaRecoConfigCacheID"] = self.injectAlcaRecoConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {
"cosmics": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
"minbias": [
"/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
],
}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
# Since this is test of the fetcher - The loading from WMBS isn't
# really necessary because the fetching happens before the workflow
# is inserted into WMBS: feed the workload instance directly into fetcher:
fetcher = PileupFetcher()
creator = SandboxCreator()
pathBase = "%s/%s" % ("/tmp", testWorkload.name())
self.dirsToErase.append(pathBase)
for topLevelTask in testWorkload.taskIterator():
for taskNode in topLevelTask.nodeIterator():
# this is how the call to PileupFetcher is happening
# from the SandboxCreator test
task = WMTask.WMTaskHelper(taskNode)
taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
fetcher.setWorkingDirectory(taskPath)
# create Sandbox for the fetcher ...
creator._makePathonPackage(taskPath)
fetcher(task)
self._queryPileUpConfigFile(defaultArguments, task, taskPath)
print sys.argv[
0], "PROCESSING_VERSION NUM_EVENTS GenConfigCacheID RecoConfigCacheID AlcaRecoConfigCacheID"
sys.exit(1)
else:
arguments["ProcessingVersion"] = sys.argv[1]
numEvents = int(sys.argv[2])
arguments["GenConfigCacheID"] = sys.argv[3]
arguments["RecoConfigCacheID"] = sys.argv[4]
arguments["AlcaRecoConfigCacheID"] = sys.argv[5]
connectToDB()
workloadName = "RelValMC-%s" % arguments["ProcessingVersion"]
workloadFile = "relValMC-%s.pkl" % arguments["ProcessingVersion"]
os.mkdir(workloadName)
workload = relValMCWorkload(workloadName, arguments)
workloadPath = os.path.join(workloadName, workloadFile)
workload.setOwner("*****@*****.**")
workload.setSpecUrl(workloadPath)
# Build a sandbox using TaskMaker
taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadPath)
myThread = threading.currentThread()
myThread.transaction.begin()
for workloadTask in workload.taskIterator():
inputFileset = Fileset(name=workloadTask.getPathName())
print sys.argv[0], "PROCESSING_VERSION NUM_EVENTS GenConfigCacheID RecoConfigCacheID AlcaRecoConfigCacheID"
sys.exit(1)
else:
arguments["ProcessingVersion"] = sys.argv[1]
numEvents = int(sys.argv[2])
arguments["GenConfigCacheID"] = sys.argv[3]
arguments["RecoConfigCacheID"] = sys.argv[4]
arguments["AlcaRecoConfigCacheID"] = sys.argv[5]
connectToDB()
workloadName = "RelValMC-%s" % arguments["ProcessingVersion"]
workloadFile = "relValMC-%s.pkl" % arguments["ProcessingVersion"]
os.mkdir(workloadName)
workload = relValMCWorkload(workloadName, arguments)
workloadPath = os.path.join(workloadName, workloadFile)
workload.setOwner("*****@*****.**")
workload.setSpecUrl(workloadPath)
# Build a sandbox using TaskMaker
taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(workloadPath)
myThread = threading.currentThread()
myThread.transaction.begin()
for workloadTask in workload.taskIterator():