def testRelValMCWithPileup(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
The input configuration includes pileup input files.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ProcConfigCacheID"] = self.injectMonteCarloConfig()
# add pile up configuration
defaultArguments["PileupConfig"] = {"mc": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"],
"data": ["/some/minbias/dataset3"]}
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
return
def testRelValMCWithPileup(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
The input configuration includes pileup input files.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
# add pile up configuration
defaultArguments["PileupConfig"] = {
"mc": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"],
"data": ["/some/minbias/dataset3"]
}
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask,
testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
return
def testMonteCarlo(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask,
testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
return
def getMCArgs():
mcArgs = getTestArguments()
mcArgs.update({
"CouchURL": None,
"CouchDBName": None,
"ConfigCacheDoc" : None
})
mcArgs.pop('ConfigCacheDoc')
return mcArgs
def getMCArgs():
mcArgs = getTestArguments()
mcArgs.update({
"CouchURL": None,
"CouchDBName": None,
"ConfigCacheDoc" : None
})
mcArgs.pop('ConfigCacheDoc')
return mcArgs
def testMonteCarloExtension(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test. This uses a non-zero first
event and lumi. Check that the splitting arguments are correctly
set for the lfn counter.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
defaultArguments["FirstEvent"] = 3571428573
defaultArguments["FirstLumi"] = 26042
defaultArguments["TimePerEvent"] = 15
defaultArguments["FilterEfficiency"] = 0.014
defaultArguments["TotalTime"] = 28800
initial_lfn_counter = 26042 # Same as the previous number of jobs + 1 which is the same value of the first lumi
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask,
testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
productionSplitting = productionTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in productionSplitting,
"No initial lfn counter was stored")
self.assertEqual(productionSplitting["initial_lfn_counter"],
initial_lfn_counter, "Wrong initial LFN counter")
for outputMod in ["OutputA", "OutputB"]:
mergeTask = testWorkload.getTaskByPath(
'/TestWorkload/Production/ProductionMerge%s' % outputMod)
mergeSplitting = mergeTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in mergeSplitting,
"No initial lfn counter was stored")
self.assertEqual(mergeSplitting["initial_lfn_counter"],
initial_lfn_counter, "Wrong initial LFN counter")
return
def testMonteCarlo(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ProdConfigCacheID"] = self.injectMonteCarloConfig()
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock")
testWMBSHelper.createSubscription()
self._commonMonteCarloTest()
def testMonteCarloExtension(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test. This uses a non-zero first
event and lumi. Check that the splitting arguments are correctly
set for the lfn counter.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ProcConfigCacheID"] = self.injectMonteCarloConfig()
defaultArguments["FirstEvent"] = 3571428573
defaultArguments["FirstLumi"] = 26042
defaultArguments["TimePerEvent"] = 15
defaultArguments["FilterEfficiency"] = 0.014
defaultArguments["TotalTime"] = 28800
initial_lfn_counter = 26042 # Same as the previous number of jobs + 1 which is the same value of the first lumi
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
productionSplitting = productionTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored")
self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter")
for outputMod in ["OutputA", "OutputB"]:
mergeTask = testWorkload.getTaskByPath('/TestWorkload/Production/ProductionMerge%s' % outputMod)
mergeSplitting = mergeTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored")
self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter")
return
from WMCore.Configuration import loadConfigurationFile
from WMCore.WMBS.File import File
from WMCore.WMBS.Fileset import Fileset
from WMCore.WMBS.Subscription import Subscription
from WMCore.WMBS.Workflow import Workflow
from WMCore.DataStructs.Run import Run
from WMCore.WMSpec.StdSpecs.MonteCarlo import MonteCarloWorkloadFactory, getTestArguments
from DBSAPI.dbsApi import DbsApi
from WMCore.WMSpec.Makers.TaskMaker import TaskMaker
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
arguments = getTestArguments()
numEvents = None
if len(sys.argv) != 3:
print("Usage:")
print("./injectMonteCarloWorkflow.py PROCESSING_VERSION NUM_EVENTS")
sys.exit(1)
else:
arguments["ProcessingVersion"] = sys.argv[1]
numEvents = int(sys.argv[2])
connectToDB()
workloadName = "MonteCarlo-%s" % arguments["ProcessingVersion"]
workloadFile = "MonteCarlo-%s.pkl" % arguments["ProcessingVersion"]
os.mkdir(workloadName)
from WMCore.Configuration import loadConfigurationFile
from WMCore.WMBS.File import File
from WMCore.WMBS.Fileset import Fileset
from WMCore.WMBS.Subscription import Subscription
from WMCore.WMBS.Workflow import Workflow
from WMCore.DataStructs.Run import Run
from WMCore.WMSpec.StdSpecs.MonteCarlo import MonteCarloWorkloadFactory, getTestArguments
from DBSAPI.dbsApi import DbsApi
from WMCore.WMSpec.Makers.TaskMaker import TaskMaker
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
arguments = getTestArguments()
numEvents = None
if len(sys.argv) != 3:
print("Usage:")
print("./injectMonteCarloWorkflow.py PROCESSING_VERSION NUM_EVENTS")
sys.exit(1)
else:
arguments["ProcessingVersion"] = sys.argv[1]
numEvents = int(sys.argv[2])
connectToDB()
workloadName = "MonteCarlo-%s" % arguments["ProcessingVersion"]
workloadFile = "MonteCarlo-%s.pkl" % arguments["ProcessingVersion"]
os.mkdir(workloadName)