def addNewFilesToOpenSubscriptions(self, *elements):
"""Inject new files to wmbs for running elements that have new files.
Assumes elements are from the same workflow"""
if not self.params['LocalQueueFlag']:
return
wmspec = None
for ele in elements:
if not ele.isRunning() or not ele['SubscriptionId'] or not ele:
continue
if not ele['Inputs'] or not ele['OpenForNewData']:
continue
if not wmspec:
wmspec = self.backend.getWMSpec(ele['RequestName'])
blockName, dbsBlock = self._getDBSBlock(ele, wmspec)
if ele['NumOfFilesAdded'] != len(dbsBlock['Files']):
self.logger.info("Adding new files to open block %s (%s)" %
(blockName, ele.id))
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
wmbsHelper = WMBSHelper(wmspec, ele['TaskName'], blockName,
ele['Mask'], self.params['CacheDir'])
ele['NumOfFilesAdded'] += wmbsHelper.createSubscriptionAndAddFiles(
block=dbsBlock)[1]
self.backend.updateElements(
ele.id, NumOfFilesAdded=ele['NumOfFilesAdded'])
if dbsBlock['IsOpen'] != ele['OpenForNewData']:
self.logger.info("Closing open block %s (%s)" %
(blockName, ele.id))
self.backend.updateInboxElements(
ele['ParentQueueId'], OpenForNewData=dbsBlock['IsOpen'])
self.backend.updateElements(ele.id,
OpenForNewData=dbsBlock['IsOpen'])
ele['OpenForNewData'] = dbsBlock['IsOpen']
def addNewFilesToOpenSubscriptions(self, *elements):
"""Inject new files to wmbs for running elements that have new files.
Assumes elements are from the same workflow"""
if not self.params['LocalQueueFlag']:
return
wmspec = None
for ele in elements:
if not ele.isRunning() or not ele['SubscriptionId'] or not ele:
continue
if not ele['Inputs'] or not ele['OpenForNewData']:
continue
if not wmspec:
wmspec = self.backend.getWMSpec(ele['RequestName'])
blockName, dbsBlock = self._getDBSBlock(ele, wmspec)
if ele['NumOfFilesAdded'] != len(dbsBlock['Files']):
self.logger.info("Adding new files to open block %s (%s)" % (blockName, ele.id))
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
wmbsHelper = WMBSHelper(wmspec, ele['TaskName'], blockName, ele['Mask'], self.params['CacheDir'])
ele['NumOfFilesAdded'] += wmbsHelper.createSubscriptionAndAddFiles(block = dbsBlock)[1]
self.backend.updateElements(ele.id, NumOfFilesAdded = ele['NumOfFilesAdded'])
if dbsBlock['IsOpen'] != ele['OpenForNewData']:
self.logger.info("Closing open block %s (%s)" % (blockName, ele.id))
self.backend.updateInboxElements(ele['ParentQueueId'], OpenForNewData = dbsBlock['IsOpen'])
self.backend.updateElements(ele.id, OpenForNewData = dbsBlock['IsOpen'])
ele['OpenForNewData'] = dbsBlock['IsOpen']
def testRelValMC(self):
"""
Configure, instantiate, install into WMBS and check that the
subscriptions in WMBS are setup correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "relvalmc_t"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Generation", "SomeBlock")
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# now run the tests on single workload instance installed into WMBS
# each of the subtests is dealing with specific tasks
self._generationTaskTest()
self._stepTwoTaskTest()
self._stepOneTaskTest()
return
def _wmbsPreparation(self, match, wmspec, blockName, dbsBlock):
"""Inject data into wmbs and create subscription.
"""
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
self.logger.info("Adding WMBS subscription for %s" %
match['RequestName'])
mask = match['Mask']
wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask,
self.params['CacheDir'])
sub, match[
'NumOfFilesAdded'] = wmbsHelper.createSubscriptionAndAddFiles(
block=dbsBlock)
self.logger.info(
"Created top level subscription %s for %s with %s files" %
(sub['id'], match['RequestName'], match['NumOfFilesAdded']))
# update couch with wmbs subscription info
match['SubscriptionId'] = sub['id']
match['Status'] = 'Running'
# do update rather than save to avoid conflicts from other thread writes
self.backend.updateElements(match.id,
Status='Running',
SubscriptionId=sub['id'],
NumOfFilesAdded=match['NumOfFilesAdded'])
return sub
def testRelValMCWithPileup(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
The input configuration includes pileup input files.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ProcConfigCacheID"] = self.injectMonteCarloConfig()
# add pile up configuration
defaultArguments["PileupConfig"] = {"mc": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"],
"data": ["/some/minbias/dataset3"]}
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
return
def testThreeStepChainedReDigi(self):
"""
_testThreeStepChaninedReDigi_
Verify that a chained ReDigi workflow that discards RAW and RECO data
can be created and installed into WMBS correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = self.injectReDigiConfigs()
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[1]
defaultArguments["StepThreeConfigCacheID"] = configs[2]
defaultArguments["KeepStepOneOutput"] = False
defaultArguments["KeepStepTwoOutput"] = False
testWorkload = reDigiWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock")
testWMBSHelper.createSubscription()
self.verifyKeepAOD()
return
def testChainedReDigi(self):
"""
_testChaninedReDigi_
Verify that a chained ReDigi workflow that discards RAW data can be
created and installed into WMBS correctly. This will only verify the
step one/step two information in WMBS as the step three information is
the same as the dependent workflow.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = self.injectReDigiConfigs()
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[1]
defaultArguments["StepThreeConfigCacheID"] = configs[2]
defaultArguments["KeepStepOneOutput"] = False
testWorkload = reDigiWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock")
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyDiscardRAW()
return
def testCombinedReDigiRecoConfig(self):
"""
_testCombinedReDigiRecoConfig_
Verify that a ReDigi workflow that uses a single step one config
installs into WMBS correctly.
"""
defaultArguments = ReDigiWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = injectReDigiConfigs(self.configDatabase, combinedStepOne = True)
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[2]
defaultArguments["StepOneOutputModuleName"] = "RECODEBUGoutput"
factory = ReDigiWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyDiscardRAW()
return
def testRelValMCWithPileup(self):
"""
Configure, instantiate, install into WMBS and check that the
subscriptions in WMBS are setup correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "relvalmc_t"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {
"cosmics": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"],
"minbias": ["/some/minbias/dataset3"],
}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock")
testWMBSHelper.createSubscription()
# now run the tests on single workload instance installed into WMBS
# each of the subtests is dealing with specific tasks
self._generationTaskTest()
self._stepOneTaskTest()
self._stepTwoTaskTest()
def testStoreResults(self):
"""
_testStoreResults_
Create a StoreResults workflow and verify it installs into WMBS
correctly.
"""
arguments = StoreResultsWorkloadFactory.getTestArguments()
arguments.update({'CmsPath': "/uscmst1/prod/sw/cms"})
factory = StoreResultsWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload",
arguments)
testWMBSHelper = WMBSHelper(testWorkload, "StoreResults", "SomeBlock", cachepath=self.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
testWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/StoreResults")
testWorkflow.load()
self.assertEqual(len(testWorkflow.outputMap.keys()), 2,
"Error: Wrong number of WF outputs.")
goldenOutputMods = ["Merged"]
for goldenOutputMod in goldenOutputMods:
mergedOutput = testWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"]
unmergedOutput = testWorkflow.outputMap[goldenOutputMod][0]["output_fileset"]
mergedOutput.loadData()
unmergedOutput.loadData()
self.assertEqual(mergedOutput.name, "/TestWorkload/StoreResults/merged-%s" % goldenOutputMod,
"Error: Merged output fileset is wrong: %s" % mergedOutput.name)
self.assertEqual(unmergedOutput.name, "/TestWorkload/StoreResults/merged-%s" % goldenOutputMod,
"Error: Unmerged output fileset is wrong: %s." % unmergedOutput.name)
logArchOutput = testWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
unmergedLogArchOutput = testWorkflow.outputMap["logArchive"][0]["output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name, "/TestWorkload/StoreResults/merged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/StoreResults/merged-logArchive",
"Error: LogArchive output fileset is wrong.")
topLevelFileset = Fileset(name="TestWorkload-StoreResults-SomeBlock")
topLevelFileset.loadData()
procSubscription = Subscription(fileset=topLevelFileset, workflow=testWorkflow)
procSubscription.loadData()
self.assertEqual(procSubscription["type"], "Merge",
"Error: Wrong subscription type.")
self.assertEqual(procSubscription["split_algo"], "ParentlessMergeBySize",
"Error: Wrong split algo.")
return
def testCombinedReDigiRecoConfig(self):
"""
_testCombinedReDigiRecoConfig_
Verify that a ReDigi workflow that uses a single step one config
installs into WMBS correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = self.injectReDigiConfigs(combinedStepOne = True)
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[2]
defaultArguments["StepThreeConfigCacheID"] = None
defaultArguments["StepOneOutputModuleName"] = "RECODEBUGoutput"
testWorkload = reDigiWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock")
testWMBSHelper.createSubscription()
self.verifyDiscardRAW()
return
def createWMBSHelperWithTopTask(self,
wmspec,
block,
mask=None,
parentFlag=False,
detail=False,
commonLocation=None):
topLevelTask = getFirstTask(wmspec)
wmbs = WMBSHelper(wmspec,
topLevelTask.name(),
block,
mask,
cachepath=self.workDir,
commonLocation=commonLocation)
if block:
blockName = block
if parentFlag:
block = self.dbs.getFileBlockWithParents(blockName)
location = self.phedex.getReplicaPhEDExNodesForBlocks(
block=[blockName], complete='y')
block['PhEDExNodeNames'] = location[blockName]
else:
block = self.dbs.getFileBlock(blockName)
location = self.phedex.getReplicaPhEDExNodesForBlocks(
block=[blockName], complete='y')
block['PhEDExNodeNames'] = location[blockName]
sub, files = wmbs.createSubscriptionAndAddFiles(block=block)
if detail:
return wmbs, sub, files
else:
return wmbs
def createWMBSHelperWithTopTask(self, wmspec, block, mask = None):
topLevelTask = getFirstTask(wmspec)
wmbs = WMBSHelper(wmspec, block, mask, cachepath = self.workDir)
if block:
block = self.dbs.getFileBlock(block)[block]
wmbs.createSubscriptionAndAddFiles(block = block)
return wmbs
def main():
start = time.time()
# blockName = match['Inputs'].keys()[0]
blockName = "/acdc/vlimant_ACDC0_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_190218_145226_481/:pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222:SUS-RunIIFall18wmLHEGS-00025_0/0/31055"
# acdcInfo = match['ACDC']
acdcInfo = {"database": "acdcserver",
"fileset": "/pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222/SUS-RunIIFall18wmLHEGS-00025_0",
"collection": "pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222",
"server": "https://cmsweb.cern.ch/couchdb"}
acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
splitedBlockName = ACDCBlock.splitBlockName(blockName)
print("Splitted block name: %s" % splitedBlockName)
fileLists = acdc.getChunkFiles(acdcInfo['collection'],
acdcInfo['fileset'],
splitedBlockName['Offset'],
splitedBlockName['NumOfFiles'])
print("Retrieved %d unique files from the ACDCServer" % len(fileLists))
block = {}
block["Files"] = fileLists
wantedLumis = set([252052, 240646])
for f in fileLists:
for run in f['runs']:
maskDict = run.json()
lumisSet = set(maskDict['Lumis'].keys())
if wantedLumis.intersection(lumisSet):
print("File: %s with events: %s, contains these lumis: %s" % (f['lfn'], f['events'], wantedLumis.intersection(lumisSet)))
# with open("chunkfiles.json", 'w') as fo:
# json.dump(block, fo)
end = time.time()
print("Spent %s secs running so far" % (end - start))
sys.exit(1)
### Now doing the WMBSHelper stuff
reqUrl = "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache"
requestName = "vlimant_ACDC0_task_HIG-RunIIFall17wmLHEGS-01122__v1_T_180808_130708_5376"
wmspec = WMWorkloadHelper()
wmspec.loadSpecFromCouch(reqUrl, requestName)
taskName = "HIG-RunIIFall17DRPremix-00788_0"
mask = None
cacheDir = "/data/srv/wmagent/v1.1.14.patch6/install/wmagent/WorkQueueManager/cache"
# wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir'])
wmbsHelper = WMBSHelper(wmspec, taskName, blockName, mask, cacheDir)
sub, numFilesAdded = wmbsHelper.createSubscriptionAndAddFiles(block=block)
def createWMBSHelperWithTopTask(self, wmspec, block, mask=None,
parentFlag=False, detail=False):
topLevelTask = getFirstTask(wmspec)
wmbs = WMBSHelper(wmspec, topLevelTask.name(), block, mask, cachepath=self.workDir)
if block:
if parentFlag:
block = self.dbs.getFileBlockWithParents(block)[block]
else:
block = self.dbs.getFileBlock(block)[block]
sub, files = wmbs.createSubscriptionAndAddFiles(block=block)
if detail:
return wmbs, sub, files
else:
return wmbs
def createWMBSHelperWithTopTask(self, wmspec, block, mask = None,
parentFlag = False, detail = False):
topLevelTask = getFirstTask(wmspec)
wmbs = WMBSHelper(wmspec, topLevelTask.name(), block, mask, cachepath = self.workDir)
if block:
if parentFlag:
block = self.dbs.getFileBlockWithParents(block)[block]
else:
block = self.dbs.getFileBlock(block)[block]
sub, files = wmbs.createSubscriptionAndAddFiles(block = block)
if detail:
return wmbs, sub, files
else:
return wmbs
def testMonteCarloExtension(self):
"""
_testMonteCarloExtension_
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test. This uses a non-zero first
lumi. Check that the splitting arguments are correctly
set for the lfn counter.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
defaultArguments["FirstLumi"] = 10001
defaultArguments["EventsPerJob"] = 100
defaultArguments["FirstEvent"] = 10001
# defaultArguments["FirstEvent"] = 10001
initial_lfn_counter = (
100
) # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath("/TestWorkload/Production")
productionSplitting = productionTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored")
self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter")
for outputMod in ["OutputA", "OutputB"]:
mergeTask = testWorkload.getTaskByPath("/TestWorkload/Production/ProductionMerge%s" % outputMod)
mergeSplitting = mergeTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored")
self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter")
return
def _wmbsPreparation(self, match, wmspec, blockName, dbsBlock):
"""Inject data into wmbs and create subscription.
"""
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
self.logger.info("Adding WMBS subscription for %s" % match['RequestName'])
mask = match['Mask']
wmbsHelper = WMBSHelper(wmspec, blockName, mask, self.params['CacheDir'])
sub, match['NumOfFilesAdded'] = wmbsHelper.createSubscriptionAndAddFiles(block = dbsBlock)
self.logger.info("Created top level subscription %s for %s with %s files" % (sub['id'],
match['RequestName'],
match['NumOfFilesAdded']))
match['SubscriptionId'] = sub['id']
match['Status'] = 'Running'
self.backend.saveElements(match)
return sub
def testMonteCarlo(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ProdConfigCacheID"] = self.injectMonteCarloConfig()
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock")
testWMBSHelper.createSubscription()
self._commonMonteCarloTest()
def testMonteCarloExtension(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test. This uses a non-zero first
event and lumi. Check that the splitting arguments are correctly
set for the lfn counter.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ProcConfigCacheID"] = self.injectMonteCarloConfig()
defaultArguments["FirstEvent"] = 3571428573
defaultArguments["FirstLumi"] = 26042
defaultArguments["TimePerEvent"] = 15
defaultArguments["FilterEfficiency"] = 0.014
defaultArguments["TotalTime"] = 28800
initial_lfn_counter = 26042 # Same as the previous number of jobs + 1 which is the same value of the first lumi
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
productionSplitting = productionTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored")
self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter")
for outputMod in ["OutputA", "OutputB"]:
mergeTask = testWorkload.getTaskByPath('/TestWorkload/Production/ProductionMerge%s' % outputMod)
mergeSplitting = mergeTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored")
self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter")
return
def testLHEStepZero(self):
"""
_testLHEStepZero_
Make sure that the workload can be created
and complies with the common MonteCarlo test
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
testWorkload = lheStepZeroWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = []
expWfTasks = ['/TestWorkload/EndOfRunDQMHarvest',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect']
expFsets = ['TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive']
subMaps = [(2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'),
(1, 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')]
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig()
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload, "EndOfRunDQMHarvest",
blockName=testArguments['InputDataset'],
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
def testChainedReDigi(self):
"""
_testChaninedReDigi_
Verify that a chained ReDigi workflow that discards RAW data can be
created and installed into WMBS correctly. This will only verify the
step one/step two information in WMBS as the step three information is
the same as the dependent workflow.
"""
defaultArguments = ReDigiWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = injectReDigiConfigs(self.configDatabase)
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[1]
defaultArguments["StepThreeConfigCacheID"] = configs[2]
defaultArguments["StepOneOutputModuleName"] = "RAWDEBUGoutput"
defaultArguments["StepTwoOutputModuleName"] = "RECODEBUGoutput"
defaultArguments["MCPileup"] = PILEUP_DATASET
defaultArguments["KeepStepOneOutput"] = False
factory = ReDigiWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
# Verify that pileup is configured for both of the cmsRun steps in the
# top level task.
topLevelTask = testWorkload.getTopLevelTask()[0]
cmsRun1Helper = topLevelTask.getStepHelper("cmsRun1")
cmsRun2Helper = topLevelTask.getStepHelper("cmsRun2")
cmsRun1PileupConfig = cmsRun1Helper.getPileup()
cmsRun2PileupConfig = cmsRun2Helper.getPileup()
self.assertTrue(cmsRun1PileupConfig.mc.dataset, "/some/cosmics/dataset")
self.assertTrue(cmsRun2PileupConfig.mc.dataset, "/some/cosmics/dataset")
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyDiscardRAW()
return
def _wmbsPreparation(self, match, wmspec, blockName, dbsBlock):
"""Inject data into wmbs and create subscription.
"""
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
self.logger.info("Adding WMBS subscription for %s" % match['RequestName'])
mask = match['Mask']
wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir'])
sub, match['NumOfFilesAdded'] = wmbsHelper.createSubscriptionAndAddFiles(block = dbsBlock)
self.logger.info("Created top level subscription %s for %s with %s files" % (sub['id'],
match['RequestName'],
match['NumOfFilesAdded']))
# update couch with wmbs subscription info
match['SubscriptionId'] = sub['id']
match['Status'] = 'Running'
# do update rather than save to avoid conflicts from other thread writes
self.backend.updateElements(match.id, Status = 'Running', SubscriptionId = sub['id'],
NumOfFilesAdded = match['NumOfFilesAdded'])
return sub
def testMonteCarlo(self):
"""
_testMonteCarlo_
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
return
def testThreeStepChainedReDigi(self):
"""
_testThreeStepChaninedReDigi_
Verify that a chained ReDigi workflow that discards RAW and RECO data
can be created and installed into WMBS correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = self.injectReDigiConfigs()
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[1]
defaultArguments["StepThreeConfigCacheID"] = configs[2]
defaultArguments["KeepStepOneOutput"] = False
defaultArguments["KeepStepTwoOutput"] = False
testWorkload = reDigiWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
self.assertTrue(len(testWorkload.getTopLevelTask()) == 1,
"Error: Wrong number of top level tasks.")
topLevelTask = testWorkload.getTopLevelTask()[0]
topLevelStep = topLevelTask.steps()
cmsRun2Step = topLevelStep.getStep("cmsRun2").getTypeHelper()
self.assertTrue(len(cmsRun2Step.listOutputModules()) == 2,
"Error: Wrong number of output modules in cmsRun2.")
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyKeepAOD()
return
def testMonteCarlo(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask,
testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
return
def testRelValMCWithPileup(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
The input configuration includes pileup input files.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
# add pile up configuration
defaultArguments["PileupConfig"] = {
"mc": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"],
"data": ["/some/minbias/dataset3"]
}
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask,
testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
return
def testRelValMCWithPileup(self):
"""
Configure, instantiate, install into WMBS and check that the
subscriptions in WMBS are setup correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "relvalmc_t"
defaultArguments["GenOutputModuleName"] = "OutputA"
defaultArguments["StepOneOutputModuleName"] = "OutputB"
defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig()
defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig()
defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig()
# add pile up information - for the generation task
defaultArguments["PileupConfig"] = {"cosmics": ["/some/cosmics/dataset1","/some/cosmics/dataset2"],
"minbias": ["/some/minbias/dataset3"]}
testWorkload = relValMCWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Generation", "SomeBlock")
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# now run the tests on single workload instance installed into WMBS
# each of the subtests is dealing with specific tasks
self._generationTaskTest()
self._stepOneTaskTest()
self._stepTwoTaskTest()
def testMCWithLHE(self):
"""
_testMCWithLHE_
Create a MonteCarlo workflow with a variation on the type of work
done, this refers to the previous LHEStepZero where the input
can be .lhe files and there is more than one lumi per job.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
defaultArguments["LheInputFiles"] = "True"
defaultArguments["EventsPerJob"] = 200
defaultArguments["EventsPerLumi"] = 50
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
splitting = productionTask.jobSplittingParameters()
self.assertEqual(splitting["events_per_job"], 200)
self.assertEqual(splitting["events_per_lumi"], 50)
self.assertEqual(splitting["lheInputFiles"], True)
self.assertFalse(splitting["deterministicPileup"])
return
def testCombinedReDigiRecoConfig(self):
"""
_testCombinedReDigiRecoConfig_
Verify that a ReDigi workflow that uses a single step one config
installs into WMBS correctly.
"""
defaultArguments = ReDigiWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = injectReDigiConfigs(self.configDatabase, combinedStepOne = True)
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[2]
defaultArguments["StepOneOutputModuleName"] = "RECODEBUGoutput"
factory = ReDigiWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyDiscardRAW()
return
def testMCWithPileup(self):
"""
_testMCWithPileup_
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
The input configuration includes pileup input files.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
# Add pileup inputs
defaultArguments["MCPileup"] = COSMICS_PU
defaultArguments["DataPileup"] = DATA_PU
defaultArguments["DeterministicPileup"] = True
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
cmsRunStep = productionTask.getStep("cmsRun1").getTypeHelper()
pileupData = cmsRunStep.getPileup()
self.assertEqual(pileupData.data.dataset, [DATA_PU])
self.assertEqual(pileupData.mc.dataset, [COSMICS_PU])
splitting = productionTask.jobSplittingParameters()
self.assertTrue(splitting["deterministicPileup"])
return
def testCombinedReDigiRecoConfig(self):
"""
_testCombinedReDigiRecoConfig_
Verify that a ReDigi workflow that uses a single step one config
installs into WMBS correctly.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = self.injectReDigiConfigs(combinedStepOne = True)
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[2]
defaultArguments["StepThreeConfigCacheID"] = None
defaultArguments["StepOneOutputModuleName"] = "RECODEBUGoutput"
testWorkload = reDigiWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyDiscardRAW()
return
def testMonteCarlo(self):
"""
_testMonteCarlo_
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
return
def testSingleStepReDigi(self):
"""
_testSingleStepReDigi_
Verify that a single step ReDigi workflow can be created and installed
correctly into WMBS.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = self.injectReDigiConfigs()
defaultArguments["StepOneConfigCacheID"] = configs[2]
defaultArguments["StepTwoConfigCacheID"] = None
defaultArguments["StepThreeConfigCacheID"] = None
testWorkload = reDigiWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyKeepAOD()
return
def testStoreResults(self):
"""
_testStoreResults_
Create a StoreResults workflow and verify it installs into WMBS
correctly.
"""
arguments = StoreResultsWorkloadFactory.getTestArguments()
factory = StoreResultsWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", arguments)
testWMBSHelper = WMBSHelper(testWorkload, "StoreResults", "SomeBlock", cachepath=self.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
testWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/StoreResults")
testWorkflow.load()
self.assertEqual(len(testWorkflow.outputMap), 2,
"Error: Wrong number of WF outputs.")
goldenOutputMods = {"Merged": "USER"}
for goldenOutputMod, tier in viewitems(goldenOutputMods):
fset = goldenOutputMod + tier
mergedOutput = testWorkflow.outputMap[fset][0]["merged_output_fileset"]
unmergedOutput = testWorkflow.outputMap[fset][0]["output_fileset"]
mergedOutput.loadData()
unmergedOutput.loadData()
self.assertEqual(mergedOutput.name, "/TestWorkload/StoreResults/merged-%s" % fset,
"Error: Merged output fileset is wrong: %s" % mergedOutput.name)
self.assertEqual(unmergedOutput.name, "/TestWorkload/StoreResults/merged-%s" % fset,
"Error: Unmerged output fileset is wrong: %s." % unmergedOutput.name)
logArchOutput = testWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
unmergedLogArchOutput = testWorkflow.outputMap["logArchive"][0]["output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name, "/TestWorkload/StoreResults/merged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/StoreResults/merged-logArchive",
"Error: LogArchive output fileset is wrong.")
topLevelFileset = Fileset(name="TestWorkload-StoreResults-SomeBlock")
topLevelFileset.loadData()
procSubscription = Subscription(fileset=topLevelFileset, workflow=testWorkflow)
procSubscription.loadData()
self.assertEqual(procSubscription["type"], "Merge",
"Error: Wrong subscription type.")
self.assertEqual(procSubscription["split_algo"], "ParentlessMergeBySize",
"Error: Wrong split algo.")
return
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = ['/TestWorkload/StoreResults']
expWfTasks = [
'/TestWorkload/StoreResults',
'/TestWorkload/StoreResults/StoreResultsLogCollect'
]
expFsets = [
'TestWorkload-StoreResults-/MinimumBias/ComissioningHI-v1/RAW',
'/TestWorkload/StoreResults/merged-Merged',
'/TestWorkload/StoreResults/merged-logArchive'
]
subMaps = [
(2, '/TestWorkload/StoreResults/merged-logArchive',
'/TestWorkload/StoreResults/StoreResultsLogCollect',
'MinFileBased', 'LogCollect'),
(1, 'TestWorkload-StoreResults-/MinimumBias/ComissioningHI-v1/RAW',
'/TestWorkload/StoreResults', 'ParentlessMergeBySize', 'Merge')
]
testArguments = StoreResultsWorkloadFactory.getTestArguments()
testArguments.update({'CmsPath': "/uscmst1/prod/sw/cms"})
factory = StoreResultsWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"StoreResults",
blockName=testArguments['InputDataset'],
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
print("Tasks producing output:\n%s" %
pformat(testWorkload.listOutputProducingTasks()))
self.assertItemsEqual(testWorkload.listOutputProducingTasks(),
expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
print("List of workflow tasks:\n%s" %
pformat([item['task'] for item in workflows]))
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
print("List of filesets:\n%s" % pformat([item[1]
for item in filesets]))
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload",
returnTuple=True)
print("List of subscriptions:\n%s" % pformat(subscriptions))
self.assertItemsEqual(subscriptions, subMaps)
def testSingleStepReDigi(self):
"""
_testSingleStepReDigi_
Verify that a single step ReDigi workflow can be created and installed
correctly into WMBS.
"""
defaultArguments = ReDigiWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = injectReDigiConfigs(self.configDatabase)
defaultArguments["StepOneConfigCacheID"] = configs[2]
factory = ReDigiWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyKeepAOD()
return
def testThreeStepChainedReDigi(self):
"""
_testThreeStepChaninedReDigi_
Verify that a chained ReDigi workflow that discards RAW and RECO data
can be created and installed into WMBS correctly.
"""
defaultArguments = ReDigiWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = injectReDigiConfigs(self.configDatabase)
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[1]
defaultArguments["StepThreeConfigCacheID"] = configs[2]
defaultArguments["KeepStepOneOutput"] = False
defaultArguments["KeepStepTwoOutput"] = False
defaultArguments["StepOneOutputModuleName"] = "RAWDEBUGoutput"
defaultArguments["StepTwoOutputModuleName"] = "RECODEBUGoutput"
factory = ReDigiWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
self.assertTrue(len(testWorkload.getTopLevelTask()) == 1,
"Error: Wrong number of top level tasks.")
topLevelTask = testWorkload.getTopLevelTask()[0]
topLevelStep = topLevelTask.steps()
cmsRun2Step = topLevelStep.getStep("cmsRun2").getTypeHelper()
self.assertTrue(len(cmsRun2Step.listOutputModules()) == 2,
"Error: Wrong number of output modules in cmsRun2.")
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyKeepAOD()
return
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = []
expWfTasks = [
'/TestWorkload/EndOfRunDQMHarvest',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect'
]
expFsets = [
'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive'
]
subMaps = [
(2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive',
'/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect',
'MinFileBased', 'LogCollect'),
(1,
'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO',
'/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')
]
testArguments = DQMHarvestWorkloadFactory.getTestArguments()
testArguments.update(REQUEST)
testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig()
testArguments.pop("ConfigCacheID", None)
factory = DQMHarvestWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"EndOfRunDQMHarvest",
blockName=testArguments['InputDataset'],
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(),
expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload",
returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
def testMonteCarloExtension(self):
"""
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test. This uses a non-zero first
event and lumi. Check that the splitting arguments are correctly
set for the lfn counter.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
defaultArguments["FirstEvent"] = 3571428573
defaultArguments["FirstLumi"] = 26042
defaultArguments["TimePerEvent"] = 15
defaultArguments["FilterEfficiency"] = 0.014
defaultArguments["TotalTime"] = 28800
initial_lfn_counter = 26042 # Same as the previous number of jobs + 1 which is the same value of the first lumi
testWorkload = monteCarloWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask,
testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
productionSplitting = productionTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in productionSplitting,
"No initial lfn counter was stored")
self.assertEqual(productionSplitting["initial_lfn_counter"],
initial_lfn_counter, "Wrong initial LFN counter")
for outputMod in ["OutputA", "OutputB"]:
mergeTask = testWorkload.getTaskByPath(
'/TestWorkload/Production/ProductionMerge%s' % outputMod)
mergeSplitting = mergeTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in mergeSplitting,
"No initial lfn counter was stored")
self.assertEqual(mergeSplitting["initial_lfn_counter"],
initial_lfn_counter, "Wrong initial LFN counter")
return
def testMonteCarloExtension(self):
"""
_testMonteCarloExtension_
Create a Monte Carlo workflow and verify that it is injected correctly
into WMBS and invoke its detailed test. This uses a non-zero first
lumi. Check that the splitting arguments are correctly
set for the lfn counter.
"""
defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = TEST_DB_NAME
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
defaultArguments["FirstLumi"] = 10001
defaultArguments["EventsPerJob"] = 100
defaultArguments["FirstEvent"] = 10001
#defaultArguments["FirstEvent"] = 10001
initial_lfn_counter = 100 # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
productionSplitting = productionTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in productionSplitting,
"No initial lfn counter was stored")
self.assertEqual(productionSplitting["initial_lfn_counter"],
initial_lfn_counter, "Wrong initial LFN counter")
for outputMod in ["OutputA", "OutputB"]:
mergeTask = testWorkload.getTaskByPath(
'/TestWorkload/Production/ProductionMerge%s' % outputMod)
mergeSplitting = mergeTask.jobSplittingParameters()
self.assertTrue("initial_lfn_counter" in mergeSplitting,
"No initial lfn counter was stored")
self.assertEqual(mergeSplitting["initial_lfn_counter"],
initial_lfn_counter, "Wrong initial LFN counter")
return
def testChainedReDigi(self):
"""
_testChaninedReDigi_
Verify that a chained ReDigi workflow that discards RAW data can be
created and installed into WMBS correctly. This will only verify the
step one/step two information in WMBS as the step three information is
the same as the dependent workflow.
"""
defaultArguments = ReDigiWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = injectReDigiConfigs(self.configDatabase)
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[1]
defaultArguments["StepThreeConfigCacheID"] = configs[2]
defaultArguments["StepOneOutputModuleName"] = "RAWDEBUGoutput"
defaultArguments["StepTwoOutputModuleName"] = "RECODEBUGoutput"
defaultArguments["MCPileup"] = "/mixing/pileup/DATASET"
defaultArguments["KeepStepOneOutput"] = False
factory = ReDigiWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
# Verify that pileup is configured for both of the cmsRun steps in the
# top level task.
topLevelTask = testWorkload.getTopLevelTask()[0]
cmsRun1Helper = topLevelTask.getStepHelper("cmsRun1")
cmsRun2Helper = topLevelTask.getStepHelper("cmsRun2")
cmsRun1PileupConfig = cmsRun1Helper.getPileup()
cmsRun2PileupConfig = cmsRun2Helper.getPileup()
self.assertTrue(cmsRun1PileupConfig.mc.dataset, "/some/cosmics/dataset")
self.assertTrue(cmsRun2PileupConfig.mc.dataset, "/some/cosmics/dataset")
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyDiscardRAW()
return
def testChainedReDigi(self):
"""
_testChaninedReDigi_
Verify that a chained ReDigi workflow that discards RAW data can be
created and installed into WMBS correctly. This will only verify the
step one/step two information in WMBS as the step three information is
the same as the dependent workflow.
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = self.injectReDigiConfigs()
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[1]
defaultArguments["StepThreeConfigCacheID"] = configs[2]
defaultArguments["KeepStepOneOutput"] = False
testWorkload = reDigiWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
# Verify that pileup is configured for both of the cmsRun steps in the
# top level task.
topLevelTask = testWorkload.getTopLevelTask()[0]
cmsRun1Helper = topLevelTask.getStepHelper("cmsRun1")
cmsRun2Helper = topLevelTask.getStepHelper("cmsRun2")
cmsRun1PileupConfig = cmsRun1Helper.getPileup()
cmsRun2PileupConfig = cmsRun2Helper.getPileup()
self.assertTrue(cmsRun1PileupConfig.mc.dataset, "/some/cosmics/dataset")
self.assertTrue(cmsRun2PileupConfig.mc.dataset, "/some/cosmics/dataset")
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyDiscardRAW()
return
def testSingleStepReDigi(self):
"""
_testSingleStepReDigi_
Verify that a single step ReDigi workflow can be created and installed
correctly into WMBS.
"""
defaultArguments = ReDigiWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = injectReDigiConfigs(self.configDatabase)
defaultArguments["StepOneConfigCacheID"] = configs[2]
factory = ReDigiWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyKeepAOD()
return
def testLHEStepZero(self):
"""
_testLHEStepZero_
Make sure that the workload can be created
and complies with the common MonteCarlo test
"""
defaultArguments = getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "rereco_t"
defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
testWorkload = lheStepZeroWorkload("TestWorkload", defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DWMWM")
testWMBSHelper = WMBSHelper(testWorkload,
"Production",
"SomeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask,
testWMBSHelper.topLevelFileset)
self._commonMonteCarloTest()
def testThreeStepChainedReDigi(self):
"""
_testThreeStepChaninedReDigi_
Verify that a chained ReDigi workflow that discards RAW and RECO data
can be created and installed into WMBS correctly.
"""
defaultArguments = ReDigiWorkloadFactory.getTestArguments()
defaultArguments["CouchURL"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "redigi_t"
configs = injectReDigiConfigs(self.configDatabase)
defaultArguments["StepOneConfigCacheID"] = configs[0]
defaultArguments["StepTwoConfigCacheID"] = configs[1]
defaultArguments["StepThreeConfigCacheID"] = configs[2]
defaultArguments["KeepStepOneOutput"] = False
defaultArguments["KeepStepTwoOutput"] = False
defaultArguments["StepOneOutputModuleName"] = "RAWDEBUGoutput"
defaultArguments["StepTwoOutputModuleName"] = "RECODEBUGoutput"
factory = ReDigiWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
self.assertTrue(len(testWorkload.getTopLevelTask()) == 1,
"Error: Wrong number of top level tasks.")
topLevelTask = testWorkload.getTopLevelTask()[0]
topLevelStep = topLevelTask.steps()
cmsRun2Step = topLevelStep.getStep("cmsRun2").getTypeHelper()
self.assertTrue(len(cmsRun2Step.listOutputModules()) == 2,
"Error: Wrong number of output modules in cmsRun2.")
testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.verifyKeepAOD()
return
def testPromptRecoWithSkims(self):
"""
_testT1PromptRecoWithSkim_
Create a T1 Prompt Reconstruction workflow with PromptSkims
and verify it installs into WMBS correctly.
"""
testArguments = PromptRecoWorkloadFactory.getTestArguments()
testArguments["CouchURL"] = os.environ["COUCHURL"]
testArguments["EnableHarvesting"] = True
factory = PromptRecoWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "T0")
testWMBSHelper = WMBSHelper(testWorkload, "Reco", "SomeBlock", cachepath=self.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
recoWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco")
recoWorkflow.load()
self.assertEqual(len(recoWorkflow.outputMap.keys()), len(testArguments["WriteTiers"]) + 1,
"Error: Wrong number of WF outputs in the Reco WF.")
goldenOutputMods = {"write_RECO": "RECO", "write_ALCARECO": "ALCARECO", "write_AOD": "AOD", "write_DQM": "DQM"}
for goldenOutputMod, tier in goldenOutputMods.items():
fset = goldenOutputMod + tier
mergedOutput = recoWorkflow.outputMap[fset][0]["merged_output_fileset"]
unmergedOutput = recoWorkflow.outputMap[fset][0]["output_fileset"]
mergedOutput.loadData()
unmergedOutput.loadData()
if goldenOutputMod != "write_ALCARECO":
self.assertEqual(mergedOutput.name,
"/TestWorkload/Reco/RecoMerge%s/merged-Merged%s" % (goldenOutputMod, tier),
"Error: Merged output fileset is wrong: %s" % mergedOutput.name)
self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/unmerged-%s" % fset,
"Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name)
logArchOutput = recoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
unmergedLogArchOutput = recoWorkflow.outputMap["logArchive"][0]["output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
alcaSkimWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/AlcaSkim")
alcaSkimWorkflow.load()
self.assertEqual(len(alcaSkimWorkflow.outputMap.keys()), len(testArguments["AlcaSkims"]) + 1,
"Error: Wrong number of WF outputs in the AlcaSkim WF.")
goldenOutputMods = []
for alcaProd in testArguments["AlcaSkims"]:
goldenOutputMods.append("ALCARECOStream%s" % alcaProd)
for goldenOutputMod in goldenOutputMods:
fset = goldenOutputMod + "ALCARECO"
mergedOutput = alcaSkimWorkflow.outputMap[fset][0]["merged_output_fileset"]
unmergedOutput = alcaSkimWorkflow.outputMap[fset][0]["output_fileset"]
mergedOutput.loadData()
unmergedOutput.loadData()
self.assertEqual(mergedOutput.name,
"/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-MergedALCARECO" % goldenOutputMod,
"Error: Merged output fileset is wrong: %s" % mergedOutput.name)
self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-%sALCARECO" % goldenOutputMod,
"Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name)
logArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
unmergedLogArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
dqmWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged")
dqmWorkflow.load()
logArchOutput = dqmWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
unmergedLogArchOutput = dqmWorkflow.outputMap["logArchive"][0]["output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name,
"/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name,
"/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
goldenOutputMods = {"write_RECO": "RECO", "write_AOD": "AOD", "write_DQM": "DQM"}
for goldenOutputMod, tier in goldenOutputMods.items():
fset = goldenOutputMod + tier
mergeWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/RecoMerge%s" % goldenOutputMod)
mergeWorkflow.load()
self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2,
"Error: Wrong number of WF outputs.")
mergedMergeOutput = mergeWorkflow.outputMap["Merged%s" % tier][0]["merged_output_fileset"]
unmergedMergeOutput = mergeWorkflow.outputMap["Merged%s" % tier][0]["output_fileset"]
mergedMergeOutput.loadData()
unmergedMergeOutput.loadData()
self.assertEqual(mergedMergeOutput.name,
"/TestWorkload/Reco/RecoMerge%s/merged-Merged%s" % (goldenOutputMod, tier),
"Error: Merged output fileset is wrong.")
self.assertEqual(unmergedMergeOutput.name,
"/TestWorkload/Reco/RecoMerge%s/merged-Merged%s" % (goldenOutputMod, tier),
"Error: Unmerged output fileset is wrong.")
logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod,
"Error: LogArchive output fileset is wrong: %s" % logArchOutput.name)
self.assertEqual(unmergedLogArchOutput.name,
"/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod,
"Error: LogArchive output fileset is wrong.")
goldenOutputMods = []
for alcaProd in testArguments["AlcaSkims"]:
goldenOutputMods.append("ALCARECOStream%s" % alcaProd)
for goldenOutputMod in goldenOutputMods:
mergeWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % goldenOutputMod)
mergeWorkflow.load()
self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2,
"Error: Wrong number of WF outputs %d." % len(mergeWorkflow.outputMap.keys()))
mergedMergeOutput = mergeWorkflow.outputMap["MergedALCARECO"][0]["merged_output_fileset"]
unmergedMergeOutput = mergeWorkflow.outputMap["MergedALCARECO"][0]["output_fileset"]
mergedMergeOutput.loadData()
unmergedMergeOutput.loadData()
self.assertEqual(mergedMergeOutput.name,
"/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-MergedALCARECO" % goldenOutputMod,
"Error: Merged output fileset is wrong.")
self.assertEqual(unmergedMergeOutput.name,
"/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-MergedALCARECO" % goldenOutputMod,
"Error: Unmerged output fileset is wrong.")
logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]
unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name,
"/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod,
"Error: LogArchive output fileset is wrong: %s" % logArchOutput.name)
self.assertEqual(unmergedLogArchOutput.name,
"/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod,
"Error: LogArchive output fileset is wrong.")
topLevelFileset = Fileset(name="TestWorkload-Reco-SomeBlock")
topLevelFileset.loadData()
recoSubscription = Subscription(fileset=topLevelFileset, workflow=recoWorkflow)
recoSubscription.loadData()
self.assertEqual(recoSubscription["type"], "Processing",
"Error: Wrong subscription type.")
self.assertEqual(recoSubscription["split_algo"], "EventAwareLumiBased",
"Error: Wrong split algorithm. %s" % recoSubscription["split_algo"])
alcaRecoFileset = Fileset(name="/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO")
alcaRecoFileset.loadData()
alcaSkimSubscription = Subscription(fileset=alcaRecoFileset, workflow=alcaSkimWorkflow)
alcaSkimSubscription.loadData()
self.assertEqual(alcaSkimSubscription["type"], "Processing",
"Error: Wrong subscription type.")
self.assertEqual(alcaSkimSubscription["split_algo"], "ParentlessMergeBySize",
"Error: Wrong split algorithm. %s" % alcaSkimSubscription["split_algo"])
mergedDQMFileset = Fileset(name="/TestWorkload/Reco/RecoMergewrite_DQM/merged-MergedDQM")
mergedDQMFileset.loadData()
dqmSubscription = Subscription(fileset=mergedDQMFileset, workflow=dqmWorkflow)
dqmSubscription.loadData()
self.assertEqual(dqmSubscription["type"], "Harvesting",
"Error: Wrong subscription type.")
self.assertEqual(dqmSubscription["split_algo"], "Harvest",
"Error: Wrong split algo.")
unmergedOutputs = {"write_RECO": "RECO", "write_AOD": "AOD", "write_DQM": "DQM"}
for unmergedOutput, tier in unmergedOutputs.items():
fset = unmergedOutput + tier
unmergedDataTier = Fileset(name="/TestWorkload/Reco/unmerged-%s" % fset)
unmergedDataTier.loadData()
dataTierMergeWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/RecoMerge%s" % unmergedOutput)
dataTierMergeWorkflow.load()
mergeSubscription = Subscription(fileset=unmergedDataTier, workflow=dataTierMergeWorkflow)
mergeSubscription.loadData()
self.assertEqual(mergeSubscription["type"], "Merge",
"Error: Wrong subscription type.")
self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize",
"Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"])
unmergedOutputs = []
for alcaProd in testArguments["AlcaSkims"]:
unmergedOutputs.append("ALCARECOStream%s" % alcaProd)
for unmergedOutput in unmergedOutputs:
unmergedAlcaSkim = Fileset(name="/TestWorkload/Reco/AlcaSkim/unmerged-%sALCARECO" % unmergedOutput)
unmergedAlcaSkim.loadData()
alcaSkimMergeWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % unmergedOutput)
alcaSkimMergeWorkflow.load()
mergeSubscription = Subscription(fileset=unmergedAlcaSkim, workflow=alcaSkimMergeWorkflow)
mergeSubscription.loadData()
self.assertEqual(mergeSubscription["type"], "Merge",
"Error: Wrong subscription type.")
self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize",
"Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"])
goldenOutputMods = {"write_RECO": "RECO", "write_AOD": "AOD", "write_DQM": "DQM"}
for goldenOutputMod, tier in goldenOutputMods.items():
fset = goldenOutputMod + tier
unmergedFileset = Fileset(name="/TestWorkload/Reco/unmerged-%s" % fset)
unmergedFileset.loadData()
cleanupWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/RecoCleanupUnmerged%s" % goldenOutputMod)
cleanupWorkflow.load()
cleanupSubscription = Subscription(fileset=unmergedFileset, workflow=cleanupWorkflow)
cleanupSubscription.loadData()
self.assertEqual(cleanupSubscription["type"], "Cleanup",
"Error: Wrong subscription type.")
self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased",
"Error: Wrong subscription type.")
goldenOutputMods = []
for alcaProd in testArguments["AlcaSkims"]:
goldenOutputMods.append("ALCARECOStream%s" % alcaProd)
for goldenOutputMod in goldenOutputMods:
unmergedFileset = Fileset(name="/TestWorkload/Reco/AlcaSkim/unmerged-%sALCARECO" % goldenOutputMod)
unmergedFileset.loadData()
cleanupWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmerged%s" % goldenOutputMod)
cleanupWorkflow.load()
cleanupSubscription = Subscription(fileset=unmergedFileset, workflow=cleanupWorkflow)
cleanupSubscription.loadData()
self.assertEqual(cleanupSubscription["type"], "Cleanup",
"Error: Wrong subscription type.")
self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased",
"Error: Wrong subscription type.")
recoLogCollect = Fileset(name="/TestWorkload/Reco/unmerged-logArchive")
recoLogCollect.loadData()
recoLogCollectWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/LogCollect")
recoLogCollectWorkflow.load()
logCollectSub = Subscription(fileset=recoLogCollect, workflow=recoLogCollectWorkflow)
logCollectSub.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algorithm.")
alcaSkimLogCollect = Fileset(name="/TestWorkload/Reco/AlcaSkim/unmerged-logArchive")
alcaSkimLogCollect.loadData()
alcaSkimLogCollectWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect")
alcaSkimLogCollectWorkflow.load()
logCollectSub = Subscription(fileset=alcaSkimLogCollect, workflow=alcaSkimLogCollectWorkflow)
logCollectSub.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algorithm.")
goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"]
for goldenOutputMod in goldenOutputMods:
recoMergeLogCollect = Fileset(name="/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod)
recoMergeLogCollect.loadData()
recoMergeLogCollectWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/RecoMerge%s/Reco%sMergeLogCollect" % (
goldenOutputMod, goldenOutputMod))
recoMergeLogCollectWorkflow.load()
logCollectSubscription = Subscription(fileset=recoMergeLogCollect, workflow=recoMergeLogCollectWorkflow)
logCollectSubscription.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algorithm.")
goldenOutputMods = []
for alcaProd in testArguments["AlcaSkims"]:
goldenOutputMods.append("ALCARECOStream%s" % alcaProd)
for goldenOutputMod in goldenOutputMods:
alcaSkimLogCollect = Fileset(
name="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod)
alcaSkimLogCollect.loadData()
alcaSkimLogCollectWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/AlcaSkim%sMergeLogCollect" % (
goldenOutputMod, goldenOutputMod))
alcaSkimLogCollectWorkflow.load()
logCollectSubscription = Subscription(fileset=alcaSkimLogCollect, workflow=alcaSkimLogCollectWorkflow)
logCollectSubscription.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algorithm.")
dqmHarvestLogCollect = Fileset(
name="/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive")
dqmHarvestLogCollect.loadData()
dqmHarvestLogCollectWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect")
dqmHarvestLogCollectWorkflow.load()
logCollectSub = Subscription(fileset=dqmHarvestLogCollect, workflow=dqmHarvestLogCollectWorkflow)
logCollectSub.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algo.")
return
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = ['/TestWorkload/Reco',
'/TestWorkload/Reco/AlcaSkim',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics',
'/TestWorkload/Reco/RecoMergewrite_AOD',
'/TestWorkload/Reco/RecoMergewrite_DQM',
'/TestWorkload/Reco/RecoMergewrite_RECO']
expWfTasks = ['/TestWorkload/Reco',
'/TestWorkload/Reco/AlcaSkim',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamHcalCalHOCosmics',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamMuAlGlobalCosmics',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamTkAlCosmics0T',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/AlcaSkimALCARECOStreamHcalCalHOCosmicsMergeLogCollect',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/AlcaSkimALCARECOStreamMuAlGlobalCosmicsMergeLogCollect',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/AlcaSkimALCARECOStreamTkAlCosmics0TMergeLogCollect',
'/TestWorkload/Reco/LogCollect',
'/TestWorkload/Reco/RecoCleanupUnmergedwrite_ALCARECO',
'/TestWorkload/Reco/RecoCleanupUnmergedwrite_AOD',
'/TestWorkload/Reco/RecoCleanupUnmergedwrite_DQM',
'/TestWorkload/Reco/RecoCleanupUnmergedwrite_RECO',
'/TestWorkload/Reco/RecoMergewrite_AOD',
'/TestWorkload/Reco/RecoMergewrite_AOD/Recowrite_AODMergeLogCollect',
'/TestWorkload/Reco/RecoMergewrite_DQM',
'/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged',
'/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect',
'/TestWorkload/Reco/RecoMergewrite_DQM/Recowrite_DQMMergeLogCollect',
'/TestWorkload/Reco/RecoMergewrite_RECO',
'/TestWorkload/Reco/RecoMergewrite_RECO/Recowrite_RECOMergeLogCollect']
expFsets = ['TestWorkload-Reco-/MinimumBias/ComissioningHI-v1/RAW',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/merged-MergedALCARECO',
'/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamHcalCalHOCosmicsALCARECO',
'/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/merged-logArchive',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/merged-logArchive',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/merged-MergedALCARECO',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/merged-logArchive',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/merged-MergedALCARECO',
'/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamMuAlGlobalCosmicsALCARECO',
'/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamTkAlCosmics0TALCARECO',
'/TestWorkload/Reco/AlcaSkim/unmerged-logArchive',
'/TestWorkload/Reco/RecoMergewrite_AOD/merged-logArchive',
'/TestWorkload/Reco/RecoMergewrite_AOD/merged-MergedAOD',
'/TestWorkload/Reco/unmerged-write_AODAOD',
'/TestWorkload/Reco/unmerged-write_DQMDQM',
'/TestWorkload/Reco/RecoMergewrite_DQM/merged-logArchive',
'/TestWorkload/Reco/RecoMergewrite_DQM/merged-MergedDQM',
'/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive',
'/TestWorkload/Reco/RecoMergewrite_RECO/merged-logArchive',
'/TestWorkload/Reco/RecoMergewrite_RECO/merged-MergedRECO',
'/TestWorkload/Reco/unmerged-logArchive',
'/TestWorkload/Reco/unmerged-write_RECORECO']
subMaps = [(4,
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/merged-logArchive',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/AlcaSkimALCARECOStreamHcalCalHOCosmicsMergeLogCollect',
'MinFileBased',
'LogCollect'),
(10,
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/merged-logArchive',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/AlcaSkimALCARECOStreamMuAlGlobalCosmicsMergeLogCollect',
'MinFileBased',
'LogCollect'),
(7,
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/merged-logArchive',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/AlcaSkimALCARECOStreamTkAlCosmics0TMergeLogCollect',
'MinFileBased',
'LogCollect'),
(5,
'/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamHcalCalHOCosmicsALCARECO',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamHcalCalHOCosmics',
'SiblingProcessingBased',
'Cleanup'),
(3,
'/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamHcalCalHOCosmicsALCARECO',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics',
'ParentlessMergeBySize',
'Merge'),
(11,
'/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamMuAlGlobalCosmicsALCARECO',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamMuAlGlobalCosmics',
'SiblingProcessingBased',
'Cleanup'),
(9,
'/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamMuAlGlobalCosmicsALCARECO',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics',
'ParentlessMergeBySize',
'Merge'),
(8,
'/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamTkAlCosmics0TALCARECO',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamTkAlCosmics0T',
'SiblingProcessingBased',
'Cleanup'),
(6,
'/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamTkAlCosmics0TALCARECO',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T',
'ParentlessMergeBySize',
'Merge'),
(12,
'/TestWorkload/Reco/AlcaSkim/unmerged-logArchive',
'/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect',
'MinFileBased',
'LogCollect'),
(15,
'/TestWorkload/Reco/RecoMergewrite_AOD/merged-logArchive',
'/TestWorkload/Reco/RecoMergewrite_AOD/Recowrite_AODMergeLogCollect',
'MinFileBased',
'LogCollect'),
(20,
'/TestWorkload/Reco/RecoMergewrite_DQM/merged-logArchive',
'/TestWorkload/Reco/RecoMergewrite_DQM/Recowrite_DQMMergeLogCollect',
'MinFileBased',
'LogCollect'),
(18,
'/TestWorkload/Reco/RecoMergewrite_DQM/merged-MergedDQM',
'/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged',
'Harvest',
'Harvesting'),
(19,
'/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive',
'/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect',
'MinFileBased',
'LogCollect'),
(23,
'/TestWorkload/Reco/RecoMergewrite_RECO/merged-logArchive',
'/TestWorkload/Reco/RecoMergewrite_RECO/Recowrite_RECOMergeLogCollect',
'MinFileBased',
'LogCollect'),
(25,
'/TestWorkload/Reco/unmerged-logArchive',
'/TestWorkload/Reco/LogCollect',
'MinFileBased',
'LogCollect'),
(2,
'/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO',
'/TestWorkload/Reco/AlcaSkim',
'ParentlessMergeBySize',
'Processing'),
(13,
'/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO',
'/TestWorkload/Reco/RecoCleanupUnmergedwrite_ALCARECO',
'SiblingProcessingBased',
'Cleanup'),
(16,
'/TestWorkload/Reco/unmerged-write_AODAOD',
'/TestWorkload/Reco/RecoCleanupUnmergedwrite_AOD',
'SiblingProcessingBased',
'Cleanup'),
(14,
'/TestWorkload/Reco/unmerged-write_AODAOD',
'/TestWorkload/Reco/RecoMergewrite_AOD',
'ParentlessMergeBySize',
'Merge'),
(21,
'/TestWorkload/Reco/unmerged-write_DQMDQM',
'/TestWorkload/Reco/RecoCleanupUnmergedwrite_DQM',
'SiblingProcessingBased',
'Cleanup'),
(17,
'/TestWorkload/Reco/unmerged-write_DQMDQM',
'/TestWorkload/Reco/RecoMergewrite_DQM',
'ParentlessMergeBySize',
'Merge'),
(24,
'/TestWorkload/Reco/unmerged-write_RECORECO',
'/TestWorkload/Reco/RecoCleanupUnmergedwrite_RECO',
'SiblingProcessingBased',
'Cleanup'),
(22,
'/TestWorkload/Reco/unmerged-write_RECORECO',
'/TestWorkload/Reco/RecoMergewrite_RECO',
'ParentlessMergeBySize',
'Merge'),
(1,
'TestWorkload-Reco-/MinimumBias/ComissioningHI-v1/RAW',
'/TestWorkload/Reco',
'EventAwareLumiBased',
'Processing')]
testArguments = PromptRecoWorkloadFactory.getTestArguments()
testArguments["CouchURL"] = os.environ["COUCHURL"]
testArguments["CouchDBName"] = "promptreco_t"
testArguments["EnableHarvesting"] = True
factory = PromptRecoWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload, "Reco", blockName=testArguments['InputDataset'],
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = [
'/TestWorkload/Repack',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW'
]
expWfTasks = [
'/TestWorkload/Repack', '/TestWorkload/Repack/LogCollect',
'/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset1_RAW',
'/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset2_RAW',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/Repackwrite_PrimaryDataset1_RAWMergeLogCollect',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/Repackwrite_PrimaryDataset2_RAWMergeLogCollect'
]
expFsets = [
'TestWorkload-Repack-StreamerFiles',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-logArchive',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-MergedRAW',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-MergedErrorRAW',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-logArchive',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-MergedRAW',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-MergedErrorRAW',
'/TestWorkload/Repack/unmerged-write_PrimaryDataset1_RAWRAW',
'/TestWorkload/Repack/unmerged-write_PrimaryDataset2_RAWRAW',
'/TestWorkload/Repack/unmerged-logArchive'
]
subMaps = [
(3,
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-logArchive',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/Repackwrite_PrimaryDataset1_RAWMergeLogCollect',
'MinFileBased', 'LogCollect'),
(6,
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-logArchive',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/Repackwrite_PrimaryDataset2_RAWMergeLogCollect',
'MinFileBased', 'LogCollect'),
(8, '/TestWorkload/Repack/unmerged-logArchive',
'/TestWorkload/Repack/LogCollect', 'MinFileBased', 'LogCollect'),
(4, '/TestWorkload/Repack/unmerged-write_PrimaryDataset1_RAWRAW',
'/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset1_RAW',
'SiblingProcessingBased', 'Cleanup'),
(2, '/TestWorkload/Repack/unmerged-write_PrimaryDataset1_RAWRAW',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW',
'RepackMerge', 'Merge'),
(7, '/TestWorkload/Repack/unmerged-write_PrimaryDataset2_RAWRAW',
'/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset2_RAW',
'SiblingProcessingBased', 'Cleanup'),
(5, '/TestWorkload/Repack/unmerged-write_PrimaryDataset2_RAWRAW',
'/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW',
'RepackMerge', 'Merge'),
(1, 'TestWorkload-Repack-StreamerFiles', '/TestWorkload/Repack',
'Repack', 'Repack')
]
testArguments = RepackWorkloadFactory.getTestArguments()
testArguments.update(deepcopy(REQUEST))
factory = RepackWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"Repack",
blockName='StreamerFiles',
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(),
expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload",
returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
def testRepack(self):
"""
_testRepack_
Create a Repack workflow
and verify it installs into WMBS correctly.
"""
testArguments = RepackWorkloadFactory.getTestArguments()
testArguments.update(deepcopy(REQUEST))
factory = RepackWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "T0")
testWMBSHelper = WMBSHelper(testWorkload,
"Repack",
cachepath=self.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
repackWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Repack")
repackWorkflow.load()
self.assertEqual(
len(repackWorkflow.outputMap),
len(testArguments["Outputs"]) + 1,
"Error: Wrong number of WF outputs in the Repack WF.")
goldenOutputMods = {
"write_PrimaryDataset1_RAW": "RAW",
"write_PrimaryDataset2_RAW": "RAW"
}
for goldenOutputMod, tier in viewitems(goldenOutputMods):
fset = goldenOutputMod + tier
mergedOutput = repackWorkflow.outputMap[fset][0][
"merged_output_fileset"]
unmergedOutput = repackWorkflow.outputMap[fset][0][
"output_fileset"]
mergedOutput.loadData()
unmergedOutput.loadData()
if goldenOutputMod != "write_PrimaryDataset1_RAW":
self.assertEqual(
mergedOutput.name,
"/TestWorkload/Repack/RepackMerge%s/merged-Merged%s" %
(goldenOutputMod, tier),
"Error: Merged output fileset is wrong: %s" %
mergedOutput.name)
self.assertEqual(
unmergedOutput.name, "/TestWorkload/Repack/unmerged-%s" % fset,
"Error: Unmerged output fileset is wrong: %s" %
unmergedOutput.name)
logArchOutput = repackWorkflow.outputMap["logArchive"][0][
"merged_output_fileset"]
unmergedLogArchOutput = repackWorkflow.outputMap["logArchive"][0][
"output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name,
"/TestWorkload/Repack/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name,
"/TestWorkload/Repack/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
for goldenOutputMod, tier in viewitems(goldenOutputMods):
mergeWorkflow = Workflow(
name="TestWorkload",
task="/TestWorkload/Repack/RepackMerge%s" % goldenOutputMod)
mergeWorkflow.load()
self.assertEqual(len(mergeWorkflow.outputMap), 3,
"Error: Wrong number of WF outputs.")
mergedMergeOutput = mergeWorkflow.outputMap[
"Merged%s" % tier][0]["merged_output_fileset"]
unmergedMergeOutput = mergeWorkflow.outputMap[
"Merged%s" % tier][0]["output_fileset"]
mergedMergeOutput.loadData()
unmergedMergeOutput.loadData()
self.assertEqual(
mergedMergeOutput.name,
"/TestWorkload/Repack/RepackMerge%s/merged-Merged%s" %
(goldenOutputMod, tier),
"Error: Merged output fileset is wrong.")
self.assertEqual(
unmergedMergeOutput.name,
"/TestWorkload/Repack/RepackMerge%s/merged-Merged%s" %
(goldenOutputMod, tier),
"Error: Unmerged output fileset is wrong.")
logArchOutput = mergeWorkflow.outputMap["logArchive"][0][
"merged_output_fileset"]
unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0][
"output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(
logArchOutput.name,
"/TestWorkload/Repack/RepackMerge%s/merged-logArchive" %
goldenOutputMod,
"Error: LogArchive output fileset is wrong: %s" %
logArchOutput.name)
self.assertEqual(
unmergedLogArchOutput.name,
"/TestWorkload/Repack/RepackMerge%s/merged-logArchive" %
goldenOutputMod, "Error: LogArchive output fileset is wrong.")
topLevelFileset = Fileset(name="TestWorkload-Repack")
topLevelFileset.loadData()
repackSubscription = Subscription(fileset=topLevelFileset,
workflow=repackWorkflow)
repackSubscription.loadData()
self.assertEqual(repackSubscription["type"], "Repack",
"Error: Wrong subscription type.")
self.assertEqual(
repackSubscription["split_algo"], "Repack",
"Error: Wrong split algorithm. %s" %
repackSubscription["split_algo"])
unmergedOutputs = {
"write_PrimaryDataset1_RAW": "RAW",
"write_PrimaryDataset2_RAW": "RAW"
}
for unmergedOutput, tier in viewitems(unmergedOutputs):
fset = unmergedOutput + tier
unmergedDataTier = Fileset(
name="/TestWorkload/Repack/unmerged-%s" % fset)
unmergedDataTier.loadData()
dataTierMergeWorkflow = Workflow(
name="TestWorkload",
task="/TestWorkload/Repack/RepackMerge%s" % unmergedOutput)
dataTierMergeWorkflow.load()
mergeSubscription = Subscription(fileset=unmergedDataTier,
workflow=dataTierMergeWorkflow)
mergeSubscription.loadData()
self.assertEqual(mergeSubscription["type"], "Merge",
"Error: Wrong subscription type.")
self.assertEqual(
mergeSubscription["split_algo"], "RepackMerge",
"Error: Wrong split algorithm. %s" %
mergeSubscription["split_algo"])
for goldenOutputMod, tier in viewitems(goldenOutputMods):
fset = goldenOutputMod + tier
unmergedFileset = Fileset(name="/TestWorkload/Repack/unmerged-%s" %
fset)
unmergedFileset.loadData()
cleanupWorkflow = Workflow(
name="TestWorkload",
task="/TestWorkload/Repack/RepackCleanupUnmerged%s" %
goldenOutputMod)
cleanupWorkflow.load()
cleanupSubscription = Subscription(fileset=unmergedFileset,
workflow=cleanupWorkflow)
cleanupSubscription.loadData()
self.assertEqual(cleanupSubscription["type"], "Cleanup",
"Error: Wrong subscription type.")
self.assertEqual(cleanupSubscription["split_algo"],
"SiblingProcessingBased",
"Error: Wrong subscription type.")
repackLogCollect = Fileset(
name="/TestWorkload/Repack/unmerged-logArchive")
repackLogCollect.loadData()
repackLogCollectWorkflow = Workflow(
name="TestWorkload", task="/TestWorkload/Repack/LogCollect")
repackLogCollectWorkflow.load()
logCollectSub = Subscription(fileset=repackLogCollect,
workflow=repackLogCollectWorkflow)
logCollectSub.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algorithm.")
for goldenOutputMod, tier in viewitems(goldenOutputMods):
repackMergeLogCollect = Fileset(
name="/TestWorkload/Repack/RepackMerge%s/merged-logArchive" %
goldenOutputMod)
repackMergeLogCollect.loadData()
repackMergeLogCollectWorkflow = Workflow(
name="TestWorkload",
task=
"/TestWorkload/Repack/RepackMerge%s/Repack%sMergeLogCollect" %
(goldenOutputMod, goldenOutputMod))
repackMergeLogCollectWorkflow.load()
logCollectSubscription = Subscription(
fileset=repackMergeLogCollect,
workflow=repackMergeLogCollectWorkflow)
logCollectSubscription.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algorithm.")
return
def testStepChainMC(self):
"""
Build a StepChain workload starting from scratch
"""
# Read in the request
request = json.load(open(self.jsonTemplate))
testArguments = request['createRequest']
testArguments.update({
"CouchURL": os.environ["COUCHURL"],
"ConfigCacheUrl": os.environ["COUCHURL"],
"CouchDBName": "stepchain_t"
})
configDocs = injectStepChainConfigMC(self.configDatabase)
for s in ['Step1', 'Step2', 'Step3']:
testArguments[s]['ConfigCacheID'] = configDocs[s]
factory = StepChainWorkloadFactory()
# test that we cannot stage out different samples with the same output module
self.assertRaises(WMSpecFactoryException,
factory.factoryWorkloadConstruction, "TestWorkload",
testArguments)
testArguments['Step2']['KeepOutput'] = False
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"ProdMinBias",
"MCFakeBlock",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "production")
self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_0_0_pre11")
self.assertEqual(testWorkload.getProcessingString(), "START70_V4")
self.assertEqual(testWorkload.getProcessingVersion(), 1)
self.assertEqual(testWorkload.getPrepID(), "Step-00")
self.assertEqual(sorted(testWorkload.getCMSSWVersions()),
['CMSSW_7_0_0_pre11', 'CMSSW_7_0_0_pre12'])
# test workload attributes
self.assertEqual(testWorkload.processingString, "START70_V4")
self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_0_0_pre11")
self.assertEqual(testWorkload.processingVersion, 1)
self.assertFalse(testWorkload.lumiList, "Wrong lumiList")
self.assertEqual(testWorkload.data.policies.start.policyName,
"MonteCarlo")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 10)
for t in [
'ProdMinBias', 'ProdMinBiasMergeRAWSIMoutput',
'RECOPROD1MergeAODSIMoutput', 'RECOPROD1MergeRECOSIMoutput'
]:
self.assertTrue(t in tasks, "Wrong task name")
self.assertFalse('ProdMinBiasMergeAODSIMoutput' in tasks,
"Wrong task name")
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.name(), "ProdMinBias")
self.assertEqual(task.getPathName(), "/TestWorkload/ProdMinBias")
self.assertEqual(task.taskType(), "Production", "Wrong task type")
splitParams = task.jobSplittingParameters()
self.assertEqual(splitParams['algorithm'], "EventBased",
"Wrong job splitting algo")
self.assertEqual(splitParams['events_per_job'], 150)
self.assertEqual(splitParams['events_per_lumi'], 50)
self.assertFalse(splitParams['lheInputFiles'], "Wrong LHE flag")
self.assertTrue(splitParams['performance']['timePerEvent'] > 4.75)
self.assertTrue(splitParams['performance']['sizePerEvent'] > 1233)
self.assertTrue(
splitParams['performance']['memoryRequirement'] == 2400)
self.assertFalse(task.getTrustSitelists().get('trustlists'),
"Wrong input location flag")
self.assertFalse(task.inputRunWhitelist(), "Wrong run white list")
# test workload step stuff
self.assertEqual(
sorted(task.listAllStepNames()),
['cmsRun1', 'cmsRun2', 'cmsRun3', 'logArch1', 'stageOut1'])
self.assertEqual(task.getTopStepName(), 'cmsRun1')
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertFalse(task.getInputStep(), "Wrong input step")
outModsAndDsets = task.listOutputDatasetsAndModules()
outMods = set([elem['outputModule'] for elem in outModsAndDsets])
outDsets = [elem['outputDataset'] for elem in outModsAndDsets]
self.assertEqual(
outMods, set(['RAWSIMoutput', 'AODSIMoutput', 'RECOSIMoutput']),
"Wrong output modules")
self.assertTrue(
'/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterA-START70_V4-v1/GEN-SIM'
in outDsets)
self.assertTrue(
'/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterD-START70_V4-v1/AODSIM'
in outDsets)
self.assertTrue(
'/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterC-START70_V4-v1/GEN-SIM-RECO'
in outDsets)
self.assertEqual(task.getSwVersion(), 'CMSSW_7_0_0_pre12')
self.assertEqual(task.getScramArch(), 'slc5_amd64_gcc481')
step = task.getStep("cmsRun1")
self.assertFalse(step.data.tree.parent)
self.assertFalse(getattr(step.data.input, 'inputStepName', None))
self.assertFalse(getattr(step.data.input, 'inputOutputModule', None))
self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName,
'FilterA')
self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier,
'GEN-SIM')
self.assertTrue(step.data.output.keep)
self.assertEqual(sorted(step.data.tree.childNames),
['cmsRun2', 'logArch1', 'stageOut1'])
self.assertEqual(step.data.application.setup.cmsswVersion,
'CMSSW_7_0_0_pre12')
self.assertEqual(step.data.application.setup.scramArch,
'slc5_amd64_gcc481')
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
'START70_V4::All')
step = task.getStep("cmsRun2")
self.assertEqual(step.data.tree.parent, "cmsRun1")
self.assertEqual(step.data.input.inputStepName, 'cmsRun1')
self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName,
'FilterB')
self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier,
'GEN-SIM-RAW')
self.assertFalse(step.data.output.keep)
self.assertEqual(step.data.tree.childNames, ["cmsRun3"])
self.assertEqual(step.data.application.setup.cmsswVersion,
'CMSSW_7_0_0_pre11')
self.assertEqual(step.data.application.setup.scramArch,
'slc5_amd64_gcc481')
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
'START70_V4::All')
step = task.getStep("cmsRun3")
self.assertEqual(step.data.tree.parent, "cmsRun2")
self.assertEqual(step.data.input.inputStepName, 'cmsRun2')
self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
self.assertEqual(step.data.output.modules.RECOSIMoutput.filterName,
'FilterC')
self.assertEqual(step.data.output.modules.AODSIMoutput.filterName,
'FilterD')
self.assertEqual(step.data.output.modules.RECOSIMoutput.dataTier,
'GEN-SIM-RECO')
self.assertEqual(step.data.output.modules.AODSIMoutput.dataTier,
'AODSIM')
self.assertTrue(step.data.output.keep)
self.assertFalse(step.data.tree.childNames)
self.assertEqual(step.data.application.setup.cmsswVersion,
'CMSSW_7_0_0_pre11')
self.assertEqual(step.data.application.setup.scramArch,
'slc5_amd64_gcc481')
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
'START70_V4::All')
return
def testStepChainSingleStep(self):
"""
Build a StepChain single step, reading AODSIM and producing MINIAODSIM
"""
testArguments = StepChainWorkloadFactory.getTestArguments()
request = {
"Campaign": "TaskForceUnitTest",
"CMSSWVersion": "CMSSW_7_5_0",
"ScramArch": "slc6_amd64_gcc491",
"DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
"GlobalTag": "PHYS14_25_V3",
"AcquisitionEra": "SingleStep",
"ProcessingString": "UnitTest_StepChain",
"ProcessingVersion": 3,
"PrepID": "MainStep",
"CouchURL": os.environ["COUCHURL"],
"CouchDBName": "stepchain_t",
"Memory": 3500,
"SizePerEvent": 2600,
"TimePerEvent": 26.5,
"Step1": {
"ConfigCacheID":
injectStepChainConfigSingle(self.configDatabase),
"GlobalTag": "PHYS14_25_V44",
"InputDataset":
"/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/RunIISpring15DR74-Asympt50ns_MCRUN2_74_V9A-v1/AODSIM",
"SplittingAlgo": "EventAwareLumiBased",
"EventsPerJob": 500,
"StepName": "StepMini"
},
"StepChain": 1
}
testArguments.update(request)
factory = StepChainWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction(
"TestWorkload", testArguments)
testWMBSHelper = WMBSHelper(testWorkload,
"StepMini",
"GravWhatever",
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
# test workload properties
self.assertEqual(testWorkload.getDashboardActivity(), "processing")
self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
self.assertEqual(testWorkload.getAcquisitionEra(), "SingleStep")
self.assertEqual(testWorkload.getProcessingString(),
"UnitTest_StepChain")
self.assertEqual(testWorkload.getProcessingVersion(), 3)
self.assertEqual(testWorkload.getPrepID(), "MainStep")
self.assertEqual(sorted(testWorkload.getCMSSWVersions()),
['CMSSW_7_5_0'])
self.assertEqual(testWorkload.data.policies.start.policyName, "Block")
# test workload tasks and steps
tasks = testWorkload.listAllTaskNames()
self.assertEqual(len(tasks), 4)
self.assertTrue('StepMiniMergeMINIAODSIMoutput' in tasks)
task = testWorkload.getTask(tasks[0])
self.assertEqual(task.taskType(), "Processing", "Wrong task type")
splitParams = task.jobSplittingParameters()
self.assertEqual(splitParams['algorithm'], "EventAwareLumiBased",
"Wrong job splitting algo")
self.assertEqual(splitParams['events_per_job'], 500)
self.assertTrue(splitParams['performance']['timePerEvent'] > 26.4)
self.assertTrue(splitParams['performance']['sizePerEvent'] > 2599)
self.assertTrue(
splitParams['performance']['memoryRequirement'] == 3500)
# test workload step stuff
self.assertEqual(sorted(task.listAllStepNames()),
['cmsRun1', 'logArch1', 'stageOut1'])
self.assertEqual(task.getTopStepName(), 'cmsRun1')
self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
self.assertFalse(task.getInputStep(), "Wrong input step")
outModsAndDsets = task.listOutputDatasetsAndModules()[0]
self.assertEqual(outModsAndDsets['outputModule'], 'MINIAODSIMoutput')
self.assertEqual(
outModsAndDsets['outputDataset'],
'/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/SingleStep-UnitTest_StepChain-v3/MINIAODSIM'
)
self.assertEqual(task.getSwVersion(), 'CMSSW_7_5_0')
self.assertEqual(task.getScramArch(), 'slc6_amd64_gcc491')
step = task.getStep("cmsRun1")
self.assertEqual(
step.data.application.configuration.arguments.globalTag,
'PHYS14_25_V44')
return
def testFilesets(self):
"""
Test workflow tasks, filesets and subscriptions creation
"""
# expected tasks, filesets, subscriptions, etc
expOutTasks = ['/TestWorkload/Production',
'/TestWorkload/Production/ProductionMergeOutputB',
'/TestWorkload/Production/ProductionMergeOutputA']
expWfTasks = ['/TestWorkload/Production',
'/TestWorkload/Production/LogCollect',
'/TestWorkload/Production/ProductionCleanupUnmergedOutputA',
'/TestWorkload/Production/ProductionCleanupUnmergedOutputB',
'/TestWorkload/Production/ProductionMergeOutputA',
'/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect',
'/TestWorkload/Production/ProductionMergeOutputB',
'/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect']
expFsets = ['FILESET_DEFINED_DURING_RUNTIME',
'/TestWorkload/Production/unmerged-OutputBUSER',
'/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive',
'/TestWorkload/Production/ProductionMergeOutputA/merged-MergedRECO',
'/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive',
'/TestWorkload/Production/ProductionMergeOutputB/merged-MergedUSER',
'/TestWorkload/Production/unmerged-logArchive',
'/TestWorkload/Production/unmerged-OutputARECO']
subMaps = ['FILESET_DEFINED_DURING_RUNTIME',
(6,
'/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive',
'/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect',
'MinFileBased',
'LogCollect'),
(3,
'/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive',
'/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect',
'MinFileBased',
'LogCollect'),
(8,
'/TestWorkload/Production/unmerged-logArchive',
'/TestWorkload/Production/LogCollect',
'MinFileBased',
'LogCollect'),
(7,
'/TestWorkload/Production/unmerged-OutputARECO',
'/TestWorkload/Production/ProductionCleanupUnmergedOutputA',
'SiblingProcessingBased',
'Cleanup'),
(5,
'/TestWorkload/Production/unmerged-OutputARECO',
'/TestWorkload/Production/ProductionMergeOutputA',
'ParentlessMergeBySize',
'Merge'),
(4,
'/TestWorkload/Production/unmerged-OutputBUSER',
'/TestWorkload/Production/ProductionCleanupUnmergedOutputB',
'SiblingProcessingBased',
'Cleanup'),
(2,
'/TestWorkload/Production/unmerged-OutputBUSER',
'/TestWorkload/Production/ProductionMergeOutputB',
'ParentlessMergeBySize',
'Merge')]
testArguments = MonteCarloWorkloadFactory.getTestArguments()
testArguments["CouchURL"] = os.environ["COUCHURL"]
testArguments["CouchDBName"] = TEST_DB_NAME
testArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
factory = MonteCarloWorkloadFactory()
testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)
myMask = Mask(FirstRun=1, FirstLumi=1, FirstEvent=1, LastRun=1, LastLumi=10, LastEvent=1000)
testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask,
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# same function as in WMBSHelper, otherwise we cannot know which fileset name is
maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)])
topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest()
expFsets[0] = topFilesetName
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subMaps[0] = (1, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production')
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
### create another top level subscription
myMask = Mask(FirstRun=1, FirstLumi=11, FirstEvent=1001, LastRun=1, LastLumi=20, LastEvent=2000)
testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask,
cachepath=self.testInit.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)
# same function as in WMBSHelper, otherwise we cannot know which fileset name is
maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)])
topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest()
expFsets.append(topFilesetName)
# returns a tuple of id, name, open and last_update
filesets = self.listFilesets.execute()
self.assertItemsEqual([item[1] for item in filesets], expFsets)
subMaps.append((9, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production'))
subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
self.assertItemsEqual(subscriptions, subMaps)
