def addNewFilesToOpenSubscriptions(self, *elements): """Inject new files to wmbs for running elements that have new files. Assumes elements are from the same workflow""" if not self.params['LocalQueueFlag']: return wmspec = None for ele in elements: if not ele.isRunning() or not ele['SubscriptionId'] or not ele: continue if not ele['Inputs'] or not ele['OpenForNewData']: continue if not wmspec: wmspec = self.backend.getWMSpec(ele['RequestName']) blockName, dbsBlock = self._getDBSBlock(ele, wmspec) if ele['NumOfFilesAdded'] != len(dbsBlock['Files']): self.logger.info("Adding new files to open block %s (%s)" % (blockName, ele.id)) from WMCore.WorkQueue.WMBSHelper import WMBSHelper wmbsHelper = WMBSHelper(wmspec, ele['TaskName'], blockName, ele['Mask'], self.params['CacheDir']) ele['NumOfFilesAdded'] += wmbsHelper.createSubscriptionAndAddFiles( block=dbsBlock)[1] self.backend.updateElements( ele.id, NumOfFilesAdded=ele['NumOfFilesAdded']) if dbsBlock['IsOpen'] != ele['OpenForNewData']: self.logger.info("Closing open block %s (%s)" % (blockName, ele.id)) self.backend.updateInboxElements( ele['ParentQueueId'], OpenForNewData=dbsBlock['IsOpen']) self.backend.updateElements(ele.id, OpenForNewData=dbsBlock['IsOpen']) ele['OpenForNewData'] = dbsBlock['IsOpen']
def addNewFilesToOpenSubscriptions(self, *elements): """Inject new files to wmbs for running elements that have new files. Assumes elements are from the same workflow""" if not self.params['LocalQueueFlag']: return wmspec = None for ele in elements: if not ele.isRunning() or not ele['SubscriptionId'] or not ele: continue if not ele['Inputs'] or not ele['OpenForNewData']: continue if not wmspec: wmspec = self.backend.getWMSpec(ele['RequestName']) blockName, dbsBlock = self._getDBSBlock(ele, wmspec) if ele['NumOfFilesAdded'] != len(dbsBlock['Files']): self.logger.info("Adding new files to open block %s (%s)" % (blockName, ele.id)) from WMCore.WorkQueue.WMBSHelper import WMBSHelper wmbsHelper = WMBSHelper(wmspec, ele['TaskName'], blockName, ele['Mask'], self.params['CacheDir']) ele['NumOfFilesAdded'] += wmbsHelper.createSubscriptionAndAddFiles(block = dbsBlock)[1] self.backend.updateElements(ele.id, NumOfFilesAdded = ele['NumOfFilesAdded']) if dbsBlock['IsOpen'] != ele['OpenForNewData']: self.logger.info("Closing open block %s (%s)" % (blockName, ele.id)) self.backend.updateInboxElements(ele['ParentQueueId'], OpenForNewData = dbsBlock['IsOpen']) self.backend.updateElements(ele.id, OpenForNewData = dbsBlock['IsOpen']) ele['OpenForNewData'] = dbsBlock['IsOpen']
def testRelValMC(self): """ Configure, instantiate, install into WMBS and check that the subscriptions in WMBS are setup correctly. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "relvalmc_t" defaultArguments["GenOutputModuleName"] = "OutputA" defaultArguments["StepOneOutputModuleName"] = "OutputB" defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig() defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig() defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig() testWorkload = relValMCWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Generation", "SomeBlock") testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) # now run the tests on single workload instance installed into WMBS # each of the subtests is dealing with specific tasks self._generationTaskTest() self._stepTwoTaskTest() self._stepOneTaskTest() return
def _wmbsPreparation(self, match, wmspec, blockName, dbsBlock): """Inject data into wmbs and create subscription. """ from WMCore.WorkQueue.WMBSHelper import WMBSHelper self.logger.info("Adding WMBS subscription for %s" % match['RequestName']) mask = match['Mask'] wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir']) sub, match[ 'NumOfFilesAdded'] = wmbsHelper.createSubscriptionAndAddFiles( block=dbsBlock) self.logger.info( "Created top level subscription %s for %s with %s files" % (sub['id'], match['RequestName'], match['NumOfFilesAdded'])) # update couch with wmbs subscription info match['SubscriptionId'] = sub['id'] match['Status'] = 'Running' # do update rather than save to avoid conflicts from other thread writes self.backend.updateElements(match.id, Status='Running', SubscriptionId=sub['id'], NumOfFilesAdded=match['NumOfFilesAdded']) return sub
def testRelValMCWithPileup(self): """ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. The input configuration includes pileup input files. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ProcConfigCacheID"] = self.injectMonteCarloConfig() # add pile up configuration defaultArguments["PileupConfig"] = {"mc": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"], "data": ["/some/minbias/dataset3"]} testWorkload = monteCarloWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() return
def testThreeStepChainedReDigi(self): """ _testThreeStepChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW and RECO data can be created and installed into WMBS correctly. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs() defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["KeepStepOneOutput"] = False defaultArguments["KeepStepTwoOutput"] = False testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock") testWMBSHelper.createSubscription() self.verifyKeepAOD() return
def testChainedReDigi(self): """ _testChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW data can be created and installed into WMBS correctly. This will only verify the step one/step two information in WMBS as the step three information is the same as the dependent workflow. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs() defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["KeepStepOneOutput"] = False testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock") testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyDiscardRAW() return
def testCombinedReDigiRecoConfig(self): """ _testCombinedReDigiRecoConfig_ Verify that a ReDigi workflow that uses a single step one config installs into WMBS correctly. """ defaultArguments = ReDigiWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = injectReDigiConfigs(self.configDatabase, combinedStepOne = True) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[2] defaultArguments["StepOneOutputModuleName"] = "RECODEBUGoutput" factory = ReDigiWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyDiscardRAW() return
def testRelValMCWithPileup(self): """ Configure, instantiate, install into WMBS and check that the subscriptions in WMBS are setup correctly. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "relvalmc_t" defaultArguments["GenOutputModuleName"] = "OutputA" defaultArguments["StepOneOutputModuleName"] = "OutputB" defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig() defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig() defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig() # add pile up information - for the generation task defaultArguments["PileupConfig"] = { "cosmics": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"], "minbias": ["/some/minbias/dataset3"], } testWorkload = relValMCWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock") testWMBSHelper.createSubscription() # now run the tests on single workload instance installed into WMBS # each of the subtests is dealing with specific tasks self._generationTaskTest() self._stepOneTaskTest() self._stepTwoTaskTest()
def testStoreResults(self): """ _testStoreResults_ Create a StoreResults workflow and verify it installs into WMBS correctly. """ arguments = StoreResultsWorkloadFactory.getTestArguments() arguments.update({'CmsPath': "/uscmst1/prod/sw/cms"}) factory = StoreResultsWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", arguments) testWMBSHelper = WMBSHelper(testWorkload, "StoreResults", "SomeBlock", cachepath=self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) testWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/StoreResults") testWorkflow.load() self.assertEqual(len(testWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") goldenOutputMods = ["Merged"] for goldenOutputMod in goldenOutputMods: mergedOutput = testWorkflow.outputMap[goldenOutputMod][0]["merged_output_fileset"] unmergedOutput = testWorkflow.outputMap[goldenOutputMod][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/StoreResults/merged-%s" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/StoreResults/merged-%s" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s." % unmergedOutput.name) logArchOutput = testWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = testWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/StoreResults/merged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/StoreResults/merged-logArchive", "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name="TestWorkload-StoreResults-SomeBlock") topLevelFileset.loadData() procSubscription = Subscription(fileset=topLevelFileset, workflow=testWorkflow) procSubscription.loadData() self.assertEqual(procSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(procSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algo.") return
def testCombinedReDigiRecoConfig(self): """ _testCombinedReDigiRecoConfig_ Verify that a ReDigi workflow that uses a single step one config installs into WMBS correctly. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs(combinedStepOne = True) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[2] defaultArguments["StepThreeConfigCacheID"] = None defaultArguments["StepOneOutputModuleName"] = "RECODEBUGoutput" testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock") testWMBSHelper.createSubscription() self.verifyDiscardRAW() return
def createWMBSHelperWithTopTask(self, wmspec, block, mask=None, parentFlag=False, detail=False, commonLocation=None): topLevelTask = getFirstTask(wmspec) wmbs = WMBSHelper(wmspec, topLevelTask.name(), block, mask, cachepath=self.workDir, commonLocation=commonLocation) if block: blockName = block if parentFlag: block = self.dbs.getFileBlockWithParents(blockName) location = self.phedex.getReplicaPhEDExNodesForBlocks( block=[blockName], complete='y') block['PhEDExNodeNames'] = location[blockName] else: block = self.dbs.getFileBlock(blockName) location = self.phedex.getReplicaPhEDExNodesForBlocks( block=[blockName], complete='y') block['PhEDExNodeNames'] = location[blockName] sub, files = wmbs.createSubscriptionAndAddFiles(block=block) if detail: return wmbs, sub, files else: return wmbs
def createWMBSHelperWithTopTask(self, wmspec, block, mask = None): topLevelTask = getFirstTask(wmspec) wmbs = WMBSHelper(wmspec, block, mask, cachepath = self.workDir) if block: block = self.dbs.getFileBlock(block)[block] wmbs.createSubscriptionAndAddFiles(block = block) return wmbs
def main(): start = time.time() # blockName = match['Inputs'].keys()[0] blockName = "/acdc/vlimant_ACDC0_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_190218_145226_481/:pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222:SUS-RunIIFall18wmLHEGS-00025_0/0/31055" # acdcInfo = match['ACDC'] acdcInfo = {"database": "acdcserver", "fileset": "/pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222/SUS-RunIIFall18wmLHEGS-00025_0", "collection": "pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222", "server": "https://cmsweb.cern.ch/couchdb"} acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) splitedBlockName = ACDCBlock.splitBlockName(blockName) print("Splitted block name: %s" % splitedBlockName) fileLists = acdc.getChunkFiles(acdcInfo['collection'], acdcInfo['fileset'], splitedBlockName['Offset'], splitedBlockName['NumOfFiles']) print("Retrieved %d unique files from the ACDCServer" % len(fileLists)) block = {} block["Files"] = fileLists wantedLumis = set([252052, 240646]) for f in fileLists: for run in f['runs']: maskDict = run.json() lumisSet = set(maskDict['Lumis'].keys()) if wantedLumis.intersection(lumisSet): print("File: %s with events: %s, contains these lumis: %s" % (f['lfn'], f['events'], wantedLumis.intersection(lumisSet))) # with open("chunkfiles.json", 'w') as fo: # json.dump(block, fo) end = time.time() print("Spent %s secs running so far" % (end - start)) sys.exit(1) ### Now doing the WMBSHelper stuff reqUrl = "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache" requestName = "vlimant_ACDC0_task_HIG-RunIIFall17wmLHEGS-01122__v1_T_180808_130708_5376" wmspec = WMWorkloadHelper() wmspec.loadSpecFromCouch(reqUrl, requestName) taskName = "HIG-RunIIFall17DRPremix-00788_0" mask = None cacheDir = "/data/srv/wmagent/v1.1.14.patch6/install/wmagent/WorkQueueManager/cache" # wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir']) wmbsHelper = WMBSHelper(wmspec, taskName, blockName, mask, cacheDir) sub, numFilesAdded = wmbsHelper.createSubscriptionAndAddFiles(block=block)
def createWMBSHelperWithTopTask(self, wmspec, block, mask=None, parentFlag=False, detail=False): topLevelTask = getFirstTask(wmspec) wmbs = WMBSHelper(wmspec, topLevelTask.name(), block, mask, cachepath=self.workDir) if block: if parentFlag: block = self.dbs.getFileBlockWithParents(block)[block] else: block = self.dbs.getFileBlock(block)[block] sub, files = wmbs.createSubscriptionAndAddFiles(block=block) if detail: return wmbs, sub, files else: return wmbs
def createWMBSHelperWithTopTask(self, wmspec, block, mask = None, parentFlag = False, detail = False): topLevelTask = getFirstTask(wmspec) wmbs = WMBSHelper(wmspec, topLevelTask.name(), block, mask, cachepath = self.workDir) if block: if parentFlag: block = self.dbs.getFileBlockWithParents(block)[block] else: block = self.dbs.getFileBlock(block)[block] sub, files = wmbs.createSubscriptionAndAddFiles(block = block) if detail: return wmbs, sub, files else: return wmbs
def testMonteCarloExtension(self): """ _testMonteCarloExtension_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. This uses a non-zero first lumi. Check that the splitting arguments are correctly set for the lfn counter. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["FirstLumi"] = 10001 defaultArguments["EventsPerJob"] = 100 defaultArguments["FirstEvent"] = 10001 # defaultArguments["FirstEvent"] = 10001 initial_lfn_counter = ( 100 ) # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath("/TestWorkload/Production") productionSplitting = productionTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored") self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") for outputMod in ["OutputA", "OutputB"]: mergeTask = testWorkload.getTaskByPath("/TestWorkload/Production/ProductionMerge%s" % outputMod) mergeSplitting = mergeTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored") self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") return
def _wmbsPreparation(self, match, wmspec, blockName, dbsBlock): """Inject data into wmbs and create subscription. """ from WMCore.WorkQueue.WMBSHelper import WMBSHelper self.logger.info("Adding WMBS subscription for %s" % match['RequestName']) mask = match['Mask'] wmbsHelper = WMBSHelper(wmspec, blockName, mask, self.params['CacheDir']) sub, match['NumOfFilesAdded'] = wmbsHelper.createSubscriptionAndAddFiles(block = dbsBlock) self.logger.info("Created top level subscription %s for %s with %s files" % (sub['id'], match['RequestName'], match['NumOfFilesAdded'])) match['SubscriptionId'] = sub['id'] match['Status'] = 'Running' self.backend.saveElements(match) return sub
def testMonteCarlo(self): """ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ProdConfigCacheID"] = self.injectMonteCarloConfig() testWorkload = monteCarloWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock") testWMBSHelper.createSubscription() self._commonMonteCarloTest()
def testMonteCarloExtension(self): """ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. This uses a non-zero first event and lumi. Check that the splitting arguments are correctly set for the lfn counter. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ProcConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["FirstEvent"] = 3571428573 defaultArguments["FirstLumi"] = 26042 defaultArguments["TimePerEvent"] = 15 defaultArguments["FilterEfficiency"] = 0.014 defaultArguments["TotalTime"] = 28800 initial_lfn_counter = 26042 # Same as the previous number of jobs + 1 which is the same value of the first lumi testWorkload = monteCarloWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') productionSplitting = productionTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored") self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") for outputMod in ["OutputA", "OutputB"]: mergeTask = testWorkload.getTaskByPath('/TestWorkload/Production/ProductionMerge%s' % outputMod) mergeSplitting = mergeTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored") self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") return
def testLHEStepZero(self): """ _testLHEStepZero_ Make sure that the workload can be created and complies with the common MonteCarlo test """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() testWorkload = lheStepZeroWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest()
def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = [] expWfTasks = ['/TestWorkload/EndOfRunDQMHarvest', '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect'] expFsets = ['TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO', '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive'] subMaps = [(2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive', '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'), (1, 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO', '/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting')] testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig() testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "EndOfRunDQMHarvest", blockName=testArguments['InputDataset'], cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)
def testChainedReDigi(self): """ _testChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW data can be created and installed into WMBS correctly. This will only verify the step one/step two information in WMBS as the step three information is the same as the dependent workflow. """ defaultArguments = ReDigiWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = injectReDigiConfigs(self.configDatabase) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["StepOneOutputModuleName"] = "RAWDEBUGoutput" defaultArguments["StepTwoOutputModuleName"] = "RECODEBUGoutput" defaultArguments["MCPileup"] = PILEUP_DATASET defaultArguments["KeepStepOneOutput"] = False factory = ReDigiWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) # Verify that pileup is configured for both of the cmsRun steps in the # top level task. topLevelTask = testWorkload.getTopLevelTask()[0] cmsRun1Helper = topLevelTask.getStepHelper("cmsRun1") cmsRun2Helper = topLevelTask.getStepHelper("cmsRun2") cmsRun1PileupConfig = cmsRun1Helper.getPileup() cmsRun2PileupConfig = cmsRun2Helper.getPileup() self.assertTrue(cmsRun1PileupConfig.mc.dataset, "/some/cosmics/dataset") self.assertTrue(cmsRun2PileupConfig.mc.dataset, "/some/cosmics/dataset") testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyDiscardRAW() return
def _wmbsPreparation(self, match, wmspec, blockName, dbsBlock): """Inject data into wmbs and create subscription. """ from WMCore.WorkQueue.WMBSHelper import WMBSHelper self.logger.info("Adding WMBS subscription for %s" % match['RequestName']) mask = match['Mask'] wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir']) sub, match['NumOfFilesAdded'] = wmbsHelper.createSubscriptionAndAddFiles(block = dbsBlock) self.logger.info("Created top level subscription %s for %s with %s files" % (sub['id'], match['RequestName'], match['NumOfFilesAdded'])) # update couch with wmbs subscription info match['SubscriptionId'] = sub['id'] match['Status'] = 'Running' # do update rather than save to avoid conflicts from other thread writes self.backend.updateElements(match.id, Status = 'Running', SubscriptionId = sub['id'], NumOfFilesAdded = match['NumOfFilesAdded']) return sub
def testMonteCarlo(self): """ _testMonteCarlo_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() return
def testThreeStepChainedReDigi(self): """ _testThreeStepChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW and RECO data can be created and installed into WMBS correctly. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs() defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["KeepStepOneOutput"] = False defaultArguments["KeepStepTwoOutput"] = False testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") self.assertTrue(len(testWorkload.getTopLevelTask()) == 1, "Error: Wrong number of top level tasks.") topLevelTask = testWorkload.getTopLevelTask()[0] topLevelStep = topLevelTask.steps() cmsRun2Step = topLevelStep.getStep("cmsRun2").getTypeHelper() self.assertTrue(len(cmsRun2Step.listOutputModules()) == 2, "Error: Wrong number of output modules in cmsRun2.") testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyKeepAOD() return
def testMonteCarlo(self): """ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() testWorkload = monteCarloWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() return
def testRelValMCWithPileup(self): """ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. The input configuration includes pileup input files. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() # add pile up configuration defaultArguments["PileupConfig"] = { "mc": ["/some/cosmics/dataset1", "/some/cosmics/dataset2"], "data": ["/some/minbias/dataset3"] } testWorkload = monteCarloWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() return
def testRelValMCWithPileup(self): """ Configure, instantiate, install into WMBS and check that the subscriptions in WMBS are setup correctly. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "relvalmc_t" defaultArguments["GenOutputModuleName"] = "OutputA" defaultArguments["StepOneOutputModuleName"] = "OutputB" defaultArguments["GenConfigCacheID"] = self.injectGenerationConfig() defaultArguments["StepOneConfigCacheID"] = self.injectStepOneConfig() defaultArguments["StepTwoConfigCacheID"] = self.injectStepTwoConfig() # add pile up information - for the generation task defaultArguments["PileupConfig"] = {"cosmics": ["/some/cosmics/dataset1","/some/cosmics/dataset2"], "minbias": ["/some/minbias/dataset3"]} testWorkload = relValMCWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Generation", "SomeBlock") testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) # now run the tests on single workload instance installed into WMBS # each of the subtests is dealing with specific tasks self._generationTaskTest() self._stepOneTaskTest() self._stepTwoTaskTest()
def testMCWithLHE(self): """ _testMCWithLHE_ Create a MonteCarlo workflow with a variation on the type of work done, this refers to the previous LHEStepZero where the input can be .lhe files and there is more than one lumi per job. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["LheInputFiles"] = "True" defaultArguments["EventsPerJob"] = 200 defaultArguments["EventsPerLumi"] = 50 factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') splitting = productionTask.jobSplittingParameters() self.assertEqual(splitting["events_per_job"], 200) self.assertEqual(splitting["events_per_lumi"], 50) self.assertEqual(splitting["lheInputFiles"], True) self.assertFalse(splitting["deterministicPileup"]) return
def testMCWithPileup(self): """ _testMCWithPileup_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. The input configuration includes pileup input files. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() # Add pileup inputs defaultArguments["MCPileup"] = COSMICS_PU defaultArguments["DataPileup"] = DATA_PU defaultArguments["DeterministicPileup"] = True factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') cmsRunStep = productionTask.getStep("cmsRun1").getTypeHelper() pileupData = cmsRunStep.getPileup() self.assertEqual(pileupData.data.dataset, [DATA_PU]) self.assertEqual(pileupData.mc.dataset, [COSMICS_PU]) splitting = productionTask.jobSplittingParameters() self.assertTrue(splitting["deterministicPileup"]) return
def testCombinedReDigiRecoConfig(self): """ _testCombinedReDigiRecoConfig_ Verify that a ReDigi workflow that uses a single step one config installs into WMBS correctly. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs(combinedStepOne = True) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[2] defaultArguments["StepThreeConfigCacheID"] = None defaultArguments["StepOneOutputModuleName"] = "RECODEBUGoutput" testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyDiscardRAW() return
def testMonteCarlo(self): """ _testMonteCarlo_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() return
def testSingleStepReDigi(self): """ _testSingleStepReDigi_ Verify that a single step ReDigi workflow can be created and installed correctly into WMBS. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs() defaultArguments["StepOneConfigCacheID"] = configs[2] defaultArguments["StepTwoConfigCacheID"] = None defaultArguments["StepThreeConfigCacheID"] = None testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyKeepAOD() return
def testStoreResults(self): """ _testStoreResults_ Create a StoreResults workflow and verify it installs into WMBS correctly. """ arguments = StoreResultsWorkloadFactory.getTestArguments() factory = StoreResultsWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", arguments) testWMBSHelper = WMBSHelper(testWorkload, "StoreResults", "SomeBlock", cachepath=self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) testWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/StoreResults") testWorkflow.load() self.assertEqual(len(testWorkflow.outputMap), 2, "Error: Wrong number of WF outputs.") goldenOutputMods = {"Merged": "USER"} for goldenOutputMod, tier in viewitems(goldenOutputMods): fset = goldenOutputMod + tier mergedOutput = testWorkflow.outputMap[fset][0]["merged_output_fileset"] unmergedOutput = testWorkflow.outputMap[fset][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/StoreResults/merged-%s" % fset, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/StoreResults/merged-%s" % fset, "Error: Unmerged output fileset is wrong: %s." % unmergedOutput.name) logArchOutput = testWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = testWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/StoreResults/merged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/StoreResults/merged-logArchive", "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name="TestWorkload-StoreResults-SomeBlock") topLevelFileset.loadData() procSubscription = Subscription(fileset=topLevelFileset, workflow=testWorkflow) procSubscription.loadData() self.assertEqual(procSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(procSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algo.") return
def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = ['/TestWorkload/StoreResults'] expWfTasks = [ '/TestWorkload/StoreResults', '/TestWorkload/StoreResults/StoreResultsLogCollect' ] expFsets = [ 'TestWorkload-StoreResults-/MinimumBias/ComissioningHI-v1/RAW', '/TestWorkload/StoreResults/merged-Merged', '/TestWorkload/StoreResults/merged-logArchive' ] subMaps = [ (2, '/TestWorkload/StoreResults/merged-logArchive', '/TestWorkload/StoreResults/StoreResultsLogCollect', 'MinFileBased', 'LogCollect'), (1, 'TestWorkload-StoreResults-/MinimumBias/ComissioningHI-v1/RAW', '/TestWorkload/StoreResults', 'ParentlessMergeBySize', 'Merge') ] testArguments = StoreResultsWorkloadFactory.getTestArguments() testArguments.update({'CmsPath': "/uscmst1/prod/sw/cms"}) factory = StoreResultsWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "StoreResults", blockName=testArguments['InputDataset'], cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) print("Tasks producing output:\n%s" % pformat(testWorkload.listOutputProducingTasks())) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") print("List of workflow tasks:\n%s" % pformat([item['task'] for item in workflows])) self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() print("List of filesets:\n%s" % pformat([item[1] for item in filesets])) self.assertItemsEqual([item[1] for item in filesets], expFsets) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) print("List of subscriptions:\n%s" % pformat(subscriptions)) self.assertItemsEqual(subscriptions, subMaps)
def testSingleStepReDigi(self): """ _testSingleStepReDigi_ Verify that a single step ReDigi workflow can be created and installed correctly into WMBS. """ defaultArguments = ReDigiWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = injectReDigiConfigs(self.configDatabase) defaultArguments["StepOneConfigCacheID"] = configs[2] factory = ReDigiWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyKeepAOD() return
def testThreeStepChainedReDigi(self): """ _testThreeStepChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW and RECO data can be created and installed into WMBS correctly. """ defaultArguments = ReDigiWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = injectReDigiConfigs(self.configDatabase) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["KeepStepOneOutput"] = False defaultArguments["KeepStepTwoOutput"] = False defaultArguments["StepOneOutputModuleName"] = "RAWDEBUGoutput" defaultArguments["StepTwoOutputModuleName"] = "RECODEBUGoutput" factory = ReDigiWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) self.assertTrue(len(testWorkload.getTopLevelTask()) == 1, "Error: Wrong number of top level tasks.") topLevelTask = testWorkload.getTopLevelTask()[0] topLevelStep = topLevelTask.steps() cmsRun2Step = topLevelStep.getStep("cmsRun2").getTypeHelper() self.assertTrue(len(cmsRun2Step.listOutputModules()) == 2, "Error: Wrong number of output modules in cmsRun2.") testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyKeepAOD() return
def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = [] expWfTasks = [ '/TestWorkload/EndOfRunDQMHarvest', '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect' ] expFsets = [ 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO', '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive' ] subMaps = [ (2, '/TestWorkload/EndOfRunDQMHarvest/unmerged-logArchive', '/TestWorkload/EndOfRunDQMHarvest/EndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'), (1, 'TestWorkload-EndOfRunDQMHarvest-/NoBPTX/Run2016F-23Sep2016-v1/DQMIO', '/TestWorkload/EndOfRunDQMHarvest', 'Harvest', 'Harvesting') ] testArguments = DQMHarvestWorkloadFactory.getTestArguments() testArguments.update(REQUEST) testArguments['DQMConfigCacheID'] = self.injectDQMHarvestConfig() testArguments.pop("ConfigCacheID", None) factory = DQMHarvestWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "EndOfRunDQMHarvest", blockName=testArguments['InputDataset'], cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)
def testMonteCarloExtension(self): """ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. This uses a non-zero first event and lumi. Check that the splitting arguments are correctly set for the lfn counter. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["FirstEvent"] = 3571428573 defaultArguments["FirstLumi"] = 26042 defaultArguments["TimePerEvent"] = 15 defaultArguments["FilterEfficiency"] = 0.014 defaultArguments["TotalTime"] = 28800 initial_lfn_counter = 26042 # Same as the previous number of jobs + 1 which is the same value of the first lumi testWorkload = monteCarloWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') productionSplitting = productionTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored") self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") for outputMod in ["OutputA", "OutputB"]: mergeTask = testWorkload.getTaskByPath( '/TestWorkload/Production/ProductionMerge%s' % outputMod) mergeSplitting = mergeTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored") self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") return
def testMonteCarloExtension(self): """ _testMonteCarloExtension_ Create a Monte Carlo workflow and verify that it is injected correctly into WMBS and invoke its detailed test. This uses a non-zero first lumi. Check that the splitting arguments are correctly set for the lfn counter. """ defaultArguments = MonteCarloWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = TEST_DB_NAME defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() defaultArguments["FirstLumi"] = 10001 defaultArguments["EventsPerJob"] = 100 defaultArguments["FirstEvent"] = 10001 #defaultArguments["FirstEvent"] = 10001 initial_lfn_counter = 100 # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", defaultArguments) testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest() productionTask = testWorkload.getTaskByPath('/TestWorkload/Production') productionSplitting = productionTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored") self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") for outputMod in ["OutputA", "OutputB"]: mergeTask = testWorkload.getTaskByPath( '/TestWorkload/Production/ProductionMerge%s' % outputMod) mergeSplitting = mergeTask.jobSplittingParameters() self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored") self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter") return
def testChainedReDigi(self): """ _testChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW data can be created and installed into WMBS correctly. This will only verify the step one/step two information in WMBS as the step three information is the same as the dependent workflow. """ defaultArguments = ReDigiWorkloadFactory.getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = injectReDigiConfigs(self.configDatabase) defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["StepOneOutputModuleName"] = "RAWDEBUGoutput" defaultArguments["StepTwoOutputModuleName"] = "RECODEBUGoutput" defaultArguments["MCPileup"] = "/mixing/pileup/DATASET" defaultArguments["KeepStepOneOutput"] = False factory = ReDigiWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments) # Verify that pileup is configured for both of the cmsRun steps in the # top level task. topLevelTask = testWorkload.getTopLevelTask()[0] cmsRun1Helper = topLevelTask.getStepHelper("cmsRun1") cmsRun2Helper = topLevelTask.getStepHelper("cmsRun2") cmsRun1PileupConfig = cmsRun1Helper.getPileup() cmsRun2PileupConfig = cmsRun2Helper.getPileup() self.assertTrue(cmsRun1PileupConfig.mc.dataset, "/some/cosmics/dataset") self.assertTrue(cmsRun2PileupConfig.mc.dataset, "/some/cosmics/dataset") testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyDiscardRAW() return
def testChainedReDigi(self): """ _testChaninedReDigi_ Verify that a chained ReDigi workflow that discards RAW data can be created and installed into WMBS correctly. This will only verify the step one/step two information in WMBS as the step three information is the same as the dependent workflow. """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "redigi_t" configs = self.injectReDigiConfigs() defaultArguments["StepOneConfigCacheID"] = configs[0] defaultArguments["StepTwoConfigCacheID"] = configs[1] defaultArguments["StepThreeConfigCacheID"] = configs[2] defaultArguments["KeepStepOneOutput"] = False testWorkload = reDigiWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") # Verify that pileup is configured for both of the cmsRun steps in the # top level task. topLevelTask = testWorkload.getTopLevelTask()[0] cmsRun1Helper = topLevelTask.getStepHelper("cmsRun1") cmsRun2Helper = topLevelTask.getStepHelper("cmsRun2") cmsRun1PileupConfig = cmsRun1Helper.getPileup() cmsRun2PileupConfig = cmsRun2Helper.getPileup() self.assertTrue(cmsRun1PileupConfig.mc.dataset, "/some/cosmics/dataset") self.assertTrue(cmsRun2PileupConfig.mc.dataset, "/some/cosmics/dataset") testWMBSHelper = WMBSHelper(testWorkload, "StepOneProc", "SomeBlock", cachepath = self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.verifyDiscardRAW() return
def testLHEStepZero(self): """ _testLHEStepZero_ Make sure that the workload can be created and complies with the common MonteCarlo test """ defaultArguments = getTestArguments() defaultArguments["CouchURL"] = os.environ["COUCHURL"] defaultArguments["CouchDBName"] = "rereco_t" defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig() testWorkload = lheStepZeroWorkload("TestWorkload", defaultArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "DWMWM") testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self._commonMonteCarloTest()
def testPromptRecoWithSkims(self): """ _testT1PromptRecoWithSkim_ Create a T1 Prompt Reconstruction workflow with PromptSkims and verify it installs into WMBS correctly. """ testArguments = PromptRecoWorkloadFactory.getTestArguments() testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["EnableHarvesting"] = True factory = PromptRecoWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "T0") testWMBSHelper = WMBSHelper(testWorkload, "Reco", "SomeBlock", cachepath=self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) recoWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco") recoWorkflow.load() self.assertEqual(len(recoWorkflow.outputMap.keys()), len(testArguments["WriteTiers"]) + 1, "Error: Wrong number of WF outputs in the Reco WF.") goldenOutputMods = {"write_RECO": "RECO", "write_ALCARECO": "ALCARECO", "write_AOD": "AOD", "write_DQM": "DQM"} for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier mergedOutput = recoWorkflow.outputMap[fset][0]["merged_output_fileset"] unmergedOutput = recoWorkflow.outputMap[fset][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() if goldenOutputMod != "write_ALCARECO": self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/unmerged-%s" % fset, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = recoWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = recoWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") alcaSkimWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim") alcaSkimWorkflow.load() self.assertEqual(len(alcaSkimWorkflow.outputMap.keys()), len(testArguments["AlcaSkims"]) + 1, "Error: Wrong number of WF outputs in the AlcaSkim WF.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: fset = goldenOutputMod + "ALCARECO" mergedOutput = alcaSkimWorkflow.outputMap[fset][0]["merged_output_fileset"] unmergedOutput = alcaSkimWorkflow.outputMap[fset][0]["output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() self.assertEqual(mergedOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-MergedALCARECO" % goldenOutputMod, "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual(unmergedOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-%sALCARECO" % goldenOutputMod, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = alcaSkimWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") dqmWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged") dqmWorkflow.load() logArchOutput = dqmWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = dqmWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") goldenOutputMods = {"write_RECO": "RECO", "write_AOD": "AOD", "write_DQM": "DQM"} for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier mergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedMergeOutput = mergeWorkflow.outputMap["Merged%s" % tier][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["Merged%s" % tier][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: mergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs %d." % len(mergeWorkflow.outputMap.keys())) mergedMergeOutput = mergeWorkflow.outputMap["MergedALCARECO"][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap["MergedALCARECO"][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual(mergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-MergedALCARECO" % goldenOutputMod, "Error: Merged output fileset is wrong.") self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-MergedALCARECO" % goldenOutputMod, "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0]["merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0]["output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name="TestWorkload-Reco-SomeBlock") topLevelFileset.loadData() recoSubscription = Subscription(fileset=topLevelFileset, workflow=recoWorkflow) recoSubscription.loadData() self.assertEqual(recoSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(recoSubscription["split_algo"], "EventAwareLumiBased", "Error: Wrong split algorithm. %s" % recoSubscription["split_algo"]) alcaRecoFileset = Fileset(name="/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO") alcaRecoFileset.loadData() alcaSkimSubscription = Subscription(fileset=alcaRecoFileset, workflow=alcaSkimWorkflow) alcaSkimSubscription.loadData() self.assertEqual(alcaSkimSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(alcaSkimSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algorithm. %s" % alcaSkimSubscription["split_algo"]) mergedDQMFileset = Fileset(name="/TestWorkload/Reco/RecoMergewrite_DQM/merged-MergedDQM") mergedDQMFileset.loadData() dqmSubscription = Subscription(fileset=mergedDQMFileset, workflow=dqmWorkflow) dqmSubscription.loadData() self.assertEqual(dqmSubscription["type"], "Harvesting", "Error: Wrong subscription type.") self.assertEqual(dqmSubscription["split_algo"], "Harvest", "Error: Wrong split algo.") unmergedOutputs = {"write_RECO": "RECO", "write_AOD": "AOD", "write_DQM": "DQM"} for unmergedOutput, tier in unmergedOutputs.items(): fset = unmergedOutput + tier unmergedDataTier = Fileset(name="/TestWorkload/Reco/unmerged-%s" % fset) unmergedDataTier.loadData() dataTierMergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMerge%s" % unmergedOutput) dataTierMergeWorkflow.load() mergeSubscription = Subscription(fileset=unmergedDataTier, workflow=dataTierMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) unmergedOutputs = [] for alcaProd in testArguments["AlcaSkims"]: unmergedOutputs.append("ALCARECOStream%s" % alcaProd) for unmergedOutput in unmergedOutputs: unmergedAlcaSkim = Fileset(name="/TestWorkload/Reco/AlcaSkim/unmerged-%sALCARECO" % unmergedOutput) unmergedAlcaSkim.loadData() alcaSkimMergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s" % unmergedOutput) alcaSkimMergeWorkflow.load() mergeSubscription = Subscription(fileset=unmergedAlcaSkim, workflow=alcaSkimMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "ParentlessMergeBySize", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) goldenOutputMods = {"write_RECO": "RECO", "write_AOD": "AOD", "write_DQM": "DQM"} for goldenOutputMod, tier in goldenOutputMods.items(): fset = goldenOutputMod + tier unmergedFileset = Fileset(name="/TestWorkload/Reco/unmerged-%s" % fset) unmergedFileset.loadData() cleanupWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset=unmergedFileset, workflow=cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: unmergedFileset = Fileset(name="/TestWorkload/Reco/AlcaSkim/unmerged-%sALCARECO" % goldenOutputMod) unmergedFileset.loadData() cleanupWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset=unmergedFileset, workflow=cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") recoLogCollect = Fileset(name="/TestWorkload/Reco/unmerged-logArchive") recoLogCollect.loadData() recoLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/LogCollect") recoLogCollectWorkflow.load() logCollectSub = Subscription(fileset=recoLogCollect, workflow=recoLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") alcaSkimLogCollect = Fileset(name="/TestWorkload/Reco/AlcaSkim/unmerged-logArchive") alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect") alcaSkimLogCollectWorkflow.load() logCollectSub = Subscription(fileset=alcaSkimLogCollect, workflow=alcaSkimLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = ["write_RECO", "write_AOD", "write_DQM"] for goldenOutputMod in goldenOutputMods: recoMergeLogCollect = Fileset(name="/TestWorkload/Reco/RecoMerge%s/merged-logArchive" % goldenOutputMod) recoMergeLogCollect.loadData() recoMergeLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMerge%s/Reco%sMergeLogCollect" % ( goldenOutputMod, goldenOutputMod)) recoMergeLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset=recoMergeLogCollect, workflow=recoMergeLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") goldenOutputMods = [] for alcaProd in testArguments["AlcaSkims"]: goldenOutputMods.append("ALCARECOStream%s" % alcaProd) for goldenOutputMod in goldenOutputMods: alcaSkimLogCollect = Fileset( name="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/merged-logArchive" % goldenOutputMod) alcaSkimLogCollect.loadData() alcaSkimLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/AlcaSkim/AlcaSkimMerge%s/AlcaSkim%sMergeLogCollect" % ( goldenOutputMod, goldenOutputMod)) alcaSkimLogCollectWorkflow.load() logCollectSubscription = Subscription(fileset=alcaSkimLogCollect, workflow=alcaSkimLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") dqmHarvestLogCollect = Fileset( name="/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive") dqmHarvestLogCollect.loadData() dqmHarvestLogCollectWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect") dqmHarvestLogCollectWorkflow.load() logCollectSub = Subscription(fileset=dqmHarvestLogCollect, workflow=dqmHarvestLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algo.") return
def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = ['/TestWorkload/Reco', '/TestWorkload/Reco/AlcaSkim', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics', '/TestWorkload/Reco/RecoMergewrite_AOD', '/TestWorkload/Reco/RecoMergewrite_DQM', '/TestWorkload/Reco/RecoMergewrite_RECO'] expWfTasks = ['/TestWorkload/Reco', '/TestWorkload/Reco/AlcaSkim', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamHcalCalHOCosmics', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamMuAlGlobalCosmics', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamTkAlCosmics0T', '/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/AlcaSkimALCARECOStreamHcalCalHOCosmicsMergeLogCollect', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/AlcaSkimALCARECOStreamMuAlGlobalCosmicsMergeLogCollect', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/AlcaSkimALCARECOStreamTkAlCosmics0TMergeLogCollect', '/TestWorkload/Reco/LogCollect', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_ALCARECO', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_AOD', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_DQM', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_RECO', '/TestWorkload/Reco/RecoMergewrite_AOD', '/TestWorkload/Reco/RecoMergewrite_AOD/Recowrite_AODMergeLogCollect', '/TestWorkload/Reco/RecoMergewrite_DQM', '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged', '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect', '/TestWorkload/Reco/RecoMergewrite_DQM/Recowrite_DQMMergeLogCollect', '/TestWorkload/Reco/RecoMergewrite_RECO', '/TestWorkload/Reco/RecoMergewrite_RECO/Recowrite_RECOMergeLogCollect'] expFsets = ['TestWorkload-Reco-/MinimumBias/ComissioningHI-v1/RAW', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/merged-MergedALCARECO', '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamHcalCalHOCosmicsALCARECO', '/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/merged-MergedALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/merged-MergedALCARECO', '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamMuAlGlobalCosmicsALCARECO', '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamTkAlCosmics0TALCARECO', '/TestWorkload/Reco/AlcaSkim/unmerged-logArchive', '/TestWorkload/Reco/RecoMergewrite_AOD/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_AOD/merged-MergedAOD', '/TestWorkload/Reco/unmerged-write_AODAOD', '/TestWorkload/Reco/unmerged-write_DQMDQM', '/TestWorkload/Reco/RecoMergewrite_DQM/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_DQM/merged-MergedDQM', '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive', '/TestWorkload/Reco/RecoMergewrite_RECO/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_RECO/merged-MergedRECO', '/TestWorkload/Reco/unmerged-logArchive', '/TestWorkload/Reco/unmerged-write_RECORECO'] subMaps = [(4, '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics/AlcaSkimALCARECOStreamHcalCalHOCosmicsMergeLogCollect', 'MinFileBased', 'LogCollect'), (10, '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics/AlcaSkimALCARECOStreamMuAlGlobalCosmicsMergeLogCollect', 'MinFileBased', 'LogCollect'), (7, '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/merged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T/AlcaSkimALCARECOStreamTkAlCosmics0TMergeLogCollect', 'MinFileBased', 'LogCollect'), (5, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamHcalCalHOCosmicsALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamHcalCalHOCosmics', 'SiblingProcessingBased', 'Cleanup'), (3, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamHcalCalHOCosmicsALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamHcalCalHOCosmics', 'ParentlessMergeBySize', 'Merge'), (11, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamMuAlGlobalCosmicsALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamMuAlGlobalCosmics', 'SiblingProcessingBased', 'Cleanup'), (9, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamMuAlGlobalCosmicsALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamMuAlGlobalCosmics', 'ParentlessMergeBySize', 'Merge'), (8, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamTkAlCosmics0TALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimCleanupUnmergedALCARECOStreamTkAlCosmics0T', 'SiblingProcessingBased', 'Cleanup'), (6, '/TestWorkload/Reco/AlcaSkim/unmerged-ALCARECOStreamTkAlCosmics0TALCARECO', '/TestWorkload/Reco/AlcaSkim/AlcaSkimMergeALCARECOStreamTkAlCosmics0T', 'ParentlessMergeBySize', 'Merge'), (12, '/TestWorkload/Reco/AlcaSkim/unmerged-logArchive', '/TestWorkload/Reco/AlcaSkim/AlcaSkimLogCollect', 'MinFileBased', 'LogCollect'), (15, '/TestWorkload/Reco/RecoMergewrite_AOD/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_AOD/Recowrite_AODMergeLogCollect', 'MinFileBased', 'LogCollect'), (20, '/TestWorkload/Reco/RecoMergewrite_DQM/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_DQM/Recowrite_DQMMergeLogCollect', 'MinFileBased', 'LogCollect'), (18, '/TestWorkload/Reco/RecoMergewrite_DQM/merged-MergedDQM', '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged', 'Harvest', 'Harvesting'), (19, '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/unmerged-logArchive', '/TestWorkload/Reco/RecoMergewrite_DQM/RecoMergewrite_DQMEndOfRunDQMHarvestMerged/RecoMergewrite_DQMMergedEndOfRunDQMHarvestLogCollect', 'MinFileBased', 'LogCollect'), (23, '/TestWorkload/Reco/RecoMergewrite_RECO/merged-logArchive', '/TestWorkload/Reco/RecoMergewrite_RECO/Recowrite_RECOMergeLogCollect', 'MinFileBased', 'LogCollect'), (25, '/TestWorkload/Reco/unmerged-logArchive', '/TestWorkload/Reco/LogCollect', 'MinFileBased', 'LogCollect'), (2, '/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO', '/TestWorkload/Reco/AlcaSkim', 'ParentlessMergeBySize', 'Processing'), (13, '/TestWorkload/Reco/unmerged-write_ALCARECOALCARECO', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_ALCARECO', 'SiblingProcessingBased', 'Cleanup'), (16, '/TestWorkload/Reco/unmerged-write_AODAOD', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_AOD', 'SiblingProcessingBased', 'Cleanup'), (14, '/TestWorkload/Reco/unmerged-write_AODAOD', '/TestWorkload/Reco/RecoMergewrite_AOD', 'ParentlessMergeBySize', 'Merge'), (21, '/TestWorkload/Reco/unmerged-write_DQMDQM', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_DQM', 'SiblingProcessingBased', 'Cleanup'), (17, '/TestWorkload/Reco/unmerged-write_DQMDQM', '/TestWorkload/Reco/RecoMergewrite_DQM', 'ParentlessMergeBySize', 'Merge'), (24, '/TestWorkload/Reco/unmerged-write_RECORECO', '/TestWorkload/Reco/RecoCleanupUnmergedwrite_RECO', 'SiblingProcessingBased', 'Cleanup'), (22, '/TestWorkload/Reco/unmerged-write_RECORECO', '/TestWorkload/Reco/RecoMergewrite_RECO', 'ParentlessMergeBySize', 'Merge'), (1, 'TestWorkload-Reco-/MinimumBias/ComissioningHI-v1/RAW', '/TestWorkload/Reco', 'EventAwareLumiBased', 'Processing')] testArguments = PromptRecoWorkloadFactory.getTestArguments() testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["CouchDBName"] = "promptreco_t" testArguments["EnableHarvesting"] = True factory = PromptRecoWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "Reco", blockName=testArguments['InputDataset'], cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)
def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = [ '/TestWorkload/Repack', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW' ] expWfTasks = [ '/TestWorkload/Repack', '/TestWorkload/Repack/LogCollect', '/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset1_RAW', '/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset2_RAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/Repackwrite_PrimaryDataset1_RAWMergeLogCollect', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/Repackwrite_PrimaryDataset2_RAWMergeLogCollect' ] expFsets = [ 'TestWorkload-Repack-StreamerFiles', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-logArchive', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-MergedRAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-MergedErrorRAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-logArchive', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-MergedRAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-MergedErrorRAW', '/TestWorkload/Repack/unmerged-write_PrimaryDataset1_RAWRAW', '/TestWorkload/Repack/unmerged-write_PrimaryDataset2_RAWRAW', '/TestWorkload/Repack/unmerged-logArchive' ] subMaps = [ (3, '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/merged-logArchive', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW/Repackwrite_PrimaryDataset1_RAWMergeLogCollect', 'MinFileBased', 'LogCollect'), (6, '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/merged-logArchive', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW/Repackwrite_PrimaryDataset2_RAWMergeLogCollect', 'MinFileBased', 'LogCollect'), (8, '/TestWorkload/Repack/unmerged-logArchive', '/TestWorkload/Repack/LogCollect', 'MinFileBased', 'LogCollect'), (4, '/TestWorkload/Repack/unmerged-write_PrimaryDataset1_RAWRAW', '/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset1_RAW', 'SiblingProcessingBased', 'Cleanup'), (2, '/TestWorkload/Repack/unmerged-write_PrimaryDataset1_RAWRAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset1_RAW', 'RepackMerge', 'Merge'), (7, '/TestWorkload/Repack/unmerged-write_PrimaryDataset2_RAWRAW', '/TestWorkload/Repack/RepackCleanupUnmergedwrite_PrimaryDataset2_RAW', 'SiblingProcessingBased', 'Cleanup'), (5, '/TestWorkload/Repack/unmerged-write_PrimaryDataset2_RAWRAW', '/TestWorkload/Repack/RepackMergewrite_PrimaryDataset2_RAW', 'RepackMerge', 'Merge'), (1, 'TestWorkload-Repack-StreamerFiles', '/TestWorkload/Repack', 'Repack', 'Repack') ] testArguments = RepackWorkloadFactory.getTestArguments() testArguments.update(deepcopy(REQUEST)) factory = RepackWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "Repack", blockName='StreamerFiles', cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)
def testRepack(self): """ _testRepack_ Create a Repack workflow and verify it installs into WMBS correctly. """ testArguments = RepackWorkloadFactory.getTestArguments() testArguments.update(deepcopy(REQUEST)) factory = RepackWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", testArguments) testWorkload.setSpecUrl("somespec") testWorkload.setOwnerDetails("*****@*****.**", "T0") testWMBSHelper = WMBSHelper(testWorkload, "Repack", cachepath=self.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) repackWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/Repack") repackWorkflow.load() self.assertEqual( len(repackWorkflow.outputMap), len(testArguments["Outputs"]) + 1, "Error: Wrong number of WF outputs in the Repack WF.") goldenOutputMods = { "write_PrimaryDataset1_RAW": "RAW", "write_PrimaryDataset2_RAW": "RAW" } for goldenOutputMod, tier in viewitems(goldenOutputMods): fset = goldenOutputMod + tier mergedOutput = repackWorkflow.outputMap[fset][0][ "merged_output_fileset"] unmergedOutput = repackWorkflow.outputMap[fset][0][ "output_fileset"] mergedOutput.loadData() unmergedOutput.loadData() if goldenOutputMod != "write_PrimaryDataset1_RAW": self.assertEqual( mergedOutput.name, "/TestWorkload/Repack/RepackMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Merged output fileset is wrong: %s" % mergedOutput.name) self.assertEqual( unmergedOutput.name, "/TestWorkload/Repack/unmerged-%s" % fset, "Error: Unmerged output fileset is wrong: %s" % unmergedOutput.name) logArchOutput = repackWorkflow.outputMap["logArchive"][0][ "merged_output_fileset"] unmergedLogArchOutput = repackWorkflow.outputMap["logArchive"][0][ "output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual(logArchOutput.name, "/TestWorkload/Repack/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Repack/unmerged-logArchive", "Error: LogArchive output fileset is wrong.") for goldenOutputMod, tier in viewitems(goldenOutputMods): mergeWorkflow = Workflow( name="TestWorkload", task="/TestWorkload/Repack/RepackMerge%s" % goldenOutputMod) mergeWorkflow.load() self.assertEqual(len(mergeWorkflow.outputMap), 3, "Error: Wrong number of WF outputs.") mergedMergeOutput = mergeWorkflow.outputMap[ "Merged%s" % tier][0]["merged_output_fileset"] unmergedMergeOutput = mergeWorkflow.outputMap[ "Merged%s" % tier][0]["output_fileset"] mergedMergeOutput.loadData() unmergedMergeOutput.loadData() self.assertEqual( mergedMergeOutput.name, "/TestWorkload/Repack/RepackMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Merged output fileset is wrong.") self.assertEqual( unmergedMergeOutput.name, "/TestWorkload/Repack/RepackMerge%s/merged-Merged%s" % (goldenOutputMod, tier), "Error: Unmerged output fileset is wrong.") logArchOutput = mergeWorkflow.outputMap["logArchive"][0][ "merged_output_fileset"] unmergedLogArchOutput = mergeWorkflow.outputMap["logArchive"][0][ "output_fileset"] logArchOutput.loadData() unmergedLogArchOutput.loadData() self.assertEqual( logArchOutput.name, "/TestWorkload/Repack/RepackMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong: %s" % logArchOutput.name) self.assertEqual( unmergedLogArchOutput.name, "/TestWorkload/Repack/RepackMerge%s/merged-logArchive" % goldenOutputMod, "Error: LogArchive output fileset is wrong.") topLevelFileset = Fileset(name="TestWorkload-Repack") topLevelFileset.loadData() repackSubscription = Subscription(fileset=topLevelFileset, workflow=repackWorkflow) repackSubscription.loadData() self.assertEqual(repackSubscription["type"], "Repack", "Error: Wrong subscription type.") self.assertEqual( repackSubscription["split_algo"], "Repack", "Error: Wrong split algorithm. %s" % repackSubscription["split_algo"]) unmergedOutputs = { "write_PrimaryDataset1_RAW": "RAW", "write_PrimaryDataset2_RAW": "RAW" } for unmergedOutput, tier in viewitems(unmergedOutputs): fset = unmergedOutput + tier unmergedDataTier = Fileset( name="/TestWorkload/Repack/unmerged-%s" % fset) unmergedDataTier.loadData() dataTierMergeWorkflow = Workflow( name="TestWorkload", task="/TestWorkload/Repack/RepackMerge%s" % unmergedOutput) dataTierMergeWorkflow.load() mergeSubscription = Subscription(fileset=unmergedDataTier, workflow=dataTierMergeWorkflow) mergeSubscription.loadData() self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual( mergeSubscription["split_algo"], "RepackMerge", "Error: Wrong split algorithm. %s" % mergeSubscription["split_algo"]) for goldenOutputMod, tier in viewitems(goldenOutputMods): fset = goldenOutputMod + tier unmergedFileset = Fileset(name="/TestWorkload/Repack/unmerged-%s" % fset) unmergedFileset.loadData() cleanupWorkflow = Workflow( name="TestWorkload", task="/TestWorkload/Repack/RepackCleanupUnmerged%s" % goldenOutputMod) cleanupWorkflow.load() cleanupSubscription = Subscription(fileset=unmergedFileset, workflow=cleanupWorkflow) cleanupSubscription.loadData() self.assertEqual(cleanupSubscription["type"], "Cleanup", "Error: Wrong subscription type.") self.assertEqual(cleanupSubscription["split_algo"], "SiblingProcessingBased", "Error: Wrong subscription type.") repackLogCollect = Fileset( name="/TestWorkload/Repack/unmerged-logArchive") repackLogCollect.loadData() repackLogCollectWorkflow = Workflow( name="TestWorkload", task="/TestWorkload/Repack/LogCollect") repackLogCollectWorkflow.load() logCollectSub = Subscription(fileset=repackLogCollect, workflow=repackLogCollectWorkflow) logCollectSub.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") for goldenOutputMod, tier in viewitems(goldenOutputMods): repackMergeLogCollect = Fileset( name="/TestWorkload/Repack/RepackMerge%s/merged-logArchive" % goldenOutputMod) repackMergeLogCollect.loadData() repackMergeLogCollectWorkflow = Workflow( name="TestWorkload", task= "/TestWorkload/Repack/RepackMerge%s/Repack%sMergeLogCollect" % (goldenOutputMod, goldenOutputMod)) repackMergeLogCollectWorkflow.load() logCollectSubscription = Subscription( fileset=repackMergeLogCollect, workflow=repackMergeLogCollectWorkflow) logCollectSubscription.loadData() self.assertEqual(logCollectSub["type"], "LogCollect", "Error: Wrong subscription type.") self.assertEqual(logCollectSub["split_algo"], "MinFileBased", "Error: Wrong split algorithm.") return
def testStepChainMC(self): """ Build a StepChain workload starting from scratch """ # Read in the request request = json.load(open(self.jsonTemplate)) testArguments = request['createRequest'] testArguments.update({ "CouchURL": os.environ["COUCHURL"], "ConfigCacheUrl": os.environ["COUCHURL"], "CouchDBName": "stepchain_t" }) configDocs = injectStepChainConfigMC(self.configDatabase) for s in ['Step1', 'Step2', 'Step3']: testArguments[s]['ConfigCacheID'] = configDocs[s] factory = StepChainWorkloadFactory() # test that we cannot stage out different samples with the same output module self.assertRaises(WMSpecFactoryException, factory.factoryWorkloadConstruction, "TestWorkload", testArguments) testArguments['Step2']['KeepOutput'] = False testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "ProdMinBias", "MCFakeBlock", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) # test workload properties self.assertEqual(testWorkload.getDashboardActivity(), "production") self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME") self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_0_0_pre11") self.assertEqual(testWorkload.getProcessingString(), "START70_V4") self.assertEqual(testWorkload.getProcessingVersion(), 1) self.assertEqual(testWorkload.getPrepID(), "Step-00") self.assertEqual(sorted(testWorkload.getCMSSWVersions()), ['CMSSW_7_0_0_pre11', 'CMSSW_7_0_0_pre12']) # test workload attributes self.assertEqual(testWorkload.processingString, "START70_V4") self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_0_0_pre11") self.assertEqual(testWorkload.processingVersion, 1) self.assertFalse(testWorkload.lumiList, "Wrong lumiList") self.assertEqual(testWorkload.data.policies.start.policyName, "MonteCarlo") # test workload tasks and steps tasks = testWorkload.listAllTaskNames() self.assertEqual(len(tasks), 10) for t in [ 'ProdMinBias', 'ProdMinBiasMergeRAWSIMoutput', 'RECOPROD1MergeAODSIMoutput', 'RECOPROD1MergeRECOSIMoutput' ]: self.assertTrue(t in tasks, "Wrong task name") self.assertFalse('ProdMinBiasMergeAODSIMoutput' in tasks, "Wrong task name") task = testWorkload.getTask(tasks[0]) self.assertEqual(task.name(), "ProdMinBias") self.assertEqual(task.getPathName(), "/TestWorkload/ProdMinBias") self.assertEqual(task.taskType(), "Production", "Wrong task type") splitParams = task.jobSplittingParameters() self.assertEqual(splitParams['algorithm'], "EventBased", "Wrong job splitting algo") self.assertEqual(splitParams['events_per_job'], 150) self.assertEqual(splitParams['events_per_lumi'], 50) self.assertFalse(splitParams['lheInputFiles'], "Wrong LHE flag") self.assertTrue(splitParams['performance']['timePerEvent'] > 4.75) self.assertTrue(splitParams['performance']['sizePerEvent'] > 1233) self.assertTrue( splitParams['performance']['memoryRequirement'] == 2400) self.assertFalse(task.getTrustSitelists().get('trustlists'), "Wrong input location flag") self.assertFalse(task.inputRunWhitelist(), "Wrong run white list") # test workload step stuff self.assertEqual( sorted(task.listAllStepNames()), ['cmsRun1', 'cmsRun2', 'cmsRun3', 'logArch1', 'stageOut1']) self.assertEqual(task.getTopStepName(), 'cmsRun1') self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW") self.assertFalse(task.getInputStep(), "Wrong input step") outModsAndDsets = task.listOutputDatasetsAndModules() outMods = set([elem['outputModule'] for elem in outModsAndDsets]) outDsets = [elem['outputDataset'] for elem in outModsAndDsets] self.assertEqual( outMods, set(['RAWSIMoutput', 'AODSIMoutput', 'RECOSIMoutput']), "Wrong output modules") self.assertTrue( '/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterA-START70_V4-v1/GEN-SIM' in outDsets) self.assertTrue( '/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterD-START70_V4-v1/AODSIM' in outDsets) self.assertTrue( '/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterC-START70_V4-v1/GEN-SIM-RECO' in outDsets) self.assertEqual(task.getSwVersion(), 'CMSSW_7_0_0_pre12') self.assertEqual(task.getScramArch(), 'slc5_amd64_gcc481') step = task.getStep("cmsRun1") self.assertFalse(step.data.tree.parent) self.assertFalse(getattr(step.data.input, 'inputStepName', None)) self.assertFalse(getattr(step.data.input, 'inputOutputModule', None)) self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName, 'FilterA') self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier, 'GEN-SIM') self.assertTrue(step.data.output.keep) self.assertEqual(sorted(step.data.tree.childNames), ['cmsRun2', 'logArch1', 'stageOut1']) self.assertEqual(step.data.application.setup.cmsswVersion, 'CMSSW_7_0_0_pre12') self.assertEqual(step.data.application.setup.scramArch, 'slc5_amd64_gcc481') self.assertEqual( step.data.application.configuration.arguments.globalTag, 'START70_V4::All') step = task.getStep("cmsRun2") self.assertEqual(step.data.tree.parent, "cmsRun1") self.assertEqual(step.data.input.inputStepName, 'cmsRun1') self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput') self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName, 'FilterB') self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier, 'GEN-SIM-RAW') self.assertFalse(step.data.output.keep) self.assertEqual(step.data.tree.childNames, ["cmsRun3"]) self.assertEqual(step.data.application.setup.cmsswVersion, 'CMSSW_7_0_0_pre11') self.assertEqual(step.data.application.setup.scramArch, 'slc5_amd64_gcc481') self.assertEqual( step.data.application.configuration.arguments.globalTag, 'START70_V4::All') step = task.getStep("cmsRun3") self.assertEqual(step.data.tree.parent, "cmsRun2") self.assertEqual(step.data.input.inputStepName, 'cmsRun2') self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput') self.assertEqual(step.data.output.modules.RECOSIMoutput.filterName, 'FilterC') self.assertEqual(step.data.output.modules.AODSIMoutput.filterName, 'FilterD') self.assertEqual(step.data.output.modules.RECOSIMoutput.dataTier, 'GEN-SIM-RECO') self.assertEqual(step.data.output.modules.AODSIMoutput.dataTier, 'AODSIM') self.assertTrue(step.data.output.keep) self.assertFalse(step.data.tree.childNames) self.assertEqual(step.data.application.setup.cmsswVersion, 'CMSSW_7_0_0_pre11') self.assertEqual(step.data.application.setup.scramArch, 'slc5_amd64_gcc481') self.assertEqual( step.data.application.configuration.arguments.globalTag, 'START70_V4::All') return
def testStepChainSingleStep(self): """ Build a StepChain single step, reading AODSIM and producing MINIAODSIM """ testArguments = StepChainWorkloadFactory.getTestArguments() request = { "Campaign": "TaskForceUnitTest", "CMSSWVersion": "CMSSW_7_5_0", "ScramArch": "slc6_amd64_gcc491", "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader", "GlobalTag": "PHYS14_25_V3", "AcquisitionEra": "SingleStep", "ProcessingString": "UnitTest_StepChain", "ProcessingVersion": 3, "PrepID": "MainStep", "CouchURL": os.environ["COUCHURL"], "CouchDBName": "stepchain_t", "Memory": 3500, "SizePerEvent": 2600, "TimePerEvent": 26.5, "Step1": { "ConfigCacheID": injectStepChainConfigSingle(self.configDatabase), "GlobalTag": "PHYS14_25_V44", "InputDataset": "/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/RunIISpring15DR74-Asympt50ns_MCRUN2_74_V9A-v1/AODSIM", "SplittingAlgo": "EventAwareLumiBased", "EventsPerJob": 500, "StepName": "StepMini" }, "StepChain": 1 } testArguments.update(request) factory = StepChainWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction( "TestWorkload", testArguments) testWMBSHelper = WMBSHelper(testWorkload, "StepMini", "GravWhatever", cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) # test workload properties self.assertEqual(testWorkload.getDashboardActivity(), "processing") self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest") self.assertEqual(testWorkload.getAcquisitionEra(), "SingleStep") self.assertEqual(testWorkload.getProcessingString(), "UnitTest_StepChain") self.assertEqual(testWorkload.getProcessingVersion(), 3) self.assertEqual(testWorkload.getPrepID(), "MainStep") self.assertEqual(sorted(testWorkload.getCMSSWVersions()), ['CMSSW_7_5_0']) self.assertEqual(testWorkload.data.policies.start.policyName, "Block") # test workload tasks and steps tasks = testWorkload.listAllTaskNames() self.assertEqual(len(tasks), 4) self.assertTrue('StepMiniMergeMINIAODSIMoutput' in tasks) task = testWorkload.getTask(tasks[0]) self.assertEqual(task.taskType(), "Processing", "Wrong task type") splitParams = task.jobSplittingParameters() self.assertEqual(splitParams['algorithm'], "EventAwareLumiBased", "Wrong job splitting algo") self.assertEqual(splitParams['events_per_job'], 500) self.assertTrue(splitParams['performance']['timePerEvent'] > 26.4) self.assertTrue(splitParams['performance']['sizePerEvent'] > 2599) self.assertTrue( splitParams['performance']['memoryRequirement'] == 3500) # test workload step stuff self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'stageOut1']) self.assertEqual(task.getTopStepName(), 'cmsRun1') self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW") self.assertFalse(task.getInputStep(), "Wrong input step") outModsAndDsets = task.listOutputDatasetsAndModules()[0] self.assertEqual(outModsAndDsets['outputModule'], 'MINIAODSIMoutput') self.assertEqual( outModsAndDsets['outputDataset'], '/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/SingleStep-UnitTest_StepChain-v3/MINIAODSIM' ) self.assertEqual(task.getSwVersion(), 'CMSSW_7_5_0') self.assertEqual(task.getScramArch(), 'slc6_amd64_gcc491') step = task.getStep("cmsRun1") self.assertEqual( step.data.application.configuration.arguments.globalTag, 'PHYS14_25_V44') return
def testFilesets(self): """ Test workflow tasks, filesets and subscriptions creation """ # expected tasks, filesets, subscriptions, etc expOutTasks = ['/TestWorkload/Production', '/TestWorkload/Production/ProductionMergeOutputB', '/TestWorkload/Production/ProductionMergeOutputA'] expWfTasks = ['/TestWorkload/Production', '/TestWorkload/Production/LogCollect', '/TestWorkload/Production/ProductionCleanupUnmergedOutputA', '/TestWorkload/Production/ProductionCleanupUnmergedOutputB', '/TestWorkload/Production/ProductionMergeOutputA', '/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect', '/TestWorkload/Production/ProductionMergeOutputB', '/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect'] expFsets = ['FILESET_DEFINED_DURING_RUNTIME', '/TestWorkload/Production/unmerged-OutputBUSER', '/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputA/merged-MergedRECO', '/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputB/merged-MergedUSER', '/TestWorkload/Production/unmerged-logArchive', '/TestWorkload/Production/unmerged-OutputARECO'] subMaps = ['FILESET_DEFINED_DURING_RUNTIME', (6, '/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect', 'MinFileBased', 'LogCollect'), (3, '/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive', '/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect', 'MinFileBased', 'LogCollect'), (8, '/TestWorkload/Production/unmerged-logArchive', '/TestWorkload/Production/LogCollect', 'MinFileBased', 'LogCollect'), (7, '/TestWorkload/Production/unmerged-OutputARECO', '/TestWorkload/Production/ProductionCleanupUnmergedOutputA', 'SiblingProcessingBased', 'Cleanup'), (5, '/TestWorkload/Production/unmerged-OutputARECO', '/TestWorkload/Production/ProductionMergeOutputA', 'ParentlessMergeBySize', 'Merge'), (4, '/TestWorkload/Production/unmerged-OutputBUSER', '/TestWorkload/Production/ProductionCleanupUnmergedOutputB', 'SiblingProcessingBased', 'Cleanup'), (2, '/TestWorkload/Production/unmerged-OutputBUSER', '/TestWorkload/Production/ProductionMergeOutputB', 'ParentlessMergeBySize', 'Merge')] testArguments = MonteCarloWorkloadFactory.getTestArguments() testArguments["CouchURL"] = os.environ["COUCHURL"] testArguments["CouchDBName"] = TEST_DB_NAME testArguments["ConfigCacheID"] = self.injectMonteCarloConfig() factory = MonteCarloWorkloadFactory() testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments) myMask = Mask(FirstRun=1, FirstLumi=1, FirstEvent=1, LastRun=1, LastLumi=10, LastEvent=1000) testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask, cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # same function as in WMBSHelper, otherwise we cannot know which fileset name is maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)]) topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest() expFsets[0] = topFilesetName # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subMaps[0] = (1, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production') subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps) ### create another top level subscription myMask = Mask(FirstRun=1, FirstLumi=11, FirstEvent=1001, LastRun=1, LastLumi=20, LastEvent=2000) testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask, cachepath=self.testInit.testDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset) workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload") self.assertItemsEqual([item['task'] for item in workflows], expWfTasks) # same function as in WMBSHelper, otherwise we cannot know which fileset name is maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)]) topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest() expFsets.append(topFilesetName) # returns a tuple of id, name, open and last_update filesets = self.listFilesets.execute() self.assertItemsEqual([item[1] for item in filesets], expFsets) subMaps.append((9, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production')) subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True) self.assertItemsEqual(subscriptions, subMaps)