def testInvalidSpecs(self): """Specs with no work""" # no dataset rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec).data.input.dataset = None for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueWMSpecError, Dataset(), processingSpec, task) # invalid dbs url processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com' for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueWMSpecError, Dataset(), processingSpec, task) # invalid dataset name processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec).data.input.dataset.primary = Globals.NOT_EXIST_DATASET for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Dataset(), processingSpec, task) # invalid run whitelist processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) processingSpec.setRunWhitelist([666]) # not in this dataset for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Dataset(), processingSpec, task)
def testInvalidSpecs(self): """Specs with no work""" dqmHarvArgs = getRequestArgs() # no dataset dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"]) factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction('NoInputDatasetTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) getFirstTask(DQMHarvWorkload).data.input.dataset = None for task in DQMHarvWorkload.taskIterator(): self.assertRaises(WorkQueueWMSpecError, Dataset(), DQMHarvWorkload, task) # invalid dataset name DQMHarvWorkload = factory.factoryWorkloadConstruction('InvalidInputDatasetTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) getFirstTask(DQMHarvWorkload).data.input.dataset.name = '/MinimumBias/FAKE-Filter-v1/RECO' for task in DQMHarvWorkload.taskIterator(): self.assertRaises(DBSReaderError, Dataset(), DQMHarvWorkload, task) # invalid run whitelist DQMHarvWorkload = factory.factoryWorkloadConstruction('InvalidRunNumberTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) DQMHarvWorkload.setRunWhitelist([666]) # not in this dataset for task in DQMHarvWorkload.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Dataset(), DQMHarvWorkload, task)
def testDatasetSingleJob(self): """ Test NumberOfRuns splitting type with very large SliceSize """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig( dqmHarvArgs["CouchDBName"]) dqmHarvArgs["DQMHarvestUnit"] = 'multiRun' factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction( 'DQMHarvestTest', dqmHarvArgs) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(1, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(4855, unit['NumberOfLumis']) self.assertEqual(72, unit['NumberOfFiles']) self.assertEqual(743201, unit['NumberOfEvents'])
def testInvalidSpecs(self): """Specs with no work""" # no dataset rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec).data.input.dataset = None for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueWMSpecError, Block(), processingSpec, task) # invalid dbs url processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com' for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueWMSpecError, Block(), processingSpec, task) # invalid dataset name processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec).data.input.dataset.primary = Globals.NOT_EXIST_DATASET for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec, task) # invalid run whitelist processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) processingSpec.setRunWhitelist([666]) # not in this dataset for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec, task) # blocks with 0 files are skipped # set all blocks in request to 0 files, no work should be found & an error is raised Globals.GlobalParams.setNumOfFilesPerBlock(0) processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec, task) Globals.GlobalParams.resetParams()
def notestParentProcessing(self): # Does not work with a RAW dataset, need a different workload """ test parent processing: should have the same results as rereco test with the parent flag and dataset. """ parentProcArgs["ConfigCacheID"] = createConfig( parentProcArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() parentProcSpec = factory.factoryWorkloadConstruction( 'testParentProcessing', parentProcArgs) inputDataset = getFirstTask(parentProcSpec).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in parentProcSpec.taskIterator(): units, _ = Block(**self.splitArgs)(parentProcSpec, task) self.assertEqual(47, len(units)) blocks = [] # fill with blocks as we get work units for them for unit in units: import pdb pdb.set_trace() self.assertTrue(1 <= unit['Jobs']) self.assertEqual(parentProcSpec, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(True, unit['ParentFlag']) self.assertEqual(1, len(unit['ParentData'])) self.assertEqual( len(units), len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
def notestParentProcessing(self): # Does not work with a RAW dataset, need a different workload """ test parent processing: should have the same results as rereco test with the parent flag and dataset. """ parentProcArgs["ConfigCacheID"] = createConfig(parentProcArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() parentProcSpec = factory.factoryWorkloadConstruction('testParentProcessing', parentProcArgs) inputDataset = getFirstTask(parentProcSpec).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in parentProcSpec.taskIterator(): units, _ = Block(**self.splitArgs)(parentProcSpec, task) self.assertEqual(47, len(units)) for unit in units: import pdb pdb.set_trace() self.assertTrue(1 <= unit['Jobs']) self.assertEqual(parentProcSpec, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(True, unit['ParentFlag']) self.assertEqual(1, len(unit['ParentData'])) self.assertEqual(len(units), len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
def testParentProcessing(self): """ test parent processing: should have the same results as rereco test with the parent flag and dataset. """ splitArgs = dict(SliceType='NumberOfLumis', SliceSize=2) parentProcArgs = getReRecoArgs(parent=True) parentProcArgs["ConfigCacheID"] = createConfig(parentProcArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() # This dataset does have parents. Adding it here to keep the test going. # It seems like "dbs" below is never used parentProcArgs2 = {} parentProcArgs2.update(parentProcArgs) parentProcArgs2.update({'InputDataset': '/SingleMu/CMSSW_6_2_0_pre4-PRE_61_V1_RelVal_mu2012A-v1/RECO'}) parentProcSpec = factory.factoryWorkloadConstruction('testParentProcessing', parentProcArgs2) parentProcSpec.setStartPolicy('Dataset', **splitArgs) inputDataset = getFirstTask(parentProcSpec).getInputDatasetPath() for task in parentProcSpec.taskIterator(): units, _ = Dataset(**splitArgs)(parentProcSpec, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(847, unit['Jobs']) self.assertEqual(1694, unit['NumberOfLumis']) self.assertEqual(parentProcSpec, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(True, unit['ParentFlag']) self.assertEqual(0, len(unit['ParentData']))
def testWithMaskedBlocks(self): """ _testWithMaskedBlocks_ Test job splitting with masked blocks """ Globals.GlobalParams.setNumOfRunsPerFile(3) Globals.GlobalParams.setNumOfLumisPerBlock(5) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 task = getFirstTask(Tier1ReRecoWorkload) inputDataset = task.inputDataset() task.data.input.splitting.runs = [181061, 180899] task.data.input.splitting.lumis = ['1,50,60,70', '1,1'] lumiMask = LumiList(compactList = {'206371': [[1, 50], [60,70]], '180899':[[1,1]], } ) dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)} units, rejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) nLumis = 0 for unit in units: nLumis += unit['NumberOfLumis'] self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testTier1ReRecoWorkload(self): """Tier1 Re-reco workflow""" rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): units, _ = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) self.assertEqual(47, len(units)) blocks = [] # fill with blocks as we get work units for them for unit in units: self.assertEqual(69, unit['Priority']) self.assertTrue(1 <= unit['Jobs']) self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertTrue(1 <= unit['NumberOfLumis']) self.assertTrue(1 <= unit['NumberOfFiles']) self.assertTrue(0 <= unit['NumberOfEvents']) self.assertEqual(len(units), len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
def testParentProcessing(self): """ test parent processing: should have the same results as rereco test with the parent flag and dataset. """ splitArgs = dict(SliceType='NumberOfLumis', SliceSize=2) parentProcArgs = getReRecoArgs(parent=True) parentProcArgs["ConfigCacheID"] = createConfig(parentProcArgs["CouchDBName"]) # This dataset does have parents. Adding it here to keep the test going. # It seems like "dbs" below is never used parentProcArgs2 = {} parentProcArgs2.update(parentProcArgs) #parentProcArgs2.update({'InputDataset': '/SingleMu/CMSSW_6_2_0_pre4-PRE_61_V1_RelVal_mu2012A-v1/RECO'}) parentProcArgs2.update({'InputDataset': '/Cosmics/ComissioningHI-PromptReco-v1/RECO'}) from pprint import pprint pprint(parentProcArgs2) parentProcSpec = rerecoWorkload('ReRecoWorkload', parentProcArgs2, assignArgs={'SiteWhitelist': ['T2_XX_SiteA']}) parentProcSpec.setStartPolicy('Dataset', **splitArgs) inputDataset = getFirstTask(parentProcSpec).getInputDatasetPath() for task in parentProcSpec.taskIterator(): units, _, _ = Dataset(**splitArgs)(parentProcSpec, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(3993, unit['Jobs']) self.assertEqual(7985, unit['NumberOfLumis']) self.assertEqual(parentProcSpec, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(True, unit['ParentFlag']) self.assertEqual(0, len(unit['ParentData']))
def testGetMaskedBlocks(self): """ _testGetMaskedBlocks_ Check that getMaskedBlocks is returning the correct information """ rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 task = getFirstTask(Tier1ReRecoWorkload) inputDataset = task.inputDataset() inputDataset.primary = 'SingleElectron' inputDataset.processed = 'StoreResults-Run2011A-WElectron-PromptSkim-v4-ALCARECO-NOLC-36cfce5a1d3f3ab4df5bd2aa0a4fa380' inputDataset.tier = 'USER' task.data.input.splitting.runs = [166921, 166429, 166911] task.data.input.splitting.lumis = ['40,70', '1,50', '1,5,16,20'] lumiMask = LumiList(compactList={'166921': [[40, 70]], '166429': [[1, 50]], '166911': [[1, 5], [16, 20]], }) inputLumis = LumiList(compactList={'166921': [[1, 67]], '166429': [[1, 91]], '166911': [[1, 104]], }) dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = DBSReader(inputDataset.dbsurl) maskedBlocks = Block(**self.splitArgs).getMaskedBlocks(task, dbs, dataset) for dummyBlock, files in maskedBlocks.iteritems(): for dummyFile, lumiList in files.iteritems(): self.assertEqual(str(lumiList), str(inputLumis & lumiMask))
def testTier1ReRecoWorkload(self): """Tier1 Re-reco workflow""" rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction( 'ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): units, _ = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) self.assertEqual(47, len(units)) blocks = [] # fill with blocks as we get work units for them for unit in units: self.assertEqual(69, unit['Priority']) self.assertTrue(1 <= unit['Jobs']) self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertTrue(1 <= unit['NumberOfLumis']) self.assertTrue(1 <= unit['NumberOfFiles']) self.assertTrue(0 <= unit['NumberOfEvents']) self.assertEqual( len(units), len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
def testParentProcessing(self): """ test parent processing: should have the same results as rereco test with the parent flag and dataset. """ splitArgs = dict(SliceType='NumberOfLumis', SliceSize=2) parentProcArgs = getReRecoArgs(parent=True) parentProcArgs["ConfigCacheID"] = createConfig( parentProcArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() # This dataset does have parents. Adding it here to keep the test going. # It seems like "dbs" below is never used parentProcArgs2 = {} parentProcArgs2.update(parentProcArgs) parentProcArgs2.update({ 'InputDataset': '/SingleMu/CMSSW_6_2_0_pre4-PRE_61_V1_RelVal_mu2012A-v1/RECO' }) parentProcSpec = factory.factoryWorkloadConstruction( 'testParentProcessing', parentProcArgs2) parentProcSpec.setStartPolicy('Dataset', **splitArgs) inputDataset = getFirstTask(parentProcSpec).getInputDatasetPath() for task in parentProcSpec.taskIterator(): units, _ = Dataset(**splitArgs)(parentProcSpec, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(847, unit['Jobs']) self.assertEqual(1694, unit['NumberOfLumis']) self.assertEqual(parentProcSpec, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(True, unit['ParentFlag']) self.assertEqual(0, len(unit['ParentData']))
def getMergeACDCSpec(self, splittingAlgo='ParentlessMergeBySize', splittingArgs=None): """ _getMergeACDCSpec_ Get a ACDC spec for the merge task of a ReReco workload """ if splittingArgs is None: splittingArgs = {} factory = ReRecoWorkloadFactory() rerecoArgs = ReRecoWorkloadFactory.getTestArguments() rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) rerecoArgs["Requestor"] = self.user rerecoArgs["Group"] = self.group Tier1ReRecoWorkload = factory.factoryWorkloadConstruction( self.workflowName, rerecoArgs) Tier1ReRecoWorkload.truncate( 'ACDC_%s' % self.workflowName, '/%s/DataProcessing/DataProcessingMergeRECOoutput' % self.workflowName, self.couchUrl, self.acdcDBName) Tier1ReRecoWorkload.setJobSplittingParameters( '/ACDC_%s/DataProcessingMergeRECOoutput' % self.workflowName, splittingAlgo, splittingArgs) return Tier1ReRecoWorkload
def testRunWhitelist(self): """ReReco lumi split with Run whitelist""" # get files with multiple runs Globals.GlobalParams.setNumOfRunsPerFile(2) # a large number of lumis to ensure we get multiple runs Globals.GlobalParams.setNumOfLumisPerBlock(10) splitArgs = dict(SliceType = 'NumberOfLumis', SliceSize = 1) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setRunWhitelist([2, 3]) Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task) self.assertEqual(1, len(units)) # Check number of jobs in element match number for # dataset in run whitelist jobs = 0 wq_jobs = 0 for unit in units: wq_jobs += unit['Jobs'] runLumis = dbs[inputDataset.dbsurl].listRunLumis(dataset = unit['Inputs'].keys()[0]) for run in runLumis: if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist(): jobs += runLumis[run] self.assertEqual(int(jobs / splitArgs['SliceSize'] ) , int(wq_jobs))
def testParentProcessing(self): """ test parent processing: should have the same results as rereco test with the parent flag and dataset. """ parentProcArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() # This dataset does have parents. Adding it here to keep the test going. It seems like "dbs" below is never used parentProcArgs2 = {} parentProcArgs2.update(parentProcArgs) parentProcArgs2.update({'InputDataset': '/SingleMu/CMSSW_6_2_0_pre4-PRE_61_V1_RelVal_mu2012A-v1/RECO'}) parentProcSpec = factory.factoryWorkloadConstruction('testParentProcessing', parentProcArgs2) parentProcSpec.setStartPolicy('Dataset', **self.splitArgs) inputDataset = getFirstTask(parentProcSpec).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dummyDBS = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in parentProcSpec.taskIterator(): units, _ = Dataset(**self.splitArgs)(parentProcSpec, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(64, unit['Jobs']) self.assertEqual(parentProcSpec, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [dataset]) self.assertEqual(True, unit['ParentFlag']) self.assertEqual(0, len(unit['ParentData']))
def testDatasetPolicy(self): """ Test ordinary NumberOfRuns splitting """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig( dqmHarvArgs["CouchDBName"]) factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction( 'DQMHarvestTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist( ["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(47, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(list(unit['Inputs']), [inputDataset]) self.assertEqual(4855, unit['NumberOfLumis']) self.assertEqual(72, unit['NumberOfFiles']) self.assertEqual(743201, unit['NumberOfEvents'])
def testDatasetLumiMask(self): """ Test NumberOfRuns splitting type with lumi mask """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig( dqmHarvArgs["CouchDBName"]) dqmHarvArgs["LumiList"] = { "181358": [[71, 80], [95, 110]], "181151": [[1, 20]] } factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction( 'DQMHarvestTest', dqmHarvArgs) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(2, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(46, unit['NumberOfLumis']) self.assertEqual(4, unit['NumberOfFiles']) self.assertEqual(12342, unit['NumberOfEvents'])
def testRunWhitelist(self): """ ReReco lumi split with Run whitelist This test may not do much of anything anymore since listRunLumis is not in DBS3 """ # get files with multiple runs Globals.GlobalParams.setNumOfRunsPerFile(2) # a large number of lumis to ensure we get multiple runs Globals.GlobalParams.setNumOfLumisPerBlock(10) splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setRunWhitelist([2, 3]) Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task) self.assertEqual(1, len(units)) # Check number of jobs in element match number for # dataset in run whitelist wq_jobs = 0 for unit in units: wq_jobs += unit['Jobs'] runLumis = dbs[inputDataset.dbsurl].listRunLumis(dataset=unit['Inputs'].keys()[0]) print "runLumis", runLumis for run in runLumis: if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist(): self.assertEqual(runLumis[run], None) # This is what it is with DBS3 unless we calculate it self.assertEqual(75, int(wq_jobs))
def testRunWhitelist(self): """ReReco lumi split with Run whitelist""" # get files with multiple runs Globals.GlobalParams.setNumOfRunsPerFile(8) # a large number of lumis to ensure we get multiple runs Globals.GlobalParams.setNumOfLumisPerBlock(20) splitArgs = dict(SliceType = 'NumberOfLumis', SliceSize = 1) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setStartPolicy('Block', **splitArgs) Tier1ReRecoWorkload.setRunWhitelist([180899, 180992]) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): units, rejectedWork = Block(**splitArgs)(Tier1ReRecoWorkload, task) # Blocks 1 and 2 match run distribution self.assertEqual(2, len(units)) self.assertEqual(len(rejectedWork), 45) # Check number of jobs in element match number for # dataset in run whitelist jobs = 0 wq_jobs = 0 for unit in units: wq_jobs += unit['Jobs'] # This fails. listRunLumis does not work correctly with DBS3, returning None for the # of lumis in a run runLumis = dbs[inputDataset.dbsurl].listRunLumis(block = unit['Inputs'].keys()[0]) for run in runLumis: if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist(): jobs += runLumis[run] self.assertEqual(int(jobs / splitArgs['SliceSize'] ) , int(wq_jobs))
def testParentProcessing(self): """ test parent processing: should have the same results as rereco test with the parent flag and dataset. """ parentProcArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() # This dataset does have parents. Adding it here to keep the test going. It seems like "dbs" below is never used parentProcArgs2 = {} parentProcArgs2.update(parentProcArgs) parentProcArgs2.update({'InputDataset': '/SingleMu/CMSSW_6_2_0_pre4-PRE_61_V1_RelVal_mu2012A-v1/RECO'}) parentProcSpec = factory.factoryWorkloadConstruction('testParentProcessing', parentProcArgs2) parentProcSpec.setStartPolicy('Dataset', **self.splitArgs) inputDataset = getFirstTask(parentProcSpec).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)} for task in parentProcSpec.taskIterator(): units, _ = Dataset(**self.splitArgs)(parentProcSpec, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(4, unit['Jobs']) self.assertEqual(parentProcSpec, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [dataset]) self.assertEqual(True, unit['ParentFlag']) self.assertEqual(0, len(unit['ParentData']))
def testRunWhitelist(self): """ ReReco lumi split with Run whitelist This test may not do much of anything anymore since listRunLumis is not in DBS3 """ splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setStartPolicy('Block', **splitArgs) Tier1ReRecoWorkload.setRunWhitelist([180899, 180992]) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): units, rejectedWork = Block(**splitArgs)(Tier1ReRecoWorkload, task) # Blocks 1 and 2 match run distribution self.assertEqual(2, len(units)) self.assertEqual(len(rejectedWork), 45) # Check number of jobs in element match number for # dataset in run whitelist wq_jobs = 0 for unit in units: wq_jobs += unit['Jobs'] # This fails. listRunLumis does not work correctly with DBS3, # returning None for the # of lumis in a run runLumis = dbs[inputDataset.dbsurl].listRunLumis(block=unit['Inputs'].keys()[0]) for run in runLumis: if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist(): # This is what it is with DBS3 unless we calculate it self.assertEqual(runLumis[run], None) self.assertEqual(2, int(wq_jobs))
def testWithMaskedBlocks(self): """ _testWithMaskedBlocks_ Test job splitting with masked blocks """ rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 task = getFirstTask(Tier1ReRecoWorkload) dummyDataset = task.inputDataset() task.data.input.splitting.runs = [181061, 180899] task.data.input.splitting.lumis = ['1,50,60,70', '1,1'] lumiMask = LumiList(compactList={'206371': [[1, 50], [60, 70]], '180899': [[1, 1]], }) units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) nLumis = 0 for unit in units: nLumis += unit['NumberOfLumis'] self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testWithMaskedBlocks(self): """ _testWithMaskedBlocks_ Test job splitting with masked blocks """ rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction( 'ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 task = getFirstTask(Tier1ReRecoWorkload) dummyDataset = task.inputDataset() task.data.input.splitting.runs = [181061, 180899] task.data.input.splitting.lumis = ['1,50,60,70', '1,1'] lumiMask = LumiList(compactList={ '206371': [[1, 50], [60, 70]], '180899': [[1, 1]], }) units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) nLumis = 0 for unit in units: nLumis += unit['NumberOfLumis'] self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testRunWhitelist(self): """ ReReco lumi split with Run whitelist This test may not do much of anything anymore since listRunLumis is not in DBS3 """ # get files with multiple runs Globals.GlobalParams.setNumOfRunsPerFile(2) # a large number of lumis to ensure we get multiple runs Globals.GlobalParams.setNumOfLumisPerBlock(10) splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setRunWhitelist([181061, 181175]) Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task) self.assertEqual(1, len(units)) # Check number of jobs in element match number for # dataset in run whitelist wq_jobs = 0 for unit in units: wq_jobs += unit['Jobs'] runLumis = dbs[inputDataset.dbsurl].listRunLumis(dataset=unit['Inputs'].keys()[0]) for run in runLumis: if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist(): # This is what it is with DBS3 unless we calculate it self.assertEqual(runLumis[run], None) self.assertEqual(250, int(wq_jobs))
def testParentProcessing(self): """ test parent processing: should have the same results as rereco test with the parent flag and dataset. """ parentProcArgs["ConfigCacheID"] = createConfig(parentProcArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() parentProcSpec = factory.factoryWorkloadConstruction('testParentProcessing', parentProcArgs) inputDataset = getFirstTask(parentProcSpec).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)} for task in parentProcSpec.taskIterator(): units, _ = Block(**self.splitArgs)(parentProcSpec, task) self.assertEqual(Globals.GlobalParams.numOfBlocksPerDataset(), len(units)) blocks = [] # fill with blocks as we get work units for them for unit in units: self.assertEqual(1, unit['Jobs']) self.assertEqual(parentProcSpec, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(True, unit['ParentFlag']) self.assertEqual(1, len(unit['ParentData'])) self.assertEqual(len(units), len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
def testInvalidSpecs(self): """Specs with no work""" # no dataset rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec).data.input.dataset = None for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueWMSpecError, Block(), processingSpec, task) # invalid dbs url processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com' for task in processingSpec.taskIterator(): self.assertRaises(DBSReaderError, Block(), processingSpec, task) # dataset non existent processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec).data.input.dataset.name = "/MinimumBias/FAKE-Filter-v1/RECO" for task in processingSpec.taskIterator(): self.assertRaises(DBSReaderError, Block(), processingSpec, task) # invalid run whitelist processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs) processingSpec.setRunWhitelist([666]) # not in this dataset for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec, task)
def testRunWhitelist(self): """ ReReco lumi split with Run whitelist This test may not do much of anything anymore since listRunLumis is not in DBS3 """ splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setRunWhitelist([181061, 181175]) Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task) self.assertEqual(1, len(units)) # Check number of jobs in element match number for # dataset in run whitelist wq_jobs = 0 for unit in units: wq_jobs += unit['Jobs'] runLumis = dbs[inputDataset.dbsurl].listRunLumis(dataset=unit['Inputs'].keys()[0]) for run in runLumis: if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist(): # This is what it is with DBS3 unless we calculate it self.assertEqual(runLumis[run], None) self.assertEqual(250, int(wq_jobs))
def testDatasetRunWhitelist(self): """ Test NumberOfRuns splitting with run white list """ dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig( dqmHarvArgs["CouchDBName"]) dqmHarvArgs["RunWhitelist"] = [181358, 181417, 180992, 181151] factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction( 'DQMHarvestTest', dqmHarvArgs) splitArgs = DQMHarvWorkload.startPolicyParameters() inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath() for task in DQMHarvWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(4, unit['Jobs']) self.assertEqual(DQMHarvWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(unit['Inputs'].keys(), [inputDataset]) self.assertEqual(217, unit['NumberOfLumis']) self.assertEqual(8, unit['NumberOfFiles']) self.assertEqual(83444, unit['NumberOfEvents'])
def getProcessingACDCSpec(self, splittingAlgo='LumiBased', splittingArgs=None, setLocationFlag=False): """ _getProcessingACDCSpec_ Get a ACDC spec for the processing task of a ReReco workload """ if splittingArgs is None: splittingArgs = {'lumis_per_job': 8} factory = ReRecoWorkloadFactory() rerecoArgs = ReRecoWorkloadFactory.getTestArguments() rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) rerecoArgs["Requestor"] = self.user rerecoArgs["Group"] = self.group Tier1ReRecoWorkload = factory.factoryWorkloadConstruction( self.workflowName, rerecoArgs) Tier1ReRecoWorkload.truncate('ACDC_%s' % self.workflowName, '/%s/DataProcessing' % self.workflowName, self.couchUrl, self.acdcDBName) Tier1ReRecoWorkload.setJobSplittingParameters( '/ACDC_%s/DataProcessing' % self.workflowName, splittingAlgo, splittingArgs) Tier1ReRecoWorkload.setSiteWhitelist(self.siteWhitelist) if setLocationFlag: Tier1ReRecoWorkload.setTrustLocationFlag(setLocationFlag) # Tier1ReRecoWorkload.setSiteWhitelist(self.siteWhitelist) return Tier1ReRecoWorkload
def createMCWMSpec(self, name='MonteCarloWorkload'): mcArgs['CouchDBName'] = rerecoArgs["CouchDBName"] mcArgs["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"]) wmspec = monteCarloWorkload(name, mcArgs) wmspec.setSpecUrl("/path/to/workload") getFirstTask(wmspec).addProduction(totalevents=10000) return wmspec
def createMCWMSpec(self, name='MonteCarloWorkload'): mcArgs = TaskChainWorkloadFactory.getTestArguments() mcArgs["CouchDBName"] = rerecoArgs["CouchDBName"] mcArgs["Task1"]["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"]) wmspec = taskChainWorkload(name, mcArgs) wmspec.setSpecUrl("/path/to/workload") getFirstTask(wmspec).addProduction(totalevents=10000) return wmspec
def createWMSpec(self, name = 'ReRecoWorkload'): factory = ReRecoWorkloadFactory() rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) wmspec = factory.factoryWorkloadConstruction(name, rerecoArgs) wmspec.setSpecUrl("/path/to/workload") wmspec.setSubscriptionInformation(custodialSites = [], nonCustodialSites = [], autoApproveSites = [], priority = "Low", custodialSubType = "Move") return wmspec
def createWMSpec(self, name='ReRecoWorkload'): factory = ReRecoWorkloadFactory() rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) wmspec = factory.factoryWorkloadConstruction(name, rerecoArgs) wmspec.setSpecUrl("/path/to/workload") wmspec.setSubscriptionInformation(custodialSites=[], nonCustodialSites=[], autoApproveSites=[], priority="Low", custodialSubType="Move") return wmspec
def testIgnore0SizeBlocks(self): """Ignore blocks with 0 files""" rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setRunWhitelist([2, 3]) for task in Tier1ReRecoWorkload.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Block(**self.splitArgs), Tier1ReRecoWorkload, task)
def testDataDirectiveFromQueue(self): """Test data directive from queue""" dqmHarvArgs = getRequestArgs() dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"]) factory = DQMHarvestWorkloadFactory() DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs) DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"]) splitArgs = DQMHarvWorkload.startPolicyParameters() for task in DQMHarvWorkload.taskIterator(): self.assertRaises(RuntimeError, Dataset(**splitArgs), DQMHarvWorkload, task, {NOT_EXIST_DATASET: []})
def testIgnore0SizeBlocks(self): """Ignore blocks with 0 files""" rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction( 'ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setRunWhitelist([2, 3]) for task in Tier1ReRecoWorkload.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Block(**self.splitArgs), Tier1ReRecoWorkload, task)
def testIgnore0SizeBlocks(self): """Ignore blocks with 0 files""" Globals.GlobalParams.setNumOfFilesPerBlock(0) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setRunWhitelist([2, 3]) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() for task in Tier1ReRecoWorkload.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Block(**self.splitArgs), Tier1ReRecoWorkload, task)
def testLumiSplitTier1ReRecoWorkload(self): """Tier1 Re-reco workflow split by Lumi""" splitArgs = dict(SliceType='NumberOfLumis', SliceSize=2) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs) for task in Tier1ReRecoWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(2428, unit['Jobs'])
def testDataDirectiveFromQueue(self): """Test data directive from queue""" rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): Dataset(**self.splitArgs)(Tier1ReRecoWorkload, task, {dataset: []}) self.assertRaises(RuntimeError, Dataset(**self.splitArgs), Tier1ReRecoWorkload, task, dbs, {dataset + '1': []})
def setUp(self): self.testInit = TestInit('CouchWorkQueueTest') self.testInit.setLogging() self.testInit.setupCouch('wq_backend_test_inbox', 'WorkQueue') self.testInit.setupCouch('wq_backend_test', 'WorkQueue') self.testInit.setupCouch('wq_backend_test_parent', 'WorkQueue') self.couch_db = self.testInit.couch.couchServer.connectDatabase('wq_backend_test') self.backend = WorkQueueBackend(db_url=self.testInit.couchUrl, db_name='wq_backend_test', inbox_name='wq_backend_test_inbox', parentQueue='%s/%s' % (self.testInit.couchUrl, 'wq_backend_test_parent')) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
def testIgnore0SizeBlocks(self): """Ignore blocks with 0 files""" Globals.GlobalParams.setNumOfFilesPerBlock(0) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction( 'ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setRunWhitelist([2, 3]) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() for task in Tier1ReRecoWorkload.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Block(**self.splitArgs), Tier1ReRecoWorkload, task)
def testContinuousSplittingSupport(self): """Can modify successfully policies for continuous splitting""" policyInstance = Block(**self.splitArgs) self.assertTrue(policyInstance.supportsWorkAddition(), "Block instance should support continuous splitting") rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction( 'ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): units, _ = policyInstance(Tier1ReRecoWorkload, task) self.assertEqual(47, len(units)) blocks = [] # fill with blocks as we get work units for them inputs = {} for unit in units: blocks.extend(unit['Inputs'].keys()) inputs.update(unit['Inputs']) self.assertEqual(69, unit['Priority']) self.assertTrue(1 <= unit['Jobs']) self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertTrue(1 <= unit['NumberOfLumis']) self.assertTrue(1 <= unit['NumberOfFiles']) self.assertTrue(0 <= unit['NumberOfEvents']) self.assertEqual( len(units), len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset))) # Modify the spec and task, get first a fresh policy instance policyInstance = Block(**self.splitArgs) for task in Tier1ReRecoWorkload.taskIterator(): policyInstance.modifyPolicyForWorkAddition( {'ProcessedInputs': inputs.keys()}) self.assertRaises(WorkQueueNoWorkError, policyInstance, Tier1ReRecoWorkload, task) # Run one last time policyInstance = Block(**self.splitArgs) for task in Tier1ReRecoWorkload.taskIterator(): policyInstance.modifyPolicyForWorkAddition( {'ProcessedInputs': inputs.keys()}) self.assertRaises(WorkQueueNoWorkError, policyInstance, Tier1ReRecoWorkload, task) return
def testLumiSplitTier1ReRecoWorkload(self): """Tier1 Re-reco workflow""" splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setStartPolicy('Block', **splitArgs) for task in Tier1ReRecoWorkload.taskIterator(): units, rejectedWork = Block(**splitArgs)(Tier1ReRecoWorkload, task) self.assertEqual(47, len(units)) for unit in units: self.assertTrue(1 <= unit['Jobs']) self.assertEqual(0, len(rejectedWork))
def getMergeACDCSpec(self, splittingAlgo = 'ParentlessMergeBySize', splittingArgs = {}): """ _getMergeACDCSpec_ Get a ACDC spec for the merge task of a ReReco workload """ factory = ReRecoWorkloadFactory() rerecoArgs = ReRecoWorkloadFactory.getTestArguments() rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(self.workflowName, rerecoArgs) Tier1ReRecoWorkload.truncate('ACDC_%s' % self.workflowName, '/%s/DataProcessing/DataProcessingMergeRECOoutput' % self.workflowName, self.couchUrl, self.acdcDBName) Tier1ReRecoWorkload.setJobSplittingParameters('/ACDC_%s/DataProcessingMergeRECOoutput' % self.workflowName, splittingAlgo, splittingArgs) return Tier1ReRecoWorkload
def testInvalidSpecs(self): """Specs with no work""" # no dataset rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() processingSpec = factory.factoryWorkloadConstruction( 'testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec).data.input.dataset = None for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueWMSpecError, Block(), processingSpec, task) # invalid dbs url processingSpec = factory.factoryWorkloadConstruction( 'testProcessingInvalid', rerecoArgs) getFirstTask( processingSpec ).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com' for task in processingSpec.taskIterator(): self.assertRaises(DBSReaderError, Block(), processingSpec, task) # invalid dataset name # This test is throwing a DBS exception but continuing on: """ Message: DbsBadRequest: DBS Server Raised An Error ModuleName : WMCore.Services.DBS.DBSErrors MethodName : __init__ ClassInstance : None FileName : /data/srv/wmagent_ewv/v0.9.94c/sw/slc5_amd64_gcc461/cms/wmagent/0.9.94c/lib/python2.6/site-packages/WMCore/Services/DBS/DBSErrors.py ClassName : None LineNumber : 29 ErrorNr : 1002 """ processingSpec = factory.factoryWorkloadConstruction( 'testProcessingInvalid', rerecoArgs) getFirstTask(processingSpec ).data.input.dataset.primary = Globals.NOT_EXIST_DATASET for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec, task) # invalid run whitelist processingSpec = factory.factoryWorkloadConstruction( 'testProcessingInvalid', rerecoArgs) processingSpec.setRunWhitelist([666]) # not in this dataset for task in processingSpec.taskIterator(): self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec, task)
def testLumiSplitTier1ReRecoWorkload(self): """Tier1 Re-reco workflow""" splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction( 'ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setStartPolicy('Block', **splitArgs) for task in Tier1ReRecoWorkload.taskIterator(): units, rejectedWork = Block(**splitArgs)(Tier1ReRecoWorkload, task) self.assertEqual(47, len(units)) for unit in units: self.assertTrue(1 <= unit['Jobs']) self.assertEqual(0, len(rejectedWork))
def setUp(self): self.testInit = TestInit('CouchWorkQueueTest') self.testInit.setLogging() self.testInit.setupCouch('wq_backend_test_inbox', 'WorkQueue') self.testInit.setupCouch('wq_backend_test', 'WorkQueue') self.testInit.setupCouch('wq_backend_test_parent', 'WorkQueue') self.couch_db = self.testInit.couch.couchServer.connectDatabase( 'wq_backend_test') self.backend = WorkQueueBackend( db_url=self.testInit.couchUrl, db_name='wq_backend_test', inbox_name='wq_backend_test_inbox', parentQueue='%s/%s' % (self.testInit.couchUrl, 'wq_backend_test_parent')) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
def testLumiSplitTier1ReRecoWorkload(self): """Tier1 Re-reco workflow split by Lumi""" splitArgs = dict(SliceType = 'NumberOfLumis', SliceSize = 2) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs) inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset() dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)} for task in Tier1ReRecoWorkload.taskIterator(): units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task) self.assertEqual(1, len(units)) for unit in units: self.assertEqual(4, unit['Jobs'])
def testLumiMask(self): """Lumi mask test""" rerecoArgs2 = {} rerecoArgs2.update(rerecoArgs) rerecoArgs2["ConfigCacheID"] = createConfig(rerecoArgs2["CouchDBName"]) factory = ReRecoWorkloadFactory() dummyWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs2) # Block blacklist lumiWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs2) task = getFirstTask(lumiWorkload) #task.data.input.splitting.runs = ['1'] task.data.input.splitting.runs = ['180992'] task.data.input.splitting.lumis = ['1,1'] units, rejectedWork = Block(**self.splitArgs)(lumiWorkload, task) self.assertEqual(len(units), 1) self.assertEqual(len(rejectedWork), 46)