def testInvalidSpecs(self):
"""Specs with no work"""
# no dataset
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset = None
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueWMSpecError, Dataset(), processingSpec, task)
# invalid dbs url
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com'
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueWMSpecError, Dataset(), processingSpec, task)
# invalid dataset name
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset.primary = Globals.NOT_EXIST_DATASET
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Dataset(), processingSpec, task)
# invalid run whitelist
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
processingSpec.setRunWhitelist([666]) # not in this dataset
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Dataset(), processingSpec, task)
def testInvalidSpecs(self):
"""Specs with no work"""
dqmHarvArgs = getRequestArgs()
# no dataset
dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction('NoInputDatasetTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
getFirstTask(DQMHarvWorkload).data.input.dataset = None
for task in DQMHarvWorkload.taskIterator():
self.assertRaises(WorkQueueWMSpecError, Dataset(), DQMHarvWorkload, task)
# invalid dataset name
DQMHarvWorkload = factory.factoryWorkloadConstruction('InvalidInputDatasetTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
getFirstTask(DQMHarvWorkload).data.input.dataset.name = '/MinimumBias/FAKE-Filter-v1/RECO'
for task in DQMHarvWorkload.taskIterator():
self.assertRaises(DBSReaderError, Dataset(), DQMHarvWorkload, task)
# invalid run whitelist
DQMHarvWorkload = factory.factoryWorkloadConstruction('InvalidRunNumberTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
DQMHarvWorkload.setRunWhitelist([666]) # not in this dataset
for task in DQMHarvWorkload.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Dataset(), DQMHarvWorkload, task)
def testDatasetSingleJob(self):
"""
Test NumberOfRuns splitting type with very large SliceSize
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(
dqmHarvArgs["CouchDBName"])
dqmHarvArgs["DQMHarvestUnit"] = 'multiRun'
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction(
'DQMHarvestTest', dqmHarvArgs)
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(1, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(4855, unit['NumberOfLumis'])
self.assertEqual(72, unit['NumberOfFiles'])
self.assertEqual(743201, unit['NumberOfEvents'])
def testInvalidSpecs(self):
"""Specs with no work"""
# no dataset
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset = None
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueWMSpecError, Block(), processingSpec, task)
# invalid dbs url
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com'
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueWMSpecError, Block(), processingSpec, task)
# invalid dataset name
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset.primary = Globals.NOT_EXIST_DATASET
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec, task)
# invalid run whitelist
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
processingSpec.setRunWhitelist([666]) # not in this dataset
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec, task)
# blocks with 0 files are skipped
# set all blocks in request to 0 files, no work should be found & an error is raised
Globals.GlobalParams.setNumOfFilesPerBlock(0)
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec, task)
Globals.GlobalParams.resetParams()
def notestParentProcessing(self):
# Does not work with a RAW dataset, need a different workload
"""
test parent processing: should have the same results as rereco test
with the parent flag and dataset.
"""
parentProcArgs["ConfigCacheID"] = createConfig(
parentProcArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
parentProcSpec = factory.factoryWorkloadConstruction(
'testParentProcessing', parentProcArgs)
inputDataset = getFirstTask(parentProcSpec).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in parentProcSpec.taskIterator():
units, _ = Block(**self.splitArgs)(parentProcSpec, task)
self.assertEqual(47, len(units))
blocks = [] # fill with blocks as we get work units for them
for unit in units:
import pdb
pdb.set_trace()
self.assertTrue(1 <= unit['Jobs'])
self.assertEqual(parentProcSpec, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(True, unit['ParentFlag'])
self.assertEqual(1, len(unit['ParentData']))
self.assertEqual(
len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
def notestParentProcessing(self):
# Does not work with a RAW dataset, need a different workload
"""
test parent processing: should have the same results as rereco test
with the parent flag and dataset.
"""
parentProcArgs["ConfigCacheID"] = createConfig(parentProcArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
parentProcSpec = factory.factoryWorkloadConstruction('testParentProcessing', parentProcArgs)
inputDataset = getFirstTask(parentProcSpec).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in parentProcSpec.taskIterator():
units, _ = Block(**self.splitArgs)(parentProcSpec, task)
self.assertEqual(47, len(units))
for unit in units:
import pdb
pdb.set_trace()
self.assertTrue(1 <= unit['Jobs'])
self.assertEqual(parentProcSpec, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(True, unit['ParentFlag'])
self.assertEqual(1, len(unit['ParentData']))
self.assertEqual(len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
def testParentProcessing(self):
"""
test parent processing: should have the same results as rereco test
with the parent flag and dataset.
"""
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=2)
parentProcArgs = getReRecoArgs(parent=True)
parentProcArgs["ConfigCacheID"] = createConfig(parentProcArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
# This dataset does have parents. Adding it here to keep the test going.
# It seems like "dbs" below is never used
parentProcArgs2 = {}
parentProcArgs2.update(parentProcArgs)
parentProcArgs2.update({'InputDataset': '/SingleMu/CMSSW_6_2_0_pre4-PRE_61_V1_RelVal_mu2012A-v1/RECO'})
parentProcSpec = factory.factoryWorkloadConstruction('testParentProcessing', parentProcArgs2)
parentProcSpec.setStartPolicy('Dataset', **splitArgs)
inputDataset = getFirstTask(parentProcSpec).getInputDatasetPath()
for task in parentProcSpec.taskIterator():
units, _ = Dataset(**splitArgs)(parentProcSpec, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(847, unit['Jobs'])
self.assertEqual(1694, unit['NumberOfLumis'])
self.assertEqual(parentProcSpec, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(True, unit['ParentFlag'])
self.assertEqual(0, len(unit['ParentData']))
def testWithMaskedBlocks(self):
"""
_testWithMaskedBlocks_
Test job splitting with masked blocks
"""
Globals.GlobalParams.setNumOfRunsPerFile(3)
Globals.GlobalParams.setNumOfLumisPerBlock(5)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
inputDataset = task.inputDataset()
task.data.input.splitting.runs = [181061, 180899]
task.data.input.splitting.lumis = ['1,50,60,70', '1,1']
lumiMask = LumiList(compactList = {'206371': [[1, 50], [60,70]], '180899':[[1,1]], } )
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
units, rejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task)
nLumis = 0
for unit in units:
nLumis += unit['NumberOfLumis']
self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testTier1ReRecoWorkload(self):
"""Tier1 Re-reco workflow"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = Block(**self.splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(47, len(units))
blocks = [] # fill with blocks as we get work units for them
for unit in units:
self.assertEqual(69, unit['Priority'])
self.assertTrue(1 <= unit['Jobs'])
self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertTrue(1 <= unit['NumberOfLumis'])
self.assertTrue(1 <= unit['NumberOfFiles'])
self.assertTrue(0 <= unit['NumberOfEvents'])
self.assertEqual(len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
def testParentProcessing(self):
"""
test parent processing: should have the same results as rereco test
with the parent flag and dataset.
"""
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=2)
parentProcArgs = getReRecoArgs(parent=True)
parentProcArgs["ConfigCacheID"] = createConfig(parentProcArgs["CouchDBName"])
# This dataset does have parents. Adding it here to keep the test going.
# It seems like "dbs" below is never used
parentProcArgs2 = {}
parentProcArgs2.update(parentProcArgs)
#parentProcArgs2.update({'InputDataset': '/SingleMu/CMSSW_6_2_0_pre4-PRE_61_V1_RelVal_mu2012A-v1/RECO'})
parentProcArgs2.update({'InputDataset': '/Cosmics/ComissioningHI-PromptReco-v1/RECO'})
from pprint import pprint
pprint(parentProcArgs2)
parentProcSpec = rerecoWorkload('ReRecoWorkload', parentProcArgs2,
assignArgs={'SiteWhitelist': ['T2_XX_SiteA']})
parentProcSpec.setStartPolicy('Dataset', **splitArgs)
inputDataset = getFirstTask(parentProcSpec).getInputDatasetPath()
for task in parentProcSpec.taskIterator():
units, _, _ = Dataset(**splitArgs)(parentProcSpec, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(3993, unit['Jobs'])
self.assertEqual(7985, unit['NumberOfLumis'])
self.assertEqual(parentProcSpec, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(True, unit['ParentFlag'])
self.assertEqual(0, len(unit['ParentData']))
def testGetMaskedBlocks(self):
"""
_testGetMaskedBlocks_
Check that getMaskedBlocks is returning the correct information
"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
inputDataset = task.inputDataset()
inputDataset.primary = 'SingleElectron'
inputDataset.processed = 'StoreResults-Run2011A-WElectron-PromptSkim-v4-ALCARECO-NOLC-36cfce5a1d3f3ab4df5bd2aa0a4fa380'
inputDataset.tier = 'USER'
task.data.input.splitting.runs = [166921, 166429, 166911]
task.data.input.splitting.lumis = ['40,70', '1,50', '1,5,16,20']
lumiMask = LumiList(compactList={'166921': [[40, 70]], '166429': [[1, 50]], '166911': [[1, 5], [16, 20]], })
inputLumis = LumiList(compactList={'166921': [[1, 67]], '166429': [[1, 91]], '166911': [[1, 104]], })
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = DBSReader(inputDataset.dbsurl)
maskedBlocks = Block(**self.splitArgs).getMaskedBlocks(task, dbs, dataset)
for dummyBlock, files in maskedBlocks.iteritems():
for dummyFile, lumiList in files.iteritems():
self.assertEqual(str(lumiList), str(inputLumis & lumiMask))
def testTier1ReRecoWorkload(self):
"""Tier1 Re-reco workflow"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
'ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = Block(**self.splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(47, len(units))
blocks = [] # fill with blocks as we get work units for them
for unit in units:
self.assertEqual(69, unit['Priority'])
self.assertTrue(1 <= unit['Jobs'])
self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertTrue(1 <= unit['NumberOfLumis'])
self.assertTrue(1 <= unit['NumberOfFiles'])
self.assertTrue(0 <= unit['NumberOfEvents'])
self.assertEqual(
len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
def testParentProcessing(self):
"""
test parent processing: should have the same results as rereco test
with the parent flag and dataset.
"""
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=2)
parentProcArgs = getReRecoArgs(parent=True)
parentProcArgs["ConfigCacheID"] = createConfig(
parentProcArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
# This dataset does have parents. Adding it here to keep the test going.
# It seems like "dbs" below is never used
parentProcArgs2 = {}
parentProcArgs2.update(parentProcArgs)
parentProcArgs2.update({
'InputDataset':
'/SingleMu/CMSSW_6_2_0_pre4-PRE_61_V1_RelVal_mu2012A-v1/RECO'
})
parentProcSpec = factory.factoryWorkloadConstruction(
'testParentProcessing', parentProcArgs2)
parentProcSpec.setStartPolicy('Dataset', **splitArgs)
inputDataset = getFirstTask(parentProcSpec).getInputDatasetPath()
for task in parentProcSpec.taskIterator():
units, _ = Dataset(**splitArgs)(parentProcSpec, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(847, unit['Jobs'])
self.assertEqual(1694, unit['NumberOfLumis'])
self.assertEqual(parentProcSpec, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(True, unit['ParentFlag'])
self.assertEqual(0, len(unit['ParentData']))
def getMergeACDCSpec(self,
splittingAlgo='ParentlessMergeBySize',
splittingArgs=None):
"""
_getMergeACDCSpec_
Get a ACDC spec for the merge task of a ReReco workload
"""
if splittingArgs is None:
splittingArgs = {}
factory = ReRecoWorkloadFactory()
rerecoArgs = ReRecoWorkloadFactory.getTestArguments()
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
rerecoArgs["Requestor"] = self.user
rerecoArgs["Group"] = self.group
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
self.workflowName, rerecoArgs)
Tier1ReRecoWorkload.truncate(
'ACDC_%s' % self.workflowName,
'/%s/DataProcessing/DataProcessingMergeRECOoutput' %
self.workflowName, self.couchUrl, self.acdcDBName)
Tier1ReRecoWorkload.setJobSplittingParameters(
'/ACDC_%s/DataProcessingMergeRECOoutput' % self.workflowName,
splittingAlgo, splittingArgs)
return Tier1ReRecoWorkload
def testRunWhitelist(self):
"""ReReco lumi split with Run whitelist"""
# get files with multiple runs
Globals.GlobalParams.setNumOfRunsPerFile(2)
# a large number of lumis to ensure we get multiple runs
Globals.GlobalParams.setNumOfLumisPerBlock(10)
splitArgs = dict(SliceType = 'NumberOfLumis', SliceSize = 1)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setRunWhitelist([2, 3])
Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs)
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(1, len(units))
# Check number of jobs in element match number for
# dataset in run whitelist
jobs = 0
wq_jobs = 0
for unit in units:
wq_jobs += unit['Jobs']
runLumis = dbs[inputDataset.dbsurl].listRunLumis(dataset = unit['Inputs'].keys()[0])
for run in runLumis:
if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist():
jobs += runLumis[run]
self.assertEqual(int(jobs / splitArgs['SliceSize'] ) , int(wq_jobs))
def testParentProcessing(self):
"""
test parent processing: should have the same results as rereco test
with the parent flag and dataset.
"""
parentProcArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
# This dataset does have parents. Adding it here to keep the test going. It seems like "dbs" below is never used
parentProcArgs2 = {}
parentProcArgs2.update(parentProcArgs)
parentProcArgs2.update({'InputDataset': '/SingleMu/CMSSW_6_2_0_pre4-PRE_61_V1_RelVal_mu2012A-v1/RECO'})
parentProcSpec = factory.factoryWorkloadConstruction('testParentProcessing', parentProcArgs2)
parentProcSpec.setStartPolicy('Dataset', **self.splitArgs)
inputDataset = getFirstTask(parentProcSpec).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier)
dummyDBS = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in parentProcSpec.taskIterator():
units, _ = Dataset(**self.splitArgs)(parentProcSpec, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(64, unit['Jobs'])
self.assertEqual(parentProcSpec, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [dataset])
self.assertEqual(True, unit['ParentFlag'])
self.assertEqual(0, len(unit['ParentData']))
def testDatasetPolicy(self):
"""
Test ordinary NumberOfRuns splitting
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(
dqmHarvArgs["CouchDBName"])
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction(
'DQMHarvestTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(
["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(47, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(list(unit['Inputs']), [inputDataset])
self.assertEqual(4855, unit['NumberOfLumis'])
self.assertEqual(72, unit['NumberOfFiles'])
self.assertEqual(743201, unit['NumberOfEvents'])
def testDatasetLumiMask(self):
"""
Test NumberOfRuns splitting type with lumi mask
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(
dqmHarvArgs["CouchDBName"])
dqmHarvArgs["LumiList"] = {
"181358": [[71, 80], [95, 110]],
"181151": [[1, 20]]
}
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction(
'DQMHarvestTest', dqmHarvArgs)
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(2, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(46, unit['NumberOfLumis'])
self.assertEqual(4, unit['NumberOfFiles'])
self.assertEqual(12342, unit['NumberOfEvents'])
def testRunWhitelist(self):
"""
ReReco lumi split with Run whitelist
This test may not do much of anything anymore since listRunLumis is not in DBS3
"""
# get files with multiple runs
Globals.GlobalParams.setNumOfRunsPerFile(2)
# a large number of lumis to ensure we get multiple runs
Globals.GlobalParams.setNumOfLumisPerBlock(10)
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setRunWhitelist([2, 3])
Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs)
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(1, len(units))
# Check number of jobs in element match number for
# dataset in run whitelist
wq_jobs = 0
for unit in units:
wq_jobs += unit['Jobs']
runLumis = dbs[inputDataset.dbsurl].listRunLumis(dataset=unit['Inputs'].keys()[0])
print "runLumis", runLumis
for run in runLumis:
if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist():
self.assertEqual(runLumis[run], None) # This is what it is with DBS3 unless we calculate it
self.assertEqual(75, int(wq_jobs))
def testRunWhitelist(self):
"""ReReco lumi split with Run whitelist"""
# get files with multiple runs
Globals.GlobalParams.setNumOfRunsPerFile(8)
# a large number of lumis to ensure we get multiple runs
Globals.GlobalParams.setNumOfLumisPerBlock(20)
splitArgs = dict(SliceType = 'NumberOfLumis', SliceSize = 1)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setStartPolicy('Block', **splitArgs)
Tier1ReRecoWorkload.setRunWhitelist([180899, 180992])
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, rejectedWork = Block(**splitArgs)(Tier1ReRecoWorkload, task)
# Blocks 1 and 2 match run distribution
self.assertEqual(2, len(units))
self.assertEqual(len(rejectedWork), 45)
# Check number of jobs in element match number for
# dataset in run whitelist
jobs = 0
wq_jobs = 0
for unit in units:
wq_jobs += unit['Jobs']
# This fails. listRunLumis does not work correctly with DBS3, returning None for the # of lumis in a run
runLumis = dbs[inputDataset.dbsurl].listRunLumis(block = unit['Inputs'].keys()[0])
for run in runLumis:
if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist():
jobs += runLumis[run]
self.assertEqual(int(jobs / splitArgs['SliceSize'] ) , int(wq_jobs))
def testGetMaskedBlocks(self):
"""
_testGetMaskedBlocks_
Check that getMaskedBlocks is returning the correct information
"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
inputDataset = task.inputDataset()
inputDataset.primary = 'SingleElectron'
inputDataset.processed = 'StoreResults-Run2011A-WElectron-PromptSkim-v4-ALCARECO-NOLC-36cfce5a1d3f3ab4df5bd2aa0a4fa380'
inputDataset.tier = 'USER'
task.data.input.splitting.runs = [166921, 166429, 166911]
task.data.input.splitting.lumis = ['40,70', '1,50', '1,5,16,20']
lumiMask = LumiList(compactList={'166921': [[40, 70]], '166429': [[1, 50]], '166911': [[1, 5], [16, 20]], })
inputLumis = LumiList(compactList={'166921': [[1, 67]], '166429': [[1, 91]], '166911': [[1, 104]], })
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = DBSReader(inputDataset.dbsurl)
maskedBlocks = Block(**self.splitArgs).getMaskedBlocks(task, dbs, dataset)
for dummyBlock, files in maskedBlocks.iteritems():
for dummyFile, lumiList in files.iteritems():
self.assertEqual(str(lumiList), str(inputLumis & lumiMask))
def testParentProcessing(self):
"""
test parent processing: should have the same results as rereco test
with the parent flag and dataset.
"""
parentProcArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
# This dataset does have parents. Adding it here to keep the test going. It seems like "dbs" below is never used
parentProcArgs2 = {}
parentProcArgs2.update(parentProcArgs)
parentProcArgs2.update({'InputDataset': '/SingleMu/CMSSW_6_2_0_pre4-PRE_61_V1_RelVal_mu2012A-v1/RECO'})
parentProcSpec = factory.factoryWorkloadConstruction('testParentProcessing', parentProcArgs2)
parentProcSpec.setStartPolicy('Dataset', **self.splitArgs)
inputDataset = getFirstTask(parentProcSpec).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
for task in parentProcSpec.taskIterator():
units, _ = Dataset(**self.splitArgs)(parentProcSpec, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(4, unit['Jobs'])
self.assertEqual(parentProcSpec, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [dataset])
self.assertEqual(True, unit['ParentFlag'])
self.assertEqual(0, len(unit['ParentData']))
def testRunWhitelist(self):
"""
ReReco lumi split with Run whitelist
This test may not do much of anything anymore since listRunLumis is not in DBS3
"""
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setStartPolicy('Block', **splitArgs)
Tier1ReRecoWorkload.setRunWhitelist([180899, 180992])
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, rejectedWork = Block(**splitArgs)(Tier1ReRecoWorkload, task)
# Blocks 1 and 2 match run distribution
self.assertEqual(2, len(units))
self.assertEqual(len(rejectedWork), 45)
# Check number of jobs in element match number for
# dataset in run whitelist
wq_jobs = 0
for unit in units:
wq_jobs += unit['Jobs']
# This fails. listRunLumis does not work correctly with DBS3,
# returning None for the # of lumis in a run
runLumis = dbs[inputDataset.dbsurl].listRunLumis(block=unit['Inputs'].keys()[0])
for run in runLumis:
if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist():
# This is what it is with DBS3 unless we calculate it
self.assertEqual(runLumis[run], None)
self.assertEqual(2, int(wq_jobs))
def testWithMaskedBlocks(self):
"""
_testWithMaskedBlocks_
Test job splitting with masked blocks
"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
dummyDataset = task.inputDataset()
task.data.input.splitting.runs = [181061, 180899]
task.data.input.splitting.lumis = ['1,50,60,70', '1,1']
lumiMask = LumiList(compactList={'206371': [[1, 50], [60, 70]], '180899': [[1, 1]], })
units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task)
nLumis = 0
for unit in units:
nLumis += unit['NumberOfLumis']
self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testWithMaskedBlocks(self):
"""
_testWithMaskedBlocks_
Test job splitting with masked blocks
"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
'ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
dummyDataset = task.inputDataset()
task.data.input.splitting.runs = [181061, 180899]
task.data.input.splitting.lumis = ['1,50,60,70', '1,1']
lumiMask = LumiList(compactList={
'206371': [[1, 50], [60, 70]],
'180899': [[1, 1]],
})
units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload,
task)
nLumis = 0
for unit in units:
nLumis += unit['NumberOfLumis']
self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testRunWhitelist(self):
"""
ReReco lumi split with Run whitelist
This test may not do much of anything anymore since listRunLumis is not in DBS3
"""
# get files with multiple runs
Globals.GlobalParams.setNumOfRunsPerFile(2)
# a large number of lumis to ensure we get multiple runs
Globals.GlobalParams.setNumOfLumisPerBlock(10)
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setRunWhitelist([181061, 181175])
Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs)
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(1, len(units))
# Check number of jobs in element match number for
# dataset in run whitelist
wq_jobs = 0
for unit in units:
wq_jobs += unit['Jobs']
runLumis = dbs[inputDataset.dbsurl].listRunLumis(dataset=unit['Inputs'].keys()[0])
for run in runLumis:
if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist():
# This is what it is with DBS3 unless we calculate it
self.assertEqual(runLumis[run], None)
self.assertEqual(250, int(wq_jobs))
def testParentProcessing(self):
"""
test parent processing: should have the same results as rereco test
with the parent flag and dataset.
"""
parentProcArgs["ConfigCacheID"] = createConfig(parentProcArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
parentProcSpec = factory.factoryWorkloadConstruction('testParentProcessing', parentProcArgs)
inputDataset = getFirstTask(parentProcSpec).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
for task in parentProcSpec.taskIterator():
units, _ = Block(**self.splitArgs)(parentProcSpec, task)
self.assertEqual(Globals.GlobalParams.numOfBlocksPerDataset(), len(units))
blocks = [] # fill with blocks as we get work units for them
for unit in units:
self.assertEqual(1, unit['Jobs'])
self.assertEqual(parentProcSpec, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(True, unit['ParentFlag'])
self.assertEqual(1, len(unit['ParentData']))
self.assertEqual(len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
def testInvalidSpecs(self):
"""Specs with no work"""
# no dataset
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset = None
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueWMSpecError, Dataset(), processingSpec, task)
# invalid dbs url
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com'
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueWMSpecError, Dataset(), processingSpec, task)
# invalid dataset name
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset.primary = Globals.NOT_EXIST_DATASET
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Dataset(), processingSpec, task)
# invalid run whitelist
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
processingSpec.setRunWhitelist([666]) # not in this dataset
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Dataset(), processingSpec, task)
def testInvalidSpecs(self):
"""Specs with no work"""
# no dataset
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset = None
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueWMSpecError, Block(), processingSpec, task)
# invalid dbs url
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com'
for task in processingSpec.taskIterator():
self.assertRaises(DBSReaderError, Block(), processingSpec, task)
# dataset non existent
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset.name = "/MinimumBias/FAKE-Filter-v1/RECO"
for task in processingSpec.taskIterator():
self.assertRaises(DBSReaderError, Block(), processingSpec, task)
# invalid run whitelist
processingSpec = factory.factoryWorkloadConstruction('testProcessingInvalid', rerecoArgs)
processingSpec.setRunWhitelist([666]) # not in this dataset
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec, task)
def testRunWhitelist(self):
"""
ReReco lumi split with Run whitelist
This test may not do much of anything anymore since listRunLumis is not in DBS3
"""
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setRunWhitelist([181061, 181175])
Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs)
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(1, len(units))
# Check number of jobs in element match number for
# dataset in run whitelist
wq_jobs = 0
for unit in units:
wq_jobs += unit['Jobs']
runLumis = dbs[inputDataset.dbsurl].listRunLumis(dataset=unit['Inputs'].keys()[0])
for run in runLumis:
if run in getFirstTask(Tier1ReRecoWorkload).inputRunWhitelist():
# This is what it is with DBS3 unless we calculate it
self.assertEqual(runLumis[run], None)
self.assertEqual(250, int(wq_jobs))
def testDatasetRunWhitelist(self):
"""
Test NumberOfRuns splitting with run white list
"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(
dqmHarvArgs["CouchDBName"])
dqmHarvArgs["RunWhitelist"] = [181358, 181417, 180992, 181151]
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction(
'DQMHarvestTest', dqmHarvArgs)
splitArgs = DQMHarvWorkload.startPolicyParameters()
inputDataset = getFirstTask(DQMHarvWorkload).getInputDatasetPath()
for task in DQMHarvWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(DQMHarvWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(4, unit['Jobs'])
self.assertEqual(DQMHarvWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(unit['Inputs'].keys(), [inputDataset])
self.assertEqual(217, unit['NumberOfLumis'])
self.assertEqual(8, unit['NumberOfFiles'])
self.assertEqual(83444, unit['NumberOfEvents'])
def getProcessingACDCSpec(self,
splittingAlgo='LumiBased',
splittingArgs=None,
setLocationFlag=False):
"""
_getProcessingACDCSpec_
Get a ACDC spec for the processing task of a ReReco workload
"""
if splittingArgs is None:
splittingArgs = {'lumis_per_job': 8}
factory = ReRecoWorkloadFactory()
rerecoArgs = ReRecoWorkloadFactory.getTestArguments()
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
rerecoArgs["Requestor"] = self.user
rerecoArgs["Group"] = self.group
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
self.workflowName, rerecoArgs)
Tier1ReRecoWorkload.truncate('ACDC_%s' % self.workflowName,
'/%s/DataProcessing' % self.workflowName,
self.couchUrl, self.acdcDBName)
Tier1ReRecoWorkload.setJobSplittingParameters(
'/ACDC_%s/DataProcessing' % self.workflowName, splittingAlgo,
splittingArgs)
Tier1ReRecoWorkload.setSiteWhitelist(self.siteWhitelist)
if setLocationFlag:
Tier1ReRecoWorkload.setTrustLocationFlag(setLocationFlag)
# Tier1ReRecoWorkload.setSiteWhitelist(self.siteWhitelist)
return Tier1ReRecoWorkload
def createMCWMSpec(self, name='MonteCarloWorkload'):
mcArgs['CouchDBName'] = rerecoArgs["CouchDBName"]
mcArgs["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"])
wmspec = monteCarloWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents=10000)
return wmspec
def createMCWMSpec(self, name='MonteCarloWorkload'):
mcArgs['CouchDBName'] = rerecoArgs["CouchDBName"]
mcArgs["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"])
wmspec = monteCarloWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents=10000)
return wmspec
def createMCWMSpec(self, name='MonteCarloWorkload'):
mcArgs = TaskChainWorkloadFactory.getTestArguments()
mcArgs["CouchDBName"] = rerecoArgs["CouchDBName"]
mcArgs["Task1"]["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"])
wmspec = taskChainWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents=10000)
return wmspec
def createMCWMSpec(self, name='MonteCarloWorkload'):
mcArgs = TaskChainWorkloadFactory.getTestArguments()
mcArgs["CouchDBName"] = rerecoArgs["CouchDBName"]
mcArgs["Task1"]["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"])
wmspec = taskChainWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents=10000)
return wmspec
def createWMSpec(self, name = 'ReRecoWorkload'):
factory = ReRecoWorkloadFactory()
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
wmspec = factory.factoryWorkloadConstruction(name, rerecoArgs)
wmspec.setSpecUrl("/path/to/workload")
wmspec.setSubscriptionInformation(custodialSites = [],
nonCustodialSites = [], autoApproveSites = [],
priority = "Low", custodialSubType = "Move")
return wmspec
def createWMSpec(self, name='ReRecoWorkload'):
factory = ReRecoWorkloadFactory()
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
wmspec = factory.factoryWorkloadConstruction(name, rerecoArgs)
wmspec.setSpecUrl("/path/to/workload")
wmspec.setSubscriptionInformation(custodialSites=[],
nonCustodialSites=[], autoApproveSites=[],
priority="Low", custodialSubType="Move")
return wmspec
def testIgnore0SizeBlocks(self):
"""Ignore blocks with 0 files"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setRunWhitelist([2, 3])
for task in Tier1ReRecoWorkload.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Block(**self.splitArgs), Tier1ReRecoWorkload, task)
def testDataDirectiveFromQueue(self):
"""Test data directive from queue"""
dqmHarvArgs = getRequestArgs()
dqmHarvArgs["DQMConfigCacheID"] = createConfig(dqmHarvArgs["CouchDBName"])
factory = DQMHarvestWorkloadFactory()
DQMHarvWorkload = factory.factoryWorkloadConstruction('DQMHarvestTest', dqmHarvArgs)
DQMHarvWorkload.setSiteWhitelist(["T2_XX_SiteA", "T2_XX_SiteB", "T2_XX_SiteC"])
splitArgs = DQMHarvWorkload.startPolicyParameters()
for task in DQMHarvWorkload.taskIterator():
self.assertRaises(RuntimeError, Dataset(**splitArgs), DQMHarvWorkload, task, {NOT_EXIST_DATASET: []})
def testIgnore0SizeBlocks(self):
"""Ignore blocks with 0 files"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
'ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setRunWhitelist([2, 3])
for task in Tier1ReRecoWorkload.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Block(**self.splitArgs),
Tier1ReRecoWorkload, task)
def testIgnore0SizeBlocks(self):
"""Ignore blocks with 0 files"""
Globals.GlobalParams.setNumOfFilesPerBlock(0)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setRunWhitelist([2, 3])
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
for task in Tier1ReRecoWorkload.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Block(**self.splitArgs), Tier1ReRecoWorkload, task)
def testLumiSplitTier1ReRecoWorkload(self):
"""Tier1 Re-reco workflow split by Lumi"""
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=2)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(2428, unit['Jobs'])
def testDataDirectiveFromQueue(self):
"""Test data directive from queue"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
Dataset(**self.splitArgs)(Tier1ReRecoWorkload, task, {dataset: []})
self.assertRaises(RuntimeError, Dataset(**self.splitArgs), Tier1ReRecoWorkload, task, dbs,
{dataset + '1': []})
def testLumiSplitTier1ReRecoWorkload(self):
"""Tier1 Re-reco workflow split by Lumi"""
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=2)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(2428, unit['Jobs'])
def testDataDirectiveFromQueue(self):
"""Test data directive from queue"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
Dataset(**self.splitArgs)(Tier1ReRecoWorkload, task, {dataset: []})
self.assertRaises(RuntimeError, Dataset(**self.splitArgs), Tier1ReRecoWorkload, task, dbs,
{dataset + '1': []})
def setUp(self):
self.testInit = TestInit('CouchWorkQueueTest')
self.testInit.setLogging()
self.testInit.setupCouch('wq_backend_test_inbox', 'WorkQueue')
self.testInit.setupCouch('wq_backend_test', 'WorkQueue')
self.testInit.setupCouch('wq_backend_test_parent', 'WorkQueue')
self.couch_db = self.testInit.couch.couchServer.connectDatabase('wq_backend_test')
self.backend = WorkQueueBackend(db_url=self.testInit.couchUrl,
db_name='wq_backend_test',
inbox_name='wq_backend_test_inbox',
parentQueue='%s/%s' % (self.testInit.couchUrl, 'wq_backend_test_parent'))
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
def testIgnore0SizeBlocks(self):
"""Ignore blocks with 0 files"""
Globals.GlobalParams.setNumOfFilesPerBlock(0)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
'ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setRunWhitelist([2, 3])
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
for task in Tier1ReRecoWorkload.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Block(**self.splitArgs),
Tier1ReRecoWorkload, task)
def testContinuousSplittingSupport(self):
"""Can modify successfully policies for continuous splitting"""
policyInstance = Block(**self.splitArgs)
self.assertTrue(policyInstance.supportsWorkAddition(),
"Block instance should support continuous splitting")
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
'ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = policyInstance(Tier1ReRecoWorkload, task)
self.assertEqual(47, len(units))
blocks = [] # fill with blocks as we get work units for them
inputs = {}
for unit in units:
blocks.extend(unit['Inputs'].keys())
inputs.update(unit['Inputs'])
self.assertEqual(69, unit['Priority'])
self.assertTrue(1 <= unit['Jobs'])
self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertTrue(1 <= unit['NumberOfLumis'])
self.assertTrue(1 <= unit['NumberOfFiles'])
self.assertTrue(0 <= unit['NumberOfEvents'])
self.assertEqual(
len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
# Modify the spec and task, get first a fresh policy instance
policyInstance = Block(**self.splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
policyInstance.modifyPolicyForWorkAddition(
{'ProcessedInputs': inputs.keys()})
self.assertRaises(WorkQueueNoWorkError, policyInstance,
Tier1ReRecoWorkload, task)
# Run one last time
policyInstance = Block(**self.splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
policyInstance.modifyPolicyForWorkAddition(
{'ProcessedInputs': inputs.keys()})
self.assertRaises(WorkQueueNoWorkError, policyInstance,
Tier1ReRecoWorkload, task)
return
def testLumiSplitTier1ReRecoWorkload(self):
"""Tier1 Re-reco workflow"""
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setStartPolicy('Block', **splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
units, rejectedWork = Block(**splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(47, len(units))
for unit in units:
self.assertTrue(1 <= unit['Jobs'])
self.assertEqual(0, len(rejectedWork))
def getMergeACDCSpec(self, splittingAlgo = 'ParentlessMergeBySize', splittingArgs = {}):
"""
_getMergeACDCSpec_
Get a ACDC spec for the merge task of a ReReco workload
"""
factory = ReRecoWorkloadFactory()
rerecoArgs = ReRecoWorkloadFactory.getTestArguments()
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(self.workflowName, rerecoArgs)
Tier1ReRecoWorkload.truncate('ACDC_%s' % self.workflowName, '/%s/DataProcessing/DataProcessingMergeRECOoutput' % self.workflowName,
self.couchUrl, self.acdcDBName)
Tier1ReRecoWorkload.setJobSplittingParameters('/ACDC_%s/DataProcessingMergeRECOoutput' % self.workflowName,
splittingAlgo, splittingArgs)
return Tier1ReRecoWorkload
def testInvalidSpecs(self):
"""Specs with no work"""
# no dataset
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
processingSpec = factory.factoryWorkloadConstruction(
'testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec).data.input.dataset = None
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueWMSpecError, Block(), processingSpec,
task)
# invalid dbs url
processingSpec = factory.factoryWorkloadConstruction(
'testProcessingInvalid', rerecoArgs)
getFirstTask(
processingSpec
).data.input.dataset.dbsurl = 'wrongprot://dbs.example.com'
for task in processingSpec.taskIterator():
self.assertRaises(DBSReaderError, Block(), processingSpec, task)
# invalid dataset name
# This test is throwing a DBS exception but continuing on:
"""
Message: DbsBadRequest: DBS Server Raised An Error
ModuleName : WMCore.Services.DBS.DBSErrors
MethodName : __init__
ClassInstance : None
FileName : /data/srv/wmagent_ewv/v0.9.94c/sw/slc5_amd64_gcc461/cms/wmagent/0.9.94c/lib/python2.6/site-packages/WMCore/Services/DBS/DBSErrors.py
ClassName : None
LineNumber : 29
ErrorNr : 1002
"""
processingSpec = factory.factoryWorkloadConstruction(
'testProcessingInvalid', rerecoArgs)
getFirstTask(processingSpec
).data.input.dataset.primary = Globals.NOT_EXIST_DATASET
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec,
task)
# invalid run whitelist
processingSpec = factory.factoryWorkloadConstruction(
'testProcessingInvalid', rerecoArgs)
processingSpec.setRunWhitelist([666]) # not in this dataset
for task in processingSpec.taskIterator():
self.assertRaises(WorkQueueNoWorkError, Block(), processingSpec,
task)
def testLumiSplitTier1ReRecoWorkload(self):
"""Tier1 Re-reco workflow"""
splitArgs = dict(SliceType='NumberOfLumis', SliceSize=1)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
'ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setStartPolicy('Block', **splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
units, rejectedWork = Block(**splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(47, len(units))
for unit in units:
self.assertTrue(1 <= unit['Jobs'])
self.assertEqual(0, len(rejectedWork))
def setUp(self):
self.testInit = TestInit('CouchWorkQueueTest')
self.testInit.setLogging()
self.testInit.setupCouch('wq_backend_test_inbox', 'WorkQueue')
self.testInit.setupCouch('wq_backend_test', 'WorkQueue')
self.testInit.setupCouch('wq_backend_test_parent', 'WorkQueue')
self.couch_db = self.testInit.couch.couchServer.connectDatabase(
'wq_backend_test')
self.backend = WorkQueueBackend(
db_url=self.testInit.couchUrl,
db_name='wq_backend_test',
inbox_name='wq_backend_test_inbox',
parentQueue='%s/%s' %
(self.testInit.couchUrl, 'wq_backend_test_parent'))
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
self.processingSpec = rerecoWorkload('testProcessing', rerecoArgs)
def testLumiSplitTier1ReRecoWorkload(self):
"""Tier1 Re-reco workflow split by Lumi"""
splitArgs = dict(SliceType = 'NumberOfLumis', SliceSize = 2)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs)
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(4, unit['Jobs'])
def testLumiMask(self):
"""Lumi mask test"""
rerecoArgs2 = {}
rerecoArgs2.update(rerecoArgs)
rerecoArgs2["ConfigCacheID"] = createConfig(rerecoArgs2["CouchDBName"])
factory = ReRecoWorkloadFactory()
dummyWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs2)
# Block blacklist
lumiWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs2)
task = getFirstTask(lumiWorkload)
#task.data.input.splitting.runs = ['1']
task.data.input.splitting.runs = ['180992']
task.data.input.splitting.lumis = ['1,1']
units, rejectedWork = Block(**self.splitArgs)(lumiWorkload, task)
self.assertEqual(len(units), 1)
self.assertEqual(len(rejectedWork), 46)
def testLumiSplitTier1ReRecoWorkload(self):
"""Tier1 Re-reco workflow split by Lumi"""
splitArgs = dict(SliceType = 'NumberOfLumis', SliceSize = 2)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.setStartPolicy('Dataset', **splitArgs)
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = Dataset(**splitArgs)(Tier1ReRecoWorkload, task)
self.assertEqual(1, len(units))
for unit in units:
self.assertEqual(4, unit['Jobs'])
