def testFilter(self):
"""
Test filtering of a list of lumis
"""
runsAndLumis = {
1: range(1, 34) + [35] + range(37, 48),
2: range(49, 76) + range(77, 131) + range(133, 137)
}
completeList = zip([1]*150, range(1, 150)) + \
zip([2]*150, range(1, 150)) + \
zip([3]*150, range(1, 150))
smallList = zip([1]*50, range(1, 10)) + zip([2]*50, range(50, 70))
overlapList = zip([1]*150, range(30, 40)) + \
zip([2]*150, range(60, 80))
overlapRes = zip([1]*9, range(30, 34)) + [(1, 35)] + \
zip([1]*9, range(37, 40)) + \
zip([2]*30, range(60, 76)) + \
zip([2]*9, range(77, 80))
runLister = LumiList(runsAndLumis = runsAndLumis)
# Test a list to be filtered which is a superset of constructed list
filterComplete = runLister.filterLumis(completeList)
# Test a list to be filtered which is a subset of constructed list
filterSmall = runLister.filterLumis(smallList)
# Test a list to be filtered which is neither
filterOverlap = runLister.filterLumis(overlapList)
self.assertTrue(filterComplete == runLister.getLumis())
self.assertTrue(filterSmall == smallList)
self.assertTrue(filterOverlap == overlapRes)
def addJobs(self, jobs):
if self.algo == 'FileBased':
self.lumisPerJob += [
sum([x.get('lumiCount', 0) for x in job['input_files']])
for job in jobs
]
self.eventsPerJob += [
sum([x['events'] for x in job['input_files']]) for job in jobs
]
self.filesPerJob += [len(job['input_files']) for job in jobs]
elif self.algo == 'EventBased':
self.lumisPerJob += [
job['mask']['LastLumi'] - job['mask']['FirstLumi']
for job in jobs
]
self.eventsPerJob += [
job['mask']['LastEvent'] - job['mask']['FirstEvent']
for job in jobs
]
else:
for job in jobs:
avgEventsPerLumi = sum([
f['avgEvtsPerLumi'] for f in job['input_files']
]) / float(len(job['input_files']))
lumis = LumiList(compactList=job['mask']['runAndLumis'])
self.lumisPerJob.append(len(lumis.getLumis()))
self.eventsPerJob.append(avgEventsPerLumi *
self.lumisPerJob[-1])
def testFilter(self):
"""
Test filtering of a list of lumis
"""
runsAndLumis = {
1: range(1, 34) + [35] + range(37, 48),
2: range(49, 76) + range(77, 131) + range(133, 137)
}
completeList = list(zip([1]*150, range(1, 150))) + \
list(zip([2]*150, range(1, 150))) + \
list(zip([3]*150, range(1, 150)))
smallList = list(zip([1]*50, range(1, 10))) + list(zip([2]*50, range(50, 70)))
overlapList = list(zip([1]*150, range(30, 40))) + \
list(zip([2]*150, range(60, 80)))
overlapRes = list(zip([1]*9, range(30, 34))) + [(1, 35)] + \
list(zip([1]*9, range(37, 40))) + \
list(zip([2]*30, range(60, 76))) + \
list(zip([2]*9, range(77, 80)))
runLister = LumiList(runsAndLumis = runsAndLumis)
# Test a list to be filtered which is a superset of constructed list
filterComplete = runLister.filterLumis(completeList)
# Test a list to be filtered which is a subset of constructed list
filterSmall = runLister.filterLumis(smallList)
# Test a list to be filtered which is neither
filterOverlap = runLister.filterLumis(overlapList)
self.assertTrue(filterComplete == runLister.getLumis())
self.assertTrue(filterSmall == smallList)
self.assertTrue(filterOverlap == overlapRes)
def testWithMaskedBlocks(self):
"""
_testWithMaskedBlocks_
Test job splitting with masked blocks
"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
'ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
dummyDataset = task.inputDataset()
task.data.input.splitting.runs = [181061, 180899]
task.data.input.splitting.lumis = ['1,50,60,70', '1,1']
lumiMask = LumiList(compactList={
'206371': [[1, 50], [60, 70]],
'180899': [[1, 1]],
})
units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload,
task)
nLumis = 0
for unit in units:
nLumis += unit['NumberOfLumis']
self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testWithMaskedBlocks(self):
"""
_testWithMaskedBlocks_
Test job splitting with masked blocks
"""
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
dummyDataset = task.inputDataset()
task.data.input.splitting.runs = [181061, 180899]
task.data.input.splitting.lumis = ['1,50,60,70', '1,1']
lumiMask = LumiList(compactList={'206371': [[1, 50], [60, 70]], '180899': [[1, 1]], })
units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task)
nLumis = 0
for unit in units:
nLumis += unit['NumberOfLumis']
self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testWithMaskedBlocks(self):
"""
_testWithMaskedBlocks_
Test job splitting with masked blocks
"""
Globals.GlobalParams.setNumOfRunsPerFile(3)
Globals.GlobalParams.setNumOfLumisPerBlock(5)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
inputDataset = task.inputDataset()
task.data.input.splitting.runs = [181061, 180899]
task.data.input.splitting.lumis = ['1,50,60,70', '1,1']
lumiMask = LumiList(compactList = {'206371': [[1, 50], [60,70]], '180899':[[1,1]], } )
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
units, rejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task)
nLumis = 0
for unit in units:
nLumis += unit['NumberOfLumis']
self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testWithMaskedBlocks(self):
"""
_testWithMaskedBlocks_
Test job splitting with masked blocks
"""
Tier1ReRecoWorkload = rerecoWorkload(
'ReRecoWorkload',
rerecoArgs,
assignArgs={'SiteWhitelist': ['T2_XX_SiteA']})
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
dummyDataset = task.inputDataset()
task.data.input.splitting.runs = [181061, 180899]
task.data.input.splitting.lumis = ['1,50,60,70', '1,1']
lumiMask = LumiList(compactList={
'206371': [[1, 50], [60, 70]],
'180899': [[1, 1]],
})
units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload,
task)
nLumis = 0
for unit in units:
nLumis += unit['NumberOfLumis']
self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testDuplicates(self):
"""
Test a list with lots of duplicates
"""
result = zip([1]*100, range(1, 34) + range(37, 48))
lumis = zip([1]*100, range(1, 34) + range(37, 48) + range(5, 25))
lister = LumiList(lumis = lumis)
self.assertTrue(lister.getLumis() == result)
def testDuplicates(self):
"""
Test a list with lots of duplicates
"""
result = list(zip([1]*100, range(1, 34) + range(37, 48)))
lumis = list(zip([1]*100, range(1, 34) + range(37, 48) + range(5, 25)))
lister = LumiList(lumis = lumis)
self.assertTrue(lister.getLumis() == result)
def testNull(self):
"""
Test a null list
"""
runLister = LumiList(lumis = None)
self.assertTrue(runLister.getCMSSWString() == '')
self.assertTrue(runLister.getLumis() == [])
self.assertTrue(runLister.getCompactList() == {})
def testNull(self):
"""
Test a null list
"""
runLister = LumiList(lumis = None)
self.assertTrue(runLister.getCMSSWString() == '')
self.assertTrue(runLister.getLumis() == [])
self.assertTrue(runLister.getCompactList() == {})
def addJobs(self, jobs):
if self.algo == 'FileBased':
self.lumisPerJob += [sum([x.get('lumiCount', 0) for x in job['input_files']]) for job in jobs]
self.eventsPerJob += [sum([x['events'] for x in job['input_files']]) for job in jobs]
elif self.algo == 'EventBased':
self.lumisPerJob += [job['mask']['LastLumi'] - job['mask']['FirstLumi'] for job in jobs]
self.eventsPerJob += [job['mask']['LastEvent'] - job['mask']['FirstEvent'] for job in jobs]
else:
for job in jobs:
avgEventsPerLumi = sum([f['avgEvtsPerLumi'] for f in job['input_files']])/float(len(job['input_files']))
lumis = LumiList(compactList=job['mask']['runAndLumis'])
self.lumisPerJob.append(len(lumis.getLumis()))
self.eventsPerJob.append(avgEventsPerLumi * self.lumisPerJob[-1])
def testWithMaskedBlocks(self):
"""
_testWithMaskedBlocks_
Test job splitting with masked blocks
"""
Tier1ReRecoWorkload = rerecoWorkload('ReRecoWorkload', rerecoArgs,
assignArgs={'SiteWhitelist': ['T2_XX_SiteA']})
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
dummyDataset = task.inputDataset()
task.data.input.splitting.runs = [181061, 180899]
task.data.input.splitting.lumis = ['1,50,60,70', '1,1']
lumiMask = LumiList(compactList={'206371': [[1, 50], [60, 70]], '180899': [[1, 1]], })
units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task)
nLumis = 0
for unit in units:
nLumis += unit['NumberOfLumis']
self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testWithMaskedBlocks(self):
"""
_testWithMaskedBlocks_
Test job splitting with masked blocks
"""
Globals.GlobalParams.setNumOfRunsPerFile(3)
Globals.GlobalParams.setNumOfLumisPerBlock(5)
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
'ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
task = getFirstTask(Tier1ReRecoWorkload)
inputDataset = task.inputDataset()
task.data.input.splitting.runs = [181061, 180899]
task.data.input.splitting.lumis = ['1,50,60,70', '1,1']
lumiMask = LumiList(compactList={
'206371': [[1, 50], [60, 70]],
'180899': [[1, 1]],
})
dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
units, rejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload,
task)
nLumis = 0
for unit in units:
nLumis += unit['NumberOfLumis']
self.assertEqual(len(lumiMask.getLumis()), nLumis)