def testRuns(self):
"""
Test constucting from run and list of lumis
"""
runsAndLumis = {
1: range(1, 34) + [35] + range(37, 48),
2: range(49, 76) + range(77, 131) + range(133, 137)
}
runsAndLumis2 = {
'1': range(1, 34) + [35] + range(37, 48),
'2': range(49, 76) + range(77, 131) + range(133, 137)
}
blank = {
'1': [],
'2': []
}
jsonLister = LumiList(filename = 'lumiTest.json')
jsonString = jsonLister.getCMSSWString()
jsonList = jsonLister.getCompactList()
runLister = LumiList(runsAndLumis = runsAndLumis)
runString = runLister.getCMSSWString()
runList = runLister.getCompactList()
runLister2 = LumiList(runsAndLumis = runsAndLumis2)
runList2 = runLister2.getCompactList()
runLister3 = LumiList(runsAndLumis = blank)
self.assertTrue(jsonString == runString)
self.assertTrue(jsonList == runList)
self.assertTrue(runList2 == runList)
self.assertTrue(len(runLister3) == 0)
def testAnd(self):
"""
a&b for lots of cases
"""
alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
'2' : range(6,20) + range (30,40),
'3' : range(10,20) + range (30,40) + range(50,60),
'4' : range(1,100),
}
blumis = {'1' : range(1,6) + range(12,13) + range(16,25) + range(25,40) + range(40,50) + range(33,36),
'2' : range(10,35),
'3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
'5' : range(1,100),
}
result = {'1' : range(2,6) + range(12,13) + range(16,20) + range(31,39) + range(45,49),
'2' : range(10,20) + range(30,35),
'3' : range(10,15) + range(35,40) + range(50,51)+ range(59,60),
}
a = LumiList(runsAndLumis = alumis)
b = LumiList(runsAndLumis = blumis)
r = LumiList(runsAndLumis = result)
self.assertTrue((a&b).getCMSSWString() == r.getCMSSWString())
self.assertTrue((a&b).getCMSSWString() == (b&a).getCMSSWString())
self.assertTrue((a|b).getCMSSWString() != r.getCMSSWString())
def testRemoveSelect(self):
"""
a-b for lots of cases
"""
alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
'2' : range(6,20) + range (30,40),
'3' : range(10,20) + range (30,40) + range(50,60),
'4' : range(10,20) + range (30,80),
}
result = {'2' : range(6,20) + range (30,40),
'4' : range(10,20) + range (30,80),
}
rem = LumiList(runsAndLumis = alumis)
sel = LumiList(runsAndLumis = alumis)
res = LumiList(runsAndLumis = result)
rem.removeRuns([1,3])
sel.selectRuns([2,4])
self.assertTrue(rem.getCMSSWString() == res.getCMSSWString())
self.assertTrue(sel.getCMSSWString() == res.getCMSSWString())
self.assertTrue(sel.getCMSSWString() == rem.getCMSSWString())
def testOr(self):
"""
a|b for lots of cases
"""
alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
'2' : range(6,20) + range (30,40),
'3' : range(10,20) + range (30,40) + range(50,60),
}
blumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
'2' : range(10,35),
'3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
}
clumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
'2' : range(10,35),
}
result = {'1' : range(2,20) + range(31,39) + range(45,49) + range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
'2' : range(6,20) + range (30,40) + range(10,35),
'3' : range(10,20) + range (30,40) + range(50,60) + range(10,15) + range(35,40) + range(45,51) + range(59,70),
}
a = LumiList(runsAndLumis = alumis)
b = LumiList(runsAndLumis = blumis)
c = LumiList(runsAndLumis = blumis)
r = LumiList(runsAndLumis = result)
self.assertTrue((a|b).getCMSSWString() == r.getCMSSWString())
self.assertTrue((a|b).getCMSSWString() == (b|a).getCMSSWString())
self.assertTrue((a|b).getCMSSWString() == (a+b).getCMSSWString())
# Test list constuction (faster)
multiple = [alumis, blumis, clumis]
easy = LumiList(runsAndLumis = multiple)
hard = a + b
hard += c
self.assertTrue(hard.getCMSSWString() == easy.getCMSSWString())
def testAnd(self):
"""
a&b for lots of cases
"""
alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
'2' : range(6,20) + range (30,40),
'3' : range(10,20) + range (30,40) + range(50,60),
'4' : range(1,100),
}
blumis = {'1' : range(1,6) + range(12,13) + range(16,25) + range(25,40) + range(40,50) + range(33,36),
'2' : range(10,35),
'3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
'5' : range(1,100),
}
result = {'1' : range(2,6) + range(12,13) + range(16,20) + range(31,39) + range(45,49),
'2' : range(10,20) + range(30,35),
'3' : range(10,15) + range(35,40) + range(50,51)+ range(59,60),
}
a = LumiList(runsAndLumis = alumis)
b = LumiList(runsAndLumis = blumis)
r = LumiList(runsAndLumis = result)
self.assertTrue((a&b).getCMSSWString() == r.getCMSSWString())
self.assertTrue((a&b).getCMSSWString() == (b&a).getCMSSWString())
self.assertTrue((a|b).getCMSSWString() != r.getCMSSWString())
def testOr(self):
"""
a|b for lots of cases
"""
alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
'2' : range(6,20) + range (30,40),
'3' : range(10,20) + range (30,40) + range(50,60),
}
blumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
'2' : range(10,35),
'3' : range(10,15) + range(35,40) + range(45,51) + range(59,70),
}
clumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
'2' : range(10,35),
}
result = {'1' : range(2,20) + range(31,39) + range(45,49) + range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80),
'2' : range(6,20) + range (30,40) + range(10,35),
'3' : range(10,20) + range (30,40) + range(50,60) + range(10,15) + range(35,40) + range(45,51) + range(59,70),
}
a = LumiList(runsAndLumis = alumis)
b = LumiList(runsAndLumis = blumis)
c = LumiList(runsAndLumis = blumis)
r = LumiList(runsAndLumis = result)
self.assertTrue((a|b).getCMSSWString() == r.getCMSSWString())
self.assertTrue((a|b).getCMSSWString() == (b|a).getCMSSWString())
self.assertTrue((a|b).getCMSSWString() == (a+b).getCMSSWString())
# Test list constuction (faster)
multiple = [alumis, blumis, clumis]
easy = LumiList(runsAndLumis = multiple)
hard = a + b
hard += c
self.assertTrue(hard.getCMSSWString() == easy.getCMSSWString())
def testRuns(self):
"""
Test constucting from run and list of lumis
"""
runsAndLumis = {
1: range(1, 34) + [35] + range(37, 48),
2: range(49, 76) + range(77, 131) + range(133, 137)
}
runsAndLumis2 = {
'1': range(1, 34) + [35] + range(37, 48),
'2': range(49, 76) + range(77, 131) + range(133, 137)
}
blank = {
'1': [],
'2': []
}
jsonLister = LumiList(filename = 'lumiTest.json')
jsonString = jsonLister.getCMSSWString()
jsonList = jsonLister.getCompactList()
runLister = LumiList(runsAndLumis = runsAndLumis)
runString = runLister.getCMSSWString()
runList = runLister.getCompactList()
runLister2 = LumiList(runsAndLumis = runsAndLumis2)
runList2 = runLister2.getCompactList()
runLister3 = LumiList(runsAndLumis = blank)
self.assertTrue(jsonString == runString)
self.assertTrue(jsonList == runList)
self.assertTrue(runList2 == runList)
self.assertTrue(len(runLister3) == 0)
def testRemoveSelect(self):
"""
a-b for lots of cases
"""
alumis = {'1' : range(2,20) + range(31,39) + range(45,49),
'2' : range(6,20) + range (30,40),
'3' : range(10,20) + range (30,40) + range(50,60),
'4' : range(10,20) + range (30,80),
}
result = {'2' : range(6,20) + range (30,40),
'4' : range(10,20) + range (30,80),
}
rem = LumiList(runsAndLumis = alumis)
sel = LumiList(runsAndLumis = alumis)
res = LumiList(runsAndLumis = result)
rem.removeRuns([1,3])
sel.selectRuns([2,4])
self.assertTrue(rem.getCMSSWString() == res.getCMSSWString())
self.assertTrue(sel.getCMSSWString() == res.getCMSSWString())
self.assertTrue(sel.getCMSSWString() == rem.getCMSSWString())
def testSubtract(self):
"""
a-b for lots of cases
"""
alumis = {
'1': range(2, 20) + range(31, 39) + range(45, 49),
'2': range(6, 20) + range(30, 40),
'3': range(10, 20) + range(30, 40) + range(50, 60),
}
blumis = {
'1':
range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) +
range(33, 36),
'2':
range(10, 35),
'3':
range(10, 15) + range(35, 40) + range(45, 51) + range(59, 70),
}
clumis = {
'1':
range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) +
range(33, 36),
'2':
range(10, 35),
}
result = {
'1': range(6, 12) + range(13, 16) + range(31, 33) + range(36, 39),
'2': range(6, 10) + range(35, 40),
'3': range(15, 20) + range(30, 35) + range(51, 59),
}
result2 = {
'1': range(6, 12) + range(13, 16) + range(31, 33) + range(36, 39),
'2': range(6, 10) + range(35, 40),
'3': range(10, 20) + range(30, 40) + range(50, 60),
}
a = LumiList(runsAndLumis=alumis)
b = LumiList(runsAndLumis=blumis)
c = LumiList(runsAndLumis=clumis)
r = LumiList(runsAndLumis=result)
r2 = LumiList(runsAndLumis=result2)
self.assertTrue((a - b).getCMSSWString() == r.getCMSSWString())
self.assertTrue((a - b).getCMSSWString() != (b - a).getCMSSWString())
# Test where c is missing runs from a
self.assertTrue((a - c).getCMSSWString() == r2.getCMSSWString())
self.assertTrue((a - c).getCMSSWString() != (c - a).getCMSSWString())
# Test empty lists
self.assertTrue(str(a - a) == '{}')
self.assertTrue(len(a - a) == 0)
def testNull(self):
"""
Test a null list
"""
runLister = LumiList(lumis = None)
self.assertTrue(runLister.getCMSSWString() == '')
self.assertTrue(runLister.getLumis() == [])
self.assertTrue(runLister.getCompactList() == {})
def testNull(self):
"""
Test a null list
"""
runLister = LumiList(lumis = None)
self.assertTrue(runLister.getCMSSWString() == '')
self.assertTrue(runLister.getLumis() == [])
self.assertTrue(runLister.getCompactList() == {})
def testList(self):
"""
Test constucting from list of pairs
"""
listLs1 = range(1, 34) + [35] + range(37, 48)
listLs2 = range(49, 76) + range(77, 131) + range(133, 137)
lumis = list(zip([1]*100, listLs1)) + list(zip([2]*100, listLs2))
jsonLister = LumiList(filename = 'lumiTest.json')
jsonString = jsonLister.getCMSSWString()
jsonList = jsonLister.getCompactList()
pairLister = LumiList(lumis = lumis)
pairString = pairLister.getCMSSWString()
pairList = pairLister.getCompactList()
self.assertTrue(jsonString == pairString)
self.assertTrue(jsonList == pairList)
def testWMagentFormat(self):
"""
Test the constuction of the double-string-based version of lumi lists
used in some places by WMAgent
"""
clumis = {
'1':
list(range(2, 20)) + list(range(31, 39)) + list(range(45, 49)),
'2': list(range(6, 20)) + list(range(30, 40)),
'3':
list(range(10, 20)) + list(range(30, 40)) + list(range(50, 60)),
'4': list(range(1, 100)),
}
runs = [1, 2, 3, 4]
lumis = ['2,19,31,38,45,48', '6,19,30,39', '10,19,30,39,50,59', '1,99']
c = LumiList(runsAndLumis=clumis)
w = LumiList(wmagentFormat=(runs, lumis))
self.assertEqual(w.getCMSSWString(), c.getCMSSWString())
clumis = {'12': [4, 5]}
runs = [12]
lumis = ['4,5']
c = LumiList(runsAndLumis=clumis)
w = LumiList(wmagentFormat=(runs, lumis))
self.assertEqual(w.getCMSSWString(), c.getCMSSWString())
aruns = [1]
alumis = ['1,10,11,20']
bruns = [1]
blumis = ['1,20']
self.assertEqual(
LumiList(wmagentFormat=(aruns, alumis)).getCMSSWString(),
LumiList(wmagentFormat=(bruns, blumis)).getCMSSWString())
with self.assertRaises(RuntimeError):
w = LumiList(wmagentFormat=([1, 2, 3])) # No lumis
with self.assertRaises(RuntimeError):
w = LumiList(wmagentFormat=([1], [
'1,2,3'
])) # Need twice as many lumis as runs
def testList(self):
"""
Test constucting from list of pairs
"""
listLs1 = range(1, 34) + [35] + range(37, 48)
listLs2 = range(49, 76) + range(77, 131) + range(133, 137)
lumis = zip([1]*100, listLs1) + zip([2]*100, listLs2)
jsonLister = LumiList(filename = 'lumiTest.json')
jsonString = jsonLister.getCMSSWString()
jsonList = jsonLister.getCompactList()
pairLister = LumiList(lumis = lumis)
pairString = pairLister.getCMSSWString()
pairList = pairLister.getCompactList()
self.assertTrue(jsonString == pairString)
self.assertTrue(jsonList == pairList)
def testAddLumiMask(self):
"""
_testAddLumiMask_
Verify that setting and getting the lumiMask objects for a task works correctly.
Do a round trip of a typical lumi mask
"""
testTask = makeWMTask("TestTask")
lumiMask = LumiList(compactList = {
'1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]],
'2':[[1,45]],
'3':[[1,45],[50,80]],
})
testTask.setLumiMask(lumiMask = lumiMask.getCompactList())
outMask = LumiList(compactList = testTask.getLumiMask())
self.assertEqual(lumiMask.getCMSSWString(), outMask.getCMSSWString())
return
def testSubtract(self):
"""
a-b for lots of cases
"""
alumis = {
"1": range(2, 20) + range(31, 39) + range(45, 49),
"2": range(6, 20) + range(30, 40),
"3": range(10, 20) + range(30, 40) + range(50, 60),
}
blumis = {
"1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(33, 36),
"2": range(10, 35),
"3": range(10, 15) + range(35, 40) + range(45, 51) + range(59, 70),
}
clumis = {"1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(33, 36), "2": range(10, 35)}
result = {
"1": range(6, 12) + range(13, 16) + range(31, 33) + range(36, 39),
"2": range(6, 10) + range(35, 40),
"3": range(15, 20) + range(30, 35) + range(51, 59),
}
result2 = {
"1": range(6, 12) + range(13, 16) + range(31, 33) + range(36, 39),
"2": range(6, 10) + range(35, 40),
"3": range(10, 20) + range(30, 40) + range(50, 60),
}
a = LumiList(runsAndLumis=alumis)
b = LumiList(runsAndLumis=blumis)
c = LumiList(runsAndLumis=clumis)
r = LumiList(runsAndLumis=result)
r2 = LumiList(runsAndLumis=result2)
self.assertTrue((a - b).getCMSSWString() == r.getCMSSWString())
self.assertTrue((a - b).getCMSSWString() != (b - a).getCMSSWString())
# Test where c is missing runs from a
self.assertTrue((a - c).getCMSSWString() == r2.getCMSSWString())
self.assertTrue((a - c).getCMSSWString() != (c - a).getCMSSWString())
# Test empty lists
self.assertTrue(str(a - a) == "{}")
self.assertTrue(len(a - a) == 0)
def testAddLumiMask(self):
"""
_testAddLumiMask_
Verify that setting and getting the lumiMask objects for a task works correctly.
Do a round trip of a typical lumi mask
"""
testTask = makeWMTask("TestTask")
lumiMask = LumiList(
compactList={
'1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130],
[133, 136]],
'2': [[1, 45]],
'3': [[1, 45], [50, 80]],
})
testTask.setLumiMask(lumiMask=lumiMask.getCompactList())
outMask = LumiList(compactList=testTask.getLumiMask())
self.assertEqual(lumiMask.getCMSSWString(), outMask.getCMSSWString())
return
def testLumisWithEvents(self):
"""Make sure that runs and lumis with event counts as used in CRAB3 works
"""
clumis = {'1': list(range(2, 20)) + list(range(31, 39)) + list(range(45, 49)),
'2': list(range(6, 20)) + list(range(30, 40)),
'3': list(range(10, 20)) + list(range(30, 40)) + list(range(50, 60)),
'4': list(range(1, 100)),
}
# create a lumilist like {'21' : {'2':None ...} ...}
lumis = {}
for run, ls in viewitems(clumis):
newrun = str(int(run) + 20)
lumis[newrun] = {str(l): None for l in ls}
c1 = LumiList(runsAndLumis=clumis)
w1 = LumiList(runsAndLumis=lumis)
c1 += w1
c2 = LumiList(runsAndLumis=clumis)
w2 = LumiList(runsAndLumis=lumis)
w2 += c2
self.assertEqual(c1.getCMSSWString(), w2.getCMSSWString())
def testWMagentFormat(self):
"""
Test the constuction of the double-string-based version of lumi lists
used in some places by WMAgent
"""
clumis = {'1': range(2, 20) + range(31, 39) + range(45, 49),
'2': range(6, 20) + range(30, 40),
'3': range(10, 20) + range(30, 40) + range(50, 60),
'4': range(1, 100),
}
runs = [1, 2, 3, 4]
lumis = ['2,19,31,38,45,48', '6,19,30,39', '10,19,30,39,50,59', '1,99']
c = LumiList(runsAndLumis=clumis)
w = LumiList(wmagentFormat=(runs, lumis))
self.assertEqual(w.getCMSSWString(), c.getCMSSWString())
clumis = {'12': [4, 5]}
runs = [12]
lumis = ['4,5']
c = LumiList(runsAndLumis=clumis)
w = LumiList(wmagentFormat=(runs, lumis))
self.assertEqual(w.getCMSSWString(), c.getCMSSWString())
aruns = [1]
alumis = ['1,10,11,20']
bruns = [1]
blumis = ['1,20']
self.assertEqual(LumiList(wmagentFormat=(aruns, alumis)).getCMSSWString(),
LumiList(wmagentFormat=(bruns, blumis)).getCMSSWString())
with self.assertRaises(RuntimeError):
w = LumiList(wmagentFormat=([1,2,3])) # No lumis
with self.assertRaises(RuntimeError):
w = LumiList(wmagentFormat=([1], ['1,2,3'])) # Need twice as many lumis as runs
def notestRead(self):
"""
Test reading from JSON
"""
exString = "1:1-1:33,1:35,1:37-1:47,2:49-2:75,2:77-2:130,2:133-2:136"
exDict = {"1": [[1, 33], [35, 35], [37, 47]], "2": [[49, 75], [77, 130], [133, 136]]}
exVLBR = cms.VLuminosityBlockRange("1:1-1:33", "1:35", "1:37-1:47", "2:49-2:75", "2:77-2:130", "2:133-2:136")
jsonList = LumiList(filename="lumiTest.json")
lumiString = jsonList.getCMSSWString()
lumiList = jsonList.getCompactList()
lumiVLBR = jsonList.getVLuminosityBlockRange(True)
self.assertTrue(lumiString == exString)
self.assertTrue(lumiList == exDict)
self.assertTrue(lumiVLBR == exVLBR)
def notestRead(self):
"""
Test reading from JSON
"""
exString = "1:1-1:33,1:35,1:37-1:47,2:49-2:75,2:77-2:130,2:133-2:136"
exDict = {'1': [[1, 33], [35, 35], [37, 47]],
'2': [[49, 75], [77, 130], [133, 136]]}
exVLBR = cms.VLuminosityBlockRange('1:1-1:33', '1:35', '1:37-1:47', '2:49-2:75', '2:77-2:130', '2:133-2:136')
jsonList = LumiList(filename = 'lumiTest.json')
lumiString = jsonList.getCMSSWString()
lumiList = jsonList.getCompactList()
lumiVLBR = jsonList.getVLuminosityBlockRange(True)
self.assertTrue(lumiString == exString)
self.assertTrue(lumiList == exDict)
self.assertTrue(lumiVLBR == exVLBR)
def testOr(self):
"""
a|b for lots of cases
"""
alumis = {
"1": range(2, 20) + range(31, 39) + range(45, 49),
"2": range(6, 20) + range(30, 40),
"3": range(10, 20) + range(30, 40) + range(50, 60),
}
blumis = {
"1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(39, 80),
"2": range(10, 35),
"3": range(10, 15) + range(35, 40) + range(45, 51) + range(59, 70),
}
clumis = {"1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(39, 80), "2": range(10, 35)}
result = {
"1": range(2, 20)
+ range(31, 39)
+ range(45, 49)
+ range(1, 6)
+ range(12, 13)
+ range(16, 30)
+ range(40, 50)
+ range(39, 80),
"2": range(6, 20) + range(30, 40) + range(10, 35),
"3": range(10, 20)
+ range(30, 40)
+ range(50, 60)
+ range(10, 15)
+ range(35, 40)
+ range(45, 51)
+ range(59, 70),
}
a = LumiList(runsAndLumis=alumis)
b = LumiList(runsAndLumis=blumis)
c = LumiList(runsAndLumis=blumis)
r = LumiList(runsAndLumis=result)
self.assertTrue((a | b).getCMSSWString() == r.getCMSSWString())
self.assertTrue((a | b).getCMSSWString() == (b | a).getCMSSWString())
self.assertTrue((a | b).getCMSSWString() == (a + b).getCMSSWString())
def testLumiMask(self):
"""
_testLumiMask_
Test that we can use a lumi-mask to filter good runs/lumis.
"""
splitter = SplitterFactory()
# Create 3 files with 100 events per lumi:
# - file1 with 1 run of 8 lumis
# - file2 with 2 runs of 2 lumis each
# - file3 with 1 run of 5 lumis
fileA = File(lfn="/this/is/file1", size=1000, events=800)
fileB = File(lfn="/this/is/file2", size=1000, events=400)
fileC = File(lfn="/this/is/file3", size=1000, events=500)
lumiListA = []
for lumi in range(8):
lumiListA.append(10 + lumi)
fileA.addRun(Run(1, *lumiListA))
fileA.setLocation("somese.cern.ch")
lumiListB1 = []
lumiListB2 = []
for lumi in range(2):
lumiListB1.append(20 + lumi)
lumiListB2.append(30 + lumi)
fileB.addRun(Run(2, *lumiListB1))
fileB.addRun(Run(3, *lumiListB2))
fileB.setLocation("somese.cern.ch")
lumiListC = []
for lumi in range(5):
lumiListC.append(40 + lumi)
fileC.addRun(Run(4, *lumiListC))
fileC.setLocation("somese.cern.ch")
testFileset = Fileset(name='Fileset')
testFileset.addFile(fileA)
testFileset.addFile(fileB)
testFileset.addFile(fileC)
testSubscription = Subscription(fileset=testFileset,
workflow=self.testWorkflow,
split_algo="EventAwareLumiByWork",
type="Processing")
jobFactory = splitter(package="WMCore.DataStructs",
subscription=testSubscription)
# Use a lumi-mask = {1: [[10,14]], 2: [[20,21]], 4: [[40,41]]}
jobGroups = jobFactory(halt_job_on_file_boundaries=False,
splitOnRun=False,
events_per_job=850,
runs=['1', '2', '4'],
lumis=['10,14', '20,21', '40,41'],
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1, "There should be only one job group")
jobs = jobGroups[0].jobs
self.assertEqual(len(jobs), 2, "Two jobs must be in the jobgroup")
processedLumis = LumiList()
for job in jobs:
processedLumis += LumiList(compactList=job['mask'].getRunAndLumis())
correctLumis = LumiList(compactList={1: [[10, 14]], 2: [[20, 21]], 4: [[40, 41]]})
self.assertEqual(processedLumis.getCMSSWString(), correctLumis.getCMSSWString())
def testGetLumiWhitelist(self):
"""
_testGetLumiWhitelist_
Verify that the ACDC whitelist generation code works correctly. We'll
add jobs with the following lumi info:
# Run 1, lumis [1, 2, 3], [4, 6], [7], [9], [11, 12]
# Run 2, lumis [5, 6, 7], [10, 11, 12], [15]
# Run 3, lumis [20]
And should get out a whitelist that looks like this:
{"1": [[1, 4], [6, 7], [9, 9], [11, 12]],
"2": [[5, 7], [10, 12], [15, 15]],
"3": [[20, 20]]}
"""
dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc")
testFileA = File(lfn=makeUUID(), size=1024, events=1024)
testFileA.addRun(Run(1, 1, 2))
testFileB = File(lfn=makeUUID(), size=1024, events=1024)
testFileB.addRun(Run(1, 3))
testJobA = self.getMinimalJob()
testJobA.addFile(testFileA)
testJobA.addFile(testFileB)
testFileC = File(lfn=makeUUID(), size=1024, events=1024)
testFileC.addRun(Run(1, 4, 6))
testJobB = self.getMinimalJob()
testJobB.addFile(testFileC)
testFileD = File(lfn=makeUUID(), size=1024, events=1024)
testFileD.addRun(Run(1, 7))
testJobC = self.getMinimalJob()
testJobC.addFile(testFileD)
testFileE = File(lfn=makeUUID(), size=1024, events=1024)
testFileE.addRun(Run(1, 11, 12))
testJobD = self.getMinimalJob()
testJobD.addFile(testFileE)
testFileF = File(lfn=makeUUID(), size=1024, events=1024)
testFileF.addRun(Run(2, 5, 6, 7))
testJobE = self.getMinimalJob()
testJobE.addFile(testFileF)
testFileG = File(lfn=makeUUID(), size=1024, events=1024)
testFileG.addRun(Run(2, 10, 11, 12))
testJobF = self.getMinimalJob()
testJobF.addFile(testFileG)
testFileH = File(lfn=makeUUID(), size=1024, events=1024)
testFileH.addRun(Run(2, 15))
testJobG = self.getMinimalJob()
testJobG.addFile(testFileH)
testFileI = File(lfn=makeUUID(), size=1024, events=1024)
testFileI.addRun(Run(3, 20))
testJobH = self.getMinimalJob()
testJobH.addFile(testFileI)
testFileJ = File(lfn=makeUUID(), size=1024, events=1024)
testFileJ.addRun(Run(1, 9))
testJobI = self.getMinimalJob()
testJobI.addFile(testFileJ)
dcs.failedJobs([testJobA, testJobB, testJobC, testJobD, testJobE,
testJobF, testJobG, testJobH, testJobI])
whiteList = dcs.getLumiWhitelist("ACDCTest", "/ACDCTest/reco")
self.assertEqual(len(whiteList.keys()), 3,
"Error: There should be 3 runs.")
self.assertEqual(whiteList["1"], [[1, 4], [6, 7], [9, 9], [11, 12]],
"Error: Whitelist for run 1 is wrong.")
self.assertEqual(whiteList["2"], [[5, 7], [10, 12], [15, 15]],
"Error: Whitelist for run 2 is wrong.")
self.assertEqual(whiteList["3"], [[20, 20]],
"Error: Whitelist for run 3 is wrong.")
correctLumiList = LumiList(compactList={"1": [[1, 4], [6, 7], [9, 9], [11, 12]],
"2": [[5, 7], [10, 12], [15, 15]],
"3": [[20, 20]]})
testLumiList = dcs.getLumilistWhitelist("ACDCTest", "/ACDCTest/reco")
self.assertEqual(correctLumiList.getCMSSWString(), testLumiList.getCMSSWString())
return
def testGetLumiWhitelist(self):
"""
_testGetLumiWhitelist_
Verify that the ACDC whitelist generation code works correctly. We'll
add jobs with the following lumi info:
# Run 1, lumis [1, 2, 3], [4, 6], [7], [9], [11, 12]
# Run 2, lumis [5, 6, 7], [10, 11, 12], [15]
# Run 3, lumis [20]
And should get out a whitelist that looks like this:
{"1": [[1, 4], [6, 7], [9, 9], [11, 12]],
"2": [[5, 7], [10, 12], [15, 15]],
"3": [[20, 20]]}
"""
dcs = DataCollectionService(url=self.testInit.couchUrl,
database="wmcore-acdc-datacollectionsvc")
testFileA = File(lfn=makeUUID(), size=1024, events=1024)
testFileA.addRun(Run(1, 1, 2))
testFileB = File(lfn=makeUUID(), size=1024, events=1024)
testFileB.addRun(Run(1, 3))
testJobA = self.getMinimalJob()
testJobA.addFile(testFileA)
testJobA.addFile(testFileB)
testFileC = File(lfn=makeUUID(), size=1024, events=1024)
testFileC.addRun(Run(1, 4, 6))
testJobB = self.getMinimalJob()
testJobB.addFile(testFileC)
testFileD = File(lfn=makeUUID(), size=1024, events=1024)
testFileD.addRun(Run(1, 7))
testJobC = self.getMinimalJob()
testJobC.addFile(testFileD)
testFileE = File(lfn=makeUUID(), size=1024, events=1024)
testFileE.addRun(Run(1, 11, 12))
testJobD = self.getMinimalJob()
testJobD.addFile(testFileE)
testFileF = File(lfn=makeUUID(), size=1024, events=1024)
testFileF.addRun(Run(2, 5, 6, 7))
testJobE = self.getMinimalJob()
testJobE.addFile(testFileF)
testFileG = File(lfn=makeUUID(), size=1024, events=1024)
testFileG.addRun(Run(2, 10, 11, 12))
testJobF = self.getMinimalJob()
testJobF.addFile(testFileG)
testFileH = File(lfn=makeUUID(), size=1024, events=1024)
testFileH.addRun(Run(2, 15))
testJobG = self.getMinimalJob()
testJobG.addFile(testFileH)
testFileI = File(lfn=makeUUID(), size=1024, events=1024)
testFileI.addRun(Run(3, 20))
testJobH = self.getMinimalJob()
testJobH.addFile(testFileI)
testFileJ = File(lfn=makeUUID(), size=1024, events=1024)
testFileJ.addRun(Run(1, 9))
testJobI = self.getMinimalJob()
testJobI.addFile(testFileJ)
dcs.failedJobs([
testJobA, testJobB, testJobC, testJobD, testJobE, testJobF,
testJobG, testJobH, testJobI
])
whiteList = dcs.getLumiWhitelist("ACDCTest", "/ACDCTest/reco")
self.assertEqual(len(whiteList.keys()), 3,
"Error: There should be 3 runs.")
self.assertEqual(whiteList["1"], [[1, 4], [6, 7], [9, 9], [11, 12]],
"Error: Whitelist for run 1 is wrong.")
self.assertEqual(whiteList["2"], [[5, 7], [10, 12], [15, 15]],
"Error: Whitelist for run 2 is wrong.")
self.assertEqual(whiteList["3"], [[20, 20]],
"Error: Whitelist for run 3 is wrong.")
correctLumiList = LumiList(
compactList={
"1": [[1, 4], [6, 7], [9, 9], [11, 12]],
"2": [[5, 7], [10, 12], [15, 15]],
"3": [[20, 20]]
})
testLumiList = dcs.getLumilistWhitelist("ACDCTest", "/ACDCTest/reco")
self.assertEqual(correctLumiList.getCMSSWString(),
testLumiList.getCMSSWString())
return
def testLumiMask(self):
"""
_testLumiMask_
Test that we can use a lumi-mask to filter good runs/lumis.
"""
splitter = SplitterFactory()
# Create 3 files with 100 events per lumi:
# - file1 with 1 run of 8 lumis
# - file2 with 2 runs of 2 lumis each
# - file3 with 1 run of 5 lumis
fileA = File(lfn="/this/is/file1", size=1000, events=800)
fileB = File(lfn="/this/is/file2", size=1000, events=400)
fileC = File(lfn="/this/is/file3", size=1000, events=500)
lumiListA = []
for lumi in range(8):
lumiListA.append(10 + lumi)
fileA.addRun(Run(1, *lumiListA))
fileA.setLocation("somese.cern.ch")
lumiListB1 = []
lumiListB2 = []
for lumi in range(2):
lumiListB1.append(20 + lumi)
lumiListB2.append(30 + lumi)
fileB.addRun(Run(2, *lumiListB1))
fileB.addRun(Run(3, *lumiListB2))
fileB.setLocation("somese.cern.ch")
lumiListC = []
for lumi in range(5):
lumiListC.append(40 + lumi)
fileC.addRun(Run(4, *lumiListC))
fileC.setLocation("somese.cern.ch")
testFileset = Fileset(name='Fileset')
testFileset.addFile(fileA)
testFileset.addFile(fileB)
testFileset.addFile(fileC)
testSubscription = Subscription(fileset=testFileset,
workflow=self.testWorkflow,
split_algo="EventAwareLumiByWork",
type="Processing")
jobFactory = splitter(package="WMCore.DataStructs",
subscription=testSubscription)
# Use a lumi-mask = {1: [[10,14]], 2: [[20,21]], 4: [[40,41]]}
jobGroups = jobFactory(halt_job_on_file_boundaries=False,
splitOnRun=False,
events_per_job=850,
runs=['1', '2', '4'],
lumis=['10,14', '20,21', '40,41'],
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1,
"There should be only one job group")
jobs = jobGroups[0].jobs
self.assertEqual(len(jobs), 2, "Two jobs must be in the jobgroup")
processedLumis = LumiList()
for job in jobs:
processedLumis += LumiList(
compactList=job['mask'].getRunAndLumis())
correctLumis = LumiList(compactList={
1: [[10, 14]],
2: [[20, 21]],
4: [[40, 41]]
})
self.assertEqual(processedLumis.getCMSSWString(),
correctLumis.getCMSSWString())