def testRuns(self): """ Test constucting from run and list of lumis """ runsAndLumis = { 1: range(1, 34) + [35] + range(37, 48), 2: range(49, 76) + range(77, 131) + range(133, 137) } runsAndLumis2 = { '1': range(1, 34) + [35] + range(37, 48), '2': range(49, 76) + range(77, 131) + range(133, 137) } blank = { '1': [], '2': [] } jsonLister = LumiList(filename = 'lumiTest.json') jsonString = jsonLister.getCMSSWString() jsonList = jsonLister.getCompactList() runLister = LumiList(runsAndLumis = runsAndLumis) runString = runLister.getCMSSWString() runList = runLister.getCompactList() runLister2 = LumiList(runsAndLumis = runsAndLumis2) runList2 = runLister2.getCompactList() runLister3 = LumiList(runsAndLumis = blank) self.assertTrue(jsonString == runString) self.assertTrue(jsonList == runList) self.assertTrue(runList2 == runList) self.assertTrue(len(runLister3) == 0)
def testAnd(self): """ a&b for lots of cases """ alumis = {'1' : range(2,20) + range(31,39) + range(45,49), '2' : range(6,20) + range (30,40), '3' : range(10,20) + range (30,40) + range(50,60), '4' : range(1,100), } blumis = {'1' : range(1,6) + range(12,13) + range(16,25) + range(25,40) + range(40,50) + range(33,36), '2' : range(10,35), '3' : range(10,15) + range(35,40) + range(45,51) + range(59,70), '5' : range(1,100), } result = {'1' : range(2,6) + range(12,13) + range(16,20) + range(31,39) + range(45,49), '2' : range(10,20) + range(30,35), '3' : range(10,15) + range(35,40) + range(50,51)+ range(59,60), } a = LumiList(runsAndLumis = alumis) b = LumiList(runsAndLumis = blumis) r = LumiList(runsAndLumis = result) self.assertTrue((a&b).getCMSSWString() == r.getCMSSWString()) self.assertTrue((a&b).getCMSSWString() == (b&a).getCMSSWString()) self.assertTrue((a|b).getCMSSWString() != r.getCMSSWString())
def testRemoveSelect(self): """ a-b for lots of cases """ alumis = {'1' : range(2,20) + range(31,39) + range(45,49), '2' : range(6,20) + range (30,40), '3' : range(10,20) + range (30,40) + range(50,60), '4' : range(10,20) + range (30,80), } result = {'2' : range(6,20) + range (30,40), '4' : range(10,20) + range (30,80), } rem = LumiList(runsAndLumis = alumis) sel = LumiList(runsAndLumis = alumis) res = LumiList(runsAndLumis = result) rem.removeRuns([1,3]) sel.selectRuns([2,4]) self.assertTrue(rem.getCMSSWString() == res.getCMSSWString()) self.assertTrue(sel.getCMSSWString() == res.getCMSSWString()) self.assertTrue(sel.getCMSSWString() == rem.getCMSSWString())
def testOr(self): """ a|b for lots of cases """ alumis = {'1' : range(2,20) + range(31,39) + range(45,49), '2' : range(6,20) + range (30,40), '3' : range(10,20) + range (30,40) + range(50,60), } blumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80), '2' : range(10,35), '3' : range(10,15) + range(35,40) + range(45,51) + range(59,70), } clumis = {'1' : range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80), '2' : range(10,35), } result = {'1' : range(2,20) + range(31,39) + range(45,49) + range(1,6) + range(12,13) + range(16,30) + range(40,50) + range(39,80), '2' : range(6,20) + range (30,40) + range(10,35), '3' : range(10,20) + range (30,40) + range(50,60) + range(10,15) + range(35,40) + range(45,51) + range(59,70), } a = LumiList(runsAndLumis = alumis) b = LumiList(runsAndLumis = blumis) c = LumiList(runsAndLumis = blumis) r = LumiList(runsAndLumis = result) self.assertTrue((a|b).getCMSSWString() == r.getCMSSWString()) self.assertTrue((a|b).getCMSSWString() == (b|a).getCMSSWString()) self.assertTrue((a|b).getCMSSWString() == (a+b).getCMSSWString()) # Test list constuction (faster) multiple = [alumis, blumis, clumis] easy = LumiList(runsAndLumis = multiple) hard = a + b hard += c self.assertTrue(hard.getCMSSWString() == easy.getCMSSWString())
def testSubtract(self): """ a-b for lots of cases """ alumis = { '1': range(2, 20) + range(31, 39) + range(45, 49), '2': range(6, 20) + range(30, 40), '3': range(10, 20) + range(30, 40) + range(50, 60), } blumis = { '1': range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(33, 36), '2': range(10, 35), '3': range(10, 15) + range(35, 40) + range(45, 51) + range(59, 70), } clumis = { '1': range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(33, 36), '2': range(10, 35), } result = { '1': range(6, 12) + range(13, 16) + range(31, 33) + range(36, 39), '2': range(6, 10) + range(35, 40), '3': range(15, 20) + range(30, 35) + range(51, 59), } result2 = { '1': range(6, 12) + range(13, 16) + range(31, 33) + range(36, 39), '2': range(6, 10) + range(35, 40), '3': range(10, 20) + range(30, 40) + range(50, 60), } a = LumiList(runsAndLumis=alumis) b = LumiList(runsAndLumis=blumis) c = LumiList(runsAndLumis=clumis) r = LumiList(runsAndLumis=result) r2 = LumiList(runsAndLumis=result2) self.assertTrue((a - b).getCMSSWString() == r.getCMSSWString()) self.assertTrue((a - b).getCMSSWString() != (b - a).getCMSSWString()) # Test where c is missing runs from a self.assertTrue((a - c).getCMSSWString() == r2.getCMSSWString()) self.assertTrue((a - c).getCMSSWString() != (c - a).getCMSSWString()) # Test empty lists self.assertTrue(str(a - a) == '{}') self.assertTrue(len(a - a) == 0)
def testNull(self): """ Test a null list """ runLister = LumiList(lumis = None) self.assertTrue(runLister.getCMSSWString() == '') self.assertTrue(runLister.getLumis() == []) self.assertTrue(runLister.getCompactList() == {})
def testList(self): """ Test constucting from list of pairs """ listLs1 = range(1, 34) + [35] + range(37, 48) listLs2 = range(49, 76) + range(77, 131) + range(133, 137) lumis = list(zip([1]*100, listLs1)) + list(zip([2]*100, listLs2)) jsonLister = LumiList(filename = 'lumiTest.json') jsonString = jsonLister.getCMSSWString() jsonList = jsonLister.getCompactList() pairLister = LumiList(lumis = lumis) pairString = pairLister.getCMSSWString() pairList = pairLister.getCompactList() self.assertTrue(jsonString == pairString) self.assertTrue(jsonList == pairList)
def testWMagentFormat(self): """ Test the constuction of the double-string-based version of lumi lists used in some places by WMAgent """ clumis = { '1': list(range(2, 20)) + list(range(31, 39)) + list(range(45, 49)), '2': list(range(6, 20)) + list(range(30, 40)), '3': list(range(10, 20)) + list(range(30, 40)) + list(range(50, 60)), '4': list(range(1, 100)), } runs = [1, 2, 3, 4] lumis = ['2,19,31,38,45,48', '6,19,30,39', '10,19,30,39,50,59', '1,99'] c = LumiList(runsAndLumis=clumis) w = LumiList(wmagentFormat=(runs, lumis)) self.assertEqual(w.getCMSSWString(), c.getCMSSWString()) clumis = {'12': [4, 5]} runs = [12] lumis = ['4,5'] c = LumiList(runsAndLumis=clumis) w = LumiList(wmagentFormat=(runs, lumis)) self.assertEqual(w.getCMSSWString(), c.getCMSSWString()) aruns = [1] alumis = ['1,10,11,20'] bruns = [1] blumis = ['1,20'] self.assertEqual( LumiList(wmagentFormat=(aruns, alumis)).getCMSSWString(), LumiList(wmagentFormat=(bruns, blumis)).getCMSSWString()) with self.assertRaises(RuntimeError): w = LumiList(wmagentFormat=([1, 2, 3])) # No lumis with self.assertRaises(RuntimeError): w = LumiList(wmagentFormat=([1], [ '1,2,3' ])) # Need twice as many lumis as runs
def testList(self): """ Test constucting from list of pairs """ listLs1 = range(1, 34) + [35] + range(37, 48) listLs2 = range(49, 76) + range(77, 131) + range(133, 137) lumis = zip([1]*100, listLs1) + zip([2]*100, listLs2) jsonLister = LumiList(filename = 'lumiTest.json') jsonString = jsonLister.getCMSSWString() jsonList = jsonLister.getCompactList() pairLister = LumiList(lumis = lumis) pairString = pairLister.getCMSSWString() pairList = pairLister.getCompactList() self.assertTrue(jsonString == pairString) self.assertTrue(jsonList == pairList)
def testAddLumiMask(self): """ _testAddLumiMask_ Verify that setting and getting the lumiMask objects for a task works correctly. Do a round trip of a typical lumi mask """ testTask = makeWMTask("TestTask") lumiMask = LumiList(compactList = { '1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]], '2':[[1,45]], '3':[[1,45],[50,80]], }) testTask.setLumiMask(lumiMask = lumiMask.getCompactList()) outMask = LumiList(compactList = testTask.getLumiMask()) self.assertEqual(lumiMask.getCMSSWString(), outMask.getCMSSWString()) return
def testSubtract(self): """ a-b for lots of cases """ alumis = { "1": range(2, 20) + range(31, 39) + range(45, 49), "2": range(6, 20) + range(30, 40), "3": range(10, 20) + range(30, 40) + range(50, 60), } blumis = { "1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(33, 36), "2": range(10, 35), "3": range(10, 15) + range(35, 40) + range(45, 51) + range(59, 70), } clumis = {"1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(33, 36), "2": range(10, 35)} result = { "1": range(6, 12) + range(13, 16) + range(31, 33) + range(36, 39), "2": range(6, 10) + range(35, 40), "3": range(15, 20) + range(30, 35) + range(51, 59), } result2 = { "1": range(6, 12) + range(13, 16) + range(31, 33) + range(36, 39), "2": range(6, 10) + range(35, 40), "3": range(10, 20) + range(30, 40) + range(50, 60), } a = LumiList(runsAndLumis=alumis) b = LumiList(runsAndLumis=blumis) c = LumiList(runsAndLumis=clumis) r = LumiList(runsAndLumis=result) r2 = LumiList(runsAndLumis=result2) self.assertTrue((a - b).getCMSSWString() == r.getCMSSWString()) self.assertTrue((a - b).getCMSSWString() != (b - a).getCMSSWString()) # Test where c is missing runs from a self.assertTrue((a - c).getCMSSWString() == r2.getCMSSWString()) self.assertTrue((a - c).getCMSSWString() != (c - a).getCMSSWString()) # Test empty lists self.assertTrue(str(a - a) == "{}") self.assertTrue(len(a - a) == 0)
def testAddLumiMask(self): """ _testAddLumiMask_ Verify that setting and getting the lumiMask objects for a task works correctly. Do a round trip of a typical lumi mask """ testTask = makeWMTask("TestTask") lumiMask = LumiList( compactList={ '1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]], '2': [[1, 45]], '3': [[1, 45], [50, 80]], }) testTask.setLumiMask(lumiMask=lumiMask.getCompactList()) outMask = LumiList(compactList=testTask.getLumiMask()) self.assertEqual(lumiMask.getCMSSWString(), outMask.getCMSSWString()) return
def testLumisWithEvents(self): """Make sure that runs and lumis with event counts as used in CRAB3 works """ clumis = {'1': list(range(2, 20)) + list(range(31, 39)) + list(range(45, 49)), '2': list(range(6, 20)) + list(range(30, 40)), '3': list(range(10, 20)) + list(range(30, 40)) + list(range(50, 60)), '4': list(range(1, 100)), } # create a lumilist like {'21' : {'2':None ...} ...} lumis = {} for run, ls in viewitems(clumis): newrun = str(int(run) + 20) lumis[newrun] = {str(l): None for l in ls} c1 = LumiList(runsAndLumis=clumis) w1 = LumiList(runsAndLumis=lumis) c1 += w1 c2 = LumiList(runsAndLumis=clumis) w2 = LumiList(runsAndLumis=lumis) w2 += c2 self.assertEqual(c1.getCMSSWString(), w2.getCMSSWString())
def testWMagentFormat(self): """ Test the constuction of the double-string-based version of lumi lists used in some places by WMAgent """ clumis = {'1': range(2, 20) + range(31, 39) + range(45, 49), '2': range(6, 20) + range(30, 40), '3': range(10, 20) + range(30, 40) + range(50, 60), '4': range(1, 100), } runs = [1, 2, 3, 4] lumis = ['2,19,31,38,45,48', '6,19,30,39', '10,19,30,39,50,59', '1,99'] c = LumiList(runsAndLumis=clumis) w = LumiList(wmagentFormat=(runs, lumis)) self.assertEqual(w.getCMSSWString(), c.getCMSSWString()) clumis = {'12': [4, 5]} runs = [12] lumis = ['4,5'] c = LumiList(runsAndLumis=clumis) w = LumiList(wmagentFormat=(runs, lumis)) self.assertEqual(w.getCMSSWString(), c.getCMSSWString()) aruns = [1] alumis = ['1,10,11,20'] bruns = [1] blumis = ['1,20'] self.assertEqual(LumiList(wmagentFormat=(aruns, alumis)).getCMSSWString(), LumiList(wmagentFormat=(bruns, blumis)).getCMSSWString()) with self.assertRaises(RuntimeError): w = LumiList(wmagentFormat=([1,2,3])) # No lumis with self.assertRaises(RuntimeError): w = LumiList(wmagentFormat=([1], ['1,2,3'])) # Need twice as many lumis as runs
def notestRead(self): """ Test reading from JSON """ exString = "1:1-1:33,1:35,1:37-1:47,2:49-2:75,2:77-2:130,2:133-2:136" exDict = {"1": [[1, 33], [35, 35], [37, 47]], "2": [[49, 75], [77, 130], [133, 136]]} exVLBR = cms.VLuminosityBlockRange("1:1-1:33", "1:35", "1:37-1:47", "2:49-2:75", "2:77-2:130", "2:133-2:136") jsonList = LumiList(filename="lumiTest.json") lumiString = jsonList.getCMSSWString() lumiList = jsonList.getCompactList() lumiVLBR = jsonList.getVLuminosityBlockRange(True) self.assertTrue(lumiString == exString) self.assertTrue(lumiList == exDict) self.assertTrue(lumiVLBR == exVLBR)
def notestRead(self): """ Test reading from JSON """ exString = "1:1-1:33,1:35,1:37-1:47,2:49-2:75,2:77-2:130,2:133-2:136" exDict = {'1': [[1, 33], [35, 35], [37, 47]], '2': [[49, 75], [77, 130], [133, 136]]} exVLBR = cms.VLuminosityBlockRange('1:1-1:33', '1:35', '1:37-1:47', '2:49-2:75', '2:77-2:130', '2:133-2:136') jsonList = LumiList(filename = 'lumiTest.json') lumiString = jsonList.getCMSSWString() lumiList = jsonList.getCompactList() lumiVLBR = jsonList.getVLuminosityBlockRange(True) self.assertTrue(lumiString == exString) self.assertTrue(lumiList == exDict) self.assertTrue(lumiVLBR == exVLBR)
def testOr(self): """ a|b for lots of cases """ alumis = { "1": range(2, 20) + range(31, 39) + range(45, 49), "2": range(6, 20) + range(30, 40), "3": range(10, 20) + range(30, 40) + range(50, 60), } blumis = { "1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(39, 80), "2": range(10, 35), "3": range(10, 15) + range(35, 40) + range(45, 51) + range(59, 70), } clumis = {"1": range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(39, 80), "2": range(10, 35)} result = { "1": range(2, 20) + range(31, 39) + range(45, 49) + range(1, 6) + range(12, 13) + range(16, 30) + range(40, 50) + range(39, 80), "2": range(6, 20) + range(30, 40) + range(10, 35), "3": range(10, 20) + range(30, 40) + range(50, 60) + range(10, 15) + range(35, 40) + range(45, 51) + range(59, 70), } a = LumiList(runsAndLumis=alumis) b = LumiList(runsAndLumis=blumis) c = LumiList(runsAndLumis=blumis) r = LumiList(runsAndLumis=result) self.assertTrue((a | b).getCMSSWString() == r.getCMSSWString()) self.assertTrue((a | b).getCMSSWString() == (b | a).getCMSSWString()) self.assertTrue((a | b).getCMSSWString() == (a + b).getCMSSWString())
def testLumiMask(self): """ _testLumiMask_ Test that we can use a lumi-mask to filter good runs/lumis. """ splitter = SplitterFactory() # Create 3 files with 100 events per lumi: # - file1 with 1 run of 8 lumis # - file2 with 2 runs of 2 lumis each # - file3 with 1 run of 5 lumis fileA = File(lfn="/this/is/file1", size=1000, events=800) fileB = File(lfn="/this/is/file2", size=1000, events=400) fileC = File(lfn="/this/is/file3", size=1000, events=500) lumiListA = [] for lumi in range(8): lumiListA.append(10 + lumi) fileA.addRun(Run(1, *lumiListA)) fileA.setLocation("somese.cern.ch") lumiListB1 = [] lumiListB2 = [] for lumi in range(2): lumiListB1.append(20 + lumi) lumiListB2.append(30 + lumi) fileB.addRun(Run(2, *lumiListB1)) fileB.addRun(Run(3, *lumiListB2)) fileB.setLocation("somese.cern.ch") lumiListC = [] for lumi in range(5): lumiListC.append(40 + lumi) fileC.addRun(Run(4, *lumiListC)) fileC.setLocation("somese.cern.ch") testFileset = Fileset(name='Fileset') testFileset.addFile(fileA) testFileset.addFile(fileB) testFileset.addFile(fileC) testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiByWork", type="Processing") jobFactory = splitter(package="WMCore.DataStructs", subscription=testSubscription) # Use a lumi-mask = {1: [[10,14]], 2: [[20,21]], 4: [[40,41]]} jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=850, runs=['1', '2', '4'], lumis=['10,14', '20,21', '40,41'], performance=self.performanceParams) self.assertEqual(len(jobGroups), 1, "There should be only one job group") jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 2, "Two jobs must be in the jobgroup") processedLumis = LumiList() for job in jobs: processedLumis += LumiList(compactList=job['mask'].getRunAndLumis()) correctLumis = LumiList(compactList={1: [[10, 14]], 2: [[20, 21]], 4: [[40, 41]]}) self.assertEqual(processedLumis.getCMSSWString(), correctLumis.getCMSSWString())
def testGetLumiWhitelist(self): """ _testGetLumiWhitelist_ Verify that the ACDC whitelist generation code works correctly. We'll add jobs with the following lumi info: # Run 1, lumis [1, 2, 3], [4, 6], [7], [9], [11, 12] # Run 2, lumis [5, 6, 7], [10, 11, 12], [15] # Run 3, lumis [20] And should get out a whitelist that looks like this: {"1": [[1, 4], [6, 7], [9, 9], [11, 12]], "2": [[5, 7], [10, 12], [15, 15]], "3": [[20, 20]]} """ dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc") testFileA = File(lfn=makeUUID(), size=1024, events=1024) testFileA.addRun(Run(1, 1, 2)) testFileB = File(lfn=makeUUID(), size=1024, events=1024) testFileB.addRun(Run(1, 3)) testJobA = self.getMinimalJob() testJobA.addFile(testFileA) testJobA.addFile(testFileB) testFileC = File(lfn=makeUUID(), size=1024, events=1024) testFileC.addRun(Run(1, 4, 6)) testJobB = self.getMinimalJob() testJobB.addFile(testFileC) testFileD = File(lfn=makeUUID(), size=1024, events=1024) testFileD.addRun(Run(1, 7)) testJobC = self.getMinimalJob() testJobC.addFile(testFileD) testFileE = File(lfn=makeUUID(), size=1024, events=1024) testFileE.addRun(Run(1, 11, 12)) testJobD = self.getMinimalJob() testJobD.addFile(testFileE) testFileF = File(lfn=makeUUID(), size=1024, events=1024) testFileF.addRun(Run(2, 5, 6, 7)) testJobE = self.getMinimalJob() testJobE.addFile(testFileF) testFileG = File(lfn=makeUUID(), size=1024, events=1024) testFileG.addRun(Run(2, 10, 11, 12)) testJobF = self.getMinimalJob() testJobF.addFile(testFileG) testFileH = File(lfn=makeUUID(), size=1024, events=1024) testFileH.addRun(Run(2, 15)) testJobG = self.getMinimalJob() testJobG.addFile(testFileH) testFileI = File(lfn=makeUUID(), size=1024, events=1024) testFileI.addRun(Run(3, 20)) testJobH = self.getMinimalJob() testJobH.addFile(testFileI) testFileJ = File(lfn=makeUUID(), size=1024, events=1024) testFileJ.addRun(Run(1, 9)) testJobI = self.getMinimalJob() testJobI.addFile(testFileJ) dcs.failedJobs([testJobA, testJobB, testJobC, testJobD, testJobE, testJobF, testJobG, testJobH, testJobI]) whiteList = dcs.getLumiWhitelist("ACDCTest", "/ACDCTest/reco") self.assertEqual(len(whiteList.keys()), 3, "Error: There should be 3 runs.") self.assertEqual(whiteList["1"], [[1, 4], [6, 7], [9, 9], [11, 12]], "Error: Whitelist for run 1 is wrong.") self.assertEqual(whiteList["2"], [[5, 7], [10, 12], [15, 15]], "Error: Whitelist for run 2 is wrong.") self.assertEqual(whiteList["3"], [[20, 20]], "Error: Whitelist for run 3 is wrong.") correctLumiList = LumiList(compactList={"1": [[1, 4], [6, 7], [9, 9], [11, 12]], "2": [[5, 7], [10, 12], [15, 15]], "3": [[20, 20]]}) testLumiList = dcs.getLumilistWhitelist("ACDCTest", "/ACDCTest/reco") self.assertEqual(correctLumiList.getCMSSWString(), testLumiList.getCMSSWString()) return
def testGetLumiWhitelist(self): """ _testGetLumiWhitelist_ Verify that the ACDC whitelist generation code works correctly. We'll add jobs with the following lumi info: # Run 1, lumis [1, 2, 3], [4, 6], [7], [9], [11, 12] # Run 2, lumis [5, 6, 7], [10, 11, 12], [15] # Run 3, lumis [20] And should get out a whitelist that looks like this: {"1": [[1, 4], [6, 7], [9, 9], [11, 12]], "2": [[5, 7], [10, 12], [15, 15]], "3": [[20, 20]]} """ dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc") testFileA = File(lfn=makeUUID(), size=1024, events=1024) testFileA.addRun(Run(1, 1, 2)) testFileB = File(lfn=makeUUID(), size=1024, events=1024) testFileB.addRun(Run(1, 3)) testJobA = self.getMinimalJob() testJobA.addFile(testFileA) testJobA.addFile(testFileB) testFileC = File(lfn=makeUUID(), size=1024, events=1024) testFileC.addRun(Run(1, 4, 6)) testJobB = self.getMinimalJob() testJobB.addFile(testFileC) testFileD = File(lfn=makeUUID(), size=1024, events=1024) testFileD.addRun(Run(1, 7)) testJobC = self.getMinimalJob() testJobC.addFile(testFileD) testFileE = File(lfn=makeUUID(), size=1024, events=1024) testFileE.addRun(Run(1, 11, 12)) testJobD = self.getMinimalJob() testJobD.addFile(testFileE) testFileF = File(lfn=makeUUID(), size=1024, events=1024) testFileF.addRun(Run(2, 5, 6, 7)) testJobE = self.getMinimalJob() testJobE.addFile(testFileF) testFileG = File(lfn=makeUUID(), size=1024, events=1024) testFileG.addRun(Run(2, 10, 11, 12)) testJobF = self.getMinimalJob() testJobF.addFile(testFileG) testFileH = File(lfn=makeUUID(), size=1024, events=1024) testFileH.addRun(Run(2, 15)) testJobG = self.getMinimalJob() testJobG.addFile(testFileH) testFileI = File(lfn=makeUUID(), size=1024, events=1024) testFileI.addRun(Run(3, 20)) testJobH = self.getMinimalJob() testJobH.addFile(testFileI) testFileJ = File(lfn=makeUUID(), size=1024, events=1024) testFileJ.addRun(Run(1, 9)) testJobI = self.getMinimalJob() testJobI.addFile(testFileJ) dcs.failedJobs([ testJobA, testJobB, testJobC, testJobD, testJobE, testJobF, testJobG, testJobH, testJobI ]) whiteList = dcs.getLumiWhitelist("ACDCTest", "/ACDCTest/reco") self.assertEqual(len(whiteList.keys()), 3, "Error: There should be 3 runs.") self.assertEqual(whiteList["1"], [[1, 4], [6, 7], [9, 9], [11, 12]], "Error: Whitelist for run 1 is wrong.") self.assertEqual(whiteList["2"], [[5, 7], [10, 12], [15, 15]], "Error: Whitelist for run 2 is wrong.") self.assertEqual(whiteList["3"], [[20, 20]], "Error: Whitelist for run 3 is wrong.") correctLumiList = LumiList( compactList={ "1": [[1, 4], [6, 7], [9, 9], [11, 12]], "2": [[5, 7], [10, 12], [15, 15]], "3": [[20, 20]] }) testLumiList = dcs.getLumilistWhitelist("ACDCTest", "/ACDCTest/reco") self.assertEqual(correctLumiList.getCMSSWString(), testLumiList.getCMSSWString()) return
def testLumiMask(self): """ _testLumiMask_ Test that we can use a lumi-mask to filter good runs/lumis. """ splitter = SplitterFactory() # Create 3 files with 100 events per lumi: # - file1 with 1 run of 8 lumis # - file2 with 2 runs of 2 lumis each # - file3 with 1 run of 5 lumis fileA = File(lfn="/this/is/file1", size=1000, events=800) fileB = File(lfn="/this/is/file2", size=1000, events=400) fileC = File(lfn="/this/is/file3", size=1000, events=500) lumiListA = [] for lumi in range(8): lumiListA.append(10 + lumi) fileA.addRun(Run(1, *lumiListA)) fileA.setLocation("somese.cern.ch") lumiListB1 = [] lumiListB2 = [] for lumi in range(2): lumiListB1.append(20 + lumi) lumiListB2.append(30 + lumi) fileB.addRun(Run(2, *lumiListB1)) fileB.addRun(Run(3, *lumiListB2)) fileB.setLocation("somese.cern.ch") lumiListC = [] for lumi in range(5): lumiListC.append(40 + lumi) fileC.addRun(Run(4, *lumiListC)) fileC.setLocation("somese.cern.ch") testFileset = Fileset(name='Fileset') testFileset.addFile(fileA) testFileset.addFile(fileB) testFileset.addFile(fileC) testSubscription = Subscription(fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiByWork", type="Processing") jobFactory = splitter(package="WMCore.DataStructs", subscription=testSubscription) # Use a lumi-mask = {1: [[10,14]], 2: [[20,21]], 4: [[40,41]]} jobGroups = jobFactory(halt_job_on_file_boundaries=False, splitOnRun=False, events_per_job=850, runs=['1', '2', '4'], lumis=['10,14', '20,21', '40,41'], performance=self.performanceParams) self.assertEqual(len(jobGroups), 1, "There should be only one job group") jobs = jobGroups[0].jobs self.assertEqual(len(jobs), 2, "Two jobs must be in the jobgroup") processedLumis = LumiList() for job in jobs: processedLumis += LumiList( compactList=job['mask'].getRunAndLumis()) correctLumis = LumiList(compactList={ 1: [[10, 14]], 2: [[20, 21]], 4: [[40, 41]] }) self.assertEqual(processedLumis.getCMSSWString(), correctLumis.getCMSSWString())