def testFilter(self): """ Test filtering of a list of lumis """ runsAndLumis = { 1: range(1, 34) + [35] + range(37, 48), 2: range(49, 76) + range(77, 131) + range(133, 137) } completeList = zip([1]*150, range(1, 150)) + \ zip([2]*150, range(1, 150)) + \ zip([3]*150, range(1, 150)) smallList = zip([1]*50, range(1, 10)) + zip([2]*50, range(50, 70)) overlapList = zip([1]*150, range(30, 40)) + \ zip([2]*150, range(60, 80)) overlapRes = zip([1]*9, range(30, 34)) + [(1, 35)] + \ zip([1]*9, range(37, 40)) + \ zip([2]*30, range(60, 76)) + \ zip([2]*9, range(77, 80)) runLister = LumiList(runsAndLumis = runsAndLumis) # Test a list to be filtered which is a superset of constructed list filterComplete = runLister.filterLumis(completeList) # Test a list to be filtered which is a subset of constructed list filterSmall = runLister.filterLumis(smallList) # Test a list to be filtered which is neither filterOverlap = runLister.filterLumis(overlapList) self.assertTrue(filterComplete == runLister.getLumis()) self.assertTrue(filterSmall == smallList) self.assertTrue(filterOverlap == overlapRes)
def addJobs(self, jobs): if self.algo == 'FileBased': self.lumisPerJob += [ sum([x.get('lumiCount', 0) for x in job['input_files']]) for job in jobs ] self.eventsPerJob += [ sum([x['events'] for x in job['input_files']]) for job in jobs ] self.filesPerJob += [len(job['input_files']) for job in jobs] elif self.algo == 'EventBased': self.lumisPerJob += [ job['mask']['LastLumi'] - job['mask']['FirstLumi'] for job in jobs ] self.eventsPerJob += [ job['mask']['LastEvent'] - job['mask']['FirstEvent'] for job in jobs ] else: for job in jobs: avgEventsPerLumi = sum([ f['avgEvtsPerLumi'] for f in job['input_files'] ]) / float(len(job['input_files'])) lumis = LumiList(compactList=job['mask']['runAndLumis']) self.lumisPerJob.append(len(lumis.getLumis())) self.eventsPerJob.append(avgEventsPerLumi * self.lumisPerJob[-1])
def testFilter(self): """ Test filtering of a list of lumis """ runsAndLumis = { 1: range(1, 34) + [35] + range(37, 48), 2: range(49, 76) + range(77, 131) + range(133, 137) } completeList = list(zip([1]*150, range(1, 150))) + \ list(zip([2]*150, range(1, 150))) + \ list(zip([3]*150, range(1, 150))) smallList = list(zip([1]*50, range(1, 10))) + list(zip([2]*50, range(50, 70))) overlapList = list(zip([1]*150, range(30, 40))) + \ list(zip([2]*150, range(60, 80))) overlapRes = list(zip([1]*9, range(30, 34))) + [(1, 35)] + \ list(zip([1]*9, range(37, 40))) + \ list(zip([2]*30, range(60, 76))) + \ list(zip([2]*9, range(77, 80))) runLister = LumiList(runsAndLumis = runsAndLumis) # Test a list to be filtered which is a superset of constructed list filterComplete = runLister.filterLumis(completeList) # Test a list to be filtered which is a subset of constructed list filterSmall = runLister.filterLumis(smallList) # Test a list to be filtered which is neither filterOverlap = runLister.filterLumis(overlapList) self.assertTrue(filterComplete == runLister.getLumis()) self.assertTrue(filterSmall == smallList) self.assertTrue(filterOverlap == overlapRes)
def testWithMaskedBlocks(self): """ _testWithMaskedBlocks_ Test job splitting with masked blocks """ rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction( 'ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 task = getFirstTask(Tier1ReRecoWorkload) dummyDataset = task.inputDataset() task.data.input.splitting.runs = [181061, 180899] task.data.input.splitting.lumis = ['1,50,60,70', '1,1'] lumiMask = LumiList(compactList={ '206371': [[1, 50], [60, 70]], '180899': [[1, 1]], }) units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) nLumis = 0 for unit in units: nLumis += unit['NumberOfLumis'] self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testWithMaskedBlocks(self): """ _testWithMaskedBlocks_ Test job splitting with masked blocks """ rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 task = getFirstTask(Tier1ReRecoWorkload) dummyDataset = task.inputDataset() task.data.input.splitting.runs = [181061, 180899] task.data.input.splitting.lumis = ['1,50,60,70', '1,1'] lumiMask = LumiList(compactList={'206371': [[1, 50], [60, 70]], '180899': [[1, 1]], }) units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) nLumis = 0 for unit in units: nLumis += unit['NumberOfLumis'] self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testWithMaskedBlocks(self): """ _testWithMaskedBlocks_ Test job splitting with masked blocks """ Globals.GlobalParams.setNumOfRunsPerFile(3) Globals.GlobalParams.setNumOfLumisPerBlock(5) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 task = getFirstTask(Tier1ReRecoWorkload) inputDataset = task.inputDataset() task.data.input.splitting.runs = [181061, 180899] task.data.input.splitting.lumis = ['1,50,60,70', '1,1'] lumiMask = LumiList(compactList = {'206371': [[1, 50], [60,70]], '180899':[[1,1]], } ) dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)} units, rejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) nLumis = 0 for unit in units: nLumis += unit['NumberOfLumis'] self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testWithMaskedBlocks(self): """ _testWithMaskedBlocks_ Test job splitting with masked blocks """ Tier1ReRecoWorkload = rerecoWorkload( 'ReRecoWorkload', rerecoArgs, assignArgs={'SiteWhitelist': ['T2_XX_SiteA']}) Tier1ReRecoWorkload.data.request.priority = 69 task = getFirstTask(Tier1ReRecoWorkload) dummyDataset = task.inputDataset() task.data.input.splitting.runs = [181061, 180899] task.data.input.splitting.lumis = ['1,50,60,70', '1,1'] lumiMask = LumiList(compactList={ '206371': [[1, 50], [60, 70]], '180899': [[1, 1]], }) units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) nLumis = 0 for unit in units: nLumis += unit['NumberOfLumis'] self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testDuplicates(self): """ Test a list with lots of duplicates """ result = zip([1]*100, range(1, 34) + range(37, 48)) lumis = zip([1]*100, range(1, 34) + range(37, 48) + range(5, 25)) lister = LumiList(lumis = lumis) self.assertTrue(lister.getLumis() == result)
def testDuplicates(self): """ Test a list with lots of duplicates """ result = list(zip([1]*100, range(1, 34) + range(37, 48))) lumis = list(zip([1]*100, range(1, 34) + range(37, 48) + range(5, 25))) lister = LumiList(lumis = lumis) self.assertTrue(lister.getLumis() == result)
def testNull(self): """ Test a null list """ runLister = LumiList(lumis = None) self.assertTrue(runLister.getCMSSWString() == '') self.assertTrue(runLister.getLumis() == []) self.assertTrue(runLister.getCompactList() == {})
def addJobs(self, jobs): if self.algo == 'FileBased': self.lumisPerJob += [sum([x.get('lumiCount', 0) for x in job['input_files']]) for job in jobs] self.eventsPerJob += [sum([x['events'] for x in job['input_files']]) for job in jobs] elif self.algo == 'EventBased': self.lumisPerJob += [job['mask']['LastLumi'] - job['mask']['FirstLumi'] for job in jobs] self.eventsPerJob += [job['mask']['LastEvent'] - job['mask']['FirstEvent'] for job in jobs] else: for job in jobs: avgEventsPerLumi = sum([f['avgEvtsPerLumi'] for f in job['input_files']])/float(len(job['input_files'])) lumis = LumiList(compactList=job['mask']['runAndLumis']) self.lumisPerJob.append(len(lumis.getLumis())) self.eventsPerJob.append(avgEventsPerLumi * self.lumisPerJob[-1])
def testWithMaskedBlocks(self): """ _testWithMaskedBlocks_ Test job splitting with masked blocks """ Tier1ReRecoWorkload = rerecoWorkload('ReRecoWorkload', rerecoArgs, assignArgs={'SiteWhitelist': ['T2_XX_SiteA']}) Tier1ReRecoWorkload.data.request.priority = 69 task = getFirstTask(Tier1ReRecoWorkload) dummyDataset = task.inputDataset() task.data.input.splitting.runs = [181061, 180899] task.data.input.splitting.lumis = ['1,50,60,70', '1,1'] lumiMask = LumiList(compactList={'206371': [[1, 50], [60, 70]], '180899': [[1, 1]], }) units, dummyRejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) nLumis = 0 for unit in units: nLumis += unit['NumberOfLumis'] self.assertEqual(len(lumiMask.getLumis()), nLumis)
def testWithMaskedBlocks(self): """ _testWithMaskedBlocks_ Test job splitting with masked blocks """ Globals.GlobalParams.setNumOfRunsPerFile(3) Globals.GlobalParams.setNumOfLumisPerBlock(5) rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) factory = ReRecoWorkloadFactory() Tier1ReRecoWorkload = factory.factoryWorkloadConstruction( 'ReRecoWorkload', rerecoArgs) Tier1ReRecoWorkload.data.request.priority = 69 task = getFirstTask(Tier1ReRecoWorkload) inputDataset = task.inputDataset() task.data.input.splitting.runs = [181061, 180899] task.data.input.splitting.lumis = ['1,50,60,70', '1,1'] lumiMask = LumiList(compactList={ '206371': [[1, 50], [60, 70]], '180899': [[1, 1]], }) dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed, inputDataset.tier) dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)} units, rejectedWork = Block(**self.splitArgs)(Tier1ReRecoWorkload, task) nLumis = 0 for unit in units: nLumis += unit['NumberOfLumis'] self.assertEqual(len(lumiMask.getLumis()), nLumis)