def testLocalQueueACDCSplit(self): """ _testLocalQueueACDCSplit_ Test that the local queue split (i.e. with something in self.data) is compatible with the information, namely the block name, produced by the global split. """ self.stuffACDCDatabase() acdcWorkload = self.getProcessingACDCSpec('LumiBased', {'lumis_per_job': 10}) acdcWorkload.data.request.priority = 10000 for task in acdcWorkload.taskIterator(): policy = ResubmitBlock() units, _, _ = policy(acdcWorkload, task) self.assertEqual(len(units), 1) for unit in units: self.assertEqual(len(unit['Inputs']), 1) inputBlock = next(iter(unit['Inputs'])) self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.PSNs)) self.assertEqual(10000, unit['Priority']) self.assertEqual(100, unit['Jobs']) self.assertEqual(acdcWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(1000, unit['NumberOfLumis']) self.assertEqual(500, unit['NumberOfFiles']) self.assertEqual(125000, unit['NumberOfEvents']) self.assertEqual( unit['ACDC'], { 'database': self.acdcDBName, 'fileset': '/%s/DataProcessing' % self.workflowName, 'collection': self.workflowName, 'server': self.couchUrl }) globalUnit = unit policy = ResubmitBlock() units, _, _ = policy(acdcWorkload, task, data=globalUnit['Inputs']) self.assertEqual(len(units), 1) for unit in units: self.assertEqual(len(unit['Inputs']), 1) inputBlock = next(iter(unit['Inputs'])) self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.PSNs)) self.assertEqual(10000, unit['Priority']) self.assertEqual(100, unit['Jobs']) self.assertEqual(acdcWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(1000, unit['NumberOfLumis']) self.assertEqual(500, unit['NumberOfFiles']) self.assertEqual(125000, unit['NumberOfEvents']) self.assertEqual( unit['ACDC'], { 'database': self.acdcDBName, 'fileset': '/%s/DataProcessing' % self.workflowName, 'collection': self.workflowName, 'server': self.couchUrl }) return
def testSingleChunksSplit(self): """ _testSingleChunksSplit_ Test splitting a resubmit block in a single chunk, it will use a ReReco ACDC of the merge task and the processing task. This is to test two algorithms and that the split is location independent. """ self.stuffACDCDatabase() acdcWorkload = self.getProcessingACDCSpec('LumiBased', {'lumis_per_job' : 10}) acdcWorkload.data.request.priority = 10000 for task in acdcWorkload.taskIterator(): policy = ResubmitBlock() units, _ = policy(acdcWorkload, task) self.assertEqual(len(units), 1) for unit in units: self.assertEqual(len(unit['Inputs']), 1) inputBlock = unit['Inputs'].keys()[0] self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.validLocationsCMSNames)) self.assertEqual(10000, unit['Priority']) self.assertEqual(100, unit['Jobs']) self.assertEqual(acdcWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(1000, unit['NumberOfLumis']) self.assertEqual(500, unit['NumberOfFiles']) self.assertEqual(125000, unit['NumberOfEvents']) self.assertEqual(unit['ACDC'], {'database' : self.acdcDBName, 'fileset' : '/%s/DataProcessing' % self.workflowName, 'collection' : self.workflowName, 'server' : self.couchUrl}) acdcWorkload = self.getMergeACDCSpec('ParentlessMergeBySize', {}) acdcWorkload.data.request.priority = 10000 for task in acdcWorkload.taskIterator(): policy = ResubmitBlock() units, _ = policy(acdcWorkload, task) self.assertEqual(len(units), 1) for unit in units: self.assertEqual(len(unit['Inputs']), 1) inputBlock = unit['Inputs'].keys()[0] self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.validLocationsCMSNames)) self.assertEqual(10000, unit['Priority']) self.assertEqual(500, unit['Jobs']) self.assertEqual(acdcWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(1000, unit['NumberOfLumis']) self.assertEqual(500, unit['NumberOfFiles']) self.assertEqual(125000, unit['NumberOfEvents']) self.assertEqual(unit['ACDC'], {'database' : self.acdcDBName, 'fileset' : '/%s/DataProcessing/DataProcessingMergeRECOoutput' % self.workflowName, 'collection' : self.workflowName, 'server' : self.couchUrl}) return
def testFixedSizeChunksSplit(self): """ _testFixedSizeChunksSplit_ Test splitting a resubmit block in fixed size chunks of 250, it will use a ReReco ACDC of the processing task. """ self.stuffACDCDatabase() acdcWorkload = self.getProcessingACDCSpec('FileBased', {'files_per_job': 5}) acdcWorkload.data.request.priority = 10000 for task in acdcWorkload.taskIterator(): policy = ResubmitBlock() units, _, _ = policy(acdcWorkload, task) self.assertEqual(len(units), 2) for unit in units: self.assertEqual(len(unit['Inputs']), 1) inputBlock = next(iter(unit['Inputs'])) self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.PSNs)) self.assertEqual(10000, unit['Priority']) self.assertEqual(50, unit['Jobs']) self.assertEqual(acdcWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(500, unit['NumberOfLumis']) self.assertEqual(250, unit['NumberOfFiles']) self.assertEqual(62500, unit['NumberOfEvents']) self.assertEqual( unit['ACDC'], { 'database': self.acdcDBName, 'fileset': '/%s/DataProcessing' % self.workflowName, 'collection': self.workflowName, 'server': self.couchUrl }) return
def testSiteWhitelistsLocation(self): """ _testSiteWhitelistsLocation_ Test splitting a resubmit block changing the block locations according to the given site whitelist. """ self.stuffACDCDatabase() acdcWorkload = self.getProcessingACDCSpec('FileBased', {'files_per_job' : 5}, setLocationFlag = True) acdcWorkload.data.request.priority = 10000 for task in acdcWorkload.taskIterator(): policy = ResubmitBlock() units, _ = policy(acdcWorkload, task) self.assertEqual(len(units), 2) for unit in units: self.assertEqual(len(unit['Inputs']), 1) inputBlock = unit['Inputs'].keys()[0] self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.siteWhitelist)) self.assertEqual(10000, unit['Priority']) self.assertEqual(50, unit['Jobs']) self.assertEqual(acdcWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(500, unit['NumberOfLumis']) self.assertEqual(250, unit['NumberOfFiles']) self.assertEqual(62500, unit['NumberOfEvents']) self.assertEqual(unit['ACDC'], {'database' : self.acdcDBName, 'fileset' : '/%s/DataProcessing' % self.workflowName, 'collection' : self.workflowName, 'server' : self.couchUrl}) return
def testMergeACDCSpec(self): """ _testMergeACDCSpec Test ACDC spec for the merge task of a ReReco workload """ self.stuffACDCDatabase() acdcWorkload = self.getMergeACDCSpec('ParentlessMergeBySize', {}) acdcWorkload.data.request.priority = 10000 for task in acdcWorkload.taskIterator(): policy = ResubmitBlock() units, _ = policy(acdcWorkload, task) self.assertEqual(len(units), 1) for unit in units: self.assertEqual(len(unit['Inputs']), 1) inputBlock = unit['Inputs'].keys()[0] self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.PSNs)) self.assertEqual(10000, unit['Priority']) self.assertEqual(500, unit['Jobs']) self.assertEqual(acdcWorkload, unit['WMSpec']) self.assertEqual(task, unit['Task']) self.assertEqual(1000, unit['NumberOfLumis']) self.assertEqual(500, unit['NumberOfFiles']) self.assertEqual(125000, unit['NumberOfEvents']) self.assertEqual(unit['ACDC'], {'database' : self.acdcDBName, 'fileset' : '/%s/DataProcessing/DataProcessingMergeRECOoutput' % self.workflowName, 'collection' : self.workflowName, 'server' : self.couchUrl}) return
def testEmptyACDC(self): """ _testEmptyACDC_ Test that the correct exception is rased when trying an ACDC without any ACDC documents. Must test a splittingAlgo that works in fixedSizeChunks and singleChunk. """ # SingleChunk acdcWorkload = self.getProcessingACDCSpec() for task in acdcWorkload.taskIterator(): policy = ResubmitBlock() self.assertRaises(WorkQueueNoWorkError, policy, acdcWorkload, task) # FixedSizeChunks acdcWorkload = self.getProcessingACDCSpec('FileBased', {'files_per_job' : 2000}) for task in acdcWorkload.taskIterator(): policy = ResubmitBlock() self.assertRaises(WorkQueueNoWorkError, policy, acdcWorkload, task) return
def testUnsupportedACDC(self): """ _testUnsupportedACDC_ Some tasks with certain algorithms won't work if resubmitted, catch those """ acdcWorkload = self.getMergeACDCSpec('WMBSMergeBySize', {}) for task in acdcWorkload.taskIterator(): policy = ResubmitBlock() self.assertRaises(WorkQueueWMSpecError, policy, acdcWorkload, task) return