def testLocalQueueACDCSplit(self):
"""
_testLocalQueueACDCSplit_
Test that the local queue split (i.e. with something in self.data) is compatible
with the information, namely the block name, produced by the global split.
"""
self.stuffACDCDatabase()
acdcWorkload = self.getProcessingACDCSpec('LumiBased',
{'lumis_per_job': 10})
acdcWorkload.data.request.priority = 10000
for task in acdcWorkload.taskIterator():
policy = ResubmitBlock()
units, _, _ = policy(acdcWorkload, task)
self.assertEqual(len(units), 1)
for unit in units:
self.assertEqual(len(unit['Inputs']), 1)
inputBlock = next(iter(unit['Inputs']))
self.assertEqual(sorted(unit['Inputs'][inputBlock]),
sorted(self.PSNs))
self.assertEqual(10000, unit['Priority'])
self.assertEqual(100, unit['Jobs'])
self.assertEqual(acdcWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(1000, unit['NumberOfLumis'])
self.assertEqual(500, unit['NumberOfFiles'])
self.assertEqual(125000, unit['NumberOfEvents'])
self.assertEqual(
unit['ACDC'], {
'database': self.acdcDBName,
'fileset': '/%s/DataProcessing' % self.workflowName,
'collection': self.workflowName,
'server': self.couchUrl
})
globalUnit = unit
policy = ResubmitBlock()
units, _, _ = policy(acdcWorkload, task, data=globalUnit['Inputs'])
self.assertEqual(len(units), 1)
for unit in units:
self.assertEqual(len(unit['Inputs']), 1)
inputBlock = next(iter(unit['Inputs']))
self.assertEqual(sorted(unit['Inputs'][inputBlock]),
sorted(self.PSNs))
self.assertEqual(10000, unit['Priority'])
self.assertEqual(100, unit['Jobs'])
self.assertEqual(acdcWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(1000, unit['NumberOfLumis'])
self.assertEqual(500, unit['NumberOfFiles'])
self.assertEqual(125000, unit['NumberOfEvents'])
self.assertEqual(
unit['ACDC'], {
'database': self.acdcDBName,
'fileset': '/%s/DataProcessing' % self.workflowName,
'collection': self.workflowName,
'server': self.couchUrl
})
return
def testSingleChunksSplit(self):
"""
_testSingleChunksSplit_
Test splitting a resubmit block in a single chunk, it will
use a ReReco ACDC of the merge task and the processing task.
This is to test two algorithms and that the split is location
independent.
"""
self.stuffACDCDatabase()
acdcWorkload = self.getProcessingACDCSpec('LumiBased', {'lumis_per_job' : 10})
acdcWorkload.data.request.priority = 10000
for task in acdcWorkload.taskIterator():
policy = ResubmitBlock()
units, _ = policy(acdcWorkload, task)
self.assertEqual(len(units), 1)
for unit in units:
self.assertEqual(len(unit['Inputs']), 1)
inputBlock = unit['Inputs'].keys()[0]
self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.validLocationsCMSNames))
self.assertEqual(10000, unit['Priority'])
self.assertEqual(100, unit['Jobs'])
self.assertEqual(acdcWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(1000, unit['NumberOfLumis'])
self.assertEqual(500, unit['NumberOfFiles'])
self.assertEqual(125000, unit['NumberOfEvents'])
self.assertEqual(unit['ACDC'], {'database' : self.acdcDBName,
'fileset' : '/%s/DataProcessing' % self.workflowName,
'collection' : self.workflowName,
'server' : self.couchUrl})
acdcWorkload = self.getMergeACDCSpec('ParentlessMergeBySize', {})
acdcWorkload.data.request.priority = 10000
for task in acdcWorkload.taskIterator():
policy = ResubmitBlock()
units, _ = policy(acdcWorkload, task)
self.assertEqual(len(units), 1)
for unit in units:
self.assertEqual(len(unit['Inputs']), 1)
inputBlock = unit['Inputs'].keys()[0]
self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.validLocationsCMSNames))
self.assertEqual(10000, unit['Priority'])
self.assertEqual(500, unit['Jobs'])
self.assertEqual(acdcWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(1000, unit['NumberOfLumis'])
self.assertEqual(500, unit['NumberOfFiles'])
self.assertEqual(125000, unit['NumberOfEvents'])
self.assertEqual(unit['ACDC'], {'database' : self.acdcDBName,
'fileset' : '/%s/DataProcessing/DataProcessingMergeRECOoutput' % self.workflowName,
'collection' : self.workflowName,
'server' : self.couchUrl})
return
def testFixedSizeChunksSplit(self):
"""
_testFixedSizeChunksSplit_
Test splitting a resubmit block in fixed size chunks of 250, it will
use a ReReco ACDC of the processing task.
"""
self.stuffACDCDatabase()
acdcWorkload = self.getProcessingACDCSpec('FileBased',
{'files_per_job': 5})
acdcWorkload.data.request.priority = 10000
for task in acdcWorkload.taskIterator():
policy = ResubmitBlock()
units, _, _ = policy(acdcWorkload, task)
self.assertEqual(len(units), 2)
for unit in units:
self.assertEqual(len(unit['Inputs']), 1)
inputBlock = next(iter(unit['Inputs']))
self.assertEqual(sorted(unit['Inputs'][inputBlock]),
sorted(self.PSNs))
self.assertEqual(10000, unit['Priority'])
self.assertEqual(50, unit['Jobs'])
self.assertEqual(acdcWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(500, unit['NumberOfLumis'])
self.assertEqual(250, unit['NumberOfFiles'])
self.assertEqual(62500, unit['NumberOfEvents'])
self.assertEqual(
unit['ACDC'], {
'database': self.acdcDBName,
'fileset': '/%s/DataProcessing' % self.workflowName,
'collection': self.workflowName,
'server': self.couchUrl
})
return
def testSiteWhitelistsLocation(self):
"""
_testSiteWhitelistsLocation_
Test splitting a resubmit block changing the block locations
according to the given site whitelist.
"""
self.stuffACDCDatabase()
acdcWorkload = self.getProcessingACDCSpec('FileBased', {'files_per_job' : 5},
setLocationFlag = True)
acdcWorkload.data.request.priority = 10000
for task in acdcWorkload.taskIterator():
policy = ResubmitBlock()
units, _ = policy(acdcWorkload, task)
self.assertEqual(len(units), 2)
for unit in units:
self.assertEqual(len(unit['Inputs']), 1)
inputBlock = unit['Inputs'].keys()[0]
self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.siteWhitelist))
self.assertEqual(10000, unit['Priority'])
self.assertEqual(50, unit['Jobs'])
self.assertEqual(acdcWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(500, unit['NumberOfLumis'])
self.assertEqual(250, unit['NumberOfFiles'])
self.assertEqual(62500, unit['NumberOfEvents'])
self.assertEqual(unit['ACDC'], {'database' : self.acdcDBName,
'fileset' : '/%s/DataProcessing' % self.workflowName,
'collection' : self.workflowName,
'server' : self.couchUrl})
return
def testMergeACDCSpec(self):
"""
_testMergeACDCSpec
Test ACDC spec for the merge task of a ReReco workload
"""
self.stuffACDCDatabase()
acdcWorkload = self.getMergeACDCSpec('ParentlessMergeBySize', {})
acdcWorkload.data.request.priority = 10000
for task in acdcWorkload.taskIterator():
policy = ResubmitBlock()
units, _ = policy(acdcWorkload, task)
self.assertEqual(len(units), 1)
for unit in units:
self.assertEqual(len(unit['Inputs']), 1)
inputBlock = unit['Inputs'].keys()[0]
self.assertEqual(sorted(unit['Inputs'][inputBlock]), sorted(self.PSNs))
self.assertEqual(10000, unit['Priority'])
self.assertEqual(500, unit['Jobs'])
self.assertEqual(acdcWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(1000, unit['NumberOfLumis'])
self.assertEqual(500, unit['NumberOfFiles'])
self.assertEqual(125000, unit['NumberOfEvents'])
self.assertEqual(unit['ACDC'], {'database' : self.acdcDBName,
'fileset' : '/%s/DataProcessing/DataProcessingMergeRECOoutput' % self.workflowName,
'collection' : self.workflowName,
'server' : self.couchUrl})
return
def testEmptyACDC(self):
"""
_testEmptyACDC_
Test that the correct exception is rased when trying an ACDC
without any ACDC documents. Must test a splittingAlgo that
works in fixedSizeChunks and singleChunk.
"""
# SingleChunk
acdcWorkload = self.getProcessingACDCSpec()
for task in acdcWorkload.taskIterator():
policy = ResubmitBlock()
self.assertRaises(WorkQueueNoWorkError, policy, acdcWorkload, task)
# FixedSizeChunks
acdcWorkload = self.getProcessingACDCSpec('FileBased', {'files_per_job' : 2000})
for task in acdcWorkload.taskIterator():
policy = ResubmitBlock()
self.assertRaises(WorkQueueNoWorkError, policy, acdcWorkload, task)
return
def testUnsupportedACDC(self):
"""
_testUnsupportedACDC_
Some tasks with certain algorithms won't work if resubmitted,
catch those
"""
acdcWorkload = self.getMergeACDCSpec('WMBSMergeBySize', {})
for task in acdcWorkload.taskIterator():
policy = ResubmitBlock()
self.assertRaises(WorkQueueWMSpecError, policy, acdcWorkload, task)
return