def testContinuousSplittingSupport(self):
"""Can modify successfully policies for continuous splitting"""
policyInstance = Block(**self.splitArgs)
self.assertTrue(policyInstance.supportsWorkAddition(),
"Block instance should support continuous splitting")
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
'ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = policyInstance(Tier1ReRecoWorkload, task)
self.assertEqual(47, len(units))
blocks = [] # fill with blocks as we get work units for them
inputs = {}
for unit in units:
blocks.extend(unit['Inputs'].keys())
inputs.update(unit['Inputs'])
self.assertEqual(69, unit['Priority'])
self.assertTrue(1 <= unit['Jobs'])
self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertTrue(1 <= unit['NumberOfLumis'])
self.assertTrue(1 <= unit['NumberOfFiles'])
self.assertTrue(0 <= unit['NumberOfEvents'])
self.assertEqual(
len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
# Modify the spec and task, get first a fresh policy instance
policyInstance = Block(**self.splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
policyInstance.modifyPolicyForWorkAddition(
{'ProcessedInputs': inputs.keys()})
self.assertRaises(WorkQueueNoWorkError, policyInstance,
Tier1ReRecoWorkload, task)
# Run one last time
policyInstance = Block(**self.splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
policyInstance.modifyPolicyForWorkAddition(
{'ProcessedInputs': inputs.keys()})
self.assertRaises(WorkQueueNoWorkError, policyInstance,
Tier1ReRecoWorkload, task)
return
def testContinuousSplittingSupport(self):
"""Can modify successfully policies for continuous splitting"""
policyInstance = Block(**self.splitArgs)
self.assertTrue(policyInstance.supportsWorkAddition(), "Block instance should support continuous splitting")
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = policyInstance(Tier1ReRecoWorkload, task)
self.assertEqual(47, len(units))
blocks = [] # fill with blocks as we get work units for them
inputs = {}
for unit in units:
blocks.extend(unit['Inputs'].keys())
inputs.update(unit['Inputs'])
self.assertEqual(69, unit['Priority'])
self.assertTrue(1 <= unit['Jobs'])
self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertTrue(1 <= unit['NumberOfLumis'])
self.assertTrue(1 <= unit['NumberOfFiles'])
self.assertTrue(0 <= unit['NumberOfEvents'])
self.assertEqual(len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
# Modify the spec and task, get first a fresh policy instance
policyInstance = Block(**self.splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
policyInstance.modifyPolicyForWorkAddition({'ProcessedInputs': inputs.keys()})
self.assertRaises(WorkQueueNoWorkError, policyInstance, Tier1ReRecoWorkload, task)
# Run one last time
policyInstance = Block(**self.splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
policyInstance.modifyPolicyForWorkAddition({'ProcessedInputs': inputs.keys()})
self.assertRaises(WorkQueueNoWorkError, policyInstance, Tier1ReRecoWorkload, task)
return
def testContinuousSplittingSupport(self):
"""Can modify successfully policies for continuous splitting"""
policyInstance = Block(**self.splitArgs)
self.assertTrue(policyInstance.supportsWorkAddition(), "Block instance should support continuous splitting")
Tier1ReRecoWorkload = rerecoWorkload('ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary,
inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl : DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = policyInstance(Tier1ReRecoWorkload, task)
self.assertEqual(Globals.GlobalParams.numOfBlocksPerDataset(), len(units))
blocks = [] # fill with blocks as we get work units for them
inputs = {}
for unit in units:
blocks.extend(unit['Inputs'].keys())
inputs.update(unit['Inputs'])
self.assertEqual(69, unit['Priority'])
self.assertEqual(1, unit['Jobs'])
self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(4, unit['NumberOfLumis'])
self.assertEqual(10, unit['NumberOfFiles'])
self.assertEqual(10000, unit['NumberOfEvents'])
self.assertEqual(len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
# Modify the spec and task, get first a fresh policy instance
policyInstance = Block(**self.splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
policyInstance.modifyPolicyForWorkAddition({'ProcessedInputs' : inputs.keys()})
self.assertRaises(WorkQueueNoWorkError, policyInstance, Tier1ReRecoWorkload, task)
# Pop up 2 more blocks for the dataset with different statistics
Globals.GlobalParams.setNumOfBlocksPerDataset(Globals.GlobalParams.numOfBlocksPerDataset() + 2)
Globals.GlobalParams.setNumOfFilesPerBlock(10) # Emulator is crooked, it gives the sum of all the files in the dataset not block
# Now run another pass of the Block policy
policyInstance = Block(**self.splitArgs)
policyInstance.modifyPolicyForWorkAddition({'ProcessedInputs' : inputs.keys()})
for task in Tier1ReRecoWorkload.taskIterator():
units, rejectedWork = policyInstance(Tier1ReRecoWorkload, task)
self.assertEqual(2, len(units))
self.assertEqual(0, len(rejectedWork))
for unit in units:
blocks.extend(unit['Inputs'].keys())
inputs.update(unit['Inputs'])
self.assertEqual(69, unit['Priority'])
self.assertEqual(1, unit['Jobs'])
self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertEqual(8, unit['NumberOfLumis'])
self.assertEqual(40, unit['NumberOfFiles'])
self.assertEqual(40000, unit['NumberOfEvents'])
self.assertEqual(len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)) - 2)
# Run one last time
policyInstance = Block(**self.splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
policyInstance.modifyPolicyForWorkAddition({'ProcessedInputs' : inputs.keys()})
self.assertRaises(WorkQueueNoWorkError, policyInstance, Tier1ReRecoWorkload, task)
return
def testContinuousSplittingSupport(self):
"""Can modify successfully policies for continuous splitting"""
policyInstance = Block(**self.splitArgs)
self.assertTrue(policyInstance.supportsWorkAddition(),
"Block instance should support continuous splitting")
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
factory = ReRecoWorkloadFactory()
Tier1ReRecoWorkload = factory.factoryWorkloadConstruction(
'ReRecoWorkload', rerecoArgs)
Tier1ReRecoWorkload.data.request.priority = 69
inputDataset = getFirstTask(Tier1ReRecoWorkload).inputDataset()
dataset = "/%s/%s/%s" % (inputDataset.primary, inputDataset.processed,
inputDataset.tier)
dbs = {inputDataset.dbsurl: DBSReader(inputDataset.dbsurl)}
for task in Tier1ReRecoWorkload.taskIterator():
units, _ = policyInstance(Tier1ReRecoWorkload, task)
self.assertEqual(47, len(units))
blocks = [] # fill with blocks as we get work units for them
inputs = {}
for unit in units:
blocks.extend(unit['Inputs'].keys())
inputs.update(unit['Inputs'])
self.assertEqual(69, unit['Priority'])
self.assertTrue(1 <= unit['Jobs'])
self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec'])
self.assertEqual(task, unit['Task'])
self.assertTrue(1 <= unit['NumberOfLumis'])
self.assertTrue(1 <= unit['NumberOfFiles'])
self.assertTrue(0 <= unit['NumberOfEvents'])
self.assertEqual(
len(units),
len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)))
# Modify the spec and task, get first a fresh policy instance
policyInstance = Block(**self.splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
policyInstance.modifyPolicyForWorkAddition(
{'ProcessedInputs': inputs.keys()})
self.assertRaises(WorkQueueNoWorkError, policyInstance,
Tier1ReRecoWorkload, task)
# Pop up 2 more blocks for the dataset with different statistics
Globals.GlobalParams.setNumOfBlocksPerDataset(
Globals.GlobalParams.numOfBlocksPerDataset() + 2)
Globals.GlobalParams.setNumOfFilesPerBlock(
10
) # Emulator is crooked, it gives the sum of all the files in the dataset not block
# Now run another pass of the Block policy
# Broken: makes no work
#policyInstance = Block(**self.splitArgs)
#policyInstance.modifyPolicyForWorkAddition({'ProcessedInputs' : inputs.keys()})
#for task in Tier1ReRecoWorkload.taskIterator():
#units, rejectedWork = policyInstance(Tier1ReRecoWorkload, task)
#self.assertEqual(2, len(units))
#self.assertEqual(0, len(rejectedWork))
#for unit in units:
#blocks.extend(unit['Inputs'].keys())
#inputs.update(unit['Inputs'])
#self.assertEqual(69, unit['Priority'])
#self.assertEqual(1, unit['Jobs'])
#self.assertEqual(Tier1ReRecoWorkload, unit['WMSpec'])
#self.assertEqual(task, unit['Task'])
#self.assertEqual(8, unit['NumberOfLumis'])
#self.assertEqual(40, unit['NumberOfFiles'])
#self.assertEqual(40000, unit['NumberOfEvents'])
#self.assertEqual(len(units),
#len(dbs[inputDataset.dbsurl].getFileBlocksInfo(dataset)) - 2)
# Run one last time
policyInstance = Block(**self.splitArgs)
for task in Tier1ReRecoWorkload.taskIterator():
policyInstance.modifyPolicyForWorkAddition(
{'ProcessedInputs': inputs.keys()})
self.assertRaises(WorkQueueNoWorkError, policyInstance,
Tier1ReRecoWorkload, task)
return