def testReqMgrOpenRequests(self): """Check the mechanics of open running requests""" # don't actually talk to ReqMgr - mock it. globalQ = self.setupGlobalWorkqueue() localQ = self.setupLocalQueue() reqMgr = fakeReqMgr(splitter='Block', openRunningTimeout=3600) reqMgrInt = WorkQueueReqMgrInterface() reqMgrInt.reqMgr = reqMgr # 1st run should pull a request self.assertEqual(len(globalQ), 0) reqMgrInt(globalQ) self.assertEqual(len(globalQ), 2) self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired') # local queue acquires and runs globalQ.updateLocationInfo() localQ.pullWork({'T2_XX_SiteA': 10000, 'T2_XX_SiteB': 10000}) self.assertEqual(len(globalQ), 0) reqMgrInt(globalQ) self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired') # start running work globalQ.setStatus('Running', WorkflowName=reqMgr.names[0]) globalQ.performQueueCleanupActions() reqMgrInt(globalQ) # report back to ReqMgr self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open') # Work should not be closed yet reqMgrInt(globalQ) globalQ.performQueueCleanupActions() self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open') # Now add 2 new blocks to dbs, the reqMgr should put more work in the queue for the request GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 2) reqMgrInt(globalQ) globalQ.performQueueCleanupActions() self.assertEqual(len(globalQ), 2) # Work that can be pulled normally and request stays in running-open globalQ.updateLocationInfo() localQ.pullWork({'T2_XX_SiteC': 10000}) self.assertEqual(len(globalQ), 0) reqMgrInt(globalQ) self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open') # Put the latest work to run globalQ.setStatus('Running', WorkflowName=reqMgr.names[0]) globalQ.performQueueCleanupActions() reqMgrInt(globalQ) # report back to ReqMgr self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open') # Change the request status manually to close it reqMgr.status[reqMgr.names[0]] = 'running-closed' globalQ.performQueueCleanupActions() reqMgrInt(globalQ) # report back to WorkQueue self.assertEqual(len(globalQ.backend.getInboxElements(OpenForNewData=False)), 1) # Put 1 more block in DBS for the dataset, request is closed so no more data is added GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 1) reqMgrInt(globalQ) globalQ.performQueueCleanupActions() self.assertEqual(len(globalQ), 0) # finish work work = globalQ.status() globalQ.setStatus('Done', elementIDs=[x.id for x in work]) reqMgrInt(globalQ) globalQ.performQueueCleanupActions() reqMgrInt(globalQ) self.assertEqual(reqMgr.status[reqMgr.names[0]], 'completed') # and removed from WorkQueue self.assertEqual(len(globalQ.status()), 0) # reqMgr problems should not crash client reqMgrInt.reqMgr = None reqMgrInt(globalQ) reqMgr._removeSpecs()
# need to clean this up at some point #// - ignore some params in dbs spec - silence pylint warnings # pylint: disable-msg=W0613,R0201 from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator from xml.dom.minidom import parseString filesInDataset = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfBlocksPerDataset() filesInBlock = GlobalParams.numOfFilesPerBlock() class PhEDEx(dict): """ """ def __init__(self, *args, **kwargs): # add the end point to prevent the existence check fails. self['endpoint'] = "phedex_emulator" self.dataBlocks = DataBlockGenerator() self.subRequests = {} def injectBlocks(self, node, xmlData, verbose = 0, strict = 1): """ do nothing don't inject block.
_PhEDEx_ PhEDEx Emulator """ # need to clean this up at some point #// - ignore some params in dbs spec - silence pylint warnings # pylint: disable-msg=W0613,R0201 from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator from xml.dom.minidom import parseString filesInDataset = GlobalParams.numOfFilesPerBlock( ) * GlobalParams.numOfBlocksPerDataset() filesInBlock = GlobalParams.numOfFilesPerBlock() class PhEDEx(dict): """ """ def __init__(self, *args, **kwargs): # add the end point to prevent the existence check fails. self['endpoint'] = "phedex_emulator" self.dataBlocks = DataBlockGenerator() self.subRequests = {} def injectBlocks(self, node, xmlData, verbose=0, strict=1): """ do nothing don't inject block.
def testReqMgrOpenRequests(self): """Check the mechanics of open running requests""" # don't actually talk to ReqMgr - mock it. globalQ = self.setupGlobalWorkqueue() localQ = self.setupLocalQueue() reqMgr = fakeReqMgr(splitter = 'Block', openRunningTimeout = 3600) reqMgrInt = WorkQueueReqMgrInterface() reqMgrInt.reqMgr = reqMgr # 1st run should pull a request self.assertEqual(len(globalQ), 0) reqMgrInt(globalQ) self.assertEqual(len(globalQ), 2) self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired') # local queue acquires and runs globalQ.updateLocationInfo() localQ.pullWork({'T2_XX_SiteA' : 10000, 'T2_XX_SiteB' : 10000}) self.assertEqual(len(globalQ), 0) reqMgrInt(globalQ) self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired') # start running work globalQ.setStatus('Running', WorkflowName = reqMgr.names[0]) globalQ.performQueueCleanupActions() reqMgrInt(globalQ) # report back to ReqMgr self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open') # Work should not be closed yet reqMgrInt(globalQ) globalQ.performQueueCleanupActions() self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open') # Now add 2 new blocks to dbs, the reqMgr should put more work in the queue for the request GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 2) reqMgrInt(globalQ) globalQ.performQueueCleanupActions() self.assertEqual(len(globalQ), 2) # Work that can be pulled normally and request stays in running-open globalQ.updateLocationInfo() localQ.pullWork({'T2_XX_SiteC' : 10000}) self.assertEqual(len(globalQ), 0) reqMgrInt(globalQ) self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open') # Put the latest work to run globalQ.setStatus('Running', WorkflowName = reqMgr.names[0]) globalQ.performQueueCleanupActions() reqMgrInt(globalQ) # report back to ReqMgr self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open') # Change the request status manually to close it reqMgr.status[reqMgr.names[0]] = 'running-closed' globalQ.performQueueCleanupActions() reqMgrInt(globalQ) # report back to WorkQueue self.assertEqual(len(globalQ.backend.getInboxElements(OpenForNewData = False)), 1) # Put 1 more block in DBS for the dataset, request is closed so no more data is added GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 1) reqMgrInt(globalQ) globalQ.performQueueCleanupActions() self.assertEqual(len(globalQ), 0) # finish work work = globalQ.status() globalQ.setStatus('Done', elementIDs = [x.id for x in work]) reqMgrInt(globalQ) globalQ.performQueueCleanupActions() reqMgrInt(globalQ) self.assertEqual(reqMgr.status[reqMgr.names[0]], 'completed') # and removed from WorkQueue self.assertEqual(len(globalQ.status()), 0) # reqMgr problems should not crash client reqMgrInt.reqMgr = None reqMgrInt(globalQ) reqMgr._removeSpecs()
# need to clean this up at some point #// - ignore some params in dbs spec - silence pylint warnings # pylint: disable=W0613,R0201 from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator from xml.dom.minidom import parseString filesInDataset = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfBlocksPerDataset() filesInBlock = GlobalParams.numOfFilesPerBlock() class PhEDEx(dict): """ """ def __init__(self, *args, **kwargs): # add the end point to prevent the existence check fails. self['endpoint'] = "phedex_emulator" self.dataBlocks = DataBlockGenerator() self.subRequests = {} self.deletionRequests = {} self.deletionRequestId = 0 def injectBlocks(self, node, xmlData, verbose = 0, strict = 1):