def testReqMgrOpenRequests(self):
"""Check the mechanics of open running requests"""
# don't actually talk to ReqMgr - mock it.
globalQ = self.setupGlobalWorkqueue()
localQ = self.setupLocalQueue()
reqMgr = fakeReqMgr(splitter='Block', openRunningTimeout=3600)
reqMgrInt = WorkQueueReqMgrInterface()
reqMgrInt.reqMgr = reqMgr
# 1st run should pull a request
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(len(globalQ), 2)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')
# local queue acquires and runs
globalQ.updateLocationInfo()
localQ.pullWork({'T2_XX_SiteA': 10000, 'T2_XX_SiteB': 10000})
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')
# start running work
globalQ.setStatus('Running', WorkflowName=reqMgr.names[0])
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to ReqMgr
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Work should not be closed yet
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Now add 2 new blocks to dbs, the reqMgr should put more work in the queue for the request
GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 2)
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(len(globalQ), 2)
# Work that can be pulled normally and request stays in running-open
globalQ.updateLocationInfo()
localQ.pullWork({'T2_XX_SiteC': 10000})
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Put the latest work to run
globalQ.setStatus('Running', WorkflowName=reqMgr.names[0])
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to ReqMgr
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Change the request status manually to close it
reqMgr.status[reqMgr.names[0]] = 'running-closed'
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to WorkQueue
self.assertEqual(len(globalQ.backend.getInboxElements(OpenForNewData=False)), 1)
# Put 1 more block in DBS for the dataset, request is closed so no more data is added
GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 1)
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(len(globalQ), 0)
# finish work
work = globalQ.status()
globalQ.setStatus('Done', elementIDs=[x.id for x in work])
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'completed')
# and removed from WorkQueue
self.assertEqual(len(globalQ.status()), 0)
# reqMgr problems should not crash client
reqMgrInt.reqMgr = None
reqMgrInt(globalQ)
reqMgr._removeSpecs()
# need to clean this up at some point
#// - ignore some params in dbs spec - silence pylint warnings
# pylint: disable-msg=W0613,R0201
from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams
from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator
from xml.dom.minidom import parseString
filesInDataset = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfBlocksPerDataset()
filesInBlock = GlobalParams.numOfFilesPerBlock()
class PhEDEx(dict):
"""
"""
def __init__(self, *args, **kwargs):
# add the end point to prevent the existence check fails.
self['endpoint'] = "phedex_emulator"
self.dataBlocks = DataBlockGenerator()
self.subRequests = {}
def injectBlocks(self, node, xmlData, verbose = 0, strict = 1):
"""
do nothing don't inject block.
_PhEDEx_
PhEDEx Emulator
"""
# need to clean this up at some point
#// - ignore some params in dbs spec - silence pylint warnings
# pylint: disable-msg=W0613,R0201
from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams
from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator
from xml.dom.minidom import parseString
filesInDataset = GlobalParams.numOfFilesPerBlock(
) * GlobalParams.numOfBlocksPerDataset()
filesInBlock = GlobalParams.numOfFilesPerBlock()
class PhEDEx(dict):
"""
"""
def __init__(self, *args, **kwargs):
# add the end point to prevent the existence check fails.
self['endpoint'] = "phedex_emulator"
self.dataBlocks = DataBlockGenerator()
self.subRequests = {}
def injectBlocks(self, node, xmlData, verbose=0, strict=1):
"""
do nothing don't inject block.
def testReqMgrOpenRequests(self):
"""Check the mechanics of open running requests"""
# don't actually talk to ReqMgr - mock it.
globalQ = self.setupGlobalWorkqueue()
localQ = self.setupLocalQueue()
reqMgr = fakeReqMgr(splitter = 'Block', openRunningTimeout = 3600)
reqMgrInt = WorkQueueReqMgrInterface()
reqMgrInt.reqMgr = reqMgr
# 1st run should pull a request
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(len(globalQ), 2)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')
# local queue acquires and runs
globalQ.updateLocationInfo()
localQ.pullWork({'T2_XX_SiteA' : 10000, 'T2_XX_SiteB' : 10000})
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')
# start running work
globalQ.setStatus('Running', WorkflowName = reqMgr.names[0])
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to ReqMgr
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Work should not be closed yet
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Now add 2 new blocks to dbs, the reqMgr should put more work in the queue for the request
GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 2)
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(len(globalQ), 2)
# Work that can be pulled normally and request stays in running-open
globalQ.updateLocationInfo()
localQ.pullWork({'T2_XX_SiteC' : 10000})
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Put the latest work to run
globalQ.setStatus('Running', WorkflowName = reqMgr.names[0])
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to ReqMgr
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Change the request status manually to close it
reqMgr.status[reqMgr.names[0]] = 'running-closed'
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to WorkQueue
self.assertEqual(len(globalQ.backend.getInboxElements(OpenForNewData = False)), 1)
# Put 1 more block in DBS for the dataset, request is closed so no more data is added
GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 1)
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(len(globalQ), 0)
# finish work
work = globalQ.status()
globalQ.setStatus('Done', elementIDs = [x.id for x in work])
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'completed')
# and removed from WorkQueue
self.assertEqual(len(globalQ.status()), 0)
# reqMgr problems should not crash client
reqMgrInt.reqMgr = None
reqMgrInt(globalQ)
reqMgr._removeSpecs()
# need to clean this up at some point
#// - ignore some params in dbs spec - silence pylint warnings
# pylint: disable=W0613,R0201
from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams
from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator
from xml.dom.minidom import parseString
filesInDataset = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfBlocksPerDataset()
filesInBlock = GlobalParams.numOfFilesPerBlock()
class PhEDEx(dict):
"""
"""
def __init__(self, *args, **kwargs):
# add the end point to prevent the existence check fails.
self['endpoint'] = "phedex_emulator"
self.dataBlocks = DataBlockGenerator()
self.subRequests = {}
self.deletionRequests = {}
self.deletionRequestId = 0
def injectBlocks(self, node, xmlData, verbose = 0, strict = 1):
