def testParentage(self):
"""
1. check whether parent files are created in wmbs.
2. check parent files are associated to child.
3. When 2 specs with the same input data (one with parent processing, one without it)
is inserted, if one without parent processing inserted first then the other with
parent processing insert, it still needs to create parent files although child files
are duplicate
"""
block = self.dataset + "#1"
wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(
self.wmspec, block, parentFlag=False, detail=True)
# file creation without parents
self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
wmbs.topLevelFileset.loadData()
for child in wmbs.topLevelFileset.files:
# no parent per child
self.assertEqual(len(child["parents"]), 0)
wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec,
block,
parentFlag=True,
detail=True)
self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
wmbs.topLevelFileset.loadData()
for child in wmbs.topLevelFileset.files:
# one parent per child
self.assertEqual(len(child["parents"]), 1)
def testParentage(self):
"""
1. check whether parent files are created in wmbs.
2. check parent files are associated to child.
3. When 2 specs with the same input data (one with parent processing, one without it)
is inserted, if one without parent processing inserted first then the other with
parent processing insert, it still needs to create parent files although child files
are duplicate
"""
block = self.dataset + "#1"
wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block,
parentFlag = False, detail = True)
# file creation without parents
self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
wmbs.topLevelFileset.loadData()
for child in wmbs.topLevelFileset.files:
# no parent per child
self.assertEqual(len(child["parents"]), 0)
wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block,
parentFlag = True, detail = True)
self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
wmbs.topLevelFileset.loadData()
for child in wmbs.topLevelFileset.files:
# one parent per child
self.assertEqual(len(child["parents"]), 1)
def setUp(self):
WorkQueueTestCase.setUp(self)
GlobalParams.resetParams()
self.globalQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
self.globalQDB)
self.localQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
self.localQInboxDB)
def testParentProcessing(self):
"""
Enqueue and get work for a processing WMSpec.
"""
specfile = self.parentProcSpec.specUrl()
njobs = [5, 10] # array of jobs per block
total = sum(njobs)
# Queue Work & check accepted
self.queue.queueWork(specfile)
self.queue.processInboundWork()
self.assertEqual(len(njobs), len(self.queue))
self.queue.updateLocationInfo()
# No resources
work = self.queue.getWork({})
self.assertEqual(len(work), 0)
work = self.queue.getWork({'T2_XX_SiteA' : 0,
'T2_XX_SiteB' : 0})
self.assertEqual(len(work), 0)
# Only 1 block at SiteB - get 1 work element when any resources free
work = self.queue.getWork({'T2_XX_SiteB' : 1})
self.assertEqual(len(work), 1)
self.assertEqual(work[0]["NumOfFilesAdded"], GlobalParams.numOfFilesPerBlock() * 2)
# claim remaining work
work = self.queue.getWork({'T2_XX_SiteA' : total, 'T2_XX_SiteB' : total})
self.assertEqual(len(work), 1)
self.assertEqual(work[0]["NumOfFilesAdded"], GlobalParams.numOfFilesPerBlock() * 2)
# no more work available
self.assertEqual(0, len(self.queue.getWork({'T2_XX_SiteA' : total})))
def setUp(self):
WorkQueueTestCase.setUp(self)
EmulatorHelper.setEmulators(phedex = True, dbs = True,
siteDB = True, requestMgr = False)
GlobalParams.resetParams()
self.globalQCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB)
self.localQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
self.localQInboxDB)
def setUp(self):
self.reqmgr2_endpoint = "https://cmsweb-testbed.cern.ch/reqmgr2"
WorkQueueTestCase.setUp(self)
GlobalParams.resetParams()
self.globalQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
self.globalQDB)
self.localQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
self.localQInboxDB)
def testLumiMaskRestrictionsOK(self):
block = self.dataset + "#1"
self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = ['1']
self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = ['1,1']
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
def testReRecoWhiteRunRestriction(self):
block = self.dataset + "#2"
# Run Whitelist
self.topLevelTask.setInputRunWhitelist([2])
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
def testReRecoWhiteRunRestriction(self):
block = self.dataset + "#2"
# Run Whitelist
self.topLevelTask.setInputRunWhitelist([2])
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
def testLumiMaskRestrictionsOK(self):
block = self.dataset + "#1"
self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = ['1']
self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = ['1,1']
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
def tearDown(self):
WorkQueueTestCase.tearDown(self)
GlobalParams.resetParams()
def setUp(self):
WorkQueueTestCase.setUp(self)
GlobalParams.resetParams()
self.globalQCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB)
self.localQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
self.localQInboxDB)
def testReqMgrOpenRequests(self):
"""Check the mechanics of open running requests"""
# don't actually talk to ReqMgr - mock it.
globalQ = self.setupGlobalWorkqueue()
localQ = self.setupLocalQueue()
reqMgr = fakeReqMgr(splitter='Block', openRunningTimeout=3600)
reqMgrInt = WorkQueueReqMgrInterface()
reqMgrInt.reqMgr = reqMgr
# 1st run should pull a request
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(len(globalQ), 2)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')
# local queue acquires and runs
globalQ.updateLocationInfo()
localQ.pullWork({'T2_XX_SiteA': 10000, 'T2_XX_SiteB': 10000})
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')
# start running work
globalQ.setStatus('Running', WorkflowName=reqMgr.names[0])
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to ReqMgr
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Work should not be closed yet
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Now add 2 new blocks to dbs, the reqMgr should put more work in the queue for the request
GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 2)
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(len(globalQ), 2)
# Work that can be pulled normally and request stays in running-open
globalQ.updateLocationInfo()
localQ.pullWork({'T2_XX_SiteC': 10000})
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Put the latest work to run
globalQ.setStatus('Running', WorkflowName=reqMgr.names[0])
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to ReqMgr
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Change the request status manually to close it
reqMgr.status[reqMgr.names[0]] = 'running-closed'
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to WorkQueue
self.assertEqual(len(globalQ.backend.getInboxElements(OpenForNewData=False)), 1)
# Put 1 more block in DBS for the dataset, request is closed so no more data is added
GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 1)
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(len(globalQ), 0)
# finish work
work = globalQ.status()
globalQ.setStatus('Done', elementIDs=[x.id for x in work])
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'completed')
# and removed from WorkQueue
self.assertEqual(len(globalQ.status()), 0)
# reqMgr problems should not crash client
reqMgrInt.reqMgr = None
reqMgrInt(globalQ)
reqMgr._removeSpecs()
# need to clean this up at some point
#// - ignore some params in dbs spec - silence pylint warnings
# pylint: disable-msg=W0613,R0201
from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams
from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator
from xml.dom.minidom import parseString
filesInDataset = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfBlocksPerDataset()
filesInBlock = GlobalParams.numOfFilesPerBlock()
class PhEDEx(dict):
"""
"""
def __init__(self, *args, **kwargs):
# add the end point to prevent the existence check fails.
self['endpoint'] = "phedex_emulator"
self.dataBlocks = DataBlockGenerator()
self.subRequests = {}
def injectBlocks(self, node, xmlData, verbose = 0, strict = 1):
"""
do nothing don't inject block.
def tearDown(self):
WorkQueueTestCase.tearDown(self)
GlobalParams.resetParams()
def tearDown(self):
WorkQueueTestCase.tearDown(self)
EmulatorHelper.resetEmulators()
GlobalParams.resetParams()
def testReqMgrOpenRequests(self):
"""Check the mechanics of open running requests"""
# don't actually talk to ReqMgr - mock it.
globalQ = self.setupGlobalWorkqueue()
localQ = self.setupLocalQueue()
reqMgr = fakeReqMgr(splitter = 'Block', openRunningTimeout = 3600)
reqMgrInt = WorkQueueReqMgrInterface()
reqMgrInt.reqMgr = reqMgr
# 1st run should pull a request
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(len(globalQ), 2)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')
# local queue acquires and runs
globalQ.updateLocationInfo()
localQ.pullWork({'T2_XX_SiteA' : 10000, 'T2_XX_SiteB' : 10000})
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')
# start running work
globalQ.setStatus('Running', WorkflowName = reqMgr.names[0])
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to ReqMgr
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Work should not be closed yet
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Now add 2 new blocks to dbs, the reqMgr should put more work in the queue for the request
GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 2)
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(len(globalQ), 2)
# Work that can be pulled normally and request stays in running-open
globalQ.updateLocationInfo()
localQ.pullWork({'T2_XX_SiteC' : 10000})
self.assertEqual(len(globalQ), 0)
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Put the latest work to run
globalQ.setStatus('Running', WorkflowName = reqMgr.names[0])
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to ReqMgr
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')
# Change the request status manually to close it
reqMgr.status[reqMgr.names[0]] = 'running-closed'
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ) # report back to WorkQueue
self.assertEqual(len(globalQ.backend.getInboxElements(OpenForNewData = False)), 1)
# Put 1 more block in DBS for the dataset, request is closed so no more data is added
GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 1)
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
self.assertEqual(len(globalQ), 0)
# finish work
work = globalQ.status()
globalQ.setStatus('Done', elementIDs = [x.id for x in work])
reqMgrInt(globalQ)
globalQ.performQueueCleanupActions()
reqMgrInt(globalQ)
self.assertEqual(reqMgr.status[reqMgr.names[0]], 'completed')
# and removed from WorkQueue
self.assertEqual(len(globalQ.status()), 0)
# reqMgr problems should not crash client
reqMgrInt.reqMgr = None
reqMgrInt(globalQ)
reqMgr._removeSpecs()
"""
_PhEDEx_
PhEDEx Emulator
"""
# need to clean this up at some point
#// - ignore some params in dbs spec - silence pylint warnings
# pylint: disable-msg=W0613,R0201
from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams
from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator
from xml.dom.minidom import parseString
filesInDataset = GlobalParams.numOfFilesPerBlock(
) * GlobalParams.numOfBlocksPerDataset()
filesInBlock = GlobalParams.numOfFilesPerBlock()
class PhEDEx(dict):
"""
"""
def __init__(self, *args, **kwargs):
# add the end point to prevent the existence check fails.
self['endpoint'] = "phedex_emulator"
self.dataBlocks = DataBlockGenerator()
self.subRequests = {}
def injectBlocks(self, node, xmlData, verbose=0, strict=1):
"""
do nothing don't inject block.