示例#1
0
    def testParentage(self):
        """
        1. check whether parent files are created in wmbs.
        2. check parent files are associated to child.
        3. When 2 specs with the same input data (one with parent processing, one without it)
           is inserted, if one without parent processing inserted first then the other with
           parent processing insert, it still needs to create parent files although child files
           are duplicate
        """
        block = self.dataset + "#1"
        wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(
            self.wmspec, block, parentFlag=False, detail=True)
        # file creation without parents
        self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
        wmbs.topLevelFileset.loadData()

        for child in wmbs.topLevelFileset.files:
            # no parent per child
            self.assertEqual(len(child["parents"]), 0)

        wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec,
                                                               block,
                                                               parentFlag=True,
                                                               detail=True)
        self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)

        wmbs.topLevelFileset.loadData()

        for child in wmbs.topLevelFileset.files:
            # one parent per child
            self.assertEqual(len(child["parents"]), 1)
示例#2
0
    def testParentage(self):
        """
        1. check whether parent files are created in wmbs.
        2. check parent files are associated to child.
        3. When 2 specs with the same input data (one with parent processing, one without it)
           is inserted, if one without parent processing inserted first then the other with
           parent processing insert, it still needs to create parent files although child files
           are duplicate
        """
        block = self.dataset + "#1"
        wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block,
                                                parentFlag = False, detail = True)
        # file creation without parents
        self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
        wmbs.topLevelFileset.loadData()

        for child in wmbs.topLevelFileset.files:
            # no parent per child
            self.assertEqual(len(child["parents"]), 0)

        wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block,
                                                parentFlag = True, detail = True)
        self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)

        wmbs.topLevelFileset.loadData()

        for child in wmbs.topLevelFileset.files:
            # one parent per child
            self.assertEqual(len(child["parents"]), 1)
 def setUp(self):
     WorkQueueTestCase.setUp(self)
     GlobalParams.resetParams()
     self.globalQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
                                       self.globalQDB)
     self.localQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
                                      self.localQInboxDB)
示例#4
0
    def testParentProcessing(self):
        """
        Enqueue and get work for a processing WMSpec.
        """
        specfile = self.parentProcSpec.specUrl()
        njobs = [5, 10] # array of jobs per block
        total = sum(njobs)

        # Queue Work & check accepted
        self.queue.queueWork(specfile)
        self.queue.processInboundWork()
        self.assertEqual(len(njobs), len(self.queue))

        self.queue.updateLocationInfo()
        # No resources
        work = self.queue.getWork({})
        self.assertEqual(len(work), 0)
        work = self.queue.getWork({'T2_XX_SiteA' : 0,
                                   'T2_XX_SiteB' : 0})
        self.assertEqual(len(work), 0)

        # Only 1 block at SiteB - get 1 work element when any resources free
        work = self.queue.getWork({'T2_XX_SiteB' : 1})
        self.assertEqual(len(work), 1)
        self.assertEqual(work[0]["NumOfFilesAdded"], GlobalParams.numOfFilesPerBlock() * 2)

        # claim remaining work
        work = self.queue.getWork({'T2_XX_SiteA' : total, 'T2_XX_SiteB' : total})
        self.assertEqual(len(work), 1)
        self.assertEqual(work[0]["NumOfFilesAdded"], GlobalParams.numOfFilesPerBlock() * 2)

        # no more work available
        self.assertEqual(0, len(self.queue.getWork({'T2_XX_SiteA' : total})))
 def setUp(self):
     WorkQueueTestCase.setUp(self)
     EmulatorHelper.setEmulators(phedex = True, dbs = True,
                                 siteDB = True, requestMgr = False)
     GlobalParams.resetParams()
     self.globalQCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB)
     self.localQCouchUrl =  "%s/%s" % (self.testInit.couchUrl,
                                       self.localQInboxDB)
    def setUp(self):
        self.reqmgr2_endpoint = "https://cmsweb-testbed.cern.ch/reqmgr2"

        WorkQueueTestCase.setUp(self)
        GlobalParams.resetParams()
        self.globalQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
                                          self.globalQDB)
        self.localQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
                                         self.localQInboxDB)
示例#7
0
 def testLumiMaskRestrictionsOK(self):
     block = self.dataset + "#1"
     self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = ['1']
     self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = ['1,1']
     wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
     files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
     self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
示例#8
0
 def testReRecoWhiteRunRestriction(self):
     block = self.dataset + "#2"
     # Run Whitelist
     self.topLevelTask.setInputRunWhitelist([2])
     wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
     files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
     self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
示例#9
0
 def testReRecoWhiteRunRestriction(self):
     block = self.dataset + "#2"
     # Run Whitelist
     self.topLevelTask.setInputRunWhitelist([2])
     wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
     files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
     self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
示例#10
0
 def testLumiMaskRestrictionsOK(self):
     block = self.dataset + "#1"
     self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = ['1']
     self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = ['1,1']
     wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
     files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
     self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
示例#11
0
 def tearDown(self):
     WorkQueueTestCase.tearDown(self)
     GlobalParams.resetParams()
示例#12
0
 def setUp(self):
     WorkQueueTestCase.setUp(self)
     GlobalParams.resetParams()
     self.globalQCouchUrl = "%s/%s" % (self.testInit.couchUrl, self.globalQDB)
     self.localQCouchUrl = "%s/%s" % (self.testInit.couchUrl,
                                      self.localQInboxDB)
示例#13
0
    def testReqMgrOpenRequests(self):
        """Check the mechanics of open running requests"""
        # don't actually talk to ReqMgr - mock it.
        globalQ = self.setupGlobalWorkqueue()
        localQ = self.setupLocalQueue()
        reqMgr = fakeReqMgr(splitter='Block', openRunningTimeout=3600)
        reqMgrInt = WorkQueueReqMgrInterface()
        reqMgrInt.reqMgr = reqMgr

        # 1st run should pull a request
        self.assertEqual(len(globalQ), 0)
        reqMgrInt(globalQ)
        self.assertEqual(len(globalQ), 2)
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')

        # local queue acquires and runs
        globalQ.updateLocationInfo()
        localQ.pullWork({'T2_XX_SiteA': 10000, 'T2_XX_SiteB': 10000})
        self.assertEqual(len(globalQ), 0)
        reqMgrInt(globalQ)
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')

        # start running work
        globalQ.setStatus('Running', WorkflowName=reqMgr.names[0])
        globalQ.performQueueCleanupActions()
        reqMgrInt(globalQ)  # report back to ReqMgr
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')

        # Work should not be closed yet
        reqMgrInt(globalQ)
        globalQ.performQueueCleanupActions()
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')

        # Now add 2 new blocks to dbs, the reqMgr should put more work in the queue for the request
        GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 2)
        reqMgrInt(globalQ)
        globalQ.performQueueCleanupActions()
        self.assertEqual(len(globalQ), 2)

        # Work that can be pulled normally and request stays in running-open
        globalQ.updateLocationInfo()
        localQ.pullWork({'T2_XX_SiteC': 10000})
        self.assertEqual(len(globalQ), 0)
        reqMgrInt(globalQ)
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')

        # Put the latest work to run
        globalQ.setStatus('Running', WorkflowName=reqMgr.names[0])
        globalQ.performQueueCleanupActions()
        reqMgrInt(globalQ)  # report back to ReqMgr
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')

        # Change the request status manually to close it
        reqMgr.status[reqMgr.names[0]] = 'running-closed'
        globalQ.performQueueCleanupActions()
        reqMgrInt(globalQ)  # report back to WorkQueue
        self.assertEqual(len(globalQ.backend.getInboxElements(OpenForNewData=False)), 1)

        # Put 1 more block in DBS for the dataset, request is closed so no more data is added
        GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 1)
        reqMgrInt(globalQ)
        globalQ.performQueueCleanupActions()
        self.assertEqual(len(globalQ), 0)

        # finish work
        work = globalQ.status()
        globalQ.setStatus('Done', elementIDs=[x.id for x in work])
        reqMgrInt(globalQ)
        globalQ.performQueueCleanupActions()
        reqMgrInt(globalQ)
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'completed')
        # and removed from WorkQueue
        self.assertEqual(len(globalQ.status()), 0)

        # reqMgr problems should not crash client
        reqMgrInt.reqMgr = None
        reqMgrInt(globalQ)
        reqMgr._removeSpecs()
示例#14
0
文件: PhEDEx.py 项目: ticoann/WMCore



# need to clean this up at some point

#//     - ignore some params in dbs spec - silence pylint warnings
# pylint: disable-msg=W0613,R0201
from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams
from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator

from xml.dom.minidom import parseString



filesInDataset = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfBlocksPerDataset()
filesInBlock = GlobalParams.numOfFilesPerBlock()

class PhEDEx(dict):
    """
    """
    def __init__(self, *args, **kwargs):
        # add the end point to prevent the existence check fails.
        self['endpoint'] = "phedex_emulator"
        self.dataBlocks = DataBlockGenerator()
        self.subRequests = {}
        
    def injectBlocks(self, node, xmlData, verbose = 0, strict = 1):

        """
        do nothing don't inject block.
 def tearDown(self):
     WorkQueueTestCase.tearDown(self)
     GlobalParams.resetParams()
 def tearDown(self):
     WorkQueueTestCase.tearDown(self)
     EmulatorHelper.resetEmulators()
     GlobalParams.resetParams()
    def testReqMgrOpenRequests(self):
        """Check the mechanics of open running requests"""
        # don't actually talk to ReqMgr - mock it.
        globalQ = self.setupGlobalWorkqueue()
        localQ = self.setupLocalQueue()
        reqMgr = fakeReqMgr(splitter = 'Block', openRunningTimeout = 3600)
        reqMgrInt = WorkQueueReqMgrInterface()
        reqMgrInt.reqMgr = reqMgr

        # 1st run should pull a request
        self.assertEqual(len(globalQ), 0)
        reqMgrInt(globalQ)
        self.assertEqual(len(globalQ), 2)
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')

        # local queue acquires and runs
        globalQ.updateLocationInfo()
        localQ.pullWork({'T2_XX_SiteA' : 10000, 'T2_XX_SiteB' : 10000})
        self.assertEqual(len(globalQ), 0)
        reqMgrInt(globalQ)
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'acquired')

        # start running work
        globalQ.setStatus('Running', WorkflowName = reqMgr.names[0])
        globalQ.performQueueCleanupActions()
        reqMgrInt(globalQ) # report back to ReqMgr
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')

        # Work should not be closed yet
        reqMgrInt(globalQ)
        globalQ.performQueueCleanupActions()
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')

        # Now add 2 new blocks to dbs, the reqMgr should put more work in the queue for the request
        GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 2)
        reqMgrInt(globalQ)
        globalQ.performQueueCleanupActions()
        self.assertEqual(len(globalQ), 2)

        # Work that can be pulled normally and request stays in running-open
        globalQ.updateLocationInfo()
        localQ.pullWork({'T2_XX_SiteC' : 10000})
        self.assertEqual(len(globalQ), 0)
        reqMgrInt(globalQ)
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')

        # Put the latest work to run
        globalQ.setStatus('Running', WorkflowName = reqMgr.names[0])
        globalQ.performQueueCleanupActions()
        reqMgrInt(globalQ) # report back to ReqMgr
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'running-open')

        # Change the request status manually to close it
        reqMgr.status[reqMgr.names[0]] = 'running-closed'
        globalQ.performQueueCleanupActions()
        reqMgrInt(globalQ) # report back to WorkQueue
        self.assertEqual(len(globalQ.backend.getInboxElements(OpenForNewData = False)), 1)

        # Put 1 more block in DBS for the dataset, request is closed so no more data is added
        GlobalParams.setNumOfBlocksPerDataset(GlobalParams.numOfBlocksPerDataset() + 1)
        reqMgrInt(globalQ)
        globalQ.performQueueCleanupActions()
        self.assertEqual(len(globalQ), 0)

        # finish work
        work = globalQ.status()
        globalQ.setStatus('Done', elementIDs = [x.id for x in work])
        reqMgrInt(globalQ)
        globalQ.performQueueCleanupActions()
        reqMgrInt(globalQ)
        self.assertEqual(reqMgr.status[reqMgr.names[0]], 'completed')
        # and removed from WorkQueue
        self.assertEqual(len(globalQ.status()), 0)

        # reqMgr problems should not crash client
        reqMgrInt.reqMgr = None
        reqMgrInt(globalQ)
        reqMgr._removeSpecs()
示例#18
0
文件: PhEDEx.py 项目: ticoann/WMCore
"""
_PhEDEx_

PhEDEx Emulator
"""

# need to clean this up at some point

#//     - ignore some params in dbs spec - silence pylint warnings
# pylint: disable-msg=W0613,R0201
from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams
from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator

from xml.dom.minidom import parseString

filesInDataset = GlobalParams.numOfFilesPerBlock(
) * GlobalParams.numOfBlocksPerDataset()
filesInBlock = GlobalParams.numOfFilesPerBlock()


class PhEDEx(dict):
    """
    """
    def __init__(self, *args, **kwargs):
        # add the end point to prevent the existence check fails.
        self['endpoint'] = "phedex_emulator"
        self.dataBlocks = DataBlockGenerator()
        self.subRequests = {}

    def injectBlocks(self, node, xmlData, verbose=0, strict=1):
        """
        do nothing don't inject block.