def testParentage(self):
"""
1. check whether parent files are created in wmbs.
2. check parent files are associated to child.
3. When 2 specs with the same input data (one with parent processing, one without it)
is inserted, if one without parent processing inserted first then the other with
parent processing insert, it still needs to create parent files although child files
are duplicate
"""
block = self.dataset + "#1"
wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(
self.wmspec, block, parentFlag=False, detail=True)
# file creation without parents
self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
wmbs.topLevelFileset.loadData()
for child in wmbs.topLevelFileset.files:
# no parent per child
self.assertEqual(len(child["parents"]), 0)
wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec,
block,
parentFlag=True,
detail=True)
self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
wmbs.topLevelFileset.loadData()
for child in wmbs.topLevelFileset.files:
# one parent per child
self.assertEqual(len(child["parents"]), 1)
def testParentage(self):
"""
1. check whether parent files are created in wmbs.
2. check parent files are associated to child.
3. When 2 specs with the same input data (one with parent processing, one without it)
is inserted, if one without parent processing inserted first then the other with
parent processing insert, it still needs to create parent files although child files
are duplicate
"""
block = self.dataset + "#1"
wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block,
parentFlag = False, detail = True)
# file creation without parents
self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
wmbs.topLevelFileset.loadData()
for child in wmbs.topLevelFileset.files:
# no parent per child
self.assertEqual(len(child["parents"]), 0)
wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block,
parentFlag = True, detail = True)
self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
wmbs.topLevelFileset.loadData()
for child in wmbs.topLevelFileset.files:
# one parent per child
self.assertEqual(len(child["parents"]), 1)
def testParentProcessing(self):
"""
Enqueue and get work for a processing WMSpec.
"""
specfile = self.parentProcSpec.specUrl()
njobs = [5, 10] # array of jobs per block
total = sum(njobs)
# Queue Work & check accepted
self.queue.queueWork(specfile)
self.queue.processInboundWork()
self.assertEqual(len(njobs), len(self.queue))
self.queue.updateLocationInfo()
# No resources
work = self.queue.getWork({})
self.assertEqual(len(work), 0)
work = self.queue.getWork({'T2_XX_SiteA' : 0,
'T2_XX_SiteB' : 0})
self.assertEqual(len(work), 0)
# Only 1 block at SiteB - get 1 work element when any resources free
work = self.queue.getWork({'T2_XX_SiteB' : 1})
self.assertEqual(len(work), 1)
self.assertEqual(work[0]["NumOfFilesAdded"], GlobalParams.numOfFilesPerBlock() * 2)
# claim remaining work
work = self.queue.getWork({'T2_XX_SiteA' : total, 'T2_XX_SiteB' : total})
self.assertEqual(len(work), 1)
self.assertEqual(work[0]["NumOfFilesAdded"], GlobalParams.numOfFilesPerBlock() * 2)
# no more work available
self.assertEqual(0, len(self.queue.getWork({'T2_XX_SiteA' : total})))
def testLumiMaskRestrictionsOK(self):
block = self.dataset + "#1"
self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = ['1']
self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = ['1,1']
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
def testReRecoWhiteRunRestriction(self):
block = self.dataset + "#2"
# Run Whitelist
self.topLevelTask.setInputRunWhitelist([2])
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
def testReRecoWhiteRunRestriction(self):
block = self.dataset + "#2"
# Run Whitelist
self.topLevelTask.setInputRunWhitelist([2])
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
def testLumiMaskRestrictionsOK(self):
block = self.dataset + "#1"
self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = ['1']
self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = ['1,1']
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
# need to clean this up at some point
#// - ignore some params in dbs spec - silence pylint warnings
# pylint: disable-msg=W0613,R0201
from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams
from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator
from xml.dom.minidom import parseString
filesInDataset = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfBlocksPerDataset()
filesInBlock = GlobalParams.numOfFilesPerBlock()
class PhEDEx(dict):
"""
"""
def __init__(self, *args, **kwargs):
# add the end point to prevent the existence check fails.
self['endpoint'] = "phedex_emulator"
self.dataBlocks = DataBlockGenerator()
self.subRequests = {}
def injectBlocks(self, node, xmlData, verbose = 0, strict = 1):
"""
do nothing don't inject block.
"""
_PhEDEx_
PhEDEx Emulator
"""
# need to clean this up at some point
#// - ignore some params in dbs spec - silence pylint warnings
# pylint: disable-msg=W0613,R0201
from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams
from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator
from xml.dom.minidom import parseString
filesInDataset = GlobalParams.numOfFilesPerBlock(
) * GlobalParams.numOfBlocksPerDataset()
filesInBlock = GlobalParams.numOfFilesPerBlock()
class PhEDEx(dict):
"""
"""
def __init__(self, *args, **kwargs):
# add the end point to prevent the existence check fails.
self['endpoint'] = "phedex_emulator"
self.dataBlocks = DataBlockGenerator()
self.subRequests = {}
def injectBlocks(self, node, xmlData, verbose=0, strict=1):
"""
do nothing don't inject block.
