def testParentage(self): """ 1. check whether parent files are created in wmbs. 2. check parent files are associated to child. 3. When 2 specs with the same input data (one with parent processing, one without it) is inserted, if one without parent processing inserted first then the other with parent processing insert, it still needs to create parent files although child files are duplicate """ block = self.dataset + "#1" wmbs, sub, numFiles = self.createWMBSHelperWithTopTask( self.wmspec, block, parentFlag=False, detail=True) # file creation without parents self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles) wmbs.topLevelFileset.loadData() for child in wmbs.topLevelFileset.files: # no parent per child self.assertEqual(len(child["parents"]), 0) wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block, parentFlag=True, detail=True) self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles) wmbs.topLevelFileset.loadData() for child in wmbs.topLevelFileset.files: # one parent per child self.assertEqual(len(child["parents"]), 1)
def testParentage(self): """ 1. check whether parent files are created in wmbs. 2. check parent files are associated to child. 3. When 2 specs with the same input data (one with parent processing, one without it) is inserted, if one without parent processing inserted first then the other with parent processing insert, it still needs to create parent files although child files are duplicate """ block = self.dataset + "#1" wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block, parentFlag = False, detail = True) # file creation without parents self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles) wmbs.topLevelFileset.loadData() for child in wmbs.topLevelFileset.files: # no parent per child self.assertEqual(len(child["parents"]), 0) wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block, parentFlag = True, detail = True) self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles) wmbs.topLevelFileset.loadData() for child in wmbs.topLevelFileset.files: # one parent per child self.assertEqual(len(child["parents"]), 1)
def testParentProcessing(self): """ Enqueue and get work for a processing WMSpec. """ specfile = self.parentProcSpec.specUrl() njobs = [5, 10] # array of jobs per block total = sum(njobs) # Queue Work & check accepted self.queue.queueWork(specfile) self.queue.processInboundWork() self.assertEqual(len(njobs), len(self.queue)) self.queue.updateLocationInfo() # No resources work = self.queue.getWork({}) self.assertEqual(len(work), 0) work = self.queue.getWork({'T2_XX_SiteA' : 0, 'T2_XX_SiteB' : 0}) self.assertEqual(len(work), 0) # Only 1 block at SiteB - get 1 work element when any resources free work = self.queue.getWork({'T2_XX_SiteB' : 1}) self.assertEqual(len(work), 1) self.assertEqual(work[0]["NumOfFilesAdded"], GlobalParams.numOfFilesPerBlock() * 2) # claim remaining work work = self.queue.getWork({'T2_XX_SiteA' : total, 'T2_XX_SiteB' : total}) self.assertEqual(len(work), 1) self.assertEqual(work[0]["NumOfFilesAdded"], GlobalParams.numOfFilesPerBlock() * 2) # no more work available self.assertEqual(0, len(self.queue.getWork({'T2_XX_SiteA' : total})))
def testLumiMaskRestrictionsOK(self): block = self.dataset + "#1" self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = ['1'] self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = ['1,1'] wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files']) self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
def testReRecoWhiteRunRestriction(self): block = self.dataset + "#2" # Run Whitelist self.topLevelTask.setInputRunWhitelist([2]) wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files']) self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
# need to clean this up at some point #// - ignore some params in dbs spec - silence pylint warnings # pylint: disable-msg=W0613,R0201 from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator from xml.dom.minidom import parseString filesInDataset = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfBlocksPerDataset() filesInBlock = GlobalParams.numOfFilesPerBlock() class PhEDEx(dict): """ """ def __init__(self, *args, **kwargs): # add the end point to prevent the existence check fails. self['endpoint'] = "phedex_emulator" self.dataBlocks = DataBlockGenerator() self.subRequests = {} def injectBlocks(self, node, xmlData, verbose = 0, strict = 1): """ do nothing don't inject block.
""" _PhEDEx_ PhEDEx Emulator """ # need to clean this up at some point #// - ignore some params in dbs spec - silence pylint warnings # pylint: disable-msg=W0613,R0201 from WMQuality.Emulators.DataBlockGenerator.Globals import GlobalParams from WMQuality.Emulators.DataBlockGenerator.DataBlockGenerator import DataBlockGenerator from xml.dom.minidom import parseString filesInDataset = GlobalParams.numOfFilesPerBlock( ) * GlobalParams.numOfBlocksPerDataset() filesInBlock = GlobalParams.numOfFilesPerBlock() class PhEDEx(dict): """ """ def __init__(self, *args, **kwargs): # add the end point to prevent the existence check fails. self['endpoint'] = "phedex_emulator" self.dataBlocks = DataBlockGenerator() self.subRequests = {} def injectBlocks(self, node, xmlData, verbose=0, strict=1): """ do nothing don't inject block.