def insertAllSEs(self, siteName, pendingSlots=0, runningSlots=0, ceName=None, plugin=None, taskList=[]): """ _insertAllSEs_ Insert all SEs into WMBS ResourceControl This uses the Services.SiteDB to insert all SEs under a common CE. It is meant to be used with WMS submission. Sites will be named siteName_SEName It expects a taskList of the following form: [{'taskType': taskType, 'priority': priority, 'maxSlots': maxSlots, 'pendingSlots' : pendingSlots}] for each entry in the taskList, a threshold is inserted into the database for EVERY SE """ from WMCore.Services.SiteDB.SiteDB import SiteDBJSON siteDB = SiteDBJSON() cmsNames = siteDB.getAllCMSNames() for cmsName in cmsNames: seNames = siteDB.cmsNametoSE(cmsName) for SE in seNames: sName = '%s_%s' % (siteName, SE) self.insertSite(siteName=sName, pendingSlots=pendingSlots, seName=SE, runningSlots=runningSlots, ceName=ceName, cmsName=cmsName, plugin=plugin) for task in taskList: if not task.has_key('maxSlots') or not task.has_key('taskType') \ or not task.has_key('priority'): msg = "Incomplete task in taskList for ResourceControl.insertAllSEs\n" msg += task raise ResourceControlException(msg) self.insertThreshold(siteName=sName, taskType=task['taskType'], maxSlots=task['maxSlots'], pendingSlots=task['pendingSlots'], priority=task['priority']) return
def insertAllSEs(self, siteName, pendingSlots = 0, runningSlots = 0, ceName = None, plugin = None, taskList = []): """ _insertAllSEs_ Insert all SEs into WMBS ResourceControl This uses the Services.SiteDB to insert all SEs under a common CE. It is meant to be used with WMS submission. Sites will be named siteName_SEName It expects a taskList of the following form: [{'taskType': taskType, 'priority': priority, 'maxSlots': maxSlots}] for each entry in the taskList, a threshold is inserted into the database for EVERY SE """ from WMCore.Services.SiteDB.SiteDB import SiteDBJSON siteDB = SiteDBJSON() cmsNames = siteDB.getAllCMSNames() for cmsName in cmsNames: seNames = siteDB.cmsNametoSE(cmsName) for SE in seNames: sName = '%s_%s' % (siteName, SE) self.insertSite(siteName = sName, pendingSlots = pendingSlots, seName = SE, runningSlots = runningSlots, ceName = ceName, cmsName = cmsName, plugin = plugin) for task in taskList: if not task.has_key('maxSlots') or not task.has_key('taskType') \ or not task.has_key('priority'): msg = "Incomplete task in taskList for ResourceControl.insertAllSEs\n" msg += task raise ResourceControlException(msg) self.insertThreshold(siteName = sName, taskType = task['taskType'], maxSlots = task['maxSlots'], priority = task['priority']) return
class SiteDBTest(unittest.TestCase): """ Unit tests for SiteScreening module """ def setUp(self): """ Setup for unit tests """ self.mySiteDB = SiteDBJSON() @attr("integration") def testCmsNametoPhEDExNode(self): """ Tests CmsNametoSE """ target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer'] results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL") self.failUnless(sorted(results) == sorted(target)) @attr("integration") def testPhEDExNodetocmsName(self): """ Tests PhEDExNodetocmsName """ result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS') self.failUnless(result == 'T1_US_FNAL') result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer') self.failUnless(result == 'T1_US_FNAL') result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC') self.failUnless(result == 'T2_UK_London_IC') # don't check this anymore, see comment in phEDExNodetocmsName function #self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName, # 'T9_DOESNT_EXIST_Buffer') @attr("integration") def testCmsNametoSE(self): """ Tests CmsNametoSE """ target = ['srm-cms.gridpp.rl.ac.uk'] results = self.mySiteDB.cmsNametoSE("T1_UK_RAL") self.failUnless(sorted(results) == sorted(target)) @attr("integration") def testSEtoCmsName(self): """ Tests CmsNametoSE """ target = 'T1_US_FNAL' results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov") self.failUnless(results == target) @attr("integration") def testCmsNametoCE(self): """ Tests CmsNametoCE """ target = ['lcgce06.gridpp.rl.ac.uk', 'lcgce07.gridpp.rl.ac.uk', 'lcgce09.gridpp.rl.ac.uk'] results = self.mySiteDB.cmsNametoCE("T1_UK_RAL") self.failUnless(sorted(results) == target) @attr("integration") def testJSONParser(self): """ Tests the JSON parser directly """ cmsName = "cmsgrid02.hep.wisc.edu" results = self.mySiteDB.getJSON("CEtoCMSName", file="CEtoCMSName", name=cmsName) self.failUnless(results['0']['name'] == "T2_US_Wisconsin") @attr("integration") def testDNUserName(self): """ Tests DN to Username lookup """ testDn = "/C=UK/O=eScience/OU=Bristol/L=IS/CN=simon metson" testUserName = "******" userName = self.mySiteDB.dnUserName(dn=testDn) self.failUnless(testUserName == userName) def testDNWithApostrophe(self): """ Tests a DN with an apostrophy in - will fail till SiteDB2 appears """ testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'" testUserName = "******" userName = self.mySiteDB.dnUserName(dn=testDn) self.failUnless(testUserName == userName) @attr("integration") def testParsingJsonWithApostrophe(self): """ Tests parsing a DN json with an apostrophe in """ json = """{"dn": "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", "user": "******"}""" d = self.mySiteDB.parser.dictParser(json) self.assertEquals("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", d['dn']) @attr("integration") def testParsingInvalidJsonWithApostrophe(self): """ Tests parsing a DN invalid json (from sitedb v1) with an apostrophe in """ json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano', 'user': '******'}""" d = self.mySiteDB.parser.dictParser(json) self.assertEquals("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano", d['dn']) json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'', 'user': '******'}""" d = self.mySiteDB.parser.dictParser(json) self.assertEquals("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", d['dn'])
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist, blockWhitelist, dbsUrl, fakeLocation=False): """ _getFiles_ Get the full information of a dataset including files, blocks, runs and lumis. Filter it using run and block white/black lists. It can receive and optional DBSUrl. """ dbsReader = DBSReader(endpoint = dbsUrl) phedexReader = PhEDEx() siteDB = SiteDBJSON() files = {} outputDatasetParts = datasetName.split("/") print "dataset",datasetName,"parts",outputDatasetParts try: #retrieve list of blocks from dataset blockNames = dbsReader.listFileBlocks(datasetName) except: raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName) has_parent = False try: parents = dbsReader.listDatasetParents( datasetName ) if parents: has_parent=True except: print "Dataset with no parent" pass #traverse each block for blockName in blockNames: #deal with white and black list. if blockBlacklist and blockName in blockBlacklist: continue if blockWhitelist and blockName not in blockWhitelist: continue #existing blocks in phedex replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName, subscribed = 'y') blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True) #has_parent = dbsReader.listBlockParents(blockName) if has_parent: try: blockFileParents = dbsReader.listFilesInBlockWithParents(blockName) except: print blockName,"does not appear to have a parent, even though it should. Very suspicious" blockFileParents = dbsReader.listFilesInBlock(blockName) else: blockFileParents = dbsReader.listFilesInBlock(blockName) blockLocations = set() #load block locations if len(replicaInfo["phedex"]["block"]) > 0: for replica in replicaInfo["phedex"]["block"][0]["replica"]: node = replica["node"] cmsSites = siteDB.PNNtoPSN(node) if type(cmsSites) != list: cmsSites = [cmsSites] for cmsName in cmsSites: se = siteDB.cmsNametoSE(cmsName) blockLocations.update(se) logging.debug("cmsName %s mapped to se %s", cmsName, se) logging.debug("PhEDEx node %s, cmsSites %s, blockLocations %s", node, cmsSites, blockLocations) # We cannot upload docs without location, so force it in case it's empty if not blockLocations: if fakeLocation: logging.info("\t\t %s\tno location", blockName) blockLocations.update([u'cmssrmdisk.fnal.gov', u'srm-eoscms.cern.ch']) elif not has_parent: ## this should be the source logging.info("Blockname: %s\tno location, ABORT", blockName) sys.exit(1) logging.info("Blockname: %s\tLocations: %s", blockName, blockLocations) #for each file on the block for blockFile in blockFiles: parentLFNs = [] #populate parent information if blockFileParents and "ParentList" in blockFileParents[0]: for fileParent in blockFileParents[0]["ParentList"]: parentLFNs.append(fileParent["LogicalFileName"]) ## remove when https://github.com/dmwm/WMCore/issues/7128 gets fixed #elif not 'RAW' in blockName: # print "no parent info" runInfo = {} #Lumis not included in file for lumiSection in blockFile["LumiList"]: if runBlacklist and lumiSection["RunNumber"] in runBlacklist: continue if runWhitelist and lumiSection["RunNumber"] not in runWhitelist: continue if lumiSection["RunNumber"] not in runInfo.keys(): runInfo[lumiSection["RunNumber"]] = [] runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"]) if len(runInfo.keys()) > 0: files[blockFile["LogicalFileName"]] = {"runs": runInfo, "events": blockFile["NumberOfEvents"], "size": blockFile["FileSize"], "locations": list(blockLocations), "parents": parentLFNs} return files
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist, blockWhitelist, dbsUrl): """ _getFiles_ Get the full information of a dataset including files, blocks, runs and lumis. Filter it using run and block white/black lists. It can receive and optional DBSUrl. """ dbsReader = DBSReader(endpoint = dbsUrl) phedexReader = PhEDEx() siteDB = SiteDBJSON() files = {} outputDatasetParts = datasetName.split("/") datasets = dbsReader.matchProcessedDatasets(outputDatasetParts[1], outputDatasetParts[3], outputDatasetParts[2]) if len(datasets) == 0: raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName) blockNames = dbsReader.listFileBlocks(datasetName) for blockName in blockNames: if blockBlacklist and blockName in blockBlacklist: continue if blockWhitelist and blockName not in blockWhitelist: continue replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName, subscribed = 'y') block = dbsReader.listFilesInBlockWithParents(blockName) blockLocations = set() if len(replicaInfo["phedex"]["block"]) > 0: for replica in replicaInfo["phedex"]["block"][0]["replica"]: node = replica["node"] cmsSites = siteDB.phEDExNodetocmsName(node) if type(cmsSites) != list: cmsSites = [cmsSites] for cmsName in cmsSites: blockLocations.update(siteDB.cmsNametoSE(cmsName)) for blockFile in block: parentLFNs = [] for fileParent in blockFile["ParentList"]: parentLFNs.append(fileParent["LogicalFileName"]) runInfo = {} for lumiSection in blockFile["LumiList"]: if runBlacklist and lumiSection["RunNumber"] in runBlacklist: continue if runWhitelist and lumiSection["RunNumber"] not in runWhitelist: continue if lumiSection["RunNumber"] not in runInfo.keys(): runInfo[lumiSection["RunNumber"]] = [] runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"]) if len(runInfo.keys()) > 0: files[blockFile["LogicalFileName"]] = {"runs": runInfo, "events": blockFile["NumberOfEvents"], "size": blockFile["FileSize"], "locations": list(blockLocations), "parents": parentLFNs} return files
class SiteDBTest(unittest.TestCase): """ Unit tests for SiteScreening module """ def setUp(self): """ Setup for unit tests """ self.mySiteDB = SiteDBJSON() def testCmsNametoPhEDExNode(self): """ Tests CmsNametoSE """ target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer'] results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL") self.failUnless(sorted(results) == sorted(target)) target = ['T1_US_FNAL_Disk'] results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL_Disk") self.failUnless(sorted(results) == sorted(target)) def testCmsNametoSE(self): """ Tests CmsNametoSE """ target = [u'srm-cms-disk.gridpp.rl.ac.uk', u'srm-cms.gridpp.rl.ac.uk'] results = self.mySiteDB.cmsNametoSE("T1_UK_RAL") self.failUnless(sorted(results) == sorted(target)) def testCmsNamePatterntoSE(self): """ Tests CmsNamePatterntoSE """ target = [u'srm-eoscms.cern.ch', u'srm-eoscms.cern.ch', u'storage01.lcg.cscs.ch', u'eoscmsftp.cern.ch'] results = self.mySiteDB.cmsNametoSE("%T2_CH") self.failUnless(sorted(results) == sorted(target)) def testSEtoCmsName(self): """ Tests CmsNametoSE """ target = [u'T1_US_FNAL'] results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov") self.failUnless(results == target) target = sorted([u'T2_CH_CERN', u'T2_CH_CERN_HLT']) results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch")) self.failUnless(sorted(results) == sorted(target)) target = sorted([u'T0_CH_CERN', u'T1_CH_CERN']) results = sorted(self.mySiteDB.seToCMSName("srm-cms.cern.ch")) self.failUnless(sorted(results) == sorted(target)) target = sorted([u'T2_CH_CERN_AI']) results = sorted(self.mySiteDB.seToCMSName("eoscmsftp.cern.ch")) self.failUnless(sorted(results) == sorted(target)) def testDNUserName(self): """ Tests DN to Username lookup """ testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gutsche/CN=582680/CN=Oliver Gutsche" testUserName = "******" userName = self.mySiteDB.dnUserName(dn=testDn) self.failUnless(testUserName == userName) def testDNWithApostrophe(self): """ Tests a DN with an apostrophy in - will fail till SiteDB2 appears """ testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'" testUserName = "******" userName = self.mySiteDB.dnUserName(dn=testDn) self.failUnless(testUserName == userName) def testSEFinder(self): """ _testSEFinder_ See if we can retrieve seNames from all sites """ seNames = self.mySiteDB.getAllSENames() self.assertTrue(len(seNames) > 1) self.assertTrue('cmssrm.fnal.gov' in seNames) return def testPNNtoPSN(self): """ _testPNNtoPSN_ Test converting PhEDEx Node Name to Processing Site Name """ result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Disk') self.failUnless(result == ['T1_US_FNAL']) result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Tape') self.failUnless(result == []) result = self.mySiteDB.PNNtoPSN('T2_UK_London_IC') self.failUnless(result == ['T2_UK_London_IC']) return def testCMSNametoList(self): result = self.mySiteDB.cmsNametoList("T1_US*", "SE") self.failUnless(result == [u'cmssrm.fnal.gov', u'cmssrmdisk.fnal.gov'])
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist, blockWhitelist, dbsUrl): """ _getFiles_ Get the full information of a dataset including files, blocks, runs and lumis. Filter it using run and block white/black lists. It can receive and optional DBSUrl. """ dbsReader = DBSReader(endpoint=dbsUrl) phedexReader = PhEDEx() siteDB = SiteDBJSON() files = {} outputDatasetParts = datasetName.split("/") print "dataset", datasetName, "parts", outputDatasetParts try: #retrieve list of blocks from dataset blockNames = dbsReader.listFileBlocks(datasetName) except: raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName) #traverse each block for blockName in blockNames: #deal with white and black list. if blockBlacklist and blockName in blockBlacklist: continue if blockWhitelist and blockName not in blockWhitelist: continue #existing blocks in phedex replicaInfo = phedexReader.getReplicaInfoForBlocks(block=blockName, subscribed='y') blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True) blockLocations = set() #load block locations if len(replicaInfo["phedex"]["block"]) > 0: for replica in replicaInfo["phedex"]["block"][0]["replica"]: node = replica["node"] cmsSites = siteDB.phEDExNodetocmsName(node) if type(cmsSites) != list: cmsSites = [cmsSites] for cmsName in cmsSites: blockLocations.update(siteDB.cmsNametoSE(cmsName)) #for each file on the block for blockFile in blockFiles: parentLFNs = [] #get parent information about file blockFileParents = dbsReader.listFilesInBlockWithParents(blockName) #populate parent information if blockFileParents and "ParentList" in blockFileParents[0]: for fileParent in blockFileParents[0]["ParentList"]: parentLFNs.append(fileParent["LogicalFileName"]) runInfo = {} #Lumis not included in file for lumiSection in blockFile["LumiList"]: if runBlacklist and lumiSection["RunNumber"] in runBlacklist: continue if runWhitelist and lumiSection[ "RunNumber"] not in runWhitelist: continue if lumiSection["RunNumber"] not in runInfo.keys(): runInfo[lumiSection["RunNumber"]] = [] runInfo[lumiSection["RunNumber"]].append( lumiSection["LumiSectionNumber"]) if len(runInfo.keys()) > 0: files[blockFile["LogicalFileName"]] = { "runs": runInfo, "events": blockFile["NumberOfEvents"], "size": blockFile["FileSize"], "locations": list(blockLocations), "parents": parentLFNs } return files
class SiteDBTest(unittest.TestCase): """ Unit tests for SiteScreening module """ def setUp(self): """ Setup for unit tests """ EmulatorHelper.setEmulators(siteDB = True) self.mySiteDB = SiteDBJSON() def tearDown(self): EmulatorHelper.resetEmulators() def testCmsNametoPhEDExNode(self): """ Tests CmsNametoSE """ target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer'] results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL") self.failUnless(sorted(results) == sorted(target)) def testPhEDExNodetocmsName(self): """ Tests PhEDExNodetocmsName """ result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS') self.failUnless(result == 'T1_US_FNAL') result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer') self.failUnless(result == 'T1_US_FNAL') result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC') self.failUnless(result == 'T2_UK_London_IC') # don't check this anymore, see comment in phEDExNodetocmsName function #self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName, # 'T9_DOESNT_EXIST_Buffer') def testCmsNametoSE(self): """ Tests CmsNametoSE """ target = ['srm-cms.gridpp.rl.ac.uk'] results = self.mySiteDB.cmsNametoSE("T1_UK_RAL") self.failUnless(sorted(results) == sorted(target)) def testSEtoCmsName(self): """ Tests CmsNametoSE """ target = ['T1_US_FNAL'] results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov") self.failUnless(results == target) target = sorted(['T2_CH_CERN', 'T2_CH_CERN_HLT']) results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch")) self.failUnless(results == target) def testCmsNametoCE(self): """ Tests CmsNametoCE """ target = ['lcgce11.gridpp.rl.ac.uk', 'lcgce10.gridpp.rl.ac.uk', 'lcgce02.gridpp.rl.ac.uk'] results = self.mySiteDB.cmsNametoCE("T1_UK_RAL") self.failUnless(sorted(results) == sorted(target)) def testDNUserName(self): """ Tests DN to Username lookup """ testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gutsche/CN=582680/CN=Oliver Gutsche" testUserName = "******" userName = self.mySiteDB.dnUserName(dn=testDn) self.failUnless(testUserName == userName) @attr("integration") def testDNWithApostrophe(self): """ Tests a DN with an apostrophy in - will fail till SiteDB2 appears """ testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'" testUserName = "******" userName = self.mySiteDB.dnUserName(dn=testDn) self.failUnless(testUserName == userName) def testSEFinder(self): """ _testSEFinder_ See if we can retrieve seNames from all sites """ seNames = self.mySiteDB.getAllSENames() self.assertTrue(len(seNames) > 1) self.assertTrue('cmssrm.fnal.gov' in seNames) return
class SiteDBTest(unittest.TestCase): """ Unit tests for SiteScreening module """ def setUp(self): """ Setup for unit tests """ self.mySiteDB = SiteDBJSON() def testCmsNametoPhEDExNode(self): """ Tests CmsNametoSE """ target = ['T1_US_FNAL_MSS', 'T1_US_FNAL_Buffer'] results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL") self.failUnless(sorted(results) == sorted(target)) def testPhEDExNodetocmsName(self): """ Tests PhEDExNodetocmsName """ result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS') self.failUnless(result == 'T1_US_FNAL') result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer') self.failUnless(result == 'T1_US_FNAL') result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC') self.failUnless(result == 'T2_UK_London_IC') # don't check this anymore, see comment in phEDExNodetocmsName function #self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName, # 'T9_DOESNT_EXIST_Buffer') def testCmsNametoSE(self): """ Tests CmsNametoSE """ target = ['srm-cms.gridpp.rl.ac.uk', 'srm-cms-disk.gridpp.rl.ac.uk'] results = self.mySiteDB.cmsNametoSE("T1_UK_RAL") self.failUnless(sorted(results) == sorted(target)) def testSEtoCmsName(self): """ Tests CmsNametoSE """ target = 'T1_US_FNAL' results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov") self.failUnless(results == target) def testCmsNametoCE(self): """ Tests CmsNametoCE """ target = [ 'lcgce09.gridpp.rl.ac.uk', 'lcgce06.gridpp.rl.ac.uk', 'lcgce07.gridpp.rl.ac.uk', 'lcgce07.gridpp.rl.ac.uk' ] results = self.mySiteDB.cmsNametoCE("T1_UK_RAL") self.failUnless(sorted(results) == sorted(target)) def testJSONParser(self): """ Tests the JSON parser directly """ cmsName = "cmsgrid02.hep.wisc.edu" results = self.mySiteDB.getJSON("CEtoCMSName", file="CEtoCMSName", name=cmsName) self.failUnless(results['0']['name'] == "T2_US_Wisconsin") def testDNUserName(self): """ Tests DN to Username lookup """ testDn = "/C=UK/O=eScience/OU=Bristol/L=IS/CN=simon metson" testUserName = "******" userName = self.mySiteDB.dnUserName(dn=testDn) self.failUnless(testUserName == userName) @attr("integration") def testDNWithApostrophe(self): """ Tests a DN with an apostrophy in - will fail till SiteDB2 appears """ testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'" testUserName = "******" userName = self.mySiteDB.dnUserName(dn=testDn) self.failUnless(testUserName == userName) def testSEFinder(self): """ _testSEFinder_ See if we can retrieve seNames from all sites """ ceNames = self.mySiteDB.getAllSENames() self.assertTrue(len(ceNames) > 1) self.assertTrue('cmssrm.fnal.gov' in ceNames) return @attr("integration") def testParsingJsonWithApostrophe(self): """ Tests parsing a DN json with an apostrophe in """ json = """{"dn": "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", "user": "******"}""" d = self.mySiteDB.parser.dictParser(json) self.assertEquals( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", d['dn']) @attr("integration") def testParsingInvalidJsonWithApostrophe(self): """ Tests parsing a DN invalid json (from sitedb v1) with an apostrophe in """ json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano', 'user': '******'}""" d = self.mySiteDB.parser.dictParser(json) self.assertEquals( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano", d['dn']) json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'', 'user': '******'}""" d = self.mySiteDB.parser.dictParser(json) self.assertEquals( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", d['dn'])
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist, blockWhitelist, dbsUrl): """ _getFiles_ Get the full information of a dataset including files, blocks, runs and lumis. Filter it using run and block white/black lists. It can receive and optional DBSUrl. """ dbsReader = DBSReader(endpoint = dbsUrl) phedexReader = PhEDEx() siteDB = SiteDBJSON() files = {} outputDatasetParts = datasetName.split("/") print "dataset",datasetName,"parts",outputDatasetParts try: #retrieve list of blocks from dataset blockNames = dbsReader.listFileBlocks(datasetName) except: raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName) #traverse each block for blockName in blockNames: #deal with white and black list. if blockBlacklist and blockName in blockBlacklist: continue if blockWhitelist and blockName not in blockWhitelist: continue #existing blocks in phedex replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName, subscribed = 'y') blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True) blockLocations = set() #load block locations if len(replicaInfo["phedex"]["block"]) > 0: for replica in replicaInfo["phedex"]["block"][0]["replica"]: node = replica["node"] cmsSites = siteDB.phEDExNodetocmsName(node) if type(cmsSites) != list: cmsSites = [cmsSites] for cmsName in cmsSites: blockLocations.update(siteDB.cmsNametoSE(cmsName)) #for each file on the block for blockFile in blockFiles: parentLFNs = [] #get parent information about file blockFileParents = dbsReader.listFilesInBlockWithParents(blockName) #populate parent information if blockFileParents and "ParentList" in blockFileParents[0]: for fileParent in blockFileParents[0]["ParentList"]: parentLFNs.append(fileParent["LogicalFileName"]) runInfo = {} #Lumis not included in file for lumiSection in blockFile["LumiList"]: if runBlacklist and lumiSection["RunNumber"] in runBlacklist: continue if runWhitelist and lumiSection["RunNumber"] not in runWhitelist: continue if lumiSection["RunNumber"] not in runInfo.keys(): runInfo[lumiSection["RunNumber"]] = [] runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"]) if len(runInfo.keys()) > 0: files[blockFile["LogicalFileName"]] = {"runs": runInfo, "events": blockFile["NumberOfEvents"], "size": blockFile["FileSize"], "locations": list(blockLocations), "parents": parentLFNs} return files
class SiteDBTest(unittest.TestCase): """ Unit tests for SiteScreening module """ def setUp(self): """ Setup for unit tests """ EmulatorHelper.setEmulators(siteDB = True) self.mySiteDB = SiteDBJSON() def tearDown(self): EmulatorHelper.resetEmulators() def testCmsNametoPhEDExNode(self): """ Tests CmsNametoSE """ target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer'] results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL") self.failUnless(sorted(results) == sorted(target)) def testPhEDExNodetocmsName(self): """ Tests PhEDExNodetocmsName """ result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS') self.failUnless(result == 'T1_US_FNAL') result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer') self.failUnless(result == 'T1_US_FNAL') result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC') self.failUnless(result == 'T2_UK_London_IC') # don't check this anymore, see comment in phEDExNodetocmsName function #self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName, # 'T9_DOESNT_EXIST_Buffer') def testCmsNametoSE(self): """ Tests CmsNametoSE """ target = ['srm-cms.gridpp.rl.ac.uk'] results = self.mySiteDB.cmsNametoSE("T1_UK_RAL") self.failUnless(sorted(results) == sorted(target)) def testCmsNamePatterntoSE(self): """ Tests CmsNamePatterntoSE """ target = ['T2_XX_SiteA', 'T2_XX_SiteB', 'T2_XX_SiteC'] results = self.mySiteDB.cmsNametoSE("%T2_XX") self.failUnless(sorted(results) == sorted(target)) def testSEtoCmsName(self): """ Tests CmsNametoSE """ target = ['T1_US_FNAL'] results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov") self.failUnless(results == target) target = sorted(['T2_CH_CERN', 'T2_CH_CERN_HLT']) results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch")) self.failUnless(results == target) def testCmsNametoCE(self): """ Tests CmsNametoCE """ target = ['lcgce11.gridpp.rl.ac.uk', 'lcgce10.gridpp.rl.ac.uk', 'lcgce02.gridpp.rl.ac.uk'] results = self.mySiteDB.cmsNametoCE("T1_UK_RAL") self.failUnless(sorted(results) == sorted(target)) def testCmsNamePatterntoCE(self): """ Tests CmsNamePatterntoCE """ target = ['T2_XX_SiteA', 'T2_XX_SiteB', 'T2_XX_SiteC'] results = self.mySiteDB.cmsNametoCE("%T2_XX") self.failUnless(sorted(results) == sorted(target)) def testDNUserName(self): """ Tests DN to Username lookup """ testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gutsche/CN=582680/CN=Oliver Gutsche" testUserName = "******" userName = self.mySiteDB.dnUserName(dn=testDn) self.failUnless(testUserName == userName) @attr("integration") def testDNWithApostrophe(self): """ Tests a DN with an apostrophy in - will fail till SiteDB2 appears """ testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'" testUserName = "******" userName = self.mySiteDB.dnUserName(dn=testDn) self.failUnless(testUserName == userName) def testSEFinder(self): """ _testSEFinder_ See if we can retrieve seNames from all sites """ seNames = self.mySiteDB.getAllSENames() self.assertTrue(len(seNames) > 1) self.assertTrue('cmssrm.fnal.gov' in seNames) return