def insertAllSEs(self,
siteName,
pendingSlots=0,
runningSlots=0,
ceName=None,
plugin=None,
taskList=[]):
"""
_insertAllSEs_
Insert all SEs into WMBS ResourceControl
This uses the Services.SiteDB to insert all SEs under a common
CE. It is meant to be used with WMS submission.
Sites will be named siteName_SEName
It expects a taskList of the following form:
[{'taskType': taskType, 'priority': priority, 'maxSlots': maxSlots, 'pendingSlots' : pendingSlots}]
for each entry in the taskList, a threshold is inserted into the database
for EVERY SE
"""
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
siteDB = SiteDBJSON()
cmsNames = siteDB.getAllCMSNames()
for cmsName in cmsNames:
seNames = siteDB.cmsNametoSE(cmsName)
for SE in seNames:
sName = '%s_%s' % (siteName, SE)
self.insertSite(siteName=sName,
pendingSlots=pendingSlots,
seName=SE,
runningSlots=runningSlots,
ceName=ceName,
cmsName=cmsName,
plugin=plugin)
for task in taskList:
if not task.has_key('maxSlots') or not task.has_key('taskType') \
or not task.has_key('priority'):
msg = "Incomplete task in taskList for ResourceControl.insertAllSEs\n"
msg += task
raise ResourceControlException(msg)
self.insertThreshold(siteName=sName,
taskType=task['taskType'],
maxSlots=task['maxSlots'],
pendingSlots=task['pendingSlots'],
priority=task['priority'])
return
def insertAllSEs(self, siteName, pendingSlots = 0, runningSlots = 0,
ceName = None, plugin = None,
taskList = []):
"""
_insertAllSEs_
Insert all SEs into WMBS ResourceControl
This uses the Services.SiteDB to insert all SEs under a common
CE. It is meant to be used with WMS submission.
Sites will be named siteName_SEName
It expects a taskList of the following form:
[{'taskType': taskType, 'priority': priority, 'maxSlots': maxSlots}]
for each entry in the taskList, a threshold is inserted into the database
for EVERY SE
"""
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON
siteDB = SiteDBJSON()
cmsNames = siteDB.getAllCMSNames()
for cmsName in cmsNames:
seNames = siteDB.cmsNametoSE(cmsName)
for SE in seNames:
sName = '%s_%s' % (siteName, SE)
self.insertSite(siteName = sName, pendingSlots = pendingSlots,
seName = SE, runningSlots = runningSlots,
ceName = ceName, cmsName = cmsName, plugin = plugin)
for task in taskList:
if not task.has_key('maxSlots') or not task.has_key('taskType') \
or not task.has_key('priority'):
msg = "Incomplete task in taskList for ResourceControl.insertAllSEs\n"
msg += task
raise ResourceControlException(msg)
self.insertThreshold(siteName = sName, taskType = task['taskType'],
maxSlots = task['maxSlots'], priority = task['priority'])
return
class SiteDBTest(unittest.TestCase):
"""
Unit tests for SiteScreening module
"""
def setUp(self):
"""
Setup for unit tests
"""
self.mySiteDB = SiteDBJSON()
@attr("integration")
def testCmsNametoPhEDExNode(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL")
self.failUnless(sorted(results) == sorted(target))
@attr("integration")
def testPhEDExNodetocmsName(self):
"""
Tests PhEDExNodetocmsName
"""
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC')
self.failUnless(result == 'T2_UK_London_IC')
# don't check this anymore, see comment in phEDExNodetocmsName function
#self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName,
# 'T9_DOESNT_EXIST_Buffer')
@attr("integration")
def testCmsNametoSE(self):
"""
Tests CmsNametoSE
"""
target = ['srm-cms.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoSE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
@attr("integration")
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = 'T1_US_FNAL'
results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov")
self.failUnless(results == target)
@attr("integration")
def testCmsNametoCE(self):
"""
Tests CmsNametoCE
"""
target = ['lcgce06.gridpp.rl.ac.uk', 'lcgce07.gridpp.rl.ac.uk', 'lcgce09.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoCE("T1_UK_RAL")
self.failUnless(sorted(results) == target)
@attr("integration")
def testJSONParser(self):
"""
Tests the JSON parser directly
"""
cmsName = "cmsgrid02.hep.wisc.edu"
results = self.mySiteDB.getJSON("CEtoCMSName",
file="CEtoCMSName",
name=cmsName)
self.failUnless(results['0']['name'] == "T2_US_Wisconsin")
@attr("integration")
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/C=UK/O=eScience/OU=Bristol/L=IS/CN=simon metson"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
@attr("integration")
def testParsingJsonWithApostrophe(self):
"""
Tests parsing a DN json with an apostrophe in
"""
json = """{"dn": "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", "user": "******"}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", d['dn'])
@attr("integration")
def testParsingInvalidJsonWithApostrophe(self):
"""
Tests parsing a DN invalid json (from sitedb v1) with an apostrophe in
"""
json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano', 'user': '******'}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano", d['dn'])
json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'', 'user': '******'}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", d['dn'])
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl, fakeLocation=False):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint = dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
files = {}
outputDatasetParts = datasetName.split("/")
print "dataset",datasetName,"parts",outputDatasetParts
try:
#retrieve list of blocks from dataset
blockNames = dbsReader.listFileBlocks(datasetName)
except:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName)
has_parent = False
try:
parents = dbsReader.listDatasetParents( datasetName )
if parents: has_parent=True
except:
print "Dataset with no parent"
pass
#traverse each block
for blockName in blockNames:
#deal with white and black list.
if blockBlacklist and blockName in blockBlacklist:
continue
if blockWhitelist and blockName not in blockWhitelist:
continue
#existing blocks in phedex
replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName,
subscribed = 'y')
blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True)
#has_parent = dbsReader.listBlockParents(blockName)
if has_parent:
try:
blockFileParents = dbsReader.listFilesInBlockWithParents(blockName)
except:
print blockName,"does not appear to have a parent, even though it should. Very suspicious"
blockFileParents = dbsReader.listFilesInBlock(blockName)
else:
blockFileParents = dbsReader.listFilesInBlock(blockName)
blockLocations = set()
#load block locations
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
node = replica["node"]
cmsSites = siteDB.PNNtoPSN(node)
if type(cmsSites) != list:
cmsSites = [cmsSites]
for cmsName in cmsSites:
se = siteDB.cmsNametoSE(cmsName)
blockLocations.update(se)
logging.debug("cmsName %s mapped to se %s", cmsName, se)
logging.debug("PhEDEx node %s, cmsSites %s, blockLocations %s", node, cmsSites, blockLocations)
# We cannot upload docs without location, so force it in case it's empty
if not blockLocations:
if fakeLocation:
logging.info("\t\t %s\tno location", blockName)
blockLocations.update([u'cmssrmdisk.fnal.gov', u'srm-eoscms.cern.ch'])
elif not has_parent: ## this should be the source
logging.info("Blockname: %s\tno location, ABORT", blockName)
sys.exit(1)
logging.info("Blockname: %s\tLocations: %s", blockName, blockLocations)
#for each file on the block
for blockFile in blockFiles:
parentLFNs = []
#populate parent information
if blockFileParents and "ParentList" in blockFileParents[0]:
for fileParent in blockFileParents[0]["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
## remove when https://github.com/dmwm/WMCore/issues/7128 gets fixed
#elif not 'RAW' in blockName:
# print "no parent info"
runInfo = {}
#Lumis not included in file
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection["RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
files[blockFile["LogicalFileName"]] = {"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs}
return files
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint = dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
files = {}
outputDatasetParts = datasetName.split("/")
datasets = dbsReader.matchProcessedDatasets(outputDatasetParts[1],
outputDatasetParts[3],
outputDatasetParts[2])
if len(datasets) == 0:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName)
blockNames = dbsReader.listFileBlocks(datasetName)
for blockName in blockNames:
if blockBlacklist and blockName in blockBlacklist:
continue
if blockWhitelist and blockName not in blockWhitelist:
continue
replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName,
subscribed = 'y')
block = dbsReader.listFilesInBlockWithParents(blockName)
blockLocations = set()
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
node = replica["node"]
cmsSites = siteDB.phEDExNodetocmsName(node)
if type(cmsSites) != list:
cmsSites = [cmsSites]
for cmsName in cmsSites:
blockLocations.update(siteDB.cmsNametoSE(cmsName))
for blockFile in block:
parentLFNs = []
for fileParent in blockFile["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
runInfo = {}
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection["RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
files[blockFile["LogicalFileName"]] = {"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs}
return files
class SiteDBTest(unittest.TestCase):
"""
Unit tests for SiteScreening module
"""
def setUp(self):
"""
Setup for unit tests
"""
self.mySiteDB = SiteDBJSON()
def testCmsNametoPhEDExNode(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL")
self.failUnless(sorted(results) == sorted(target))
target = ['T1_US_FNAL_Disk']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL_Disk")
self.failUnless(sorted(results) == sorted(target))
def testCmsNametoSE(self):
"""
Tests CmsNametoSE
"""
target = [u'srm-cms-disk.gridpp.rl.ac.uk', u'srm-cms.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoSE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testCmsNamePatterntoSE(self):
"""
Tests CmsNamePatterntoSE
"""
target = [u'srm-eoscms.cern.ch', u'srm-eoscms.cern.ch', u'storage01.lcg.cscs.ch', u'eoscmsftp.cern.ch']
results = self.mySiteDB.cmsNametoSE("%T2_CH")
self.failUnless(sorted(results) == sorted(target))
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = [u'T1_US_FNAL']
results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov")
self.failUnless(results == target)
target = sorted([u'T2_CH_CERN', u'T2_CH_CERN_HLT'])
results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch"))
self.failUnless(sorted(results) == sorted(target))
target = sorted([u'T0_CH_CERN', u'T1_CH_CERN'])
results = sorted(self.mySiteDB.seToCMSName("srm-cms.cern.ch"))
self.failUnless(sorted(results) == sorted(target))
target = sorted([u'T2_CH_CERN_AI'])
results = sorted(self.mySiteDB.seToCMSName("eoscmsftp.cern.ch"))
self.failUnless(sorted(results) == sorted(target))
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gutsche/CN=582680/CN=Oliver Gutsche"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
seNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(seNames) > 1)
self.assertTrue('cmssrm.fnal.gov' in seNames)
return
def testPNNtoPSN(self):
"""
_testPNNtoPSN_
Test converting PhEDEx Node Name to Processing Site Name
"""
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Disk')
self.failUnless(result == ['T1_US_FNAL'])
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Tape')
self.failUnless(result == [])
result = self.mySiteDB.PNNtoPSN('T2_UK_London_IC')
self.failUnless(result == ['T2_UK_London_IC'])
return
def testCMSNametoList(self):
result = self.mySiteDB.cmsNametoList("T1_US*", "SE")
self.failUnless(result == [u'cmssrm.fnal.gov', u'cmssrmdisk.fnal.gov'])
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint=dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
files = {}
outputDatasetParts = datasetName.split("/")
print "dataset", datasetName, "parts", outputDatasetParts
try:
#retrieve list of blocks from dataset
blockNames = dbsReader.listFileBlocks(datasetName)
except:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" %
datasetName)
#traverse each block
for blockName in blockNames:
#deal with white and black list.
if blockBlacklist and blockName in blockBlacklist:
continue
if blockWhitelist and blockName not in blockWhitelist:
continue
#existing blocks in phedex
replicaInfo = phedexReader.getReplicaInfoForBlocks(block=blockName,
subscribed='y')
blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True)
blockLocations = set()
#load block locations
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
node = replica["node"]
cmsSites = siteDB.phEDExNodetocmsName(node)
if type(cmsSites) != list:
cmsSites = [cmsSites]
for cmsName in cmsSites:
blockLocations.update(siteDB.cmsNametoSE(cmsName))
#for each file on the block
for blockFile in blockFiles:
parentLFNs = []
#get parent information about file
blockFileParents = dbsReader.listFilesInBlockWithParents(blockName)
#populate parent information
if blockFileParents and "ParentList" in blockFileParents[0]:
for fileParent in blockFileParents[0]["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
runInfo = {}
#Lumis not included in file
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection[
"RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(
lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
files[blockFile["LogicalFileName"]] = {
"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs
}
return files
class SiteDBTest(unittest.TestCase):
"""
Unit tests for SiteScreening module
"""
def setUp(self):
"""
Setup for unit tests
"""
EmulatorHelper.setEmulators(siteDB = True)
self.mySiteDB = SiteDBJSON()
def tearDown(self):
EmulatorHelper.resetEmulators()
def testCmsNametoPhEDExNode(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL")
self.failUnless(sorted(results) == sorted(target))
def testPhEDExNodetocmsName(self):
"""
Tests PhEDExNodetocmsName
"""
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC')
self.failUnless(result == 'T2_UK_London_IC')
# don't check this anymore, see comment in phEDExNodetocmsName function
#self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName,
# 'T9_DOESNT_EXIST_Buffer')
def testCmsNametoSE(self):
"""
Tests CmsNametoSE
"""
target = ['srm-cms.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoSE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL']
results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov")
self.failUnless(results == target)
target = sorted(['T2_CH_CERN', 'T2_CH_CERN_HLT'])
results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch"))
self.failUnless(results == target)
def testCmsNametoCE(self):
"""
Tests CmsNametoCE
"""
target = ['lcgce11.gridpp.rl.ac.uk', 'lcgce10.gridpp.rl.ac.uk',
'lcgce02.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoCE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gutsche/CN=582680/CN=Oliver Gutsche"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
@attr("integration")
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
seNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(seNames) > 1)
self.assertTrue('cmssrm.fnal.gov' in seNames)
return
class SiteDBTest(unittest.TestCase):
"""
Unit tests for SiteScreening module
"""
def setUp(self):
"""
Setup for unit tests
"""
self.mySiteDB = SiteDBJSON()
def testCmsNametoPhEDExNode(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL_MSS', 'T1_US_FNAL_Buffer']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL")
self.failUnless(sorted(results) == sorted(target))
def testPhEDExNodetocmsName(self):
"""
Tests PhEDExNodetocmsName
"""
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC')
self.failUnless(result == 'T2_UK_London_IC')
# don't check this anymore, see comment in phEDExNodetocmsName function
#self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName,
# 'T9_DOESNT_EXIST_Buffer')
def testCmsNametoSE(self):
"""
Tests CmsNametoSE
"""
target = ['srm-cms.gridpp.rl.ac.uk', 'srm-cms-disk.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoSE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = 'T1_US_FNAL'
results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov")
self.failUnless(results == target)
def testCmsNametoCE(self):
"""
Tests CmsNametoCE
"""
target = [
'lcgce09.gridpp.rl.ac.uk', 'lcgce06.gridpp.rl.ac.uk',
'lcgce07.gridpp.rl.ac.uk', 'lcgce07.gridpp.rl.ac.uk'
]
results = self.mySiteDB.cmsNametoCE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testJSONParser(self):
"""
Tests the JSON parser directly
"""
cmsName = "cmsgrid02.hep.wisc.edu"
results = self.mySiteDB.getJSON("CEtoCMSName",
file="CEtoCMSName",
name=cmsName)
self.failUnless(results['0']['name'] == "T2_US_Wisconsin")
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/C=UK/O=eScience/OU=Bristol/L=IS/CN=simon metson"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
@attr("integration")
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
ceNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(ceNames) > 1)
self.assertTrue('cmssrm.fnal.gov' in ceNames)
return
@attr("integration")
def testParsingJsonWithApostrophe(self):
"""
Tests parsing a DN json with an apostrophe in
"""
json = """{"dn": "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", "user": "******"}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals(
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'",
d['dn'])
@attr("integration")
def testParsingInvalidJsonWithApostrophe(self):
"""
Tests parsing a DN invalid json (from sitedb v1) with an apostrophe in
"""
json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano', 'user': '******'}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals(
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano",
d['dn'])
json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'', 'user': '******'}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals(
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'",
d['dn'])
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint = dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
files = {}
outputDatasetParts = datasetName.split("/")
print "dataset",datasetName,"parts",outputDatasetParts
try:
#retrieve list of blocks from dataset
blockNames = dbsReader.listFileBlocks(datasetName)
except:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName)
#traverse each block
for blockName in blockNames:
#deal with white and black list.
if blockBlacklist and blockName in blockBlacklist:
continue
if blockWhitelist and blockName not in blockWhitelist:
continue
#existing blocks in phedex
replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName,
subscribed = 'y')
blockFiles = dbsReader.listFilesInBlock(blockName, lumis=True)
blockLocations = set()
#load block locations
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
node = replica["node"]
cmsSites = siteDB.phEDExNodetocmsName(node)
if type(cmsSites) != list:
cmsSites = [cmsSites]
for cmsName in cmsSites:
blockLocations.update(siteDB.cmsNametoSE(cmsName))
#for each file on the block
for blockFile in blockFiles:
parentLFNs = []
#get parent information about file
blockFileParents = dbsReader.listFilesInBlockWithParents(blockName)
#populate parent information
if blockFileParents and "ParentList" in blockFileParents[0]:
for fileParent in blockFileParents[0]["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
runInfo = {}
#Lumis not included in file
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection["RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
files[blockFile["LogicalFileName"]] = {"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs}
return files
def getFiles(datasetName, runBlacklist, runWhitelist, blockBlacklist,
blockWhitelist, dbsUrl):
"""
_getFiles_
Get the full information of a dataset including files, blocks, runs and lumis.
Filter it using run and block white/black lists.
It can receive and optional DBSUrl.
"""
dbsReader = DBSReader(endpoint = dbsUrl)
phedexReader = PhEDEx()
siteDB = SiteDBJSON()
files = {}
outputDatasetParts = datasetName.split("/")
datasets = dbsReader.matchProcessedDatasets(outputDatasetParts[1],
outputDatasetParts[3],
outputDatasetParts[2])
if len(datasets) == 0:
raise RuntimeError("Dataset %s doesn't exist in given DBS instance" % datasetName)
blockNames = dbsReader.listFileBlocks(datasetName)
for blockName in blockNames:
if blockBlacklist and blockName in blockBlacklist:
continue
if blockWhitelist and blockName not in blockWhitelist:
continue
replicaInfo = phedexReader.getReplicaInfoForBlocks(block = blockName,
subscribed = 'y')
block = dbsReader.listFilesInBlockWithParents(blockName)
blockLocations = set()
if len(replicaInfo["phedex"]["block"]) > 0:
for replica in replicaInfo["phedex"]["block"][0]["replica"]:
node = replica["node"]
cmsSites = siteDB.phEDExNodetocmsName(node)
if type(cmsSites) != list:
cmsSites = [cmsSites]
for cmsName in cmsSites:
blockLocations.update(siteDB.cmsNametoSE(cmsName))
for blockFile in block:
parentLFNs = []
for fileParent in blockFile["ParentList"]:
parentLFNs.append(fileParent["LogicalFileName"])
runInfo = {}
for lumiSection in blockFile["LumiList"]:
if runBlacklist and lumiSection["RunNumber"] in runBlacklist:
continue
if runWhitelist and lumiSection["RunNumber"] not in runWhitelist:
continue
if lumiSection["RunNumber"] not in runInfo.keys():
runInfo[lumiSection["RunNumber"]] = []
runInfo[lumiSection["RunNumber"]].append(lumiSection["LumiSectionNumber"])
if len(runInfo.keys()) > 0:
files[blockFile["LogicalFileName"]] = {"runs": runInfo,
"events": blockFile["NumberOfEvents"],
"size": blockFile["FileSize"],
"locations": list(blockLocations),
"parents": parentLFNs}
return files
class SiteDBTest(unittest.TestCase):
"""
Unit tests for SiteScreening module
"""
def setUp(self):
"""
Setup for unit tests
"""
EmulatorHelper.setEmulators(siteDB = True)
self.mySiteDB = SiteDBJSON()
def tearDown(self):
EmulatorHelper.resetEmulators()
def testCmsNametoPhEDExNode(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL")
self.failUnless(sorted(results) == sorted(target))
def testPhEDExNodetocmsName(self):
"""
Tests PhEDExNodetocmsName
"""
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC')
self.failUnless(result == 'T2_UK_London_IC')
# don't check this anymore, see comment in phEDExNodetocmsName function
#self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName,
# 'T9_DOESNT_EXIST_Buffer')
def testCmsNametoSE(self):
"""
Tests CmsNametoSE
"""
target = ['srm-cms.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoSE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testCmsNamePatterntoSE(self):
"""
Tests CmsNamePatterntoSE
"""
target = ['T2_XX_SiteA', 'T2_XX_SiteB', 'T2_XX_SiteC']
results = self.mySiteDB.cmsNametoSE("%T2_XX")
self.failUnless(sorted(results) == sorted(target))
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL']
results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov")
self.failUnless(results == target)
target = sorted(['T2_CH_CERN', 'T2_CH_CERN_HLT'])
results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch"))
self.failUnless(results == target)
def testCmsNametoCE(self):
"""
Tests CmsNametoCE
"""
target = ['lcgce11.gridpp.rl.ac.uk', 'lcgce10.gridpp.rl.ac.uk',
'lcgce02.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoCE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testCmsNamePatterntoCE(self):
"""
Tests CmsNamePatterntoCE
"""
target = ['T2_XX_SiteA', 'T2_XX_SiteB', 'T2_XX_SiteC']
results = self.mySiteDB.cmsNametoCE("%T2_XX")
self.failUnless(sorted(results) == sorted(target))
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gutsche/CN=582680/CN=Oliver Gutsche"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
@attr("integration")
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
seNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(seNames) > 1)
self.assertTrue('cmssrm.fnal.gov' in seNames)
return