def sitesFromStorageEelements(ses):
"""Return Sites given Storage Elements"""
global __sitedb
if not __sitedb:
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
__sitedb = SiteDB()
result = set()
for se in ses:
try:
sites = __sitedb.seToCMSName(se)
except:
print "Unable to get site name for %s" % se
else:
result.update(sites)
return list(result)
def sitesFromStorageEelements(ses):
"""Return Sites given Storage Elements"""
global __sitedb
if not __sitedb:
from WMCore.Services.SiteDB.SiteDB import SiteDBJSON as SiteDB
__sitedb = SiteDB()
result = set()
for se in ses:
try:
site = __sitedb.seToCMSName(se)
except:
print "Unable to get site name for %s" % se
else:
result.add(site)
return list(result)
def formatOutput(self, task, requestname, datasetfiles, locations):
"""
Receives as input the result of the data location
discovery operations and fill up the WMCore objects.
"""
self.logger.debug(" Formatting data discovery output ")
# TEMPORARY
secmsmap = {}
sbj = SiteDBJSON({"key": self.config.TaskWorker.cmskey, "cert": self.config.TaskWorker.cmscert})
wmfiles = []
event_counter = 0
lumi_counter = 0
file_counter = 0
uniquelumis = set()
## Loop over the sorted list of files.
for lfn, infos in datasetfiles.iteritems():
## Skip the file if the block has not been found or has no locations.
if not infos['BlockName'] in locations or not locations[infos['BlockName']]:
self.logger.warning("Skipping %s because its block (%s) has no locations" % (lfn, infos['BlockName']))
continue
## Skip the file if it is not in VALID state.
if not infos.get('ValidFile', True):
self.logger.warning("Skipping invalid file %s" % lfn)
continue
## Createa a WMCore File object.
wmfile = File(lfn = lfn, events = infos['NumberOfEvents'], size = infos['Size'], checksums = infos['Checksums'])
wmfile['block'] = infos['BlockName']
wmfile['locations'] = []
for se in locations[infos['BlockName']]:
if se and se not in secmsmap:
self.logger.debug("Translating SE %s" %se)
try:
secmsmap[se] = sbj.seToCMSName(se)
except KeyError, ke:
self.logger.error("Impossible translating %s to a CMS name through SiteDB" %se)
secmsmap[se] = ''
except httplib.HTTPException, ex:
self.logger.error("Couldn't map SE to site: %s" % se)
print "Couldn't map SE to site: %s" % se
print "got problem: %s" % ex
print "got another problem: %s" % ex.__dict__
if se and se in secmsmap:
if type(secmsmap[se]) == list:
wmfile['locations'].extend(secmsmap[se])
else:
wmfile['locations'].append(secmsmap[se])
def phedexIt():
x = PhEDEx(responseType = "json")
phedexNodes = x.getNodeMap()['phedex']['node']
phedexMap = {}
sePhedexMap = {}
knownPhedexNodes = set()
for node in phedexNodes:
phedexMap[node['name']] = node['kind']
#print '%s -> %s, %s' % (node['name'], node['kind'], node['se'])
if node['se'] not in sePhedexMap:
sePhedexMap[node['se']] = set()
sePhedexMap[node['se']].add(node['name'])
knownPhedexNodes.add(node['name'])
y = SiteDBJSON()
seNames = y.getAllSENames()
cmsNamesMap = {}
for se in seNames:
cmsNames = y.seToCMSName(se)
cmsNamesMap[se] = cmsNames
seToNodeMap = {}
for se in cmsNamesMap:
candidates = set()
for cmsName in cmsNamesMap[se]:
phedexNodes = y.cmsNametoPhEDExNode(cmsName)
candidates.update(set(phedexNodes))
validCandidates = set()
for candidate in candidates:
if candidate in knownPhedexNodes:
validCandidates.add(candidate)
seToNodeMap[se] = validCandidates
#print '%s to %s' % (se, candidates)
for se in sePhedexMap:
if se not in seToNodeMap:
print "SE: %s is not in new mapping for sites %s" % (se, list(sePhedexMap[se]))
for se in seToNodeMap:
if se not in sePhedexMap:
print "SE: %s is not in old mapping for sites %s" % (se, list(seToNodeMap[se]))
continue
for se in set(seToNodeMap.keys()).intersection(set(sePhedexMap.keys())):
diff = sePhedexMap[se] - seToNodeMap[se]
if diff:
print "%s are in old mapping but not in new for %s" %(str(list(diff)), se)
diff = seToNodeMap[se] - sePhedexMap[se]
if diff:
print "%s are in new mapping but not in old for %s" %(str(list(diff)), se)
def formatOutput(self, task, requestname, datasetfiles, locations):
"""Receives as input the result of the data location
discovery operations and fill up the WMCore objects."""
self.logger.debug(" Formatting data discovery output ")
# TEMPORARY
secmsmap = {}
sbj = SiteDBJSON({"key":self.config.MyProxy.serverhostkey,
"cert":self.config.MyProxy.serverhostcert})
wmfiles = []
lumicounter = evecounter = 0
for lfn, infos in datasetfiles.iteritems():
wmfile = File(lfn=lfn, events=infos['NumberOfEvents'], size=infos['Size'], checksums=infos['Checksums'])
wmfile['block'] = infos['BlockName']
wmfile['locations'] = []
if locations.has_key(infos['BlockName']):
for se in locations[infos['BlockName']]:
if se not in secmsmap:
self.logger.debug("Translating SE %s" %se)
try:
secmsmap[se] = sbj.seToCMSName(se)
except KeyError, ke:
self.logger.error("Impossible translating %s to a CMS name through SiteDB" %se)
secmsmap[se] = ''
if se in secmsmap:
if type(secmsmap[se]) == list:
wmfile['locations'].extend(secmsmap[se])
else:
wmfile['locations'].append(secmsmap[se])
wmfile['workflow'] = requestname
evecounter += infos['NumberOfEvents']
for run, lumis in infos['Lumis'].iteritems():
#self.logger.debug(' - adding run %d and lumis %s' %(run, lumis))
wmfile.addRun(Run(run, *lumis))
lumicounter += len(lumis)
wmfiles.append(wmfile)
class SiteDBTest(unittest.TestCase):
"""
Unit tests for SiteScreening module
"""
def setUp(self):
"""
Setup for unit tests
"""
self.mySiteDB = SiteDBJSON()
@attr("integration")
def testCmsNametoPhEDExNode(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL")
self.failUnless(sorted(results) == sorted(target))
@attr("integration")
def testPhEDExNodetocmsName(self):
"""
Tests PhEDExNodetocmsName
"""
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC')
self.failUnless(result == 'T2_UK_London_IC')
# don't check this anymore, see comment in phEDExNodetocmsName function
#self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName,
# 'T9_DOESNT_EXIST_Buffer')
@attr("integration")
def testCmsNametoSE(self):
"""
Tests CmsNametoSE
"""
target = ['srm-cms.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoSE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
@attr("integration")
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = 'T1_US_FNAL'
results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov")
self.failUnless(results == target)
@attr("integration")
def testCmsNametoCE(self):
"""
Tests CmsNametoCE
"""
target = ['lcgce06.gridpp.rl.ac.uk', 'lcgce07.gridpp.rl.ac.uk', 'lcgce09.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoCE("T1_UK_RAL")
self.failUnless(sorted(results) == target)
@attr("integration")
def testJSONParser(self):
"""
Tests the JSON parser directly
"""
cmsName = "cmsgrid02.hep.wisc.edu"
results = self.mySiteDB.getJSON("CEtoCMSName",
file="CEtoCMSName",
name=cmsName)
self.failUnless(results['0']['name'] == "T2_US_Wisconsin")
@attr("integration")
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/C=UK/O=eScience/OU=Bristol/L=IS/CN=simon metson"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
@attr("integration")
def testParsingJsonWithApostrophe(self):
"""
Tests parsing a DN json with an apostrophe in
"""
json = """{"dn": "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", "user": "******"}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", d['dn'])
@attr("integration")
def testParsingInvalidJsonWithApostrophe(self):
"""
Tests parsing a DN invalid json (from sitedb v1) with an apostrophe in
"""
json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano', 'user': '******'}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano", d['dn'])
json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'', 'user': '******'}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals("/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", d['dn'])
class SiteDBTest(unittest.TestCase):
"""
Unit tests for SiteScreening module
"""
def setUp(self):
"""
Setup for unit tests
"""
self.mySiteDB = SiteDBJSON()
def testCmsNametoPhEDExNode(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL")
self.failUnless(sorted(results) == sorted(target))
target = ['T1_US_FNAL_Disk']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL_Disk")
self.failUnless(sorted(results) == sorted(target))
def testCmsNametoSE(self):
"""
Tests CmsNametoSE
"""
target = [u'srm-cms-disk.gridpp.rl.ac.uk', u'srm-cms.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoSE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testCmsNamePatterntoSE(self):
"""
Tests CmsNamePatterntoSE
"""
target = [u'srm-eoscms.cern.ch', u'srm-eoscms.cern.ch', u'storage01.lcg.cscs.ch', u'eoscmsftp.cern.ch']
results = self.mySiteDB.cmsNametoSE("%T2_CH")
self.failUnless(sorted(results) == sorted(target))
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = [u'T1_US_FNAL']
results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov")
self.failUnless(results == target)
target = sorted([u'T2_CH_CERN', u'T2_CH_CERN_HLT'])
results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch"))
self.failUnless(sorted(results) == sorted(target))
target = sorted([u'T0_CH_CERN', u'T1_CH_CERN'])
results = sorted(self.mySiteDB.seToCMSName("srm-cms.cern.ch"))
self.failUnless(sorted(results) == sorted(target))
target = sorted([u'T2_CH_CERN_AI'])
results = sorted(self.mySiteDB.seToCMSName("eoscmsftp.cern.ch"))
self.failUnless(sorted(results) == sorted(target))
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gutsche/CN=582680/CN=Oliver Gutsche"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
seNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(seNames) > 1)
self.assertTrue('cmssrm.fnal.gov' in seNames)
return
def testPNNtoPSN(self):
"""
_testPNNtoPSN_
Test converting PhEDEx Node Name to Processing Site Name
"""
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Disk')
self.failUnless(result == ['T1_US_FNAL'])
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Tape')
self.failUnless(result == [])
result = self.mySiteDB.PNNtoPSN('T2_UK_London_IC')
self.failUnless(result == ['T2_UK_London_IC'])
return
def testCMSNametoList(self):
result = self.mySiteDB.cmsNametoList("T1_US*", "SE")
self.failUnless(result == [u'cmssrm.fnal.gov', u'cmssrmdisk.fnal.gov'])
class SiteDBTest(EmulatedUnitTestCase):
"""
Unit tests for SiteScreening module
"""
def __init__(self, methodName='runTest'):
super(SiteDBTest, self).__init__(methodName=methodName)
def setUp(self):
"""
Setup for unit tests
"""
super(SiteDBTest, self).setUp()
EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=False, requestMgr=True)
self.mySiteDB = SiteDBJSON()
def tearDown(self):
"""
_tearDown_
"""
super(SiteDBTest, self).tearDown()
EmulatorHelper.resetEmulators()
return
def testCmsNametoPhEDExNode(self):
"""
#Tests CmsNametoSE
"""
target = ['T1_US_FNAL_Buffer', 'T1_US_FNAL_MSS']
results = self.mySiteDB.cmsNametoPhEDExNode('T1_US_FNAL')
self.assertItemsEqual(results, target)
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = [u'T1_US_FNAL', u'T1_US_FNAL_Disk']
results = self.mySiteDB.seToCMSName("cmsdcadisk01.fnal.gov")
self.assertTrue(results == target)
target = sorted([u'T2_CH_CERN', u'T2_CH_CERN_HLT'])
results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch"))
self.assertItemsEqual(results, target)
target = sorted([u'T0_CH_CERN', u'T1_CH_CERN'])
results = sorted(self.mySiteDB.seToCMSName("srm-cms.cern.ch"))
self.assertItemsEqual(results, target)
target = sorted([u'T2_CH_CERN_AI'])
results = sorted(self.mySiteDB.seToCMSName("eoscmsftp.cern.ch"))
self.assertItemsEqual(results, target)
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jha/CN=618566/CN=Manoj Jha"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.assertTrue(testUserName == userName)
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.assertTrue(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
seNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(seNames) > 1)
self.assertTrue('cmsdcadisk01.fnal.gov' in seNames)
return
def testPNNtoPSN(self):
"""
_testPNNtoPSN_
Test converting PhEDEx Node Name to Processing Site Name
"""
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Disk')
self.assertTrue(result == ['T1_US_FNAL'])
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Tape')
self.assertTrue(result == [])
result = self.mySiteDB.PNNtoPSN('T2_UK_London_IC')
self.assertTrue(result == ['T2_UK_London_IC'])
return
def testCMSNametoList(self):
"""
Test PNN to storage list
"""
result = self.mySiteDB.cmsNametoList("T1_US*", "SE")
self.assertItemsEqual(result, [u'cmsdcadisk01.fnal.gov'])
def testCheckAndConvertSENameToPNN(self):
"""
Test the conversion of SE name to PNN for single
and multiple sites/PNNs using checkAndConvertSENameToPNN
"""
fnalSE = u'cmsdcadisk01.fnal.gov'
purdueSE = u'srm.rcac.purdue.edu'
fnalPNNs = [u'T1_US_FNAL_Buffer', u'T1_US_FNAL_MSS', u'T1_US_FNAL_Disk']
purduePNN = [u'T2_US_Purdue']
pnnList = fnalPNNs + purduePNN
seList = [fnalSE, purdueSE]
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN(fnalSE), fnalPNNs)
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN([fnalSE]), fnalPNNs)
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN(purdueSE), purduePNN)
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN([purdueSE]), purduePNN)
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN(seList), purduePNN + fnalPNNs)
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN(pnnList), pnnList)
return
class SiteDBTest(unittest.TestCase):
"""
Unit tests for SiteScreening module
"""
def setUp(self):
"""
Setup for unit tests
"""
EmulatorHelper.setEmulators(siteDB = True)
self.mySiteDB = SiteDBJSON()
def tearDown(self):
EmulatorHelper.resetEmulators()
def testCmsNametoPhEDExNode(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL")
self.failUnless(sorted(results) == sorted(target))
def testPhEDExNodetocmsName(self):
"""
Tests PhEDExNodetocmsName
"""
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC')
self.failUnless(result == 'T2_UK_London_IC')
# don't check this anymore, see comment in phEDExNodetocmsName function
#self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName,
# 'T9_DOESNT_EXIST_Buffer')
def testCmsNametoSE(self):
"""
Tests CmsNametoSE
"""
target = ['srm-cms.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoSE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL']
results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov")
self.failUnless(results == target)
target = sorted(['T2_CH_CERN', 'T2_CH_CERN_HLT'])
results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch"))
self.failUnless(results == target)
def testCmsNametoCE(self):
"""
Tests CmsNametoCE
"""
target = ['lcgce11.gridpp.rl.ac.uk', 'lcgce10.gridpp.rl.ac.uk',
'lcgce02.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoCE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gutsche/CN=582680/CN=Oliver Gutsche"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
@attr("integration")
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
seNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(seNames) > 1)
self.assertTrue('cmssrm.fnal.gov' in seNames)
return
class SiteDBTest(EmulatedUnitTestCase):
"""
Unit tests for SiteScreening module
"""
def __init__(self, methodName='runTest'):
super(SiteDBTest, self).__init__(methodName=methodName)
def setUp(self):
"""
Setup for unit tests
"""
super(SiteDBTest, self).setUp()
self.mySiteDB = SiteDBJSON()
def testCmsNametoPhEDExNode(self):
"""
#Tests CmsNametoSE
"""
target = ['T1_US_FNAL_Buffer', 'T1_US_FNAL_MSS']
results = self.mySiteDB.cmsNametoPhEDExNode('T1_US_FNAL')
self.assertItemsEqual(results, target)
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = [u'T1_US_FNAL', u'T1_US_FNAL_Disk']
results = self.mySiteDB.seToCMSName("cmsdcadisk01.fnal.gov")
self.assertTrue(results == target)
target = sorted([u'T2_CH_CERN', u'T2_CH_CERN_HLT'])
results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch"))
self.assertItemsEqual(results, target)
target = sorted([u'T0_CH_CERN', u'T1_CH_CERN'])
results = sorted(self.mySiteDB.seToCMSName("srm-cms.cern.ch"))
self.assertItemsEqual(results, target)
target = sorted([u'T2_CH_CERN_AI'])
results = sorted(self.mySiteDB.seToCMSName("eoscmsftp.cern.ch"))
self.assertItemsEqual(results, target)
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jha/CN=618566/CN=Manoj Jha"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.assertTrue(testUserName == userName)
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.assertTrue(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
seNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(seNames) > 1)
self.assertTrue('cmsdcadisk01.fnal.gov' in seNames)
return
def testPNNtoPSN(self):
"""
_testPNNtoPSN_
Test converting PhEDEx Node Name to Processing Site Name
"""
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Disk')
self.assertTrue(result == ['T1_US_FNAL'])
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Tape')
self.assertTrue(result == [])
result = self.mySiteDB.PNNtoPSN('T2_UK_London_IC')
self.assertTrue(result == ['T2_UK_London_IC'])
return
def testCMSNametoList(self):
"""
Test PNN to storage list
"""
result = self.mySiteDB.cmsNametoList("T1_US*", "SE")
self.assertItemsEqual(result, [u'cmsdcadisk01.fnal.gov'])
def testPNNstoPSNs(self):
"""
_testPNNstoPSNs_
Test converting PhEDEx Node Names to Processing Site Names
"""
result = self.mySiteDB.PNNstoPSNs(['T1_US_FNAL_Disk', 'T1_US_FNAL_Buffer', 'T1_US_FNAL_MSS'])
self.assertTrue(result == ['T1_US_FNAL'])
result = self.mySiteDB.PNNstoPSNs(['T2_UK_London_IC', 'T2_US_Purdue'])
self.assertItemsEqual(result, ['T2_UK_London_IC', 'T2_US_Purdue'])
return
class SiteDBTest(unittest.TestCase):
"""
Unit tests for SiteScreening module
"""
def setUp(self):
"""
Setup for unit tests
"""
self.mySiteDB = SiteDBJSON()
def testCmsNametoPhEDExNode(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL_MSS', 'T1_US_FNAL_Buffer']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL")
self.failUnless(sorted(results) == sorted(target))
def testPhEDExNodetocmsName(self):
"""
Tests PhEDExNodetocmsName
"""
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC')
self.failUnless(result == 'T2_UK_London_IC')
# don't check this anymore, see comment in phEDExNodetocmsName function
#self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName,
# 'T9_DOESNT_EXIST_Buffer')
def testCmsNametoSE(self):
"""
Tests CmsNametoSE
"""
target = ['srm-cms.gridpp.rl.ac.uk', 'srm-cms-disk.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoSE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = 'T1_US_FNAL'
results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov")
self.failUnless(results == target)
def testCmsNametoCE(self):
"""
Tests CmsNametoCE
"""
target = [
'lcgce09.gridpp.rl.ac.uk', 'lcgce06.gridpp.rl.ac.uk',
'lcgce07.gridpp.rl.ac.uk', 'lcgce07.gridpp.rl.ac.uk'
]
results = self.mySiteDB.cmsNametoCE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testJSONParser(self):
"""
Tests the JSON parser directly
"""
cmsName = "cmsgrid02.hep.wisc.edu"
results = self.mySiteDB.getJSON("CEtoCMSName",
file="CEtoCMSName",
name=cmsName)
self.failUnless(results['0']['name'] == "T2_US_Wisconsin")
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/C=UK/O=eScience/OU=Bristol/L=IS/CN=simon metson"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
@attr("integration")
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
ceNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(ceNames) > 1)
self.assertTrue('cmssrm.fnal.gov' in ceNames)
return
@attr("integration")
def testParsingJsonWithApostrophe(self):
"""
Tests parsing a DN json with an apostrophe in
"""
json = """{"dn": "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'", "user": "******"}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals(
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'",
d['dn'])
@attr("integration")
def testParsingInvalidJsonWithApostrophe(self):
"""
Tests parsing a DN invalid json (from sitedb v1) with an apostrophe in
"""
json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano', 'user': '******'}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals(
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio' Fano",
d['dn'])
json = """{'dn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'', 'user': '******'}"""
d = self.mySiteDB.parser.dictParser(json)
self.assertEquals(
"/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'",
d['dn'])
class SiteDBTest(EmulatedUnitTestCase):
"""
Unit tests for SiteScreening module
"""
def __init__(self, methodName='runTest'):
super(SiteDBTest, self).__init__(methodName=methodName)
def setUp(self):
"""
Setup for unit tests
"""
super(SiteDBTest, self).setUp()
EmulatorHelper.setEmulators(phedex=False, dbs=False, siteDB=False, requestMgr=True)
self.mySiteDB = SiteDBJSON()
def tearDown(self):
"""
_tearDown_
"""
super(SiteDBTest, self).tearDown()
EmulatorHelper.resetEmulators()
return
def testCmsNametoPhEDExNode(self):
"""
#Tests CmsNametoSE
"""
target = ['T1_US_FNAL_Buffer', 'T1_US_FNAL_MSS']
results = self.mySiteDB.cmsNametoPhEDExNode('T1_US_FNAL')
self.assertItemsEqual(results, target)
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = [u'T1_US_FNAL', u'T1_US_FNAL_Disk']
results = self.mySiteDB.seToCMSName("cmsdcadisk01.fnal.gov")
self.assertTrue(results == target)
target = sorted([u'T2_CH_CERN', u'T2_CH_CERN_HLT'])
results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch"))
self.assertItemsEqual(results, target)
target = sorted([u'T0_CH_CERN', u'T1_CH_CERN'])
results = sorted(self.mySiteDB.seToCMSName("srm-cms.cern.ch"))
self.assertItemsEqual(results, target)
target = sorted([u'T2_CH_CERN_AI'])
results = sorted(self.mySiteDB.seToCMSName("eoscmsftp.cern.ch"))
self.assertItemsEqual(results, target)
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jha/CN=618566/CN=Manoj Jha"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.assertTrue(testUserName == userName)
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.assertTrue(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
seNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(seNames) > 1)
self.assertTrue('cmsdcadisk01.fnal.gov' in seNames)
return
def testPNNtoPSN(self):
"""
_testPNNtoPSN_
Test converting PhEDEx Node Name to Processing Site Name
"""
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Disk')
self.assertTrue(result == ['T1_US_FNAL'])
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Tape')
self.assertTrue(result == [])
result = self.mySiteDB.PNNtoPSN('T2_UK_London_IC')
self.assertTrue(result == ['T2_UK_London_IC'])
return
def testCMSNametoList(self):
"""
Test PNN to storage list
"""
result = self.mySiteDB.cmsNametoList("T1_US*", "SE")
self.assertItemsEqual(result, [u'cmsdcadisk01.fnal.gov'])
def testCheckAndConvertSENameToPNN(self):
"""
Test the conversion of SE name to PNN for single
and multiple sites/PNNs using checkAndConvertSENameToPNN
"""
fnalSE = u'cmsdcadisk01.fnal.gov'
purdueSE = u'srm.rcac.purdue.edu'
fnalPNNs = [u'T1_US_FNAL_Buffer', u'T1_US_FNAL_MSS', u'T1_US_FNAL_Disk']
purduePNN = [u'T2_US_Purdue']
pnnList = fnalPNNs + purduePNN
seList = [fnalSE, purdueSE]
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN(fnalSE), fnalPNNs)
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN([fnalSE]), fnalPNNs)
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN(purdueSE), purduePNN)
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN([purdueSE]), purduePNN)
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN(seList), purduePNN + fnalPNNs)
self.assertItemsEqual(self.mySiteDB.checkAndConvertSENameToPNN(pnnList), pnnList)
return
def testPNNstoPSNs(self):
"""
_testPNNstoPSNs_
Test converting PhEDEx Node Names to Processing Site Names
"""
result = self.mySiteDB.PNNstoPSNs(['T1_US_FNAL_Disk', 'T1_US_FNAL_Buffer', 'T1_US_FNAL_MSS'])
self.assertTrue(result == ['T1_US_FNAL'])
result = self.mySiteDB.PNNstoPSNs(['T2_UK_London_IC', 'T2_US_Purdue'])
self.assertItemsEqual(result, ['T2_UK_London_IC', 'T2_US_Purdue'])
return
class SiteDBTest(EmulatedUnitTestCase):
"""
Unit tests for SiteScreening module
"""
def __init__(self, methodName='runTest'):
super(SiteDBTest, self).__init__(methodName=methodName)
def setUp(self):
"""
Setup for unit tests
"""
super(SiteDBTest, self).setUp()
self.mySiteDB = SiteDBJSON()
def testCmsNametoPhEDExNode(self):
"""
#Tests CmsNametoSE
"""
target = ['T1_US_FNAL_Buffer', 'T1_US_FNAL_MSS']
results = self.mySiteDB.cmsNametoPhEDExNode('T1_US_FNAL')
self.assertItemsEqual(results, target)
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = [u'T1_US_FNAL', u'T1_US_FNAL_Disk']
results = self.mySiteDB.seToCMSName("cmsdcadisk01.fnal.gov")
self.assertTrue(results == target)
target = sorted([u'T2_CH_CERN', u'T2_CH_CERN_HLT'])
results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch"))
self.assertItemsEqual(results, target)
target = sorted([u'T0_CH_CERN', u'T1_CH_CERN'])
results = sorted(self.mySiteDB.seToCMSName("srm-cms.cern.ch"))
self.assertItemsEqual(results, target)
target = sorted([u'T2_CH_CERN_AI'])
results = sorted(self.mySiteDB.seToCMSName("eoscmsftp.cern.ch"))
self.assertItemsEqual(results, target)
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=jha/CN=618566/CN=Manoj Jha"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.assertTrue(testUserName == userName)
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.assertTrue(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
seNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(seNames) > 1)
self.assertTrue('cmsdcadisk01.fnal.gov' in seNames)
return
def testPNNtoPSN(self):
"""
_testPNNtoPSN_
Test converting PhEDEx Node Name to Processing Site Name
"""
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Disk')
self.assertTrue(result == ['T1_US_FNAL'])
result = self.mySiteDB.PNNtoPSN('T1_US_FNAL_Tape')
self.assertTrue(result == [])
result = self.mySiteDB.PNNtoPSN('T2_UK_London_IC')
self.assertTrue(result == ['T2_UK_London_IC'])
return
def testCMSNametoList(self):
"""
Test PNN to storage list
"""
result = self.mySiteDB.cmsNametoList("T1_US*", "SE")
self.assertItemsEqual(result, [u'cmsdcadisk01.fnal.gov'])
def testPNNstoPSNs(self):
"""
_testPNNstoPSNs_
Test converting PhEDEx Node Names to Processing Site Names
"""
result = self.mySiteDB.PNNstoPSNs(
['T1_US_FNAL_Disk', 'T1_US_FNAL_Buffer', 'T1_US_FNAL_MSS'])
self.assertTrue(result == ['T1_US_FNAL'])
result = self.mySiteDB.PNNstoPSNs(['T2_UK_London_IC', 'T2_US_Purdue'])
self.assertItemsEqual(result, ['T2_UK_London_IC', 'T2_US_Purdue'])
return
class SiteDBTest(unittest.TestCase):
"""
Unit tests for SiteScreening module
"""
def setUp(self):
"""
Setup for unit tests
"""
EmulatorHelper.setEmulators(siteDB = True)
self.mySiteDB = SiteDBJSON()
def tearDown(self):
EmulatorHelper.resetEmulators()
def testCmsNametoPhEDExNode(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL_MSS','T1_US_FNAL_Buffer']
results = self.mySiteDB.cmsNametoPhEDExNode("T1_US_FNAL")
self.failUnless(sorted(results) == sorted(target))
def testPhEDExNodetocmsName(self):
"""
Tests PhEDExNodetocmsName
"""
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_MSS')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T1_US_FNAL_Buffer')
self.failUnless(result == 'T1_US_FNAL')
result = self.mySiteDB.phEDExNodetocmsName('T2_UK_London_IC')
self.failUnless(result == 'T2_UK_London_IC')
# don't check this anymore, see comment in phEDExNodetocmsName function
#self.assertRaises(ValueError, self.mySiteDB.phEDExNodetocmsName,
# 'T9_DOESNT_EXIST_Buffer')
def testCmsNametoSE(self):
"""
Tests CmsNametoSE
"""
target = ['srm-cms.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoSE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testCmsNamePatterntoSE(self):
"""
Tests CmsNamePatterntoSE
"""
target = ['T2_XX_SiteA', 'T2_XX_SiteB', 'T2_XX_SiteC']
results = self.mySiteDB.cmsNametoSE("%T2_XX")
self.failUnless(sorted(results) == sorted(target))
def testSEtoCmsName(self):
"""
Tests CmsNametoSE
"""
target = ['T1_US_FNAL']
results = self.mySiteDB.seToCMSName("cmssrm.fnal.gov")
self.failUnless(results == target)
target = sorted(['T2_CH_CERN', 'T2_CH_CERN_HLT'])
results = sorted(self.mySiteDB.seToCMSName("srm-eoscms.cern.ch"))
self.failUnless(results == target)
def testCmsNametoCE(self):
"""
Tests CmsNametoCE
"""
target = ['lcgce11.gridpp.rl.ac.uk', 'lcgce10.gridpp.rl.ac.uk',
'lcgce02.gridpp.rl.ac.uk']
results = self.mySiteDB.cmsNametoCE("T1_UK_RAL")
self.failUnless(sorted(results) == sorted(target))
def testCmsNamePatterntoCE(self):
"""
Tests CmsNamePatterntoCE
"""
target = ['T2_XX_SiteA', 'T2_XX_SiteB', 'T2_XX_SiteC']
results = self.mySiteDB.cmsNametoCE("%T2_XX")
self.failUnless(sorted(results) == sorted(target))
def testDNUserName(self):
"""
Tests DN to Username lookup
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=gutsche/CN=582680/CN=Oliver Gutsche"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
@attr("integration")
def testDNWithApostrophe(self):
"""
Tests a DN with an apostrophy in - will fail till SiteDB2 appears
"""
testDn = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=liviof/CN=472739/CN=Livio Fano'"
testUserName = "******"
userName = self.mySiteDB.dnUserName(dn=testDn)
self.failUnless(testUserName == userName)
def testSEFinder(self):
"""
_testSEFinder_
See if we can retrieve seNames from all sites
"""
seNames = self.mySiteDB.getAllSENames()
self.assertTrue(len(seNames) > 1)
self.assertTrue('cmssrm.fnal.gov' in seNames)
return