def test_02_unparse( self ):
""" pfnunparse and pfnunparse_old
:param self: self reference
"""
for pfn, result in self.pfns.items():
if result["OK"]:
self.assertEqual( pfnunparse( result["Value"] ), { "OK" : True, "Value" : pfn } )
self.assertEqual( pfnunparse( None )['OK'], False )
self.assertEqual( pfnunparse( "Path" )['OK'], False )
def test_03_default_unparse( self ):
""" pfnunparse and pfnunparse_old
:param self: self reference
"""
for pfn, result in self.default_pfns.items():
if result["OK"]:
unparseResult = pfnunparse( result["Value"], srmSpecific = False )
self.assertEqual( unparseResult, { "OK" : True, "Value" : pfn } )
self.assertEqual( pfnunparse( None )['OK'], False )
self.assertEqual( pfnunparse( "Path" )['OK'], False )
def test_03_srm_unparse( self ):
""" pfnunparse and pfnunparse_old
:param self: self reference
"""
for pfn, result in self.srm_pfns.items():
if result["OK"]:
unparseResult = pfnunparse( result["Value"] )
self.assertEqual( unparseResult, { "OK" : True, "Value" : pfn } )
self.assertEqual( pfnunparse( None )['OK'], False )
self.assertEqual( pfnunparse( "Path" )['OK'], False )
def test_02_unparse( self ):
""" pfnunparse and pfnunparse_old
:param self: self reference
"""
for pfn, result in self.pfns.items():
if result["OK"]:
self.assertEqual( pfnunparse( result["Value"] ), { "OK" : True, "Value" : pfn } )
self.assertEqual( pfnunparse( None ),
{'Message': "pfnunparse: wrong type for pfnDict argument, expected a dict, got <type 'NoneType'>", 'OK': False} )
self.assertEqual( pfnunparse( "Path" ),
{'Message': "pfnunparse: wrong type for pfnDict argument, expected a dict, got <type 'str'>", 'OK': False} )
def constructURLFromLFN(self, lfn, withWSUrl=False):
"""Construct URL from the given LFN according to the VO convention for the
primary protocol of the storage plagin
:param str lfn: file LFN
:param boolean withWSUrl: flag to include the web service part into the resulting URL
:return result: result['Value'] - resulting URL
"""
# Check the LFN convention:
# 1. LFN must start with the VO name as the top level directory
# 2. VO name must not appear as any subdirectory or file name
lfnSplitList = lfn.split("/")
voLFN = lfnSplitList[1]
# TODO comparison to Sandbox below is for backward compatibility, should
# be removed in the next release
if voLFN != self.se.vo and voLFN != "SandBox" and voLFN != "Sandbox":
return S_ERROR("LFN (%s) path must start with VO name (%s)" %
(lfn, self.se.vo))
urlDict = dict(self.protocolParameters)
urlDict["Options"] = "&".join(
"%s=%s" % (optionName, urlDict[paramName])
for paramName, optionName in self.DYNAMIC_OPTIONS.items()
if urlDict.get(paramName))
if not withWSUrl:
urlDict["WSUrl"] = ""
urlDict["FileName"] = lfn.lstrip("/")
return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def constructURLFromLFN( self, lfn, withWSUrl = False ):
""" Construct URL from the given LFN according to the VO convention for the
primary protocol of the storage plagin
:param str lfn: file LFN
:param boolean withWSUrl: flag to include the web service part into the resulting URL
:return result: result['Value'] - resulting URL
"""
# Check the LFN convention:
# 1. LFN must start with the VO name as the top level directory
# 2. VO name must not appear as any subdirectory or file name
lfnSplitList = lfn.split( '/' )
voLFN = lfnSplitList[1]
# TODO comparison to Sandbox below is for backward compatibility, should be removed in the next release
if voLFN != self.se.vo and voLFN != "SandBox" and voLFN != "Sandbox":
return S_ERROR( 'LFN does not follow the DIRAC naming convention %s' % lfn )
urlDict = dict( self.protocolParameters )
urlDict['Options'] = '&'.join( "%s=%s" % ( optionName, urlDict[paramName] )
for paramName, optionName in self.DYNAMIC_OPTIONS.iteritems()
if urlDict.get( paramName ) )
if not withWSUrl:
urlDict['WSUrl'] = ''
urlDict['FileName'] = lfn.lstrip( '/' )
return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
def _resolvePFN(self, lfn, se):
resSE = self.db.seManager.getSEDefinition(se)
if not resSE['OK']:
return resSE
pfnDict = dict(resSE['Value']['SEDict'])
if "PFNPrefix" in pfnDict:
return S_OK(pfnDict['PFNPrefix'] + lfn)
pfnDict['FileName'] = lfn
return pfnunparse(pfnDict)
def getProtocolPfn(self, pfnDict, withPort):
""" From the pfn dict construct the pfn to be used
"""
# pfnDict['Protocol'] = ''
# pfnDict['Host'] = ''
# pfnDict['Port'] = ''
# pfnDict['WSUrl'] = ''
res = pfnunparse(pfnDict)
return res
def getProtocolPfn( self, pfnDict, withPort ): """ From the pfn dict construct the pfn to be used """ pfnDict['Protocol'] = '' pfnDict['Host'] = '' pfnDict['Port'] = '' pfnDict['WSUrl'] = '' res = pfnunparse( pfnDict ) return res
def getProtocolPfn(self, pfnDict, withPort):
""" From the pfn dict construct the pfn to be used
"""
pfnDict["Protocol"] = ""
pfnDict["Host"] = ""
pfnDict["Port"] = ""
pfnDict["WSUrl"] = ""
res = pfnunparse(pfnDict)
return res
def _resolvePFN(self, lfn, se):
resSE = self.db.seManager.getSEDefinition(se)
if not resSE["OK"]:
return resSE
pfnDict = dict(resSE["Value"]["SEDict"])
if "PFNPrefix" in pfnDict:
return S_OK(pfnDict["PFNPrefix"] + lfn)
else:
pfnDict["FileName"] = lfn
return pfnunparse(pfnDict)
def getURLBase(self, withWSUrl=False):
"""This will get the URL base. This is then appended with the LFN in DIRAC convention.
:param self: self reference
:param bool withWSUrl: flag to include Web Service part of the url
:returns: URL
"""
urlDict = dict(self.protocolParameters)
if not withWSUrl:
urlDict["WSUrl"] = ""
return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def getEndpoint(self):
"""This will get endpoint of the storage. It basically is the same as :py:meth:`getURLBase`
but without the basePath
:returns: 'proto://hostname<:port>'
"""
urlDict = dict(self.protocolParameters)
# We remove the basePath
urlDict["Path"] = ""
return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def getURLBase( self, withWSUrl = False ):
""" This will get the URL base. This is then appended with the LFN in DIRAC convention.
:param self: self reference
:param bool withWSUrl: flag to include Web Service part of the url
:returns: URL
"""
urlDict = dict( self.protocolParameters )
if not withWSUrl:
urlDict['WSUrl'] = ''
return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
def test_02_unparse(self):
""" pfnunparse and pfnunparse_old
:param self: self reference
"""
for pfn, result in self.pfns.items():
if result["OK"]:
self.assertEqual(pfnunparse(result["Value"]), {
"OK": True,
"Value": pfn
})
self.assertEqual(
pfnunparse(None), {
'Message':
"pfnunparse: wrong type for pfnDict argument, expected a dict, got <type 'NoneType'>",
'OK': False
})
self.assertEqual(
pfnunparse("Path"), {
'Message':
"pfnunparse: wrong type for pfnDict argument, expected a dict, got <type 'str'>",
'OK': False
})
def getSEDefinition(self, seID):
""" Get the Storage Element definition
"""
if type(seID) in StringTypes:
result = self.getSEID(seID)
if not result['OK']:
return result
seID = result['Value']
if seID in self.db.seDefinitions:
if (time.time() - self.db.seDefinitions[seID]['LastUpdate']
) < self.seUpdatePeriod:
if self.db.seDefinitions[seID]['SEDict']:
return S_OK(self.db.seDefinitions[seID])
se = self.db.seDefinitions[seID]['SEName']
else:
result = self.getSEName(seID)
if not result['OK']:
return result
se = result['Value']
self.db.seDefinitions[seID] = {}
self.db.seDefinitions[seID]['SEName'] = se
self.db.seDefinitions[seID]['SEDict'] = {}
self.db.seDefinitions[seID]['LastUpdate'] = 0.
# We have to refresh the SE definition from the CS
result = gConfig.getOptionsDict(
'/Resources/StorageElements/%s/AccessProtocol.1' % se)
if not result['OK']:
return result
seDict = result['Value']
self.db.seDefinitions[seID]['SEDict'] = seDict
if seDict:
# A.T. Ports can be multiple, this can be better done using the Storage plugin
# to provide the replica prefix to keep implementations in one place
if 'Port' in seDict:
ports = seDict['Port']
if ',' in ports:
portList = [x.strip() for x in ports.split(',')]
random.shuffle(portList)
seDict['Port'] = portList[0]
tmpDict = dict(seDict)
tmpDict['FileName'] = ''
result = pfnunparse(tmpDict)
if result['OK']:
self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result[
'Value']
self.db.seDefinitions[seID]['LastUpdate'] = time.time()
return S_OK(self.db.seDefinitions[seID])
def updateURL(self, url, withWSUrl=False):
"""Update the URL according to the current SE parameters"""
result = pfnparse(url, srmSpecific=self.srmSpecificParse)
if not result["OK"]:
return result
urlDict = result["Value"]
urlDict["Protocol"] = self.protocolParameters["Protocol"]
urlDict["Host"] = self.protocolParameters["Host"]
urlDict["Port"] = self.protocolParameters["Port"]
urlDict["WSUrl"] = ""
if withWSUrl:
urlDict["WSUrl"] = self.protocolParameters["WSUrl"]
return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def __addDoubleSlash(self, res):
""" Utilities to add the double slash between the host(:port) and the path
:param res: DIRAC return structure which contains an URL if S_OK
:return: DIRAC structure with corrected URL
"""
if not res['OK']:
return res
url = res['Value']
res = pfnparse(url, srmSpecific=self.srmSpecificParse)
if not res['OK']:
return res
urlDict = res['Value']
urlDict['Path'] = '/' + urlDict['Path']
return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def __addDoubleSlash( self, res ):
""" Utilities to add the double slash between the host(:port) and the path
:param res: DIRAC return structure which contains an URL if S_OK
:return: DIRAC structure with corrected URL
"""
if not res['OK']:
return res
url = res['Value']
res = pfnparse( url, srmSpecific = self.srmSpecificParse )
if not res['OK']:
return res
urlDict = res['Value']
urlDict['Path'] = '/' + urlDict['Path']
return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
def getCurrentURL( self, fileName ):
""" Obtain the current file URL from the current working directory and the filename
:param self: self reference
:param str fileName: path on storage
"""
urlDict = dict( self.protocolParameters )
if not fileName.startswith( '/' ):
# Relative path is given
urlDict['Path'] = self.cwd
result = pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
if not result['OK']:
return result
cwdUrl = result['Value']
fullUrl = '%s%s' % ( cwdUrl, fileName )
return S_OK( fullUrl )
def getCurrentURL(self, fileName):
"""Obtain the current file URL from the current working directory and the filename
:param self: self reference
:param str fileName: path on storage
"""
urlDict = dict(self.protocolParameters)
if not fileName.startswith("/"):
# Relative path is given
urlDict["Path"] = self.cwd
result = pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
if not result["OK"]:
return result
cwdUrl = result["Value"]
fullUrl = "%s%s" % (cwdUrl, fileName)
return S_OK(fullUrl)
def getCurrentURL(self, fileName):
""" Obtain the current file URL from the current working directory and the filename
:param self: self reference
:param str fileName: path on storage
"""
urlDict = dict(self.protocolParameters)
if not fileName.startswith('/'):
# Relative path is given
urlDict['Path'] = self.cwd
result = pfnunparse(urlDict)
if not result['OK']:
return result
cwdUrl = result['Value']
fullUrl = '%s%s' % (cwdUrl, fileName)
return S_OK(fullUrl)
def updateURL(self, url, withWSUrl=False):
""" Update the URL according to the current SE parameters
"""
result = pfnparse(url, srmSpecific=self.srmSpecificParse)
if not result['OK']:
return result
urlDict = result['Value']
urlDict['Protocol'] = self.protocolParameters['Protocol']
urlDict['Host'] = self.protocolParameters['Host']
urlDict['Port'] = self.protocolParameters['Port']
urlDict['WSUrl'] = ''
if withWSUrl:
urlDict['WSUrl'] = self.protocolParameters['WSUrl']
return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def updateURL( self, url, withWSUrl = False ):
""" Update the URL according to the current SE parameters
"""
result = pfnparse( url, srmSpecific = self.srmSpecificParse )
if not result['OK']:
return result
urlDict = result['Value']
urlDict['Protocol'] = self.protocolParameters['Protocol']
urlDict['Host'] = self.protocolParameters['Host']
urlDict['Port'] = self.protocolParameters['Port']
urlDict['WSUrl'] = ''
if withWSUrl:
urlDict['WSUrl'] = self.protocolParameters['WSUrl']
return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
def getSEDefinition(self,seID):
""" Get the Storage Element definition
"""
if type(seID) in StringTypes:
result = self.getSEID(seID)
if not result['OK']:
return result
seID = result['Value']
if seID in self.db.seDefinitions:
if (time.time()-self.db.seDefinitions[seID]['LastUpdate']) < self.seUpdatePeriod:
if self.db.seDefinitions[seID]['SEDict']:
return S_OK(self.db.seDefinitions[seID])
se = self.db.seDefinitions[seID]['SEName']
else:
result = self.getSEName(seID)
if not result['OK']:
return result
se = result['Value']
self.db.seDefinitions[seID] = {}
self.db.seDefinitions[seID]['SEName'] = se
self.db.seDefinitions[seID]['SEDict'] = {}
self.db.seDefinitions[seID]['LastUpdate'] = 0.
# We have to refresh the SE definition from the CS
result = gConfig.getOptionsDict('/Resources/StorageElements/%s/AccessProtocol.1' % se)
if not result['OK']:
return result
seDict = result['Value']
self.db.seDefinitions[seID]['SEDict'] = seDict
if seDict:
# A.T. Ports can be multiple, this can be better done using the Storage plugin
# to provide the replica prefix to keep implementations in one place
if 'Port' in seDict:
ports = seDict['Port']
if ',' in ports:
portList = [ x.strip() for x in ports.split(',') ]
random.shuffle( portList )
seDict['Port'] = portList[0]
tmpDict = dict(seDict)
tmpDict['FileName'] = ''
result = pfnunparse(tmpDict)
if result['OK']:
self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result['Value']
self.db.seDefinitions[seID]['LastUpdate'] = time.time()
return S_OK(self.db.seDefinitions[seID])
def _checkLFNPFNConvention(self, lfn, pfn, se):
""" Check that the PFN corresponds to the LFN-PFN convention
"""
# Check if the PFN corresponds to the LFN convention
if pfn == lfn:
return S_OK()
lfn_pfn = True # flag that the lfn is contained in the pfn
if (len(pfn) < len(lfn)) or (pfn[-len(lfn):] != lfn):
return S_ERROR('PFN does not correspond to the LFN convention')
if not pfn.endswith(lfn):
return S_ERROR()
# Check if the pfn corresponds to the SE definition
result = self._getStorageElement(se)
if not result['OK']:
return result
selement = result['Value']
res = pfnparse(pfn)
if not res['OK']:
return res
pfnDict = res['Value']
protocol = pfnDict['Protocol']
pfnpath = pfnDict['Path']
result = selement.getStorageParameters(protocol)
if not result['OK']:
return result
seDict = result['Value']
sePath = seDict['Path']
ind = pfnpath.find(sePath)
if ind == -1:
return S_ERROR(
'The given PFN %s does not correspond to the %s SE definition'
% (pfn, se))
# Check the full LFN-PFN-SE convention
lfn_pfn_se = True
if lfn_pfn:
seAccessDict = dict(seDict)
seAccessDict['Path'] = sePath + '/' + lfn
check_pfn = pfnunparse(seAccessDict)
if check_pfn != pfn:
return S_ERROR('PFN does not correspond to the LFN convention')
return S_OK()
def getSEDefinition(self, seID):
""" Get the Storage Element definition
"""
if type(seID) in StringTypes:
result = self.getSEID(seID)
if not result['OK']:
return result
seID = result['Value']
if seID in self.db.seDefinitions:
if (time.time() - self.db.seDefinitions[seID]['LastUpdate']
) < self.seUpdatePeriod:
if self.db.seDefinitions[seID]['SEDict']:
return S_OK(self.db.seDefinitions[seID])
se = self.db.seDefinitions[seID]['SEName']
else:
result = self.getSEName(seID)
if not result['OK']:
return result
se = result['Value']
self.db.seDefinitions[seID] = {}
self.db.seDefinitions[seID]['SEName'] = se
self.db.seDefinitions[seID]['SEDict'] = {}
self.db.seDefinitions[seID]['LastUpdate'] = 0.
# We have to refresh the SE definition from the CS
result = gConfig.getOptionsDict(
'/Resources/StorageElements/%s/AccessProtocol.1' % se)
if not result['OK']:
return result
seDict = result['Value']
self.db.seDefinitions[seID]['SEDict'] = seDict
if seDict:
tmpDict = dict(seDict)
tmpDict['FileName'] = ''
result = pfnunparse(tmpDict)
if result['OK']:
self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result[
'Value']
self.db.seDefinitions[seID]['LastUpdate'] = time.time()
return S_OK(self.db.seDefinitions[seID])
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
# Make the service URL from the file URL by stripping off the file part
serviceDict = dict(srcDict)
serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
serviceDict['FileName'] = ''
res = pfnunparse(serviceDict)
if not res['OK']:
return res
srcSEURL = res['Value']
localCache = True
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'],
os.path.join(srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
# Make the service URL from the file URL by stripping off the file part
serviceDict = dict(srcDict)
serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
serviceDict['FileName'] = ''
res = pfnunparse(serviceDict)
if not res['OK']:
return res
srcSEURL = res['Value']
localCache = True
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'], os.path.join(
srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def __init__(self, storageName, parameters):
""" """
StorageBase.__init__(self, storageName, parameters)
self.pluginName = "DIP"
self.log = gLogger.getSubLogger(self.__class__.__name__)
# Several ports can be specified as comma separated list, choose
# randomly one of those ports
ports = self.protocolParameters["Port"].split(",")
random.shuffle(ports)
self.protocolParameters["Port"] = ports[0]
pathDict = dict(self.protocolParameters)
pathDict["Path"] = self.basePath
result = pfnunparse(pathDict)
if result["OK"]:
self.url = result["Value"]
self.checkSum = "CheckSum"
self.isok = True
def __init__(self, storageName, parameters):
"""
"""
StorageBase.__init__(self, storageName, parameters)
self.pluginName = 'DIP'
self.log = gLogger.getSubLogger("DIPStorage", True)
# Several ports can be specified as comma separated list, choose
# randomly one of those ports
ports = self.protocolParameters['Port'].split(',')
random.shuffle(ports)
self.protocolParameters['Port'] = ports[0]
pathDict = dict(self.protocolParameters)
pathDict['Path'] = self.basePath
result = pfnunparse(pathDict)
if result['OK']:
self.url = result['Value']
self.checkSum = "CheckSum"
self.isok = True
def getTransportURL( self, path, protocols = False ):
""" Obtain the TURLs for the supplied path and protocols
"""
res = self.exists( path )
if res['OK']:
for url in res['Value']['Successful']:
if protocols and not self.protocol in protocols:
res['Value']['Successful'].pop( url )
res['Value']['Failed'][url] = 'Protocol not supported'
continue
if url[0] == '/':
nameDict = self.getParameters()['Value']
nameDict['FileName'] = url
ret = pfnunparse( nameDict )
if ret['OK']:
res['Value']['Successful'][url] = ret['Value']
else:
res['Value']['Successful'].pop( url )
res['Value']['Failed'][url] = ret['Message']
else:
res['Value']['Successful'][url] = url
return res
def getTransportURL(self, path, protocols=False):
""" Obtain the TURLs for the supplied path and protocols
"""
res = self.exists(path)
if res['OK']:
for url in res['Value']['Successful']:
if protocols and not self.protocol in protocols:
res['Value']['Successful'].pop(url)
res['Value']['Failed'][url] = 'Protocol not supported'
continue
if url[0] == '/':
nameDict = self.getParameters()['Value']
nameDict['FileName'] = url
ret = pfnunparse(nameDict)
if ret['OK']:
res['Value']['Successful'][url] = ret['Value']
else:
res['Value']['Successful'].pop(url)
res['Value']['Failed'][url] = ret['Message']
else:
res['Value']['Successful'][url] = url
return res
def __init__( self, storageName, parameters ):
"""
"""
StorageBase.__init__( self, storageName, parameters )
self.pluginName = 'DIP'
self.log = gLogger.getSubLogger( "DIPStorage", True )
# Several ports can be specified as comma separated list, choose
# randomly one of those ports
ports = self.protocolParameters['Port'].split( ',' )
random.shuffle( ports )
self.protocolParameters['Port'] = ports[0]
pathDict = dict( self.protocolParameters )
pathDict['Path'] = self.basePath
result = pfnunparse( pathDict )
if result['OK']:
self.url = result['Value']
self.checkSum = "CheckSum"
self.isok = True
def _checkLFNPFNConvention(self, lfn, pfn, se):
""" Check that the PFN corresponds to the LFN-PFN convention
"""
# Check if the PFN corresponds to the LFN convention
if pfn == lfn:
return S_OK()
lfn_pfn = True # flag that the lfn is contained in the pfn
if (len(pfn) < len(lfn)) or (pfn[-len(lfn) :] != lfn):
return S_ERROR("PFN does not correspond to the LFN convention")
if not pfn.endswith(lfn):
return S_ERROR()
# Check if the pfn corresponds to the SE definition
result = self._getStorageElement(se)
if not result["OK"]:
return result
selement = result["Value"]
res = pfnparse(pfn)
if not res["OK"]:
return res
pfnDict = res["Value"]
protocol = pfnDict["Protocol"]
pfnpath = pfnDict["Path"]
result = selement.getStorageParameters(protocol)
if not result["OK"]:
return result
seDict = result["Value"]
sePath = seDict["Path"]
ind = pfnpath.find(sePath)
if ind == -1:
return S_ERROR("The given PFN %s does not correspond to the %s SE definition" % (pfn, se))
# Check the full LFN-PFN-SE convention
lfn_pfn_se = True
if lfn_pfn:
seAccessDict = dict(seDict)
seAccessDict["Path"] = sePath + "/" + lfn
check_pfn = pfnunparse(seAccessDict)
if check_pfn != pfn:
return S_ERROR("PFN does not correspond to the LFN convention")
return S_OK()
def getSEDefinition(self,seID):
""" Get the Storage Element definition
"""
if type(seID) in StringTypes:
result = self.getSEID(seID)
if not result['OK']:
return result
seID = result['Value']
if seID in self.db.seDefinitions:
if (time.time()-self.db.seDefinitions[seID]['LastUpdate']) < self.seUpdatePeriod:
if self.db.seDefinitions[seID]['SEDict']:
return S_OK(self.db.seDefinitions[seID])
se = self.db.seDefinitions[seID]['SEName']
else:
result = self.getSEName(seID)
if not result['OK']:
return result
se = result['Value']
self.db.seDefinitions[seID] = {}
self.db.seDefinitions[seID]['SEName'] = se
self.db.seDefinitions[seID]['SEDict'] = {}
self.db.seDefinitions[seID]['LastUpdate'] = 0.
# We have to refresh the SE definition from the CS
result = gConfig.getOptionsDict('/Resources/StorageElements/%s/AccessProtocol.1' % se)
if not result['OK']:
return result
seDict = result['Value']
self.db.seDefinitions[seID]['SEDict'] = seDict
if seDict:
tmpDict = dict(seDict)
tmpDict['FileName'] = ''
result = pfnunparse(tmpDict)
if result['OK']:
self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result['Value']
self.db.seDefinitions[seID]['LastUpdate'] = time.time()
return S_OK(self.db.seDefinitions[seID])
def getSEDefinition(self, seID):
""" Get the Storage Element definition
"""
if type(seID) in StringTypes:
result = self.getSEID(seID)
if not result["OK"]:
return result
seID = result["Value"]
if seID in self.db.seDefinitions:
if (time.time() - self.db.seDefinitions[seID]["LastUpdate"]) < self.seUpdatePeriod:
if self.db.seDefinitions[seID]["SEDict"]:
return S_OK(self.db.seDefinitions[seID])
se = self.db.seDefinitions[seID]["SEName"]
else:
result = self.getSEName(seID)
if not result["OK"]:
return result
se = result["Value"]
self.db.seDefinitions[seID] = {}
self.db.seDefinitions[seID]["SEName"] = se
self.db.seDefinitions[seID]["SEDict"] = {}
self.db.seDefinitions[seID]["LastUpdate"] = 0.0
# We have to refresh the SE definition from the CS
result = gConfig.getOptionsDict("/Resources/StorageElements/%s/AccessProtocol.1" % se)
if not result["OK"]:
return result
seDict = result["Value"]
self.db.seDefinitions[seID]["SEDict"] = seDict
if seDict:
tmpDict = dict(seDict)
tmpDict["FileName"] = ""
result = pfnunparse(tmpDict)
if result["OK"]:
self.db.seDefinitions[seID]["SEDict"]["PFNPrefix"] = result["Value"]
self.db.seDefinitions[seID]["LastUpdate"] = time.time()
return S_OK(self.db.seDefinitions[seID])
def __addDoubleSlash(self, res):
"""Utilities to add the double slash between the host(:port) and the path
:param res: DIRAC return structure which contains an URL if S_OK
:return: DIRAC structure with corrected URL
"""
if not res["OK"]:
return res
url = res["Value"]
res = pfnparse(url, srmSpecific=self.srmSpecificParse)
if not res["OK"]:
return res
urlDict = res["Value"]
urlDict["Path"] = "/" + urlDict["Path"]
# Now, that's one heck of a disgusting hack
# xrootd client is a bit faulty when managing
# the connection cache, and ends up reusing an
# existing connection for different users (security flaw...)
# they have fixed it (to some extent starting from xrootd 4.10)
# (https://github.com/xrootd/xrootd/issues/976)
# BUT. They still can't consume properly the information when
# the identity is passed in the url (root://url?gsiusrpxy=/tmp/myproxy)
# So we apply a trick here which is to specify the proxy filename as a virtual user
# This has no consequence (developer's own words), but to distinguish between users
# Another ticket has been opened for that https://github.com/xrootd/xrootd/issues/992
try:
proxyLoc = getProxyLocation()
if proxyLoc:
# xroot does not support dots in the virtual user
proxyLoc = os.path.basename(proxyLoc).replace(".", "")
urlDict["Host"] = "%s@%s" % (proxyLoc, urlDict["Host"])
except Exception as e:
self.log.warn("Exception trying to add virtual user in the url",
repr(e))
return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def test_pfnparse(self):
castorPfn = '/castor/cern.ch/grid/lhcb/production/somefile.file'
res = pfnparse(castorPfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(castorPfn, newPfn)
rfioPfn = 'rfio:/castor/cern.ch/grid/lhcb/production/somefile.file'
res = pfnparse(rfioPfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(rfioPfn, newPfn)
miniSrmPfn = 'srm://srm.cern.ch/castor/cern.ch/grid/lhcb/prodution/somefile.file'
res = pfnparse(miniSrmPfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(miniSrmPfn, newPfn)
portSrmPfn = 'srm://srm.cern.ch:8443/castor/cern.ch/grid/lhcb/prodution/somefile.file'
res = pfnparse(portSrmPfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(portSrmPfn, newPfn)
srmPfn = 'srm://srm.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/grid/lhcb/prodution/somefile.file'
res = pfnparse(srmPfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(srmPfn, newPfn)
filePfn = 'file:/opt/dirac/somefile.file'
res = pfnparse(filePfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(filePfn, newPfn)
def test_pfnparse(self):
castorPfn = '/castor/cern.ch/grid/lhcb/production/somefile.file'
res = pfnparse(castorPfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(castorPfn,newPfn)
rfioPfn = 'rfio:/castor/cern.ch/grid/lhcb/production/somefile.file'
res = pfnparse(rfioPfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(rfioPfn,newPfn)
miniSrmPfn = 'srm://srm.cern.ch/castor/cern.ch/grid/lhcb/prodution/somefile.file'
res = pfnparse(miniSrmPfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(miniSrmPfn,newPfn)
portSrmPfn = 'srm://srm.cern.ch:8443/castor/cern.ch/grid/lhcb/prodution/somefile.file'
res = pfnparse(portSrmPfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(portSrmPfn,newPfn)
srmPfn = 'srm://srm.cern.ch:8443/srm/managerv2?SFN=/castor/cern.ch/grid/lhcb/prodution/somefile.file'
res = pfnparse(srmPfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(srmPfn,newPfn)
filePfn = 'file:/opt/dirac/somefile.file'
res = pfnparse(filePfn)
pfnDict = res['Value']
res = pfnunparse(pfnDict)
newPfn = res['Value']
self.assertEqual(filePfn,newPfn)
def getProtocolPfn( self, pfnDict, withPort ): """ From the pfn dict construct the SURL to be used """ return pfnunparse( pfnDict )
def getProtocolPfn(self, pfnDict, withPort):
""" From the pfn dict construct the SURL to be used
"""
return pfnunparse(pfnDict)
def getSEDefinition(self, seID):
""" Get the Storage Element definition
"""
if isinstance(seID, str):
result = self.getSEID(seID)
if not result['OK']:
return result
seID = result['Value']
if seID in self.db.seDefinitions:
if (time.time() - self.db.seDefinitions[seID]['LastUpdate']
) < self.seUpdatePeriod:
if self.db.seDefinitions[seID]['SEDict']:
return S_OK(self.db.seDefinitions[seID])
se = self.db.seDefinitions[seID]['SEName']
else:
result = self.getSEName(seID)
if not result['OK']:
return result
se = result['Value']
self.db.seDefinitions[seID] = {}
self.db.seDefinitions[seID]['SEName'] = se
self.db.seDefinitions[seID]['SEDict'] = {}
self.db.seDefinitions[seID]['LastUpdate'] = 0.
# We have to refresh the SE definition from the CS
result = gConfig.getSections('/Resources/StorageElements/%s' % se)
if not result['OK']:
return result
pluginSection = result['Value'][0]
result = gConfig.getOptionsDict('/Resources/StorageElements/%s/%s' %
(se, pluginSection))
if not result['OK']:
return result
seDict = result['Value']
self.db.seDefinitions[seID]['SEDict'] = seDict
# Get VO paths if any
voPathDict = None
result = gConfig.getOptionsDict(
'/Resources/StorageElements/%s/%s/VOPath' % (se, pluginSection))
if result['OK']:
voPathDict = result['Value']
if seDict:
# A.T. Ports can be multiple, this can be better done using the Storage plugin
# to provide the replica prefix to keep implementations in one place
if 'Port' in seDict:
ports = seDict['Port']
if ',' in ports:
portList = [x.strip() for x in ports.split(',')]
random.shuffle(portList)
seDict['Port'] = portList[0]
tmpDict = dict(seDict)
tmpDict['FileName'] = ''
result = pfnunparse(tmpDict)
if result['OK']:
self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result[
'Value']
if voPathDict is not None:
for vo in voPathDict:
tmpDict['Path'] = voPathDict[vo]
result = pfnunparse(tmpDict)
if result['OK']:
self.db.seDefinitions[seID]['SEDict'].setdefault(
"VOPrefix", {})[vo] = result['Value']
self.db.seDefinitions[seID]['LastUpdate'] = time.time()
return S_OK(self.db.seDefinitions[seID])