def test_01_parse( self ):
""" pfnparse and pfnparse_old
:param self: self reference
"""
for pfn, result in self.pfns.items():
self.assertEqual( pfnparse( pfn )['OK'], result['OK'] )
self.assertEqual( pfnparse( pfn ).get('Errno'), result.get('Errno') )
def getJobStatus(self, jobIDList):
""" Get status of the jobs in the given list
"""
hostDict = {}
for job in jobIDList:
result = pfnparse(job)
if not result['OK']:
continue
host = result['Value']['Host']
hostDict.setdefault(host, [])
hostDict[host].append(job)
resultDict = {}
failed = []
for host, jobIDList in hostDict.items():
result = self._getJobStatusOnHost(jobIDList, host)
if not result['OK']:
failed.extend(jobIDList)
continue
resultDict.update(result['Value'])
for job in failed:
if not job in resultDict:
resultDict[job] = 'Unknown'
return S_OK(resultDict)
def __generatePfnDict( self, lfns, storage ):
""" Generates a dictionnary (pfn : lfn ), where the pfn are constructed
from the lfn using the getProtocolPfn method of the storage plugins.
:param: lfns : dictionnary {lfn:whatever}
:returns dictionnary {constructed pfn : lfn}
"""
self.log.verbose( "StorageElement.__generatePfnDict: generating pfn dict for %s lfn in %s." % ( len( lfns ), self.name ) )
pfnDict = {} # pfn : lfn
failed = {} # lfn : string with errors
for lfn in lfns:
if ":" in lfn:
errStr = "StorageElement.__generatePfnDict: received a pfn as input. It should not happen anymore, please check your code"
self.log.verbose( errStr, lfn )
res = pfnparse( lfn ) # pfnparse can take an lfn as input, it will just fill the path and filename
if not res['OK']:
errStr = "StorageElement.__generatePfnDict: Failed to parse supplied LFN."
self.log.debug( errStr, "%s: %s" % ( lfn, res['Message'] ) )
if lfn not in failed:
failed[lfn] = ''
failed[lfn] = "%s %s" % ( failed[lfn], errStr )
else:
res = storage.getProtocolPfn( res['Value'], True )
if not res['OK']:
errStr = "StorageElement.__generatePfnDict %s." % res['Message']
self.log.debug( errStr, 'for %s' % ( lfn ) )
if lfn not in failed:
failed[lfn] = ''
failed[lfn] = "%s %s" % ( failed[lfn], errStr )
else:
pfnDict[res['Value']] = lfn
res = S_OK( pfnDict )
res['Failed'] = failed
return res
def __generatePfnDict(self, pfns, storage):
""" whatever, it creates PFN dict """
pfnDict = {}
failed = {}
for pfn in pfns:
res = pfnparse(pfn)
if not res['OK']:
errStr = "__generatePfnDict: Failed to parse supplied PFN."
self.log.error(errStr, "%s: %s" % (pfn, res['Message']))
if pfn not in failed:
failed[pfn] = ''
failed[pfn] = "%s %s" % (failed[pfn], errStr)
else:
res = storage.getProtocolPfn(res['Value'], True)
if not res['OK']:
errStr = "__generatePfnDict %s." % res['Message']
self.log.error(errStr, 'for %s' % (pfn))
if pfn not in failed:
failed[pfn] = ''
failed[pfn] = "%s %s" % (failed[pfn], errStr)
else:
pfnDict[res['Value']] = pfn
res = S_OK(pfnDict)
res['Failed'] = failed
return res
def __putFile( self, src_file, dest_url ):
res = pfnparse( src_file )
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
localCache = True
srcSEURL = srcDict['Protocol'] + '://' + srcDict['Host'] + ':' + srcDict['Port'] + srcDict['WSUrl']
transferClient = TransferClient( srcSEURL )
res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'] ) )
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists( src_file ):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error( errStr, src_file )
return S_ERROR( errStr )
sourceSize = getSize( src_file )
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error( errStr, src_file )
return S_ERROR( errStr )
transferClient = TransferClient( self.url )
res = transferClient.sendFile( src_file, dest_url, token = self.checkSum )
if localCache:
os.unlink( src_file )
if res['OK']:
return S_OK( sourceSize )
else:
return res
def killJob( self, jobIDs ):
""" Kill specified jobs
"""
jobIDList = list( jobIDs )
if type( jobIDs ) == type( ' ' ):
jobIDList = [jobIDs]
hostDict = {}
for job in jobIDList:
result = pfnparse( job )
if not result['OK']:
continue
host = result['Value']['Host']
hostDict.setdefault(host,[])
hostDict[host].append( job )
failed = []
for host,jobIDList in hostDict.items():
result = self._killJobOnHost( jobIDList, host )
if not result['OK']:
failed.extend( jobIDList )
message = result['Message']
if failed:
result = S_ERROR(message)
result['Failed'] = failed
else:
result = S_OK()
return result
def getPfnPath(self, pfn):
""" Get the part of the PFN path below the basic storage path.
This path must coincide with the LFN of the file in order to be compliant with the LHCb conventions.
"""
if not self.valid:
return S_ERROR(self.errorReason)
res = pfnparse(pfn)
if not res['OK']:
return res
fullPfnPath = '%s/%s' % (res['Value']['Path'],
res['Value']['FileName'])
# Check all available storages and check whether the pfn is for that protocol
pfnPath = ''
for storage in self.storages:
res = storage.isPfnForProtocol(pfn)
if res['OK']:
if res['Value']:
res = storage.getParameters()
saPath = res['Value']['Path']
if not saPath:
# If the sa path doesn't exist then the pfn path is the entire string
pfnPath = fullPfnPath
else:
if re.search(saPath, fullPfnPath):
# Remove the sa path from the fullPfnPath
pfnPath = fullPfnPath.replace(saPath, '')
if pfnPath:
return S_OK(pfnPath)
# This should never happen. DANGER!!
errStr = "StorageElement.getPfnPath: Failed to get the pfn path for any of the protocols!!"
gLogger.error(errStr)
return S_ERROR(errStr)
def __getURLPath(self, url):
""" Get the part of the URL path below the basic storage path.
This path must coincide with the LFN of the file in order to be compliant with the DIRAC conventions.
"""
log = self.log.getSubLogger('__getURLPath')
log.verbose("Getting path from url in %s." % self.name)
if not self.valid:
return S_ERROR(self.errorReason)
res = pfnparse(url)
if not res['OK']:
return res
fullURLPath = '%s/%s' % (res['Value']['Path'],
res['Value']['FileName'])
# Check all available storages and check whether the url is for that protocol
urlPath = ''
for storage in self.storages:
res = storage.isNativeURL(url)
if res['OK']:
if res['Value']:
parameters = storage.getParameters()
saPath = parameters['Path']
if not saPath:
# If the sa path doesn't exist then the url path is the entire string
urlPath = fullURLPath
else:
if re.search(saPath, fullURLPath):
# Remove the sa path from the fullURLPath
urlPath = fullURLPath.replace(saPath, '')
if urlPath:
return S_OK(urlPath)
# This should never happen. DANGER!!
errStr = "Failed to get the url path for any of the protocols!!"
log.debug(errStr)
return S_ERROR(errStr)
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
localCache = True
srcSEURL = srcDict['Protocol'] + '://' + srcDict[
'Host'] + ':' + srcDict['Port'] + srcDict['WSUrl']
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'],
os.path.join(srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def _getKeyFromURL(self, url):
"""Extract the Key from the URL.
The key is basically the LFN without trailing slash
I despise such path mangling, expecially after all the efforts to
get ride of such method. However, since in the case of S3 we need
to go back and forth between URL and LFN (for finding keys, checking
accesses in the gw, etc), there is no other option...
:param url: s3 url
:returns: S_OK(key) / S_ERROR
"""
res = pfnparse(url, srmSpecific=False)
if not res["OK"]:
return res
splitURL = res["Value"]
# The path originally looks like '/bucket/lhcb/user/c/chaen
# We remove the trailing slash, and get the relative path
# of bucket/lhcb/user/c/chaen starting from bucket,
# which gives you basically the LFN without trailing slash
path = os.path.relpath(splitURL["Path"].lstrip("/"), start=self.bucketName)
key = os.path.join(path, splitURL["FileName"])
return S_OK(key)
def __getURLPath(self, url):
""" Get the part of the URL path below the basic storage path.
This path must coincide with the LFN of the file in order to be compliant with the DIRAC conventions.
"""
log = self.log.getSubLogger('__getURLPath')
log.verbose("Getting path from url in %s." % self.name)
if not self.valid:
return S_ERROR(self.errorReason)
res = pfnparse(url)
if not res['OK']:
return res
fullURLPath = '%s/%s' % (res['Value']['Path'], res['Value']['FileName'])
# Check all available storages and check whether the url is for that protocol
urlPath = ''
for storage in self.storages:
res = storage.isNativeURL(url)
if res['OK']:
if res['Value']:
parameters = storage.getParameters()
saPath = parameters['Path']
if not saPath:
# If the sa path doesn't exist then the url path is the entire string
urlPath = fullURLPath
else:
if re.search(saPath, fullURLPath):
# Remove the sa path from the fullURLPath
urlPath = fullURLPath.replace(saPath, '')
if urlPath:
return S_OK(urlPath)
# This should never happen. DANGER!!
errStr = "Failed to get the url path for any of the protocols!!"
log.debug(errStr)
return S_ERROR(errStr)
def getPfnPath( self, pfn ):
""" Get the part of the PFN path below the basic storage path.
This path must coincide with the LFN of the file in order to be compliant with the LHCb conventions.
"""
if not self.valid:
return S_ERROR( self.errorReason )
res = pfnparse( pfn )
if not res['OK']:
return res
fullPfnPath = '%s/%s' % ( res['Value']['Path'], res['Value']['FileName'] )
# Check all available storages and check whether the pfn is for that protocol
pfnPath = ''
for storage in self.storages:
res = storage.isPfnForProtocol( pfn )
if res['OK']:
if res['Value']:
res = storage.getParameters()
saPath = res['Value']['Path']
if not saPath:
# If the sa path doesn't exist then the pfn path is the entire string
pfnPath = fullPfnPath
else:
if re.search( saPath, fullPfnPath ):
# Remove the sa path from the fullPfnPath
pfnPath = fullPfnPath.replace( saPath, '' )
if pfnPath:
return S_OK( pfnPath )
# This should never happen. DANGER!!
errStr = "getPfnPath: Failed to get the pfn path for any of the protocols!!"
self.log.error( errStr )
return S_ERROR( errStr )
def getJobStatus ( self, jobIDList ):
""" Get status of the jobs in the given list
"""
hostDict = {}
for job in jobIDList:
result = pfnparse( job )
if not result['OK']:
continue
host = result['Value']['Host']
hostDict.setdefault(host,[])
hostDict[host].append( job )
resultDict = {}
failed = []
for host,jobIDList in hostDict.items():
result = self._getJobStatusOnHost( jobIDList, host )
if not result['OK']:
failed.extend( jobIDList )
continue
resultDict.update( result['Value'] )
for job in failed:
if not job in resultDict:
resultDict[job] = 'Unknown'
return S_OK( resultDict )
def killJob(self, jobIDs):
""" Kill specified jobs
"""
jobIDList = list(jobIDs)
if type(jobIDs) == type(' '):
jobIDList = [jobIDs]
hostDict = {}
for job in jobIDList:
result = pfnparse(job)
if not result['OK']:
continue
host = result['Value']['Host']
hostDict.setdefault(host, [])
hostDict[host].append(job)
failed = []
for host, jobIDList in hostDict.items():
result = self._killJobOnHost(jobIDList, host)
if not result['OK']:
failed.extend(jobIDList)
message = result['Message']
if failed:
result = S_ERROR(message)
result['Failed'] = failed
else:
result = S_OK()
return result
def __generatePfnDict( self, pfns, storage ):
""" whatever, it creates PFN dict """
pfnDict = {}
failed = {}
for pfn in pfns:
res = pfnparse( pfn )
if not res['OK']:
errStr = "__generatePfnDict: Failed to parse supplied PFN."
self.log.error( errStr, "%s: %s" % ( pfn, res['Message'] ) )
if pfn not in failed:
failed[pfn] = ''
failed[pfn] = "%s %s" % ( failed[pfn], errStr )
else:
res = storage.getProtocolPfn( res['Value'], True )
if not res['OK']:
errStr = "__generatePfnDict %s." % res['Message']
self.log.error( errStr, 'for %s' % ( pfn ) )
if pfn not in failed:
failed[pfn] = ''
failed[pfn] = "%s %s" % ( failed[pfn], errStr )
else:
pfnDict[res['Value']] = pfn
res = S_OK( pfnDict )
res['Failed'] = failed
return res
def test_01_parse(self):
""" pfnparse and pfnparse_old
:param self: self reference
"""
for pfn, result in self.pfns.items():
self.assertEqual(pfnparse(pfn), result)
def test_01_parse( self ):
""" pfnparse and pfnparse_old
:param self: self reference
"""
for pfn, result in self.pfns.items():
self.assertEqual( pfnparse( pfn ), result )
def _getJobStatusOnHost(self, jobIDList, host=None):
""" Get the status information for the given list of jobs
"""
# self.log.verbose( '*** getUnitJobStatus %s - %s\n' % ( jobIDList, host) )
resultDict = {}
ssh = SSH(host=host, parameters=self.ceParameters)
jobDict = {}
for job in jobIDList:
result = pfnparse(job)
if result["OK"]:
stamp = result["Value"]["FileName"]
else:
self.log.error("Invalid job id", job)
continue
jobDict[stamp] = job
stampList = jobDict.keys()
cmd = "bash --login -c '%s/%s job_status %s %s %s'" % (
self.sharedArea,
self.controlScript,
"#".join(stampList),
self.infoArea,
self.ceParameters["SSHUser"],
)
result = ssh.sshCall(30, cmd)
if not result["OK"]:
return result
sshStatus = result["Value"][0]
sshStdout = result["Value"][1]
sshStderr = result["Value"][2]
if sshStatus == 0:
outputLines = sshStdout.strip().replace("\r", "").split("\n")
try:
index = outputLines.index("============= Start output ===============")
outputLines = outputLines[index + 1 :]
except:
return S_ERROR("Invalid output from job get status: %s" % outputLines[0])
try:
status = int(outputLines[0])
except:
return S_ERROR("Failed local batch job status: %s" % outputLines[0])
if status != 0:
message = "Unknown reason"
if len(outputLines) > 1:
message = outputLines[1]
return S_ERROR("Failed job kill, reason: %s" % message)
else:
for line in outputLines[1:]:
jbundle = line.split(":::")
if len(jbundle) == 2:
resultDict[jobDict[jbundle[0]]] = jbundle[1]
else:
return S_ERROR("\n".join([sshStdout, sshStderr]))
# self.log.verbose( ' !!! getUnitJobStatus will return : %s\n' % resultDict )
return S_OK(resultDict)
def getGridSRMs( vo, bdiiInfo = None, srmBlackList = None, unUsed = False ):
result = ldapService( serviceType = 'SRM', vo = vo )
if not result['OK']:
return result
srmBdiiDict = result['Value']
knownSRMs = set()
if srmBlackList is not None:
knownSRMs = knownSRMs.union( set( srmBlackList ) )
siteSRMDict = {}
for srm in srmBdiiDict:
endPoint = srm.get( 'GlueServiceEndpoint', '')
srmHost = ''
if endPoint:
result = pfnparse( endPoint )
if not result['OK']:
continue
srmHost = result['Value']['Host']
if not srmHost:
continue
if srmHost in knownSRMs:
continue
if unUsed:
result = getDIRACSesForSRM( srmHost )
if not result['OK']:
return result
diracSEs = result['Value']
if diracSEs:
# If it is a known SRM and only new SRMs are requested, continue
continue
site = srm.get( 'GlueForeignKey', '' ).replace( 'GlueSiteUniqueID=', '' )
siteSRMDict.setdefault( site, {} )
siteSRMDict[site][srmHost] = srm
if bdiiInfo is None:
result = getBdiiSEInfo( vo )
if not result['OK']:
return result
seBdiiDict = result['Value']
else:
seBdiiDict = bdiiInfo
srmSeDict = {}
for site in siteSRMDict:
srms = siteSRMDict[site].keys()
for srm in srms:
if seBdiiDict.get( site, {} ).get( 'SEs', {} ).get( srm, {} ):
srmSeDict.setdefault( site, {} )
srmSeDict[site].setdefault( srm, {} )
srmSeDict[site][srm]['SRM'] = siteSRMDict[site][srm]
srmSeDict[site][srm]['SE'] = seBdiiDict[site]['SEs'][srm]
return S_OK( srmSeDict )
def getGridSRMs( vo, bdiiInfo = None, srmBlackList = None, unUsed = False ):
result = ldapService( serviceType = 'SRM', vo = vo )
if not result['OK']:
return result
srmBdiiDict = result['Value']
knownSRMs = set()
if srmBlackList is not None:
knownSRMs = knownSRMs.union( set( srmBlackList ) )
siteSRMDict = {}
for srm in srmBdiiDict:
endPoint = srm.get( 'GlueServiceEndpoint', '')
srmHost = ''
if endPoint:
result = pfnparse( endPoint )
if not result['OK']:
continue
srmHost = result['Value']['Host']
if not srmHost:
continue
if srmHost in knownSRMs:
continue
if unUsed:
result = getDIRACSesForSRM( srmHost )
if not result['OK']:
return result
diracSEs = result['Value']
if diracSEs:
# If it is a known SRM and only new SRMs are requested, continue
continue
site = srm.get( 'GlueForeignKey', '' ).replace( 'GlueSiteUniqueID=', '' )
siteSRMDict.setdefault( site, {} )
siteSRMDict[site][srmHost] = srm
if bdiiInfo is None:
result = getBdiiSEInfo( vo )
if not result['OK']:
return result
seBdiiDict = result['Value']
else:
seBdiiDict = bdiiInfo
srmSeDict = {}
for site in siteSRMDict:
srms = siteSRMDict[site].keys()
for srm in srms:
if seBdiiDict.get( site, {} ).get( 'SEs', {} ).get( srm, {} ):
srmSeDict.setdefault( site, {} )
srmSeDict[site].setdefault( srm, {} )
srmSeDict[site][srm]['SRM'] = siteSRMDict[site][srm]
srmSeDict[site][srm]['SE'] = seBdiiDict[site]['SEs'][srm]
return S_OK( srmSeDict )
def _getJobStatus(self, jobIDList):
""" Get the status information for the given list of jobs
"""
resultDict = {}
jobDict = {}
for job in jobIDList:
result = pfnparse(job)
if result['OK']:
stamp = result['Value']['FileName']
else:
self.log.error('Invalid job id', job)
continue
jobDict[stamp] = job
stampList = jobDict.keys()
cmdTuple = [
self.finalScript, 'job_status', '#'.join(stampList), self.infoArea,
self.userName
]
result = systemCall(10, cmdTuple)
if not result['OK']:
return result
status = result['Value'][0]
stdout = result['Value'][1]
stderr = result['Value'][2]
# Examine results of the job status
if status == 0:
outputLines = stdout.strip().replace('\r', '').split('\n')
try:
index = outputLines.index(
'============= Start output ===============')
outputLines = outputLines[index + 1:]
except:
return S_ERROR("Invalid output from CE get status: %s" %
outputLines[0])
try:
status = int(outputLines[0])
except:
return S_ERROR("Failed to get CE status: %s" % outputLines[0])
if status != 0:
message = "Unknown reason"
if len(outputLines) > 1:
message = outputLines[1]
return S_ERROR('Failed to get CE status, reason: %s' % message)
for line in outputLines[1:]:
if ':::' in line:
jbundle = line.split(':::')
if (len(jbundle) == 2):
resultDict[jobDict[jbundle[0]]] = jbundle[1]
else:
return S_ERROR('\n'.join([stdout, stderr]))
return S_OK(resultDict)
def isPfnForProtocol(self, pfn):
res = pfnparse(pfn)
if not res['OK']:
return res
pfnDict = res['Value']
if pfnDict['Protocol'] == self.protocol:
return S_OK(True)
else:
return S_OK(False)
def isPfnForProtocol( self, pfn ):
res = pfnparse( pfn )
if not res['OK']:
return res
pfnDict = res['Value']
if pfnDict['Protocol'] == self.protocol:
return S_OK( True )
else:
return S_OK( False )
def _getJobStatusOnHost(self, jobIDList, host=None):
""" Get the status information for the given list of jobs
"""
# self.log.verbose( '*** getUnitJobStatus %s - %s\n' % ( jobIDList, host) )
resultDict = {}
ssh = SSH(host=host, parameters=self.ceParameters)
jobDict = {}
for job in jobIDList:
result = pfnparse(job)
if result['OK']:
stamp = result['Value']['FileName']
else:
self.log.error('Invalid job id', job)
continue
jobDict[stamp] = job
stampList = jobDict.keys()
cmd = "bash --login -c '%s/%s job_status %s %s %s'" % (
self.sharedArea, self.controlScript, '#'.join(stampList),
self.infoArea, self.ceParameters['SSHUser'])
result = ssh.sshCall(10, cmd)
if not result['OK']:
return result
sshStatus = result['Value'][0]
sshStdout = result['Value'][1]
sshStderr = result['Value'][2]
if sshStatus == 0:
outputLines = sshStdout.strip().replace('\r', '').split('\n')
try:
index = outputLines.index(
'============= Start output ===============')
outputLines = outputLines[index + 1:]
except:
return S_ERROR("Invalid output from job get status: %s" %
outputLines[0])
try:
status = int(outputLines[0])
except:
return S_ERROR("Failed local batch job status: %s" %
outputLines[0])
if status != 0:
message = "Unknown reason"
if len(outputLines) > 1:
message = outputLines[1]
return S_ERROR('Failed job kill, reason: %s' % message)
else:
for line in outputLines[1:]:
jbundle = line.split(':::')
if (len(jbundle) == 2):
resultDict[jobDict[jbundle[0]]] = jbundle[1]
else:
return S_ERROR('\n'.join([sshStdout, sshStderr]))
# self.log.verbose( ' !!! getUnitJobStatus will return : %s\n' % resultDict )
return S_OK(resultDict)
def isPfnForProtocol(self, pfn):
res = pfnparse(pfn)
if not res["OK"]:
return res
pfnDict = res["Value"]
if pfnDict["Protocol"] == self.protocol:
return S_OK(True)
else:
return S_OK(False)
def test_02_default_parse( self ):
""" pfnparse and pfnparse_old
:param self: self reference
"""
for pfn, result in self.default_pfns.items():
parseResult = pfnparse( pfn, srmSpecific = False )
self.assertEqual( parseResult['OK'], result['OK'] )
if result['OK']:
self.assertEqual( parseResult['Value'], result['Value'] )
def isNativeURL(self, url):
"""Check if URL :url: is valid for :self.protocol:
:param self: self reference
:param str url: URL
"""
res = pfnparse(url, srmSpecific=self.srmSpecificParse)
if not res["OK"]:
return res
urlDict = res["Value"]
return S_OK(urlDict["Protocol"] == self.protocolParameters["Protocol"])
def test_02_default_parse( self ):
""" pfnparse and pfnparse_old
:param self: self reference
"""
for pfn, result in self.default_pfns.iteritems():
parseResult = pfnparse( pfn, srmSpecific = False )
self.assertEqual( parseResult['OK'], result['OK'] )
if result['OK']:
self.assertEqual( parseResult['Value'], result['Value'] )
def isNativeURL(self, url):
""" Check if URL :url: is valid for :self.protocol:
:param self: self reference
:param str url: URL
"""
res = pfnparse(url)
if not res['OK']:
return res
urlDict = res['Value']
return S_OK(urlDict['Protocol'] == self.protocolParameters['Protocol'])
def isNativeURL( self, url ):
""" Check if URL :url: is valid for :self.protocol:
:param self: self reference
:param str url: URL
"""
res = pfnparse( url, srmSpecific = self.srmSpecificParse )
if not res['OK']:
return res
urlDict = res['Value']
return S_OK( urlDict['Protocol'] == self.protocolParameters['Protocol'] )
def test_01_srm_parse( self ):
""" pfnparse and pfnparse_old
:param self: self reference
"""
for pfn, result in self.srm_pfns.items():
parseResult = pfnparse( pfn )
self.assertEqual( parseResult['OK'], result['OK'] )
if result['OK']:
self.assertEqual( parseResult['Value'], result['Value'] )
def test_01_srm_parse( self ):
""" pfnparse and pfnparse_old
:param self: self reference
"""
for pfn, result in self.srm_pfns.iteritems():
parseResult = pfnparse( pfn )
self.assertEqual( parseResult['OK'], result['OK'] )
if result['OK']:
self.assertEqual( parseResult['Value'], result['Value'] )
def _getJobStatus( self, jobIDList ):
""" Get the status information for the given list of jobs
"""
resultDict = {}
jobDict = {}
for job in jobIDList:
result = pfnparse( job )
if result['OK']:
stamp = result['Value']['FileName']
else:
self.log.error( 'Invalid job id', job )
continue
jobDict[stamp] = job
stampList = jobDict.keys()
cmdTuple = [ self.finalScript, 'job_status', '#'.join( stampList ), self.infoArea, self.userName ]
result = systemCall( 10, cmdTuple )
if not result['OK']:
return result
status = result['Value'][0]
stdout = result['Value'][1]
stderr = result['Value'][2]
# Examine results of the job status
if status == 0:
outputLines = stdout.strip().replace( '\r', '' ).split( '\n' )
try:
index = outputLines.index( '============= Start output ===============' )
outputLines = outputLines[index + 1:]
except:
return S_ERROR( "Invalid output from CE get status: %s" % outputLines[0] )
try:
status = int( outputLines[0] )
except:
return S_ERROR( "Failed to get CE status: %s" % outputLines[0] )
if status != 0:
message = "Unknown reason"
if len( outputLines ) > 1:
message = outputLines[1]
return S_ERROR( 'Failed to get CE status, reason: %s' % message )
for line in outputLines[1:]:
if ':::' in line:
jbundle = line.split( ':::' )
if ( len( jbundle ) == 2 ):
resultDict[jobDict[jbundle[0]]] = jbundle[1]
else:
return S_ERROR( '\n'.join( [stdout, stderr] ) )
return S_OK( resultDict )
def _getJobOutputFiles( self, jobID ):
""" Get output file names for the specific CE
"""
result = pfnparse( jobID )
if not result['OK']:
return result
jobStamp = result['Value']['FileName']
host = result['Value']['Host']
output = '%s/DIRACPilot.o%s' % ( self.batchOutput, jobStamp )
error = '%s/DIRACPilot.e%s' % ( self.batchError, jobStamp )
return S_OK( (jobStamp,host,output,error) )
def _getJobOutputFiles(self, jobID):
""" Get output file names for the specific CE
"""
result = pfnparse(jobID)
if not result['OK']:
return result
jobStamp = result['Value']['FileName']
host = result['Value']['Host']
output = '%s/DIRACPilot.o%s' % (self.batchOutput, jobStamp)
error = '%s/DIRACPilot.e%s' % (self.batchError, jobStamp)
return S_OK((jobStamp, host, output, error))
def _getJobOutputFiles(self, jobID):
""" Get output file names for the specific CE
"""
result = pfnparse(jobID)
if not result["OK"]:
return result
jobStamp = result["Value"]["FileName"]
host = result["Value"]["Host"]
output = "%s/%s.out" % (self.batchOutput, jobStamp)
error = "%s/%s.err" % (self.batchError, jobStamp)
return S_OK((jobStamp, host, output, error))
def _killJobOnHost(self, jobIDList, host=None):
""" Kill the jobs for the given list of job IDs
"""
resultDict = {}
ssh = SSH(host=host, parameters=self.ceParameters)
jobDict = {}
for job in jobIDList:
result = pfnparse(job)
if result['OK']:
stamp = result['Value']['FileName']
else:
self.log.error('Invalid job id', job)
continue
jobDict[stamp] = job
stampList = jobDict.keys()
cmd = "bash --login -c '%s/%s kill_job %s %s'" % (
self.sharedArea, self.controlScript, '#'.join(stampList),
self.infoArea)
result = ssh.sshCall(10, cmd)
if not result['OK']:
return result
sshStatus = result['Value'][0]
sshStdout = result['Value'][1]
sshStderr = result['Value'][2]
# Examine results of the job submission
if sshStatus == 0:
outputLines = sshStdout.strip().replace('\r', '').split('\n')
try:
index = outputLines.index(
'============= Start output ===============')
outputLines = outputLines[index + 1:]
except:
return S_ERROR("Invalid output from job kill: %s" %
outputLines[0])
try:
status = int(outputLines[0])
except:
return S_ERROR("Failed local batch job kill: %s" %
outputLines[0])
if status != 0:
message = "Unknown reason"
if len(outputLines) > 1:
message = outputLines[1]
return S_ERROR('Failed job kill, reason: %s' % message)
else:
return S_ERROR('\n'.join([sshStdout, sshStderr]))
return S_OK()
def _killJobOnHost(self, jobIDList, host=None):
""" Kill the jobs for the given list of job IDs
"""
resultDict = {}
ssh = SSH(host=host, parameters=self.ceParameters)
jobDict = {}
for job in jobIDList:
result = pfnparse(job)
if result["OK"]:
stamp = result["Value"]["FileName"]
else:
self.log.error("Invalid job id", job)
continue
jobDict[stamp] = job
stampList = jobDict.keys()
cmd = "bash --login -c '%s/%s kill_job %s %s'" % (
self.sharedArea,
self.controlScript,
"#".join(stampList),
self.infoArea,
)
result = ssh.sshCall(10, cmd)
if not result["OK"]:
return result
sshStatus = result["Value"][0]
sshStdout = result["Value"][1]
sshStderr = result["Value"][2]
# Examine results of the job submission
if sshStatus == 0:
outputLines = sshStdout.strip().replace("\r", "").split("\n")
try:
index = outputLines.index("============= Start output ===============")
outputLines = outputLines[index + 1 :]
except:
return S_ERROR("Invalid output from job kill: %s" % outputLines[0])
try:
status = int(outputLines[0])
except:
return S_ERROR("Failed local batch job kill: %s" % outputLines[0])
if status != 0:
message = "Unknown reason"
if len(outputLines) > 1:
message = outputLines[1]
return S_ERROR("Failed job kill, reason: %s" % message)
else:
return S_ERROR("\n".join([sshStdout, sshStderr]))
return S_OK()
def _killJobs(self, jobIDList, host=None):
""" Kill the jobs for the given list of job IDs
"""
resultDict = {}
jobDict = {}
for job in jobIDList:
result = pfnparse(job)
if result['OK']:
stamp = result['Value']['FileName']
else:
self.log.error('Invalid job id', job)
continue
jobDict[stamp] = job
stampList = jobDict.keys()
cmdTuple = [
self.finalScript, 'kill_job', '#'.join(stampList), self.infoArea
]
result = systemCall(10, cmdTuple)
if not result['OK']:
return result
status = result['Value'][0]
stdout = result['Value'][1]
stderr = result['Value'][2]
# Examine results of the job submission
if status != 0:
outputLines = stdout.strip().replace('\r', '').split('\n')
try:
index = outputLines.index(
'============= Start output ===============')
outputLines = outputLines[index + 1:]
except:
return S_ERROR("Invalid output from kill Job: %s" %
outputLines[0])
try:
status = int(outputLines[0])
except:
return S_ERROR("Failed to kill Job: %s" % outputLines[0])
if status != 0:
message = "Unknown reason"
if len(outputLines) > 1:
message = outputLines[1]
return S_ERROR('Failed to kill Job, reason: %s' % message)
return S_ERROR('\n'.join([stdout, stderr]))
return S_OK()
def __addDoubleSlash(self, res):
""" Utilities to add the double slash between the host(:port) and the path
:param res: DIRAC return structure which contains an URL if S_OK
:return: DIRAC structure with corrected URL
"""
if not res['OK']:
return res
url = res['Value']
res = pfnparse(url, srmSpecific=self.srmSpecificParse)
if not res['OK']:
return res
urlDict = res['Value']
urlDict['Path'] = '/' + urlDict['Path']
return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def __addDoubleSlash( self, res ):
""" Utilities to add the double slash between the host(:port) and the path
:param res: DIRAC return structure which contains an URL if S_OK
:return: DIRAC structure with corrected URL
"""
if not res['OK']:
return res
url = res['Value']
res = pfnparse( url, srmSpecific = self.srmSpecificParse )
if not res['OK']:
return res
urlDict = res['Value']
urlDict['Path'] = '/' + urlDict['Path']
return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
def updateURL(self, url, withWSUrl=False):
"""Update the URL according to the current SE parameters"""
result = pfnparse(url, srmSpecific=self.srmSpecificParse)
if not result["OK"]:
return result
urlDict = result["Value"]
urlDict["Protocol"] = self.protocolParameters["Protocol"]
urlDict["Host"] = self.protocolParameters["Host"]
urlDict["Port"] = self.protocolParameters["Port"]
urlDict["WSUrl"] = ""
if withWSUrl:
urlDict["WSUrl"] = self.protocolParameters["WSUrl"]
return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def getJobStatus(self, jobIDList):
""" Get the status information for the given list of jobs
"""
resultDict = {}
ssh = SSH(parameters=self.ceParameters)
for jobList in breakListIntoChunks(jobIDList, 100):
jobDict = {}
for job in jobList:
result = pfnparse(job)
if result['OK']:
stamp = result['Value']['FileName'].split('.')[0]
else:
self.log.error('Invalid job id', job)
continue
jobDict[stamp] = job
stampList = jobDict.keys()
cmd = ['qstat', ' '.join(stampList)]
result = ssh.sshCall(10, cmd)
if not result['OK']:
return result
status = result['Value'][0]
if status == -1:
return S_ERROR('Timeout while SSH call')
elif status != 0:
return S_ERROR('Error while SSH call')
output = result['Value'][1].replace('\r', '')
lines = output.split('\n')
for job in jobDict:
resultDict[jobDict[job]] = 'Unknown'
for line in lines:
if line.find(job) != -1:
if line.find('Unknown') != -1:
resultDict[jobDict[job]] = 'Unknown'
else:
torqueStatus = line.split()[4]
if torqueStatus in ['E', 'C']:
resultDict[jobDict[job]] = 'Done'
elif torqueStatus in ['R']:
resultDict[jobDict[job]] = 'Running'
elif torqueStatus in ['S', 'W', 'Q', 'H', 'T']:
resultDict[jobDict[job]] = 'Waiting'
return S_OK(resultDict)
def getJobStatus( self, jobIDList ):
""" Get the status information for the given list of jobs
"""
resultDict = {}
ssh = SSH( parameters = self.ceParameters )
for jobList in breakListIntoChunks( jobIDList, 100 ):
jobDict = {}
for job in jobList:
result = pfnparse( job )
if result['OK']:
stamp = result['Value']['FileName'].split('.')[0]
else:
self.log.error( 'Invalid job id', job )
continue
jobDict[stamp] = job
stampList = jobDict.keys()
cmd = [ 'qstat', ' '.join( stampList ) ]
result = ssh.sshCall( 10, cmd )
if not result['OK']:
return result
status = result['Value'][0]
if status == -1:
return S_ERROR( 'Timeout while SSH call' )
elif status != 0:
return S_ERROR( 'Error while SSH call' )
output = result['Value'][1].replace( '\r', '' )
lines = output.split( '\n' )
for job in jobDict:
resultDict[jobDict[job]] = 'Unknown'
for line in lines:
if line.find( job ) != -1:
if line.find( 'Unknown' ) != -1:
resultDict[jobDict[job]] = 'Unknown'
else:
torqueStatus = line.split()[4]
if torqueStatus in ['E', 'C']:
resultDict[jobDict[job]] = 'Done'
elif torqueStatus in ['R']:
resultDict[jobDict[job]] = 'Running'
elif torqueStatus in ['S', 'W', 'Q', 'H', 'T']:
resultDict[jobDict[job]] = 'Waiting'
return S_OK( resultDict )
def updateURL( self, url, withWSUrl = False ):
""" Update the URL according to the current SE parameters
"""
result = pfnparse( url, srmSpecific = self.srmSpecificParse )
if not result['OK']:
return result
urlDict = result['Value']
urlDict['Protocol'] = self.protocolParameters['Protocol']
urlDict['Host'] = self.protocolParameters['Host']
urlDict['Port'] = self.protocolParameters['Port']
urlDict['WSUrl'] = ''
if withWSUrl:
urlDict['WSUrl'] = self.protocolParameters['WSUrl']
return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse )
def _isInputURL( self, url ):
""" Check if the given url can be taken as input
:param self: self reference
:param str url: URL
"""
res = pfnparse( url )
if not res['OK']:
return res
urlDict = res['Value']
# Special case of 'file' protocol which can be just a URL
if not urlDict['Protocol'] and 'file' in self.protocolParameters['InputProtocols']:
return S_OK( True )
return S_OK( urlDict['Protocol'] == self.protocolParameters['Protocol'] )
def updateURL(self, url, withWSUrl=False):
""" Update the URL according to the current SE parameters
"""
result = pfnparse(url, srmSpecific=self.srmSpecificParse)
if not result['OK']:
return result
urlDict = result['Value']
urlDict['Protocol'] = self.protocolParameters['Protocol']
urlDict['Host'] = self.protocolParameters['Host']
urlDict['Port'] = self.protocolParameters['Port']
urlDict['WSUrl'] = ''
if withWSUrl:
urlDict['WSUrl'] = self.protocolParameters['WSUrl']
return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse)
def _isInputURL(self, url):
""" Check if the given url can be taken as input
:param self: self reference
:param str url: URL
"""
res = pfnparse(url)
if not res['OK']:
return res
urlDict = res['Value']
# Special case of 'file' protocol which can be just a URL
if not urlDict['Protocol'] and 'file' in self.protocolParameters['InputProtocols']:
return S_OK(True)
return S_OK(urlDict['Protocol'] == self.protocolParameters['Protocol'])
def __getSinglePfnForProtocol(self, pfn, protocol, withPort=True):
""" Transform the input pfn into a pfn with the given protocol for the Storage Element.
:param pfn : input PFN
:param protocol : string or list of string of the protocol we want
:param withPort : includes the port in the returned pfn
"""
self.log.verbose(
"StorageElement.getSinglePfnForProtocol: Getting pfn for given protocols in %s."
% self.name)
# This test of the available protocols could actually be done in getPfnForProtocol once for all
# but it is safer to put it here in case we decide to call this method internally (which I doubt!)
res = self.getProtocols()
if not res['OK']:
return res
if type(protocol) == StringType:
protocols = [protocol]
elif type(protocol) == ListType:
protocols = protocol
else:
errStr = "StorageElement.getSinglePfnForProtocol: Supplied protocol must be string or list of strings."
self.log.debug(errStr, "%s %s" % (protocol, self.name))
return S_ERROR(errStr)
availableProtocols = res['Value']
protocolsToTry = []
for protocol in protocols:
if protocol in availableProtocols:
protocolsToTry.append(protocol)
else:
errStr = "StorageElement.getSinglePfnForProtocol: Requested protocol not available for SE."
self.log.debug(errStr, '%s for %s' % (protocol, self.name))
if not protocolsToTry:
errStr = "StorageElement.getSinglePfnForProtocol: None of the requested protocols were available for SE."
self.log.debug(errStr, '%s for %s' % (protocol, self.name))
return S_ERROR(errStr)
# Check all available storages for required protocol then contruct the PFN
for storage in self.storages:
res = storage.getParameters()
if res['Value']['ProtocolName'] in protocolsToTry:
res = pfnparse(pfn)
if res['OK']:
res = storage.getProtocolPfn(res['Value'], withPort)
if res['OK']:
return res
errStr = "StorageElement.getSinglePfnForProtocol: Failed to get PFN for requested protocols."
self.log.debug(errStr, "%s for %s" % (protocols, self.name))
return S_ERROR(errStr)
def __getSinglePfnForProtocol( self, pfn, protocol, withPort = True ):
""" Transform the input pfn into a pfn with the given protocol for the Storage Element.
:param pfn : input PFN
:param protocol : string or list of string of the protocol we want
:param withPort : includes the port in the returned pfn
"""
self.log.verbose( "StorageElement.getSinglePfnForProtocol: Getting pfn for given protocols in %s." % self.name )
# This test of the available protocols could actually be done in getPfnForProtocol once for all
# but it is safer to put it here in case we decide to call this method internally (which I doubt!)
res = self.getProtocols()
if not res['OK']:
return res
if type( protocol ) == StringType:
protocols = [protocol]
elif type( protocol ) == ListType:
protocols = protocol
else:
errStr = "StorageElement.getSinglePfnForProtocol: Supplied protocol must be string or list of strings."
self.log.debug( errStr, "%s %s" % ( protocol, self.name ) )
return S_ERROR( errStr )
availableProtocols = res['Value']
protocolsToTry = []
for protocol in protocols:
if protocol in availableProtocols:
protocolsToTry.append( protocol )
else:
errStr = "StorageElement.getSinglePfnForProtocol: Requested protocol not available for SE."
self.log.debug( errStr, '%s for %s' % ( protocol, self.name ) )
if not protocolsToTry:
errStr = "StorageElement.getSinglePfnForProtocol: None of the requested protocols were available for SE."
self.log.debug( errStr, '%s for %s' % ( protocol, self.name ) )
return S_ERROR( errStr )
# Check all available storages for required protocol then contruct the PFN
for storage in self.storages:
res = storage.getParameters()
if res['Value']['ProtocolName'] in protocolsToTry:
res = pfnparse( pfn )
if res['OK']:
res = storage.getProtocolPfn( res['Value'], withPort )
if res['OK']:
return res
errStr = "StorageElement.getSinglePfnForProtocol: Failed to get PFN for requested protocols."
self.log.debug( errStr, "%s for %s" % ( protocols, self.name ) )
return S_ERROR( errStr )
def _isInputURL(self, url):
"""Check if the given url can be taken as input
:param self: self reference
:param str url: URL
"""
res = pfnparse(url)
if not res["OK"]:
return res
urlDict = res["Value"]
# Special case of 'file' protocol which can be just a URL
if not urlDict["Protocol"] and "file" in self.protocolParameters[
"InputProtocols"]:
return S_OK(True)
return S_OK(urlDict["Protocol"] == self.protocolParameters["Protocol"])
def _killJobs( self, jobIDList, host = None ):
""" Kill the jobs for the given list of job IDs
"""
resultDict = {}
jobDict = {}
for job in jobIDList:
result = pfnparse( job )
if result['OK']:
stamp = result['Value']['FileName']
else:
self.log.error( 'Invalid job id', job )
continue
jobDict[stamp] = job
stampList = jobDict.keys()
cmdTuple = [ self.finalScript, 'kill_job', '#'.join( stampList ), self.infoArea ]
result = systemCall( 10, cmdTuple )
if not result['OK']:
return result
status = result['Value'][0]
stdout = result['Value'][1]
stderr = result['Value'][2]
# Examine results of the job submission
if status != 0:
outputLines = stdout.strip().replace( '\r', '' ).split( '\n' )
try:
index = outputLines.index( '============= Start output ===============' )
outputLines = outputLines[index + 1:]
except:
return S_ERROR( "Invalid output from kill Job: %s" % outputLines[0] )
try:
status = int( outputLines[0] )
except:
return S_ERROR( "Failed to kill Job: %s" % outputLines[0] )
if status != 0:
message = "Unknown reason"
if len( outputLines ) > 1:
message = outputLines[1]
return S_ERROR( 'Failed to kill Job, reason: %s' % message )
return S_ERROR( '\n'.join( [stdout, stderr] ) )
return S_OK()
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
# Make the service URL from the file URL by stripping off the file part
serviceDict = dict(srcDict)
serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
serviceDict['FileName'] = ''
res = pfnunparse(serviceDict)
if not res['OK']:
return res
srcSEURL = res['Value']
localCache = True
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'], os.path.join(
srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
# Make the service URL from the file URL by stripping off the file part
serviceDict = dict(srcDict)
serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
serviceDict['FileName'] = ''
res = pfnunparse(serviceDict)
if not res['OK']:
return res
srcSEURL = res['Value']
localCache = True
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'],
os.path.join(srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def _checkLFNPFNConvention(self, lfn, pfn, se):
""" Check that the PFN corresponds to the LFN-PFN convention
"""
# Check if the PFN corresponds to the LFN convention
if pfn == lfn:
return S_OK()
lfn_pfn = True # flag that the lfn is contained in the pfn
if (len(pfn) < len(lfn)) or (pfn[-len(lfn):] != lfn):
return S_ERROR('PFN does not correspond to the LFN convention')
if not pfn.endswith(lfn):
return S_ERROR()
# Check if the pfn corresponds to the SE definition
result = self._getStorageElement(se)
if not result['OK']:
return result
selement = result['Value']
res = pfnparse(pfn)
if not res['OK']:
return res
pfnDict = res['Value']
protocol = pfnDict['Protocol']
pfnpath = pfnDict['Path']
result = selement.getStorageParameters(protocol)
if not result['OK']:
return result
seDict = result['Value']
sePath = seDict['Path']
ind = pfnpath.find(sePath)
if ind == -1:
return S_ERROR(
'The given PFN %s does not correspond to the %s SE definition'
% (pfn, se))
# Check the full LFN-PFN-SE convention
lfn_pfn_se = True
if lfn_pfn:
seAccessDict = dict(seDict)
seAccessDict['Path'] = sePath + '/' + lfn
check_pfn = pfnunparse(seAccessDict)
if check_pfn != pfn:
return S_ERROR('PFN does not correspond to the LFN convention')
return S_OK()
def isURL( self, path ):
""" Guess if the path looks like a URL
:param self: self reference
:param string path: input file LFN or URL
:returns boolean: True if URL, False otherwise
"""
if self.basePath and path.startswith( self.basePath ):
return S_OK( True )
result = pfnparse( path, srmSpecific = self.srmSpecificParse )
if not result['OK']:
return result
if len( result['Value']['Protocol'] ) != 0:
return S_OK( True )
if result['Value']['Path'].startswith( self.basePath ):
return S_OK( True )
return S_OK( False )
def isURL(self, path):
""" Guess if the path looks like a URL
:param self: self reference
:param string path: input file LFN or URL
:returns boolean: True if URL, False otherwise
"""
if self.basePath and path.startswith(self.basePath):
return S_OK(True)
result = pfnparse(path)
if not result['OK']:
return result
if len(result['Value']['Protocol']) != 0:
return S_OK(True)
if result['Value']['Path'].startswith(self.basePath):
return S_OK(True)
return S_OK(False)
def isURL(self, path):
"""Guess if the path looks like a URL
:param self: self reference
:param string path: input file LFN or URL
:returns boolean: True if URL, False otherwise
"""
if self.basePath and path.startswith(self.basePath):
return S_OK(True)
result = pfnparse(path, srmSpecific=self.srmSpecificParse)
if not result["OK"]:
return result
if len(result["Value"]["Protocol"]) != 0:
return S_OK(True)
if result["Value"]["Path"].startswith(self.basePath):
return S_OK(True)
return S_OK(False)
def _checkLFNPFNConvention(self, lfn, pfn, se):
""" Check that the PFN corresponds to the LFN-PFN convention
"""
# Check if the PFN corresponds to the LFN convention
if pfn == lfn:
return S_OK()
lfn_pfn = True # flag that the lfn is contained in the pfn
if (len(pfn) < len(lfn)) or (pfn[-len(lfn) :] != lfn):
return S_ERROR("PFN does not correspond to the LFN convention")
if not pfn.endswith(lfn):
return S_ERROR()
# Check if the pfn corresponds to the SE definition
result = self._getStorageElement(se)
if not result["OK"]:
return result
selement = result["Value"]
res = pfnparse(pfn)
if not res["OK"]:
return res
pfnDict = res["Value"]
protocol = pfnDict["Protocol"]
pfnpath = pfnDict["Path"]
result = selement.getStorageParameters(protocol)
if not result["OK"]:
return result
seDict = result["Value"]
sePath = seDict["Path"]
ind = pfnpath.find(sePath)
if ind == -1:
return S_ERROR("The given PFN %s does not correspond to the %s SE definition" % (pfn, se))
# Check the full LFN-PFN-SE convention
lfn_pfn_se = True
if lfn_pfn:
seAccessDict = dict(seDict)
seAccessDict["Path"] = sePath + "/" + lfn
check_pfn = pfnunparse(seAccessDict)
if check_pfn != pfn:
return S_ERROR("PFN does not correspond to the LFN convention")
return S_OK()
def getPfnForProtocol( self, pfn, protocol, withPort = True ):
""" Transform the input pfn into another with the given protocol for the Storage Element.
"""
res = self.getProtocols()
if not res['OK']:
return res
if type( protocol ) == StringType:
protocols = [protocol]
elif type( protocol ) == ListType:
protocols = protocol
else:
errStr = "getPfnForProtocol: Supplied protocol must be string or list of strings."
self.log.error( errStr, "%s %s" % ( protocol, self.name ) )
return S_ERROR( errStr )
availableProtocols = res['Value']
protocolsToTry = []
for protocol in protocols:
if protocol in availableProtocols:
protocolsToTry.append( protocol )
else:
errStr = "getPfnForProtocol: Requested protocol not available for SE."
self.log.debug( errStr, '%s for %s' % ( protocol, self.name ) )
if not protocolsToTry:
errStr = "getPfnForProtocol: None of the requested protocols were available for SE."
self.log.error( errStr, '%s for %s' % ( protocol, self.name ) )
return S_ERROR( errStr )
# Check all available storages for required protocol then contruct the PFN
for storage in self.storages:
res = storage.getParameters()
if res['Value']['ProtocolName'] in protocolsToTry:
res = pfnparse( pfn )
if res['OK']:
res = storage.getProtocolPfn( res['Value'], withPort )
if res['OK']:
return res
errStr = "getPfnForProtocol: Failed to get PFN for requested protocols."
self.log.error( errStr, "%s for %s" % ( protocols, self.name ) )
return S_ERROR( errStr )
