def getDirectoryMetadata(self, path):
"""Get metadata associated to the directory"""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
gLogger.debug("DIPStorage.getFileMetadata: Attempting to obtain metadata for %s directories." % len(urls))
serviceClient = Client(url=self.url)
for url in urls:
res = serviceClient.getMetadata(url)
if res["OK"]:
if res["Value"]["Exists"]:
if res["Value"]["Type"] == "Directory":
res["Value"]["Directory"] = True
gLogger.debug("DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % url)
successful[url] = res["Value"]
else:
failed[url] = "Supplied path is not a directory"
else:
failed[url] = "Directory does not exist"
else:
gLogger.error(
"DIPStorage.getFileMetadata: Failed to get metadata for url", "%s: %s" % (url, res["Message"])
)
failed[url] = res["Message"]
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def putDirectory( self, path ):
""" Put a local directory to the physical storage together with all its files and subdirectories.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "RFIOStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) )
for destDir, sourceDir in urls.items():
res = self.__putDir( sourceDir, destDir )
if res['OK']:
if res['Value']['AllPut']:
gLogger.debug( "RFIOStorage.putDirectory: Successfully put directory to remote storage: %s" % destDir )
successful[destDir] = {'Files':res['Value']['Files'], 'Size':res['Value']['Size']}
else:
gLogger.error( "RFIOStorage.putDirectory: Failed to put entire directory to remote storage.", destDir )
failed[destDir] = {'Files':res['Value']['Files'], 'Size':res['Value']['Size']}
else:
gLogger.error( "RFIOStorage.putDirectory: Completely failed to put directory to remote storage.", destDir )
failed[destDir] = {'Files':0, 'Size':0}
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getTransportURL(self, pathDict, protocols):
"""Get a transport URL for a given URL. For a simple storage plugin
it is just returning input URL if the plugin protocol is one of the
requested protocols
:param dict pathDict: URL obtained from File Catalog or constructed according
to convention
:param protocols: a list of acceptable transport protocols in priority order
:type protocols: `python:list`
"""
res = checkArgumentFormat(pathDict)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
if protocols and not self.protocolParameters["Protocol"] in protocols:
return S_ERROR(errno.EPROTONOSUPPORT,
"No native protocol requested")
for url in urls:
successful[url] = url
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def getDirectoryMetadata( self, path ):
""" Get metadata associated to the directory(ies)
:param self: self reference
:param path: url (or list of urls) on storage
:returns Successful dict {path : metadata}
Failed dict {path : error message }
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for url in urls:
res = self.__stat( url )
if not res['OK']:
failed[url] = res['Message']
elif not res['Value']['Directory']:
failed[url] = os.strerror( errno.ENOTDIR )
else:
successful[url] = res['Value']
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def getFileSize(self, path):
"""Get the physical size of the given file
:param self: self reference
:param path: path (or list of path) on storage
:returns Successful dict {path : size}
Failed dict {path : error message }
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for url in urls:
try:
# We check the filesize first because if it does not exist
# it raises an exception, while os.path.isfile just return False
filesize = os.path.getsize(url)
if os.path.isfile(url):
successful[url] = filesize
else:
failed[url] = os.strerror(errno.EISDIR)
except OSError as ose:
failed[url] = str(ose)
return S_OK({'Failed': failed, 'Successful': successful})
def removeDirectory(self, path, recursive=False):
""" Remove a directory from the storage together with all its files and subdirectories.
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug("DIPStorage.removeDirectory: Attempting to remove %s directories." % len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.removeDirectory(url, '')
if res['OK']:
gLogger.debug(
"DIPStorage.removeDirectory: Successfully removed directory on storage: %s" %
url)
successful[url] = {'FilesRemoved': 0, 'SizeRemoved': 0}
else:
gLogger.error(
"DIPStorage.removeDirectory: Failed to remove directory from storage.", "%s: %s" %
(url, res['Message']))
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def removeFile(self, paths):
""" Remove file
Uses the name to react.
if /error_removeFile/ is in the lfn, returns S_ERROR
if /removeFile/ is in the lfn: adds it in failed
"""
lfns = checkArgumentFormat(paths)['Value']
if any(['/error_removeFile/' in lfn for lfn in lfns]):
for lfn in lfns:
self.removeCalls.append(lfn)
self.failedRemove.append(lfn)
return S_ERROR("ERROR to remove File")
failed = {}
successful = {}
for lfn in lfns:
self.removeCalls.append(lfn)
if '/fail_removeFile/' in lfn:
self.failedRemove.append(lfn)
failed[lfn] = 'It failed'
else:
self.successfulRemove.append(lfn)
successful[lfn] = 'It worked :-)'
return S_OK({'Successful': successful, 'Failed': failed})
def createDirectory(self, path):
""" Create the remote directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug("DIPStorage.createDirectory: Attempting to create %s directories." % len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.createDirectory(url)
if res['OK']:
gLogger.debug(
"DIPStorage.createDirectory: Successfully created directory on storage: %s" %
url)
successful[url] = True
else:
gLogger.error(
"DIPStorage.createDirectory: Failed to create directory on storage.", "%s: %s" %
(url, res['Message']))
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getDirectoryMetadata(self, path):
"""Get metadata associated to the directory(ies)
:param self: self reference
:param path: url (or list of urls) on storage
:returns: Successful dict {path : metadata}
Failed dict {path : error message }
"""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
failed = {}
successful = {}
for url in urls:
res = self.__stat(url)
if not res["OK"]:
failed[url] = res["Message"]
elif not res["Value"]["Directory"]:
failed[url] = os.strerror(errno.ENOTDIR)
else:
successful[url] = res["Value"]
return S_OK({"Failed": failed, "Successful": successful})
def getDirectory(self, path, localPath=False):
""" Get a local copy in the current directory of a physical file specified by its path
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
gLogger.debug(
"DIPStorage.getDirectory: Attempting to get local copies of %s directories."
% len(urls))
transferClient = TransferClient(self.url)
for src_dir in urls:
if localPath:
dest_dir = localPath
else:
dest_dir = os.getcwd()
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
res = transferClient.receiveBulk(dest_dir, src_dir)
if res['OK']:
gLogger.debug(
"DIPStorage.getDirectory: Successfully got local copy of %s"
% src_dir)
successful[src_dir] = {'Files': 0, 'Size': 0}
else:
gLogger.error(
"DIPStorage.getDirectory: Failed to get entire directory.",
src_dir)
failed[src_dir] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def removeFile(self, path):
"""Remove physically the file specified by its path
A non existing file will be considered as successfully removed.
:param path: path (or list of path) on storage
:returns Successful dict {path : True}
Failed dict {path : error message }
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
gLogger.debug(
"FileStorage.removeFile: Attempting to remove %s files." %
len(urls))
failed = {}
successful = {}
for url in urls:
try:
os.unlink(url)
successful[url] = True
except OSError as ose:
# Removing a non existing file is a success
if ose.errno == errno.ENOENT:
successful[url] = True
else:
failed[url] = str(ose)
except Exception as e:
failed[url] = str(e)
return S_OK({'Failed': failed, 'Successful': successful})
def removeDirectory(self, path, recursive=False):
""" Remove a directory from the storage together with all its files and subdirectories.
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.removeDirectory: Attempting to remove %s directories."
% len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.removeDirectory(url, '')
if res['OK']:
gLogger.debug(
"DIPStorage.removeDirectory: Successfully removed directory on storage: %s"
% url)
successful[url] = {'FilesRemoved': 0, 'SizeRemoved': 0}
else:
gLogger.error(
"DIPStorage.removeDirectory: Failed to remove directory from storage.",
"%s: %s" % (url, res['Message']))
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def createDirectory(self, path):
""" Create the remote directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.createDirectory: Attempting to create %s directories."
% len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.createDirectory(url)
if res['OK']:
gLogger.debug(
"DIPStorage.createDirectory: Successfully created directory on storage: %s"
% url)
successful[url] = True
else:
gLogger.error(
"DIPStorage.createDirectory: Failed to create directory on storage.",
"%s: %s" % (url, res['Message']))
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getDirectoryMetadata( self, path ):
""" Get metadata associated to the directory
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.getFileMetadata: Attempting to obtain metadata for %s directories." % len( urls ) )
serviceClient = RPCClient( self.url )
for url in urls:
res = serviceClient.getMetadata( url )
if res['OK']:
if res['Value']['Exists']:
if res['Value']['Type'] == 'Directory':
res['Value']['Directory'] = True
gLogger.debug( "DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % url )
successful[url] = res['Value']
else:
failed[url] = 'Supplied path is not a directory'
else:
failed[url] = 'Directory does not exist'
else:
gLogger.error( "DIPStorage.getFileMetadata: Failed to get metadata for url",
"%s: %s" % ( url, res['Message'] ) )
failed[url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putDirectory(self, path):
""" Put a local directory to the physical storage together with all its files and subdirectories.
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug("RFIOStorage.putDirectory: Attemping to put %s directories to remote storage." % len(urls))
for destDir, sourceDir in urls.items():
res = self.__putDir(sourceDir, destDir)
if res['OK']:
if res['Value']['AllPut']:
gLogger.debug("RFIOStorage.putDirectory: Successfully put directory to remote storage: %s" % destDir)
successful[destDir] = {'Files': res['Value']['Files'], 'Size': res['Value']['Size']}
else:
gLogger.error("RFIOStorage.putDirectory: Failed to put entire directory to remote storage.", destDir)
failed[destDir] = {'Files': res['Value']['Files'], 'Size': res['Value']['Size']}
else:
gLogger.error("RFIOStorage.putDirectory: Completely failed to put directory to remote storage.", destDir)
failed[destDir] = {'Files': 0, 'Size': 0}
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getDirectoryMetadata(self, path):
""" Get metadata associated to the directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.getFileMetadata: Attempting to obtain metadata for %s directories."
% len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.getMetadata(url)
if res['OK']:
if res['Value']['Exists']:
if res['Value']['Type'] == 'Directory':
res['Value']['Directory'] = True
gLogger.debug(
"DIPStorage.getFileMetadata: Successfully obtained metadata for %s."
% url)
successful[url] = res['Value']
else:
failed[url] = 'Supplied path is not a directory'
else:
failed[url] = 'Directory does not exist'
else:
gLogger.error(
"DIPStorage.getFileMetadata: Failed to get metadata for url",
"%s: %s" % (url, res['Message']))
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getDirectory(self, path, localPath=False):
"""Get locally a directory from the physical storage together with all its
files and subdirectories.
:param: path: path (or list of path) on storage
:param: localPath: local path where to store what is downloaded
:return: successful and failed dictionaries. The keys are the pathes,
the values are dictionary {'Files': amount of files downloaded, 'Size': amount of data downloaded}
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
self.log.debug(
"FileStorage.getDirectory: Attempting to get local copies of %s directories."
% len(urls))
failed = {}
successful = {}
if not localPath:
localPath = os.getcwd()
for src_dir in urls:
try:
dirName = os.path.basename(src_dir)
dest_dir = "%s/%s" % (localPath, dirName)
successful[src_dir] = self.__copyDirectory(src_dir, dest_dir)
except OSError:
failed[src_dir] = {'Files': 0, 'Size': 0}
return S_OK({'Failed': failed, 'Successful': successful})
def getDirectoryMetadata(self, path):
""" Get metadata associated to the directory(ies)
:param self: self reference
:param path: url (or list of urls) on storage
:returns Successful dict {path : metadata}
Failed dict {path : error message }
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for url in urls:
res = self.__stat(url)
if not res['OK']:
failed[url] = res['Message']
elif not res['Value']['Directory']:
failed[url] = os.strerror(errno.ENOTDIR)
else:
successful[url] = res['Value']
return S_OK({'Failed': failed, 'Successful': successful})
def createDirectory( self, path ):
""" Make a/several new directory on the physical storage
This method creates all the intermediate directory
:param self: self reference
:param str path: path (or list of path) on storage
:returns Successful dict {path : True}
Failed dict {path : error message }
"""
urls = checkArgumentFormat( path )
if not urls['OK']:
return urls
urls = urls['Value']
successful = {}
failed = {}
self.log.debug( "FileStorage.createDirectory: Attempting to create %s directories." % len( urls ) )
for url in urls:
try:
# Create all the path
os.makedirs( url )
successful[url] = True
except OSError as ose:
failed[url] = str( ose )
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def getFileSize( self, path ):
"""Get the physical size of the given file
:param self: self reference
:param path: path (or list of path) on storage
:returns Successful dict {path : size}
Failed dict {path : error message }
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for url in urls:
try:
# We check the filesize first because if it does not exist
# it raises an exception, while os.path.isfile just return False
filesize = os.path.getsize( url )
if os.path.isfile( url ):
successful[url] = filesize
else:
failed[url] = os.strerror( errno.EISDIR )
except OSError as ose:
failed[url] = str( ose )
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def getDirectory( self, path, localPath = False ):
"""Get locally a directory from the physical storage together with all its
files and subdirectories.
:param: path: path (or list of path) on storage
:param: localPath: local path where to store what is downloaded
:return: successful and failed dictionaries. The keys are the pathes,
the values are dictionary {'Files': amount of files downloaded, 'Size': amount of data downloaded}
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.getDirectory: Attempting to get local copies of %s directories." % len( urls ) )
failed = {}
successful = {}
if not localPath:
localPath = os.getcwd()
for src_dir in urls:
try:
dirName = os.path.basename( src_dir )
dest_dir = "%s/%s" % ( localPath, dirName )
successful[src_dir] = self.__copyDirectory( src_dir, dest_dir )
except OSError:
failed[src_dir] = {'Files':0, 'Size':0}
return S_OK( {'Failed' : failed, 'Successful' : successful } )
def removeFile( self, path ):
"""Remove physically the file specified by its path
A non existing file will be considered as successfully removed.
:param path: path (or list of path) on storage
:returns Successful dict {path : True}
Failed dict {path : error message }
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
gLogger.debug( "FileStorage.removeFile: Attempting to remove %s files." % len( urls ) )
failed = {}
successful = {}
for url in urls:
try:
os.unlink(url)
successful[url] = True
except OSError as ose:
# Removing a non existing file is a success
if ose.errno == errno.ENOENT:
successful[url] = True
else:
failed[url] = str( ose )
except Exception as e:
failed[url] = str( e )
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def getFile( self, path, localPath = False ):
""" make a local copy of a storage :path:
:param self: self reference
:param str path: path on storage
:param localPath: if not specified, self.cwd
:returns Successful dict {path : size}
Failed dict {path : error message }
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.getFile: Trying to download %s files." % len( urls ) )
failed = {}
successful = {}
if not localPath:
localPath = os.getcwd()
for src_url in urls:
try:
fileName = os.path.basename( src_url )
dest_url = os.path.join( localPath, fileName )
shutil.copy2( src_url, dest_url )
fileSize = os.path.getsize( dest_url )
successful[src_url] = fileSize
except ( OSError, IOError ) as ose:
failed[src_url] = str( ose )
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def getTransportURL(self, pathDict, protocols):
""" Get a transport URL for a given URL. For a simple storage plugin
it is just returning input URL if the plugin protocol is one of the
requested protocols
:param dict pathDict: URL obtained from File Catalog or constructed according
to convention
:param protocols: a list of acceptable transport protocols in priority order
:type protocols: `python:list`
"""
res = checkArgumentFormat(pathDict)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
if protocols and not self.protocolParameters['Protocol'] in protocols:
return S_ERROR('No native protocol requested')
for url in urls:
successful[url] = url
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def isFile( self, path ):
"""Check if the given path exists and it is a file
:param self: self reference
:param path: path (or list of path) on storage
:returns: Successful dict {path : boolean}
Failed dict {path : error message }
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.isFile: Determining whether %s paths are files." % len( urls ) )
successful = {}
failed = {}
for url in urls:
if os.path.exists( url ):
successful[url] = os.path.isfile( url )
else:
failed[url] = "No such file or directory"
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def createDirectory(self, path):
""" Make a/several new directory on the physical storage
This method creates all the intermediate directory
:param self: self reference
:param str path: path (or list of path) on storage
:returns Successful dict {path : True}
Failed dict {path : error message }
"""
urls = checkArgumentFormat(path)
if not urls['OK']:
return urls
urls = urls['Value']
successful = {}
failed = {}
self.log.debug(
"FileStorage.createDirectory: Attempting to create %s directories."
% len(urls))
for url in urls:
try:
# Create all the path
os.makedirs(url)
successful[url] = True
except OSError as ose:
failed[url] = str(ose)
return S_OK({'Failed': failed, 'Successful': successful})
def listDirectory(self, path):
"""List the contents of the directory"""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
gLogger.debug("DIPStorage.listDirectory: Attempting to list %s directories." % len(urls))
serviceClient = Client(url=self.url)
for url in urls:
res = serviceClient.listDirectory(url, "l")
if not res["OK"]:
failed[url] = res["Message"]
else:
files = {}
subDirs = {}
for subPath, pathDict in res["Value"].items():
if pathDict["Type"] == "File":
files[subPath] = pathDict
elif pathDict["Type"] == "Directory":
subDirs[subPath] = pathDict
successful[url] = {}
successful[url]["SubDirs"] = subDirs
successful[url]["Files"] = files
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def isFile(self, path):
"""Check if the given path exists and it is a file
:param self: self reference
:param path: path (or list of path) on storage
:returns: Successful dict {path : boolean}
Failed dict {path : error message }
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
self.log.debug(
"FileStorage.isFile: Determining whether %s paths are files." %
len(urls))
successful = {}
failed = {}
for url in urls:
if os.path.exists(url):
successful[url] = os.path.isfile(url)
else:
failed[url] = "No such file or directory"
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def listDirectory( self, path ):
""" List the contents of the directory
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.listDirectory: Attempting to list %s directories." % len( urls ) )
serviceClient = RPCClient( self.url )
for url in urls:
res = serviceClient.listDirectory( url, 'l' )
if not res['OK']:
failed[url] = res['Message']
else:
files = {}
subDirs = {}
for subPath, pathDict in res['Value'].items():
if pathDict['Type'] == 'File':
files[subPath] = pathDict
elif pathDict['Type'] == 'Directory':
subDirs[subPath] = pathDict
successful[url] = {}
successful[url]['SubDirs'] = subDirs
successful[url]['Files'] = files
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def isDirectory( self, path ):
""" Determine whether the path is a directory
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len( urls ) )
serviceClient = RPCClient( self.url )
for url in urls:
res = serviceClient.getMetadata( url )
if res['OK']:
if res['Value']['Exists']:
if res['Value']['Type'] == 'Directory':
gLogger.debug( "DIPStorage.isDirectory: Successfully obtained metadata for %s." % url )
successful[url] = True
else:
successful[url] = False
else:
failed[url] = 'Path does not exist'
else:
gLogger.error( "DIPStorage.isDirectory: Failed to get metadata for url",
"%s: %s" % ( url, res['Message'] ) )
failed[url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def listDirectory(self, path):
""" List the contents of the directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.listDirectory: Attempting to list %s directories." %
len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.listDirectory(url, 'l')
if not res['OK']:
failed[url] = res['Message']
else:
files = {}
subDirs = {}
for subPath, pathDict in res['Value'].items():
if pathDict['Type'] == 'File':
files[subPath] = pathDict
elif pathDict['Type'] == 'Directory':
subDirs[subPath] = pathDict
successful[url] = {}
successful[url]['SubDirs'] = subDirs
successful[url]['Files'] = files
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getFile(self, path, localPath=False):
"""Get a local copy in the current directory of a physical file specified by its path
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for src_url in urls:
fileName = os.path.basename(src_url)
if localPath:
dest_file = "%s/%s" % (localPath, fileName)
else:
dest_file = "%s/%s" % (os.getcwd(), fileName)
gLogger.debug(
"DIPStorage.getFile: Executing transfer of %s to %s" %
(src_url, dest_file))
res = self.__getFile(src_url, dest_file)
if res['OK']:
successful[src_url] = res['Value']
else:
failed[src_url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getDirectory( self, path, localPath = False ):
""" Get a local copy in the current directory of a physical file specified by its path
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
gLogger.debug( "DIPStorage.getDirectory: Attempting to get local copies of %s directories." % len( urls ) )
transferClient = TransferClient( self.url )
for src_dir in urls:
if localPath:
dest_dir = localPath
else:
dest_dir = os.getcwd()
if not os.path.exists( dest_dir ):
os.mkdir( dest_dir )
res = transferClient.receiveBulk( dest_dir, src_dir )
if res['OK']:
gLogger.debug( "DIPStorage.getDirectory: Successfully got local copy of %s" % src_dir )
successful[src_dir] = {'Files':0, 'Size':0}
else:
gLogger.error( "DIPStorage.getDirectory: Failed to get entire directory.", src_dir )
failed[src_dir] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def isDirectory(self, path):
""" Determine whether the path is a directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.isDirectory: Attempting to determine whether %s paths are directories."
% len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.getMetadata(url)
if res['OK']:
if res['Value']['Exists']:
if res['Value']['Type'] == 'Directory':
gLogger.debug(
"DIPStorage.isDirectory: Successfully obtained metadata for %s."
% url)
successful[url] = True
else:
successful[url] = False
else:
failed[url] = 'Path does not exist'
else:
gLogger.error(
"DIPStorage.isDirectory: Failed to get metadata for url",
"%s: %s" % (url, res['Message']))
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def isDirectory(self, path):
"""Determine whether the path is a directory"""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
gLogger.debug("DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len(urls))
serviceClient = Client(url=self.url)
for url in urls:
res = serviceClient.getMetadata(url)
if res["OK"]:
if res["Value"]["Exists"]:
if res["Value"]["Type"] == "Directory":
gLogger.debug("DIPStorage.isDirectory: Successfully obtained metadata for %s." % url)
successful[url] = True
else:
successful[url] = False
else:
failed[url] = "Path does not exist"
else:
gLogger.error(
"DIPStorage.isDirectory: Failed to get metadata for url", "%s: %s" % (url, res["Message"])
)
failed[url] = res["Message"]
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def getDirectorySize(self, path):
""" Get the size of the contents of the directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.isDirectory: Attempting to determine whether %s paths are directories."
% len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.getDirectorySize(url)
if not res['OK']:
failed[url] = res['Message']
else:
successful[url] = {
'Files': 0,
'Size': res['Value'],
'SubDirs': 0
}
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getTransportURL( self, path, protocols = False ):
""" obtain the tURLs for the supplied path and protocols
:param self: self reference
:param str path: path on storage
:param mixed protocols: protocols to use
:returns Failed dict {path : error message}
Successful dict {path : transport url}
S_ERROR in case of argument problems
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( 'GFAL2_SRM2Storage.getTransportURL: Attempting to retrieve tURL for %s paths' % len( urls ) )
failed = {}
successful = {}
if not protocols:
protocols = self.__getProtocols()
if not protocols['OK']:
return protocols
listProtocols = protocols['Value']
elif isinstance( protocols, basestring ):
listProtocols = [protocols]
elif isinstance( protocols, list ):
listProtocols = protocols
else:
return S_ERROR( "getTransportURL: Must supply desired protocols to this plug-in." )
# I doubt this can happen... 'srm' is not in the listProtocols,
# it is normally, gsiftp, root, etc
if self.protocolParameters['Protocol'] in listProtocols:
successful = {}
failed = {}
for url in urls:
if self.isURL( url )['Value']:
successful[url] = url
else:
failed[url] = 'getTransportURL: Failed to obtain turls.'
return S_OK( {'Successful' : successful, 'Failed' : failed} )
for url in urls:
res = self.__getSingleTransportURL( url, listProtocols )
self.log.debug( 'res = %s' % res )
if not res['OK']:
failed[url] = res['Message']
else:
successful[url] = res['Value']
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def getDirectorySize(self, path):
""" Get the size of the directory on the storage
.. warning:: the size is not recursive, and does not go into subfolders
:param self: self reference
:param path: path (or list of path) on storage
:returns: list of successfull and failed dictionnary, both indexed by the path
* In the failed, the value is the error message
* In the successful the values are dictionnaries :
* Files : amount of files in the directory
* Size : summed up size of files
* subDirs : amount of sub directories
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
self.log.debug(
"FileStorage.getDirectorySize: Attempting to get size of %s directories."
% len(urls))
failed = {}
successful = {}
for url in urls:
try:
totSize = 0
nbOfFiles = 0
nbOfSubDirs = 0
for filename in os.listdir(url):
fullPath = os.path.join(url, filename)
if os.path.isfile(fullPath):
nbOfFiles += 1
totSize += os.path.getsize(fullPath)
else:
nbOfSubDirs += 1
successful[url] = {
'Files': nbOfFiles,
'Size': totSize,
'SubDirs': nbOfSubDirs
}
except OSError as ose:
failed[url] = str(ose)
return S_OK({'Failed': failed, 'Successful': successful})
def getTransportURL(self, pathDict, protocols):
res = checkArgumentFormat(pathDict)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
for url in urls:
successful[url] = url
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def getTransportURL(self, pathDict, protocols):
res = checkArgumentFormat(pathDict)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for url in urls:
successful[url] = url
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def putFile(self, path, sourceSize=0):
client = Client(url=self.url)
if sourceSize:
gLogger.debug(
"ProxyStorage.putFile: The client has provided the source file size implying\
a replication is requested.")
return client.callProxyMethod(self.name, "putFile", [path],
{"sourceSize": sourceSize})
gLogger.debug(
"ProxyStorage.putFile: No source size was provided therefore a simple put will be performed."
)
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
failed = {}
successful = {}
# make sure transferClient uses the same ProxyStorage instance we uploaded the file to
transferClient = TransferClient(client.serviceURL)
for dest_url, src_file in urls.items():
fileName = os.path.basename(dest_url)
res = transferClient.sendFile(src_file, "putFile/%s" % fileName)
if not res["OK"]:
gLogger.error(
"ProxyStorage.putFile: Failed to send file to proxy server.",
res["Message"])
failed[dest_url] = res["Message"]
else:
res = client.uploadFile(self.name, dest_url)
if not res["OK"]:
gLogger.error(
"ProxyStorage.putFile: Failed to upload file to storage element from proxy server.",
res["Message"],
)
failed[dest_url] = res["Message"]
else:
res = self.__executeOperation(dest_url, "getFileSize")
if not res["OK"]:
gLogger.error(
"ProxyStorage.putFile: Failed to determine destination file size.",
res["Message"])
failed[dest_url] = res["Message"]
else:
successful[dest_url] = res["Value"]
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def listDirectory(self, path):
"""List the supplied path
.. warning:: It is not recursive!
:param path: single or list of url
:return: successful and failed dictionaries. The keys are the pathes,
the values are dictionary 'SubDirs' and 'Files'. Each are dictionaries with
path as key and metadata as values (for Files only, SubDirs has just True as value)
"""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
self.log.debug(
"FileStorage.listDirectory: Attempting to list %s directories." %
len(urls))
successful = {}
failed = {}
for url in urls:
try:
dirs = {}
files = {}
# os.listdir returns files and directories together
for child in os.listdir(url):
fullpath = os.path.join(url, child)
lfnPath = os.path.join(
"/", os.path.relpath(fullpath, self.basePath))
res = self.__stat(fullpath)
if not res["OK"]:
failed[lfnPath] = res["Message"]
continue
if os.path.isfile(fullpath):
files[lfnPath] = res["Value"]
else:
dirs[lfnPath] = res["Value"]
successful[url] = {"SubDirs": dirs, "Files": files}
except OSError as ose:
failed[url] = str(ose)
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def getDirectorySize( self, path ):
""" Get the size of the directory on the storage
.. warning:: the size is not recursive, and does not go into subfolders
:param self: self reference
:param path: path (or list of path) on storage
:returns: list of successfull and failed dictionnary, both indexed by the path
* In the failed, the value is the error message
* In the successful the values are dictionnaries :
* Files : amount of files in the directory
* Size : summed up size of files
* subDirs : amount of sub directories
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.getDirectorySize: Attempting to get size of %s directories." % len( urls ) )
failed = {}
successful = {}
for url in urls:
try:
totSize = 0
nbOfFiles = 0
nbOfSubDirs = 0
for filename in os.listdir( url ):
fullPath = os.path.join( url, filename )
if os.path.isfile( fullPath ):
nbOfFiles += 1
totSize += os.path.getsize( fullPath )
else:
nbOfSubDirs += 1
successful[url] = { 'Files' : nbOfFiles, 'Size' : totSize, 'SubDirs' : nbOfSubDirs }
except OSError as ose:
failed[url] = str( ose )
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def putFile(self, path, sourceSize=0):
"""Put a file to the physical storage"""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
for dest_url, src_file in urls.items():
gLogger.debug("DIPStorage.putFile: Executing transfer of %s to %s" % (src_file, dest_url))
res = self.__putFile(src_file, dest_url)
if res["OK"]:
successful[dest_url] = res["Value"]
else:
failed[dest_url] = res["Message"]
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def getFile(self, path, localPath=False):
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
client = RPCClient(self.url)
# Make sure transferClient uses the same ProxyStorage instance.
# Only the this one holds the file we want to transfer.
transferClient = TransferClient(client.serviceURL)
for src_url in urls.keys():
res = client.prepareFile(self.name, src_url)
if not res['OK']:
gLogger.error(
"ProxyStorage.getFile: Failed to prepare file on remote server.",
res['Message'])
failed[src_url] = res['Message']
else:
fileName = os.path.basename(src_url)
if localPath:
dest_file = "%s/%s" % (localPath, fileName)
else:
dest_file = "%s/%s" % (os.getcwd(), fileName)
res = transferClient.receiveFile(dest_file,
'getFile/%s' % fileName)
if not res['OK']:
gLogger.error(
"ProxyStorage.getFile: Failed to recieve file from proxy server.",
res['Message'])
failed[src_url] = res['Message']
elif not os.path.exists(dest_file):
errStr = "ProxyStorage.getFile: The destination local file does not exist."
gLogger.error(errStr, dest_file)
failed[src_url] = errStr
else:
destSize = getSize(dest_file)
if destSize == -1:
errStr = "ProxyStorage.getFile: Failed to get the local file size."
gLogger.error(errStr, dest_file)
failed[src_url] = errStr
else:
successful[src_url] = destSize
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def putFile( self, path, sourceSize = 0 ):
"""Put a file to the physical storage
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for dest_url, src_file in urls.items():
gLogger.debug( "DIPStorage.putFile: Executing transfer of %s to %s" % ( src_file, dest_url ) )
res = self.__putFile( src_file, dest_url )
if res['OK']:
successful[dest_url] = res['Value']
else:
failed[dest_url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def exists(self, path):
"""Check if the given path exists. The 'path' variable can be a string or a list of strings."""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
serviceClient = Client(url=self.url)
for url in urls:
gLogger.debug("DIPStorage.exists: Determining existence of %s." % url)
res = serviceClient.exists(url)
if res["OK"]:
successful[url] = res["Value"]
else:
failed[url] = res["Message"]
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def exists( self, path ):
"""Check if the given path exists.
:param self: self reference
:param path: path (or list of path) on storage
:returns Failed dictionary: {pfn : errorMsg}
Successful dictionary: {pfn : bool}
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.exists: Checking the existence of %s path(s)" % len( urls ) )
successful = dict( ( url, os.path.exists( url ) ) for url in urls )
resDict = {'Failed':{}, 'Successful':successful}
return S_OK( resDict )
def getDirectorySize( self, path ):
""" Get the size of the contents of the directory
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len( urls ) )
serviceClient = RPCClient( self.url )
for url in urls:
res = serviceClient.getDirectorySize( url )
if not res['OK']:
failed[url] = res['Message']
else:
successful[url] = {'Files':0, 'Size':res['Value'], 'SubDirs':0}
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putFile( self, path, sourceSize = 0 ):
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for dest_url, src_file in urls.items():
res = self.__executeOperation( os.path.dirname( dest_url ), 'createDirectory' )
if not res['OK']:
failed[dest_url] = res['Message']
else:
res = self.__putFile( src_file, dest_url, sourceSize )
if res['OK']:
successful[dest_url] = res['Value']
else:
failed[dest_url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def exists( self, path ):
""" Check if the given path exists. The 'path' variable can be a string or a list of strings.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
serviceClient = RPCClient( self.url )
for url in urls:
gLogger.debug( "DIPStorage.exists: Determining existence of %s." % url )
res = serviceClient.exists( url )
if res['OK']:
successful[url] = res['Value']
else:
failed[url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def listDirectory( self, path ):
""" List the supplied path
.. warning:: It is not recursive!
:param path: single or list of url
:return: successful and failed dictionaries. The keys are the pathes,
the values are dictionary 'SubDirs' and 'Files'. Each are dictionaries with
path as key and metadata as values (for Files only, SubDirs has just True as value)
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.listDirectory: Attempting to list %s directories." % len( urls ) )
successful = {}
failed = {}
for url in urls:
try:
dirs = []
files = []
# os.listdir returns files and directories together
for child in os.listdir(url):
fullpath= os.path.join(url, child)
lfnPath = os.path.join('/', os.path.relpath(fullpath, self.basePath))
if os.path.isfile(fullpath):
files.append(lfnPath)
else:
dirs.append(lfnPath)
successful[url] = { 'SubDirs' : dirs, 'Files' : files}
except OSError as ose:
failed[url] = str( ose )
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getLFNFromURL( self, urls ):
""" Get the LFN from the PFNS .
:param lfn : input lfn or lfns (list/dict)
"""
result = checkArgumentFormat( urls )
if result['OK']:
urlDict = result['Value']
else:
errStr = "Supplied urls must be string, list of strings or a dictionary."
self.log.getSubLogger( 'getLFNFromURL' ).debug( errStr )
return S_ERROR( errno.EINVAL, errStr )
retDict = { "Successful" : {}, "Failed" : {} }
for url in urlDict:
res = self.__getURLPath( url )
if res["OK"]:
retDict["Successful"][url] = res["Value"]
else:
retDict["Failed"][url] = res["Message"]
return S_OK( retDict )
def removeFile( self, path ):
"""Remove physically the file specified by its path
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
if not len( urls ) > 0:
return S_ERROR( "DIPStorage.removeFile: No surls supplied." )
successful = {}
failed = {}
serviceClient = RPCClient( self.url )
for url in urls:
gLogger.debug( "DIPStorage.removeFile: Attempting to remove %s." % url )
res = serviceClient.remove( url, '' )
if res['OK']:
successful[url] = True
else:
failed[url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putDirectory( self, path ):
""" Put a local directory to the physical storage together with all its files and subdirectories.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) )
transferClient = TransferClient( self.url )
for destDir, sourceDir in urls.items():
tmpList = os.listdir( sourceDir )
sourceFiles = [ "%s/%s" % ( sourceDir, x ) for x in tmpList ]
res = transferClient.sendBulk( sourceFiles, destDir )
if res['OK']:
successful[destDir] = {'Files':0, 'Size':0}
else:
failed[destDir] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getFile( self, path, localPath = False ):
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
client = RPCClient( self.url )
# Make sure transferClient uses the same ProxyStorage instance.
# Only the this one holds the file we want to transfer.
transferClient = TransferClient( client.serviceURL )
for src_url in urls.keys():
res = client.prepareFile( self.name, src_url )
if not res['OK']:
gLogger.error( "ProxyStorage.getFile: Failed to prepare file on remote server.", res['Message'] )
failed[src_url] = res['Message']
else:
fileName = os.path.basename( src_url )
if localPath:
dest_file = "%s/%s" % ( localPath, fileName )
else:
dest_file = "%s/%s" % ( os.getcwd(), fileName )
res = transferClient.receiveFile( dest_file, 'getFile/%s' % fileName )
if not res['OK']:
gLogger.error( "ProxyStorage.getFile: Failed to recieve file from proxy server.", res['Message'] )
failed[src_url] = res['Message']
elif not os.path.exists( dest_file ):
errStr = "ProxyStorage.getFile: The destination local file does not exist."
gLogger.error( errStr, dest_file )
failed[src_url] = errStr
else:
destSize = getSize( dest_file )
if destSize == -1:
errStr = "ProxyStorage.getFile: Failed to get the local file size."
gLogger.error( errStr, dest_file )
failed[src_url] = errStr
else:
successful[src_url] = destSize
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putDirectory( self, path ):
""" puts a or several local directory to the physical storage together with all its files and subdirectories
:param self: self reference
:param str path: dictionnary {url : local dir}
:return: successful and failed dictionaries. The keys are the pathes,
the values are dictionary {'Files': amount of files uploaded, 'Size': amount of data uploaded}
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for destDir, sourceDir in urls.items():
try:
successful[destDir] = self.__copyDirectory( sourceDir, destDir )
except OSError:
failed[destDir] = {'Files':0, 'Size':0}
return S_OK( { "Failed" : failed, "Successful" : successful } )
def getFileSize( self, path ):
""" Get size of supplied files
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.getFileSize: Attempting to obtain size for %s files." % len( urls ) )
res = self.getFileMetadata( urls )
if not res['OK']:
return res
for url, urlDict in res['Value']['Successful'].items():
if urlDict['Exists']:
successful[url] = urlDict['Size']
else:
failed[url] = 'File does not exist'
for url, error in res['Value']['Failed'].items():
failed[url] = error
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getFile( self, path, localPath = False ):
"""Get a local copy in the current directory of a physical file specified by its path
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for src_url in urls:
fileName = os.path.basename( src_url )
if localPath:
dest_file = "%s/%s" % ( localPath, fileName )
else:
dest_file = "%s/%s" % ( os.getcwd(), fileName )
gLogger.debug( "DIPStorage.getFile: Executing transfer of %s to %s" % ( src_url, dest_file ) )
res = self.__getFile( src_url, dest_file )
if res['OK']:
successful[src_url] = res['Value']
else:
failed[src_url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
