def isDirectory( self, path ):
""" Determine whether the path is a directory
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len( urls ) )
serviceClient = RPCClient( self.url )
for url in urls:
res = serviceClient.getMetadata( url )
if res['OK']:
if res['Value']['Exists']:
if res['Value']['Type'] == 'Directory':
gLogger.debug( "DIPStorage.isDirectory: Successfully obtained metadata for %s." % url )
successful[url] = True
else:
successful[url] = False
else:
failed[url] = 'Path does not exist'
else:
gLogger.error( "DIPStorage.isDirectory: Failed to get metadata for url",
"%s: %s" % ( url, res['Message'] ) )
failed[url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def setReplicaHost(self, lfn):
res = checkArgumentFormat(lfn)
if not res['OK']:
return res
lfndict = res['Value']
rpcClient = self._getRPC()
return rpcClient.setReplicaHost(lfndict)
def putDirectory( self, path ):
""" Put a local directory to the physical storage together with all its files and subdirectories.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "RFIOStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) )
for destDir, sourceDir in urls.items():
res = self.__putDir( sourceDir, destDir )
if res['OK']:
if res['Value']['AllPut']:
gLogger.debug( "RFIOStorage.putDirectory: Successfully put directory to remote storage: %s" % destDir )
successful[destDir] = {'Files':res['Value']['Files'], 'Size':res['Value']['Size']}
else:
gLogger.error( "RFIOStorage.putDirectory: Failed to put entire directory to remote storage.", destDir )
failed[destDir] = {'Files':res['Value']['Files'], 'Size':res['Value']['Size']}
else:
gLogger.error( "RFIOStorage.putDirectory: Completely failed to put directory to remote storage.", destDir )
failed[destDir] = {'Files':0, 'Size':0}
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def removeDirectory(self, path, recursive=False):
""" Remove a directory from the storage together with all its files and subdirectories.
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.removeDirectory: Attempting to remove %s directories."
% len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.removeDirectory(url, '')
if res['OK']:
gLogger.debug(
"DIPStorage.removeDirectory: Successfully removed directory on storage: %s"
% url)
successful[url] = {'FilesRemoved': 0, 'SizeRemoved': 0}
else:
gLogger.error(
"DIPStorage.removeDirectory: Failed to remove directory from storage.",
"%s: %s" % (url, res['Message']))
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def listDirectory(self, path):
""" List the contents of the directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.listDirectory: Attempting to list %s directories." %
len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.listDirectory(url, 'l')
if not res['OK']:
failed[url] = res['Message']
else:
files = {}
subDirs = {}
for subPath, pathDict in res['Value'].items():
if pathDict['Type'] == 'File':
files[subPath] = pathDict
elif pathDict['Type'] == 'Directory':
subDirs[subPath] = pathDict
successful[url] = {}
successful[url]['SubDirs'] = subDirs
successful[url]['Files'] = files
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getDirectory( self, path, localPath = False ):
""" Get a local copy in the current directory of a physical file specified by its path
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
gLogger.debug( "DIPStorage.getDirectory: Attempting to get local copies of %s directories." % len( urls ) )
transferClient = TransferClient( self.url )
for src_dir in urls:
if localPath:
dest_dir = localPath
else:
dest_dir = os.getcwd()
if not os.path.exists( dest_dir ):
os.mkdir( dest_dir )
res = transferClient.receiveBulk( dest_dir, src_dir )
if res['OK']:
gLogger.debug( "DIPStorage.getDirectory: Successfully got local copy of %s" % src_dir )
successful[src_dir] = {'Files':0, 'Size':0}
else:
gLogger.error( "DIPStorage.getDirectory: Failed to get entire directory.", src_dir )
failed[src_dir] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getReplicas(self, lfn):
res = checkArgumentFormat(lfn)
if not res['OK']:
return res
lfns = res['Value'].keys()
rpcClient = self._getRPC()
return rpcClient.getReplicas(lfns)
def createDirectory(self, path):
""" Create the remote directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.createDirectory: Attempting to create %s directories."
% len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.createDirectory(url)
if res['OK']:
gLogger.debug(
"DIPStorage.createDirectory: Successfully created directory on storage: %s"
% url)
successful[url] = True
else:
gLogger.error(
"DIPStorage.createDirectory: Failed to create directory on storage.",
"%s: %s" % (url, res['Message']))
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def createDirectory(self, path):
""" Make a/several new directory on the physical storage
This method creates all the intermediate directory
:param self: self reference
:param str path: path (or list of path) on storage
:returns Successful dict {path : True}
Failed dict {path : error message }
"""
urls = checkArgumentFormat(path)
if not urls['OK']:
return urls
urls = urls['Value']
successful = {}
failed = {}
self.log.debug(
"FileStorage.createDirectory: Attempting to create %s directories."
% len(urls))
for url in urls:
try:
# Create all the path
os.makedirs(url)
successful[url] = True
except OSError as ose:
failed[url] = str(ose)
return S_OK({'Failed': failed, 'Successful': successful})
def addReplica(self, lfn, force=False):
res = checkArgumentFormat(lfn)
if not res['OK']:
return res
lfndicts = res['Value']
rpcClient = self._getRPC()
return rpcClient.addReplica(lfndicts, force)
def getFileSize(self, path):
"""Get the physical size of the given file
:param self: self reference
:param path: path (or list of path) on storage
:returns Successful dict {path : size}
Failed dict {path : error message }
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for url in urls:
try:
# We check the filesize first because if it does not exist
# it raises an exception, while os.path.isfile just return False
filesize = os.path.getsize(url)
if os.path.isfile(url):
successful[url] = filesize
else:
failed[url] = os.strerror(errno.EISDIR)
except OSError as ose:
failed[url] = str(ose)
return S_OK({'Failed': failed, 'Successful': successful})
def isDirectory(self, path):
""" Determine whether the path is a directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.isDirectory: Attempting to determine whether %s paths are directories."
% len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.getMetadata(url)
if res['OK']:
if res['Value']['Exists']:
if res['Value']['Type'] == 'Directory':
gLogger.debug(
"DIPStorage.isDirectory: Successfully obtained metadata for %s."
% url)
successful[url] = True
else:
successful[url] = False
else:
failed[url] = 'Path does not exist'
else:
gLogger.error(
"DIPStorage.isDirectory: Failed to get metadata for url",
"%s: %s" % (url, res['Message']))
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def isFile(self, path):
"""Check if the given path exists and it is a file
:param self: self reference
:param path: path (or list of path) on storage
:returns: Successful dict {path : boolean}
Failed dict {path : error message }
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
self.log.debug(
"FileStorage.isFile: Determining whether %s paths are files." %
len(urls))
successful = {}
failed = {}
for url in urls:
if os.path.exists(url):
successful[url] = os.path.isfile(url)
else:
failed[url] = "No such file or directory"
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def createDirectory( self, path ):
""" Make a/several new directory on the physical storage
This method creates all the intermediate directory
:param self: self reference
:param str path: path (or list of path) on storage
:returns Successful dict {path : True}
Failed dict {path : error message }
"""
urls = checkArgumentFormat( path )
if not urls['OK']:
return urls
urls = urls['Value']
successful = {}
failed = {}
self.log.debug( "FileStorage.createDirectory: Attempting to create %s directories." % len( urls ) )
for url in urls:
try:
# Create all the path
os.makedirs( url )
successful[url] = True
except OSError as ose:
failed[url] = str( ose )
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def getDirectory( self, path, localPath = False ):
"""Get locally a directory from the physical storage together with all its
files and subdirectories.
:param: path: path (or list of path) on storage
:param: localPath: local path where to store what is downloaded
:return: successful and failed dictionaries. The keys are the pathes,
the values are dictionary {'Files': amount of files downloaded, 'Size': amount of data downloaded}
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.getDirectory: Attempting to get local copies of %s directories." % len( urls ) )
failed = {}
successful = {}
if not localPath:
localPath = os.getcwd()
for src_dir in urls:
try:
dirName = os.path.basename( src_dir )
dest_dir = "%s/%s" % ( localPath, dirName )
successful[src_dir] = self.__copyDirectory( src_dir, dest_dir )
except OSError:
failed[src_dir] = {'Files':0, 'Size':0}
return S_OK( {'Failed' : failed, 'Successful' : successful } )
def getDirectorySize(self, path):
""" Get the size of the contents of the directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.isDirectory: Attempting to determine whether %s paths are directories."
% len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.getDirectorySize(url)
if not res['OK']:
failed[url] = res['Message']
else:
successful[url] = {
'Files': 0,
'Size': res['Value'],
'SubDirs': 0
}
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getFileSize( self, path ):
"""Get the physical size of the given file
:param self: self reference
:param path: path (or list of path) on storage
:returns Successful dict {path : size}
Failed dict {path : error message }
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for url in urls:
try:
# We check the filesize first because if it does not exist
# it raises an exception, while os.path.isfile just return False
filesize = os.path.getsize( url )
if os.path.isfile( url ):
successful[url] = filesize
else:
failed[url] = os.strerror( errno.EISDIR )
except OSError as ose:
failed[url] = str( ose )
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def setReplicaHost( self, lfn ):
res = checkArgumentFormat( lfn )
if not res['OK']:
return res
lfndict = res['Value']
rpcClient = self._getRPC()
return rpcClient.setReplicaHost( lfndict )
def removeFile(self, path):
"""Remove physically the file specified by its path
A non existing file will be considered as successfully removed.
:param path: path (or list of path) on storage
:returns Successful dict {path : True}
Failed dict {path : error message }
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
gLogger.debug(
"FileStorage.removeFile: Attempting to remove %s files." %
len(urls))
failed = {}
successful = {}
for url in urls:
try:
os.unlink(url)
successful[url] = True
except OSError as ose:
# Removing a non existing file is a success
if ose.errno == errno.ENOENT:
successful[url] = True
else:
failed[url] = str(ose)
except Exception as e:
failed[url] = str(e)
return S_OK({'Failed': failed, 'Successful': successful})
def getDirectoryMetadata(self, path):
""" Get metadata associated to the directory(ies)
:param self: self reference
:param path: url (or list of urls) on storage
:returns Successful dict {path : metadata}
Failed dict {path : error message }
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for url in urls:
res = self.__stat(url)
if not res['OK']:
failed[url] = res['Message']
elif not res['Value']['Directory']:
failed[url] = os.strerror(errno.ENOTDIR)
else:
successful[url] = res['Value']
return S_OK({'Failed': failed, 'Successful': successful})
def getReplicas( self, lfn ):
res = checkArgumentFormat( lfn )
if not res['OK']:
return res
lfns = res['Value'].keys()
rpcClient = self._getRPC()
return rpcClient.getReplicas( lfns )
def listDirectory( self, path ):
""" List the contents of the directory
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.listDirectory: Attempting to list %s directories." % len( urls ) )
serviceClient = RPCClient( self.url )
for url in urls:
res = serviceClient.listDirectory( url, 'l' )
if not res['OK']:
failed[url] = res['Message']
else:
files = {}
subDirs = {}
for subPath, pathDict in res['Value'].items():
if pathDict['Type'] == 'File':
files[subPath] = pathDict
elif pathDict['Type'] == 'Directory':
subDirs[subPath] = pathDict
successful[url] = {}
successful[url]['SubDirs'] = subDirs
successful[url]['Files'] = files
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getDirectoryMetadata(self, path):
""" Get metadata associated to the directory
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.getFileMetadata: Attempting to obtain metadata for %s directories."
% len(urls))
serviceClient = RPCClient(self.url)
for url in urls:
res = serviceClient.getMetadata(url)
if res['OK']:
if res['Value']['Exists']:
if res['Value']['Type'] == 'Directory':
res['Value']['Directory'] = True
gLogger.debug(
"DIPStorage.getFileMetadata: Successfully obtained metadata for %s."
% url)
successful[url] = res['Value']
else:
failed[url] = 'Supplied path is not a directory'
else:
failed[url] = 'Directory does not exist'
else:
gLogger.error(
"DIPStorage.getFileMetadata: Failed to get metadata for url",
"%s: %s" % (url, res['Message']))
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def addReplica( self, lfn, force = False ):
res = checkArgumentFormat( lfn )
if not res['OK']:
return res
lfndicts = res['Value']
rpcClient = self._getRPC()
return rpcClient.addReplica( lfndicts, force )
def getDirectoryMetadata( self, path ):
""" Get metadata associated to the directory
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.getFileMetadata: Attempting to obtain metadata for %s directories." % len( urls ) )
serviceClient = RPCClient( self.url )
for url in urls:
res = serviceClient.getMetadata( url )
if res['OK']:
if res['Value']['Exists']:
if res['Value']['Type'] == 'Directory':
res['Value']['Directory'] = True
gLogger.debug( "DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % url )
successful[url] = res['Value']
else:
failed[url] = 'Supplied path is not a directory'
else:
failed[url] = 'Directory does not exist'
else:
gLogger.error( "DIPStorage.getFileMetadata: Failed to get metadata for url",
"%s: %s" % ( url, res['Message'] ) )
failed[url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getFile(self, path, localPath=False):
"""Get a local copy in the current directory of a physical file specified by its path
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for src_url in urls:
fileName = os.path.basename(src_url)
if localPath:
dest_file = "%s/%s" % (localPath, fileName)
else:
dest_file = "%s/%s" % (os.getcwd(), fileName)
gLogger.debug(
"DIPStorage.getFile: Executing transfer of %s to %s" %
(src_url, dest_file))
res = self.__getFile(src_url, dest_file)
if res['OK']:
successful[src_url] = res['Value']
else:
failed[src_url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getDirectory(self, path, localPath=False):
"""Get locally a directory from the physical storage together with all its
files and subdirectories.
:param: path: path (or list of path) on storage
:param: localPath: local path where to store what is downloaded
:return: successful and failed dictionaries. The keys are the pathes,
the values are dictionary {'Files': amount of files downloaded, 'Size': amount of data downloaded}
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
self.log.debug(
"FileStorage.getDirectory: Attempting to get local copies of %s directories."
% len(urls))
failed = {}
successful = {}
if not localPath:
localPath = os.getcwd()
for src_dir in urls:
try:
dirName = os.path.basename(src_dir)
dest_dir = "%s/%s" % (localPath, dirName)
successful[src_dir] = self.__copyDirectory(src_dir, dest_dir)
except OSError:
failed[src_dir] = {'Files': 0, 'Size': 0}
return S_OK({'Failed': failed, 'Successful': successful})
def removeFile( self, path ):
"""Remove physically the file specified by its path
A non existing file will be considered as successfully removed.
:param path: path (or list of path) on storage
:returns Successful dict {path : True}
Failed dict {path : error message }
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
gLogger.debug( "FileStorage.removeFile: Attempting to remove %s files." % len( urls ) )
failed = {}
successful = {}
for url in urls:
try:
os.unlink(url)
successful[url] = True
except OSError as ose:
# Removing a non existing file is a success
if ose.errno == errno.ENOENT:
successful[url] = True
else:
failed[url] = str( ose )
except Exception as e:
failed[url] = str( e )
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def getFile( self, path, localPath = False ):
""" make a local copy of a storage :path:
:param self: self reference
:param str path: path on storage
:param localPath: if not specified, self.cwd
:returns Successful dict {path : size}
Failed dict {path : error message }
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.getFile: Trying to download %s files." % len( urls ) )
failed = {}
successful = {}
if not localPath:
localPath = os.getcwd()
for src_url in urls:
try:
fileName = os.path.basename( src_url )
dest_url = os.path.join( localPath, fileName )
shutil.copy2( src_url, dest_url )
fileSize = os.path.getsize( dest_url )
successful[src_url] = fileSize
except OSError as ose:
failed[src_url] = str( ose )
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def getDirectory(self, path, localPath=False):
""" Get a local copy in the current directory of a physical file specified by its path
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
gLogger.debug(
"DIPStorage.getDirectory: Attempting to get local copies of %s directories."
% len(urls))
transferClient = TransferClient(self.url)
for src_dir in urls:
if localPath:
dest_dir = localPath
else:
dest_dir = os.getcwd()
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
res = transferClient.receiveBulk(dest_dir, src_dir)
if res['OK']:
gLogger.debug(
"DIPStorage.getDirectory: Successfully got local copy of %s"
% src_dir)
successful[src_dir] = {'Files': 0, 'Size': 0}
else:
gLogger.error(
"DIPStorage.getDirectory: Failed to get entire directory.",
src_dir)
failed[src_dir] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getDirectoryMetadata( self, path ):
""" Get metadata associated to the directory(ies)
:param self: self reference
:param path: url (or list of urls) on storage
:returns Successful dict {path : metadata}
Failed dict {path : error message }
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for url in urls:
res = self.__stat( url )
if not res['OK']:
failed[url] = res['Message']
elif not res['Value']['Directory']:
failed[url] = os.strerror( errno.ENOTDIR )
else:
successful[url] = res['Value']
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def w_execute(self, *parms, **kws):
""" Write method executor.
"""
successful = {}
failed = {}
failedCatalogs = []
fileInfo = parms[0]
res = checkArgumentFormat(fileInfo)
if not res['OK']:
return res
fileInfo = res['Value']
allLfns = fileInfo.keys()
parms1 = parms[1:]
lfnsFlag = False
for catalogName, oCatalog, master in self.writeCatalogs:
# Skip if metadata related method on pure File Catalog
if self.call in FileCatalog.write_meta_methods and not catalogName in self.metaCatalogs:
continue
method = getattr(oCatalog, self.call)
if self.call in FileCatalog.write_meta_methods:
res = method(*parms, **kws)
else:
res = method(fileInfo, *parms1, **kws)
if not res['OK']:
if master:
# If this is the master catalog and it fails we dont want to continue with the other catalogs
gLogger.error(
"FileCatalog.w_execute: Failed to execute call on master catalog",
"%s on %s: %s" %
(self.call, catalogName, res['Message']))
return res
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs.append((catalogName, res['Message']))
else:
if 'Failed' in res['Value']:
lfnsFlag = True
for lfn, message in res['Value']['Failed'].items():
# Save the error message for the failed operations
failed.setdefault(lfn, {})[catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop(lfn, None)
for lfn, result in res['Value']['Successful'].items():
# Save the result return for each file for the successful operations
successful.setdefault(lfn, {})[catalogName] = result
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
if lfnsFlag:
for catalogName, errorMessage in failedCatalogs:
for lfn in allLfns:
failed.setdefault(lfn, {})[catalogName] = errorMessage
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
else:
return res
def _returnOK( lfn ):
res = checkArgumentFormat( lfn )
if not res['OK']:
return res
successful = {}
for lfn in res['Value']:
successful[lfn] = True
resDict = {'Successful':successful, 'Failed':{}}
return S_OK( resDict )
def _returnOK(lfn):
res = checkArgumentFormat(lfn)
if not res['OK']:
return res
successful = {}
for lfn in res['Value']:
successful[lfn] = True
resDict = {'Successful': successful, 'Failed': {}}
return S_OK(resDict)
def getTransportURL(self, path, protocols=False):
""" obtain the tURLs for the supplied path and protocols
:param self: self reference
:param str path: path on storage
:param mixed protocols: protocols to use
:returns Failed dict {path : error message}
Successful dict {path : transport url}
S_ERROR in case of argument problems
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
self.log.debug(
'GFAL2_SRM2Storage.getTransportURL: Attempting to retrieve tURL for %s paths'
% len(urls))
failed = {}
successful = {}
if not protocols:
protocols = self.__getProtocols()
if not protocols['OK']:
return protocols
listProtocols = protocols['Value']
elif type(protocols) == StringType:
listProtocols = [protocols]
elif type(protocols) == ListType:
listProtocols = protocols
else:
return S_ERROR(
"getTransportURL: Must supply desired protocols to this plug-in."
)
if self.protocolParameters['Protocol'] in listProtocols:
successful = {}
failed = {}
for url in urls:
if self.isURL(url)['Value']:
successful[url] = url
else:
failed[url] = 'getTransportURL: Failed to obtain turls.'
return S_OK({'Successful': successful, 'Failed': failed})
for url in urls:
res = self.__getSingleTransportURL(url, listProtocols)
self.log.debug('res = %s' % res)
if not res['OK']:
failed[url] = res['Message']
else:
successful[url] = res['Value']
return S_OK({'Failed': failed, 'Successful': successful})
def w_execute( self, *parms, **kws ):
""" Write method executor.
"""
successful = {}
failed = {}
failedCatalogs = []
fileInfo = parms[0]
res = checkArgumentFormat( fileInfo )
if not res['OK']:
return res
fileInfo = res['Value']
allLfns = fileInfo.keys()
parms1 = parms[1:]
lfnsFlag = False
for catalogName, oCatalog, master in self.writeCatalogs:
# Skip if metadata related method on pure File Catalog
if self.call in FileCatalog.write_meta_methods and not catalogName in self.metaCatalogs:
continue
method = getattr( oCatalog, self.call )
if self.call in FileCatalog.write_meta_methods:
res = method( *parms, **kws )
else:
res = method( fileInfo, *parms1, **kws )
if not res['OK']:
if master:
# If this is the master catalog and it fails we dont want to continue with the other catalogs
gLogger.error( "FileCatalog.w_execute: Failed to execute call on master catalog",
"%s on %s: %s" % ( self.call, catalogName, res['Message'] ) )
return res
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs.append( ( catalogName, res['Message'] ) )
else:
if 'Failed' in res['Value']:
lfnsFlag = True
for lfn, message in res['Value']['Failed'].items():
# Save the error message for the failed operations
failed.setdefault( lfn, {} )[catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop( lfn, None )
for lfn, result in res['Value']['Successful'].items():
# Save the result return for each file for the successful operations
successful.setdefault( lfn, {} )[catalogName] = result
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
if lfnsFlag:
for catalogName, errorMessage in failedCatalogs:
for lfn in allLfns:
failed.setdefault( lfn, {} )[catalogName] = errorMessage
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
else:
return res
def getTransportURL( self, path, protocols = False ):
""" obtain the tURLs for the supplied path and protocols
:param self: self reference
:param str path: path on storage
:param mixed protocols: protocols to use
:returns Failed dict {path : error message}
Successful dict {path : transport url}
S_ERROR in case of argument problems
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( 'GFAL2_SRM2Storage.getTransportURL: Attempting to retrieve tURL for %s paths' % len( urls ) )
failed = {}
successful = {}
if not protocols:
protocols = self.__getProtocols()
if not protocols['OK']:
return protocols
listProtocols = protocols['Value']
elif type( protocols ) == StringType:
listProtocols = [protocols]
elif type( protocols ) == ListType:
listProtocols = protocols
else:
return S_ERROR( "getTransportURL: Must supply desired protocols to this plug-in." )
if self.protocolParameters['Protocol'] in listProtocols:
successful = {}
failed = {}
for url in urls:
if self.isURL( url )['Value']:
successful[url] = url
else:
failed[url] = 'getTransportURL: Failed to obtain turls.'
return S_OK( {'Successful' : successful, 'Failed' : failed} )
for url in urls:
res = self.__getSingleTransportURL( url, listProtocols )
self.log.debug( 'res = %s' % res )
if not res['OK']:
failed[url] = res['Message']
else:
successful[url] = res['Value']
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def getDirectorySize(self, path):
""" Get the size of the directory on the storage
CAUTION : the size is not recursive, and does not go into subfolders
:param self: self reference
:param path: path (or list of path) on storage
:returns: list of successfull and failed dictionnary, both indexed by the path
In the failed, the value is the error message
In the successful the values are dictionnaries : Files : amount of files in the directory
Size : summed up size of files
subDirs : amount of sub directories
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
self.log.debug(
"FileStorage.getDirectorySize: Attempting to get size of %s directories."
% len(urls))
failed = {}
successful = {}
for url in urls:
try:
totSize = 0
nbOfFiles = 0
nbOfSubDirs = 0
for filename in os.listdir(url):
fullPath = os.path.join(url, filename)
if os.path.isfile(fullPath):
nbOfFiles += 1
totSize += os.path.getsize(fullPath)
else:
nbOfSubDirs += 1
successful[url] = {
'Files': nbOfFiles,
'Size': totSize,
'SubDirs': nbOfSubDirs
}
except OSError as ose:
failed[url] = str(ose)
return S_OK({'Failed': failed, 'Successful': successful})
def removeFile( self, lfn ):
res = checkArgumentFormat( lfn )
if not res['OK']:
return res
lfns = res['Value'].keys()
rpcClient = self._getRPC()
successful = {}
failed = {}
listOfLists = breakListIntoChunks( lfns, 100 )
for fList in listOfLists:
res = rpcClient.removeFile( fList )
if not res['OK']:
return res
successful.update( res['Value']['Successful'] )
failed.update( res['Value']['Failed'] )
resDict = {'Successful': successful, 'Failed':failed}
return S_OK( resDict )
def removeFile(self, lfn):
res = checkArgumentFormat(lfn)
if not res['OK']:
return res
lfns = res['Value'].keys()
rpcClient = self._getRPC()
successful = {}
failed = {}
listOfLists = breakListIntoChunks(lfns, 100)
for fList in listOfLists:
res = rpcClient.removeFile(fList)
if not res['OK']:
return res
successful.update(res['Value']['Successful'])
failed.update(res['Value']['Failed'])
resDict = {'Successful': successful, 'Failed': failed}
return S_OK(resDict)
def putFile( self, path, sourceSize = 0 ):
"""Put a file to the physical storage
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for dest_url, src_file in urls.items():
gLogger.debug( "DIPStorage.putFile: Executing transfer of %s to %s" % ( src_file, dest_url ) )
res = self.__putFile( src_file, dest_url )
if res['OK']:
successful[dest_url] = res['Value']
else:
failed[dest_url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def exists( self, path ):
""" Check if the given path exists. The 'path' variable can be a string or a list of strings.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
serviceClient = RPCClient( self.url )
for url in urls:
gLogger.debug( "DIPStorage.exists: Determining existence of %s." % url )
res = serviceClient.exists( url )
if res['OK']:
successful[url] = res['Value']
else:
failed[url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def exists( self, path ):
"""Check if the given path exists.
:param self: self reference
:param path: path (or list of path) on storage
:returns Failed dictionary: {pfn : errorMsg}
Successful dictionary: {pfn : bool}
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.exists: Checking the existence of %s path(s)" % len( urls ) )
successful = dict( ( url, os.path.exists( url ) ) for url in urls )
resDict = {'Failed':{}, 'Successful':successful}
return S_OK( resDict )
def putFile( self, path, sourceSize = 0 ):
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for dest_url, src_file in urls.items():
res = self.__executeOperation( os.path.dirname( dest_url ), 'createDirectory' )
if not res['OK']:
failed[dest_url] = res['Message']
else:
res = self.__putFile( src_file, dest_url, sourceSize )
if res['OK']:
successful[dest_url] = res['Value']
else:
failed[dest_url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getDirectorySize( self, path ):
""" Get the size of the contents of the directory
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len( urls ) )
serviceClient = RPCClient( self.url )
for url in urls:
res = serviceClient.getDirectorySize( url )
if not res['OK']:
failed[url] = res['Message']
else:
successful[url] = {'Files':0, 'Size':res['Value'], 'SubDirs':0}
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getDirectorySize( self, path ):
""" Get the size of the directory on the storage
CAUTION : the size is not recursive, and does not go into subfolders
:param self: self reference
:param path: path (or list of path) on storage
:returns: list of successfull and failed dictionnary, both indexed by the path
In the failed, the value is the error message
In the successful the values are dictionnaries : Files : amount of files in the directory
Size : summed up size of files
subDirs : amount of sub directories
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.getDirectorySize: Attempting to get size of %s directories." % len( urls ) )
failed = {}
successful = {}
for url in urls:
try:
totSize = 0
nbOfFiles = 0
nbOfSubDirs = 0
for filename in os.listdir( url ):
fullPath = os.path.join( url, filename )
if os.path.isfile( fullPath ):
nbOfFiles += 1
totSize += os.path.getsize( fullPath )
else:
nbOfSubDirs += 1
successful[url] = { 'Files' : nbOfFiles, 'Size' : totSize, 'SubDirs' : nbOfSubDirs }
except OSError as ose:
failed[url] = str( ose )
return S_OK( { 'Failed' : failed, 'Successful' : successful } )
def listDirectory(self, path):
""" List the supplied path
CAUTION : It is not recursive!
:param path : single or list of url
:return: successful and failed dictionaries. The keys are the pathes,
the values are dictionary 'SubDirs' and 'Files'. Each are dictionaries with
path as key and metadata as values (for Files only, SubDirs has just True as value)
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
self.log.debug(
"FileStorage.listDirectory: Attempting to list %s directories." %
len(urls))
successful = {}
failed = {}
for url in urls:
try:
dirs = []
files = []
# os.listdir returns files and directories together
for child in os.listdir(url):
fullpath = os.path.join(url, child)
lfnPath = os.path.join(
'/', os.path.relpath(fullpath, self.basePath))
if os.path.isfile(fullpath):
files.append(lfnPath)
else:
dirs.append(lfnPath)
successful[url] = {'SubDirs': dirs, 'Files': files}
except OSError as ose:
failed[url] = str(ose)
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def putFile(self, path, sourceSize=0):
"""Put a file to the physical storage
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for dest_url, src_file in urls.items():
gLogger.debug(
"DIPStorage.putFile: Executing transfer of %s to %s" %
(src_file, dest_url))
res = self.__putFile(src_file, dest_url)
if res['OK']:
successful[dest_url] = res['Value']
else:
failed[dest_url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def exists(self, path):
""" Check if the given path exists. The 'path' variable can be a string or a list of strings.
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
serviceClient = RPCClient(self.url)
for url in urls:
gLogger.debug("DIPStorage.exists: Determining existence of %s." %
url)
res = serviceClient.exists(url)
if res['OK']:
successful[url] = res['Value']
else:
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def putFile(self, path, sourceSize=0):
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for dest_url, src_file in urls.items():
res = self.__executeOperation(os.path.dirname(dest_url),
'createDirectory')
if not res['OK']:
failed[dest_url] = res['Message']
else:
res = self.__putFile(src_file, dest_url, sourceSize)
if res['OK']:
successful[dest_url] = res['Value']
else:
failed[dest_url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getLFNFromURL( self, urls ):
""" Get the LFN from the PFNS .
:param lfn : input lfn or lfns (list/dict)
"""
result = checkArgumentFormat( urls )
if result['OK']:
urlDict = result['Value']
else:
errStr = "Supplied urls must be string, list of strings or a dictionary."
self.log.getSubLogger( 'getLFNFromURL' ).debug( errStr )
return S_ERROR( errStr )
retDict = { "Successful" : {}, "Failed" : {} }
for url in urlDict:
res = self.__getURLPath( url )
if res["OK"]:
retDict["Successful"][url] = res["Value"]
else:
retDict["Failed"][url] = res["Message"]
return S_OK( retDict )
def getLFNFromURL(self, urls):
""" Get the LFN from the PFNS .
:param lfn : input lfn or lfns (list/dict)
"""
result = checkArgumentFormat(urls)
if result['OK']:
urlDict = result['Value']
else:
errStr = "Supplied urls must be string, list of strings or a dictionary."
self.log.getSubLogger('getLFNFromURL').debug(errStr)
return S_ERROR(errStr)
retDict = {"Successful": {}, "Failed": {}}
for url in urlDict:
res = self.__getURLPath(url)
if res["OK"]:
retDict["Successful"][url] = res["Value"]
else:
retDict["Failed"][url] = res["Message"]
return S_OK(retDict)
def listDirectory( self, path ):
""" List the supplied path
CAUTION : It is not recursive!
:param path : single or list of url
:return: successful and failed dictionaries. The keys are the pathes,
the values are dictionary 'SubDirs' and 'Files'. Each are dictionaries with
path as key and metadata as values (for Files only, SubDirs has just True as value)
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
self.log.debug( "FileStorage.listDirectory: Attempting to list %s directories." % len( urls ) )
successful = {}
failed = {}
for url in urls:
try:
dirs = []
files = []
# os.listdir returns files and directories together
for child in os.listdir(url):
fullpath= os.path.join(url, child)
lfnPath = os.path.join('/', os.path.relpath(fullpath, self.basePath))
if os.path.isfile(fullpath):
files.append(lfnPath)
else:
dirs.append(lfnPath)
successful[url] = { 'SubDirs' : dirs, 'Files' : files}
except OSError as ose:
failed[url] = str( ose )
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def createDirectory( self, path ):
""" Create the remote directory
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.createDirectory: Attempting to create %s directories." % len( urls ) )
serviceClient = RPCClient( self.url )
for url in urls:
res = serviceClient.createDirectory( url )
if res['OK']:
gLogger.debug( "DIPStorage.createDirectory: Successfully created directory on storage: %s" % url )
successful[url] = True
else:
gLogger.error( "DIPStorage.createDirectory: Failed to create directory on storage.", "%s: %s" % ( url, res['Message'] ) )
failed[url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putDirectory( self, path ):
""" Put a local directory to the physical storage together with all its files and subdirectories.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) )
transferClient = TransferClient( self.url )
for destDir, sourceDir in urls.items():
tmpList = os.listdir( sourceDir )
sourceFiles = [ "%s/%s" % ( sourceDir, x ) for x in tmpList ]
res = transferClient.sendBulk( sourceFiles, destDir )
if res['OK']:
successful[destDir] = {'Files':0, 'Size':0}
else:
failed[destDir] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def removeDirectory( self, path, recursive = False ):
""" Remove a directory from the storage together with all its files and subdirectories.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.removeDirectory: Attempting to remove %s directories." % len( urls ) )
serviceClient = RPCClient( self.url )
for url in urls:
res = serviceClient.removeDirectory( url, '' )
if res['OK']:
gLogger.debug( "DIPStorage.removeDirectory: Successfully removed directory on storage: %s" % url )
successful[url] = {'FilesRemoved':0, 'SizeRemoved':0}
else:
gLogger.error( "DIPStorage.removeDirectory: Failed to remove directory from storage.", "%s: %s" % ( url, res['Message'] ) )
failed[url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def exists(self, path):
"""Check if the given path exists.
:param self: self reference
:param path: path (or list of path) on storage
:returns Failed dictionary: {pfn : errorMsg}
Successful dictionary: {pfn : bool}
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
self.log.debug(
"FileStorage.exists: Checking the existence of %s path(s)" %
len(urls))
successful = dict((url, os.path.exists(url)) for url in urls)
resDict = {'Failed': {}, 'Successful': successful}
return S_OK(resDict)
def removeFile( self, path ):
"""Remove physically the file specified by its path
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
if not len( urls ) > 0:
return S_ERROR( "DIPStorage.removeFile: No surls supplied." )
successful = {}
failed = {}
serviceClient = RPCClient( self.url )
for url in urls:
gLogger.debug( "DIPStorage.removeFile: Attempting to remove %s." % url )
res = serviceClient.remove( url, '' )
if res['OK']:
successful[url] = True
else:
failed[url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putDirectory(self, path):
""" puts a or several local directory to the physical storage together with all its files and subdirectories
:param self: self reference
:param str path: dictionnary {url : local dir}
:return: successful and failed dictionaries. The keys are the pathes,
the values are dictionary {'Files': amount of files uploaded, 'Size': amount of data uploaded}
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for destDir, sourceDir in urls.items():
try:
successful[destDir] = self.__copyDirectory(sourceDir, destDir)
except OSError:
failed[destDir] = {'Files': 0, 'Size': 0}
return S_OK({"Failed": failed, "Successful": successful})
def removeFile(self, path):
"""Remove physically the file specified by its path
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
if not len(urls) > 0:
return S_ERROR("DIPStorage.removeFile: No surls supplied.")
successful = {}
failed = {}
serviceClient = RPCClient(self.url)
for url in urls:
gLogger.debug("DIPStorage.removeFile: Attempting to remove %s." %
url)
res = serviceClient.remove(url, '')
if res['OK']:
successful[url] = True
else:
failed[url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
