def putDirectory( self, path ):
""" Put a local directory to the physical storage together with all its files and subdirectories.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "RFIOStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) )
for destDir, sourceDir in urls.items():
res = self.__putDir( sourceDir, destDir )
if res['OK']:
if res['Value']['AllPut']:
gLogger.debug( "RFIOStorage.putDirectory: Successfully put directory to remote storage: %s" % destDir )
successful[destDir] = {'Files':res['Value']['Files'], 'Size':res['Value']['Size']}
else:
gLogger.error( "RFIOStorage.putDirectory: Failed to put entire directory to remote storage.", destDir )
failed[destDir] = {'Files':res['Value']['Files'], 'Size':res['Value']['Size']}
else:
gLogger.error( "RFIOStorage.putDirectory: Completely failed to put directory to remote storage.", destDir )
failed[destDir] = {'Files':0, 'Size':0}
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def w_execute(self, *parms, **kws):
""" Write method executor.
"""
successful = {}
failed = {}
failedCatalogs = []
fileInfo = parms[0]
res = checkArgumentFormat(fileInfo)
if not res['OK']:
return res
fileInfo = res['Value']
allLfns = fileInfo.keys()
parms = parms[1:]
for catalogName, oCatalog, master in self.writeCatalogs:
# Skip if metadata related method on pure File Catalog
if self.call in FileCatalog.write_meta_methods and not catalogName in self.metaCatalogs:
continue
method = getattr(oCatalog, self.call)
res = method(fileInfo, *parms, **kws)
if not res['OK']:
if master:
# If this is the master catalog and it fails we dont want to continue with the other catalogs
gLogger.error(
"FileCatalog.w_execute: Failed to execute %s on master catalog %s."
% (self.call, catalogName), res['Message'])
return res
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs.append((catalogName, res['Message']))
else:
for lfn, message in res['Value']['Failed'].items():
# Save the error message for the failed operations
failed.setdefault(lfn, {})[catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop(lfn, None)
for lfn, result in res['Value']['Successful'].items():
# Save the result return for each file for the successful operations
successful.setdefault(lfn, {})[catalogName] = result
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
for catalogName, errorMessage in failedCatalogs:
for lfn in allLfns:
failed.setdefault(lfn, {})[catalogName] = errorMessage
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def putFile(self, path, sourceSize=0):
client = RPCClient(self.url)
if sourceSize:
gLogger.debug(
"ProxyStorage.putFile: The client has provided the source file size implying a replication is requested."
)
return client.callProxyMethod(self.name, 'putFile', [path],
{'sourceSize': sourceSize})
gLogger.debug(
"ProxyStorage.putFile: No source size was provided therefore a simple put will be performed."
)
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
client = RPCClient(self.url)
transferClient = TransferClient(self.url)
for dest_url, src_file in urls.items():
fileName = os.path.basename(dest_url)
res = transferClient.sendFile(src_file, 'putFile/%s' % fileName)
if not res['OK']:
gLogger.error(
"ProxyStorage.putFile: Failed to send file to proxy server.",
res['Message'])
failed[dest_url] = res['Message']
else:
res = client.uploadFile(self.name, dest_url)
if not res['OK']:
gLogger.error(
"ProxyStorage.putFile: Failed to upload file to storage element from proxy server.",
res['Message'])
failed[dest_url] = res['Message']
else:
res = self.__executeOperation(dest_url, 'getFileSize')
if not res['OK']:
gLogger.error(
"ProxyStorage.putFile: Failed to determine destination file size.",
res['Message'])
failed[dest_url] = res['Message']
else:
successful[dest_url] = res['Value']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def w_execute( self, *parms, **kws ):
""" Write method executor.
"""
successful = {}
failed = {}
failedCatalogs = []
fileInfo = parms[0]
res = checkArgumentFormat( fileInfo )
if not res['OK']:
return res
fileInfo = res['Value']
allLfns = fileInfo.keys()
parms = parms[1:]
for catalogName, oCatalog, master in self.writeCatalogs:
# Skip if metadata related method on pure File Catalog
if self.call in FileCatalog.write_meta_methods and not catalogName in self.metaCatalogs:
continue
method = getattr( oCatalog, self.call )
res = method( fileInfo, *parms, **kws )
if not res['OK']:
if master:
# If this is the master catalog and it fails we dont want to continue with the other catalogs
gLogger.error( "FileCatalog.w_execute: Failed to execute %s on master catalog %s." % ( self.call, catalogName ), res['Message'] )
return res
else:
# Otherwise we keep the failed catalogs so we can update their state later
failedCatalogs.append( ( catalogName, res['Message'] ) )
else:
for lfn, message in res['Value']['Failed'].items():
# Save the error message for the failed operations
failed.setdefault( lfn, {} )[catalogName] = message
if master:
# If this is the master catalog then we should not attempt the operation on other catalogs
fileInfo.pop( lfn, None )
for lfn, result in res['Value']['Successful'].items():
# Save the result return for each file for the successful operations
successful.setdefault( lfn, {} )[catalogName] = result
# This recovers the states of the files that completely failed i.e. when S_ERROR is returned by a catalog
for catalogName, errorMessage in failedCatalogs:
for lfn in allLfns:
failed.setdefault( lfn, {} )[catalogName] = errorMessage
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putFile( self, path, sourceSize = 0 ):
"""Put a file to the physical storage
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for dest_url, src_file in urls.items():
gLogger.debug( "DIPStorage.putFile: Executing transfer of %s to %s" % ( src_file, dest_url ) )
res = self.__putFile( src_file, dest_url )
if res['OK']:
successful[dest_url] = res['Value']
else:
failed[dest_url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putFile( self, path, sourceSize = 0 ):
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for dest_url, src_file in urls.items():
res = self.__executeOperation( os.path.dirname( dest_url ), 'createDirectory' )
if not res['OK']:
failed[dest_url] = res['Message']
else:
res = self.__putFile( src_file, dest_url, sourceSize )
if res['OK']:
successful[dest_url] = res['Value']
else:
failed[dest_url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getFile(self, path, localPath=False):
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
client = RPCClient(self.url)
transferClient = TransferClient(self.url)
for src_url in urls.keys():
res = client.prepareFile(self.name, src_url)
if not res['OK']:
gLogger.error(
"ProxyStorage.getFile: Failed to prepare file on remote server.",
res['Message'])
failed[src_url] = res['Message']
else:
fileName = os.path.basename(src_url)
if localPath:
dest_file = "%s/%s" % (localPath, fileName)
else:
dest_file = "%s/%s" % (os.getcwd(), fileName)
res = transferClient.receiveFile(dest_file,
'getFile/%s' % fileName)
if not res['OK']:
gLogger.error(
"ProxyStorage.getFile: Failed to recieve file from proxy server.",
res['Message'])
failed[src_url] = res['Message']
elif not os.path.exists(dest_file):
errStr = "ProxyStorage.getFile: The destination local file does not exist."
gLogger.error(errStr, dest_file)
failed[src_url] = errStr
else:
destSize = getSize(dest_file)
if destSize == -1:
errStr = "ProxyStorage.getFile: Failed to get the local file size."
gLogger.error(errStr, dest_file)
failed[src_url] = errStr
else:
successful[src_url] = destSize
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def putFile(self, path, sourceSize=0):
"""Put a file to the physical storage
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
for dest_url, src_file in urls.items():
gLogger.debug(
"DIPStorage.putFile: Executing transfer of %s to %s" %
(src_file, dest_url))
res = self.__putFile(src_file, dest_url)
if res['OK']:
successful[dest_url] = res['Value']
else:
failed[dest_url] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def putDirectory( self, path ):
""" Put a local directory to the physical storage together with all its files and subdirectories.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) )
transferClient = TransferClient( self.url )
for destDir, sourceDir in urls.items():
tmpList = os.listdir( sourceDir )
sourceFiles = [ "%s/%s" % ( sourceDir, x ) for x in tmpList ]
res = transferClient.sendBulk( sourceFiles, destDir )
if res['OK']:
successful[destDir] = {'Files':0, 'Size':0}
else:
failed[destDir] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def getFile( self, path, localPath = False ):
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
client = RPCClient( self.url )
transferClient = TransferClient( self.url )
for src_url in urls.keys():
res = client.prepareFile( self.name, src_url )
if not res['OK']:
gLogger.error( "ProxyStorage.getFile: Failed to prepare file on remote server.", res['Message'] )
failed[src_url] = res['Message']
else:
fileName = os.path.basename( src_url )
if localPath:
dest_file = "%s/%s" % ( localPath, fileName )
else:
dest_file = "%s/%s" % ( os.getcwd(), fileName )
res = transferClient.receiveFile( dest_file, 'getFile/%s' % fileName )
if not res['OK']:
gLogger.error( "ProxyStorage.getFile: Failed to recieve file from proxy server.", res['Message'] )
failed[src_url] = res['Message']
elif not os.path.exists( dest_file ):
errStr = "ProxyStorage.getFile: The destination local file does not exist."
gLogger.error( errStr, dest_file )
failed[src_url] = errStr
else:
destSize = getSize( dest_file )
if destSize == -1:
errStr = "ProxyStorage.getFile: Failed to get the local file size."
gLogger.error( errStr, dest_file )
failed[src_url] = errStr
else:
successful[src_url] = destSize
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putDirectory(self, path):
""" Put a local directory to the physical storage together with all its files and subdirectories.
"""
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug(
"DIPStorage.putDirectory: Attemping to put %s directories to remote storage."
% len(urls))
transferClient = TransferClient(self.url)
for destDir, sourceDir in urls.items():
tmpList = os.listdir(sourceDir)
sourceFiles = ["%s/%s" % (sourceDir, x) for x in tmpList]
res = transferClient.sendBulk(sourceFiles, destDir)
if res['OK']:
successful[destDir] = {'Files': 0, 'Size': 0}
else:
failed[destDir] = res['Message']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def getFile( self, path, localPath = False ):
""" Get a local copy in the current directory of a physical file specified by its path
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
for src_url in urls.keys():
fileName = os.path.basename( src_url )
if localPath:
dest_file = "%s/%s" % ( localPath, fileName )
else:
dest_file = "%s/%s" % ( os.getcwd(), fileName )
res = self.__getFile( src_url, dest_file )
if res['OK']:
successful[src_url] = res['Value']
else:
failed[src_url] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putFile( self, path, sourceSize = 0 ):
client = RPCClient( self.url )
if sourceSize:
gLogger.debug( "ProxyStorage.putFile: The client has provided the source file size implying a replication is requested." )
return client.callProxyMethod( self.name, 'putFile', [path], {'sourceSize':sourceSize} )
gLogger.debug( "ProxyStorage.putFile: No source size was provided therefore a simple put will be performed." )
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
client = RPCClient( self.url )
transferClient = TransferClient( self.url )
for dest_url, src_file in urls.items():
fileName = os.path.basename( dest_url )
res = transferClient.sendFile( src_file, 'putFile/%s' % fileName )
if not res['OK']:
gLogger.error( "ProxyStorage.putFile: Failed to send file to proxy server.", res['Message'] )
failed[dest_url] = res['Message']
else:
res = client.uploadFile( self.name, dest_url )
if not res['OK']:
gLogger.error( "ProxyStorage.putFile: Failed to upload file to storage element from proxy server.", res['Message'] )
failed[dest_url] = res['Message']
else:
res = self.__executeOperation( dest_url, 'getFileSize' )
if not res['OK']:
gLogger.error( "ProxyStorage.putFile: Failed to determine destination file size.", res['Message'] )
failed[dest_url] = res['Message']
else:
successful[dest_url] = res['Value']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def prestageFileStatus( self, path ):
""" Monitor the status of a prestage request
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
requestFiles = {}
for url, requestID in urls.items():
if not requestFiles.has_key( requestID ):
requestFiles[requestID] = []
requestFiles[requestID].append( url )
for requestID, urls in requestFiles.items():
comm = "stager_qry -S %s -U %s " % ( self.spaceToken, requestID )
res = shellCall( 100, comm )
if res['OK']:
returncode, stdout, stderr = res['Value']
if returncode in [0, 1]:
for line in stdout.splitlines():
pfn = line.split()[0]
status = line.split()[-1]
if status in ['STAGED', 'CANBEMIGR']:
successful[pfn] = True
else:
successful[pfn] = False
else:
errStr = "RFIOStorage.prestageFileStatus: Got unexpected return code from stager_get."
gLogger.error( errStr, stderr )
return S_ERROR( errStr )
else:
errStr = "RFIOStorage.prestageFileStatus: Completely failed to obtain prestage status."
gLogger.error( errStr, res['Message'] )
return S_ERROR( errStr )
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
