def getDataFromService( self, srvMethod, params ):
self.infoMsg( "Sending header request to %s" % self.getDestinationService(), str( params ) )
result = self._sendTransferHeader( srvMethod, params )
if not result[ 'OK' ]:
self.errMsg( "Could not send header", result[ 'Message' ] )
return result
self.infoMsg( "Starting to receive data from service" )
srvTransport = result[ 'Value' ]
srvFileHelper = FileHelper( srvTransport )
srvFileHelper.setDirection( "receive" )
sIO = cStringIO.StringIO()
result = srvFileHelper.networkToDataSink( sIO, self.__transferBytesLimit )
if not result[ 'OK' ]:
self.errMsg( "Could receive data from server", result[ 'Message' ] )
srvTransport.close()
sIO.close()
return result
dataReceived = sIO.getvalue()
sIO.close()
self.infoMsg( "Received %s bytes from service" % len( dataReceived ) )
retVal = srvTransport.receiveData()
srvTransport.close()
if not retVal[ 'OK' ]:
return retVal
return S_OK( { 'data' : dataReceived, 'srvResponse' : retVal } )
def receiveBulk(self, destDir, bulkId, token="", compress=True):
"""
Receive a bulk of files from server
@type destDir : list of ( string / file descriptor / file object )
@param destDir : Files to receive from server
@type bulkId : any
@param bulkId : Identification of the files being received
@type token : string
@param token : Token for the bulk
@type compress : boolean
@param compress : Enable compression for the bulk. By default its True
@return : S_OK/S_ERROR
"""
if not os.path.isdir(destDir):
return S_ERROR("%s is not a directory for bulk receival" % destDir)
if compress:
bulkId = "%s.tar.bz2" % bulkId
else:
bulkId = "%s.tar" % bulkId
try:
retVal = self._sendTransferHeader("BulkToClient", (bulkId, token))
if not retVal['OK']:
return retVal
transport = retVal['Value']
fileHelper = FileHelper(transport)
retVal = fileHelper.networkToBulk(destDir, compress)
if not retVal['OK']:
return retVal
retVal = transport.receiveData()
self._disconnect()
return retVal
finally:
self._disconnect()
def receiveFile( self, filename, fileId, token = "" ):
"""
Receive a file from the server
@type filename : string / file descriptor / file object
@param filename : File to receive from server
@type fileId : any
@param fileId : Identification of the file being received
@type token : string
@param token : Optional token for the file
@return : S_OK/S_ERROR
"""
fileHelper = FileHelper()
retVal = fileHelper.getDataSink( filename )
if not retVal[ 'OK' ]:
return retVal
dS = retVal[ 'Value' ]
closeAfterUse = retVal[ 'closeAfterUse' ]
retVal = self._sendTransferHeader( "ToClient", ( fileId, token ) )
if not retVal[ 'OK' ]:
return retVal
trid, transport = retVal[ 'Value' ]
try:
fileHelper.setTransport( transport )
retVal = fileHelper.networkToDataSink( dS )
if not retVal[ 'OK' ]:
return retVal
retVal = transport.receiveData()
if closeAfterUse:
dS.close()
return retVal
finally:
self._disconnect( trid )
def receiveBulk( self, destDir, bulkId, token = "", compress = True ):
"""
Receive a bulk of files from server
@type destDir : list of ( string / file descriptor / file object )
@param destDir : Files to receive from server
@type bulkId : any
@param bulkId : Identification of the files being received
@type token : string
@param token : Token for the bulk
@type compress : boolean
@param compress : Enable compression for the bulk. By default its True
@return : S_OK/S_ERROR
"""
if not os.path.isdir( destDir ):
return S_ERROR( "%s is not a directory for bulk receival" % destDir )
if compress:
bulkId = "%s.tar.bz2" % bulkId
else:
bulkId = "%s.tar" % bulkId
retVal = self._sendTransferHeader( "BulkToClient", ( bulkId, token ) )
if not retVal[ 'OK' ]:
return retVal
trid, transport = retVal[ 'Value' ]
try:
fileHelper = FileHelper( transport )
retVal = fileHelper.networkToBulk( destDir, compress )
if not retVal[ 'OK' ]:
return retVal
retVal = transport.receiveData()
return retVal
finally:
def sendFile( self, filename, fileId, token = "" ):
"""
Send a file to server
@type filename : string / file descriptor / file object
@param filename : File to send to server
@type fileId : any
@param fileId : Identification of the file being sent
@type token : string
@param token : Optional token for the file
@return : S_OK/S_ERROR
"""
fileHelper = FileHelper()
retVal = fileHelper.getFileDescriptor( filename, "r" )
if not retVal[ 'OK' ]:
return retVal
fd = retVal[ 'Value' ]
retVal = self._sendTransferHeader( "FromClient", ( fileId, token, File.getSize( filename ) ) )
if not retVal[ 'OK' ]:
return retVal
trid, transport = retVal[ 'Value' ]
try:
fileHelper.setTransport( transport )
retVal = fileHelper.FDToNetwork( fd )
if not retVal[ 'OK' ]:
return retVal
retVal = transport.receiveData()
return retVal
finally:
self._disconnect( trid )
def sendBulk( self, fileList, bulkId, token = "", compress = True, bulkSize = -1, onthefly = True ):
"""
Send a bulk of files to server
@type fileList : list of ( string / file descriptor / file object )
@param fileList : Files to send to server
@type bulkId : any
@param bulkId : Identification of the files being sent
@type token : string
@param token : Token for the bulk
@type compress : boolean
@param compress : Enable compression for the bulk. By default its True
@type bulkSize : integer
@param bulkSize : Optional size of the bulk
@return : S_OK/S_ERROR
"""
bogusEntries = self.__checkFileList( fileList )
if bogusEntries:
return S_ERROR( "Some files or directories don't exist :\n\t%s" % "\n\t".join( bogusEntries ) )
if compress:
bulkId = "%s.tar.bz2" % bulkId
else:
bulkId = "%s.tar" % bulkId
retVal = self._sendTransferHeader( "BulkFromClient", ( bulkId, token, bulkSize ) )
if not retVal[ 'OK' ]:
return retVal
trid, transport = retVal[ 'Value' ]
try:
fileHelper = FileHelper( transport )
retVal = fileHelper.bulkToNetwork( fileList, compress, onthefly )
if not retVal[ 'OK' ]:
return retVal
retVal = transport.receiveData()
return retVal
finally:
def getDataFromService(self, srvMethod, params):
self.infoMsg("Sending header request to %s" % self.getDestinationService(), str(params))
result = self._sendTransferHeader(srvMethod, params)
if not result["OK"]:
self.errMsg("Could not send header", result["Message"])
return result
self.infoMsg("Starting to receive data from service")
_, srvTransport = result["Value"]
srvFileHelper = FileHelper(srvTransport)
srvFileHelper.setDirection("receive")
sIO = BytesIO()
result = srvFileHelper.networkToDataSink(sIO, self.__transferBytesLimit)
if not result["OK"]:
self.errMsg("Could not receive data from server", result["Message"])
srvTransport.close()
sIO.close()
return result
dataReceived = sIO.getvalue()
sIO.close()
self.infoMsg("Received %s bytes from service" % len(dataReceived))
retVal = srvTransport.receiveData()
srvTransport.close()
if not retVal["OK"]:
return retVal
return S_OK({"data": dataReceived, "srvResponse": retVal})
def receiveFile(self, filename, fileId, token=""):
"""
Receive a file from the server
:type filename : string / file descriptor / file object
:param filename : File to receive from server
:type fileId : any
:param fileId : Identification of the file being received
:type token : string
:param token : Optional token for the file
:return : S_OK/S_ERROR
"""
fileHelper = FileHelper()
if "NoCheckSum" in token:
fileHelper.disableCheckSum()
retVal = fileHelper.getDataSink(filename)
if not retVal['OK']:
return retVal
dS = retVal['Value']
closeAfterUse = retVal['closeAfterUse']
retVal = self._sendTransferHeader("ToClient", (fileId, token))
if not retVal['OK']:
return retVal
trid, transport = retVal['Value']
try:
fileHelper.setTransport(transport)
retVal = fileHelper.networkToDataSink(dS)
if not retVal['OK']:
return retVal
retVal = transport.receiveData()
if closeAfterUse:
dS.close()
return retVal
finally:
self._disconnect(trid)
def sendFile(self, filename, fileId, token=""):
"""
Send a file to server
:type filename : string / file descriptor / file object
:param filename : File to send to server
:type fileId : any
:param fileId : Identification of the file being sent
:type token : string
:param token : Optional token for the file
:return : S_OK/S_ERROR
"""
fileHelper = FileHelper()
if "NoCheckSum" in token:
fileHelper.disableCheckSum()
retVal = fileHelper.getFileDescriptor(filename, "r")
if not retVal['OK']:
return retVal
fd = retVal['Value']
retVal = self._sendTransferHeader(
"FromClient", (fileId, token, File.getSize(filename)))
if not retVal['OK']:
return retVal
trid, transport = retVal['Value']
try:
fileHelper.setTransport(transport)
retVal = fileHelper.FDToNetwork(fd)
if not retVal['OK']:
return retVal
retVal = transport.receiveData()
return retVal
finally:
self._disconnect(trid)
def sendDataToService( self, srvMethod, params, data ):
self.infoMsg( "Sending header request to %s" % self.getDestinationService(), str( params ) )
result = self._sendTransferHeader( srvMethod, params )
if not result[ 'OK' ]:
self.errMsg( "Could not send header", result[ 'Message' ] )
return result
self.infoMsg( "Starting to send data to service" )
srvTransport = result[ 'Value' ]
srvFileHelper = FileHelper( srvTransport )
srvFileHelper.setDirection( "send" )
result = srvFileHelper.BufferToNetwork( data )
if not result[ 'OK' ]:
self.errMsg( "Could send data to server", result[ 'Message' ] )
srvTransport.close()
return result
self.infoMsg( "Data sent to service (%s bytes)" % len( data ) )
retVal = srvTransport.receiveData()
srvTransport.close()
return retVal
def sendDataToService(self, srvMethod, params, data):
self.infoMsg("Sending header request to %s" % self.getDestinationService(), str(params))
result = self._sendTransferHeader(srvMethod, params)
if not result["OK"]:
self.errMsg("Could not send header", result["Message"])
return result
self.infoMsg("Starting to send data to service")
_, srvTransport = result["Value"]
srvFileHelper = FileHelper(srvTransport)
srvFileHelper.setDirection("send")
result = srvFileHelper.BufferToNetwork(data)
if not result["OK"]:
self.errMsg("Could send data to server", result["Message"])
srvTransport.close()
return result
self.infoMsg("Data sent to service (%s bytes)" % len(data))
retVal = srvTransport.receiveData()
srvTransport.close()
return retVal
def __forwardFileTransferCall(self, targetService, clientInitArgs, method, params, clientTransport):
transferRelay = TransferRelay(targetService, **clientInitArgs)
transferRelay.setTransferLimit(self.__transferBytesLimit)
cliFH = FileHelper(clientTransport)
# Check file size
if method.find("ToClient") > -1:
cliFH.setDirection("send")
elif method.find("FromClient") > -1:
cliFH.setDirection("receive")
if not self.__ftCheckMaxTransferSize(params[2]):
cliFH.markAsTransferred()
return S_ERROR("Transfer size is too big")
# Forward queries
try:
relayMethodObject = getattr(transferRelay, "forward%s" % method)
except Exception:
return S_ERROR("Cannot forward unknown method %s" % method)
result = relayMethodObject(cliFH, params)
return result
def __forwardFileTransferCall(self, targetService, clientInitArgs, method, params, clientTransport):
transferRelay = TransferRelay(targetService, **clientInitArgs)
transferRelay.setTransferLimit(self.__transferBytesLimit)
cliFH = FileHelper(clientTransport)
# Check file size
if method.find("ToClient") > -1:
cliFH.setDirection("send")
elif method.find("FromClient") > -1:
cliFH.setDirection("receive")
if not self.__ftCheckMaxTransferSize(params[2]):
cliFH.markAsTransferred()
return S_ERROR("Transfer size is too big")
# Forward queries
try:
relayMethodObject = getattr(transferRelay, "forward%s" % method)
except:
return S_ERROR("Cannot forward unknown method %s" % method)
result = relayMethodObject(cliFH, params)
return result
def __doFileTransfer(self, sDirection):
"""
Execute a file transfer action
@type sDirection: string
@param sDirection: Direction of the transfer
@return: S_OK/S_ERROR
"""
retVal = self.__trPool.receive(self.__trid)
if not retVal['OK']:
raise RequestHandler.ConnectionError(
"Error while receiving file description %s %s" %
(self.srv_getFormattedRemoteCredentials(), retVal['Message']))
fileInfo = retVal['Value']
sDirection = "%s%s" % (sDirection[0].lower(), sDirection[1:])
if "transfer_%s" % sDirection not in dir(self):
self.__trPool.send(
self.__trid,
S_ERROR("Service can't transfer files %s" % sDirection))
return
retVal = self.__trPool.send(self.__trid, S_OK("Accepted"))
if not retVal['OK']:
return retVal
self.__logRemoteQuery("FileTransfer/%s" % sDirection, fileInfo)
self.__lockManager.lock("FileTransfer/%s" % sDirection)
try:
try:
fileHelper = FileHelper(self.__trPool.get(self.__trid))
if sDirection == "fromClient":
fileHelper.setDirection("fromClient")
uRetVal = self.transfer_fromClient(fileInfo[0],
fileInfo[1],
fileInfo[2], fileHelper)
elif sDirection == "toClient":
fileHelper.setDirection("toClient")
uRetVal = self.transfer_toClient(fileInfo[0], fileInfo[1],
fileHelper)
elif sDirection == "bulkFromClient":
fileHelper.setDirection("fromClient")
uRetVal = self.transfer_bulkFromClient(
fileInfo[0], fileInfo[1], fileInfo[2], fileHelper)
elif sDirection == "bulkToClient":
fileHelper.setDirection("toClient")
uRetVal = self.transfer_bulkToClient(
fileInfo[0], fileInfo[1], fileHelper)
elif sDirection == "listBulk":
fileHelper.setDirection("toClient")
uRetVal = self.transfer_listBulk(fileInfo[0], fileInfo[1],
fileHelper)
else:
return S_ERROR("Direction %s does not exist!!!" %
sDirection)
if uRetVal['OK'] and not fileHelper.finishedTransmission():
gLogger.error(
"You haven't finished receiving/sending the file",
str(fileInfo))
return S_ERROR("Incomplete transfer")
return uRetVal
finally:
self.__lockManager.unlock("FileTransfer/%s" % sDirection)
except Exception, v:
gLogger.exception("Uncaught exception when serving Transfer",
"%s" % sDirection)
return S_ERROR("Server error while serving %s: %s" %
(sDirection, str(v)))
def __doFileTransfer( self, sDirection ):
"""
Execute a file transfer action
@type sDirection: string
@param sDirection: Direction of the transfer
@return: S_OK/S_ERROR
"""
retVal = self.__trPool.receive( self.__trid )
if not retVal[ 'OK' ]:
gLogger.error( "Error while receiving file description", "%s %s" % ( self.srv_getFormattedRemoteCredentials(),
retVal[ 'Message' ] ) )
return S_ERROR( "Error while receiving file description: %s" % retVal[ 'Message' ] )
fileInfo = retVal[ 'Value' ]
sDirection = "%s%s" % ( sDirection[0].lower(), sDirection[1:] )
if "transfer_%s" % sDirection not in dir( self ):
self.__trPool.send( self.__trid, S_ERROR( "Service can't transfer files %s" % sDirection ) )
return
retVal = self.__trPool.send( self.__trid, S_OK( "Accepted" ) )
if not retVal[ 'OK' ]:
return retVal
self.__logRemoteQuery( "FileTransfer/%s" % sDirection, fileInfo )
self.__lockManager.lock( sDirection )
try:
try:
fileHelper = FileHelper( self.__trPool.get( self.__trid ) )
if sDirection == "fromClient":
fileHelper.setDirection( "fromClient" )
uRetVal = self.transfer_fromClient( fileInfo[0], fileInfo[1], fileInfo[2], fileHelper )
elif sDirection == "toClient" :
fileHelper.setDirection( "toClient" )
uRetVal = self.transfer_toClient( fileInfo[0], fileInfo[1], fileHelper )
elif sDirection == "bulkFromClient" :
fileHelper.setDirection( "fromClient" )
uRetVal = self.transfer_bulkFromClient( fileInfo[0], fileInfo[1], fileInfo[2], fileHelper )
elif sDirection == "bulkToClient" :
fileHelper.setDirection( "toClient" )
uRetVal = self.transfer_bulkToClient( fileInfo[0], fileInfo[1], fileHelper )
elif sDirection == "listBulk":
fileHelper.setDirection( "toClient" )
uRetVal = self.transfer_listBulk( fileInfo[0], fileInfo[1], fileHelper )
else:
return S_ERROR( "Direction %s does not exist!!!" % sDirection )
if uRetVal[ 'OK' ] and not fileHelper.finishedTransmission():
gLogger.error( "You haven't finished receiving/sending the file", str( fileInfo ) )
return S_ERROR( "Incomplete transfer" )
return uRetVal
finally:
self.__lockManager.unlock( sDirection )
except Exception, v:
gLogger.exception( "Uncaught exception when serving Transfer", "%s" % sDirection )
return S_ERROR( "Server error while serving %s: %s" % ( sDirection, str( v ) ) )