def getDataFromService( self, srvMethod, params ): self.infoMsg( "Sending header request to %s" % self.getDestinationService(), str( params ) ) result = self._sendTransferHeader( srvMethod, params ) if not result[ 'OK' ]: self.errMsg( "Could not send header", result[ 'Message' ] ) return result self.infoMsg( "Starting to receive data from service" ) srvTransport = result[ 'Value' ] srvFileHelper = FileHelper( srvTransport ) srvFileHelper.setDirection( "receive" ) sIO = cStringIO.StringIO() result = srvFileHelper.networkToDataSink( sIO, self.__transferBytesLimit ) if not result[ 'OK' ]: self.errMsg( "Could receive data from server", result[ 'Message' ] ) srvTransport.close() sIO.close() return result dataReceived = sIO.getvalue() sIO.close() self.infoMsg( "Received %s bytes from service" % len( dataReceived ) ) retVal = srvTransport.receiveData() srvTransport.close() if not retVal[ 'OK' ]: return retVal return S_OK( { 'data' : dataReceived, 'srvResponse' : retVal } )
def receiveBulk(self, destDir, bulkId, token="", compress=True): """ Receive a bulk of files from server @type destDir : list of ( string / file descriptor / file object ) @param destDir : Files to receive from server @type bulkId : any @param bulkId : Identification of the files being received @type token : string @param token : Token for the bulk @type compress : boolean @param compress : Enable compression for the bulk. By default its True @return : S_OK/S_ERROR """ if not os.path.isdir(destDir): return S_ERROR("%s is not a directory for bulk receival" % destDir) if compress: bulkId = "%s.tar.bz2" % bulkId else: bulkId = "%s.tar" % bulkId try: retVal = self._sendTransferHeader("BulkToClient", (bulkId, token)) if not retVal['OK']: return retVal transport = retVal['Value'] fileHelper = FileHelper(transport) retVal = fileHelper.networkToBulk(destDir, compress) if not retVal['OK']: return retVal retVal = transport.receiveData() self._disconnect() return retVal finally: self._disconnect()
def receiveFile( self, filename, fileId, token = "" ): """ Receive a file from the server @type filename : string / file descriptor / file object @param filename : File to receive from server @type fileId : any @param fileId : Identification of the file being received @type token : string @param token : Optional token for the file @return : S_OK/S_ERROR """ fileHelper = FileHelper() retVal = fileHelper.getDataSink( filename ) if not retVal[ 'OK' ]: return retVal dS = retVal[ 'Value' ] closeAfterUse = retVal[ 'closeAfterUse' ] retVal = self._sendTransferHeader( "ToClient", ( fileId, token ) ) if not retVal[ 'OK' ]: return retVal trid, transport = retVal[ 'Value' ] try: fileHelper.setTransport( transport ) retVal = fileHelper.networkToDataSink( dS ) if not retVal[ 'OK' ]: return retVal retVal = transport.receiveData() if closeAfterUse: dS.close() return retVal finally: self._disconnect( trid )
def receiveBulk( self, destDir, bulkId, token = "", compress = True ): """ Receive a bulk of files from server @type destDir : list of ( string / file descriptor / file object ) @param destDir : Files to receive from server @type bulkId : any @param bulkId : Identification of the files being received @type token : string @param token : Token for the bulk @type compress : boolean @param compress : Enable compression for the bulk. By default its True @return : S_OK/S_ERROR """ if not os.path.isdir( destDir ): return S_ERROR( "%s is not a directory for bulk receival" % destDir ) if compress: bulkId = "%s.tar.bz2" % bulkId else: bulkId = "%s.tar" % bulkId retVal = self._sendTransferHeader( "BulkToClient", ( bulkId, token ) ) if not retVal[ 'OK' ]: return retVal trid, transport = retVal[ 'Value' ] try: fileHelper = FileHelper( transport ) retVal = fileHelper.networkToBulk( destDir, compress ) if not retVal[ 'OK' ]: return retVal retVal = transport.receiveData() return retVal finally:
def sendFile( self, filename, fileId, token = "" ): """ Send a file to server @type filename : string / file descriptor / file object @param filename : File to send to server @type fileId : any @param fileId : Identification of the file being sent @type token : string @param token : Optional token for the file @return : S_OK/S_ERROR """ fileHelper = FileHelper() retVal = fileHelper.getFileDescriptor( filename, "r" ) if not retVal[ 'OK' ]: return retVal fd = retVal[ 'Value' ] retVal = self._sendTransferHeader( "FromClient", ( fileId, token, File.getSize( filename ) ) ) if not retVal[ 'OK' ]: return retVal trid, transport = retVal[ 'Value' ] try: fileHelper.setTransport( transport ) retVal = fileHelper.FDToNetwork( fd ) if not retVal[ 'OK' ]: return retVal retVal = transport.receiveData() return retVal finally: self._disconnect( trid )
def sendBulk( self, fileList, bulkId, token = "", compress = True, bulkSize = -1, onthefly = True ): """ Send a bulk of files to server @type fileList : list of ( string / file descriptor / file object ) @param fileList : Files to send to server @type bulkId : any @param bulkId : Identification of the files being sent @type token : string @param token : Token for the bulk @type compress : boolean @param compress : Enable compression for the bulk. By default its True @type bulkSize : integer @param bulkSize : Optional size of the bulk @return : S_OK/S_ERROR """ bogusEntries = self.__checkFileList( fileList ) if bogusEntries: return S_ERROR( "Some files or directories don't exist :\n\t%s" % "\n\t".join( bogusEntries ) ) if compress: bulkId = "%s.tar.bz2" % bulkId else: bulkId = "%s.tar" % bulkId retVal = self._sendTransferHeader( "BulkFromClient", ( bulkId, token, bulkSize ) ) if not retVal[ 'OK' ]: return retVal trid, transport = retVal[ 'Value' ] try: fileHelper = FileHelper( transport ) retVal = fileHelper.bulkToNetwork( fileList, compress, onthefly ) if not retVal[ 'OK' ]: return retVal retVal = transport.receiveData() return retVal finally:
def getDataFromService(self, srvMethod, params): self.infoMsg("Sending header request to %s" % self.getDestinationService(), str(params)) result = self._sendTransferHeader(srvMethod, params) if not result["OK"]: self.errMsg("Could not send header", result["Message"]) return result self.infoMsg("Starting to receive data from service") _, srvTransport = result["Value"] srvFileHelper = FileHelper(srvTransport) srvFileHelper.setDirection("receive") sIO = BytesIO() result = srvFileHelper.networkToDataSink(sIO, self.__transferBytesLimit) if not result["OK"]: self.errMsg("Could not receive data from server", result["Message"]) srvTransport.close() sIO.close() return result dataReceived = sIO.getvalue() sIO.close() self.infoMsg("Received %s bytes from service" % len(dataReceived)) retVal = srvTransport.receiveData() srvTransport.close() if not retVal["OK"]: return retVal return S_OK({"data": dataReceived, "srvResponse": retVal})
def receiveFile(self, filename, fileId, token=""): """ Receive a file from the server :type filename : string / file descriptor / file object :param filename : File to receive from server :type fileId : any :param fileId : Identification of the file being received :type token : string :param token : Optional token for the file :return : S_OK/S_ERROR """ fileHelper = FileHelper() if "NoCheckSum" in token: fileHelper.disableCheckSum() retVal = fileHelper.getDataSink(filename) if not retVal['OK']: return retVal dS = retVal['Value'] closeAfterUse = retVal['closeAfterUse'] retVal = self._sendTransferHeader("ToClient", (fileId, token)) if not retVal['OK']: return retVal trid, transport = retVal['Value'] try: fileHelper.setTransport(transport) retVal = fileHelper.networkToDataSink(dS) if not retVal['OK']: return retVal retVal = transport.receiveData() if closeAfterUse: dS.close() return retVal finally: self._disconnect(trid)
def sendFile(self, filename, fileId, token=""): """ Send a file to server :type filename : string / file descriptor / file object :param filename : File to send to server :type fileId : any :param fileId : Identification of the file being sent :type token : string :param token : Optional token for the file :return : S_OK/S_ERROR """ fileHelper = FileHelper() if "NoCheckSum" in token: fileHelper.disableCheckSum() retVal = fileHelper.getFileDescriptor(filename, "r") if not retVal['OK']: return retVal fd = retVal['Value'] retVal = self._sendTransferHeader( "FromClient", (fileId, token, File.getSize(filename))) if not retVal['OK']: return retVal trid, transport = retVal['Value'] try: fileHelper.setTransport(transport) retVal = fileHelper.FDToNetwork(fd) if not retVal['OK']: return retVal retVal = transport.receiveData() return retVal finally: self._disconnect(trid)
def sendDataToService( self, srvMethod, params, data ): self.infoMsg( "Sending header request to %s" % self.getDestinationService(), str( params ) ) result = self._sendTransferHeader( srvMethod, params ) if not result[ 'OK' ]: self.errMsg( "Could not send header", result[ 'Message' ] ) return result self.infoMsg( "Starting to send data to service" ) srvTransport = result[ 'Value' ] srvFileHelper = FileHelper( srvTransport ) srvFileHelper.setDirection( "send" ) result = srvFileHelper.BufferToNetwork( data ) if not result[ 'OK' ]: self.errMsg( "Could send data to server", result[ 'Message' ] ) srvTransport.close() return result self.infoMsg( "Data sent to service (%s bytes)" % len( data ) ) retVal = srvTransport.receiveData() srvTransport.close() return retVal
def sendDataToService(self, srvMethod, params, data): self.infoMsg("Sending header request to %s" % self.getDestinationService(), str(params)) result = self._sendTransferHeader(srvMethod, params) if not result["OK"]: self.errMsg("Could not send header", result["Message"]) return result self.infoMsg("Starting to send data to service") _, srvTransport = result["Value"] srvFileHelper = FileHelper(srvTransport) srvFileHelper.setDirection("send") result = srvFileHelper.BufferToNetwork(data) if not result["OK"]: self.errMsg("Could send data to server", result["Message"]) srvTransport.close() return result self.infoMsg("Data sent to service (%s bytes)" % len(data)) retVal = srvTransport.receiveData() srvTransport.close() return retVal
def __forwardFileTransferCall(self, targetService, clientInitArgs, method, params, clientTransport): transferRelay = TransferRelay(targetService, **clientInitArgs) transferRelay.setTransferLimit(self.__transferBytesLimit) cliFH = FileHelper(clientTransport) # Check file size if method.find("ToClient") > -1: cliFH.setDirection("send") elif method.find("FromClient") > -1: cliFH.setDirection("receive") if not self.__ftCheckMaxTransferSize(params[2]): cliFH.markAsTransferred() return S_ERROR("Transfer size is too big") # Forward queries try: relayMethodObject = getattr(transferRelay, "forward%s" % method) except Exception: return S_ERROR("Cannot forward unknown method %s" % method) result = relayMethodObject(cliFH, params) return result
def __forwardFileTransferCall(self, targetService, clientInitArgs, method, params, clientTransport): transferRelay = TransferRelay(targetService, **clientInitArgs) transferRelay.setTransferLimit(self.__transferBytesLimit) cliFH = FileHelper(clientTransport) # Check file size if method.find("ToClient") > -1: cliFH.setDirection("send") elif method.find("FromClient") > -1: cliFH.setDirection("receive") if not self.__ftCheckMaxTransferSize(params[2]): cliFH.markAsTransferred() return S_ERROR("Transfer size is too big") # Forward queries try: relayMethodObject = getattr(transferRelay, "forward%s" % method) except: return S_ERROR("Cannot forward unknown method %s" % method) result = relayMethodObject(cliFH, params) return result
def __doFileTransfer(self, sDirection): """ Execute a file transfer action @type sDirection: string @param sDirection: Direction of the transfer @return: S_OK/S_ERROR """ retVal = self.__trPool.receive(self.__trid) if not retVal['OK']: raise RequestHandler.ConnectionError( "Error while receiving file description %s %s" % (self.srv_getFormattedRemoteCredentials(), retVal['Message'])) fileInfo = retVal['Value'] sDirection = "%s%s" % (sDirection[0].lower(), sDirection[1:]) if "transfer_%s" % sDirection not in dir(self): self.__trPool.send( self.__trid, S_ERROR("Service can't transfer files %s" % sDirection)) return retVal = self.__trPool.send(self.__trid, S_OK("Accepted")) if not retVal['OK']: return retVal self.__logRemoteQuery("FileTransfer/%s" % sDirection, fileInfo) self.__lockManager.lock("FileTransfer/%s" % sDirection) try: try: fileHelper = FileHelper(self.__trPool.get(self.__trid)) if sDirection == "fromClient": fileHelper.setDirection("fromClient") uRetVal = self.transfer_fromClient(fileInfo[0], fileInfo[1], fileInfo[2], fileHelper) elif sDirection == "toClient": fileHelper.setDirection("toClient") uRetVal = self.transfer_toClient(fileInfo[0], fileInfo[1], fileHelper) elif sDirection == "bulkFromClient": fileHelper.setDirection("fromClient") uRetVal = self.transfer_bulkFromClient( fileInfo[0], fileInfo[1], fileInfo[2], fileHelper) elif sDirection == "bulkToClient": fileHelper.setDirection("toClient") uRetVal = self.transfer_bulkToClient( fileInfo[0], fileInfo[1], fileHelper) elif sDirection == "listBulk": fileHelper.setDirection("toClient") uRetVal = self.transfer_listBulk(fileInfo[0], fileInfo[1], fileHelper) else: return S_ERROR("Direction %s does not exist!!!" % sDirection) if uRetVal['OK'] and not fileHelper.finishedTransmission(): gLogger.error( "You haven't finished receiving/sending the file", str(fileInfo)) return S_ERROR("Incomplete transfer") return uRetVal finally: self.__lockManager.unlock("FileTransfer/%s" % sDirection) except Exception, v: gLogger.exception("Uncaught exception when serving Transfer", "%s" % sDirection) return S_ERROR("Server error while serving %s: %s" % (sDirection, str(v)))
def __doFileTransfer( self, sDirection ): """ Execute a file transfer action @type sDirection: string @param sDirection: Direction of the transfer @return: S_OK/S_ERROR """ retVal = self.__trPool.receive( self.__trid ) if not retVal[ 'OK' ]: gLogger.error( "Error while receiving file description", "%s %s" % ( self.srv_getFormattedRemoteCredentials(), retVal[ 'Message' ] ) ) return S_ERROR( "Error while receiving file description: %s" % retVal[ 'Message' ] ) fileInfo = retVal[ 'Value' ] sDirection = "%s%s" % ( sDirection[0].lower(), sDirection[1:] ) if "transfer_%s" % sDirection not in dir( self ): self.__trPool.send( self.__trid, S_ERROR( "Service can't transfer files %s" % sDirection ) ) return retVal = self.__trPool.send( self.__trid, S_OK( "Accepted" ) ) if not retVal[ 'OK' ]: return retVal self.__logRemoteQuery( "FileTransfer/%s" % sDirection, fileInfo ) self.__lockManager.lock( sDirection ) try: try: fileHelper = FileHelper( self.__trPool.get( self.__trid ) ) if sDirection == "fromClient": fileHelper.setDirection( "fromClient" ) uRetVal = self.transfer_fromClient( fileInfo[0], fileInfo[1], fileInfo[2], fileHelper ) elif sDirection == "toClient" : fileHelper.setDirection( "toClient" ) uRetVal = self.transfer_toClient( fileInfo[0], fileInfo[1], fileHelper ) elif sDirection == "bulkFromClient" : fileHelper.setDirection( "fromClient" ) uRetVal = self.transfer_bulkFromClient( fileInfo[0], fileInfo[1], fileInfo[2], fileHelper ) elif sDirection == "bulkToClient" : fileHelper.setDirection( "toClient" ) uRetVal = self.transfer_bulkToClient( fileInfo[0], fileInfo[1], fileHelper ) elif sDirection == "listBulk": fileHelper.setDirection( "toClient" ) uRetVal = self.transfer_listBulk( fileInfo[0], fileInfo[1], fileHelper ) else: return S_ERROR( "Direction %s does not exist!!!" % sDirection ) if uRetVal[ 'OK' ] and not fileHelper.finishedTransmission(): gLogger.error( "You haven't finished receiving/sending the file", str( fileInfo ) ) return S_ERROR( "Incomplete transfer" ) return uRetVal finally: self.__lockManager.unlock( sDirection ) except Exception, v: gLogger.exception( "Uncaught exception when serving Transfer", "%s" % sDirection ) return S_ERROR( "Server error while serving %s: %s" % ( sDirection, str( v ) ) )