def __putFile( self, src_file, dest_url ): res = pfnparse( src_file ) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: localCache = True srcSEURL = srcDict['Protocol'] + '://' + srcDict['Host'] + ':' + srcDict['Port'] + srcDict['WSUrl'] transferClient = TransferClient( srcSEURL ) res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'] ) ) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists( src_file ): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error( errStr, src_file ) return S_ERROR( errStr ) sourceSize = getSize( src_file ) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error( errStr, src_file ) return S_ERROR( errStr ) transferClient = TransferClient( self.url ) res = transferClient.sendFile( src_file, dest_url, token = self.checkSum ) if localCache: os.unlink( src_file ) if res['OK']: return S_OK( sourceSize ) else: return res
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: localCache = True srcSEURL = srcDict['Protocol'] + '://' + srcDict[ 'Host'] + ':' + srcDict['Port'] + srcDict['WSUrl'] transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join(srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res
def putFile( self, path, sourceSize = 0 ): client = RPCClient( self.url ) if sourceSize: gLogger.debug( "ProxyStorage.putFile: The client has provided the source file size implying a replication is requested." ) return client.callProxyMethod( self.name, 'putFile', path, {'sourceSize':sourceSize} ) gLogger.debug( "ProxyStorage.putFile: No source size was provided therefore a simple put will be performed." ) res = self.__checkArgumentFormatDict( path ) if not res['OK']: return res urls = res['Value'] failed = {} successful = {} client = RPCClient( self.url ) transferClient = TransferClient( self.url ) for dest_url, src_file in urls.items(): fileName = os.path.basename( dest_url ) res = transferClient.sendFile( src_file, 'putFile/%s' % fileName ) if not res['OK']: gLogger.error( "ProxyStorage.putFile: Failed to send file to proxy server.", res['Message'] ) failed[dest_url] = res['Message'] else: res = client.uploadFile( self.name, src_file ) if not res['OK']: gLogger.error( "ProxyStorage.putFile: Failed to upload file to storage element from proxy server.", res['Message'] ) failed[dest_url] = res['Message'] else: res = self.__executeOperation( dest_url, 'getFileSize' ) if not res['OK']: gLogger.error( "ProxyStorage.putFile: Failed to determine destination file size.", res['Message'] ) failed[dest_url] = res['Message'] else: successful[dest_url] = res['Value'] resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict )
def sendFile(self, jobID, fname): """ Send a file specified by fname to Sandbox service for job with jobID """ if os.path.exists(fname): if os.path.isdir(fname): dname = os.path.dirname(fname) bname = os.path.basename(fname) bzname = bname + '.tar.gz' if dname: comm = 'tar czf ' + bzname + ' -C ' + dname + ' ' + bname else: comm = 'tar czf ' + bzname + ' ' + bname result = shellCall(0, comm) if not result['OK'] or result['Value'][0]: return S_ERROR('Failed to send directory ' + fname) sendName = ` jobID ` + "::" + bzname sandbox = TransferClient('WorkloadManagement/%sSandbox' % self.sandbox_type) result = sandbox.sendFile(sendName, bzname) if not result['OK']: gLogger.error('Failed to send directory ' + bzname + ' to Sandbox service for job ' + ` jobID `) os.remove(bzname) return result os.remove(bzname) else: # This is a file bname = os.path.basename(fname) sendName = ` jobID ` + "::" + bname sandbox = TransferClient('WorkloadManagement/%sSandbox' % self.sandbox_type) result = sandbox.sendFile(bname, sendName) if not result['OK']: gLogger.error('Failed to send file ' + bname + ' to Sandbox service for job ' + ` jobID `) return result # We are done OK return S_OK() else: gLogger.error("Can't find file " + fname) return S_ERROR("Can't find file " + fname)
def sendFile(self,jobID,fname): """ Send a file specified by fname to Sandbox service for job with jobID """ if os.path.exists(fname): if os.path.isdir(fname): dname = os.path.dirname(fname) bname = os.path.basename(fname) bzname = bname+'.tar.gz' if dname: comm = 'tar czf '+bzname+' -C '+dname+' '+bname else: comm = 'tar czf '+bzname+' '+bname result = shellCall(0,comm) if not result['OK'] or result['Value'][0]: return S_ERROR('Failed to send directory '+fname) sendName = `jobID`+"::"+bzname sandbox = TransferClient('WorkloadManagement/%sSandbox' % self.sandbox_type) result = sandbox.sendFile(sendName,bzname) if not result['OK']: gLogger.error('Failed to send directory '+bzname+' to Sandbox service for job '+`jobID`) os.remove(bzname) return result os.remove(bzname) else: # This is a file bname = os.path.basename(fname) sendName = `jobID`+"::"+bname sandbox = TransferClient('WorkloadManagement/%sSandbox' % self.sandbox_type) result = sandbox.sendFile(bname, sendName) if not result['OK']: gLogger.error('Failed to send file '+bname+' to Sandbox service for job '+`jobID`) return result # We are done OK return S_OK() else: gLogger.error("Can't find file "+ fname) return S_ERROR("Can't find file "+ fname)
def putFile(self, path, sourceSize=0): client = RPCClient(self.url) if sourceSize: gLogger.debug( "ProxyStorage.putFile: The client has provided the source file size implying a replication is requested." ) return client.callProxyMethod(self.name, 'putFile', [path], {'sourceSize': sourceSize}) gLogger.debug( "ProxyStorage.putFile: No source size was provided therefore a simple put will be performed." ) res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] failed = {} successful = {} # make sure transferClient uses the same ProxyStorage instance we uploaded the file to transferClient = TransferClient(client.serviceURL) for dest_url, src_file in urls.items(): fileName = os.path.basename(dest_url) res = transferClient.sendFile(src_file, 'putFile/%s' % fileName) if not res['OK']: gLogger.error( "ProxyStorage.putFile: Failed to send file to proxy server.", res['Message']) failed[dest_url] = res['Message'] else: res = client.uploadFile(self.name, dest_url) if not res['OK']: gLogger.error( "ProxyStorage.putFile: Failed to upload file to storage element from proxy server.", res['Message']) failed[dest_url] = res['Message'] else: res = self.__executeOperation(dest_url, 'getFileSize') if not res['OK']: gLogger.error( "ProxyStorage.putFile: Failed to determine destination file size.", res['Message']) failed[dest_url] = res['Message'] else: successful[dest_url] = res['Value'] resDict = {'Failed': failed, 'Successful': successful} return S_OK(resDict)
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: # Make the service URL from the file URL by stripping off the file part serviceDict = dict(srcDict) serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3]) serviceDict['FileName'] = '' res = pfnunparse(serviceDict) if not res['OK']: return res srcSEURL = res['Value'] localCache = True transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join(srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res
def __putFile(self, src_file, dest_url): res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: # Make the service URL from the file URL by stripping off the file part serviceDict = dict(srcDict) serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3]) serviceDict['FileName'] = '' res = pfnunparse(serviceDict) if not res['OK']: return res srcSEURL = res['Value'] localCache = True transferClient = TransferClient(srcSEURL) res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." gLogger.error(errStr, src_file) return S_ERROR(errStr) sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." gLogger.error(errStr, src_file) return S_ERROR(errStr) transferClient = TransferClient(self.url) res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: os.unlink(src_file) if res['OK']: return S_OK(sourceSize) else: return res