def __putFile( self, src_file, dest_url ):
res = pfnparse( src_file )
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
localCache = True
srcSEURL = srcDict['Protocol'] + '://' + srcDict['Host'] + ':' + srcDict['Port'] + srcDict['WSUrl']
transferClient = TransferClient( srcSEURL )
res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'] ) )
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists( src_file ):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error( errStr, src_file )
return S_ERROR( errStr )
sourceSize = getSize( src_file )
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error( errStr, src_file )
return S_ERROR( errStr )
transferClient = TransferClient( self.url )
res = transferClient.sendFile( src_file, dest_url, token = self.checkSum )
if localCache:
os.unlink( src_file )
if res['OK']:
return S_OK( sourceSize )
else:
return res
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
localCache = True
srcSEURL = srcDict['Protocol'] + '://' + srcDict[
'Host'] + ':' + srcDict['Port'] + srcDict['WSUrl']
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'],
os.path.join(srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def putFile( self, path, sourceSize = 0 ):
client = RPCClient( self.url )
if sourceSize:
gLogger.debug( "ProxyStorage.putFile: The client has provided the source file size implying a replication is requested." )
return client.callProxyMethod( self.name, 'putFile', path, {'sourceSize':sourceSize} )
gLogger.debug( "ProxyStorage.putFile: No source size was provided therefore a simple put will be performed." )
res = self.__checkArgumentFormatDict( path )
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
client = RPCClient( self.url )
transferClient = TransferClient( self.url )
for dest_url, src_file in urls.items():
fileName = os.path.basename( dest_url )
res = transferClient.sendFile( src_file, 'putFile/%s' % fileName )
if not res['OK']:
gLogger.error( "ProxyStorage.putFile: Failed to send file to proxy server.", res['Message'] )
failed[dest_url] = res['Message']
else:
res = client.uploadFile( self.name, src_file )
if not res['OK']:
gLogger.error( "ProxyStorage.putFile: Failed to upload file to storage element from proxy server.", res['Message'] )
failed[dest_url] = res['Message']
else:
res = self.__executeOperation( dest_url, 'getFileSize' )
if not res['OK']:
gLogger.error( "ProxyStorage.putFile: Failed to determine destination file size.", res['Message'] )
failed[dest_url] = res['Message']
else:
successful[dest_url] = res['Value']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def sendFile(self, jobID, fname):
""" Send a file specified by fname to Sandbox service for job with jobID
"""
if os.path.exists(fname):
if os.path.isdir(fname):
dname = os.path.dirname(fname)
bname = os.path.basename(fname)
bzname = bname + '.tar.gz'
if dname:
comm = 'tar czf ' + bzname + ' -C ' + dname + ' ' + bname
else:
comm = 'tar czf ' + bzname + ' ' + bname
result = shellCall(0, comm)
if not result['OK'] or result['Value'][0]:
return S_ERROR('Failed to send directory ' + fname)
sendName = ` jobID ` + "::" + bzname
sandbox = TransferClient('WorkloadManagement/%sSandbox' %
self.sandbox_type)
result = sandbox.sendFile(sendName, bzname)
if not result['OK']:
gLogger.error('Failed to send directory ' + bzname +
' to Sandbox service for job ' + ` jobID `)
os.remove(bzname)
return result
os.remove(bzname)
else: # This is a file
bname = os.path.basename(fname)
sendName = ` jobID ` + "::" + bname
sandbox = TransferClient('WorkloadManagement/%sSandbox' %
self.sandbox_type)
result = sandbox.sendFile(bname, sendName)
if not result['OK']:
gLogger.error('Failed to send file ' + bname +
' to Sandbox service for job ' + ` jobID `)
return result
# We are done OK
return S_OK()
else:
gLogger.error("Can't find file " + fname)
return S_ERROR("Can't find file " + fname)
def sendFile(self,jobID,fname):
""" Send a file specified by fname to Sandbox service for job with jobID
"""
if os.path.exists(fname):
if os.path.isdir(fname):
dname = os.path.dirname(fname)
bname = os.path.basename(fname)
bzname = bname+'.tar.gz'
if dname:
comm = 'tar czf '+bzname+' -C '+dname+' '+bname
else:
comm = 'tar czf '+bzname+' '+bname
result = shellCall(0,comm)
if not result['OK'] or result['Value'][0]:
return S_ERROR('Failed to send directory '+fname)
sendName = `jobID`+"::"+bzname
sandbox = TransferClient('WorkloadManagement/%sSandbox' % self.sandbox_type)
result = sandbox.sendFile(sendName,bzname)
if not result['OK']:
gLogger.error('Failed to send directory '+bzname+' to Sandbox service for job '+`jobID`)
os.remove(bzname)
return result
os.remove(bzname)
else: # This is a file
bname = os.path.basename(fname)
sendName = `jobID`+"::"+bname
sandbox = TransferClient('WorkloadManagement/%sSandbox' % self.sandbox_type)
result = sandbox.sendFile(bname, sendName)
if not result['OK']:
gLogger.error('Failed to send file '+bname+' to Sandbox service for job '+`jobID`)
return result
# We are done OK
return S_OK()
else:
gLogger.error("Can't find file "+ fname)
return S_ERROR("Can't find file "+ fname)
def putFile(self, path, sourceSize=0):
client = RPCClient(self.url)
if sourceSize:
gLogger.debug(
"ProxyStorage.putFile: The client has provided the source file size implying a replication is requested."
)
return client.callProxyMethod(self.name, 'putFile', [path],
{'sourceSize': sourceSize})
gLogger.debug(
"ProxyStorage.putFile: No source size was provided therefore a simple put will be performed."
)
res = checkArgumentFormat(path)
if not res['OK']:
return res
urls = res['Value']
failed = {}
successful = {}
# make sure transferClient uses the same ProxyStorage instance we uploaded the file to
transferClient = TransferClient(client.serviceURL)
for dest_url, src_file in urls.items():
fileName = os.path.basename(dest_url)
res = transferClient.sendFile(src_file, 'putFile/%s' % fileName)
if not res['OK']:
gLogger.error(
"ProxyStorage.putFile: Failed to send file to proxy server.",
res['Message'])
failed[dest_url] = res['Message']
else:
res = client.uploadFile(self.name, dest_url)
if not res['OK']:
gLogger.error(
"ProxyStorage.putFile: Failed to upload file to storage element from proxy server.",
res['Message'])
failed[dest_url] = res['Message']
else:
res = self.__executeOperation(dest_url, 'getFileSize')
if not res['OK']:
gLogger.error(
"ProxyStorage.putFile: Failed to determine destination file size.",
res['Message'])
failed[dest_url] = res['Message']
else:
successful[dest_url] = res['Value']
resDict = {'Failed': failed, 'Successful': successful}
return S_OK(resDict)
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
# Make the service URL from the file URL by stripping off the file part
serviceDict = dict(srcDict)
serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
serviceDict['FileName'] = ''
res = pfnunparse(serviceDict)
if not res['OK']:
return res
srcSEURL = res['Value']
localCache = True
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'],
os.path.join(srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
def __putFile(self, src_file, dest_url):
res = pfnparse(src_file)
if not res['OK']:
return res
localCache = False
srcDict = res['Value']
if srcDict['Protocol'] in ['dips', 'dip']:
# Make the service URL from the file URL by stripping off the file part
serviceDict = dict(srcDict)
serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3])
serviceDict['FileName'] = ''
res = pfnunparse(serviceDict)
if not res['OK']:
return res
srcSEURL = res['Value']
localCache = True
transferClient = TransferClient(srcSEURL)
res = transferClient.receiveFile(
srcDict['FileName'], os.path.join(
srcDict['Path'], srcDict['FileName']))
if not res['OK']:
return res
src_file = srcDict['FileName']
if not os.path.exists(src_file):
errStr = "DIPStorage.__putFile: The source local file does not exist."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
sourceSize = getSize(src_file)
if sourceSize == -1:
errStr = "DIPStorage.__putFile: Failed to get file size."
gLogger.error(errStr, src_file)
return S_ERROR(errStr)
transferClient = TransferClient(self.url)
res = transferClient.sendFile(src_file, dest_url, token=self.checkSum)
if localCache:
os.unlink(src_file)
if res['OK']:
return S_OK(sourceSize)
else:
return res
