def sendFiles(self, jobID, fileList, sizeLimit=0):
""" Send files in the fileList to a Sandbox service for the given jobID.
This is the preferable method to upload sandboxes. fileList can contain
both files and directories
"""
print "sendFiles: sizeLimit =", sizeLimit
error_files = []
files_to_send = []
for file in fileList:
if re.search('^lfn:', file) or re.search('^LFN:', file):
pass
else:
if os.path.exists(file):
files_to_send.append(file)
else:
error_files.append(file)
if error_files:
return S_ERROR('Failed to locate files: \n' +
string.join(error_files, ','))
if sizeLimit > 0:
# Evaluate the compressed size of the sandbox
if getGlobbedTotalSize(files_to_send) > sizeLimit:
tname = 'Sandbox_' + str(jobID) + '.tar.gz'
import tarfile
tarFile = tarfile.open(tname, 'w:gz')
for file in files_to_send:
tarFile.add(file)
tarFile.close()
result = S_ERROR('Size over the limit')
result['SandboxFileName'] = tname
return result
sendName = str(jobID) + "::Job__Sandbox__"
sandbox = TransferClient('WorkloadManagement/%sSandbox' %
self.sandbox_type)
result = sandbox.sendBulk(files_to_send, sendName)
return result
def putDirectory( self, path ):
""" Put a local directory to the physical storage together with all its files and subdirectories.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) )
transferClient = TransferClient( self.url )
for destDir, sourceDir in urls.items():
tmpList = os.listdir( sourceDir )
sourceFiles = [ "%s/%s" % ( sourceDir, x ) for x in tmpList ]
res = transferClient.sendBulk( sourceFiles, destDir )
if res['OK']:
successful[destDir] = {'Files':0, 'Size':0}
else:
failed[destDir] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def putDirectory( self, path ):
""" Put a local directory to the physical storage together with all its files and subdirectories.
"""
res = checkArgumentFormat( path )
if not res['OK']:
return res
urls = res['Value']
successful = {}
failed = {}
gLogger.debug( "DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) )
transferClient = TransferClient( self.url )
for destDir, sourceDir in urls.items():
tmpList = os.listdir( sourceDir )
sourceFiles = [ "%s/%s" % ( sourceDir, x ) for x in tmpList ]
res = transferClient.sendBulk( sourceFiles, destDir )
if res['OK']:
successful[destDir] = {'Files':0, 'Size':0}
else:
failed[destDir] = res['Message']
resDict = {'Failed':failed, 'Successful':successful}
return S_OK( resDict )
def sendFiles(self,jobID,fileList,sizeLimit=0):
""" Send files in the fileList to a Sandbox service for the given jobID.
This is the preferable method to upload sandboxes. fileList can contain
both files and directories
"""
print "sendFiles: sizeLimit =", sizeLimit
error_files = []
files_to_send = []
for file in fileList:
if re.search('^lfn:',file) or re.search('^LFN:',file):
pass
else:
if os.path.exists(file):
files_to_send.append(file)
else:
error_files.append(file)
if error_files:
return S_ERROR('Failed to locate files: \n'+string.join(error_files,','))
if sizeLimit > 0:
# Evaluate the compressed size of the sandbox
if getGlobbedTotalSize( files_to_send ) > sizeLimit:
tname = 'Sandbox_'+str(jobID)+'.tar.gz'
import tarfile
tarFile = tarfile.open( tname, 'w:gz' )
for file in files_to_send:
tarFile.add( file )
tarFile.close()
result = S_ERROR('Size over the limit')
result['SandboxFileName'] = tname
return result
sendName = str(jobID)+"::Job__Sandbox__"
sandbox = TransferClient('WorkloadManagement/%sSandbox' % self.sandbox_type)
result = sandbox.sendBulk(files_to_send,sendName)
return result
