def __uploadInputSandbox( self, classAdJob, jobDescriptionObject = None ):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries( classAdJob )
realFiles = []
badFiles = []
diskFiles = []
for isFile in inputSandbox:
valid = True
for tag in ( 'lfn:', 'LFN:', 'SB:', '%s' ): # in case of parametric input sandbox, there is %s passed, so have to ignore it also
if isFile.find( tag ) == 0:
valid = False
break
if valid:
realFiles.append( isFile )
stringIOFiles = []
stringIOFilesSize = 0
if jobDescriptionObject is not None:
if isinstance( jobDescriptionObject, StringIO.StringIO ):
stringIOFiles = [jobDescriptionObject]
stringIOFilesSize = len( jobDescriptionObject.buf )
gLogger.debug( "Size of the stringIOFiles: " + str( stringIOFilesSize ) )
else:
return S_ERROR( "jobDescriptionObject is not a StringIO object" )
# Check real files
for isFile in realFiles:
if not os.path.exists( isFile ): # we are passing in real files, we expect them to be on disk
badFiles.append( isFile )
gLogger.warn( "inputSandbox file/directory " + isFile + " not found. Keep looking for the others" )
continue
diskFiles.append( isFile )
diskFilesSize = File.getGlobbedTotalSize( diskFiles )
gLogger.debug( "Size of the diskFiles: " + str( diskFilesSize ) )
totalSize = diskFilesSize + stringIOFilesSize
gLogger.verbose( "Total size of the inputSandbox: " + str( totalSize ) )
okFiles = stringIOFiles + diskFiles
if badFiles:
result = S_ERROR( 'Input Sandbox is not valid' )
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient( useCertificates = self.useCertificates )
result = self.sandboxClient.uploadFilesAsSandbox( okFiles )
if not result[ 'OK' ]:
return result
inputSandbox.append( result[ 'Value' ] )
classAdJob.insertAttributeVectorString( "InputSandbox", inputSandbox )
return S_OK()
def __uploadInputSandbox(self, classAdJob, jobDescriptionObject=None):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
diskFiles = []
for isFile in inputSandbox:
if not isFile.startswith(("lfn:", "LFN:", "SB:", "%s", "%(")):
realFiles.append(isFile)
stringIOFiles = []
stringIOFilesSize = 0
if jobDescriptionObject is not None:
if isinstance(jobDescriptionObject, StringIO):
stringIOFiles = [jobDescriptionObject]
stringIOFilesSize = len(jobDescriptionObject.getvalue())
gLogger.debug("Size of the stringIOFiles: " + str(stringIOFilesSize))
else:
return S_ERROR(EWMSJDL, "jobDescriptionObject is not a StringIO object")
# Check real files
for isFile in realFiles:
if not os.path.exists(isFile): # we are passing in real files, we expect them to be on disk
badFiles.append(isFile)
gLogger.warn("inputSandbox file/directory " + isFile + " not found. Keep looking for the others")
continue
diskFiles.append(isFile)
diskFilesSize = File.getGlobbedTotalSize(diskFiles)
gLogger.debug("Size of the diskFiles: " + str(diskFilesSize))
totalSize = diskFilesSize + stringIOFilesSize
gLogger.verbose("Total size of the inputSandbox: " + str(totalSize))
okFiles = stringIOFiles + diskFiles
if badFiles:
result = S_ERROR(EWMSJDL, "Input Sandbox is not valid")
result["BadFile"] = badFiles
result["TotalSize"] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates,
delegatedDN=self.delegatedDN,
delegatedGroup=self.delegatedGroup,
)
result = self.sandboxClient.uploadFilesAsSandbox(okFiles)
if not result["OK"]:
return result
inputSandbox.append(result["Value"])
classAdJob.insertAttributeVectorString("InputSandbox", inputSandbox)
return S_OK()
def __uploadInputSandbox(self, classAdJob):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates, rpcClient=self.sbRPCClient, transferClient=self.sbTransferClient
)
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
okFiles = []
realFiles = []
for file in inputSandbox:
valid = True
for tag in (
"lfn:",
"LFN:",
"SB:",
"%s",
): # in case of parametric input sandbox, there is %s passed, so have to ignore it also
if file.find(tag) == 0:
valid = False
break
if valid:
realFiles.append(file)
# If there are no files, skip!
if not realFiles:
return S_OK()
# Check real files
for file in realFiles:
if not os.path.exists(file):
badFiles.append(file)
print "inputSandbox file/directory " + file + " not found"
continue
okFiles.append(file)
# print "Total size of the inputSandbox: "+str(totalSize)
totalSize = File.getGlobbedTotalSize(okFiles)
if badFiles:
result = S_ERROR("Input Sandbox is not valid")
result["BadFile"] = badFiles
result["TotalSize"] = totalSize
return result
if okFiles:
result = sandboxClient.uploadFilesAsSandbox(okFiles)
if not result["OK"]:
return result
inputSandbox.append(result["Value"])
classAdJob.insertAttributeVectorString("InputSandbox", inputSandbox)
return S_OK()
def __uploadInputSandbox(self, classAdJob):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates,
rpcClient=self.sbRPCClient,
transferClient=self.sbTransferClient)
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
okFiles = []
realFiles = []
for file in inputSandbox:
valid = True
for tag in (
'lfn:', 'LFN:', 'SB:', '%s'
): #in case of parametric input sandbox, there is %s passed, so have to ignore it also
if file.find(tag) == 0:
valid = False
break
if valid:
realFiles.append(file)
#If there are no files, skip!
if not realFiles:
return S_OK()
#Check real files
for file in realFiles:
if not os.path.exists(file):
badFiles.append(file)
print "inputSandbox file/directory " + file + " not found"
continue
okFiles.append(file)
#print "Total size of the inputSandbox: "+str(totalSize)
totalSize = File.getGlobbedTotalSize(okFiles)
if badFiles:
result = S_ERROR('Input Sandbox is not valid')
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
result = sandboxClient.uploadFilesAsSandbox(okFiles)
if not result['OK']:
return result
inputSandbox.append(result['Value'])
classAdJob.insertAttributeVectorString("InputSandbox",
inputSandbox)
return S_OK()
def __uploadInputSandbox( self, classAdJob ):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries( classAdJob )
badFiles = []
okFiles = []
realFiles = []
for isFile in inputSandbox:
valid = True
for tag in ( 'lfn:', 'LFN:', 'SB:', '%s' ): # in case of parametric input sandbox, there is %s passed, so have to ignore it also
if isFile.find( tag ) == 0:
valid = False
break
if valid:
realFiles.append( isFile )
# If there are no files, skip!
if not realFiles:
return S_OK()
# Check real files
for isFile in realFiles:
if not os.path.exists( isFile ):
badFiles.append( isFile )
gLogger.warn( "inputSandbox file/directory " + isFile + " not found. Keep looking for the others" )
continue
okFiles.append( isFile )
totalSize = File.getGlobbedTotalSize( okFiles )
gLogger.verbose( "Total size of the inputSandbox: " + str( totalSize ) )
if badFiles:
result = S_ERROR( 'Input Sandbox is not valid' )
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient( useCertificates = self.useCertificates )
result = self.sandboxClient.uploadFilesAsSandbox( okFiles )
if not result[ 'OK' ]:
return result
inputSandbox.append( result[ 'Value' ] )
classAdJob.insertAttributeVectorString( "InputSandbox", inputSandbox )
return S_OK()
def __uploadInputSandbox(self, classAdJob, jobDescriptionObject=None):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
diskFiles = []
for isFile in inputSandbox:
valid = True
for tag in (
'lfn:', 'LFN:', 'SB:', '%s'
): # in case of parametric input sandbox, there is %s passed, so have to ignore it also
if isFile.find(tag) == 0:
valid = False
break
if valid:
realFiles.append(isFile)
stringIOFiles = []
stringIOFilesSize = 0
if jobDescriptionObject is not None:
if isinstance(jobDescriptionObject, StringIO.StringIO):
stringIOFiles = [jobDescriptionObject]
stringIOFilesSize = len(jobDescriptionObject.buf)
gLogger.debug("Size of the stringIOFiles: " +
str(stringIOFilesSize))
else:
return S_ERROR("jobDescriptionObject is not a StringIO object")
# Check real files
for isFile in realFiles:
if not os.path.exists(
isFile
): # we are passing in real files, we expect them to be on disk
badFiles.append(isFile)
gLogger.warn("inputSandbox file/directory " + isFile +
" not found. Keep looking for the others")
continue
diskFiles.append(isFile)
diskFilesSize = File.getGlobbedTotalSize(diskFiles)
gLogger.debug("Size of the diskFiles: " + str(diskFilesSize))
totalSize = diskFilesSize + stringIOFilesSize
gLogger.verbose("Total size of the inputSandbox: " + str(totalSize))
okFiles = stringIOFiles + diskFiles
if badFiles:
result = S_ERROR('Input Sandbox is not valid')
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates)
result = self.sandboxClient.uploadFilesAsSandbox(okFiles)
if not result['OK']:
return result
inputSandbox.append(result['Value'])
classAdJob.insertAttributeVectorString("InputSandbox",
inputSandbox)
return S_OK()
