def __uploadInputSandbox( self, classAdJob, jobDescriptionObject = None ): """Checks the validity of the job Input Sandbox. The function returns the list of Input Sandbox files. The total volume of the input sandbox is evaluated """ inputSandbox = self.__getInputSandboxEntries( classAdJob ) realFiles = [] badFiles = [] diskFiles = [] for isFile in inputSandbox: valid = True for tag in ( 'lfn:', 'LFN:', 'SB:', '%s' ): # in case of parametric input sandbox, there is %s passed, so have to ignore it also if isFile.find( tag ) == 0: valid = False break if valid: realFiles.append( isFile ) stringIOFiles = [] stringIOFilesSize = 0 if jobDescriptionObject is not None: if isinstance( jobDescriptionObject, StringIO.StringIO ): stringIOFiles = [jobDescriptionObject] stringIOFilesSize = len( jobDescriptionObject.buf ) gLogger.debug( "Size of the stringIOFiles: " + str( stringIOFilesSize ) ) else: return S_ERROR( "jobDescriptionObject is not a StringIO object" ) # Check real files for isFile in realFiles: if not os.path.exists( isFile ): # we are passing in real files, we expect them to be on disk badFiles.append( isFile ) gLogger.warn( "inputSandbox file/directory " + isFile + " not found. Keep looking for the others" ) continue diskFiles.append( isFile ) diskFilesSize = File.getGlobbedTotalSize( diskFiles ) gLogger.debug( "Size of the diskFiles: " + str( diskFilesSize ) ) totalSize = diskFilesSize + stringIOFilesSize gLogger.verbose( "Total size of the inputSandbox: " + str( totalSize ) ) okFiles = stringIOFiles + diskFiles if badFiles: result = S_ERROR( 'Input Sandbox is not valid' ) result['BadFile'] = badFiles result['TotalSize'] = totalSize return result if okFiles: if not self.sandboxClient: self.sandboxClient = SandboxStoreClient( useCertificates = self.useCertificates ) result = self.sandboxClient.uploadFilesAsSandbox( okFiles ) if not result[ 'OK' ]: return result inputSandbox.append( result[ 'Value' ] ) classAdJob.insertAttributeVectorString( "InputSandbox", inputSandbox ) return S_OK()
def __uploadInputSandbox(self, classAdJob, jobDescriptionObject=None): """Checks the validity of the job Input Sandbox. The function returns the list of Input Sandbox files. The total volume of the input sandbox is evaluated """ inputSandbox = self.__getInputSandboxEntries(classAdJob) realFiles = [] badFiles = [] diskFiles = [] for isFile in inputSandbox: if not isFile.startswith(("lfn:", "LFN:", "SB:", "%s", "%(")): realFiles.append(isFile) stringIOFiles = [] stringIOFilesSize = 0 if jobDescriptionObject is not None: if isinstance(jobDescriptionObject, StringIO): stringIOFiles = [jobDescriptionObject] stringIOFilesSize = len(jobDescriptionObject.getvalue()) gLogger.debug("Size of the stringIOFiles: " + str(stringIOFilesSize)) else: return S_ERROR(EWMSJDL, "jobDescriptionObject is not a StringIO object") # Check real files for isFile in realFiles: if not os.path.exists(isFile): # we are passing in real files, we expect them to be on disk badFiles.append(isFile) gLogger.warn("inputSandbox file/directory " + isFile + " not found. Keep looking for the others") continue diskFiles.append(isFile) diskFilesSize = File.getGlobbedTotalSize(diskFiles) gLogger.debug("Size of the diskFiles: " + str(diskFilesSize)) totalSize = diskFilesSize + stringIOFilesSize gLogger.verbose("Total size of the inputSandbox: " + str(totalSize)) okFiles = stringIOFiles + diskFiles if badFiles: result = S_ERROR(EWMSJDL, "Input Sandbox is not valid") result["BadFile"] = badFiles result["TotalSize"] = totalSize return result if okFiles: if not self.sandboxClient: self.sandboxClient = SandboxStoreClient( useCertificates=self.useCertificates, delegatedDN=self.delegatedDN, delegatedGroup=self.delegatedGroup, ) result = self.sandboxClient.uploadFilesAsSandbox(okFiles) if not result["OK"]: return result inputSandbox.append(result["Value"]) classAdJob.insertAttributeVectorString("InputSandbox", inputSandbox) return S_OK()
def __uploadInputSandbox(self, classAdJob): """Checks the validity of the job Input Sandbox. The function returns the list of Input Sandbox files. The total volume of the input sandbox is evaluated """ sandboxClient = SandboxStoreClient( useCertificates=self.useCertificates, rpcClient=self.sbRPCClient, transferClient=self.sbTransferClient ) inputSandbox = self.__getInputSandboxEntries(classAdJob) realFiles = [] badFiles = [] okFiles = [] realFiles = [] for file in inputSandbox: valid = True for tag in ( "lfn:", "LFN:", "SB:", "%s", ): # in case of parametric input sandbox, there is %s passed, so have to ignore it also if file.find(tag) == 0: valid = False break if valid: realFiles.append(file) # If there are no files, skip! if not realFiles: return S_OK() # Check real files for file in realFiles: if not os.path.exists(file): badFiles.append(file) print "inputSandbox file/directory " + file + " not found" continue okFiles.append(file) # print "Total size of the inputSandbox: "+str(totalSize) totalSize = File.getGlobbedTotalSize(okFiles) if badFiles: result = S_ERROR("Input Sandbox is not valid") result["BadFile"] = badFiles result["TotalSize"] = totalSize return result if okFiles: result = sandboxClient.uploadFilesAsSandbox(okFiles) if not result["OK"]: return result inputSandbox.append(result["Value"]) classAdJob.insertAttributeVectorString("InputSandbox", inputSandbox) return S_OK()
def __uploadInputSandbox(self, classAdJob): """Checks the validity of the job Input Sandbox. The function returns the list of Input Sandbox files. The total volume of the input sandbox is evaluated """ sandboxClient = SandboxStoreClient( useCertificates=self.useCertificates, rpcClient=self.sbRPCClient, transferClient=self.sbTransferClient) inputSandbox = self.__getInputSandboxEntries(classAdJob) realFiles = [] badFiles = [] okFiles = [] realFiles = [] for file in inputSandbox: valid = True for tag in ( 'lfn:', 'LFN:', 'SB:', '%s' ): #in case of parametric input sandbox, there is %s passed, so have to ignore it also if file.find(tag) == 0: valid = False break if valid: realFiles.append(file) #If there are no files, skip! if not realFiles: return S_OK() #Check real files for file in realFiles: if not os.path.exists(file): badFiles.append(file) print "inputSandbox file/directory " + file + " not found" continue okFiles.append(file) #print "Total size of the inputSandbox: "+str(totalSize) totalSize = File.getGlobbedTotalSize(okFiles) if badFiles: result = S_ERROR('Input Sandbox is not valid') result['BadFile'] = badFiles result['TotalSize'] = totalSize return result if okFiles: result = sandboxClient.uploadFilesAsSandbox(okFiles) if not result['OK']: return result inputSandbox.append(result['Value']) classAdJob.insertAttributeVectorString("InputSandbox", inputSandbox) return S_OK()
def __uploadInputSandbox( self, classAdJob ): """Checks the validity of the job Input Sandbox. The function returns the list of Input Sandbox files. The total volume of the input sandbox is evaluated """ inputSandbox = self.__getInputSandboxEntries( classAdJob ) badFiles = [] okFiles = [] realFiles = [] for isFile in inputSandbox: valid = True for tag in ( 'lfn:', 'LFN:', 'SB:', '%s' ): # in case of parametric input sandbox, there is %s passed, so have to ignore it also if isFile.find( tag ) == 0: valid = False break if valid: realFiles.append( isFile ) # If there are no files, skip! if not realFiles: return S_OK() # Check real files for isFile in realFiles: if not os.path.exists( isFile ): badFiles.append( isFile ) gLogger.warn( "inputSandbox file/directory " + isFile + " not found. Keep looking for the others" ) continue okFiles.append( isFile ) totalSize = File.getGlobbedTotalSize( okFiles ) gLogger.verbose( "Total size of the inputSandbox: " + str( totalSize ) ) if badFiles: result = S_ERROR( 'Input Sandbox is not valid' ) result['BadFile'] = badFiles result['TotalSize'] = totalSize return result if okFiles: if not self.sandboxClient: self.sandboxClient = SandboxStoreClient( useCertificates = self.useCertificates ) result = self.sandboxClient.uploadFilesAsSandbox( okFiles ) if not result[ 'OK' ]: return result inputSandbox.append( result[ 'Value' ] ) classAdJob.insertAttributeVectorString( "InputSandbox", inputSandbox ) return S_OK()
def __uploadInputSandbox(self, classAdJob, jobDescriptionObject=None): """Checks the validity of the job Input Sandbox. The function returns the list of Input Sandbox files. The total volume of the input sandbox is evaluated """ inputSandbox = self.__getInputSandboxEntries(classAdJob) realFiles = [] badFiles = [] diskFiles = [] for isFile in inputSandbox: valid = True for tag in ( 'lfn:', 'LFN:', 'SB:', '%s' ): # in case of parametric input sandbox, there is %s passed, so have to ignore it also if isFile.find(tag) == 0: valid = False break if valid: realFiles.append(isFile) stringIOFiles = [] stringIOFilesSize = 0 if jobDescriptionObject is not None: if isinstance(jobDescriptionObject, StringIO.StringIO): stringIOFiles = [jobDescriptionObject] stringIOFilesSize = len(jobDescriptionObject.buf) gLogger.debug("Size of the stringIOFiles: " + str(stringIOFilesSize)) else: return S_ERROR("jobDescriptionObject is not a StringIO object") # Check real files for isFile in realFiles: if not os.path.exists( isFile ): # we are passing in real files, we expect them to be on disk badFiles.append(isFile) gLogger.warn("inputSandbox file/directory " + isFile + " not found. Keep looking for the others") continue diskFiles.append(isFile) diskFilesSize = File.getGlobbedTotalSize(diskFiles) gLogger.debug("Size of the diskFiles: " + str(diskFilesSize)) totalSize = diskFilesSize + stringIOFilesSize gLogger.verbose("Total size of the inputSandbox: " + str(totalSize)) okFiles = stringIOFiles + diskFiles if badFiles: result = S_ERROR('Input Sandbox is not valid') result['BadFile'] = badFiles result['TotalSize'] = totalSize return result if okFiles: if not self.sandboxClient: self.sandboxClient = SandboxStoreClient( useCertificates=self.useCertificates) result = self.sandboxClient.uploadFilesAsSandbox(okFiles) if not result['OK']: return result inputSandbox.append(result['Value']) classAdJob.insertAttributeVectorString("InputSandbox", inputSandbox) return S_OK()