def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): log = self._log.getSubLogger("_prepareNewJobs", child=True) filesToSubmit = self._getFilesToSubmit( maxAttemptsPerFile=maxAttemptsPerFile) log.debug("%s ftsFiles to submit" % len(filesToSubmit)) newJobs = [] # {targetSE : [FTS3Files] } res = FTS3Utilities.groupFilesByTarget(filesToSubmit) if not res['OK']: return res filesGroupedByTarget = res['Value'] for targetSE, ftsFiles in filesGroupedByTarget.iteritems(): res = self._checkSEAccess(targetSE, 'WriteAccess', vo=self.vo) if not res['OK']: # If the SE is currently banned, we just skip it if cmpError(res, errno.EACCES): log.info( "Write access currently not permitted to %s, skipping." % targetSE) else: log.error(res) for ftsFile in ftsFiles: ftsFile.attempt += 1 continue sourceSEs = self.sourceSEs.split( ',') if self.sourceSEs is not None else [] # { sourceSE : [FTSFiles] } res = FTS3Utilities.selectUniqueRandomSource( ftsFiles, allowedSources=sourceSEs) if not res['OK']: return res uniqueTransfersBySource = res['Value'] # We don't need to check the source, since it is already filtered by the DataManager for sourceSE, ftsFiles in uniqueTransfersBySource.iteritems(): for ftsFilesChunk in breakListIntoChunks( ftsFiles, maxFilesPerJob): newJob = self._createNewJob('Transfer', ftsFilesChunk, targetSE, sourceSE=sourceSE) newJobs.append(newJob) return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): log = self._log.getSubLogger("_prepareNewJobs", child=True) filesToSubmit = self._getFilesToSubmit(maxAttemptsPerFile=maxAttemptsPerFile) log.debug("%s ftsFiles to submit" % len(filesToSubmit)) newJobs = [] # {targetSE : [FTS3Files] } res = FTS3Utilities.groupFilesByTarget(filesToSubmit) if not res['OK']: return res filesGroupedByTarget = res['Value'] for targetSE, ftsFiles in filesGroupedByTarget.iteritems(): res = self._checkSEAccess(targetSE, 'WriteAccess', vo=self.vo) if not res['OK']: # If the SE is currently banned, we just skip it if cmpError(res, errno.EACCES): log.info("Write access currently not permitted to %s, skipping." % targetSE) else: log.error(res) for ftsFile in ftsFiles: ftsFile.attempt += 1 continue sourceSEs = self.sourceSEs.split(',') if self.sourceSEs is not None else [] # { sourceSE : [FTSFiles] } res = FTS3Utilities.selectUniqueRandomSource(ftsFiles, allowedSources=sourceSEs) if not res['OK']: return res uniqueTransfersBySource = res['Value'] # We don't need to check the source, since it is already filtered by the DataManager for sourceSE, ftsFiles in uniqueTransfersBySource.iteritems(): for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob): newJob = self._createNewJob('Transfer', ftsFilesChunk, targetSE, sourceSE=sourceSE) newJobs.append(newJob) return S_OK(newJobs)
def __readConf(self): """ read configurations """ # Getting all the possible servers res = getFTS3ServerDict() if not res['OK']: gLogger.error(res['Message']) return res srvDict = res['Value'] serverPolicyType = opHelper().getValue('DataManagement/FTSPlacement/FTS3/ServerPolicy', 'Random') self._serverPolicy = FTS3Utilities.FTS3ServerPolicy(srvDict, serverPolicy=serverPolicyType) self.maxNumberOfThreads = self.am_getOption("MaxThreads", 10) # Number of Operation we treat in one loop self.operationBulkSize = self.am_getOption("OperationBulkSize", 20) # Number of Jobs we treat in one loop self.jobBulkSize = self.am_getOption("JobBulkSize", 20) self.maxFilesPerJob = self.am_getOption("MaxFilesPerJob", 100) self.maxAttemptsPerFile = self.am_getOption("MaxAttemptsPerFile", 256) self.kickDelay = self.am_getOption("KickAssignedHours", 1) self.maxKick = self.am_getOption("KickLimitPerCycle", 100) self.deleteDelay = self.am_getOption("DeleteGraceDays", 180) self.maxDelete = self.am_getOption("DeleteLimitPerCycle", 100) return S_OK()
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): log = gLogger.getSubLogger("_prepareNewJobs", child=True) filesToSubmit = self._getFilesToSubmit(maxAttemptsPerFile=maxAttemptsPerFile) log.debug("%s ftsFiles to submit" % len(filesToSubmit)) newJobs = [] # {targetSE : [FTS3Files] } filesGroupedByTarget = FTS3Utilities.groupFilesByTarget(filesToSubmit) for targetSE, ftsFiles in filesGroupedByTarget.iteritems(): res = self._checkSEAccess(targetSE, 'ReadAccess', vo=self.vo) if not res['OK']: log.error(res) continue for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob): newJob = self._createNewJob('Staging', ftsFilesChunk, targetSE, sourceSE=targetSE) newJobs.append(newJob) return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): log = gLogger.getSubLogger("_prepareNewJobs", child=True) filesToSubmit = self._getFilesToSubmit( maxAttemptsPerFile=maxAttemptsPerFile) log.debug("%s ftsFiles to submit" % len(filesToSubmit)) newJobs = [] # {targetSE : [FTS3Files] } filesGroupedByTarget = FTS3Utilities.groupFilesByTarget(filesToSubmit) for targetSE, ftsFiles in filesGroupedByTarget.iteritems(): res = self._checkSEAccess(targetSE, 'ReadAccess', vo=self.vo) if not res['OK']: log.error(res) continue for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob): newJob = self._createNewJob('Staging', ftsFilesChunk, targetSE, sourceSE=targetSE) newJobs.append(newJob) return S_OK(newJobs)
def __readConf(self): """Read configurations :return: S_OK()/S_ERROR() """ # Getting all the possible servers res = getFTS3ServerDict() if not res["OK"]: gLogger.error(res["Message"]) return res srvDict = res["Value"] serverPolicyType = opHelper().getValue("DataManagement/FTSPlacement/FTS3/ServerPolicy", "Random") self._serverPolicy = FTS3Utilities.FTS3ServerPolicy(srvDict, serverPolicy=serverPolicyType) self.maxNumberOfThreads = self.am_getOption("MaxThreads", 10) # Number of Operation we treat in one loop self.operationBulkSize = self.am_getOption("OperationBulkSize", 20) # Number of Jobs we treat in one loop self.jobBulkSize = self.am_getOption("JobBulkSize", 20) self.maxFilesPerJob = self.am_getOption("MaxFilesPerJob", 100) self.maxAttemptsPerFile = self.am_getOption("MaxAttemptsPerFile", 256) self.kickDelay = self.am_getOption("KickAssignedHours", 1) self.maxKick = self.am_getOption("KickLimitPerCycle", 100) self.deleteDelay = self.am_getOption("DeleteGraceDays", 180) self.maxDelete = self.am_getOption("DeleteLimitPerCycle", 100) # lifetime of the proxy we download to delegate to FTS self.proxyLifetime = self.am_getOption("ProxyLifetime", PROXY_LIFETIME) return S_OK()
def init_on_load(self): """This method initializes some attributes. It is called by sqlalchemy (which does not call __init__) """ self._vo = None # Note that in the case of an FTS3Operation created from an RMS # object, the members here will probably be "wrong" in the sense # that the VO will not be known by then. # It does not really matter however, since we do not perform anything # on an operation created this way, it's just to be then serialized # in the DB. self.dManager = DataManager() self.rssClient = ResourceStatus() self.fts3Plugin = FTS3Utilities.getFTS3Plugin(vo=self.vo) opID = getattr(self, "operationID", None) loggerName = "%s/" % opID if opID else "" loggerName += "req_%s/op_%s" % (self.rmsReqID, self.rmsOpID) self._log = gLogger.getSubLogger(loggerName)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): log = self._log.getSubLogger("_prepareNewJobs", child=True) filesToSubmit = self._getFilesToSubmit( maxAttemptsPerFile=maxAttemptsPerFile) log.debug("%s ftsFiles to submit" % len(filesToSubmit)) newJobs = [] # {targetSE : [FTS3Files] } res = FTS3Utilities.groupFilesByTarget(filesToSubmit) if not res['OK']: return res filesGroupedByTarget = res['Value'] for targetSE, ftsFiles in filesGroupedByTarget.items(): res = self._checkSEAccess(targetSE, 'WriteAccess', vo=self.vo) if not res['OK']: # If the SE is currently banned, we just skip it if cmpError(res, errno.EACCES): log.info( "Write access currently not permitted to %s, skipping." % targetSE) else: log.error(res) for ftsFile in ftsFiles: ftsFile.attempt += 1 continue sourceSEs = self.sourceSEs.split( ',') if self.sourceSEs is not None else [] # { sourceSE : [FTSFiles] } res = FTS3Utilities.selectUniqueRandomSource( ftsFiles, allowedSources=sourceSEs) if not res['OK']: return res uniqueTransfersBySource, failedFiles = res['Value'] # Treat the errors of the failed files for ftsFile, errMsg in failedFiles.items(): log.error("Error when selecting random sources", "%s, %s" % (ftsFile.lfn, errMsg)) # If the error is that the file does not exist in the catalog # fail it ! if cmpError(errMsg, errno.ENOENT): log.error("The file does not exist, setting it Defunct", "%s" % ftsFile.lfn) ftsFile.status = 'Defunct' # We don't need to check the source, since it is already filtered by the DataManager for sourceSE, ftsFiles in uniqueTransfersBySource.items(): if self.__needsMultiHopStaging(sourceSE, targetSE): log.verbose( "Needs multihop staging, max files per job is 1") maxFilesPerJob = 1 for ftsFilesChunk in breakListIntoChunks( ftsFiles, maxFilesPerJob): newJob = self._createNewJob('Transfer', ftsFilesChunk, targetSE, sourceSE=sourceSE) newJobs.append(newJob) return S_OK(newJobs)
def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): log = self._log.getSubLogger("_prepareNewJobs") filesToSubmit = self._getFilesToSubmit( maxAttemptsPerFile=maxAttemptsPerFile) log.debug("%s ftsFiles to submit" % len(filesToSubmit)) newJobs = [] # {targetSE : [FTS3Files] } res = FTS3Utilities.groupFilesByTarget(filesToSubmit) if not res["OK"]: return res filesGroupedByTarget = res["Value"] for targetSE, ftsFiles in filesGroupedByTarget.items(): res = self._checkSEAccess(targetSE, "WriteAccess", vo=self.vo) if not res["OK"]: # If the SE is currently banned, we just skip it if cmpError(res, errno.EACCES): log.info( "Write access currently not permitted to %s, skipping." % targetSE) else: log.error(res) for ftsFile in ftsFiles: ftsFile.attempt += 1 continue sourceSEs = self.sourceSEs.split( ",") if self.sourceSEs is not None else [] # { sourceSE : [FTSFiles] } res = FTS3Utilities.selectUniqueSource(ftsFiles, self.fts3Plugin, allowedSources=sourceSEs) if not res["OK"]: return res uniqueTransfersBySource, failedFiles = res["Value"] # Treat the errors of the failed files for ftsFile, errMsg in failedFiles.items(): log.error("Error when selecting random sources", "%s, %s" % (ftsFile.lfn, errMsg)) # If the error is that the file does not exist in the catalog # fail it ! if cmpError(errMsg, errno.ENOENT): log.error("The file does not exist, setting it Defunct", "%s" % ftsFile.lfn) ftsFile.status = "Defunct" # We don't need to check the source, since it is already filtered by the DataManager for sourceSE, ftsFiles in uniqueTransfersBySource.items(): # Checking whether we will need multiHop transfer multiHopSE = self.fts3Plugin.findMultiHopSEToCoverUpForWLCGFailure( sourceSE, targetSE) if multiHopSE: log.verbose( "WLCG failure manifestation, use %s for multihop, max files per job is 1" % multiHopSE) # Check that we can write and read from it try: for accessType in ("Read", "Write"): res = self._checkSEAccess(multiHopSE, "%sAccess" % accessType, vo=self.vo) if not res["OK"]: # If the SE is currently banned, we just skip it if cmpError(res, errno.EACCES): log.info( "Access currently not permitted", "%s to %s" % (accessType, multiHopSE)) else: log.error("CheckSEAccess error", res) for ftsFile in ftsFiles: ftsFile.attempt += 1 # If we have a problem with the multiHop SE, # we skip the whole loop for the pair # (targetSE, sourceSE) raise RuntimeError("MultiHopSE unavailable") except RuntimeError as e: log.info( "Problem with multiHop SE, skipping transfers from %s to %s." % (sourceSE, targetSE)) continue maxFilesPerJob = 1 # Check if we need a multihop staging elif self.__needsMultiHopStaging(sourceSE, targetSE): log.verbose( "Needs multihop staging, max files per job is 1") maxFilesPerJob = 1 for ftsFilesChunk in breakListIntoChunks( ftsFiles, maxFilesPerJob): newJob = self._createNewJob("Transfer", ftsFilesChunk, targetSE, sourceSE=sourceSE, multiHopSE=multiHopSE) newJobs.append(newJob) return S_OK(newJobs)