def __createFTSFiles( self ): """ create LFNs file for glite-transfer-submit command This file consists one line for each fiel to be transferred: sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM] :param self: self reference """ self.__updateMetadataCache() for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get( 'Status' ) if lfnStatus not in self.noSubmitStatus: cksmStr = "" # # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself if self.__cksmTest and self.__cksmType: checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' ) if checkSum: cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) ) ftsFile = FTSFile() ftsFile.LFN = lfn ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' ) ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' ) ftsFile.SourceSE = self.sourceSE ftsFile.TargetSE = self.targetSE ftsFile.Status = self.fileDict[lfn].get( 'Status' ) ftsFile.Checksum = cksmStr ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' ) self.ftsFiles.append( ftsFile ) self.submittedFiles += 1 return S_OK()
def test01Ctor( self ): """ test ctor and (de-)serialization """ ftsFile = FTSFile( self.fromDict ) self.assertEqual( isinstance( ftsFile, FTSFile ), True ) for k, v in self.fromDict.items(): self.assertEqual( getattr( ftsFile, k ), v ) json = ftsFile.toJSON() ftsFileJSON = FTSFile( json["Value"] ) self.assertEqual( isinstance( ftsFileJSON, FTSFile ), True ) for k, v in self.fromDict.items(): self.assertEqual( getattr( ftsFileJSON, k ), v )
def test01Ctor(self): """ test ctor and (de-)serialization """ ftsFile = FTSFile(self.fromDict) self.assertEqual(isinstance(ftsFile, FTSFile), True) for k, v in self.fromDict.items(): self.assertEqual(getattr(ftsFile, k), v) json = ftsFile.toJSON() ftsFileJSON = FTSFile(json["Value"]) self.assertEqual(isinstance(ftsFileJSON, FTSFile), True) for k, v in self.fromDict.items(): self.assertEqual(getattr(ftsFileJSON, k), v)
def getFTSJobList(self, statusList=None, limit=500): """ select FTS jobs with statuses in :statusList: """ statusList = statusList if statusList else list(FTSJob.INITSTATES + FTSJob.TRANSSTATES) query = "SELECT * FROM `FTSJob` WHERE `Status` IN (%s) ORDER BY `LastUpdate` DESC LIMIT %s;" % ( stringListToString(statusList), limit) trn = self._transaction([query]) if not trn['OK']: self.log.error('Failed ftsJobSQL', "getFTSJobList: %s" % trn['Message']) return trn ftsJobs = [FTSJob(ftsJobDict) for ftsJobDict in trn['Value'][query]] for ftsJob in ftsJobs: query = "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID trn = self._transaction(query) if not trn['OK']: self.log.error('Failed ftsFileSQL', "getFTSJobList: %s" % trn['Message']) return trn ftsFiles = [ FTSFile(ftsFileDict) for ftsFileDict in trn['Value'][query] ] for ftsFile in ftsFiles: ftsJob.addFile(ftsFile) return S_OK(ftsJobs)
def getFTSJob(self, ftsJobID=None, readOnly=False): """ get FTSJob given FTSJobID """ getJob = ["SELECT * FROM `FTSJob` WHERE `FTSJobID` = %s;" % ftsJobID] getJob = self._transaction(getJob) if not getJob["OK"]: self.log.error(getJob["Message"]) return getJob getJob = getJob["Value"] if not getJob: return S_OK() ftsJob = FTSJob(getJob.values()[0][0]) selectFiles = self._transaction([ "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID ]) if not selectFiles["OK"]: self.log.error(selectFiles["Message"]) return selectFiles selectFiles = selectFiles["Value"] ftsFiles = [FTSFile(item) for item in selectFiles.values()[0]] for ftsFile in ftsFiles: ftsJob.addFile(ftsFile) if not readOnly: setAssigned = "UPDATE `FTSJob` SET `Status`='Assigned' WHERE `FTSJobID` = %s;" % ftsJobID setAssigned = self._query(setAssigned) if not setAssigned["OK"]: self.log.error(setAssigned["Message"]) return setAssigned return S_OK(ftsJob)
def export_putFTSFileList(self, ftsFilesJSONList): """ put FTS files list """ ftsFiles = [] for ftsFileJSON in ftsFilesJSONList: ftsFiles.append(FTSFile(ftsFileJSON)) return self.ftsDB.putFTSFileList(ftsFiles)
def getFTSJobsForRequest(self, requestID, statusList=None): """ get list of FTSJobs with status in :statusList: for request given its requestID TODO: should be more smart, i.e. one query to select all ftsfiles """ statusList = statusList if statusList else list(FTSJob.INITSTATES + FTSJob.TRANSSTATES) query = "SELECT * FROM `FTSJob` WHERE `RequestID` = %s AND `Status` in (%s)" % ( requestID, stringListToString(statusList)) ftsJobs = self._transaction([query]) if not ftsJobs['OK']: self.log.error("Failed getFTSJobsForRequest", "%s" % ftsJobs['Message']) return ftsJobs ftsJobs = ftsJobs['Value'][query] if query in ftsJobs['Value'] else [] ftsJobs = [FTSJob(ftsJobDict) for ftsJobDict in ftsJobs] for ftsJob in ftsJobs: query = "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s' AND `RequestID`=%s" % ( ftsJob.FTSGUID, requestID) ftsFiles = self._transaction([query]) if not ftsFiles['OK']: self.log.error("Failed getFTSJobsForRequest", "%s" % ftsFiles['Message']) return ftsFiles ftsFiles = ftsFiles['Value'][query] if query in ftsFiles[ 'Value'] else [] for ftsFileDict in ftsFiles: ftsJob.addFile(FTSFile(ftsFileDict)) return S_OK(ftsJobs)
def __init__(self, fromDict=None): """c'tor :param self: self reference :param dict fromDict: data dict """ Record.__init__(self) now = datetime.datetime.utcnow().replace(microsecond=0) self.__data__["CreationTime"] = now self.__data__["SubmitTime"] = now self.__data__["LastUpdate"] = now self.__data__["Status"] = "Submitted" self.__data__["Completeness"] = 0 self.__data__["FTSJobID"] = 0 self.__files__ = TypedList(allowedTypes=FTSFile) self._log = gLogger.getSubLogger("FTSJob-%s" % self.FTSJobID, True) fromDict = fromDict if fromDict else {} for ftsFileDict in fromDict.get("FTSFiles", []): self += FTSFile(ftsFileDict) if "FTSFiles" in fromDict: del fromDict["FTSFiles"] for key, value in fromDict.items(): if key not in self.__data__: raise AttributeError("Unknown FTSJob attribute '%s'" % key) if value: setattr(self, key, value)
def getFTSFileList(self, statusList=None, limit=1000): """ get at most :limit: FTSFiles with status in :statusList: :param statusList: list with FTSFiles statuses :type statusList: python:list :param int limit: select query limit """ statusList = statusList if statusList else ["Waiting"] reStatus = [] inStatus = [] for status in statusList: if "%" in status or ".*" in status or ".+" in status: reStatus.append(status) else: inStatus.append(status) reQuery = "`Status` REGEXP '%s'" % "|".join( reStatus) if reStatus else "" inQuery = "`Status` IN (%s)" % stringListToString( inStatus) if inStatus else "" whereClause = " OR ".join([q for q in (reQuery, inQuery) if q]) if whereClause: whereClause = "WHERE %s" % whereClause query = "SELECT * FROM `FTSFile` %s ORDER BY `LastUpdate` DESC LIMIT %s;" % ( whereClause, limit) trn = self._transaction([query]) if not trn['OK']: self.log.error("Failed getFTSFileList", "%s" % trn['Message']) return trn return S_OK([FTSFile(fileDict) for fileDict in trn['Value'][query]])
def _getFTSFileProperties(self, ftsFileID, columnNames=None): """ select :columnNames: from FTSJobFile table """ columnNames = columnNames if columnNames else FTSFile.tableDesc( )["Fields"].keys() columnNames = ",".join( ['`%s`' % str(columnName) for columnName in columnNames]) return "SELECT %s FROM `FTSFile` WHERE `FTSFileID` = %s;" % ( columnNames, int(ftsFileID))
def getFTSFileList( self, statusList = None, limit = None ): """ get list of FTSFiles with status in statusList """ statusList = statusList if statusList else [ "Waiting" ] limit = limit if limit else 1000 getFTSFileList = self.ftsManager.getFTSFileList( statusList, limit ) if not getFTSFileList['OK']: self.log.error( "Failed getFTSFileList", "%s" % getFTSFileList['Message'] ) return getFTSFileList getFTSFileList = getFTSFileList['Value'] return S_OK( [ FTSFile( ftsFile ) for ftsFile in getFTSFileList ] )
def getAllFTSFilesForRequest( self, requestID ): """ read FTSFiles for a given :requestID: :param int requestID: ReqDB.Request.RequestID """ ftsFiles = self.ftsManager.getAllFTSFilesForRequest( requestID ) if not ftsFiles['OK']: self.log.error( "Failed getFTSFilesForRequest", "%s" % ftsFiles['Message'] ) return ftsFiles return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles['Value'] ] )
def getAllFTSFilesForRequest(self, requestID): """ get FTSFiles with status in :statusList: for request given its :requestID: """ requestID = int(requestID) query = "SELECT * FROM `FTSFile` WHERE `RequestID` = %s;" % (requestID) ftsFiles = self._transaction([query]) if not ftsFiles['OK']: self.log.error("getFTSFilesForRequest: %s" % ftsFiles['Message']) return ftsFiles ftsFiles = ftsFiles['Value'].get(query, []) return S_OK([FTSFile(ftsFileDict) for ftsFileDict in ftsFiles])
def getFTSFileList( self, statusList = None, limit = None ): """ get list of FTSFiles with status in statusList """ statusList = statusList if statusList else [ "Waiting" ] limit = limit if limit else 1000 getFTSFileList = self.ftsManager().getFTSFileList( statusList, limit ) if not getFTSFileList["OK"]: self.log.error( "getFTSFileList: %s" % getFTSFileList["Message"] ) return getFTSFileList getFTSFileList = getFTSFileList["Value"] return S_OK( [ FTSFile( ftsFile ) for ftsFile in getFTSFileList ] )
def getFTSFilesForRequest( self, requestID, statusList = None ): """ read FTSFiles for a given :requestID: :param int requestID: ReqDB.Request.RequestID :param list statusList: List of statuses (default: Waiting) """ ftsFiles = self.ftsManager.getFTSFilesForRequest( requestID, statusList ) if not ftsFiles['OK']: self.log.error( "Failed getFTSFilesForRequest", "%s" % ftsFiles['Message'] ) return ftsFiles return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles['Value'] ] )
def getFTSFilesForRequest( self, requestID, operationID = None ): """ read FTSFiles for a given :requestID: :param int requestID: ReqDB.Request.RequestID :param int operationID: ReqDB.Operation.OperationID """ ftsFiles = self.ftsManager().getFTSFilesForRequest( requestID, operationID ) if not ftsFiles["OK"]: self.log.error( "getFTSFilesForRequest: %s" % ftsFiles["Message"] ) return ftsFiles return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles["Value"] ] )
def getFTSFile( self, ftsFileID = None ): """ get FTSFile :param int ftsFileID: FTSFileID """ getFile = self.ftsManager.getFTSFile( ftsFileID ) if not getFile['OK']: self.log.error( 'Failed to get FTS file', getFile['Message'] ) # # de-serialize if getFile['Value']: ftsFile = FTSFile( getFile['Value'] ) return S_OK( ftsFile )
def getFTSFilesForRequest(self, requestID, statusList=None): """ get FTSFiles with status in :statusList: for request given its :requestID: """ requestID = int(requestID) statusList = statusList if statusList else ["Waiting"] query = "SELECT * FROM `FTSFile` WHERE `RequestID` = %s AND `Status` IN (%s);" % ( requestID, stringListToString(statusList)) ftsFiles = self._transaction([query]) if not ftsFiles['OK']: self.log.error("getFTSFilesForRequest: %s" % ftsFiles['Message']) return ftsFiles ftsFiles = ftsFiles['Value'].get(query, []) return S_OK([FTSFile(ftsFileDict) for ftsFileDict in ftsFiles])
def export_putFTSFile(cls, ftsFileJSON): """ put FTSFile into FTSDB """ try: ftsFile = FTSFile(ftsFileJSON) isValid = cls.ftsValidator().validate(ftsFile) if not isValid["OK"]: gLogger.error(isValid["Message"]) return isValid return cls.__ftsDB.putFTSFile(ftsFile) except Exception, error: gLogger.exception(error) return S_ERROR(error)
def getFTSFile(self, ftsFileID): """ read FTSFile from db given FTSFileID """ select = "SELECT * FROM `FTSFile` WHERE `FTSFileID` = %s;" % ftsFileID select = self._transaction([select]) if not select["OK"]: self.log.error(select["Message"]) return select select = select["Value"] if not select.values()[0]: return S_OK() ftsFile = FTSFile(select.values()[0][0]) return S_OK(ftsFile)
def getFTSFile(self, ftsFileID): """ read FTSFile from db given FTSFileID """ select = "SELECT * FROM `FTSFile` WHERE `FTSFileID` = %s;" % ftsFileID select = self._transaction([select]) if not select['OK']: self.log.error('Failed FTS file selection', select['Message']) return select select = select['Value'] if not select.values()[0]: return S_OK() ftsFile = FTSFile(select.values()[0][0]) return S_OK(ftsFile)
def setUp(self): """ test set up """ self.fileA = FTSFile({ "LFN": "/a", "ChecksumType": "ADLER32", "Checksum": "123456", "Size": 10, "SourceSE": "CERN-USER", "TargetSE": "PIC-USER", "SourceSURL": "sourceSURL", "TargetSURL": "targetSURL" }) self.fileB = FTSFile({ "LFN": "/b", "ChecksumType": "ADLER32", "Checksum": "654321", "Size": 10, "SourceSE": "CERN-USER", "TargetSE": "PIC-USER", "SourceSURL": "sourceSURL", "TargetSURL": "targetSURL" })
def getFTSFile( self, ftsFileID = None ): """ get FTSFile :param int fileID: FileID :param int ftsFileID: FTSFileID """ getFile = self.ftsManager().getFTSFile( ftsFileID ) if not getFile["OK"]: self.log.error( getFile["Message"] ) # # de-serialize if getFile["Value"]: getFile = FTSFile( getFile["Value"] ) if not getFile["OK"]: self.log.error( getFile["Message"] ) return getFile
def export_putFTSJob(self, ftsJobJSON): """ put FTSJob (serialized in JSON into FTSDB """ ftsFiles = [] if "FTSFiles" in ftsJobJSON: ftsFiles = ftsJobJSON.get("FTSFiles", []) del ftsJobJSON["FTSFiles"] try: ftsJob = FTSJob(ftsJobJSON) for ftsFile in ftsFiles: ftsJob.addFile(FTSFile(ftsFile)) except Exception, error: gLogger.exception(error) return S_ERROR(error)
def __init__(self, fromDict=None): """c'tor :param self: self reference :param dict fromDict: data dict """ self.__data__ = dict.fromkeys(self.tableDesc()["Fields"].keys(), None) now = datetime.datetime.utcnow().replace(microsecond=0) self.__data__["CreationTime"] = now self.__data__["SubmitTime"] = now self.__data__["LastUpdate"] = now self.__data__["Status"] = "Submitted" self.__data__["Completeness"] = 0 self.__data__["FTSJobID"] = 0 self._regTime = 0. self._regSuccess = 0 self._regTotal = 0 self.__files__ = TypedList(allowedTypes=FTSFile) self._fc = FileCatalog() self._fts3context = None self._states = tuple( set(self.INITSTATES + self.TRANSSTATES + self.FAILEDSTATES + self.FINALSTATES)) fromDict = fromDict if fromDict else {} for ftsFileDict in fromDict.get("FTSFiles", []): self += FTSFile(ftsFileDict) if "FTSFiles" in fromDict: del fromDict["FTSFiles"] for key, value in fromDict.items(): if key not in self.__data__: raise AttributeError("Unknown FTSJob attribute '%s'" % key) if value: setattr(self, key, value) self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % (self.RequestID, self.FTSGUID), True)
def getFTSJob(self, ftsJobID=None): """ get FTSJob given FTSJobID """ getJob = ["SELECT * FROM `FTSJob` WHERE `FTSJobID` = %s;" % ftsJobID] getJob = self._transaction(getJob) if not getJob['OK']: self.log.error('Failed ftsFileSQL', getJob['Message']) return getJob getJob = getJob['Value'] if not getJob: return S_OK() ftsJob = FTSJob(getJob.values()[0][0]) selectFiles = self._transaction([ "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID ]) if not selectFiles['OK']: self.log.error('Failed ftsFileSQL', selectFiles['Message']) return selectFiles selectFiles = selectFiles['Value'] ftsFiles = [FTSFile(item) for item in selectFiles.values()[0]] for ftsFile in ftsFiles: ftsJob.addFile(ftsFile) return S_OK(ftsJob)
def monitorFTS2(self, command="glite-transfer-status", full=False): """ monitor fts job """ if not self.FTSGUID: return S_ERROR("FTSGUID not set, FTS job not submitted?") monitorCommand = command.split() + \ ["--verbose", "-s", self.FTSServer, self.FTSGUID ] if full: monitorCommand.append("-l") monitor = executeGridCommand("", monitorCommand) if not monitor["OK"]: return monitor returnCode, outputStr, errStr = monitor["Value"] # Returns a non zero status if error if returnCode != 0: if 'was not found' in outputStr and not errStr: errStr = 'Job was not found' return S_ERROR(errStr) outputStr = outputStr.replace("'", "").replace("<", "").replace(">", "") # # set FTS job status regExp = re.compile("Status:\\s+(\\S+)") # with FTS3 this can be uppercase self.Status = re.search(regExp, outputStr).group(1) statusSummary = {} # This is capitalized, even in FTS3! for state in FTSFile.ALL_STATES: regExp = re.compile("\\s+%s:\\s+(\\d+)" % state) if regExp.search(outputStr): statusSummary[state] = int( re.search(regExp, outputStr).group(1)) total = sum(statusSummary.values()) completed = sum( statusSummary.get(state, 0) for state in FTSFile.FINAL_STATES) self.Completeness = 100 * completed / total if total else 0 if not full: return S_OK(statusSummary) # The order of informations is not the same for glite- and fts- !!! # In order: new fts-, old fts-, glite- realJob = len(self) != 0 iExptr = None for iExptr, exptr in enumerate( ('[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)' )): regExp = re.compile(exptr, re.S) fileInfo = re.findall(regExp, outputStr) if fileInfo: break if not fileInfo: return S_ERROR("Error monitoring job (no regexp match)") for info in fileInfo: if iExptr == 0: # version >= 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info elif iExptr == 1: # version FTS3 < 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries = info elif iExptr == 2: # version FTS2 sourceURL, targetURL, fileStatus, _retries, reason, duration = info else: return S_ERROR('Error monitoring job (implement match %d)' % iExptr) candidateFile = None if not realJob: # This is used by the CLI monitoring of jobs in case no file was specified candidateFile = FTSFile() candidateFile.LFN = overlap(sourceURL, targetURL) candidateFile.SourceSURL = sourceURL candidateFile.Size = 0 self += candidateFile else: for ftsFile in self: if ftsFile.SourceSURL == sourceURL: candidateFile = ftsFile break if not candidateFile: continue # Can be uppercase for FTS3 if not candidateFile.TargetSURL: candidateFile.TargetSURL = targetURL candidateFile.Status = fileStatus candidateFile.Error = reason candidateFile._duration = duration if candidateFile.Status == "Failed": for missingSource in self.missingSourceErrors: if missingSource.match(reason): candidateFile.Error = "MissingSource" # If the staging info was present, record it if len(info) > 6: candidateFile._staging = info[6] # # register successful files if self.Status in FTSJob.FINALSTATES: return self.finalize() return S_OK()
def setUp(self): """ test case set up """ gLogger.setLevel('NOTICE') self.ftsSites = [ FTSSite( ftsServer= 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name='CERN.ch'), FTSSite( ftsServer= 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name='PIC.es'), FTSSite( ftsServer= 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name='RAL.uk'), ] self.ses = ['CERN-USER', 'RAL-USER'] self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ] self.submitted = 0 self.numberOfJobs = 10 self.opIDs = [] self.ftsJobs = [] for i in xrange(self.numberOfJobs): opID = i % 3 if opID not in self.opIDs: self.opIDs.append(opID) ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = self.statuses[i % len(self.statuses)] ftsJob.OperationID = opID if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == 'Active': ftsJob.Completeness = 90 ftsJob.SourceSE = self.ses[i % len(self.ses)] ftsJob.TargetSE = 'PIC-USER' ftsJob.RequestID = 12345 ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = '/a/b/c/%d' % i ftsFile.Size = 1000000 ftsFile.OperationID = opID ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed' ftsFile.RequestID = 12345 ftsFile.Checksum = 'addler' ftsFile.ChecksumType = 'adler32' ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == 'FinishedDirty': ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile(ftsFile) self.ftsJobs.append(ftsJob) self.submitted = len( [i for i in self.ftsJobs if i.Status == 'Submitted']) self.ftsClient = FTSClient()
def setUp(self): """ test case set up """ # ## set some defaults gConfig.setOptionValue('DIRAC/Setup', 'Test') gConfig.setOptionValue('/DIRAC/Setups/Test/DataManagement', 'Test') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac') self.ftsSites = [ FTSSite({ "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" }), FTSSite({ "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" }), FTSSite({ "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" }), FTSSite({ "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" }), FTSSite({ "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" }), FTSSite({ "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" }), FTSSite({ "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" }), FTSSite({ "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" }) ] self.ftsFiles = [] for i in range(100): ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = 9999 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 10 ftsFile.SourceSE = "CERN-USER" ftsFile.TargetSE = "PIC-USER" ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" self.ftsFiles.append(ftsFile) ses = ["CERN-USER", "RAL-USER"] statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ] self.submitted = 0 self.ftsJobs = [] for i in range(7200): ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = statuses[i % len(statuses)] if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == "Active": ftsJob.Completeness = 90 ftsJob.SourceSE = ses[i % len(ses)] ftsJob.TargetSE = "PIC-USER" ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 1000000 ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed" ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == "FinishedDirty": ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile(ftsFile) self.ftsJobs.append(ftsJob) self.submitted = len( [i for i in self.ftsJobs if i.Status == "Submitted"])
def _getFTSFileProperties( self, ftsFileID, columnNames = None ): """ select :columnNames: from FTSJobFile table """ columnNames = columnNames if columnNames else FTSFile.tableDesc()["Fields"].keys() columnNames = ",".join( [ '`%s`' % str( columnName ) for columnName in columnNames ] ) return "SELECT %s FROM `FTSFile` WHERE `FTSFileID` = %s;" % ( columnNames, int( ftsFileID ) )
def export_ftsSchedule(self, requestID, operationID, fileJSONList): """ call FTS scheduler :param int requestID: ReqDB.Request.RequestID :param int operationID: ReqDB.Operation.OperationID :param list fileJSONList: [ (fileJSON, [sourceSE, ...], [ targetSE, ...] ) ] """ # # this will be returned on success ret = {"Successful": [], "Failed": {}} requestID = int(requestID) operationID = int(operationID) fileIDs = [] for fileJSON, sourceSEs, targetSEs in fileJSONList: fileID = int(fileJSON.get("FileID")) fileIDs.append(fileID) cleanUpFTSFiles = self.__ftsDB.cleanUpFTSFiles(requestID, fileIDs) if not cleanUpFTSFiles["OK"]: self.log.error("ftsSchedule: %s" % cleanUpFTSFiles["Message"]) return S_ERROR(cleanUpFTSFiles["Message"]) ftsFiles = [] for fileJSON, sourceSEs, targetSEs in fileJSONList: lfn = fileJSON.get("LFN", "") size = int(fileJSON.get("Size", 0)) fileID = int(fileJSON.get("FileID", 0)) opID = int(fileJSON.get("OperationID", 0)) gLogger.info( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % (lfn, fileID, opID, sourceSEs, targetSEs)) replicaDict = self.replicaManager().getActiveReplicas(lfn) if not replicaDict["OK"]: gLogger.error("ftsSchedule: %s" % replicaDict["Message"]) ret["Failed"][fileID] = replicaDict["Message"] continue replicaDict = replicaDict["Value"] if lfn in replicaDict["Failed"] and lfn not in replicaDict[ "Successful"]: ret["Failed"][fileID] = "no active replicas found" continue replicaDict = replicaDict["Successful"][lfn] if lfn in replicaDict[ "Successful"] else {} # # use valid replicas only replicaDict = dict([(se, pfn) for se, pfn in replicaDict.items() if se in sourceSEs]) if not replicaDict: ret["Failed"][fileID] = "no active replicas found in sources" continue tree = self.ftsStrategy().replicationTree(sourceSEs, targetSEs, size) if not tree["OK"]: gLogger.error("ftsSchedule: %s cannot be scheduled: %s" % (lfn, tree["Message"])) ret["Failed"][fileID] = tree["Message"] continue tree = tree["Value"] gLogger.info("LFN=%s tree=%s" % (lfn, tree)) for repDict in tree.values(): gLogger.info( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % (repDict["Strategy"], repDict["Ancestor"], repDict["SourceSE"], repDict["TargetSE"])) transferSURLs = self._getTransferURLs(lfn, repDict, sourceSEs, replicaDict) if not transferSURLs["OK"]: ret["Failed"][fileID] = transferSURLs["Message"] continue sourceSURL, targetSURL, fileStatus = transferSURLs["Value"] if sourceSURL == targetSURL: ret["Failed"][ fileID] = "sourceSURL equals to targetSURL for %s" % lfn continue gLogger.info("sourceURL=%s targetURL=%s FTSFile.Status=%s" % (sourceSURL, targetSURL, fileStatus)) ftsFile = FTSFile() for key in ("LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size"): setattr(ftsFile, key, fileJSON.get(key)) ftsFile.RequestID = requestID ftsFile.OperationID = operationID ftsFile.SourceSURL = sourceSURL ftsFile.TargetSURL = targetSURL ftsFile.SourceSE = repDict["SourceSE"] ftsFile.TargetSE = repDict["TargetSE"] ftsFile.Status = fileStatus ftsFiles.append(ftsFile) if not ftsFiles: return S_ERROR("ftsSchedule: no FTSFiles to put") put = self.__ftsDB.putFTSFileList(ftsFiles) if not put["OK"]: gLogger.error("ftsSchedule: %s" % put["Message"]) return put for fileJSON, sources, targets in fileJSONList: lfn = fileJSON.get("LFN", "") fileID = fileJSON.get("FileID", 0) if fileID not in ret["Failed"]: ret["Successful"].append(int(fileID)) # # if we land here some files have been properly scheduled return S_OK(ret)
def ftsSchedule( self, requestID, operationID, opFileList ): """ schedule lfn for FTS job :param int requestID: RequestDB.Request.RequestID :param int operationID: RequestDB.Operation.OperationID :param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList ) """ fileIDs = [int( fileJSON.get( 'FileID', 0 ) ) for fileJSON, _sourceSEs, _targetSEs in opFileList ] res = self.ftsManager.cleanUpFTSFiles( requestID, fileIDs ) if not res['OK']: self.log.error( "ftsSchedule: %s" % res['Message'] ) return S_ERROR( "ftsSchedule: %s" % res['Message'] ) ftsFiles = [] # # this will be returned on success result = { "Successful": [], "Failed": {} } for fileJSON, sourceSEs, targetSEs in opFileList: lfn = fileJSON.get( "LFN", "" ) size = int( fileJSON.get( "Size", 0 ) ) fileID = int( fileJSON.get( "FileID", 0 ) ) opID = int( fileJSON.get( "OperationID", 0 ) ) self.log.verbose( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID, sourceSEs, targetSEs ) ) res = self.dataManager.getActiveReplicas( lfn ) if not res['OK']: self.log.error( "ftsSchedule: %s" % res['Message'] ) result["Failed"][fileID] = res['Message'] continue replicaDict = res['Value'] if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]: result["Failed"][fileID] = "no active replicas found" continue replicaDict = replicaDict["Successful"].get( lfn, {} ) # # use valid replicas only validReplicasDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] ) if not validReplicasDict: self.log.warn( "No active replicas found in sources" ) result["Failed"][fileID] = "no active replicas found in sources" continue tree = self.ftsManager.getReplicationTree( sourceSEs, targetSEs, size ) if not tree['OK']: self.log.error( "ftsSchedule: %s cannot be scheduled: %s" % ( lfn, tree['Message'] ) ) result["Failed"][fileID] = tree['Message'] continue tree = tree['Value'] self.log.verbose( "LFN=%s tree=%s" % ( lfn, tree ) ) for repDict in tree.values(): self.log.verbose( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"], repDict["Ancestor"], repDict["SourceSE"], repDict["TargetSE"] ) ) transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, validReplicasDict ) if not transferSURLs['OK']: result["Failed"][fileID] = transferSURLs['Message'] continue sourceSURL, targetSURL, fileStatus = transferSURLs['Value'] if sourceSURL == targetSURL: result["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn continue self.log.verbose( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) ) ftsFile = FTSFile() for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ): if fileJSON.get( key ): setattr( ftsFile, key, fileJSON.get( key ) ) ftsFile.RequestID = requestID ftsFile.OperationID = operationID ftsFile.SourceSURL = sourceSURL ftsFile.TargetSURL = targetSURL ftsFile.SourceSE = repDict["SourceSE"] ftsFile.TargetSE = repDict["TargetSE"] ftsFile.Status = fileStatus ftsFiles.append( ftsFile ) if not ftsFiles: self.log.info( "ftsSchedule: no FTSFiles to put for request %d" % requestID ) return S_OK( result ) ftsFilesJSONList = [ftsFile.toJSON()['Value'] for ftsFile in ftsFiles] res = self.ftsManager.putFTSFileList( ftsFilesJSONList ) if not res['OK']: self.log.error( "ftsSchedule: %s" % res['Message'] ) return S_ERROR( "ftsSchedule: %s" % res['Message'] ) result['Successful'] += [ fileID for fileID in fileIDs if fileID not in result['Failed']] # # if we land here some files have been properly scheduled return S_OK( result )
def setUp( self ): """ test case set up """ # ## set some defaults gConfig.setOptionValue( 'DIRAC/Setup', 'Test' ) gConfig.setOptionValue( '/DIRAC/Setups/Test/DataManagement', 'Test' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac' ) self.ftsSites = [ FTSSite( { "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" } ), FTSSite( { "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" } ), FTSSite( { "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" } ), FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" } ), FTSSite( { "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" } ), FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" } ), FTSSite( { "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" } ), FTSSite( { "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" } ) ] self.ftsFiles = [] for i in range ( 100 ): ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = 9999 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 10 ftsFile.SourceSE = "CERN-USER" ftsFile.TargetSE = "PIC-USER" ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" self.ftsFiles.append( ftsFile ) ses = [ "CERN-USER", "RAL-USER" ] statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ] self.submitted = 0 self.ftsJobs = [] for i in range( 7200 ): ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = statuses[ i % len( statuses ) ] if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == "Active": ftsJob.Completeness = 90 ftsJob.SourceSE = ses[ i % len( ses ) ] ftsJob.TargetSE = "PIC-USER" ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 1000000 ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed" ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == "FinishedDirty": ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile( ftsFile ) self.ftsJobs.append( ftsJob ) self.submitted = len( [ i for i in self.ftsJobs if i.Status == "Submitted" ] )
def export_ftsSchedule( self, requestID, operationID, fileJSONList ): """ call FTS scheduler :param int requestID: ReqDB.Request.RequestID :param int operationID: ReqDB.Operation.OperationID :param list fileJSONList: [ (fileJSON, [sourceSE, ...], [ targetSE, ...] ) ] """ # # this will be returned on success ret = { "Successful": [], "Failed": {} } requestID = int( requestID ) operationID = int( operationID ) fileIDs = [] for fileJSON, sourceSEs, targetSEs in fileJSONList: fileID = int( fileJSON.get( "FileID" ) ) fileIDs.append( fileID ) cleanUpFTSFiles = self.__ftsDB.cleanUpFTSFiles( requestID, fileIDs ) if not cleanUpFTSFiles["OK"]: self.log.error( "ftsSchedule: %s" % cleanUpFTSFiles["Message"] ) return S_ERROR( cleanUpFTSFiles["Message"] ) ftsFiles = [] for fileJSON, sourceSEs, targetSEs in fileJSONList: lfn = fileJSON.get( "LFN", "" ) size = int( fileJSON.get( "Size", 0 ) ) fileID = int( fileJSON.get( "FileID", 0 ) ) opID = int( fileJSON.get( "OperationID", 0 ) ) gLogger.info( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID, sourceSEs, targetSEs ) ) replicaDict = self.replicaManager().getActiveReplicas( lfn ) if not replicaDict["OK"]: gLogger.error( "ftsSchedule: %s" % replicaDict["Message"] ) ret["Failed"][fileID] = replicaDict["Message"] continue replicaDict = replicaDict["Value"] if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]: ret["Failed"][fileID] = "no active replicas found" continue replicaDict = replicaDict["Successful"][lfn] if lfn in replicaDict["Successful"] else {} # # use valid replicas only replicaDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] ) if not replicaDict: ret["Failed"][fileID] = "no active replicas found in sources" continue tree = self.ftsStrategy().replicationTree( sourceSEs, targetSEs, size ) if not tree["OK"]: gLogger.error( "ftsSchedule: %s cannot be scheduled: %s" % ( lfn, tree["Message"] ) ) ret["Failed"][fileID] = tree["Message"] continue tree = tree["Value"] gLogger.info( "LFN=%s tree=%s" % ( lfn, tree ) ) for repDict in tree.values(): gLogger.info( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"], repDict["Ancestor"], repDict["SourceSE"], repDict["TargetSE"] ) ) transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, replicaDict ) if not transferSURLs["OK"]: ret["Failed"][fileID] = transferSURLs["Message"] continue sourceSURL, targetSURL, fileStatus = transferSURLs["Value"] if sourceSURL == targetSURL: ret["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn continue gLogger.info( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) ) ftsFile = FTSFile() for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ): setattr( ftsFile, key, fileJSON.get( key ) ) ftsFile.RequestID = requestID ftsFile.OperationID = operationID ftsFile.SourceSURL = sourceSURL ftsFile.TargetSURL = targetSURL ftsFile.SourceSE = repDict["SourceSE"] ftsFile.TargetSE = repDict["TargetSE"] ftsFile.Status = fileStatus ftsFiles.append( ftsFile ) if not ftsFiles: return S_ERROR( "ftsSchedule: no FTSFiles to put" ) put = self.__ftsDB.putFTSFileList( ftsFiles ) if not put["OK"]: gLogger.error( "ftsSchedule: %s" % put["Message"] ) return put for fileJSON, sources, targets in fileJSONList: lfn = fileJSON.get( "LFN", "" ) fileID = fileJSON.get( "FileID", 0 ) if fileID not in ret["Failed"]: ret["Successful"].append( int( fileID ) ) # # if we land here some files have been properly scheduled return S_OK( ret )
def setUp( self ): """ test case set up """ gLogger.setLevel( 'NOTICE' ) self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ), FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ), FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ), ] self.ses = [ 'CERN-USER', 'RAL-USER' ] self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ] self.submitted = 0 self.numberOfJobs = 10 self.opIDs = [] self.ftsJobs = [] for i in xrange( self.numberOfJobs ): opID = i % 3 if opID not in self.opIDs: self.opIDs.append( opID ) ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = self.statuses[ i % len( self.statuses ) ] ftsJob.OperationID = opID if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == 'Active': ftsJob.Completeness = 90 ftsJob.SourceSE = self.ses[ i % len( self.ses ) ] ftsJob.TargetSE = 'PIC-USER' ftsJob.RequestID = 12345 ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = '/a/b/c/%d' % i ftsFile.Size = 1000000 ftsFile.OperationID = opID ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed' ftsFile.RequestID = 12345 ftsFile.Checksum = 'addler' ftsFile.ChecksumType = 'adler32' ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == 'FinishedDirty': ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile( ftsFile ) self.ftsJobs.append( ftsJob ) self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] ) self.ftsClient = FTSClient() self.ftsClient.replicaManager = mock.Mock() self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True, 'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******', 'RAL-USER':'******'}, '/a/b/c/2':{'CERN-USER':'******', 'RAL-USER':'******'}, '/a/b/c/3':{'CERN-USER':'******', 'RAL-USER':'******'} }, 'Failed': {'/a/b/c/4':'/aa/a/b/c/4d', '/a/b/c/5':'/aa/a/b/c/5d'} } }
def monitorFTS2( self, command = "glite-transfer-status", full = False ): """ monitor fts job """ if not self.FTSGUID: return S_ERROR( "FTSGUID not set, FTS job not submitted?" ) monitorCommand = command.split() + \ ["--verbose", "-s", self.FTSServer, self.FTSGUID ] if full: monitorCommand.append( "-l" ) monitor = executeGridCommand( "", monitorCommand ) if not monitor["OK"]: return monitor returnCode, outputStr, errStr = monitor["Value"] # Returns a non zero status if error if returnCode != 0: if 'was not found' in outputStr and not errStr: errStr = 'Job was not found' return S_ERROR( errStr ) outputStr = outputStr.replace( "'" , "" ).replace( "<", "" ).replace( ">", "" ) # # set FTS job status regExp = re.compile( "Status:\\s+(\\S+)" ) # with FTS3 this can be uppercase self.Status = re.search( regExp, outputStr ).group( 1 ) statusSummary = {} # This is capitalized, even in FTS3! for state in FTSFile.ALL_STATES: regExp = re.compile( "\\s+%s:\\s+(\\d+)" % state ) if regExp.search( outputStr ): statusSummary[state] = int( re.search( regExp, outputStr ).group( 1 ) ) total = sum( statusSummary.values() ) completed = sum( statusSummary.get( state, 0 ) for state in FTSFile.FINAL_STATES ) self.Completeness = 100 * completed / total if total else 0 if not full: return S_OK( statusSummary ) # The order of informations is not the same for glite- and fts- !!! # In order: new fts-, old fts-, glite- realJob = len( self ) != 0 iExptr = None for iExptr, exptr in enumerate( ( '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)' ) ): regExp = re.compile( exptr, re.S ) fileInfo = re.findall( regExp, outputStr ) if fileInfo: break if not fileInfo: return S_ERROR( "Error monitoring job (no regexp match)" ) for info in fileInfo: if iExptr == 0: # version >= 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info elif iExptr == 1: # version FTS3 < 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries = info elif iExptr == 2: # version FTS2 sourceURL, targetURL, fileStatus, _retries, reason, duration = info else: return S_ERROR( 'Error monitoring job (implement match %d)' % iExptr ) candidateFile = None if not realJob: # This is used by the CLI monitoring of jobs in case no file was specified candidateFile = FTSFile() candidateFile.LFN = overlap( sourceURL, targetURL ) candidateFile.SourceSURL = sourceURL candidateFile.Size = 0 self +=candidateFile else: for ftsFile in self: if ftsFile.SourceSURL == sourceURL: candidateFile = ftsFile break if not candidateFile: continue # Can be uppercase for FTS3 if not candidateFile.TargetSURL: candidateFile.TargetSURL = targetURL candidateFile.Status = fileStatus candidateFile.Error = reason candidateFile._duration = duration if candidateFile.Status == "Failed": for missingSource in self.missingSourceErrors: if missingSource.match( reason ): candidateFile.Error = "MissingSource" # If the staging info was present, record it if len( info ) > 6: candidateFile._staging = info[6] # # register successful files if self.Status in FTSJob.FINALSTATES: return self.finalize() return S_OK()
def ftsSchedule( self, requestID, operationID, opFileList ): """ schedule lfn for FTS job :param int requestID: RequestDB.Request.RequestID :param int operationID: RequestDB.Operation.OperationID :param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList ) """ # Check whether there are duplicates fList = [] for fileJSON, sourceSEs, targetSEs in opFileList: fTuple = ( json.loads( fileJSON ), sourceSEs, targetSEs ) if fTuple not in fList: fList.append( fTuple ) else: self.log.warn( 'File list for FTS scheduling has duplicates, fix it:\n', fTuple ) fileIDs = [int( fileJSON.get( 'FileID', 0 ) ) for fileJSON, _sourceSEs, _targetSEs in fList ] res = self.ftsManager.cleanUpFTSFiles( requestID, fileIDs ) if not res['OK']: self.log.error( "Failed ftsSchedule", "%s" % res['Message'] ) return S_ERROR( "ftsSchedule: %s" % res['Message'] ) ftsFiles = [] # # this will be returned on success result = { "Successful": [], "Failed": {} } for fileJSON, sourceSEs, targetSEs in fList: lfn = fileJSON.get( "LFN", "" ) size = int( fileJSON.get( "Size", 0 ) ) fileID = int( fileJSON.get( "FileID", 0 ) ) opID = int( fileJSON.get( "OperationID", 0 ) ) self.log.verbose( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID, sourceSEs, targetSEs ) ) res = self.dataManager.getActiveReplicas( lfn ) if not res['OK']: self.log.error( "Failed ftsSchedule", "%s" % res['Message'] ) result["Failed"][fileID] = res['Message'] continue replicaDict = res['Value'] if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]: result["Failed"][fileID] = "no active replicas found" continue replicaDict = replicaDict["Successful"].get( lfn, {} ) # # use valid replicas only validReplicasDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] ) if not validReplicasDict: self.log.warn( "No active replicas found in sources" ) result["Failed"][fileID] = "no active replicas found in sources" continue tree = self.ftsManager.getReplicationTree( sourceSEs, targetSEs, size ) if not tree['OK']: self.log.error( "Failed ftsSchedule", "%s cannot be scheduled: %s" % ( lfn, tree['Message'] ) ) result["Failed"][fileID] = tree['Message'] continue tree = tree['Value'] self.log.verbose( "LFN=%s tree=%s" % ( lfn, tree ) ) treeBranches = [] printed = False for repDict in tree.values(): if repDict in treeBranches: if not printed: self.log.warn( 'Duplicate tree branch', str( tree ) ) printed = True else: treeBranches.append( repDict ) for repDict in treeBranches: self.log.verbose( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"], repDict["Ancestor"], repDict["SourceSE"], repDict["TargetSE"] ) ) transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, validReplicasDict ) if not transferSURLs['OK']: result["Failed"][fileID] = transferSURLs['Message'] continue sourceSURL, targetSURL, fileStatus = transferSURLs['Value'] if sourceSURL == targetSURL: result["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn continue self.log.verbose( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) ) ftsFile = FTSFile() for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ): if fileJSON.get( key ): setattr( ftsFile, key, fileJSON.get( key ) ) ftsFile.RequestID = requestID ftsFile.OperationID = operationID ftsFile.SourceSURL = sourceSURL ftsFile.TargetSURL = targetSURL ftsFile.SourceSE = repDict["SourceSE"] ftsFile.TargetSE = repDict["TargetSE"] ftsFile.Status = fileStatus ftsFiles.append( ftsFile ) if not ftsFiles: self.log.info( "ftsSchedule: no FTSFiles to put for request %d" % requestID ) return S_OK( result ) ftsFilesJSONList = [ftsFile.toJSON()['Value'] for ftsFile in ftsFiles] res = self.ftsManager.putFTSFileList( ftsFilesJSONList ) if not res['OK']: self.log.error( "Failed ftsSchedule", "%s" % res['Message'] ) return S_ERROR( "ftsSchedule: %s" % res['Message'] ) result['Successful'] += [ fileID for fileID in fileIDs if fileID not in result['Failed']] # # if we land here some files have been properly scheduled return S_OK( result )