def monitorFTS2( self, command = "glite-transfer-status", full = False ): """ monitor fts job """ if not self.FTSGUID: return S_ERROR( "FTSGUID not set, FTS job not submitted?" ) monitorCommand = command.split() + \ ["--verbose", "-s", self.FTSServer, self.FTSGUID ] if full: monitorCommand.append( "-l" ) monitor = executeGridCommand( "", monitorCommand ) if not monitor["OK"]: return monitor returnCode, outputStr, errStr = monitor["Value"] # Returns a non zero status if error if returnCode != 0: if 'was not found' in outputStr and not errStr: errStr = 'Job was not found' return S_ERROR( errStr ) outputStr = outputStr.replace( "'" , "" ).replace( "<", "" ).replace( ">", "" ) # # set FTS job status regExp = re.compile( "Status:\\s+(\\S+)" ) # with FTS3 this can be uppercase self.Status = re.search( regExp, outputStr ).group( 1 ) statusSummary = {} # This is capitalized, even in FTS3! for state in FTSFile.ALL_STATES: regExp = re.compile( "\\s+%s:\\s+(\\d+)" % state ) if regExp.search( outputStr ): statusSummary[state] = int( re.search( regExp, outputStr ).group( 1 ) ) total = sum( statusSummary.values() ) completed = sum( statusSummary.get( state, 0 ) for state in FTSFile.FINAL_STATES ) self.Completeness = 100 * completed / total if total else 0 if not full: return S_OK( statusSummary ) # The order of informations is not the same for glite- and fts- !!! # In order: new fts-, old fts-, glite- realJob = len( self ) != 0 iExptr = None for iExptr, exptr in enumerate( ( '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)' ) ): regExp = re.compile( exptr, re.S ) fileInfo = re.findall( regExp, outputStr ) if fileInfo: break if not fileInfo: return S_ERROR( "Error monitoring job (no regexp match)" ) for info in fileInfo: if iExptr == 0: # version >= 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info elif iExptr == 1: # version FTS3 < 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries = info elif iExptr == 2: # version FTS2 sourceURL, targetURL, fileStatus, _retries, reason, duration = info else: return S_ERROR( 'Error monitoring job (implement match %d)' % iExptr ) candidateFile = None if not realJob: # This is used by the CLI monitoring of jobs in case no file was specified candidateFile = FTSFile() candidateFile.LFN = overlap( sourceURL, targetURL ) candidateFile.SourceSURL = sourceURL candidateFile.Size = 0 self +=candidateFile else: for ftsFile in self: if ftsFile.SourceSURL == sourceURL: candidateFile = ftsFile break if not candidateFile: continue # Can be uppercase for FTS3 if not candidateFile.TargetSURL: candidateFile.TargetSURL = targetURL candidateFile.Status = fileStatus candidateFile.Error = reason candidateFile._duration = duration if candidateFile.Status == "Failed": for missingSource in self.missingSourceErrors: if missingSource.match( reason ): candidateFile.Error = "MissingSource" # If the staging info was present, record it if len( info ) > 6: candidateFile._staging = info[6] # # register successful files if self.Status in FTSJob.FINALSTATES: return self.finalize() return S_OK()
def monitorFTS2(self, command="glite-transfer-status", full=False): """ monitor fts job """ if not self.FTSGUID: return S_ERROR("FTSGUID not set, FTS job not submitted?") monitorCommand = command.split() + \ ["--verbose", "-s", self.FTSServer, self.FTSGUID ] if full: monitorCommand.append("-l") monitor = executeGridCommand("", monitorCommand) if not monitor["OK"]: return monitor returnCode, outputStr, errStr = monitor["Value"] # Returns a non zero status if error if returnCode != 0: if 'was not found' in outputStr and not errStr: errStr = 'Job was not found' return S_ERROR(errStr) outputStr = outputStr.replace("'", "").replace("<", "").replace(">", "") # # set FTS job status regExp = re.compile("Status:\\s+(\\S+)") # with FTS3 this can be uppercase self.Status = re.search(regExp, outputStr).group(1) statusSummary = {} # This is capitalized, even in FTS3! for state in FTSFile.ALL_STATES: regExp = re.compile("\\s+%s:\\s+(\\d+)" % state) if regExp.search(outputStr): statusSummary[state] = int( re.search(regExp, outputStr).group(1)) total = sum(statusSummary.values()) completed = sum( statusSummary.get(state, 0) for state in FTSFile.FINAL_STATES) self.Completeness = 100 * completed / total if total else 0 if not full: return S_OK(statusSummary) # The order of informations is not the same for glite- and fts- !!! # In order: new fts-, old fts-, glite- realJob = len(self) != 0 iExptr = None for iExptr, exptr in enumerate( ('[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)' )): regExp = re.compile(exptr, re.S) fileInfo = re.findall(regExp, outputStr) if fileInfo: break if not fileInfo: return S_ERROR("Error monitoring job (no regexp match)") for info in fileInfo: if iExptr == 0: # version >= 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info elif iExptr == 1: # version FTS3 < 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries = info elif iExptr == 2: # version FTS2 sourceURL, targetURL, fileStatus, _retries, reason, duration = info else: return S_ERROR('Error monitoring job (implement match %d)' % iExptr) candidateFile = None if not realJob: # This is used by the CLI monitoring of jobs in case no file was specified candidateFile = FTSFile() candidateFile.LFN = overlap(sourceURL, targetURL) candidateFile.SourceSURL = sourceURL candidateFile.Size = 0 self += candidateFile else: for ftsFile in self: if ftsFile.SourceSURL == sourceURL: candidateFile = ftsFile break if not candidateFile: continue # Can be uppercase for FTS3 if not candidateFile.TargetSURL: candidateFile.TargetSURL = targetURL candidateFile.Status = fileStatus candidateFile.Error = reason candidateFile._duration = duration if candidateFile.Status == "Failed": for missingSource in self.missingSourceErrors: if missingSource.match(reason): candidateFile.Error = "MissingSource" # If the staging info was present, record it if len(info) > 6: candidateFile._staging = info[6] # # register successful files if self.Status in FTSJob.FINALSTATES: return self.finalize() return S_OK()