def __createFTSFiles( self ): """ create LFNs file for glite-transfer-submit command This file consists one line for each fiel to be transferred: sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM] :param self: self reference """ self.__updateMetadataCache() for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get( 'Status' ) if lfnStatus not in self.noSubmitStatus: cksmStr = "" # # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself if self.__cksmTest and self.__cksmType: checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' ) if checkSum: cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) ) ftsFile = FTSFile() ftsFile.LFN = lfn ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' ) ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' ) ftsFile.SourceSE = self.sourceSE ftsFile.TargetSE = self.targetSE ftsFile.Status = self.fileDict[lfn].get( 'Status' ) ftsFile.Checksum = cksmStr ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' ) self.ftsFiles.append( ftsFile ) self.submittedFiles += 1 return S_OK()
def monitorFTS2( self, command = "glite-transfer-status", full = False ): """ monitor fts job """ if not self.FTSGUID: return S_ERROR( "FTSGUID not set, FTS job not submitted?" ) monitorCommand = command.split() + \ ["--verbose", "-s", self.FTSServer, self.FTSGUID ] if full: monitorCommand.append( "-l" ) monitor = executeGridCommand( "", monitorCommand ) if not monitor["OK"]: return monitor returnCode, outputStr, errStr = monitor["Value"] # Returns a non zero status if error if returnCode != 0: if 'was not found' in outputStr and not errStr: errStr = 'Job was not found' return S_ERROR( errStr ) outputStr = outputStr.replace( "'" , "" ).replace( "<", "" ).replace( ">", "" ) # # set FTS job status regExp = re.compile( "Status:\\s+(\\S+)" ) # with FTS3 this can be uppercase self.Status = re.search( regExp, outputStr ).group( 1 ) statusSummary = {} # This is capitalized, even in FTS3! for state in FTSFile.ALL_STATES: regExp = re.compile( "\\s+%s:\\s+(\\d+)" % state ) if regExp.search( outputStr ): statusSummary[state] = int( re.search( regExp, outputStr ).group( 1 ) ) total = sum( statusSummary.values() ) completed = sum( statusSummary.get( state, 0 ) for state in FTSFile.FINAL_STATES ) self.Completeness = 100 * completed / total if total else 0 if not full: return S_OK( statusSummary ) # The order of informations is not the same for glite- and fts- !!! # In order: new fts-, old fts-, glite- realJob = len( self ) != 0 iExptr = None for iExptr, exptr in enumerate( ( '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)' ) ): regExp = re.compile( exptr, re.S ) fileInfo = re.findall( regExp, outputStr ) if fileInfo: break if not fileInfo: return S_ERROR( "Error monitoring job (no regexp match)" ) for info in fileInfo: if iExptr == 0: # version >= 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info elif iExptr == 1: # version FTS3 < 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries = info elif iExptr == 2: # version FTS2 sourceURL, targetURL, fileStatus, _retries, reason, duration = info else: return S_ERROR( 'Error monitoring job (implement match %d)' % iExptr ) candidateFile = None if not realJob: # This is used by the CLI monitoring of jobs in case no file was specified candidateFile = FTSFile() candidateFile.LFN = overlap( sourceURL, targetURL ) candidateFile.SourceSURL = sourceURL candidateFile.Size = 0 self +=candidateFile else: for ftsFile in self: if ftsFile.SourceSURL == sourceURL: candidateFile = ftsFile break if not candidateFile: continue # Can be uppercase for FTS3 if not candidateFile.TargetSURL: candidateFile.TargetSURL = targetURL candidateFile.Status = fileStatus candidateFile.Error = reason candidateFile._duration = duration if candidateFile.Status == "Failed": for missingSource in self.missingSourceErrors: if missingSource.match( reason ): candidateFile.Error = "MissingSource" # If the staging info was present, record it if len( info ) > 6: candidateFile._staging = info[6] # # register successful files if self.Status in FTSJob.FINALSTATES: return self.finalize() return S_OK()
def monitorFTS2(self, command="glite-transfer-status", full=False): """ monitor fts job """ if not self.FTSGUID: return S_ERROR("FTSGUID not set, FTS job not submitted?") monitorCommand = command.split() + \ ["--verbose", "-s", self.FTSServer, self.FTSGUID ] if full: monitorCommand.append("-l") monitor = executeGridCommand("", monitorCommand) if not monitor["OK"]: return monitor returnCode, outputStr, errStr = monitor["Value"] # Returns a non zero status if error if returnCode != 0: if 'was not found' in outputStr and not errStr: errStr = 'Job was not found' return S_ERROR(errStr) outputStr = outputStr.replace("'", "").replace("<", "").replace(">", "") # # set FTS job status regExp = re.compile("Status:\\s+(\\S+)") # with FTS3 this can be uppercase self.Status = re.search(regExp, outputStr).group(1) statusSummary = {} # This is capitalized, even in FTS3! for state in FTSFile.ALL_STATES: regExp = re.compile("\\s+%s:\\s+(\\d+)" % state) if regExp.search(outputStr): statusSummary[state] = int( re.search(regExp, outputStr).group(1)) total = sum(statusSummary.values()) completed = sum( statusSummary.get(state, 0) for state in FTSFile.FINAL_STATES) self.Completeness = 100 * completed / total if total else 0 if not full: return S_OK(statusSummary) # The order of informations is not the same for glite- and fts- !!! # In order: new fts-, old fts-, glite- realJob = len(self) != 0 iExptr = None for iExptr, exptr in enumerate( ('[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)', '[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)' )): regExp = re.compile(exptr, re.S) fileInfo = re.findall(regExp, outputStr) if fileInfo: break if not fileInfo: return S_ERROR("Error monitoring job (no regexp match)") for info in fileInfo: if iExptr == 0: # version >= 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info elif iExptr == 1: # version FTS3 < 3.2.30 sourceURL, targetURL, fileStatus, reason, duration, _retries = info elif iExptr == 2: # version FTS2 sourceURL, targetURL, fileStatus, _retries, reason, duration = info else: return S_ERROR('Error monitoring job (implement match %d)' % iExptr) candidateFile = None if not realJob: # This is used by the CLI monitoring of jobs in case no file was specified candidateFile = FTSFile() candidateFile.LFN = overlap(sourceURL, targetURL) candidateFile.SourceSURL = sourceURL candidateFile.Size = 0 self += candidateFile else: for ftsFile in self: if ftsFile.SourceSURL == sourceURL: candidateFile = ftsFile break if not candidateFile: continue # Can be uppercase for FTS3 if not candidateFile.TargetSURL: candidateFile.TargetSURL = targetURL candidateFile.Status = fileStatus candidateFile.Error = reason candidateFile._duration = duration if candidateFile.Status == "Failed": for missingSource in self.missingSourceErrors: if missingSource.match(reason): candidateFile.Error = "MissingSource" # If the staging info was present, record it if len(info) > 6: candidateFile._staging = info[6] # # register successful files if self.Status in FTSJob.FINALSTATES: return self.finalize() return S_OK()
def setUp( self ): """ test case set up """ # ## set some defaults gConfig.setOptionValue( 'DIRAC/Setup', 'Test' ) gConfig.setOptionValue( '/DIRAC/Setups/Test/DataManagement', 'Test' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac' ) self.ftsSites = [ FTSSite( { "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" } ), FTSSite( { "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" } ), FTSSite( { "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" } ), FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" } ), FTSSite( { "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" } ), FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" } ), FTSSite( { "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" } ), FTSSite( { "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" } ) ] self.ftsFiles = [] for i in range ( 100 ): ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = 9999 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 10 ftsFile.SourceSE = "CERN-USER" ftsFile.TargetSE = "PIC-USER" ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" self.ftsFiles.append( ftsFile ) ses = [ "CERN-USER", "RAL-USER" ] statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ] self.submitted = 0 self.ftsJobs = [] for i in range( 7200 ): ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = statuses[ i % len( statuses ) ] if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == "Active": ftsJob.Completeness = 90 ftsJob.SourceSE = ses[ i % len( ses ) ] ftsJob.TargetSE = "PIC-USER" ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 1000000 ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed" ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == "FinishedDirty": ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile( ftsFile ) self.ftsJobs.append( ftsJob ) self.submitted = len( [ i for i in self.ftsJobs if i.Status == "Submitted" ] )
def setUp( self ): """ test case set up """ gLogger.setLevel( 'NOTICE' ) self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ), FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ), FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ), ] self.ses = [ 'CERN-USER', 'RAL-USER' ] self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ] self.submitted = 0 self.numberOfJobs = 10 self.opIDs = [] self.ftsJobs = [] for i in xrange( self.numberOfJobs ): opID = i % 3 if opID not in self.opIDs: self.opIDs.append( opID ) ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = self.statuses[ i % len( self.statuses ) ] ftsJob.OperationID = opID if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == 'Active': ftsJob.Completeness = 90 ftsJob.SourceSE = self.ses[ i % len( self.ses ) ] ftsJob.TargetSE = 'PIC-USER' ftsJob.RequestID = 12345 ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = '/a/b/c/%d' % i ftsFile.Size = 1000000 ftsFile.OperationID = opID ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed' ftsFile.RequestID = 12345 ftsFile.Checksum = 'addler' ftsFile.ChecksumType = 'adler32' ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == 'FinishedDirty': ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile( ftsFile ) self.ftsJobs.append( ftsJob ) self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] ) self.ftsClient = FTSClient() self.ftsClient.replicaManager = mock.Mock() self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True, 'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******', 'RAL-USER':'******'}, '/a/b/c/2':{'CERN-USER':'******', 'RAL-USER':'******'}, '/a/b/c/3':{'CERN-USER':'******', 'RAL-USER':'******'} }, 'Failed': {'/a/b/c/4':'/aa/a/b/c/4d', '/a/b/c/5':'/aa/a/b/c/5d'} } }
def setUp(self): """ test case set up """ # ## set some defaults gConfig.setOptionValue('DIRAC/Setup', 'Test') gConfig.setOptionValue('/DIRAC/Setups/Test/DataManagement', 'Test') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac') self.ftsSites = [ FTSSite({ "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" }), FTSSite({ "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" }), FTSSite({ "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" }), FTSSite({ "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" }), FTSSite({ "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" }), FTSSite({ "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" }), FTSSite({ "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" }), FTSSite({ "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" }) ] self.ftsFiles = [] for i in range(100): ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = 9999 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 10 ftsFile.SourceSE = "CERN-USER" ftsFile.TargetSE = "PIC-USER" ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" self.ftsFiles.append(ftsFile) ses = ["CERN-USER", "RAL-USER"] statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ] self.submitted = 0 self.ftsJobs = [] for i in range(7200): ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = statuses[i % len(statuses)] if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == "Active": ftsJob.Completeness = 90 ftsJob.SourceSE = ses[i % len(ses)] ftsJob.TargetSE = "PIC-USER" ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 1000000 ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed" ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == "FinishedDirty": ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile(ftsFile) self.ftsJobs.append(ftsJob) self.submitted = len( [i for i in self.ftsJobs if i.Status == "Submitted"])
def setUp(self): """ test case set up """ gLogger.setLevel('NOTICE') self.ftsSites = [ FTSSite( ftsServer= 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name='CERN.ch'), FTSSite( ftsServer= 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name='PIC.es'), FTSSite( ftsServer= 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name='RAL.uk'), ] self.ses = ['CERN-USER', 'RAL-USER'] self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ] self.submitted = 0 self.numberOfJobs = 10 self.opIDs = [] self.ftsJobs = [] for i in xrange(self.numberOfJobs): opID = i % 3 if opID not in self.opIDs: self.opIDs.append(opID) ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = self.statuses[i % len(self.statuses)] ftsJob.OperationID = opID if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == 'Active': ftsJob.Completeness = 90 ftsJob.SourceSE = self.ses[i % len(self.ses)] ftsJob.TargetSE = 'PIC-USER' ftsJob.RequestID = 12345 ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = '/a/b/c/%d' % i ftsFile.Size = 1000000 ftsFile.OperationID = opID ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed' ftsFile.RequestID = 12345 ftsFile.Checksum = 'addler' ftsFile.ChecksumType = 'adler32' ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == 'FinishedDirty': ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile(ftsFile) self.ftsJobs.append(ftsJob) self.submitted = len( [i for i in self.ftsJobs if i.Status == 'Submitted']) self.ftsClient = FTSClient()