def __createFTSFiles( self ): """ create LFNs file for glite-transfer-submit command This file consists one line for each fiel to be transferred: sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM] :param self: self reference """ self.__updateMetadataCache() for lfn in self.fileDict: lfnStatus = self.fileDict[lfn].get( 'Status' ) if lfnStatus not in self.noSubmitStatus: cksmStr = "" # # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself if self.__cksmTest and self.__cksmType: checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' ) if checkSum: cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) ) ftsFile = FTSFile() ftsFile.LFN = lfn ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' ) ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' ) ftsFile.SourceSE = self.sourceSE ftsFile.TargetSE = self.targetSE ftsFile.Status = self.fileDict[lfn].get( 'Status' ) ftsFile.Checksum = cksmStr ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' ) self.ftsFiles.append( ftsFile ) self.submittedFiles += 1 return S_OK()
def setUp( self ): """ test case set up """ gLogger.setLevel( 'NOTICE' ) self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ), FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ), FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ), ] self.ses = [ 'CERN-USER', 'RAL-USER' ] self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ] self.submitted = 0 self.numberOfJobs = 10 self.opIDs = [] self.ftsJobs = [] for i in xrange( self.numberOfJobs ): opID = i % 3 if opID not in self.opIDs: self.opIDs.append( opID ) ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = self.statuses[ i % len( self.statuses ) ] ftsJob.OperationID = opID if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == 'Active': ftsJob.Completeness = 90 ftsJob.SourceSE = self.ses[ i % len( self.ses ) ] ftsJob.TargetSE = 'PIC-USER' ftsJob.RequestID = 12345 ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = '/a/b/c/%d' % i ftsFile.Size = 1000000 ftsFile.OperationID = opID ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed' ftsFile.RequestID = 12345 ftsFile.Checksum = 'addler' ftsFile.ChecksumType = 'adler32' ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == 'FinishedDirty': ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile( ftsFile ) self.ftsJobs.append( ftsJob ) self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] ) self.ftsClient = FTSClient() self.ftsClient.replicaManager = mock.Mock() self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True, 'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******', 'RAL-USER':'******'}, '/a/b/c/2':{'CERN-USER':'******', 'RAL-USER':'******'}, '/a/b/c/3':{'CERN-USER':'******', 'RAL-USER':'******'} }, 'Failed': {'/a/b/c/4':'/aa/a/b/c/4d', '/a/b/c/5':'/aa/a/b/c/5d'} } }
def setUp(self): """ test case set up """ gLogger.setLevel('NOTICE') self.ftsSites = [ FTSSite( ftsServer= 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name='CERN.ch'), FTSSite( ftsServer= 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name='PIC.es'), FTSSite( ftsServer= 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name='RAL.uk'), ] self.ses = ['CERN-USER', 'RAL-USER'] self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ] self.submitted = 0 self.numberOfJobs = 10 self.opIDs = [] self.ftsJobs = [] for i in xrange(self.numberOfJobs): opID = i % 3 if opID not in self.opIDs: self.opIDs.append(opID) ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = self.statuses[i % len(self.statuses)] ftsJob.OperationID = opID if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == 'Active': ftsJob.Completeness = 90 ftsJob.SourceSE = self.ses[i % len(self.ses)] ftsJob.TargetSE = 'PIC-USER' ftsJob.RequestID = 12345 ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = '/a/b/c/%d' % i ftsFile.Size = 1000000 ftsFile.OperationID = opID ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed' ftsFile.RequestID = 12345 ftsFile.Checksum = 'addler' ftsFile.ChecksumType = 'adler32' ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == 'FinishedDirty': ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile(ftsFile) self.ftsJobs.append(ftsJob) self.submitted = len( [i for i in self.ftsJobs if i.Status == 'Submitted']) self.ftsClient = FTSClient()