def __createFTSJob( self, guid = None ): self.__createFTSFiles() ftsJob = FTSJob() ftsJob.RequestID = 0 ftsJob.OperationID = 0 ftsJob.SourceSE = self.sourceSE ftsJob.TargetSE = self.targetSE ftsJob.SourceToken = self.sourceToken ftsJob.TargetToken = self.targetToken ftsJob.FTSServer = self.ftsServer if guid: ftsJob.FTSGUID = guid for ftsFile in self.ftsFiles: ftsFile.Attempt += 1 ftsFile.Error = "" ftsJob.addFile( ftsFile ) self.ftsJob = ftsJob
def test02Files( self ): """ FTSFiles arithmetic """ ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) self.assertEqual( len( ftsJob ), 0, "1. len(ftsJob) error" ) self.assertEqual( ftsJob.Files, 0 , "1. Files prop error" ) self.assertEqual( ftsJob.Size, 0, "1. Size prop error" ) ftsJob.addFile( self.fileA ) ftsJob.addFile( self.fileB ) self.assertEqual( self.fileA.FTSGUID, ftsJob.FTSGUID, "FTSGUID mismatch for fileA" ) self.assertEqual( self.fileB.FTSGUID, ftsJob.FTSGUID, "FTSGUID mismatch for fileB" ) self.assertEqual( len( ftsJob ), 2, "2. len(ftsJob) error" ) self.assertEqual( ftsJob.Files, 2, "2. Files prop error" ) self.assertEqual( ftsJob.Size, 20, "2. Size prop error" )
def test02Files(self): """ FTSFiles arithmetic """ ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) self.assertEqual(len(ftsJob), 0, "1. len(ftsJob) error") self.assertEqual(ftsJob.Files, 0, "1. Files prop error") self.assertEqual(ftsJob.Size, 0, "1. Size prop error") ftsJob.addFile(self.fileA) ftsJob.addFile(self.fileB) self.assertEqual(self.fileA.FTSGUID, ftsJob.FTSGUID, "FTSGUID mismatch for fileA") self.assertEqual(self.fileB.FTSGUID, ftsJob.FTSGUID, "FTSGUID mismatch for fileB") self.assertEqual(len(ftsJob), 2, "2. len(ftsJob) error") self.assertEqual(ftsJob.Files, 2, "2. Files prop error") self.assertEqual(ftsJob.Size, 20, "2. Size prop error")
def setUp( self ): """ test case set up """ # ## set some defaults gConfig.setOptionValue( 'DIRAC/Setup', 'Test' ) gConfig.setOptionValue( '/DIRAC/Setups/Test/DataManagement', 'Test' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac' ) self.ftsSites = [ FTSSite( { "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" } ), FTSSite( { "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" } ), FTSSite( { "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" } ), FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" } ), FTSSite( { "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" } ), FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" } ), FTSSite( { "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" } ), FTSSite( { "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" } ) ] self.ftsFiles = [] for i in range ( 100 ): ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = 9999 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 10 ftsFile.SourceSE = "CERN-USER" ftsFile.TargetSE = "PIC-USER" ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" self.ftsFiles.append( ftsFile ) ses = [ "CERN-USER", "RAL-USER" ] statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ] self.submitted = 0 self.ftsJobs = [] for i in range( 7200 ): ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = statuses[ i % len( statuses ) ] if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == "Active": ftsJob.Completeness = 90 ftsJob.SourceSE = ses[ i % len( ses ) ] ftsJob.TargetSE = "PIC-USER" ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 1000000 ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed" ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == "FinishedDirty": ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile( ftsFile ) self.ftsJobs.append( ftsJob ) self.submitted = len( [ i for i in self.ftsJobs if i.Status == "Submitted" ] )
def setUp( self ): """ test case set up """ gLogger.setLevel( 'NOTICE' ) self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ), FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ), FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ), ] self.ses = [ 'CERN-USER', 'RAL-USER' ] self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ] self.submitted = 0 self.numberOfJobs = 10 self.opIDs = [] self.ftsJobs = [] for i in xrange( self.numberOfJobs ): opID = i % 3 if opID not in self.opIDs: self.opIDs.append( opID ) ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = self.statuses[ i % len( self.statuses ) ] ftsJob.OperationID = opID if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == 'Active': ftsJob.Completeness = 90 ftsJob.SourceSE = self.ses[ i % len( self.ses ) ] ftsJob.TargetSE = 'PIC-USER' ftsJob.RequestID = 12345 ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = '/a/b/c/%d' % i ftsFile.Size = 1000000 ftsFile.OperationID = opID ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed' ftsFile.RequestID = 12345 ftsFile.Checksum = 'addler' ftsFile.ChecksumType = 'adler32' ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == 'FinishedDirty': ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile( ftsFile ) self.ftsJobs.append( ftsJob ) self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] ) self.ftsClient = FTSClient() self.ftsClient.replicaManager = mock.Mock() self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True, 'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******', 'RAL-USER':'******'}, '/a/b/c/2':{'CERN-USER':'******', 'RAL-USER':'******'}, '/a/b/c/3':{'CERN-USER':'******', 'RAL-USER':'******'} }, 'Failed': {'/a/b/c/4':'/aa/a/b/c/4d', '/a/b/c/5':'/aa/a/b/c/5d'} } }
def setUp(self): """ test case set up """ # ## set some defaults gConfig.setOptionValue('DIRAC/Setup', 'Test') gConfig.setOptionValue('/DIRAC/Setups/Test/DataManagement', 'Test') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac') self.ftsSites = [ FTSSite({ "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" }), FTSSite({ "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" }), FTSSite({ "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" }), FTSSite({ "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" }), FTSSite({ "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" }), FTSSite({ "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" }), FTSSite({ "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" }), FTSSite({ "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" }) ] self.ftsFiles = [] for i in range(100): ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = 9999 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 10 ftsFile.SourceSE = "CERN-USER" ftsFile.TargetSE = "PIC-USER" ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" self.ftsFiles.append(ftsFile) ses = ["CERN-USER", "RAL-USER"] statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ] self.submitted = 0 self.ftsJobs = [] for i in range(7200): ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = statuses[i % len(statuses)] if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == "Active": ftsJob.Completeness = 90 ftsJob.SourceSE = ses[i % len(ses)] ftsJob.TargetSE = "PIC-USER" ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 1000000 ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed" ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == "FinishedDirty": ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile(ftsFile) self.ftsJobs.append(ftsJob) self.submitted = len( [i for i in self.ftsJobs if i.Status == "Submitted"])
def setUp(self): """ test case set up """ gLogger.setLevel('NOTICE') self.ftsSites = [ FTSSite( ftsServer= 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name='CERN.ch'), FTSSite( ftsServer= 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name='PIC.es'), FTSSite( ftsServer= 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name='RAL.uk'), ] self.ses = ['CERN-USER', 'RAL-USER'] self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ] self.submitted = 0 self.numberOfJobs = 10 self.opIDs = [] self.ftsJobs = [] for i in xrange(self.numberOfJobs): opID = i % 3 if opID not in self.opIDs: self.opIDs.append(opID) ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = self.statuses[i % len(self.statuses)] ftsJob.OperationID = opID if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == 'Active': ftsJob.Completeness = 90 ftsJob.SourceSE = self.ses[i % len(self.ses)] ftsJob.TargetSE = 'PIC-USER' ftsJob.RequestID = 12345 ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = '/a/b/c/%d' % i ftsFile.Size = 1000000 ftsFile.OperationID = opID ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed' ftsFile.RequestID = 12345 ftsFile.Checksum = 'addler' ftsFile.ChecksumType = 'adler32' ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == 'FinishedDirty': ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile(ftsFile) self.ftsJobs.append(ftsJob) self.submitted = len( [i for i in self.ftsJobs if i.Status == 'Submitted']) self.ftsClient = FTSClient()