コード例 #1
0
ファイル: FTSRequest.py プロジェクト: Teddy22/DIRAC
  def __createFTSJob( self, guid = None ):
    self.__createFTSFiles()
    ftsJob = FTSJob()
    ftsJob.RequestID = 0
    ftsJob.OperationID = 0
    ftsJob.SourceSE = self.sourceSE
    ftsJob.TargetSE = self.targetSE
    ftsJob.SourceToken = self.sourceToken
    ftsJob.TargetToken = self.targetToken
    ftsJob.FTSServer = self.ftsServer
    if guid:
      ftsJob.FTSGUID = guid

    for ftsFile in self.ftsFiles:
      ftsFile.Attempt += 1
      ftsFile.Error = ""
      ftsJob.addFile( ftsFile )
    self.ftsJob = ftsJob
コード例 #2
0
  def test02Files( self ):
    """ FTSFiles arithmetic """
    ftsJob = FTSJob()
    ftsJob.FTSGUID = str( uuid.uuid4() )

    self.assertEqual( len( ftsJob ), 0, "1. len(ftsJob) error" )
    self.assertEqual( ftsJob.Files, 0 , "1. Files prop error" )
    self.assertEqual( ftsJob.Size, 0, "1. Size prop error" )

    ftsJob.addFile( self.fileA )
    ftsJob.addFile( self.fileB )

    self.assertEqual( self.fileA.FTSGUID, ftsJob.FTSGUID, "FTSGUID mismatch for fileA" )
    self.assertEqual( self.fileB.FTSGUID, ftsJob.FTSGUID, "FTSGUID mismatch for fileB" )

    self.assertEqual( len( ftsJob ), 2, "2. len(ftsJob) error" )
    self.assertEqual( ftsJob.Files, 2, "2. Files prop error" )
    self.assertEqual( ftsJob.Size, 20, "2. Size prop error" )
コード例 #3
0
ファイル: FTSRequest.py プロジェクト: vingar/DIRAC
  def __createFTSJob( self, guid = None ):
    self.__createFTSFiles()
    ftsJob = FTSJob()
    ftsJob.RequestID = 0
    ftsJob.OperationID = 0
    ftsJob.SourceSE = self.sourceSE
    ftsJob.TargetSE = self.targetSE
    ftsJob.SourceToken = self.sourceToken
    ftsJob.TargetToken = self.targetToken
    ftsJob.FTSServer = self.ftsServer
    if guid:
      ftsJob.FTSGUID = guid

    for ftsFile in self.ftsFiles:
      ftsFile.Attempt += 1
      ftsFile.Error = ""
      ftsJob.addFile( ftsFile )
    self.ftsJob = ftsJob
コード例 #4
0
    def test02Files(self):
        """ FTSFiles arithmetic """
        ftsJob = FTSJob()
        ftsJob.FTSGUID = str(uuid.uuid4())

        self.assertEqual(len(ftsJob), 0, "1. len(ftsJob) error")
        self.assertEqual(ftsJob.Files, 0, "1. Files prop error")
        self.assertEqual(ftsJob.Size, 0, "1. Size prop error")

        ftsJob.addFile(self.fileA)
        ftsJob.addFile(self.fileB)

        self.assertEqual(self.fileA.FTSGUID, ftsJob.FTSGUID,
                         "FTSGUID mismatch for fileA")
        self.assertEqual(self.fileB.FTSGUID, ftsJob.FTSGUID,
                         "FTSGUID mismatch for fileB")

        self.assertEqual(len(ftsJob), 2, "2. len(ftsJob) error")
        self.assertEqual(ftsJob.Files, 2, "2. Files prop error")
        self.assertEqual(ftsJob.Size, 20, "2. Size prop error")
コード例 #5
0
ファイル: FTSDBTests.py プロジェクト: IgorPelevanyuk/DIRAC
  def setUp( self ):
    """ test case set up """
    # ## set some defaults
    gConfig.setOptionValue( 'DIRAC/Setup', 'Test' )
    gConfig.setOptionValue( '/DIRAC/Setups/Test/DataManagement', 'Test' )
    gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost' )
    gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB' )
    gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac' )

    self.ftsSites = [ FTSSite( { "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" } ),
                      FTSSite( { "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" } ),
                      FTSSite( { "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" } ),
                      FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" } ),
                      FTSSite( { "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" } ),
                      FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" } ),
                      FTSSite( { "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" } ),
                      FTSSite( { "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" } ) ]


    self.ftsFiles = []
    for i in range ( 100 ):
      ftsFile = FTSFile()
      ftsFile.FileID = i + 1
      ftsFile.OperationID = 9999
      ftsFile.LFN = "/a/b/c/%d" % i
      ftsFile.Size = 10
      ftsFile.SourceSE = "CERN-USER"
      ftsFile.TargetSE = "PIC-USER"
      ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
      ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
      ftsFile.Status = "Waiting"
      self.ftsFiles.append( ftsFile )

    ses = [ "CERN-USER", "RAL-USER" ]
    statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ]

    self.submitted = 0

    self.ftsJobs = []
    for i in range( 7200 ):

      ftsJob = FTSJob()
      ftsJob.FTSGUID = str( uuid.uuid4() )
      ftsJob.FTSServer = self.ftsSites[0].FTSServer
      ftsJob.Status = statuses[ i % len( statuses ) ]

      if ftsJob.Status in FTSJob.FINALSTATES:
        ftsJob.Completeness = 100
      if ftsJob.Status == "Active":
        ftsJob.Completeness = 90

      ftsJob.SourceSE = ses[ i % len( ses ) ]
      ftsJob.TargetSE = "PIC-USER"

      ftsFile = FTSFile()
      ftsFile.FileID = i + 1
      ftsFile.OperationID = i + 1
      ftsFile.LFN = "/a/b/c/%d" % i
      ftsFile.Size = 1000000
      ftsFile.SourceSE = ftsJob.SourceSE
      ftsFile.TargetSE = ftsJob.TargetSE
      ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
      ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
      ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed"

      ftsFile.FTSGUID = ftsJob.FTSGUID
      if ftsJob.Status == "FinishedDirty":
        ftsJob.FailedFiles = 1
        ftsJob.FailedSize = ftsFile.Size

      ftsJob.addFile( ftsFile )
      self.ftsJobs.append( ftsJob )

    self.submitted = len( [ i for i in self.ftsJobs if i.Status == "Submitted" ] )
コード例 #6
0
ファイル: TestClientFTS.py プロジェクト: DIRACGrid/TestDIRAC
  def setUp( self ):
    """ test case set up """

    gLogger.setLevel( 'NOTICE' )

    self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ),
                      FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ),
                      FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ),
                    ]

    self.ses = [ 'CERN-USER', 'RAL-USER' ]
    self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ]

    self.submitted = 0
    self.numberOfJobs = 10
    self.opIDs = []

    self.ftsJobs = []
    for i in xrange( self.numberOfJobs ):

      opID = i % 3
      if opID not in self.opIDs:
        self.opIDs.append( opID )

      ftsJob = FTSJob()
      ftsJob.FTSGUID = str( uuid.uuid4() )
      ftsJob.FTSServer = self.ftsSites[0].FTSServer
      ftsJob.Status = self.statuses[ i % len( self.statuses ) ]
      ftsJob.OperationID = opID
      if ftsJob.Status in FTSJob.FINALSTATES:
        ftsJob.Completeness = 100
      if ftsJob.Status == 'Active':
        ftsJob.Completeness = 90
      ftsJob.SourceSE = self.ses[ i % len( self.ses ) ]
      ftsJob.TargetSE = 'PIC-USER'
      ftsJob.RequestID = 12345

      ftsFile = FTSFile()
      ftsFile.FileID = i + 1
      ftsFile.OperationID = i + 1
      ftsFile.LFN = '/a/b/c/%d' % i
      ftsFile.Size = 1000000
      ftsFile.OperationID = opID
      ftsFile.SourceSE = ftsJob.SourceSE
      ftsFile.TargetSE = ftsJob.TargetSE
      ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
      ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
      ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
      ftsFile.RequestID = 12345
      ftsFile.Checksum = 'addler'
      ftsFile.ChecksumType = 'adler32'

      ftsFile.FTSGUID = ftsJob.FTSGUID
      if ftsJob.Status == 'FinishedDirty':
        ftsJob.FailedFiles = 1
        ftsJob.FailedSize = ftsFile.Size

      ftsJob.addFile( ftsFile )
      self.ftsJobs.append( ftsJob )

    self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] )

    self.ftsClient = FTSClient()
    self.ftsClient.replicaManager = mock.Mock()
    self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True,
                                                                    'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******',
                                                                                                         'RAL-USER':'******'},
                                                                                             '/a/b/c/2':{'CERN-USER':'******',
                                                                                                         'RAL-USER':'******'},
                                                                                             '/a/b/c/3':{'CERN-USER':'******',
                                                                                                         'RAL-USER':'******'}
                                                                                             },
                                                                              'Failed': {'/a/b/c/4':'/aa/a/b/c/4d',
                                                                                         '/a/b/c/5':'/aa/a/b/c/5d'}
                                                                              }
                                                                    }
コード例 #7
0
ファイル: FTSDBTests.py プロジェクト: sposs/DIRAC
    def setUp(self):
        """ test case set up """
        # ## set some defaults
        gConfig.setOptionValue('DIRAC/Setup', 'Test')
        gConfig.setOptionValue('/DIRAC/Setups/Test/DataManagement', 'Test')
        gConfig.setOptionValue(
            '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost')
        gConfig.setOptionValue(
            '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB')
        gConfig.setOptionValue(
            '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac')

        self.ftsSites = [
            FTSSite({
                "FTSServer":
                "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "CERN.ch"
            }),
            FTSSite({
                "FTSServer":
                "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "PIC.es"
            }),
            FTSSite({
                "FTSServer":
                "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "RAL.uk"
            }),
            FTSSite({
                "FTSServer":
                "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "SARA.nl"
            }),
            FTSSite({
                "FTSServer":
                "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "CNAF.it"
            }),
            FTSSite({
                "FTSServer":
                "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "NIKHEF.nl"
            }),
            FTSSite({
                "FTSServer":
                "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "GRIDKA.de"
            }),
            FTSSite({
                "FTSServer":
                "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer",
                "Name": "IN2P3.fr"
            })
        ]

        self.ftsFiles = []
        for i in range(100):
            ftsFile = FTSFile()
            ftsFile.FileID = i + 1
            ftsFile.OperationID = 9999
            ftsFile.LFN = "/a/b/c/%d" % i
            ftsFile.Size = 10
            ftsFile.SourceSE = "CERN-USER"
            ftsFile.TargetSE = "PIC-USER"
            ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
            ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
            ftsFile.Status = "Waiting"
            self.ftsFiles.append(ftsFile)

        ses = ["CERN-USER", "RAL-USER"]
        statuses = [
            "Submitted", "Finished", "FinishedDirty", "Active", "Ready"
        ]

        self.submitted = 0

        self.ftsJobs = []
        for i in range(7200):

            ftsJob = FTSJob()
            ftsJob.FTSGUID = str(uuid.uuid4())
            ftsJob.FTSServer = self.ftsSites[0].FTSServer
            ftsJob.Status = statuses[i % len(statuses)]

            if ftsJob.Status in FTSJob.FINALSTATES:
                ftsJob.Completeness = 100
            if ftsJob.Status == "Active":
                ftsJob.Completeness = 90

            ftsJob.SourceSE = ses[i % len(ses)]
            ftsJob.TargetSE = "PIC-USER"

            ftsFile = FTSFile()
            ftsFile.FileID = i + 1
            ftsFile.OperationID = i + 1
            ftsFile.LFN = "/a/b/c/%d" % i
            ftsFile.Size = 1000000
            ftsFile.SourceSE = ftsJob.SourceSE
            ftsFile.TargetSE = ftsJob.TargetSE
            ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
            ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
            ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed"

            ftsFile.FTSGUID = ftsJob.FTSGUID
            if ftsJob.Status == "FinishedDirty":
                ftsJob.FailedFiles = 1
                ftsJob.FailedSize = ftsFile.Size

            ftsJob.addFile(ftsFile)
            self.ftsJobs.append(ftsJob)

        self.submitted = len(
            [i for i in self.ftsJobs if i.Status == "Submitted"])
コード例 #8
0
    def setUp(self):
        """ test case set up """

        gLogger.setLevel('NOTICE')

        self.ftsSites = [
            FTSSite(
                ftsServer=
                'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer',
                name='CERN.ch'),
            FTSSite(
                ftsServer=
                'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer',
                name='PIC.es'),
            FTSSite(
                ftsServer=
                'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer',
                name='RAL.uk'),
        ]

        self.ses = ['CERN-USER', 'RAL-USER']
        self.statuses = [
            'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready'
        ]

        self.submitted = 0
        self.numberOfJobs = 10
        self.opIDs = []

        self.ftsJobs = []
        for i in xrange(self.numberOfJobs):

            opID = i % 3
            if opID not in self.opIDs:
                self.opIDs.append(opID)

            ftsJob = FTSJob()
            ftsJob.FTSGUID = str(uuid.uuid4())
            ftsJob.FTSServer = self.ftsSites[0].FTSServer
            ftsJob.Status = self.statuses[i % len(self.statuses)]
            ftsJob.OperationID = opID
            if ftsJob.Status in FTSJob.FINALSTATES:
                ftsJob.Completeness = 100
            if ftsJob.Status == 'Active':
                ftsJob.Completeness = 90
            ftsJob.SourceSE = self.ses[i % len(self.ses)]
            ftsJob.TargetSE = 'PIC-USER'
            ftsJob.RequestID = 12345

            ftsFile = FTSFile()
            ftsFile.FileID = i + 1
            ftsFile.OperationID = i + 1
            ftsFile.LFN = '/a/b/c/%d' % i
            ftsFile.Size = 1000000
            ftsFile.OperationID = opID
            ftsFile.SourceSE = ftsJob.SourceSE
            ftsFile.TargetSE = ftsJob.TargetSE
            ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
            ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
            ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
            ftsFile.RequestID = 12345
            ftsFile.Checksum = 'addler'
            ftsFile.ChecksumType = 'adler32'

            ftsFile.FTSGUID = ftsJob.FTSGUID
            if ftsJob.Status == 'FinishedDirty':
                ftsJob.FailedFiles = 1
                ftsJob.FailedSize = ftsFile.Size

            ftsJob.addFile(ftsFile)
            self.ftsJobs.append(ftsJob)

        self.submitted = len(
            [i for i in self.ftsJobs if i.Status == 'Submitted'])

        self.ftsClient = FTSClient()