Ejemplo n.º 1
0
  def __createFTSJob( self, guid = None ):
    self.__createFTSFiles()
    ftsJob = FTSJob()
    ftsJob.RequestID = 0
    ftsJob.OperationID = 0
    ftsJob.SourceSE = self.sourceSE
    ftsJob.TargetSE = self.targetSE
    ftsJob.SourceToken = self.sourceToken
    ftsJob.TargetToken = self.targetToken
    ftsJob.FTSServer = self.ftsServer
    if guid:
      ftsJob.FTSGUID = guid

    for ftsFile in self.ftsFiles:
      ftsFile.Attempt += 1
      ftsFile.Error = ""
      ftsJob.addFile( ftsFile )
    self.ftsJob = ftsJob
Ejemplo n.º 2
0
  def __submit( self, request, operation, toSubmit ):
    """ create and submit new FTSJobs using list of FTSFiles

    :param Request request: ReqDB.Request instance
    :param list ftsFiles: list of FTSFile instances

    :return: [ FTSJob, FTSJob, ...]
    """
    log = self.log.getSubLogger( "req_%s/%s/submit" % ( request.RequestID, request.RequestName ) )

    bySourceAndTarget = {}
    for ftsFile in toSubmit:
      if ftsFile.SourceSE not in bySourceAndTarget:
        bySourceAndTarget.setdefault( ftsFile.SourceSE, {} )
      if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]:
        bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] )
      bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile )

    ftsJobs = []

    for source, targetDict in bySourceAndTarget.iteritems():

      for target, ftsFileList in targetDict.iteritems():

        log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) )

        route = self.__ftsPlacement.findRoute( source, target )
        if not route["OK"]:
          log.error( route["Message"] )
          continue
        route = route["Value"]

        routeValid = self.__ftsPlacement.isRouteValid( route )

        if not routeValid['OK']:
          log.error( "Route invalid : %s" % routeValid['Message'] )
          continue

        sourceSE = StorageElement( source )
        sourceToken = sourceSE.getStorageParameters( "SRM2" )
        if not sourceToken["OK"]:
          log.error( "unable to get sourceSE parameters:", "(%s) %s" % ( source, sourceToken["Message"] ) )
          continue
        seStatus = sourceSE.getStatus()['Value']

        targetSE = StorageElement( target )
        targetToken = targetSE.getStorageParameters( "SRM2" )
        if not targetToken["OK"]:
          log.error( "unable to get targetSE parameters:", "(%s) %s" % ( target, targetToken["Message"] ) )
          continue

        # # create FTSJob
        for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ):
          ftsJob = FTSJob()
          ftsJob.RequestID = request.RequestID
          ftsJob.OperationID = operation.OperationID
          ftsJob.SourceSE = source
          ftsJob.TargetSE = target
          ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" )
          ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" )
          ftsJob.FTSServer = route.ftsServer

          for ftsFile in fileList:
            ftsFile.Attempt += 1
            ftsFile.Error = ""
            ftsJob.addFile( ftsFile )

          submit = ftsJob.submitFTS( self.__ftsVersion, command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 )
          if not submit["OK"]:
            log.error( "unable to submit FTSJob:", submit["Message"] )
            continue

          log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) )

          # # update statuses for job files
          for ftsFile in ftsJob:
            ftsFile.FTSGUID = ftsJob.FTSGUID
            ftsFile.Status = "Submitted"
            ftsFile.Attempt += 1



          # # update placement route
          try:
            self.updateLock().acquire()
            self.__ftsPlacement.startTransferOnRoute( route )
          finally:
            self.updateLock().release()

          ftsJobs.append( ftsJob )

    log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) )
    return S_OK( ftsJobs )
Ejemplo n.º 3
0
    def __submit(self, request, operation, toSubmit):
        """ create and submit new FTSJobs using list of FTSFiles

    :param Request request: ReqDB.Request instance
    :param list ftsFiles: list of FTSFile instances

    :return: [ FTSJob, FTSJob, ...]
    """
        log = self.log.getSubLogger("%s/submit" % request.RequestName)

        bySourceAndTarget = {}
        for ftsFile in toSubmit:
            if ftsFile.SourceSE not in bySourceAndTarget:
                bySourceAndTarget.setdefault(ftsFile.SourceSE, {})
            if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]:
                bySourceAndTarget[ftsFile.SourceSE].setdefault(
                    ftsFile.TargetSE, [])
            bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append(
                ftsFile)

        ftsJobs = []

        for source, targetDict in bySourceAndTarget.items():

            for target, ftsFileList in targetDict.items():

                log.info("found %s files to submit from %s to %s" %
                         (len(ftsFileList), source, target))

                route = self.__ftsGraph.findRoute(source, target)
                if not route["OK"]:
                    log.error(route["Message"])
                    continue
                route = route["Value"]

                sourceRead = route.fromNode.SEs[source]["read"]
                if not sourceRead:
                    log.error("SourceSE %s is banned for reading right now" %
                              source)
                    continue

                targetWrite = route.toNode.SEs[target]["write"]
                if not targetWrite:
                    log.error("TargetSE %s is banned for writing right now" %
                              target)
                    continue

                if route.ActiveJobs > route.toNode.MaxActiveJobs:
                    log.warn(
                        "unable to submit new FTS job, max active jobs reached"
                    )
                    continue

                # # create FTSJob
                ftsJob = FTSJob()
                ftsJob.RequestID = request.RequestID
                ftsJob.OperationID = operation.OperationID
                ftsJob.SourceSE = source
                ftsJob.TargetSE = target

                sourceSE = self.getSE(source)
                sourceToken = sourceSE.getStorageParameters("SRM2")
                if not sourceToken["OK"]:
                    log.error("unable to get sourceSE '%s' parameters: %s" %
                              (source, sourceToken["Message"]))
                    continue
                ftsJob.SourceToken = sourceToken["Value"].get("SpaceToken", "")

                targetSE = self.getSE(target)
                targetToken = targetSE.getStorageParameters("SRM2")
                if not targetToken["OK"]:
                    log.error("unable to get targetSE '%s' parameters: %s" %
                              (target, targetToken["Message"]))
                    continue
                ftsJob.TargetToken = targetToken["Value"].get("SpaceToken", "")

                ftsJob.FTSServer = route.toNode.FTSServer

                for ftsFile in ftsFileList:
                    ftsFile.Attempt += 1
                    ftsFile.Error = ""
                    ftsJob.addFile(ftsFile)

                submit = ftsJob.submitFTS2(self.STAGE_FILES)
                if not submit["OK"]:
                    log.error("unable to submit FTSJob: %s" %
                              submit["Message"])
                    continue

                log.info("FTSJob '%s'@'%s' has been submitted" %
                         (ftsJob.FTSGUID, ftsJob.FTSServer))

                # # update statuses for job files
                for ftsFile in ftsJob:
                    ftsFile.FTSGUID = ftsJob.FTSGUID
                    ftsFile.Status = "Submitted"
                    ftsFile.Attempt += 1

                # # update graph route
                try:
                    self.updateLock().acquire()
                    route.ActiveJobs += 1
                finally:
                    self.updateLock().release()

                ftsJobs.append(ftsJob)

        log.info("%s new FTSJobs have been submitted" % len(ftsJobs))
        return S_OK(ftsJobs)
Ejemplo n.º 4
0
  def __submit( self, request, operation, toSubmit ):
    """ create and submit new FTSJobs using list of FTSFiles

    :param Request request: ReqDB.Request instance
    :param list ftsFiles: list of FTSFile instances

    :return: [ FTSJob, FTSJob, ...]
    """
    log = self.log.getSubLogger( "%s/submit" % request.RequestName )

    bySourceAndTarget = {}
    for ftsFile in toSubmit:
      if ftsFile.SourceSE not in bySourceAndTarget:
        bySourceAndTarget.setdefault( ftsFile.SourceSE, {} )
      if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]:
        bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] )
      bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile )

    ftsJobs = []

    for source, targetDict in bySourceAndTarget.items():

      for target, ftsFileList in targetDict.items():

        log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) )

        route = self.__ftsGraph.findRoute( source, target )
        if not route["OK"]:
          log.error( route["Message"] )
          continue
        route = route["Value"]

        sourceRead = route.fromNode.SEs[source]["read"]
        if not sourceRead:
          log.error( "SourceSE %s is banned for reading right now" % source )
          continue

        targetWrite = route.toNode.SEs[target]["write"]
        if not targetWrite:
          log.error( "TargetSE %s is banned for writing right now" % target )
          continue

        if route.ActiveJobs > route.toNode.MaxActiveJobs:
          log.warn( "unable to submit new FTS job, max active jobs reached" )
          continue

        sourceSE = self.getSE( source )
        sourceToken = sourceSE.getStorageParameters( "SRM2" )
        if not sourceToken["OK"]:
          log.error( "unable to get sourceSE '%s' parameters: %s" % ( source, sourceToken["Message"] ) )
          continue
        seStatus = sourceSE.getStatus()['Value']

        targetSE = self.getSE( target )
        targetToken = targetSE.getStorageParameters( "SRM2" )
        if not targetToken["OK"]:
          log.error( "unable to get targetSE '%s' parameters: %s" % ( target, targetToken["Message"] ) )
          continue

        # # create FTSJob
        for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ):
          ftsJob = FTSJob()
          ftsJob.RequestID = request.RequestID
          ftsJob.OperationID = operation.OperationID
          ftsJob.SourceSE = source
          ftsJob.TargetSE = target
          ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" )
          ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" )
          ftsJob.FTSServer = route.toNode.FTSServer

          for ftsFile in fileList:
            ftsFile.Attempt += 1
            ftsFile.Error = ""
            ftsJob.addFile( ftsFile )

          submit = ftsJob.submitFTS2( command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 )
          if not submit["OK"]:
            log.error( "unable to submit FTSJob: %s" % submit["Message"] )
            continue

          log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) )

          # # update statuses for job files
          for ftsFile in ftsJob:
            ftsFile.FTSGUID = ftsJob.FTSGUID
            ftsFile.Status = "Submitted"
            ftsFile.Attempt += 1

          # # update graph route
          try:
            self.updateLock().acquire()
            route.ActiveJobs += 1
          finally:
            self.updateLock().release()

          ftsJobs.append( ftsJob )

    log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) )
    return S_OK( ftsJobs )
Ejemplo n.º 5
0
  def setUp( self ):
    """ test case set up """

    gLogger.setLevel( 'NOTICE' )

    self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ),
                      FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ),
                      FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ),
                    ]

    self.ses = [ 'CERN-USER', 'RAL-USER' ]
    self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ]

    self.submitted = 0
    self.numberOfJobs = 10
    self.opIDs = []

    self.ftsJobs = []
    for i in xrange( self.numberOfJobs ):

      opID = i % 3
      if opID not in self.opIDs:
        self.opIDs.append( opID )

      ftsJob = FTSJob()
      ftsJob.FTSGUID = str( uuid.uuid4() )
      ftsJob.FTSServer = self.ftsSites[0].FTSServer
      ftsJob.Status = self.statuses[ i % len( self.statuses ) ]
      ftsJob.OperationID = opID
      if ftsJob.Status in FTSJob.FINALSTATES:
        ftsJob.Completeness = 100
      if ftsJob.Status == 'Active':
        ftsJob.Completeness = 90
      ftsJob.SourceSE = self.ses[ i % len( self.ses ) ]
      ftsJob.TargetSE = 'PIC-USER'
      ftsJob.RequestID = 12345

      ftsFile = FTSFile()
      ftsFile.FileID = i + 1
      ftsFile.OperationID = i + 1
      ftsFile.LFN = '/a/b/c/%d' % i
      ftsFile.Size = 1000000
      ftsFile.OperationID = opID
      ftsFile.SourceSE = ftsJob.SourceSE
      ftsFile.TargetSE = ftsJob.TargetSE
      ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
      ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
      ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
      ftsFile.RequestID = 12345
      ftsFile.Checksum = 'addler'
      ftsFile.ChecksumType = 'adler32'

      ftsFile.FTSGUID = ftsJob.FTSGUID
      if ftsJob.Status == 'FinishedDirty':
        ftsJob.FailedFiles = 1
        ftsJob.FailedSize = ftsFile.Size

      ftsJob.addFile( ftsFile )
      self.ftsJobs.append( ftsJob )

    self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] )

    self.ftsClient = FTSClient()
    self.ftsClient.replicaManager = mock.Mock()
    self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True,
                                                                    'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******',
                                                                                                         'RAL-USER':'******'},
                                                                                             '/a/b/c/2':{'CERN-USER':'******',
                                                                                                         'RAL-USER':'******'},
                                                                                             '/a/b/c/3':{'CERN-USER':'******',
                                                                                                         'RAL-USER':'******'}
                                                                                             },
                                                                              'Failed': {'/a/b/c/4':'/aa/a/b/c/4d',
                                                                                         '/a/b/c/5':'/aa/a/b/c/5d'}
                                                                              }
                                                                    }
Ejemplo n.º 6
0
  def __submit( self, request, operation, toSubmit ):
    """ create and submit new FTSJobs using list of FTSFiles

    :param Request request: ReqDB.Request instance
    :param list ftsFiles: list of FTSFile instances

    :return: [ FTSJob, FTSJob, ...]
    """
    log = self.log.getSubLogger( "req_%s/%s/submit" % ( request.RequestID, request.RequestName ) )

    bySourceAndTarget = {}
    for ftsFile in toSubmit:
      if ftsFile.SourceSE not in bySourceAndTarget:
        bySourceAndTarget.setdefault( ftsFile.SourceSE, {} )
      if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]:
        bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] )
      bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile )

    ftsJobs = []

    for source, targetDict in bySourceAndTarget.iteritems():

      for target, ftsFileList in targetDict.iteritems():

        log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) )

        route = self.__ftsPlacement.findRoute( source, target )
        if not route["OK"]:
          log.error( route["Message"] )
          continue
        route = route["Value"]

        routeValid = self.__ftsPlacement.isRouteValid( route )

        if not routeValid['OK']:
          log.error( "Route invalid : %s" % routeValid['Message'] )
          continue

        sourceSE = StorageElement( source )
        sourceToken = sourceSE.getStorageParameters( protocol = 'srm' )
        if not sourceToken["OK"]:
          log.error( "unable to get sourceSE parameters:", "(%s) %s" % ( source, sourceToken["Message"] ) )
          continue
        seStatus = sourceSE.getStatus()['Value']

        targetSE = StorageElement( target )
        targetToken = targetSE.getStorageParameters( protocol = 'srm' )
        if not targetToken["OK"]:
          log.error( "unable to get targetSE parameters:", "(%s) %s" % ( target, targetToken["Message"] ) )
          continue

        # # create FTSJob
        for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ):
          ftsJob = FTSJob()
          ftsJob.RequestID = request.RequestID
          ftsJob.OperationID = operation.OperationID
          ftsJob.SourceSE = source
          ftsJob.TargetSE = target
          ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" )
          ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" )
          ftsJob.FTSServer = route.ftsServer

          for ftsFile in fileList:
            ftsFile.Attempt += 1
            ftsFile.Error = ""
            ftsJob.addFile( ftsFile )

          submit = ftsJob.submitFTS( self.__ftsVersion, command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 )
          if not submit["OK"]:
            log.error( "unable to submit FTSJob:", submit["Message"] )
            continue

          log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) )

          # # update statuses for job files
          for ftsFile in ftsJob:
            ftsFile.FTSGUID = ftsJob.FTSGUID
            ftsFile.Status = "Submitted"
            ftsFile.Attempt += 1



          # # update placement route
          try:
            self.updateLock().acquire()
            self.__ftsPlacement.startTransferOnRoute( route )
          finally:
            self.updateLock().release()

          ftsJobs.append( ftsJob )

    log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) )
    return S_OK( ftsJobs )
Ejemplo n.º 7
0
    def setUp(self):
        """ test case set up """

        gLogger.setLevel('NOTICE')

        self.ftsSites = [
            FTSSite(
                ftsServer=
                'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer',
                name='CERN.ch'),
            FTSSite(
                ftsServer=
                'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer',
                name='PIC.es'),
            FTSSite(
                ftsServer=
                'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer',
                name='RAL.uk'),
        ]

        self.ses = ['CERN-USER', 'RAL-USER']
        self.statuses = [
            'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready'
        ]

        self.submitted = 0
        self.numberOfJobs = 10
        self.opIDs = []

        self.ftsJobs = []
        for i in xrange(self.numberOfJobs):

            opID = i % 3
            if opID not in self.opIDs:
                self.opIDs.append(opID)

            ftsJob = FTSJob()
            ftsJob.FTSGUID = str(uuid.uuid4())
            ftsJob.FTSServer = self.ftsSites[0].FTSServer
            ftsJob.Status = self.statuses[i % len(self.statuses)]
            ftsJob.OperationID = opID
            if ftsJob.Status in FTSJob.FINALSTATES:
                ftsJob.Completeness = 100
            if ftsJob.Status == 'Active':
                ftsJob.Completeness = 90
            ftsJob.SourceSE = self.ses[i % len(self.ses)]
            ftsJob.TargetSE = 'PIC-USER'
            ftsJob.RequestID = 12345

            ftsFile = FTSFile()
            ftsFile.FileID = i + 1
            ftsFile.OperationID = i + 1
            ftsFile.LFN = '/a/b/c/%d' % i
            ftsFile.Size = 1000000
            ftsFile.OperationID = opID
            ftsFile.SourceSE = ftsJob.SourceSE
            ftsFile.TargetSE = ftsJob.TargetSE
            ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
            ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
            ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
            ftsFile.RequestID = 12345
            ftsFile.Checksum = 'addler'
            ftsFile.ChecksumType = 'adler32'

            ftsFile.FTSGUID = ftsJob.FTSGUID
            if ftsJob.Status == 'FinishedDirty':
                ftsJob.FailedFiles = 1
                ftsJob.FailedSize = ftsFile.Size

            ftsJob.addFile(ftsFile)
            self.ftsJobs.append(ftsJob)

        self.submitted = len(
            [i for i in self.ftsJobs if i.Status == 'Submitted'])

        self.ftsClient = FTSClient()