예제 #1
0
  def submitTransfer( self, channelDict ):
    """ create and submit FTS jobs based on information it gets from the DB

    :param self: self reference
    :param dict channelDict: dict with channel info as read from TransferDB.selectChannelsForSubmission
    """

    # Create the FTSRequest object for preparing the submission
    oFTSRequest = FTSRequest()
    channelID = channelDict['ChannelID']
    filesPerJob = channelDict['NumFiles']

    #########################################################################
    #  Obtain the first files in the selected channel.
    self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s" % channelID )
    res = self.transferDB.getFilesForChannel( channelID, 2 * filesPerJob )
    if not res['OK']:
      errStr = 'FTSSubmitAgent.%s' % res['Message']
      self.log.error( errStr )
      return S_OK()
    if not res['Value']:
      self.log.info( "FTSSubmitAgent.submitTransfer: No files to found for channel." )
      return S_OK()
    filesDict = res['Value']
    self.log.info( 'Obtained %s files for channel' % len( filesDict['Files'] ) )

    sourceSE = filesDict['SourceSE']
    oFTSRequest.setSourceSE( sourceSE )
    targetSE = filesDict['TargetSE']
    oFTSRequest.setTargetSE( targetSE )
    self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel." % ( sourceSE, 
                                                                                                         targetSE ) )
    files = filesDict['Files']

    ## enable/disable cksm test
    oFTSRequest.setCksmTest( self.cksmTest )
    if self.cksmType:
      oFTSRequest.setCksmType( self.cksmType )

    #########################################################################
    #  Populate the FTS Request with the files.
    self.log.info( 'Populating the FTS request with file information' )
    fileIDs = []
    totalSize = 0
    fileIDSizes = {}
    for fileMeta in files:
      lfn = fileMeta['LFN']
      oFTSRequest.setLFN( lfn )
      oFTSRequest.setSourceSURL( lfn, fileMeta['SourceSURL'] )
      oFTSRequest.setTargetSURL( lfn, fileMeta['TargetSURL'] )
      fileID = fileMeta['FileID']
      fileIDs.append( fileID )
      totalSize += fileMeta['Size']
      fileIDSizes[fileID] = fileMeta['Size']

    #########################################################################
    #  Submit the FTS request and retrieve the FTS GUID/Server
    self.log.info( 'Submitting the FTS request' )
    res = oFTSRequest.submit()
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )
      self.log.info( 'Updating the Channel table for files to retry' )
      res = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
      if not res['OK']:
        self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
      return S_ERROR( errStr )
    ftsGUID = res['Value']['ftsGUID']
    ftsServer = res['Value']['ftsServer']
    infoStr = """Submitted FTS Job:
    
              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( len( files ) ) )
    self.log.info( infoStr )

    ## filter out skipped files
    failedFiles = oFTSRequest.getFailed()
    if not failedFiles["OK"]:
      self.log.warn("Unable to read skipped LFNs.")
    failedFiles = failedFiles["Value"] if "Value" in failedFiles else []
    failedIDs = [ meta["FileID"] for meta in files if meta["LFN"] in failedFiles ]
    ## only submitted
    fileIDs = [ fileID for fileID in fileIDs if fileID not in failedIDs ]
    ## sub failed from total size
    totalSize -= sum( [ meta["Size"] for meta in files if meta["LFN"] in failedFiles ] )

    #########################################################################
    #  Insert the FTS Req details and add the number of files and size
    res = self.transferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )
      return S_ERROR( errStr )
    ftsReqID = res['Value']
    self.log.info( 'Obtained FTS RequestID %s' % ftsReqID )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
    if not res['OK']:
      self.log.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
    if not res['OK']:
      self.log.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( fileIDs ) )
    if not res['OK']:
      self.log.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
    if not res['OK']:
      self.log.error( "Failed to set TotalSize for FTSRequest", res['Message'] )

    #########################################################################
    #  Insert the submission event in the FTSReqLogging table
    event = 'Submitted'
    res = self.transferDB.addLoggingEvent( ftsReqID, event )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )

    #########################################################################
    #  Insert the FileToFTS details and remove the files from the channel
    self.log.info( 'Setting the files as Executing in the Channel table' )
    res = self.transferDB.setChannelFilesExecuting( channelID, fileIDs )
    if not res['OK']:
      self.log.error( 'Failed to update the Channel tables for files.', res['Message'] )

    lfns = []
    fileToFTSFileAttributes = []
    for fileMeta in files:
      lfn = fileMeta['LFN']
      fileID = fileMeta['FileID']
      lfns.append( lfn )
      fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )

    self.log.info( 'Populating the FileToFTS table with file information' )
    res = self.transferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
    if not res['OK']:
      self.log.error( 'Failed to populate the FileToFTS table with files.' )
예제 #2
0
  def submitTransfer( self, channelDict ):
    """ create and submit FTS jobs based on information it gets from the DB

    :param self: self reference
    :param dict channelDict: dict with channel info as read from TransferDB.selectChannelsForSubmission
    """

    channelID = channelDict['ChannelID']
    filesPerJob = channelDict['NumFiles']

    #########################################################################
    #  Obtain the first files in the selected channel.
    self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for channel %s: %s to %s"
                   % ( channelID, channelDict['Source'], channelDict['Destination'] ) )
    res = self.transferDB.getFilesForChannel( channelID, 2 * filesPerJob )
    if not res['OK']:
      errStr = 'FTSSubmitAgent.%s' % res['Message']
      self.log.error( errStr )
      return S_OK()
    if not res['Value']:
      self.log.info( "FTSSubmitAgent.submitTransfer: No files found for channel." )
      return S_OK()
    filesDict = res['Value']
    sourceSE = filesDict['SourceSE']
    targetSE = filesDict['TargetSE']
    self.log.info( 'Obtained %s files for channel %s to %s' % ( len( filesDict['Files'] ), sourceSE, targetSE ) )

    # Create the FTSRequest object for preparing the submission
    oFTSRequest = FTSRequest()
    oFTSRequest.setSourceSE( sourceSE )
    oFTSRequest.setTargetSE( targetSE )
    files = filesDict['Files']

    # # enable/disable cksm test
    oFTSRequest.setCksmTest( self.cksmTest )
    if self.cksmType:
      oFTSRequest.setCksmType( self.cksmType )

    #########################################################################
    #  Populate the FTS Request with the files.
    self.log.info( 'Populating the FTS request with file information' )
    fileIDs = []
    totalSize = 0
    fileIDSizes = {}
    lfns = set()
    for fileMeta in files:
      lfn = fileMeta['LFN']
      lfns.add( lfn )
      oFTSRequest.setLFN( lfn )
      oFTSRequest.setSourceSURL( lfn, fileMeta['SourceSURL'] )
      oFTSRequest.setTargetSURL( lfn, fileMeta['TargetSURL'] )
      if lfn in self.filesBeingStaged:
        oFTSRequest.setStatus( lfn, 'Staging' )
      fileID = fileMeta['FileID']
      fileIDs.append( fileID )
      totalSize += fileMeta['Size']
      fileIDSizes[fileID] = fileMeta['Size']

    oFTSRequest.resolveSource()
    noSource = [ lfn for lfn, fileInfo in oFTSRequest.fileDict.items()
                     if fileInfo.get( "Status", "" ) == "Failed" and fileInfo.get( "Reason", "" ) in ( "No replica at SourceSE",
                                                                                                   "Source file does not exist" ) ]
    toReschedule = [fileMeta["FileID"] for fileMeta in files if fileMeta["LFN"] in noSource]

    if toReschedule:
      self.log.info( "Found %s files to reschedule" % len( toReschedule ) )
      for fileID in toReschedule:
        res = self.transferDB.setFileToReschedule( fileID )
        if not res["OK"]:
          self.log.error( "Failed to update Channel table for failed files.", res["Message"] )
        elif res["Value"] == "max reschedule attempt reached":
          self.log.error( "setting Channel status to 'Failed' : " % res["Value"] )
          res = self.transferDB.setFileChannelStatus( channelID, fileID, 'Failed' )
          if not res["OK"]:
            self.log.error( "Failed to update Channel table for failed files.", res["Message"] )

    #########################################################################
    #  Submit the FTS request and retrieve the FTS GUID/Server
    self.log.info( 'Submitting the FTS request' )
    res = oFTSRequest.submit()
    if not res['OK']:
      errStr = "FTSSubmitAgent.submit: %s" % res['Message']
      self.log.error( errStr )
      self.log.info( 'Updating the Channel table for files to retry' )
      res = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
      if not res['OK']:
        self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
      return S_ERROR( errStr )
    ftsGUID = res['Value']['ftsGUID']
    ftsServer = res['Value']['ftsServer']
    nbSubmitted = res['Value']['submittedFiles']
    infoStr = """Submitted FTS Job:

              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( nbSubmitted ) )
    self.log.info( infoStr )

    # # filter out skipped files
    failedFiles = oFTSRequest.getFailed()['Value']
    stagingFiles = oFTSRequest.getStaging()['Value']
    # cache files being staged
    self.filesBeingStaged.update( stagingFiles )
    submittedFiles = lfns.difference( failedFiles, stagingFiles )
    # files being submitted are staged
    self.filesBeingStaged -= submittedFiles
    failedIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in failedFiles ] )
    stagingIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in stagingFiles ] )
    # # only submitted
    submittedIDs = set( fileIDs ) - failedIDs - stagingIDs
    # # only count the submitted size
    totalSize = sum( [ meta["Size"] for meta in files if meta["FileID"] in submittedIDs ] )

    #########################################################################
    #  Insert the FTS Req details and add the number of files and size
    res = self.transferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )
      return S_ERROR( errStr )
    ftsReqID = res['Value']
    self.log.info( 'Obtained FTS RequestID %s' % ftsReqID )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
    if not res['OK']:
      self.log.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
    if not res['OK']:
      self.log.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( submittedIDs ) )
    if not res['OK']:
      self.log.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
    if not res['OK']:
      self.log.error( "Failed to set TotalSize for FTSRequest", res['Message'] )

    #########################################################################
    #  Insert the submission event in the FTSReqLogging table
    event = 'Submitted'
    res = self.transferDB.addLoggingEvent( ftsReqID, event )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )

    #########################################################################
    #  Insert the FileToFTS details and remove the files from the channel
    self.log.info( 'Setting the files as Executing in the Channel table' )
    res = self.transferDB.setChannelFilesExecuting( channelID, list( submittedIDs ) )
    if not res['OK']:
      self.log.error( 'Failed to update the Channel tables for files.', res['Message'] )

    lfns = []
    fileToFTSFileAttributes = []
    for fileMeta in files:
      fileID = fileMeta['FileID']
      # Staging is not an error case
      if fileID not in stagingIDs:
        lfns.append( fileMeta['LFN'] )
        fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )

    self.log.info( 'Populating the FileToFTS table with file information' )
    res = self.transferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
    if not res['OK']:
      self.log.error( 'Failed to populate the FileToFTS table with files.' )
예제 #3
0
  def submitTransfer( self, channelDict ):
    """ This method creates and submits FTS jobs based on information it gets from the DB
    """

    # Create the FTSRequest object for preparing the submission
    oFTSRequest = FTSRequest()
    channelID = channelDict['ChannelID']
    filesPerJob = channelDict['NumFiles']

    #########################################################################
    #  Obtain the first files in the selected channel.
    gLogger.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s" % channelID )
    res = self.TransferDB.getFilesForChannel( channelID, 2 * filesPerJob )
    if not res['OK']:
      errStr = 'FTSSubmitAgent.%s' % res['Message']
      gLogger.error( errStr )
      return S_OK()
    if not res['Value']:
      gLogger.info( "FTSSubmitAgent.submitTransfer: No files to found for channel." )
      return S_OK()
    filesDict = res['Value']
    gLogger.info( 'Obtained %s files for channel' % len( filesDict['Files'] ) )

    sourceSE = filesDict['SourceSE']
    oFTSRequest.setSourceSE( sourceSE )
    targetSE = filesDict['TargetSE']
    oFTSRequest.setTargetSE( targetSE )
    gLogger.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel." % ( sourceSE, targetSE ) )
    files = filesDict['Files']

    #########################################################################
    #  Populate the FTS Request with the files.
    gLogger.info( 'Populating the FTS request with file information' )
    fileIDs = []
    totalSize = 0
    fileIDSizes = {}
    for file in files:
      lfn = file['LFN']
      oFTSRequest.setLFN( lfn )
      oFTSRequest.setSourceSURL( lfn, file['SourceSURL'] )
      oFTSRequest.setTargetSURL( lfn, file['TargetSURL'] )
      fileID = file['FileID']
      fileIDs.append( fileID )
      totalSize += file['Size']
      fileIDSizes[fileID] = file['Size']

    #########################################################################
    #  Submit the FTS request and retrieve the FTS GUID/Server
    gLogger.info( 'Submitting the FTS request' )
    res = oFTSRequest.submit()
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      gLogger.error( errStr )
      gLogger.info( 'Updating the Channel table for files to retry' )
      res = self.TransferDB.resetFileChannelStatus( channelID, fileIDs )
      if not res['OK']:
        gLogger.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
      return S_ERROR( errStr )
    ftsGUID = res['Value']['ftsGUID']
    ftsServer = res['Value']['ftsServer']
    infoStr = """Submitted FTS Job:
    
              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( len( files ) ) )
    gLogger.info( infoStr )

    #########################################################################
    #  Insert the FTS Req details and add the number of files and size
    res = self.TransferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      gLogger.error( errStr )
      return S_ERROR( errStr )
    ftsReqID = res['Value']
    gLogger.info( 'Obtained FTS RequestID %s' % ftsReqID )
    res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
    if not res['OK']:
      gLogger.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
    res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
    if not res['OK']:
      gLogger.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
    res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( fileIDs ) )
    if not res['OK']:
      gLogger.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
    res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
    if not res['OK']:
      gLogger.error( "Failed to set TotalSize for FTSRequest", res['Message'] )

    #########################################################################
    #  Insert the submission event in the FTSReqLogging table
    event = 'Submitted'
    res = self.TransferDB.addLoggingEvent( ftsReqID, event )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      gLogger.error( errStr )

    #########################################################################
    #  Insert the FileToFTS details and remove the files from the channel
    gLogger.info( 'Setting the files as Executing in the Channel table' )
    res = self.TransferDB.setChannelFilesExecuting( channelID, fileIDs )
    if not res['OK']:
      gLogger.error( 'Failed to update the Channel tables for files.', res['Message'] )

    lfns = []
    fileToFTSFileAttributes = []
    for file in files:
      lfn = file['LFN']
      fileID = file['FileID']
      lfns.append( lfn )
      fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )

    gLogger.info( 'Populating the FileToFTS table with file information' )
    res = self.TransferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
    if not res['OK']:
      gLogger.error( 'Failed to populate the FileToFTS table with files.' )
예제 #4
0
    def submitTransfer(self, channelDict):
        """ This method creates and submits FTS jobs based on information it gets from the DB
    """

        # Create the FTSRequest object for preparing the submission
        oFTSRequest = FTSRequest()
        channelID = channelDict['ChannelID']
        filesPerJob = channelDict['NumFiles']

        #########################################################################
        #  Obtain the first files in the selected channel.
        gLogger.info(
            "FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s"
            % channelID)
        res = self.TransferDB.getFilesForChannel(channelID, 2 * filesPerJob)
        if not res['OK']:
            errStr = 'FTSSubmitAgent.%s' % res['Message']
            gLogger.error(errStr)
            return S_OK()
        if not res['Value']:
            gLogger.info(
                "FTSSubmitAgent.submitTransfer: No files to found for channel."
            )
            return S_OK()
        filesDict = res['Value']
        gLogger.info('Obtained %s files for channel' % len(filesDict['Files']))

        sourceSE = filesDict['SourceSE']
        oFTSRequest.setSourceSE(sourceSE)
        targetSE = filesDict['TargetSE']
        oFTSRequest.setTargetSE(targetSE)
        gLogger.info(
            "FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel."
            % (sourceSE, targetSE))
        files = filesDict['Files']

        #########################################################################
        #  Populate the FTS Request with the files.
        gLogger.info('Populating the FTS request with file information')
        fileIDs = []
        totalSize = 0
        fileIDSizes = {}
        for file in files:
            lfn = file['LFN']
            oFTSRequest.setLFN(lfn)
            oFTSRequest.setSourceSURL(lfn, file['SourceSURL'])
            oFTSRequest.setTargetSURL(lfn, file['TargetSURL'])
            fileID = file['FileID']
            fileIDs.append(fileID)
            totalSize += file['Size']
            fileIDSizes[fileID] = file['Size']

        #########################################################################
        #  Submit the FTS request and retrieve the FTS GUID/Server
        gLogger.info('Submitting the FTS request')
        res = oFTSRequest.submit()
        if not res['OK']:
            errStr = "FTSSubmitAgent.%s" % res['Message']
            gLogger.error(errStr)
            gLogger.info('Updating the Channel table for files to retry')
            res = self.TransferDB.resetFileChannelStatus(channelID, fileIDs)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the Channel table for file to retry.',
                    res['Message'])
            return S_ERROR(errStr)
        ftsGUID = res['Value']['ftsGUID']
        ftsServer = res['Value']['ftsServer']
        infoStr = """Submitted FTS Job:
    
              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % (ftsGUID, ftsServer, str(channelID), sourceSE, targetSE, str(len(files)))
        gLogger.info(infoStr)

        #########################################################################
        #  Insert the FTS Req details and add the number of files and size
        res = self.TransferDB.insertFTSReq(ftsGUID, ftsServer, channelID)
        if not res['OK']:
            errStr = "FTSSubmitAgent.%s" % res['Message']
            gLogger.error(errStr)
            return S_ERROR(errStr)
        ftsReqID = res['Value']
        gLogger.info('Obtained FTS RequestID %s' % ftsReqID)
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'SourceSE',
                                                 sourceSE)
        if not res['OK']:
            gLogger.error("Failed to set SourceSE for FTSRequest",
                          res['Message'])
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'TargetSE',
                                                 targetSE)
        if not res['OK']:
            gLogger.error("Failed to set TargetSE for FTSRequest",
                          res['Message'])
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'NumberOfFiles',
                                                 len(fileIDs))
        if not res['OK']:
            gLogger.error("Failed to set NumberOfFiles for FTSRequest",
                          res['Message'])
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'TotalSize',
                                                 totalSize)
        if not res['OK']:
            gLogger.error("Failed to set TotalSize for FTSRequest",
                          res['Message'])

        #########################################################################
        #  Insert the submission event in the FTSReqLogging table
        event = 'Submitted'
        res = self.TransferDB.addLoggingEvent(ftsReqID, event)
        if not res['OK']:
            errStr = "FTSSubmitAgent.%s" % res['Message']
            gLogger.error(errStr)

        #########################################################################
        #  Insert the FileToFTS details and remove the files from the channel
        gLogger.info('Setting the files as Executing in the Channel table')
        res = self.TransferDB.setChannelFilesExecuting(channelID, fileIDs)
        if not res['OK']:
            gLogger.error('Failed to update the Channel tables for files.',
                          res['Message'])

        lfns = []
        fileToFTSFileAttributes = []
        for file in files:
            lfn = file['LFN']
            fileID = file['FileID']
            lfns.append(lfn)
            fileToFTSFileAttributes.append((fileID, fileIDSizes[fileID]))

        gLogger.info('Populating the FileToFTS table with file information')
        res = self.TransferDB.setFTSReqFiles(ftsReqID, channelID,
                                             fileToFTSFileAttributes)
        if not res['OK']:
            gLogger.error('Failed to populate the FileToFTS table with files.')
예제 #5
0
  def submitTransfer( self, channelDict ):
    """ create and submit FTS jobs based on information it gets from the DB

    :param self: self reference
    :param dict channelDict: dict with channel info as read from TransferDB.selectChannelsForSubmission
    """

    channelID = channelDict['ChannelID']
    filesPerJob = channelDict['NumFiles']

    #########################################################################
    #  Obtain the first files in the selected channel.
    self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for channel %s: %s to %s"
                   % ( channelID, channelDict['Source'], channelDict['Destination'] ) )
    res = self.transferDB.getFilesForChannel( channelID, 2 * filesPerJob )
    if not res['OK']:
      errStr = 'FTSSubmitAgent.%s' % res['Message']
      self.log.error( errStr )
      return S_OK()
    if not res['Value']:
      self.log.info( "FTSSubmitAgent.submitTransfer: No files found for channel." )
      return S_OK()
    filesDict = res['Value']
    sourceSE = filesDict['SourceSE']
    targetSE = filesDict['TargetSE']
    self.log.info( 'Obtained %s files for channel %s to %s' % ( len( filesDict['Files'] ), sourceSE, targetSE ) )
    if self.filesBeingStaged.get( channelID ):
      self.log.info( '%d files are currently in staging status' % len( self.filesBeingStaged[channelID] ) )

    # Create the FTSRequest object for preparing the submission
    oFTSRequest = FTSRequest()
    oFTSRequest.setSourceSE( sourceSE )
    oFTSRequest.setTargetSE( targetSE )
    files = filesDict['Files']

    # # enable/disable cksm test
    oFTSRequest.setCksmTest( self.cksmTest )
    if self.cksmType:
      oFTSRequest.setCksmType( self.cksmType )

    #########################################################################
    #  Populate the FTS Request with the files.
    self.log.info( 'Populating the FTS request with file information' )
    fileIDs = []
    totalSize = 0
    fileIDSizes = {}
    lfns = set()
    for fileMeta in files:
      lfn = fileMeta['LFN']
      lfns.add( lfn )
      oFTSRequest.setLFN( lfn )
      oFTSRequest.setSourceSURL( lfn, fileMeta['SourceSURL'] )
      oFTSRequest.setTargetSURL( lfn, fileMeta['TargetSURL'] )
      if lfn in self.filesBeingStaged.get( channelID, [] ):
        oFTSRequest.setStatus( lfn, 'Staging' )
      fileID = fileMeta['FileID']
      fileIDs.append( fileID )
      totalSize += fileMeta['Size']
      fileIDSizes[fileID] = fileMeta['Size']

    oFTSRequest.resolveSource()
    noSource = [ lfn for lfn, fileInfo in oFTSRequest.fileDict.items()
                     if fileInfo.get( "Status", "" ) == "Failed" and fileInfo.get( "Reason", "" ) in ( "No replica at SourceSE",
                                                                                                   "Source file does not exist" ) ]
    toReschedule = [fileMeta["FileID"] for fileMeta in files if fileMeta["LFN"] in noSource]

    if toReschedule:
      self.log.info( "Found %s files to reschedule" % len( toReschedule ) )
      for fileID in toReschedule:
        res = self.transferDB.setFileToReschedule( fileID )
        if not res["OK"]:
          self.log.error( "Failed to update Channel table for failed files.", res["Message"] )
        elif res["Value"] == "max reschedule attempt reached":
          self.log.error( "setting Channel status to 'Failed' : " % res["Value"] )
          res = self.transferDB.setFileChannelStatus( channelID, fileID, 'Failed' )
          if not res["OK"]:
            self.log.error( "Failed to update Channel table for failed files.", res["Message"] )

    #########################################################################
    #  Submit the FTS request and retrieve the FTS GUID/Server
    self.log.info( 'Submitting the FTS request' )
    res = oFTSRequest.submit()
    if not res['OK']:
      errStr = "FTSSubmitAgent.submit: %s" % res['Message']
      self.log.error( errStr )
      self.log.info( 'Updating the Channel table for files to retry' )
      res = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
      if not res['OK']:
        self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
      return S_ERROR( errStr )
    ftsGUID = res['Value']['ftsGUID']
    ftsServer = res['Value']['ftsServer']
    nbSubmitted = res['Value']['submittedFiles']
    infoStr = """Submitted FTS Job:

              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( nbSubmitted ) )
    self.log.info( infoStr )

    # # filter out skipped files
    failedFiles = oFTSRequest.getFailed()['Value']
    stagingFiles = oFTSRequest.getStaging()['Value']
    # cache files being staged
    self.filesBeingStaged.setdefault( channelID, set() ).update( stagingFiles )
    submittedFiles = lfns.difference( failedFiles, stagingFiles )
    # files being submitted are staged
    self.filesBeingStaged[channelID] -= submittedFiles
    failedIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in failedFiles ] )
    stagingIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in stagingFiles ] )
    # # only submitted
    submittedIDs = set( fileIDs ) - failedIDs - stagingIDs
    # # only count the submitted size
    totalSize = sum( [ meta["Size"] for meta in files if meta["FileID"] in submittedIDs ] )

    #########################################################################
    #  Insert the FTS Req details and add the number of files and size
    res = self.transferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )
      return S_ERROR( errStr )
    ftsReqID = res['Value']
    self.log.info( 'Obtained FTS RequestID %s' % ftsReqID )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
    if not res['OK']:
      self.log.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
    if not res['OK']:
      self.log.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( submittedIDs ) )
    if not res['OK']:
      self.log.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
    if not res['OK']:
      self.log.error( "Failed to set TotalSize for FTSRequest", res['Message'] )

    #########################################################################
    #  Insert the submission event in the FTSReqLogging table
    event = 'Submitted'
    res = self.transferDB.addLoggingEvent( ftsReqID, event )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )

    #########################################################################
    #  Insert the FileToFTS details and remove the files from the channel
    self.log.info( 'Setting the files as Executing in the Channel table' )
    res = self.transferDB.setChannelFilesExecuting( channelID, list( submittedIDs ) )
    if not res['OK']:
      self.log.error( 'Failed to update the Channel tables for files.', res['Message'] )

    lfns = []
    fileToFTSFileAttributes = []
    for fileMeta in files:
      fileID = fileMeta['FileID']
      # Staging is not an error case
      if fileID not in stagingIDs:
        lfns.append( fileMeta['LFN'] )
        fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )

    self.log.info( 'Populating the FileToFTS table with file information' )
    res = self.transferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
    if not res['OK']:
      self.log.error( 'Failed to populate the FileToFTS table with files.' )