예제 #1
0
    server = args.pop(0)
    guid = args.pop(0)
else:
    Script.showHelp()
    DIRAC.exit(1)
if args:
    sourceSE = args.pop(0)
if args:
    targetSE = args.pop(0)
for arg in args:
    for lfn in arg.split(","):
        if not os.path.exists(lfn):
            lfns.append(lfn)
        else:
            inputFile = open(lfn, "r")
            string = inputFile.read()
            inputFile.close()
            lfns += string.splitlines()

oFTSRequest = FTSRequest()
oFTSRequest.setSourceSE(sourceSE)
oFTSRequest.setTargetSE(targetSE)
for lfn in lfns:
    oFTSRequest.setLFN(lfn)
oFTSRequest.setFTSGUID(guid)
oFTSRequest.setFTSServer(server)
result = oFTSRequest.monitor(untilTerminal=False, printOutput=True)
if not result["OK"]:
    DIRAC.gLogger.error("Failed to issue FTS Request", result["Message"])
    DIRAC.exit(-1)
예제 #2
0
  def monitorTransfer( self, ftsReqDict ):
    """ monitors transfer  obtained from TransferDB """

    ftsReqID = ftsReqDict['FTSReqID']
    ftsGUID = ftsReqDict['FTSGuid']
    ftsServer = ftsReqDict['FTSServer']
    channelID = ftsReqDict['ChannelID']
    sourceSE = ftsReqDict['SourceSE']
    targetSE = ftsReqDict['TargetSE']

    oFTSRequest = FTSRequest()
    oFTSRequest.setFTSServer( ftsServer )
    oFTSRequest.setFTSGUID( ftsGUID )
    oFTSRequest.setSourceSE( sourceSE )
    oFTSRequest.setTargetSE( targetSE )

    #########################################################################
    # Perform summary update of the FTS Request and update FTSReq entries.
    self.log.info( 'Perform summary update of the FTS Request' )
    infoStr = "Monitoring FTS Job:\n\n"
    infoStr = "%sglite-transfer-status -s %s -l %s\n" % ( infoStr, ftsServer, ftsGUID )
    infoStr = "%s%s%s\n" % ( infoStr, 'FTS GUID:'.ljust( 20 ), ftsGUID )
    infoStr = "%s%s%s\n\n" % ( infoStr, 'FTS Server:'.ljust( 20 ), ftsServer )
    self.log.info( infoStr )
    res = oFTSRequest.summary()
    self.transferDB.setFTSReqLastMonitor( ftsReqID )
    if not res['OK']:
      self.log.error( "Failed to update the FTS request summary", res['Message'] )
      if "getTransferJobSummary2: Not authorised to query request" in res["Message"]:
        self.log.error("FTS job is not existing at the FTS server anymore, will clean it up on TransferDB side")

        ## get fileIDs
        fileIDs = self.transferDB.getFTSReqFileIDs( ftsReqID )
        if not fileIDs["OK"]:
          self.log.error("Unable to retrieve FileIDs associated to %s request" % ftsReqID )
          return fileIDs
        fileIDs = fileIDs["Value"]
      
        ## update FileToFTS table, this is just a clean up, no worry if somethings goes wrong
        for fileID in fileIDs:
          fileStatus = self.transferDB.setFileToFTSFileAttribute( ftsReqID, fileID, 
                                                                  "Status", "Failed" )
          if not fileStatus["OK"]:
            self.log.error("Unable to set FileToFTS status to Failed for FileID %s: %s" % ( fileID, 
                                                                                           fileStatus["Message"] ) )
                          
          failReason = self.transferDB.setFileToFTSFileAttribute( ftsReqID, fileID, 
                                                                  "Reason", "FTS job expired on server" )
          if not failReason["OK"]:
            self.log.error("Unable to set FileToFTS reason for FileID %s: %s" % ( fileID, 
                                                                                 failReason["Message"] ) )

        ## update Channel table
        resetChannels = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
        if not resetChannels["OK"]:
          self.log.error("Failed to reset Channel table for files to retry")
          return resetChannels   

        ## update FTSReq table
        self.log.info( 'Setting FTS request status to Finished' )
        ftsReqStatus = self.transferDB.setFTSReqStatus( ftsReqID, 'Finished' )
        if not ftsReqStatus['OK']:
          self.log.error( 'Failed update FTS Request status', ftsReqStatus['Message'] )
          return ftsReqStatus
        ## if we land here, everything should be OK
        return S_OK()

      return res
    res = oFTSRequest.dumpSummary()
    if not res['OK']:
      self.log.error( "Failed to get FTS request summary", res['Message'] )
      return res
    self.log.info( res['Value'] )
    res = oFTSRequest.getPercentageComplete()
    if not res['OK']:
      self.log.error( "Failed to get FTS percentage complete", res['Message'] )
      return res
    self.log.info( 'FTS Request found to be %.1f percent complete' % res['Value'] )
    self.transferDB.setFTSReqAttribute( ftsReqID, 'PercentageComplete', res['Value'] )
    self.transferDB.addLoggingEvent( ftsReqID, res['Value'] )

    #########################################################################
    # Update the information in the TransferDB if the transfer is terminal.
    res = oFTSRequest.isRequestTerminal()
    if not res['OK']:
      self.log.error( "Failed to determine whether FTS request terminal", res['Message'] )
      return res
    if not res['Value']:
      return S_OK()
    self.log.info( 'FTS Request found to be terminal, updating file states' )

    #########################################################################
    # Get the LFNS associated to the FTS request
    self.log.info( 'Obtaining the LFNs associated to this request' )
    res = self.transferDB.getFTSReqLFNs( ftsReqID, channelID, sourceSE )
    if not res['OK']:
      self.log.error( "Failed to obtain FTS request LFNs", res['Message'] )
      return res
    files = res['Value']
    if not files:
      self.log.error( 'No files present for transfer' )
      return S_ERROR( 'No files were found in the DB' )
    lfns = files.keys()
    self.log.info( 'Obtained %s files' % len( lfns ) )
    for lfn in lfns:
      oFTSRequest.setLFN( lfn )

    res = oFTSRequest.monitor()
    if not res['OK']:
      self.log.error( "Failed to perform detailed monitoring of FTS request", res['Message'] )
      return res
    res = oFTSRequest.getFailed()
    if not res['OK']:
      self.log.error( "Failed to obtained failed files for FTS request", res['Message'] )
      return res
    failedFiles = res['Value']
    res = oFTSRequest.getDone()
    if not res['OK']:
      self.log.error( "Failed to obtained successful files for FTS request", res['Message'] )
      return res
    completedFiles = res['Value']

    # An LFN can be included more than once if it was entered into more than one Request. 
    # FTS will only do the transfer once. We need to identify all FileIDs
    res = self.transferDB.getFTSReqFileIDs( ftsReqID )
    if not res['OK']:
      self.log.error( 'Failed to get FileIDs associated to FTS Request', res['Message'] )
      return res
    fileIDs = res['Value']
    res = self.transferDB.getAttributesForFilesList( fileIDs, ['LFN'] )
    if not res['OK']:
      self.log.error( 'Failed to get LFNs associated to FTS Request', res['Message'] )
      return res
    fileIDDict = res['Value']

    fileToFTSUpdates = []
    completedFileIDs = []

    filesToRetry = []
    filesToFail = []

    for fileID, fileDict in fileIDDict.items():
      lfn = fileDict['LFN']
      if lfn in completedFiles:
        completedFileIDs.append( fileID )
        transferTime = 0
        res = oFTSRequest.getTransferTime( lfn )
        if res['OK']:
          transferTime = res['Value']
        fileToFTSUpdates.append( ( fileID, 'Completed', '', 0, transferTime ) )

      if lfn in failedFiles:
        failReason = ''
        res = oFTSRequest.getFailReason( lfn )
        if res['OK']:
          failReason = res['Value']
        if self.missingSource( failReason ):
          self.log.error( 'The source SURL does not exist.', '%s %s' % ( lfn, oFTSRequest.getSourceSURL( lfn ) ) )
          filesToFail.append( fileID )
        else:
          filesToRetry.append( fileID )
        self.log.error( 'Failed to replicate file on channel.', "%s %s" % ( channelID, failReason ) )
        fileToFTSUpdates.append( ( fileID, 'Failed', failReason, 0, 0 ) )

    allUpdated = True
    if filesToRetry:
      self.log.info( 'Updating the Channel table for files to retry' )
      res = self.transferDB.resetFileChannelStatus( channelID, filesToRetry )
      if not res['OK']:
        self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
        allUpdated = False
    for fileID in filesToFail:
      self.log.info( 'Updating the Channel table for files to reschedule' )
      res = self.transferDB.setFileChannelStatus( channelID, fileID, 'Failed' )
      if not res['OK']:
        self.log.error( 'Failed to update Channel table for failed files.', res['Message'] )
        allUpdated = False

    if completedFileIDs:
      self.log.info( 'Updating the Channel table for successful files' )
      res = self.transferDB.updateCompletedChannelStatus( channelID, completedFileIDs )
      if not res['OK']:
        self.log.error( 'Failed to update the Channel table for successful files.', res['Message'] )
        allUpdated = False
      self.log.info( 'Updating the Channel table for ancestors of successful files' )
      res = self.transferDB.updateAncestorChannelStatus( channelID, completedFileIDs )
      if not res['OK']:
        self.log.error( 'Failed to update the Channel table for ancestors of successful files.', res['Message'] )
        allUpdated = False

    if fileToFTSUpdates:
      self.log.info( 'Updating the FileToFTS table for files' )
      res = self.transferDB.setFileToFTSFileAttributes( ftsReqID, channelID, fileToFTSUpdates )
      if not res['OK']:
        self.log.error( 'Failed to update the FileToFTS table for files.', res['Message'] )
        allUpdated = False

    if allUpdated:
      res = oFTSRequest.finalize()
      if not res['OK']:
        self.log.error( "Failed to perform the finalization for the FTS request", res['Message'] )
        return res

      self.log.info( 'Adding logging event for FTS request' )
      # Now set the FTSReq status to terminal so that it is not monitored again
      res = self.transferDB.addLoggingEvent( ftsReqID, 'Finished' )
      if not res['OK']:
        self.log.error( 'Failed to add logging event for FTS Request', res['Message'] )

      res = oFTSRequest.getFailedRegistrations()
      failedRegistrations = res['Value']
      regFailedFileIDs = []
      regDoneFileIDs = []
      regForgetFileIDs = []
      for fileID, fileDict in fileIDDict.items():
        lfn = fileDict['LFN']
        if lfn in failedRegistrations:
          self.log.info( 'Setting failed registration in FileToCat back to Waiting', lfn )
          regFailedFileIDs.append( fileID )
          # if the LFN appears more than once, FileToCat needs to be reset only once 
          del failedRegistrations[lfn]
        elif lfn in completedFiles:
          regDoneFileIDs.append( fileID )
        elif fileID in filesToFail:
          regForgetFileIDs.append( fileID )


      if regFailedFileIDs:
        res = self.transferDB.setRegistrationWaiting( channelID, regFailedFileIDs )
        if not res['OK']:
          self.log.error( 'Failed to reset entries in FileToCat', res['Message'] )
          return res

      if regDoneFileIDs:
        res = self.transferDB.setRegistrationDone( channelID, regDoneFileIDs )
        if not res['OK']:
          self.log.error( 'Failed to set entries Done in FileToCat', res['Message'] )
          return res

      if regForgetFileIDs:
        # This entries could also be set to Failed, but currently there is no method to do so.
        res = self.transferDB.setRegistrationDone( channelID, regForgetFileIDs )
        if not res['OK']:
          self.log.error( 'Failed to set entries Done in FileToCat', res['Message'] )
          return res

      self.log.info( 'Updating FTS request status' )
      res = self.transferDB.setFTSReqStatus( ftsReqID, 'Finished' )
      if not res['OK']:
        self.log.error( 'Failed update FTS Request status', res['Message'] )
    return S_OK()
예제 #3
0
  def submitTransfer( self, channelDict ):
    """ create and submit FTS jobs based on information it gets from the DB

    :param self: self reference
    :param dict channelDict: dict with channel info as read from TransferDB.selectChannelsForSubmission
    """

    # Create the FTSRequest object for preparing the submission
    oFTSRequest = FTSRequest()
    channelID = channelDict['ChannelID']
    filesPerJob = channelDict['NumFiles']

    #########################################################################
    #  Obtain the first files in the selected channel.
    self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s" % channelID )
    res = self.transferDB.getFilesForChannel( channelID, 2 * filesPerJob )
    if not res['OK']:
      errStr = 'FTSSubmitAgent.%s' % res['Message']
      self.log.error( errStr )
      return S_OK()
    if not res['Value']:
      self.log.info( "FTSSubmitAgent.submitTransfer: No files to found for channel." )
      return S_OK()
    filesDict = res['Value']
    self.log.info( 'Obtained %s files for channel' % len( filesDict['Files'] ) )

    sourceSE = filesDict['SourceSE']
    oFTSRequest.setSourceSE( sourceSE )
    targetSE = filesDict['TargetSE']
    oFTSRequest.setTargetSE( targetSE )
    self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel." % ( sourceSE, 
                                                                                                         targetSE ) )
    files = filesDict['Files']

    ## enable/disable cksm test
    oFTSRequest.setCksmTest( self.cksmTest )
    if self.cksmType:
      oFTSRequest.setCksmType( self.cksmType )

    #########################################################################
    #  Populate the FTS Request with the files.
    self.log.info( 'Populating the FTS request with file information' )
    fileIDs = []
    totalSize = 0
    fileIDSizes = {}
    for fileMeta in files:
      lfn = fileMeta['LFN']
      oFTSRequest.setLFN( lfn )
      oFTSRequest.setSourceSURL( lfn, fileMeta['SourceSURL'] )
      oFTSRequest.setTargetSURL( lfn, fileMeta['TargetSURL'] )
      fileID = fileMeta['FileID']
      fileIDs.append( fileID )
      totalSize += fileMeta['Size']
      fileIDSizes[fileID] = fileMeta['Size']

    #########################################################################
    #  Submit the FTS request and retrieve the FTS GUID/Server
    self.log.info( 'Submitting the FTS request' )
    res = oFTSRequest.submit()
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )
      self.log.info( 'Updating the Channel table for files to retry' )
      res = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
      if not res['OK']:
        self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
      return S_ERROR( errStr )
    ftsGUID = res['Value']['ftsGUID']
    ftsServer = res['Value']['ftsServer']
    infoStr = """Submitted FTS Job:
    
              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( len( files ) ) )
    self.log.info( infoStr )

    ## filter out skipped files
    failedFiles = oFTSRequest.getFailed()
    if not failedFiles["OK"]:
      self.log.warn("Unable to read skipped LFNs.")
    failedFiles = failedFiles["Value"] if "Value" in failedFiles else []
    failedIDs = [ meta["FileID"] for meta in files if meta["LFN"] in failedFiles ]
    ## only submitted
    fileIDs = [ fileID for fileID in fileIDs if fileID not in failedIDs ]
    ## sub failed from total size
    totalSize -= sum( [ meta["Size"] for meta in files if meta["LFN"] in failedFiles ] )

    #########################################################################
    #  Insert the FTS Req details and add the number of files and size
    res = self.transferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )
      return S_ERROR( errStr )
    ftsReqID = res['Value']
    self.log.info( 'Obtained FTS RequestID %s' % ftsReqID )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
    if not res['OK']:
      self.log.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
    if not res['OK']:
      self.log.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( fileIDs ) )
    if not res['OK']:
      self.log.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
    if not res['OK']:
      self.log.error( "Failed to set TotalSize for FTSRequest", res['Message'] )

    #########################################################################
    #  Insert the submission event in the FTSReqLogging table
    event = 'Submitted'
    res = self.transferDB.addLoggingEvent( ftsReqID, event )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )

    #########################################################################
    #  Insert the FileToFTS details and remove the files from the channel
    self.log.info( 'Setting the files as Executing in the Channel table' )
    res = self.transferDB.setChannelFilesExecuting( channelID, fileIDs )
    if not res['OK']:
      self.log.error( 'Failed to update the Channel tables for files.', res['Message'] )

    lfns = []
    fileToFTSFileAttributes = []
    for fileMeta in files:
      lfn = fileMeta['LFN']
      fileID = fileMeta['FileID']
      lfns.append( lfn )
      fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )

    self.log.info( 'Populating the FileToFTS table with file information' )
    res = self.transferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
    if not res['OK']:
      self.log.error( 'Failed to populate the FileToFTS table with files.' )
예제 #4
0
  def submitTransfer( self, channelDict ):
    """ create and submit FTS jobs based on information it gets from the DB

    :param self: self reference
    :param dict channelDict: dict with channel info as read from TransferDB.selectChannelsForSubmission
    """

    channelID = channelDict['ChannelID']
    filesPerJob = channelDict['NumFiles']

    #########################################################################
    #  Obtain the first files in the selected channel.
    self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for channel %s: %s to %s"
                   % ( channelID, channelDict['Source'], channelDict['Destination'] ) )
    res = self.transferDB.getFilesForChannel( channelID, 2 * filesPerJob )
    if not res['OK']:
      errStr = 'FTSSubmitAgent.%s' % res['Message']
      self.log.error( errStr )
      return S_OK()
    if not res['Value']:
      self.log.info( "FTSSubmitAgent.submitTransfer: No files found for channel." )
      return S_OK()
    filesDict = res['Value']
    sourceSE = filesDict['SourceSE']
    targetSE = filesDict['TargetSE']
    self.log.info( 'Obtained %s files for channel %s to %s' % ( len( filesDict['Files'] ), sourceSE, targetSE ) )

    # Create the FTSRequest object for preparing the submission
    oFTSRequest = FTSRequest()
    oFTSRequest.setSourceSE( sourceSE )
    oFTSRequest.setTargetSE( targetSE )
    files = filesDict['Files']

    # # enable/disable cksm test
    oFTSRequest.setCksmTest( self.cksmTest )
    if self.cksmType:
      oFTSRequest.setCksmType( self.cksmType )

    #########################################################################
    #  Populate the FTS Request with the files.
    self.log.info( 'Populating the FTS request with file information' )
    fileIDs = []
    totalSize = 0
    fileIDSizes = {}
    lfns = set()
    for fileMeta in files:
      lfn = fileMeta['LFN']
      lfns.add( lfn )
      oFTSRequest.setLFN( lfn )
      oFTSRequest.setSourceSURL( lfn, fileMeta['SourceSURL'] )
      oFTSRequest.setTargetSURL( lfn, fileMeta['TargetSURL'] )
      if lfn in self.filesBeingStaged:
        oFTSRequest.setStatus( lfn, 'Staging' )
      fileID = fileMeta['FileID']
      fileIDs.append( fileID )
      totalSize += fileMeta['Size']
      fileIDSizes[fileID] = fileMeta['Size']

    oFTSRequest.resolveSource()
    noSource = [ lfn for lfn, fileInfo in oFTSRequest.fileDict.items()
                     if fileInfo.get( "Status", "" ) == "Failed" and fileInfo.get( "Reason", "" ) in ( "No replica at SourceSE",
                                                                                                   "Source file does not exist" ) ]
    toReschedule = [fileMeta["FileID"] for fileMeta in files if fileMeta["LFN"] in noSource]

    if toReschedule:
      self.log.info( "Found %s files to reschedule" % len( toReschedule ) )
      for fileID in toReschedule:
        res = self.transferDB.setFileToReschedule( fileID )
        if not res["OK"]:
          self.log.error( "Failed to update Channel table for failed files.", res["Message"] )
        elif res["Value"] == "max reschedule attempt reached":
          self.log.error( "setting Channel status to 'Failed' : " % res["Value"] )
          res = self.transferDB.setFileChannelStatus( channelID, fileID, 'Failed' )
          if not res["OK"]:
            self.log.error( "Failed to update Channel table for failed files.", res["Message"] )

    #########################################################################
    #  Submit the FTS request and retrieve the FTS GUID/Server
    self.log.info( 'Submitting the FTS request' )
    res = oFTSRequest.submit()
    if not res['OK']:
      errStr = "FTSSubmitAgent.submit: %s" % res['Message']
      self.log.error( errStr )
      self.log.info( 'Updating the Channel table for files to retry' )
      res = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
      if not res['OK']:
        self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
      return S_ERROR( errStr )
    ftsGUID = res['Value']['ftsGUID']
    ftsServer = res['Value']['ftsServer']
    nbSubmitted = res['Value']['submittedFiles']
    infoStr = """Submitted FTS Job:

              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( nbSubmitted ) )
    self.log.info( infoStr )

    # # filter out skipped files
    failedFiles = oFTSRequest.getFailed()['Value']
    stagingFiles = oFTSRequest.getStaging()['Value']
    # cache files being staged
    self.filesBeingStaged.update( stagingFiles )
    submittedFiles = lfns.difference( failedFiles, stagingFiles )
    # files being submitted are staged
    self.filesBeingStaged -= submittedFiles
    failedIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in failedFiles ] )
    stagingIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in stagingFiles ] )
    # # only submitted
    submittedIDs = set( fileIDs ) - failedIDs - stagingIDs
    # # only count the submitted size
    totalSize = sum( [ meta["Size"] for meta in files if meta["FileID"] in submittedIDs ] )

    #########################################################################
    #  Insert the FTS Req details and add the number of files and size
    res = self.transferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )
      return S_ERROR( errStr )
    ftsReqID = res['Value']
    self.log.info( 'Obtained FTS RequestID %s' % ftsReqID )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
    if not res['OK']:
      self.log.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
    if not res['OK']:
      self.log.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( submittedIDs ) )
    if not res['OK']:
      self.log.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
    if not res['OK']:
      self.log.error( "Failed to set TotalSize for FTSRequest", res['Message'] )

    #########################################################################
    #  Insert the submission event in the FTSReqLogging table
    event = 'Submitted'
    res = self.transferDB.addLoggingEvent( ftsReqID, event )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )

    #########################################################################
    #  Insert the FileToFTS details and remove the files from the channel
    self.log.info( 'Setting the files as Executing in the Channel table' )
    res = self.transferDB.setChannelFilesExecuting( channelID, list( submittedIDs ) )
    if not res['OK']:
      self.log.error( 'Failed to update the Channel tables for files.', res['Message'] )

    lfns = []
    fileToFTSFileAttributes = []
    for fileMeta in files:
      fileID = fileMeta['FileID']
      # Staging is not an error case
      if fileID not in stagingIDs:
        lfns.append( fileMeta['LFN'] )
        fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )

    self.log.info( 'Populating the FileToFTS table with file information' )
    res = self.transferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
    if not res['OK']:
      self.log.error( 'Failed to populate the FileToFTS table with files.' )
예제 #5
0
    def monitorTransfer(self, ftsReqDict):
        """ Monitors transfer  obtained from TransferDB
    """

        ftsReqID = ftsReqDict['FTSReqID']
        ftsGUID = ftsReqDict['FTSGuid']
        ftsServer = ftsReqDict['FTSServer']
        channelID = ftsReqDict['ChannelID']
        sourceSE = ftsReqDict['SourceSE']
        targetSE = ftsReqDict['TargetSE']

        oFTSRequest = FTSRequest()
        oFTSRequest.setFTSServer(ftsServer)
        oFTSRequest.setFTSGUID(ftsGUID)
        oFTSRequest.setSourceSE(sourceSE)
        oFTSRequest.setTargetSE(targetSE)

        #########################################################################
        # Perform summary update of the FTS Request and update FTSReq entries.
        gLogger.info('Perform summary update of the FTS Request')
        infoStr = "Monitoring FTS Job:\n\n"
        infoStr = "%sglite-transfer-status -s %s -l %s\n" % (
            infoStr, ftsServer, ftsGUID)
        infoStr = "%s%s%s\n" % (infoStr, 'FTS GUID:'.ljust(20), ftsGUID)
        infoStr = "%s%s%s\n\n" % (infoStr, 'FTS Server:'.ljust(20), ftsServer)
        gLogger.info(infoStr)
        res = oFTSRequest.summary()
        self.TransferDB.setFTSReqLastMonitor(ftsReqID)
        if not res['OK']:
            gLogger.error("Failed to update the FTS request summary",
                          res['Message'])
            return res
        res = oFTSRequest.dumpSummary()
        if not res['OK']:
            gLogger.error("Failed to get FTS request summary", res['Message'])
            return res
        gLogger.info(res['Value'])
        res = oFTSRequest.getPercentageComplete()
        if not res['OK']:
            gLogger.error("Failed to get FTS percentage complete",
                          res['Message'])
            return res
        gLogger.info('FTS Request found to be %.1f percent complete' %
                     res['Value'])
        self.TransferDB.setFTSReqAttribute(ftsReqID, 'PercentageComplete',
                                           res['Value'])
        self.TransferDB.addLoggingEvent(ftsReqID, res['Value'])

        #########################################################################
        # Update the information in the TransferDB if the transfer is terminal.
        res = oFTSRequest.isRequestTerminal()
        if not res['OK']:
            gLogger.error("Failed to determine whether FTS request terminal",
                          res['Message'])
            return res
        if not res['Value']:
            return S_OK()
        gLogger.info('FTS Request found to be terminal, updating file states')

        #########################################################################
        # Get the LFNS associated to the FTS request
        gLogger.info('Obtaining the LFNs associated to this request')
        res = self.TransferDB.getFTSReqLFNs(ftsReqID)
        if not res['OK']:
            gLogger.error("Failed to obtain FTS request LFNs", res['Message'])
            return res
        files = res['Value']
        if not files:
            gLogger.error('No files present for transfer')
            return S_ERROR('No files were found in the DB')
        lfns = files.keys()
        gLogger.info('Obtained %s files' % len(lfns))
        for lfn in lfns:
            oFTSRequest.setLFN(lfn)

        res = oFTSRequest.monitor()
        if not res['OK']:
            gLogger.error(
                "Failed to perform detailed monitoring of FTS request",
                res['Message'])
            return res
        res = oFTSRequest.getFailed()
        if not res['OK']:
            gLogger.error("Failed to obtained failed files for FTS request",
                          res['Message'])
            return res
        failedFiles = res['Value']
        res = oFTSRequest.getDone()
        if not res['OK']:
            gLogger.error(
                "Failed to obtained successful files for FTS request",
                res['Message'])
            return res
        completedFiles = res['Value']

        fileToFTSUpdates = []
        completedFileIDs = []
        for lfn in completedFiles:
            fileID = files[lfn]
            completedFileIDs.append(fileID)
            transferTime = 0
            res = oFTSRequest.getTransferTime(lfn)
            if res['OK']:
                transferTime = res['Value']
            fileToFTSUpdates.append((fileID, 'Completed', '', 0, transferTime))

        filesToRetry = []
        filesToReschedule = []
        for lfn in failedFiles:
            fileID = files[lfn]
            failReason = ''
            res = oFTSRequest.getFailReason(lfn)
            if res['OK']:
                failReason = res['Value']
            if self.missingSource(failReason):
                gLogger.error('The source SURL does not exist.',
                              '%s %s' % (lfn, oFTSRequest.getSourceSURL(lfn)))
                filesToReschedule.append(fileID)
            else:
                filesToRetry.append(fileID)
            gLogger.error('Failed to replicate file on channel.',
                          "%s %s" % (channelID, failReason))
            fileToFTSUpdates.append((fileID, 'Failed', failReason, 0, 0))

        allUpdated = True
        if filesToRetry:
            gLogger.info('Updating the Channel table for files to retry')
            res = self.TransferDB.resetFileChannelStatus(
                channelID, filesToRetry)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the Channel table for file to retry.',
                    res['Message'])
                allUpdated = False
        for fileID in filesToReschedule:
            gLogger.info('Updating the Channel table for files to reschedule')
            res = self.TransferDB.setFileChannelStatus(channelID, fileID,
                                                       'Failed')
            if not res['OK']:
                gLogger.error(
                    'Failed to update Channel table for failed files.',
                    res['Message'])
                allUpdated = False

        if completedFileIDs:
            gLogger.info('Updating the Channel table for successful files')
            res = self.TransferDB.updateCompletedChannelStatus(
                channelID, completedFileIDs)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the Channel table for successful files.',
                    res['Message'])
                allUpdated = False
            gLogger.info(
                'Updating the Channel table for ancestors of successful files')
            res = self.TransferDB.updateAncestorChannelStatus(
                channelID, completedFileIDs)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the Channel table for ancestors of successful files.',
                    res['Message'])
                allUpdated = False

            #
            #  cibak: temporary switch off of regitstration in FileToCat
            #
            #gLogger.info( 'Updating the FileToCat table for successful files' )
            #res = self.TransferDB.setRegistrationWaiting( channelID, completedFileIDs )
            #if not res['OK']:
            #  gLogger.error( 'Failed to update the FileToCat table for successful files.', res['Message'] )
            #  allUpdated = False

        if fileToFTSUpdates:
            gLogger.info('Updating the FileToFTS table for files')
            res = self.TransferDB.setFileToFTSFileAttributes(
                ftsReqID, channelID, fileToFTSUpdates)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the FileToFTS table for files.',
                    res['Message'])
                allUpdated = False

        if allUpdated:
            res = oFTSRequest.finalize()
            if not res['OK']:
                gLogger.error(
                    "Failed to perform the finalization for the FTS request",
                    res['Message'])
                return res

            gLogger.info('Adding logging event for FTS request')
            # Now set the FTSReq status to terminal so that it is not monitored again
            res = self.TransferDB.addLoggingEvent(ftsReqID, 'Finished')
            if not res['OK']:
                gLogger.error('Failed to add logging event for FTS Request',
                              res['Message'])

            gLogger.info('Updating FTS request status')
            res = self.TransferDB.setFTSReqStatus(ftsReqID, 'Finished')
            if not res['OK']:
                gLogger.error('Failed update FTS Request status',
                              res['Message'])
        return S_OK()
예제 #6
0
  def monitorTransfer( self, ftsReqDict ):
    """ Monitors transfer  obtained from TransferDB
    """

    ftsReqID = ftsReqDict['FTSReqID']
    ftsGUID = ftsReqDict['FTSGuid']
    ftsServer = ftsReqDict['FTSServer']
    channelID = ftsReqDict['ChannelID']
    sourceSE = ftsReqDict['SourceSE']
    targetSE = ftsReqDict['TargetSE']

    oFTSRequest = FTSRequest()
    oFTSRequest.setFTSServer( ftsServer )
    oFTSRequest.setFTSGUID( ftsGUID )
    oFTSRequest.setSourceSE( sourceSE )
    oFTSRequest.setTargetSE( targetSE )

    #########################################################################
    # Perform summary update of the FTS Request and update FTSReq entries.
    gLogger.info( 'Perform summary update of the FTS Request' )
    infoStr = "Monitoring FTS Job:\n\n"
    infoStr = "%sglite-transfer-status -s %s -l %s\n" % ( infoStr, ftsServer, ftsGUID )
    infoStr = "%s%s%s\n" % ( infoStr, 'FTS GUID:'.ljust( 20 ), ftsGUID )
    infoStr = "%s%s%s\n\n" % ( infoStr, 'FTS Server:'.ljust( 20 ), ftsServer )
    gLogger.info( infoStr )
    res = oFTSRequest.summary()
    self.TransferDB.setFTSReqLastMonitor( ftsReqID )
    if not res['OK']:
      gLogger.error( "Failed to update the FTS request summary", res['Message'] )
      return res
    res = oFTSRequest.dumpSummary()
    if not res['OK']:
      gLogger.error( "Failed to get FTS request summary", res['Message'] )
      return res
    gLogger.info( res['Value'] )
    res = oFTSRequest.getPercentageComplete()
    if not res['OK']:
      gLogger.error( "Failed to get FTS percentage complete", res['Message'] )
      return res
    gLogger.info( 'FTS Request found to be %.1f percent complete' % res['Value'] )
    self.TransferDB.setFTSReqAttribute( ftsReqID, 'PercentageComplete', res['Value'] )
    self.TransferDB.addLoggingEvent( ftsReqID, res['Value'] )

    #########################################################################
    # Update the information in the TransferDB if the transfer is terminal.
    res = oFTSRequest.isRequestTerminal()
    if not res['OK']:
      gLogger.error( "Failed to determine whether FTS request terminal", res['Message'] )
      return res
    if not res['Value']:
      return S_OK()
    gLogger.info( 'FTS Request found to be terminal, updating file states' )

    #########################################################################
    # Get the LFNS associated to the FTS request
    gLogger.info( 'Obtaining the LFNs associated to this request' )
    res = self.TransferDB.getFTSReqLFNs( ftsReqID )
    if not res['OK']:
      gLogger.error( "Failed to obtain FTS request LFNs", res['Message'] )
      return res
    files = res['Value']
    if not files:
      gLogger.error( 'No files present for transfer' )
      return S_ERROR( 'No files were found in the DB' )
    lfns = files.keys()
    gLogger.info( 'Obtained %s files' % len( lfns ) )
    for lfn in lfns:
      oFTSRequest.setLFN( lfn )

    res = oFTSRequest.monitor()
    if not res['OK']:
      gLogger.error( "Failed to perform detailed monitoring of FTS request", res['Message'] )
      return res
    res = oFTSRequest.getFailed()
    if not res['OK']:
      gLogger.error( "Failed to obtained failed files for FTS request", res['Message'] )
      return res
    failedFiles = res['Value']
    res = oFTSRequest.getDone()
    if not res['OK']:
      gLogger.error( "Failed to obtained successful files for FTS request", res['Message'] )
      return res
    completedFiles = res['Value']

    fileToFTSUpdates = []
    completedFileIDs = []
    for lfn in completedFiles:
      fileID = files[lfn]
      completedFileIDs.append( fileID )
      transferTime = 0
      res = oFTSRequest.getTransferTime( lfn )
      if res['OK']:
        transferTime = res['Value']
      fileToFTSUpdates.append( ( fileID, 'Completed', '', 0, transferTime ) )

    filesToRetry = []
    filesToReschedule = []
    for lfn in failedFiles:
      fileID = files[lfn]
      failReason = ''
      res = oFTSRequest.getFailReason( lfn )
      if res['OK']:
        failReason = res['Value']
      if self.missingSource( failReason ):
        gLogger.error( 'The source SURL does not exist.', '%s %s' % ( lfn, oFTSRequest.getSourceSURL( lfn ) ) )
        filesToReschedule.append( fileID )
      else:
        filesToRetry.append( fileID )
      gLogger.error( 'Failed to replicate file on channel.', "%s %s" % ( channelID, failReason ) )
      fileToFTSUpdates.append( ( fileID, 'Failed', failReason, 0, 0 ) )

    allUpdated = True
    if filesToRetry:
      gLogger.info( 'Updating the Channel table for files to retry' )
      res = self.TransferDB.resetFileChannelStatus( channelID, filesToRetry )
      if not res['OK']:
        gLogger.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
        allUpdated = False
    for fileID in filesToReschedule:
      gLogger.info( 'Updating the Channel table for files to reschedule' )
      res = self.TransferDB.setFileChannelStatus( channelID, fileID, 'Failed' )
      if not res['OK']:
        gLogger.error( 'Failed to update Channel table for failed files.', res['Message'] )
        allUpdated = False

    if completedFileIDs:
      gLogger.info( 'Updating the Channel table for successful files' )
      res = self.TransferDB.updateCompletedChannelStatus( channelID, completedFileIDs )
      if not res['OK']:
        gLogger.error( 'Failed to update the Channel table for successful files.', res['Message'] )
        allUpdated = False
      gLogger.info( 'Updating the Channel table for ancestors of successful files' )
      res = self.TransferDB.updateAncestorChannelStatus( channelID, completedFileIDs )
      if not res['OK']:
        gLogger.error( 'Failed to update the Channel table for ancestors of successful files.', res['Message'] )
        allUpdated = False
      
      #
      #  cibak: temporary switch off of regitstration in FileToCat
      #
      #gLogger.info( 'Updating the FileToCat table for successful files' )
      #res = self.TransferDB.setRegistrationWaiting( channelID, completedFileIDs )
      #if not res['OK']:
      #  gLogger.error( 'Failed to update the FileToCat table for successful files.', res['Message'] )
      #  allUpdated = False

    if fileToFTSUpdates:
      gLogger.info( 'Updating the FileToFTS table for files' )
      res = self.TransferDB.setFileToFTSFileAttributes( ftsReqID, channelID, fileToFTSUpdates )
      if not res['OK']:
        gLogger.error( 'Failed to update the FileToFTS table for files.', res['Message'] )
        allUpdated = False

    if allUpdated:
      res = oFTSRequest.finalize()
      if not res['OK']:
        gLogger.error( "Failed to perform the finalization for the FTS request", res['Message'] )
        return res

      gLogger.info( 'Adding logging event for FTS request' )
      # Now set the FTSReq status to terminal so that it is not monitored again
      res = self.TransferDB.addLoggingEvent( ftsReqID, 'Finished' )
      if not res['OK']:
        gLogger.error( 'Failed to add logging event for FTS Request', res['Message'] )

      gLogger.info( 'Updating FTS request status' )
      res = self.TransferDB.setFTSReqStatus( ftsReqID, 'Finished' )
      if not res['OK']:
        gLogger.error( 'Failed update FTS Request status', res['Message'] )
    return S_OK()
예제 #7
0
from DIRAC.DataManagementSystem.Client.FTSRequest import FTSRequest
import os, sys

if not len(sys.argv) >= 6:
    Script.showHelp()
    DIRAC.exit(-1)
else:
    inputFileName = sys.argv[1]
    sourceSE = sys.argv[2]
    targetSE = sys.argv[3]
    guid = sys.argv[4]
    server = sys.argv[5]

if not os.path.exists(inputFileName):
    lfns = [inputFileName]
else:
    inputFile = open(inputFileName, 'r')
    string = inputFile.read()
    inputFile.close()
    lfns = string.splitlines()

oFTSRequest = FTSRequest()
for lfn in lfns:
    oFTSRequest.setLFN(lfn)
oFTSRequest.setFTSGUID(guid)
oFTSRequest.setFTSServer(server)
oFTSRequest.setSourceSE(sourceSE)
oFTSRequest.setTargetSE(targetSE)
oFTSRequest.monitor(untilTerminal=True)
예제 #8
0
    def monitorTransfer(self, ftsReqDict):
        """ monitors transfer obtained from TransferDB

    :param dict ftsReqDict: FTS job dictionary
    """
        ftsReqID = ftsReqDict.get("FTSReqID")
        ftsGUID = ftsReqDict.get("FTSGuid")
        ftsServer = ftsReqDict.get("FTSServer")
        channelID = ftsReqDict.get("ChannelID")
        sourceSE = ftsReqDict.get("SourceSE")
        targetSE = ftsReqDict.get("TargetSE")

        oFTSRequest = FTSRequest()
        oFTSRequest.setFTSServer(ftsServer)
        oFTSRequest.setFTSGUID(ftsGUID)
        oFTSRequest.setSourceSE(sourceSE)
        oFTSRequest.setTargetSE(targetSE)

        log = gLogger.getSubLogger("@%s" % str(ftsReqID))

        #########################################################################
        # Perform summary update of the FTS Request and update FTSReq entries.
        log.info("Perform summary update of the FTS Request")
        infoStr = ["glite-transfer-status -s %s -l %s" % (ftsServer, ftsGUID)]
        infoStr.append("FTS GUID:   %s" % ftsGUID)
        infoStr.append("FTS Server: %s" % ftsServer)
        log.info("\n".join(infoStr))
        res = oFTSRequest.summary()
        self.transferDB.setFTSReqLastMonitor(ftsReqID)
        if not res["OK"]:
            log.error("Failed to update the FTS request summary",
                      res["Message"])
            if "getTransferJobSummary2: Not authorised to query request" in res[
                    "Message"]:
                log.error(
                    "FTS job is not existing at the FTS server anymore, will clean it up on TransferDB side"
                )
                cleanUp = self.ftsJobExpired(ftsReqID, channelID)
                if not cleanUp["OK"]:
                    log.error(cleanUp["Message"])
                return cleanUp
            return res

        res = oFTSRequest.dumpSummary()
        if not res['OK']:
            log.error("Failed to get FTS request summary", res["Message"])
            return res
        log.info(res['Value'])
        res = oFTSRequest.getPercentageComplete()
        if not res['OK']:
            log.error("Failed to get FTS percentage complete", res["Message"])
            return res
        log.info('FTS Request found to be %.1f percent complete' %
                 res["Value"])
        self.transferDB.setFTSReqAttribute(ftsReqID, "PercentageComplete",
                                           res["Value"])
        self.transferDB.addLoggingEvent(ftsReqID, res["Value"])

        #########################################################################
        # Update the information in the TransferDB if the transfer is terminal.
        res = oFTSRequest.isRequestTerminal()
        if not res["OK"]:
            log.error("Failed to determine whether FTS request terminal",
                      res["Message"])
            return res
        if not res["Value"]:
            return S_OK()
        # # request is terminal
        return self.terminalRequest(oFTSRequest, ftsReqID, channelID, sourceSE)
예제 #9
0
  def monitorTransfer( self, ftsReqDict ):
    """ monitors transfer obtained from TransferDB

    :param dict ftsReqDict: FTS job dictionary
    """
    ftsReqID = ftsReqDict.get( "FTSReqID" )
    ftsGUID = ftsReqDict.get( "FTSGuid" )
    ftsServer = ftsReqDict.get( "FTSServer" )
    channelID = ftsReqDict.get( "ChannelID" )
    sourceSE = ftsReqDict.get( "SourceSE" )
    targetSE = ftsReqDict.get( "TargetSE" )

    oFTSRequest = FTSRequest()
    oFTSRequest.setFTSServer( ftsServer )
    oFTSRequest.setFTSGUID( ftsGUID )
    oFTSRequest.setSourceSE( sourceSE )
    oFTSRequest.setTargetSE( targetSE )

    log = gLogger.getSubLogger( "@%s" % str( ftsReqID ) )

    #########################################################################
    # Perform summary update of the FTS Request and update FTSReq entries.
    log.info( "Perform summary update of the FTS Request" )
    infoStr = [ "glite-transfer-status -s %s -l %s" % ( ftsServer, ftsGUID ) ]
    infoStr.append( "FTS GUID:   %s" % ftsGUID )
    infoStr.append( "FTS Server: %s" % ftsServer )
    log.info( "\n".join( infoStr ) )
    res = oFTSRequest.summary()
    self.transferDB.setFTSReqLastMonitor( ftsReqID )
    if not res["OK"]:
      log.error( "Failed to update the FTS request summary", res["Message"] )
      if "getTransferJobSummary2: Not authorised to query request" in res["Message"]:
        log.error( "FTS job is not existing at the FTS server anymore, will clean it up on TransferDB side" )
        cleanUp = self.ftsJobExpired( ftsReqID, channelID )
        if not cleanUp["OK"]:
          log.error( cleanUp["Message"] )
        return cleanUp
      return res

    res = oFTSRequest.dumpSummary()
    if not res['OK']:
      log.error( "Failed to get FTS request summary", res["Message"] )
      return res
    log.info( res['Value'] )
    res = oFTSRequest.getPercentageComplete()
    if not res['OK']:
      log.error( "Failed to get FTS percentage complete", res["Message"] )
      return res
    log.info( 'FTS Request found to be %.1f percent complete' % res["Value"] )
    self.transferDB.setFTSReqAttribute( ftsReqID, "PercentageComplete", res["Value"] )
    self.transferDB.addLoggingEvent( ftsReqID, res["Value"] )

    #########################################################################
    # Update the information in the TransferDB if the transfer is terminal.
    res = oFTSRequest.isRequestTerminal()
    if not res["OK"]:
      log.error( "Failed to determine whether FTS request terminal", res["Message"] )
      return res
    if not res["Value"]:
      return S_OK()
    # # request is terminal
    return self.terminalRequest( oFTSRequest, ftsReqID, channelID, sourceSE )
예제 #10
0
    def submitTransfer(self, channelDict):
        """ This method creates and submits FTS jobs based on information it gets from the DB
    """

        # Create the FTSRequest object for preparing the submission
        oFTSRequest = FTSRequest()
        channelID = channelDict['ChannelID']
        filesPerJob = channelDict['NumFiles']

        #########################################################################
        #  Obtain the first files in the selected channel.
        gLogger.info(
            "FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s"
            % channelID)
        res = self.TransferDB.getFilesForChannel(channelID, 2 * filesPerJob)
        if not res['OK']:
            errStr = 'FTSSubmitAgent.%s' % res['Message']
            gLogger.error(errStr)
            return S_OK()
        if not res['Value']:
            gLogger.info(
                "FTSSubmitAgent.submitTransfer: No files to found for channel."
            )
            return S_OK()
        filesDict = res['Value']
        gLogger.info('Obtained %s files for channel' % len(filesDict['Files']))

        sourceSE = filesDict['SourceSE']
        oFTSRequest.setSourceSE(sourceSE)
        targetSE = filesDict['TargetSE']
        oFTSRequest.setTargetSE(targetSE)
        gLogger.info(
            "FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel."
            % (sourceSE, targetSE))
        files = filesDict['Files']

        #########################################################################
        #  Populate the FTS Request with the files.
        gLogger.info('Populating the FTS request with file information')
        fileIDs = []
        totalSize = 0
        fileIDSizes = {}
        for file in files:
            lfn = file['LFN']
            oFTSRequest.setLFN(lfn)
            oFTSRequest.setSourceSURL(lfn, file['SourceSURL'])
            oFTSRequest.setTargetSURL(lfn, file['TargetSURL'])
            fileID = file['FileID']
            fileIDs.append(fileID)
            totalSize += file['Size']
            fileIDSizes[fileID] = file['Size']

        #########################################################################
        #  Submit the FTS request and retrieve the FTS GUID/Server
        gLogger.info('Submitting the FTS request')
        res = oFTSRequest.submit()
        if not res['OK']:
            errStr = "FTSSubmitAgent.%s" % res['Message']
            gLogger.error(errStr)
            gLogger.info('Updating the Channel table for files to retry')
            res = self.TransferDB.resetFileChannelStatus(channelID, fileIDs)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the Channel table for file to retry.',
                    res['Message'])
            return S_ERROR(errStr)
        ftsGUID = res['Value']['ftsGUID']
        ftsServer = res['Value']['ftsServer']
        infoStr = """Submitted FTS Job:
    
              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % (ftsGUID, ftsServer, str(channelID), sourceSE, targetSE, str(len(files)))
        gLogger.info(infoStr)

        #########################################################################
        #  Insert the FTS Req details and add the number of files and size
        res = self.TransferDB.insertFTSReq(ftsGUID, ftsServer, channelID)
        if not res['OK']:
            errStr = "FTSSubmitAgent.%s" % res['Message']
            gLogger.error(errStr)
            return S_ERROR(errStr)
        ftsReqID = res['Value']
        gLogger.info('Obtained FTS RequestID %s' % ftsReqID)
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'SourceSE',
                                                 sourceSE)
        if not res['OK']:
            gLogger.error("Failed to set SourceSE for FTSRequest",
                          res['Message'])
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'TargetSE',
                                                 targetSE)
        if not res['OK']:
            gLogger.error("Failed to set TargetSE for FTSRequest",
                          res['Message'])
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'NumberOfFiles',
                                                 len(fileIDs))
        if not res['OK']:
            gLogger.error("Failed to set NumberOfFiles for FTSRequest",
                          res['Message'])
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'TotalSize',
                                                 totalSize)
        if not res['OK']:
            gLogger.error("Failed to set TotalSize for FTSRequest",
                          res['Message'])

        #########################################################################
        #  Insert the submission event in the FTSReqLogging table
        event = 'Submitted'
        res = self.TransferDB.addLoggingEvent(ftsReqID, event)
        if not res['OK']:
            errStr = "FTSSubmitAgent.%s" % res['Message']
            gLogger.error(errStr)

        #########################################################################
        #  Insert the FileToFTS details and remove the files from the channel
        gLogger.info('Setting the files as Executing in the Channel table')
        res = self.TransferDB.setChannelFilesExecuting(channelID, fileIDs)
        if not res['OK']:
            gLogger.error('Failed to update the Channel tables for files.',
                          res['Message'])

        lfns = []
        fileToFTSFileAttributes = []
        for file in files:
            lfn = file['LFN']
            fileID = file['FileID']
            lfns.append(lfn)
            fileToFTSFileAttributes.append((fileID, fileIDSizes[fileID]))

        gLogger.info('Populating the FileToFTS table with file information')
        res = self.TransferDB.setFTSReqFiles(ftsReqID, channelID,
                                             fileToFTSFileAttributes)
        if not res['OK']:
            gLogger.error('Failed to populate the FileToFTS table with files.')
예제 #11
0
  def submitTransfer( self, channelDict ):
    """ This method creates and submits FTS jobs based on information it gets from the DB
    """

    # Create the FTSRequest object for preparing the submission
    oFTSRequest = FTSRequest()
    channelID = channelDict['ChannelID']
    filesPerJob = channelDict['NumFiles']

    #########################################################################
    #  Obtain the first files in the selected channel.
    gLogger.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s" % channelID )
    res = self.TransferDB.getFilesForChannel( channelID, 2 * filesPerJob )
    if not res['OK']:
      errStr = 'FTSSubmitAgent.%s' % res['Message']
      gLogger.error( errStr )
      return S_OK()
    if not res['Value']:
      gLogger.info( "FTSSubmitAgent.submitTransfer: No files to found for channel." )
      return S_OK()
    filesDict = res['Value']
    gLogger.info( 'Obtained %s files for channel' % len( filesDict['Files'] ) )

    sourceSE = filesDict['SourceSE']
    oFTSRequest.setSourceSE( sourceSE )
    targetSE = filesDict['TargetSE']
    oFTSRequest.setTargetSE( targetSE )
    gLogger.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel." % ( sourceSE, targetSE ) )
    files = filesDict['Files']

    #########################################################################
    #  Populate the FTS Request with the files.
    gLogger.info( 'Populating the FTS request with file information' )
    fileIDs = []
    totalSize = 0
    fileIDSizes = {}
    for file in files:
      lfn = file['LFN']
      oFTSRequest.setLFN( lfn )
      oFTSRequest.setSourceSURL( lfn, file['SourceSURL'] )
      oFTSRequest.setTargetSURL( lfn, file['TargetSURL'] )
      fileID = file['FileID']
      fileIDs.append( fileID )
      totalSize += file['Size']
      fileIDSizes[fileID] = file['Size']

    #########################################################################
    #  Submit the FTS request and retrieve the FTS GUID/Server
    gLogger.info( 'Submitting the FTS request' )
    res = oFTSRequest.submit()
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      gLogger.error( errStr )
      gLogger.info( 'Updating the Channel table for files to retry' )
      res = self.TransferDB.resetFileChannelStatus( channelID, fileIDs )
      if not res['OK']:
        gLogger.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
      return S_ERROR( errStr )
    ftsGUID = res['Value']['ftsGUID']
    ftsServer = res['Value']['ftsServer']
    infoStr = """Submitted FTS Job:
    
              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( len( files ) ) )
    gLogger.info( infoStr )

    #########################################################################
    #  Insert the FTS Req details and add the number of files and size
    res = self.TransferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      gLogger.error( errStr )
      return S_ERROR( errStr )
    ftsReqID = res['Value']
    gLogger.info( 'Obtained FTS RequestID %s' % ftsReqID )
    res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
    if not res['OK']:
      gLogger.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
    res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
    if not res['OK']:
      gLogger.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
    res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( fileIDs ) )
    if not res['OK']:
      gLogger.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
    res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
    if not res['OK']:
      gLogger.error( "Failed to set TotalSize for FTSRequest", res['Message'] )

    #########################################################################
    #  Insert the submission event in the FTSReqLogging table
    event = 'Submitted'
    res = self.TransferDB.addLoggingEvent( ftsReqID, event )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      gLogger.error( errStr )

    #########################################################################
    #  Insert the FileToFTS details and remove the files from the channel
    gLogger.info( 'Setting the files as Executing in the Channel table' )
    res = self.TransferDB.setChannelFilesExecuting( channelID, fileIDs )
    if not res['OK']:
      gLogger.error( 'Failed to update the Channel tables for files.', res['Message'] )

    lfns = []
    fileToFTSFileAttributes = []
    for file in files:
      lfn = file['LFN']
      fileID = file['FileID']
      lfns.append( lfn )
      fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )

    gLogger.info( 'Populating the FileToFTS table with file information' )
    res = self.TransferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
    if not res['OK']:
      gLogger.error( 'Failed to populate the FileToFTS table with files.' )
예제 #12
0
  def submitTransfer( self, channelDict ):
    """ create and submit FTS jobs based on information it gets from the DB

    :param self: self reference
    :param dict channelDict: dict with channel info as read from TransferDB.selectChannelsForSubmission
    """

    channelID = channelDict['ChannelID']
    filesPerJob = channelDict['NumFiles']

    #########################################################################
    #  Obtain the first files in the selected channel.
    self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for channel %s: %s to %s"
                   % ( channelID, channelDict['Source'], channelDict['Destination'] ) )
    res = self.transferDB.getFilesForChannel( channelID, 2 * filesPerJob )
    if not res['OK']:
      errStr = 'FTSSubmitAgent.%s' % res['Message']
      self.log.error( errStr )
      return S_OK()
    if not res['Value']:
      self.log.info( "FTSSubmitAgent.submitTransfer: No files found for channel." )
      return S_OK()
    filesDict = res['Value']
    sourceSE = filesDict['SourceSE']
    targetSE = filesDict['TargetSE']
    self.log.info( 'Obtained %s files for channel %s to %s' % ( len( filesDict['Files'] ), sourceSE, targetSE ) )
    if self.filesBeingStaged.get( channelID ):
      self.log.info( '%d files are currently in staging status' % len( self.filesBeingStaged[channelID] ) )

    # Create the FTSRequest object for preparing the submission
    oFTSRequest = FTSRequest()
    oFTSRequest.setSourceSE( sourceSE )
    oFTSRequest.setTargetSE( targetSE )
    files = filesDict['Files']

    # # enable/disable cksm test
    oFTSRequest.setCksmTest( self.cksmTest )
    if self.cksmType:
      oFTSRequest.setCksmType( self.cksmType )

    #########################################################################
    #  Populate the FTS Request with the files.
    self.log.info( 'Populating the FTS request with file information' )
    fileIDs = []
    totalSize = 0
    fileIDSizes = {}
    lfns = set()
    for fileMeta in files:
      lfn = fileMeta['LFN']
      lfns.add( lfn )
      oFTSRequest.setLFN( lfn )
      oFTSRequest.setSourceSURL( lfn, fileMeta['SourceSURL'] )
      oFTSRequest.setTargetSURL( lfn, fileMeta['TargetSURL'] )
      if lfn in self.filesBeingStaged.get( channelID, [] ):
        oFTSRequest.setStatus( lfn, 'Staging' )
      fileID = fileMeta['FileID']
      fileIDs.append( fileID )
      totalSize += fileMeta['Size']
      fileIDSizes[fileID] = fileMeta['Size']

    oFTSRequest.resolveSource()
    noSource = [ lfn for lfn, fileInfo in oFTSRequest.fileDict.items()
                     if fileInfo.get( "Status", "" ) == "Failed" and fileInfo.get( "Reason", "" ) in ( "No replica at SourceSE",
                                                                                                   "Source file does not exist" ) ]
    toReschedule = [fileMeta["FileID"] for fileMeta in files if fileMeta["LFN"] in noSource]

    if toReschedule:
      self.log.info( "Found %s files to reschedule" % len( toReschedule ) )
      for fileID in toReschedule:
        res = self.transferDB.setFileToReschedule( fileID )
        if not res["OK"]:
          self.log.error( "Failed to update Channel table for failed files.", res["Message"] )
        elif res["Value"] == "max reschedule attempt reached":
          self.log.error( "setting Channel status to 'Failed' : " % res["Value"] )
          res = self.transferDB.setFileChannelStatus( channelID, fileID, 'Failed' )
          if not res["OK"]:
            self.log.error( "Failed to update Channel table for failed files.", res["Message"] )

    #########################################################################
    #  Submit the FTS request and retrieve the FTS GUID/Server
    self.log.info( 'Submitting the FTS request' )
    res = oFTSRequest.submit()
    if not res['OK']:
      errStr = "FTSSubmitAgent.submit: %s" % res['Message']
      self.log.error( errStr )
      self.log.info( 'Updating the Channel table for files to retry' )
      res = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
      if not res['OK']:
        self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
      return S_ERROR( errStr )
    ftsGUID = res['Value']['ftsGUID']
    ftsServer = res['Value']['ftsServer']
    nbSubmitted = res['Value']['submittedFiles']
    infoStr = """Submitted FTS Job:

              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( nbSubmitted ) )
    self.log.info( infoStr )

    # # filter out skipped files
    failedFiles = oFTSRequest.getFailed()['Value']
    stagingFiles = oFTSRequest.getStaging()['Value']
    # cache files being staged
    self.filesBeingStaged.setdefault( channelID, set() ).update( stagingFiles )
    submittedFiles = lfns.difference( failedFiles, stagingFiles )
    # files being submitted are staged
    self.filesBeingStaged[channelID] -= submittedFiles
    failedIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in failedFiles ] )
    stagingIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in stagingFiles ] )
    # # only submitted
    submittedIDs = set( fileIDs ) - failedIDs - stagingIDs
    # # only count the submitted size
    totalSize = sum( [ meta["Size"] for meta in files if meta["FileID"] in submittedIDs ] )

    #########################################################################
    #  Insert the FTS Req details and add the number of files and size
    res = self.transferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )
      return S_ERROR( errStr )
    ftsReqID = res['Value']
    self.log.info( 'Obtained FTS RequestID %s' % ftsReqID )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
    if not res['OK']:
      self.log.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
    if not res['OK']:
      self.log.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( submittedIDs ) )
    if not res['OK']:
      self.log.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
    if not res['OK']:
      self.log.error( "Failed to set TotalSize for FTSRequest", res['Message'] )

    #########################################################################
    #  Insert the submission event in the FTSReqLogging table
    event = 'Submitted'
    res = self.transferDB.addLoggingEvent( ftsReqID, event )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )

    #########################################################################
    #  Insert the FileToFTS details and remove the files from the channel
    self.log.info( 'Setting the files as Executing in the Channel table' )
    res = self.transferDB.setChannelFilesExecuting( channelID, list( submittedIDs ) )
    if not res['OK']:
      self.log.error( 'Failed to update the Channel tables for files.', res['Message'] )

    lfns = []
    fileToFTSFileAttributes = []
    for fileMeta in files:
      fileID = fileMeta['FileID']
      # Staging is not an error case
      if fileID not in stagingIDs:
        lfns.append( fileMeta['LFN'] )
        fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )

    self.log.info( 'Populating the FileToFTS table with file information' )
    res = self.transferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
    if not res['OK']:
      self.log.error( 'Failed to populate the FileToFTS table with files.' )