Example #1
0
    def monitorTransfer(self, ftsReqDict):
        """ Monitors transfer  obtained from TransferDB
    """

        ftsReqID = ftsReqDict['FTSReqID']
        ftsGUID = ftsReqDict['FTSGuid']
        ftsServer = ftsReqDict['FTSServer']
        channelID = ftsReqDict['ChannelID']
        sourceSE = ftsReqDict['SourceSE']
        targetSE = ftsReqDict['TargetSE']

        oFTSRequest = FTSRequest()
        oFTSRequest.setFTSServer(ftsServer)
        oFTSRequest.setFTSGUID(ftsGUID)
        oFTSRequest.setSourceSE(sourceSE)
        oFTSRequest.setTargetSE(targetSE)

        #########################################################################
        # Perform summary update of the FTS Request and update FTSReq entries.
        gLogger.info('Perform summary update of the FTS Request')
        infoStr = "Monitoring FTS Job:\n\n"
        infoStr = "%sglite-transfer-status -s %s -l %s\n" % (
            infoStr, ftsServer, ftsGUID)
        infoStr = "%s%s%s\n" % (infoStr, 'FTS GUID:'.ljust(20), ftsGUID)
        infoStr = "%s%s%s\n\n" % (infoStr, 'FTS Server:'.ljust(20), ftsServer)
        gLogger.info(infoStr)
        res = oFTSRequest.summary()
        self.TransferDB.setFTSReqLastMonitor(ftsReqID)
        if not res['OK']:
            gLogger.error("Failed to update the FTS request summary",
                          res['Message'])
            return res
        res = oFTSRequest.dumpSummary()
        if not res['OK']:
            gLogger.error("Failed to get FTS request summary", res['Message'])
            return res
        gLogger.info(res['Value'])
        res = oFTSRequest.getPercentageComplete()
        if not res['OK']:
            gLogger.error("Failed to get FTS percentage complete",
                          res['Message'])
            return res
        gLogger.info('FTS Request found to be %.1f percent complete' %
                     res['Value'])
        self.TransferDB.setFTSReqAttribute(ftsReqID, 'PercentageComplete',
                                           res['Value'])
        self.TransferDB.addLoggingEvent(ftsReqID, res['Value'])

        #########################################################################
        # Update the information in the TransferDB if the transfer is terminal.
        res = oFTSRequest.isRequestTerminal()
        if not res['OK']:
            gLogger.error("Failed to determine whether FTS request terminal",
                          res['Message'])
            return res
        if not res['Value']:
            return S_OK()
        gLogger.info('FTS Request found to be terminal, updating file states')

        #########################################################################
        # Get the LFNS associated to the FTS request
        gLogger.info('Obtaining the LFNs associated to this request')
        res = self.TransferDB.getFTSReqLFNs(ftsReqID)
        if not res['OK']:
            gLogger.error("Failed to obtain FTS request LFNs", res['Message'])
            return res
        files = res['Value']
        if not files:
            gLogger.error('No files present for transfer')
            return S_ERROR('No files were found in the DB')
        lfns = files.keys()
        gLogger.info('Obtained %s files' % len(lfns))
        for lfn in lfns:
            oFTSRequest.setLFN(lfn)

        res = oFTSRequest.monitor()
        if not res['OK']:
            gLogger.error(
                "Failed to perform detailed monitoring of FTS request",
                res['Message'])
            return res
        res = oFTSRequest.getFailed()
        if not res['OK']:
            gLogger.error("Failed to obtained failed files for FTS request",
                          res['Message'])
            return res
        failedFiles = res['Value']
        res = oFTSRequest.getDone()
        if not res['OK']:
            gLogger.error(
                "Failed to obtained successful files for FTS request",
                res['Message'])
            return res
        completedFiles = res['Value']

        fileToFTSUpdates = []
        completedFileIDs = []
        for lfn in completedFiles:
            fileID = files[lfn]
            completedFileIDs.append(fileID)
            transferTime = 0
            res = oFTSRequest.getTransferTime(lfn)
            if res['OK']:
                transferTime = res['Value']
            fileToFTSUpdates.append((fileID, 'Completed', '', 0, transferTime))

        filesToRetry = []
        filesToReschedule = []
        for lfn in failedFiles:
            fileID = files[lfn]
            failReason = ''
            res = oFTSRequest.getFailReason(lfn)
            if res['OK']:
                failReason = res['Value']
            if self.missingSource(failReason):
                gLogger.error('The source SURL does not exist.',
                              '%s %s' % (lfn, oFTSRequest.getSourceSURL(lfn)))
                filesToReschedule.append(fileID)
            else:
                filesToRetry.append(fileID)
            gLogger.error('Failed to replicate file on channel.',
                          "%s %s" % (channelID, failReason))
            fileToFTSUpdates.append((fileID, 'Failed', failReason, 0, 0))

        allUpdated = True
        if filesToRetry:
            gLogger.info('Updating the Channel table for files to retry')
            res = self.TransferDB.resetFileChannelStatus(
                channelID, filesToRetry)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the Channel table for file to retry.',
                    res['Message'])
                allUpdated = False
        for fileID in filesToReschedule:
            gLogger.info('Updating the Channel table for files to reschedule')
            res = self.TransferDB.setFileChannelStatus(channelID, fileID,
                                                       'Failed')
            if not res['OK']:
                gLogger.error(
                    'Failed to update Channel table for failed files.',
                    res['Message'])
                allUpdated = False

        if completedFileIDs:
            gLogger.info('Updating the Channel table for successful files')
            res = self.TransferDB.updateCompletedChannelStatus(
                channelID, completedFileIDs)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the Channel table for successful files.',
                    res['Message'])
                allUpdated = False
            gLogger.info(
                'Updating the Channel table for ancestors of successful files')
            res = self.TransferDB.updateAncestorChannelStatus(
                channelID, completedFileIDs)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the Channel table for ancestors of successful files.',
                    res['Message'])
                allUpdated = False

            #
            #  cibak: temporary switch off of regitstration in FileToCat
            #
            #gLogger.info( 'Updating the FileToCat table for successful files' )
            #res = self.TransferDB.setRegistrationWaiting( channelID, completedFileIDs )
            #if not res['OK']:
            #  gLogger.error( 'Failed to update the FileToCat table for successful files.', res['Message'] )
            #  allUpdated = False

        if fileToFTSUpdates:
            gLogger.info('Updating the FileToFTS table for files')
            res = self.TransferDB.setFileToFTSFileAttributes(
                ftsReqID, channelID, fileToFTSUpdates)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the FileToFTS table for files.',
                    res['Message'])
                allUpdated = False

        if allUpdated:
            res = oFTSRequest.finalize()
            if not res['OK']:
                gLogger.error(
                    "Failed to perform the finalization for the FTS request",
                    res['Message'])
                return res

            gLogger.info('Adding logging event for FTS request')
            # Now set the FTSReq status to terminal so that it is not monitored again
            res = self.TransferDB.addLoggingEvent(ftsReqID, 'Finished')
            if not res['OK']:
                gLogger.error('Failed to add logging event for FTS Request',
                              res['Message'])

            gLogger.info('Updating FTS request status')
            res = self.TransferDB.setFTSReqStatus(ftsReqID, 'Finished')
            if not res['OK']:
                gLogger.error('Failed update FTS Request status',
                              res['Message'])
        return S_OK()
from DIRAC.DataManagementSystem.Client.FTSRequest import FTSRequest
import os, sys

if not len(sys.argv) >= 6:
    Script.showHelp()
    DIRAC.exit(-1)
else:
    inputFileName = sys.argv[1]
    sourceSE = sys.argv[2]
    targetSE = sys.argv[3]
    guid = sys.argv[4]
    server = sys.argv[5]

if not os.path.exists(inputFileName):
    lfns = [inputFileName]
else:
    inputFile = open(inputFileName, 'r')
    string = inputFile.read()
    inputFile.close()
    lfns = string.splitlines()

oFTSRequest = FTSRequest()
for lfn in lfns:
    oFTSRequest.setLFN(lfn)
oFTSRequest.setFTSGUID(guid)
oFTSRequest.setFTSServer(server)
oFTSRequest.setSourceSE(sourceSE)
oFTSRequest.setTargetSE(targetSE)
oFTSRequest.monitor(untilTerminal=True)
Example #3
0
    def submitTransfer(self, channelDict):
        """ This method creates and submits FTS jobs based on information it gets from the DB
    """

        # Create the FTSRequest object for preparing the submission
        oFTSRequest = FTSRequest()
        channelID = channelDict['ChannelID']
        filesPerJob = channelDict['NumFiles']

        #########################################################################
        #  Obtain the first files in the selected channel.
        gLogger.info(
            "FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s"
            % channelID)
        res = self.TransferDB.getFilesForChannel(channelID, 2 * filesPerJob)
        if not res['OK']:
            errStr = 'FTSSubmitAgent.%s' % res['Message']
            gLogger.error(errStr)
            return S_OK()
        if not res['Value']:
            gLogger.info(
                "FTSSubmitAgent.submitTransfer: No files to found for channel."
            )
            return S_OK()
        filesDict = res['Value']
        gLogger.info('Obtained %s files for channel' % len(filesDict['Files']))

        sourceSE = filesDict['SourceSE']
        oFTSRequest.setSourceSE(sourceSE)
        targetSE = filesDict['TargetSE']
        oFTSRequest.setTargetSE(targetSE)
        gLogger.info(
            "FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel."
            % (sourceSE, targetSE))
        files = filesDict['Files']

        #########################################################################
        #  Populate the FTS Request with the files.
        gLogger.info('Populating the FTS request with file information')
        fileIDs = []
        totalSize = 0
        fileIDSizes = {}
        for file in files:
            lfn = file['LFN']
            oFTSRequest.setLFN(lfn)
            oFTSRequest.setSourceSURL(lfn, file['SourceSURL'])
            oFTSRequest.setTargetSURL(lfn, file['TargetSURL'])
            fileID = file['FileID']
            fileIDs.append(fileID)
            totalSize += file['Size']
            fileIDSizes[fileID] = file['Size']

        #########################################################################
        #  Submit the FTS request and retrieve the FTS GUID/Server
        gLogger.info('Submitting the FTS request')
        res = oFTSRequest.submit()
        if not res['OK']:
            errStr = "FTSSubmitAgent.%s" % res['Message']
            gLogger.error(errStr)
            gLogger.info('Updating the Channel table for files to retry')
            res = self.TransferDB.resetFileChannelStatus(channelID, fileIDs)
            if not res['OK']:
                gLogger.error(
                    'Failed to update the Channel table for file to retry.',
                    res['Message'])
            return S_ERROR(errStr)
        ftsGUID = res['Value']['ftsGUID']
        ftsServer = res['Value']['ftsServer']
        infoStr = """Submitted FTS Job:
    
              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % (ftsGUID, ftsServer, str(channelID), sourceSE, targetSE, str(len(files)))
        gLogger.info(infoStr)

        #########################################################################
        #  Insert the FTS Req details and add the number of files and size
        res = self.TransferDB.insertFTSReq(ftsGUID, ftsServer, channelID)
        if not res['OK']:
            errStr = "FTSSubmitAgent.%s" % res['Message']
            gLogger.error(errStr)
            return S_ERROR(errStr)
        ftsReqID = res['Value']
        gLogger.info('Obtained FTS RequestID %s' % ftsReqID)
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'SourceSE',
                                                 sourceSE)
        if not res['OK']:
            gLogger.error("Failed to set SourceSE for FTSRequest",
                          res['Message'])
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'TargetSE',
                                                 targetSE)
        if not res['OK']:
            gLogger.error("Failed to set TargetSE for FTSRequest",
                          res['Message'])
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'NumberOfFiles',
                                                 len(fileIDs))
        if not res['OK']:
            gLogger.error("Failed to set NumberOfFiles for FTSRequest",
                          res['Message'])
        res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'TotalSize',
                                                 totalSize)
        if not res['OK']:
            gLogger.error("Failed to set TotalSize for FTSRequest",
                          res['Message'])

        #########################################################################
        #  Insert the submission event in the FTSReqLogging table
        event = 'Submitted'
        res = self.TransferDB.addLoggingEvent(ftsReqID, event)
        if not res['OK']:
            errStr = "FTSSubmitAgent.%s" % res['Message']
            gLogger.error(errStr)

        #########################################################################
        #  Insert the FileToFTS details and remove the files from the channel
        gLogger.info('Setting the files as Executing in the Channel table')
        res = self.TransferDB.setChannelFilesExecuting(channelID, fileIDs)
        if not res['OK']:
            gLogger.error('Failed to update the Channel tables for files.',
                          res['Message'])

        lfns = []
        fileToFTSFileAttributes = []
        for file in files:
            lfn = file['LFN']
            fileID = file['FileID']
            lfns.append(lfn)
            fileToFTSFileAttributes.append((fileID, fileIDSizes[fileID]))

        gLogger.info('Populating the FileToFTS table with file information')
        res = self.TransferDB.setFTSReqFiles(ftsReqID, channelID,
                                             fileToFTSFileAttributes)
        if not res['OK']:
            gLogger.error('Failed to populate the FileToFTS table with files.')
Example #4
0
    def monitorTransfer(self, ftsReqDict):
        """ monitors transfer obtained from TransferDB

    :param dict ftsReqDict: FTS job dictionary
    """
        ftsReqID = ftsReqDict.get("FTSReqID")
        ftsGUID = ftsReqDict.get("FTSGuid")
        ftsServer = ftsReqDict.get("FTSServer")
        channelID = ftsReqDict.get("ChannelID")
        sourceSE = ftsReqDict.get("SourceSE")
        targetSE = ftsReqDict.get("TargetSE")

        oFTSRequest = FTSRequest()
        oFTSRequest.setFTSServer(ftsServer)
        oFTSRequest.setFTSGUID(ftsGUID)
        oFTSRequest.setSourceSE(sourceSE)
        oFTSRequest.setTargetSE(targetSE)

        log = gLogger.getSubLogger("@%s" % str(ftsReqID))

        #########################################################################
        # Perform summary update of the FTS Request and update FTSReq entries.
        log.info("Perform summary update of the FTS Request")
        infoStr = ["glite-transfer-status -s %s -l %s" % (ftsServer, ftsGUID)]
        infoStr.append("FTS GUID:   %s" % ftsGUID)
        infoStr.append("FTS Server: %s" % ftsServer)
        log.info("\n".join(infoStr))
        res = oFTSRequest.summary()
        self.transferDB.setFTSReqLastMonitor(ftsReqID)
        if not res["OK"]:
            log.error("Failed to update the FTS request summary",
                      res["Message"])
            if "getTransferJobSummary2: Not authorised to query request" in res[
                    "Message"]:
                log.error(
                    "FTS job is not existing at the FTS server anymore, will clean it up on TransferDB side"
                )
                cleanUp = self.ftsJobExpired(ftsReqID, channelID)
                if not cleanUp["OK"]:
                    log.error(cleanUp["Message"])
                return cleanUp
            return res

        res = oFTSRequest.dumpSummary()
        if not res['OK']:
            log.error("Failed to get FTS request summary", res["Message"])
            return res
        log.info(res['Value'])
        res = oFTSRequest.getPercentageComplete()
        if not res['OK']:
            log.error("Failed to get FTS percentage complete", res["Message"])
            return res
        log.info('FTS Request found to be %.1f percent complete' %
                 res["Value"])
        self.transferDB.setFTSReqAttribute(ftsReqID, "PercentageComplete",
                                           res["Value"])
        self.transferDB.addLoggingEvent(ftsReqID, res["Value"])

        #########################################################################
        # Update the information in the TransferDB if the transfer is terminal.
        res = oFTSRequest.isRequestTerminal()
        if not res["OK"]:
            log.error("Failed to determine whether FTS request terminal",
                      res["Message"])
            return res
        if not res["Value"]:
            return S_OK()
        # # request is terminal
        return self.terminalRequest(oFTSRequest, ftsReqID, channelID, sourceSE)
Example #5
0
  def submitTransfer( self, channelDict ):
    """ create and submit FTS jobs based on information it gets from the DB

    :param self: self reference
    :param dict channelDict: dict with channel info as read from TransferDB.selectChannelsForSubmission
    """

    channelID = channelDict['ChannelID']
    filesPerJob = channelDict['NumFiles']

    #########################################################################
    #  Obtain the first files in the selected channel.
    self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for channel %s: %s to %s"
                   % ( channelID, channelDict['Source'], channelDict['Destination'] ) )
    res = self.transferDB.getFilesForChannel( channelID, 2 * filesPerJob )
    if not res['OK']:
      errStr = 'FTSSubmitAgent.%s' % res['Message']
      self.log.error( errStr )
      return S_OK()
    if not res['Value']:
      self.log.info( "FTSSubmitAgent.submitTransfer: No files found for channel." )
      return S_OK()
    filesDict = res['Value']
    sourceSE = filesDict['SourceSE']
    targetSE = filesDict['TargetSE']
    self.log.info( 'Obtained %s files for channel %s to %s' % ( len( filesDict['Files'] ), sourceSE, targetSE ) )
    if self.filesBeingStaged.get( channelID ):
      self.log.info( '%d files are currently in staging status' % len( self.filesBeingStaged[channelID] ) )

    # Create the FTSRequest object for preparing the submission
    oFTSRequest = FTSRequest()
    oFTSRequest.setSourceSE( sourceSE )
    oFTSRequest.setTargetSE( targetSE )
    files = filesDict['Files']

    # # enable/disable cksm test
    oFTSRequest.setCksmTest( self.cksmTest )
    if self.cksmType:
      oFTSRequest.setCksmType( self.cksmType )

    #########################################################################
    #  Populate the FTS Request with the files.
    self.log.info( 'Populating the FTS request with file information' )
    fileIDs = []
    totalSize = 0
    fileIDSizes = {}
    lfns = set()
    for fileMeta in files:
      lfn = fileMeta['LFN']
      lfns.add( lfn )
      oFTSRequest.setLFN( lfn )
      oFTSRequest.setSourceSURL( lfn, fileMeta['SourceSURL'] )
      oFTSRequest.setTargetSURL( lfn, fileMeta['TargetSURL'] )
      if lfn in self.filesBeingStaged.get( channelID, [] ):
        oFTSRequest.setStatus( lfn, 'Staging' )
      fileID = fileMeta['FileID']
      fileIDs.append( fileID )
      totalSize += fileMeta['Size']
      fileIDSizes[fileID] = fileMeta['Size']

    oFTSRequest.resolveSource()
    noSource = [ lfn for lfn, fileInfo in oFTSRequest.fileDict.items()
                     if fileInfo.get( "Status", "" ) == "Failed" and fileInfo.get( "Reason", "" ) in ( "No replica at SourceSE",
                                                                                                   "Source file does not exist" ) ]
    toReschedule = [fileMeta["FileID"] for fileMeta in files if fileMeta["LFN"] in noSource]

    if toReschedule:
      self.log.info( "Found %s files to reschedule" % len( toReschedule ) )
      for fileID in toReschedule:
        res = self.transferDB.setFileToReschedule( fileID )
        if not res["OK"]:
          self.log.error( "Failed to update Channel table for failed files.", res["Message"] )
        elif res["Value"] == "max reschedule attempt reached":
          self.log.error( "setting Channel status to 'Failed' : " % res["Value"] )
          res = self.transferDB.setFileChannelStatus( channelID, fileID, 'Failed' )
          if not res["OK"]:
            self.log.error( "Failed to update Channel table for failed files.", res["Message"] )

    #########################################################################
    #  Submit the FTS request and retrieve the FTS GUID/Server
    self.log.info( 'Submitting the FTS request' )
    res = oFTSRequest.submit()
    if not res['OK']:
      errStr = "FTSSubmitAgent.submit: %s" % res['Message']
      self.log.error( errStr )
      self.log.info( 'Updating the Channel table for files to retry' )
      res = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
      if not res['OK']:
        self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
      return S_ERROR( errStr )
    ftsGUID = res['Value']['ftsGUID']
    ftsServer = res['Value']['ftsServer']
    nbSubmitted = res['Value']['submittedFiles']
    infoStr = """Submitted FTS Job:

              FTS Guid: %s
              FTS Server: %s
              ChannelID: %s
              SourceSE: %s
              TargetSE: %s
              Files: %s

""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( nbSubmitted ) )
    self.log.info( infoStr )

    # # filter out skipped files
    failedFiles = oFTSRequest.getFailed()['Value']
    stagingFiles = oFTSRequest.getStaging()['Value']
    # cache files being staged
    self.filesBeingStaged.setdefault( channelID, set() ).update( stagingFiles )
    submittedFiles = lfns.difference( failedFiles, stagingFiles )
    # files being submitted are staged
    self.filesBeingStaged[channelID] -= submittedFiles
    failedIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in failedFiles ] )
    stagingIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in stagingFiles ] )
    # # only submitted
    submittedIDs = set( fileIDs ) - failedIDs - stagingIDs
    # # only count the submitted size
    totalSize = sum( [ meta["Size"] for meta in files if meta["FileID"] in submittedIDs ] )

    #########################################################################
    #  Insert the FTS Req details and add the number of files and size
    res = self.transferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )
      return S_ERROR( errStr )
    ftsReqID = res['Value']
    self.log.info( 'Obtained FTS RequestID %s' % ftsReqID )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
    if not res['OK']:
      self.log.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
    if not res['OK']:
      self.log.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( submittedIDs ) )
    if not res['OK']:
      self.log.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
    res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
    if not res['OK']:
      self.log.error( "Failed to set TotalSize for FTSRequest", res['Message'] )

    #########################################################################
    #  Insert the submission event in the FTSReqLogging table
    event = 'Submitted'
    res = self.transferDB.addLoggingEvent( ftsReqID, event )
    if not res['OK']:
      errStr = "FTSSubmitAgent.%s" % res['Message']
      self.log.error( errStr )

    #########################################################################
    #  Insert the FileToFTS details and remove the files from the channel
    self.log.info( 'Setting the files as Executing in the Channel table' )
    res = self.transferDB.setChannelFilesExecuting( channelID, list( submittedIDs ) )
    if not res['OK']:
      self.log.error( 'Failed to update the Channel tables for files.', res['Message'] )

    lfns = []
    fileToFTSFileAttributes = []
    for fileMeta in files:
      fileID = fileMeta['FileID']
      # Staging is not an error case
      if fileID not in stagingIDs:
        lfns.append( fileMeta['LFN'] )
        fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )

    self.log.info( 'Populating the FileToFTS table with file information' )
    res = self.transferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
    if not res['OK']:
      self.log.error( 'Failed to populate the FileToFTS table with files.' )