def monitorTransfer(self, ftsReqDict):
""" Monitors transfer obtained from TransferDB
"""
ftsReqID = ftsReqDict['FTSReqID']
ftsGUID = ftsReqDict['FTSGuid']
ftsServer = ftsReqDict['FTSServer']
channelID = ftsReqDict['ChannelID']
sourceSE = ftsReqDict['SourceSE']
targetSE = ftsReqDict['TargetSE']
oFTSRequest = FTSRequest()
oFTSRequest.setFTSServer(ftsServer)
oFTSRequest.setFTSGUID(ftsGUID)
oFTSRequest.setSourceSE(sourceSE)
oFTSRequest.setTargetSE(targetSE)
#########################################################################
# Perform summary update of the FTS Request and update FTSReq entries.
gLogger.info('Perform summary update of the FTS Request')
infoStr = "Monitoring FTS Job:\n\n"
infoStr = "%sglite-transfer-status -s %s -l %s\n" % (
infoStr, ftsServer, ftsGUID)
infoStr = "%s%s%s\n" % (infoStr, 'FTS GUID:'.ljust(20), ftsGUID)
infoStr = "%s%s%s\n\n" % (infoStr, 'FTS Server:'.ljust(20), ftsServer)
gLogger.info(infoStr)
res = oFTSRequest.summary()
self.TransferDB.setFTSReqLastMonitor(ftsReqID)
if not res['OK']:
gLogger.error("Failed to update the FTS request summary",
res['Message'])
return res
res = oFTSRequest.dumpSummary()
if not res['OK']:
gLogger.error("Failed to get FTS request summary", res['Message'])
return res
gLogger.info(res['Value'])
res = oFTSRequest.getPercentageComplete()
if not res['OK']:
gLogger.error("Failed to get FTS percentage complete",
res['Message'])
return res
gLogger.info('FTS Request found to be %.1f percent complete' %
res['Value'])
self.TransferDB.setFTSReqAttribute(ftsReqID, 'PercentageComplete',
res['Value'])
self.TransferDB.addLoggingEvent(ftsReqID, res['Value'])
#########################################################################
# Update the information in the TransferDB if the transfer is terminal.
res = oFTSRequest.isRequestTerminal()
if not res['OK']:
gLogger.error("Failed to determine whether FTS request terminal",
res['Message'])
return res
if not res['Value']:
return S_OK()
gLogger.info('FTS Request found to be terminal, updating file states')
#########################################################################
# Get the LFNS associated to the FTS request
gLogger.info('Obtaining the LFNs associated to this request')
res = self.TransferDB.getFTSReqLFNs(ftsReqID)
if not res['OK']:
gLogger.error("Failed to obtain FTS request LFNs", res['Message'])
return res
files = res['Value']
if not files:
gLogger.error('No files present for transfer')
return S_ERROR('No files were found in the DB')
lfns = files.keys()
gLogger.info('Obtained %s files' % len(lfns))
for lfn in lfns:
oFTSRequest.setLFN(lfn)
res = oFTSRequest.monitor()
if not res['OK']:
gLogger.error(
"Failed to perform detailed monitoring of FTS request",
res['Message'])
return res
res = oFTSRequest.getFailed()
if not res['OK']:
gLogger.error("Failed to obtained failed files for FTS request",
res['Message'])
return res
failedFiles = res['Value']
res = oFTSRequest.getDone()
if not res['OK']:
gLogger.error(
"Failed to obtained successful files for FTS request",
res['Message'])
return res
completedFiles = res['Value']
fileToFTSUpdates = []
completedFileIDs = []
for lfn in completedFiles:
fileID = files[lfn]
completedFileIDs.append(fileID)
transferTime = 0
res = oFTSRequest.getTransferTime(lfn)
if res['OK']:
transferTime = res['Value']
fileToFTSUpdates.append((fileID, 'Completed', '', 0, transferTime))
filesToRetry = []
filesToReschedule = []
for lfn in failedFiles:
fileID = files[lfn]
failReason = ''
res = oFTSRequest.getFailReason(lfn)
if res['OK']:
failReason = res['Value']
if self.missingSource(failReason):
gLogger.error('The source SURL does not exist.',
'%s %s' % (lfn, oFTSRequest.getSourceSURL(lfn)))
filesToReschedule.append(fileID)
else:
filesToRetry.append(fileID)
gLogger.error('Failed to replicate file on channel.',
"%s %s" % (channelID, failReason))
fileToFTSUpdates.append((fileID, 'Failed', failReason, 0, 0))
allUpdated = True
if filesToRetry:
gLogger.info('Updating the Channel table for files to retry')
res = self.TransferDB.resetFileChannelStatus(
channelID, filesToRetry)
if not res['OK']:
gLogger.error(
'Failed to update the Channel table for file to retry.',
res['Message'])
allUpdated = False
for fileID in filesToReschedule:
gLogger.info('Updating the Channel table for files to reschedule')
res = self.TransferDB.setFileChannelStatus(channelID, fileID,
'Failed')
if not res['OK']:
gLogger.error(
'Failed to update Channel table for failed files.',
res['Message'])
allUpdated = False
if completedFileIDs:
gLogger.info('Updating the Channel table for successful files')
res = self.TransferDB.updateCompletedChannelStatus(
channelID, completedFileIDs)
if not res['OK']:
gLogger.error(
'Failed to update the Channel table for successful files.',
res['Message'])
allUpdated = False
gLogger.info(
'Updating the Channel table for ancestors of successful files')
res = self.TransferDB.updateAncestorChannelStatus(
channelID, completedFileIDs)
if not res['OK']:
gLogger.error(
'Failed to update the Channel table for ancestors of successful files.',
res['Message'])
allUpdated = False
#
# cibak: temporary switch off of regitstration in FileToCat
#
#gLogger.info( 'Updating the FileToCat table for successful files' )
#res = self.TransferDB.setRegistrationWaiting( channelID, completedFileIDs )
#if not res['OK']:
# gLogger.error( 'Failed to update the FileToCat table for successful files.', res['Message'] )
# allUpdated = False
if fileToFTSUpdates:
gLogger.info('Updating the FileToFTS table for files')
res = self.TransferDB.setFileToFTSFileAttributes(
ftsReqID, channelID, fileToFTSUpdates)
if not res['OK']:
gLogger.error(
'Failed to update the FileToFTS table for files.',
res['Message'])
allUpdated = False
if allUpdated:
res = oFTSRequest.finalize()
if not res['OK']:
gLogger.error(
"Failed to perform the finalization for the FTS request",
res['Message'])
return res
gLogger.info('Adding logging event for FTS request')
# Now set the FTSReq status to terminal so that it is not monitored again
res = self.TransferDB.addLoggingEvent(ftsReqID, 'Finished')
if not res['OK']:
gLogger.error('Failed to add logging event for FTS Request',
res['Message'])
gLogger.info('Updating FTS request status')
res = self.TransferDB.setFTSReqStatus(ftsReqID, 'Finished')
if not res['OK']:
gLogger.error('Failed update FTS Request status',
res['Message'])
return S_OK()
from DIRAC.DataManagementSystem.Client.FTSRequest import FTSRequest
import os, sys
if not len(sys.argv) >= 6:
Script.showHelp()
DIRAC.exit(-1)
else:
inputFileName = sys.argv[1]
sourceSE = sys.argv[2]
targetSE = sys.argv[3]
guid = sys.argv[4]
server = sys.argv[5]
if not os.path.exists(inputFileName):
lfns = [inputFileName]
else:
inputFile = open(inputFileName, 'r')
string = inputFile.read()
inputFile.close()
lfns = string.splitlines()
oFTSRequest = FTSRequest()
for lfn in lfns:
oFTSRequest.setLFN(lfn)
oFTSRequest.setFTSGUID(guid)
oFTSRequest.setFTSServer(server)
oFTSRequest.setSourceSE(sourceSE)
oFTSRequest.setTargetSE(targetSE)
oFTSRequest.monitor(untilTerminal=True)
def submitTransfer(self, channelDict):
""" This method creates and submits FTS jobs based on information it gets from the DB
"""
# Create the FTSRequest object for preparing the submission
oFTSRequest = FTSRequest()
channelID = channelDict['ChannelID']
filesPerJob = channelDict['NumFiles']
#########################################################################
# Obtain the first files in the selected channel.
gLogger.info(
"FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s"
% channelID)
res = self.TransferDB.getFilesForChannel(channelID, 2 * filesPerJob)
if not res['OK']:
errStr = 'FTSSubmitAgent.%s' % res['Message']
gLogger.error(errStr)
return S_OK()
if not res['Value']:
gLogger.info(
"FTSSubmitAgent.submitTransfer: No files to found for channel."
)
return S_OK()
filesDict = res['Value']
gLogger.info('Obtained %s files for channel' % len(filesDict['Files']))
sourceSE = filesDict['SourceSE']
oFTSRequest.setSourceSE(sourceSE)
targetSE = filesDict['TargetSE']
oFTSRequest.setTargetSE(targetSE)
gLogger.info(
"FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel."
% (sourceSE, targetSE))
files = filesDict['Files']
#########################################################################
# Populate the FTS Request with the files.
gLogger.info('Populating the FTS request with file information')
fileIDs = []
totalSize = 0
fileIDSizes = {}
for file in files:
lfn = file['LFN']
oFTSRequest.setLFN(lfn)
oFTSRequest.setSourceSURL(lfn, file['SourceSURL'])
oFTSRequest.setTargetSURL(lfn, file['TargetSURL'])
fileID = file['FileID']
fileIDs.append(fileID)
totalSize += file['Size']
fileIDSizes[fileID] = file['Size']
#########################################################################
# Submit the FTS request and retrieve the FTS GUID/Server
gLogger.info('Submitting the FTS request')
res = oFTSRequest.submit()
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
gLogger.error(errStr)
gLogger.info('Updating the Channel table for files to retry')
res = self.TransferDB.resetFileChannelStatus(channelID, fileIDs)
if not res['OK']:
gLogger.error(
'Failed to update the Channel table for file to retry.',
res['Message'])
return S_ERROR(errStr)
ftsGUID = res['Value']['ftsGUID']
ftsServer = res['Value']['ftsServer']
infoStr = """Submitted FTS Job:
FTS Guid: %s
FTS Server: %s
ChannelID: %s
SourceSE: %s
TargetSE: %s
Files: %s
""" % (ftsGUID, ftsServer, str(channelID), sourceSE, targetSE, str(len(files)))
gLogger.info(infoStr)
#########################################################################
# Insert the FTS Req details and add the number of files and size
res = self.TransferDB.insertFTSReq(ftsGUID, ftsServer, channelID)
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
gLogger.error(errStr)
return S_ERROR(errStr)
ftsReqID = res['Value']
gLogger.info('Obtained FTS RequestID %s' % ftsReqID)
res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'SourceSE',
sourceSE)
if not res['OK']:
gLogger.error("Failed to set SourceSE for FTSRequest",
res['Message'])
res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'TargetSE',
targetSE)
if not res['OK']:
gLogger.error("Failed to set TargetSE for FTSRequest",
res['Message'])
res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'NumberOfFiles',
len(fileIDs))
if not res['OK']:
gLogger.error("Failed to set NumberOfFiles for FTSRequest",
res['Message'])
res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'TotalSize',
totalSize)
if not res['OK']:
gLogger.error("Failed to set TotalSize for FTSRequest",
res['Message'])
#########################################################################
# Insert the submission event in the FTSReqLogging table
event = 'Submitted'
res = self.TransferDB.addLoggingEvent(ftsReqID, event)
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
gLogger.error(errStr)
#########################################################################
# Insert the FileToFTS details and remove the files from the channel
gLogger.info('Setting the files as Executing in the Channel table')
res = self.TransferDB.setChannelFilesExecuting(channelID, fileIDs)
if not res['OK']:
gLogger.error('Failed to update the Channel tables for files.',
res['Message'])
lfns = []
fileToFTSFileAttributes = []
for file in files:
lfn = file['LFN']
fileID = file['FileID']
lfns.append(lfn)
fileToFTSFileAttributes.append((fileID, fileIDSizes[fileID]))
gLogger.info('Populating the FileToFTS table with file information')
res = self.TransferDB.setFTSReqFiles(ftsReqID, channelID,
fileToFTSFileAttributes)
if not res['OK']:
gLogger.error('Failed to populate the FileToFTS table with files.')
def monitorTransfer(self, ftsReqDict):
""" monitors transfer obtained from TransferDB
:param dict ftsReqDict: FTS job dictionary
"""
ftsReqID = ftsReqDict.get("FTSReqID")
ftsGUID = ftsReqDict.get("FTSGuid")
ftsServer = ftsReqDict.get("FTSServer")
channelID = ftsReqDict.get("ChannelID")
sourceSE = ftsReqDict.get("SourceSE")
targetSE = ftsReqDict.get("TargetSE")
oFTSRequest = FTSRequest()
oFTSRequest.setFTSServer(ftsServer)
oFTSRequest.setFTSGUID(ftsGUID)
oFTSRequest.setSourceSE(sourceSE)
oFTSRequest.setTargetSE(targetSE)
log = gLogger.getSubLogger("@%s" % str(ftsReqID))
#########################################################################
# Perform summary update of the FTS Request and update FTSReq entries.
log.info("Perform summary update of the FTS Request")
infoStr = ["glite-transfer-status -s %s -l %s" % (ftsServer, ftsGUID)]
infoStr.append("FTS GUID: %s" % ftsGUID)
infoStr.append("FTS Server: %s" % ftsServer)
log.info("\n".join(infoStr))
res = oFTSRequest.summary()
self.transferDB.setFTSReqLastMonitor(ftsReqID)
if not res["OK"]:
log.error("Failed to update the FTS request summary",
res["Message"])
if "getTransferJobSummary2: Not authorised to query request" in res[
"Message"]:
log.error(
"FTS job is not existing at the FTS server anymore, will clean it up on TransferDB side"
)
cleanUp = self.ftsJobExpired(ftsReqID, channelID)
if not cleanUp["OK"]:
log.error(cleanUp["Message"])
return cleanUp
return res
res = oFTSRequest.dumpSummary()
if not res['OK']:
log.error("Failed to get FTS request summary", res["Message"])
return res
log.info(res['Value'])
res = oFTSRequest.getPercentageComplete()
if not res['OK']:
log.error("Failed to get FTS percentage complete", res["Message"])
return res
log.info('FTS Request found to be %.1f percent complete' %
res["Value"])
self.transferDB.setFTSReqAttribute(ftsReqID, "PercentageComplete",
res["Value"])
self.transferDB.addLoggingEvent(ftsReqID, res["Value"])
#########################################################################
# Update the information in the TransferDB if the transfer is terminal.
res = oFTSRequest.isRequestTerminal()
if not res["OK"]:
log.error("Failed to determine whether FTS request terminal",
res["Message"])
return res
if not res["Value"]:
return S_OK()
# # request is terminal
return self.terminalRequest(oFTSRequest, ftsReqID, channelID, sourceSE)
def submitTransfer( self, channelDict ):
""" create and submit FTS jobs based on information it gets from the DB
:param self: self reference
:param dict channelDict: dict with channel info as read from TransferDB.selectChannelsForSubmission
"""
channelID = channelDict['ChannelID']
filesPerJob = channelDict['NumFiles']
#########################################################################
# Obtain the first files in the selected channel.
self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for channel %s: %s to %s"
% ( channelID, channelDict['Source'], channelDict['Destination'] ) )
res = self.transferDB.getFilesForChannel( channelID, 2 * filesPerJob )
if not res['OK']:
errStr = 'FTSSubmitAgent.%s' % res['Message']
self.log.error( errStr )
return S_OK()
if not res['Value']:
self.log.info( "FTSSubmitAgent.submitTransfer: No files found for channel." )
return S_OK()
filesDict = res['Value']
sourceSE = filesDict['SourceSE']
targetSE = filesDict['TargetSE']
self.log.info( 'Obtained %s files for channel %s to %s' % ( len( filesDict['Files'] ), sourceSE, targetSE ) )
if self.filesBeingStaged.get( channelID ):
self.log.info( '%d files are currently in staging status' % len( self.filesBeingStaged[channelID] ) )
# Create the FTSRequest object for preparing the submission
oFTSRequest = FTSRequest()
oFTSRequest.setSourceSE( sourceSE )
oFTSRequest.setTargetSE( targetSE )
files = filesDict['Files']
# # enable/disable cksm test
oFTSRequest.setCksmTest( self.cksmTest )
if self.cksmType:
oFTSRequest.setCksmType( self.cksmType )
#########################################################################
# Populate the FTS Request with the files.
self.log.info( 'Populating the FTS request with file information' )
fileIDs = []
totalSize = 0
fileIDSizes = {}
lfns = set()
for fileMeta in files:
lfn = fileMeta['LFN']
lfns.add( lfn )
oFTSRequest.setLFN( lfn )
oFTSRequest.setSourceSURL( lfn, fileMeta['SourceSURL'] )
oFTSRequest.setTargetSURL( lfn, fileMeta['TargetSURL'] )
if lfn in self.filesBeingStaged.get( channelID, [] ):
oFTSRequest.setStatus( lfn, 'Staging' )
fileID = fileMeta['FileID']
fileIDs.append( fileID )
totalSize += fileMeta['Size']
fileIDSizes[fileID] = fileMeta['Size']
oFTSRequest.resolveSource()
noSource = [ lfn for lfn, fileInfo in oFTSRequest.fileDict.items()
if fileInfo.get( "Status", "" ) == "Failed" and fileInfo.get( "Reason", "" ) in ( "No replica at SourceSE",
"Source file does not exist" ) ]
toReschedule = [fileMeta["FileID"] for fileMeta in files if fileMeta["LFN"] in noSource]
if toReschedule:
self.log.info( "Found %s files to reschedule" % len( toReschedule ) )
for fileID in toReschedule:
res = self.transferDB.setFileToReschedule( fileID )
if not res["OK"]:
self.log.error( "Failed to update Channel table for failed files.", res["Message"] )
elif res["Value"] == "max reschedule attempt reached":
self.log.error( "setting Channel status to 'Failed' : " % res["Value"] )
res = self.transferDB.setFileChannelStatus( channelID, fileID, 'Failed' )
if not res["OK"]:
self.log.error( "Failed to update Channel table for failed files.", res["Message"] )
#########################################################################
# Submit the FTS request and retrieve the FTS GUID/Server
self.log.info( 'Submitting the FTS request' )
res = oFTSRequest.submit()
if not res['OK']:
errStr = "FTSSubmitAgent.submit: %s" % res['Message']
self.log.error( errStr )
self.log.info( 'Updating the Channel table for files to retry' )
res = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
if not res['OK']:
self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
return S_ERROR( errStr )
ftsGUID = res['Value']['ftsGUID']
ftsServer = res['Value']['ftsServer']
nbSubmitted = res['Value']['submittedFiles']
infoStr = """Submitted FTS Job:
FTS Guid: %s
FTS Server: %s
ChannelID: %s
SourceSE: %s
TargetSE: %s
Files: %s
""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( nbSubmitted ) )
self.log.info( infoStr )
# # filter out skipped files
failedFiles = oFTSRequest.getFailed()['Value']
stagingFiles = oFTSRequest.getStaging()['Value']
# cache files being staged
self.filesBeingStaged.setdefault( channelID, set() ).update( stagingFiles )
submittedFiles = lfns.difference( failedFiles, stagingFiles )
# files being submitted are staged
self.filesBeingStaged[channelID] -= submittedFiles
failedIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in failedFiles ] )
stagingIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in stagingFiles ] )
# # only submitted
submittedIDs = set( fileIDs ) - failedIDs - stagingIDs
# # only count the submitted size
totalSize = sum( [ meta["Size"] for meta in files if meta["FileID"] in submittedIDs ] )
#########################################################################
# Insert the FTS Req details and add the number of files and size
res = self.transferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
self.log.error( errStr )
return S_ERROR( errStr )
ftsReqID = res['Value']
self.log.info( 'Obtained FTS RequestID %s' % ftsReqID )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
if not res['OK']:
self.log.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
if not res['OK']:
self.log.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( submittedIDs ) )
if not res['OK']:
self.log.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
if not res['OK']:
self.log.error( "Failed to set TotalSize for FTSRequest", res['Message'] )
#########################################################################
# Insert the submission event in the FTSReqLogging table
event = 'Submitted'
res = self.transferDB.addLoggingEvent( ftsReqID, event )
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
self.log.error( errStr )
#########################################################################
# Insert the FileToFTS details and remove the files from the channel
self.log.info( 'Setting the files as Executing in the Channel table' )
res = self.transferDB.setChannelFilesExecuting( channelID, list( submittedIDs ) )
if not res['OK']:
self.log.error( 'Failed to update the Channel tables for files.', res['Message'] )
lfns = []
fileToFTSFileAttributes = []
for fileMeta in files:
fileID = fileMeta['FileID']
# Staging is not an error case
if fileID not in stagingIDs:
lfns.append( fileMeta['LFN'] )
fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )
self.log.info( 'Populating the FileToFTS table with file information' )
res = self.transferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
if not res['OK']:
self.log.error( 'Failed to populate the FileToFTS table with files.' )