class FTSSubmitAgent( AgentModule ):
"""
.. class:: FTSSubmitAgent
This class is submitting previosly scheduled files to the FTS system using helper class FTSRequest.
Files to be transferred are read from TransferDB.Channel table, only those with Status = 'Waiting'.
After submision TransferDB.Channel.Status is set to 'Executing'. The rest of state propagation is
done in FTSMonitorAgent.
An information about newly created FTS request is hold in TransferDB.FTSReq (request itself) table and
TransferDB.FileToFTS (files in request) and TransferDB.FileToCat (files to be registered, for
failover only).
"""
## placaholder for TransferDB reference
transferDB = None
## placeholder for max job per channel
maxJobsPerChannel = 2
## placeholder for checksum test option flag
cksmTest = False
## placeholder for checksum type
cksmType = ""
## default checksum type
__defaultCksmType = "ADLER32"
def initialize( self ):
""" agent's initalisation
:param self: self reference
"""
## save tarsnferDB handler
self.transferDB = TransferDB()
## read config options
self.maxJobsPerChannel = self.am_getOption( 'MaxJobsPerChannel', 2 )
self.log.info("max jobs/channel = %s" % self.maxJobsPerChannel )
## checksum test
self.cksmTest = bool( self.am_getOption( "ChecksumTest", False ) )
## ckecksum type
if self.cksmTest:
self.cksmType = str( self.am_getOption( "ChecksumType", self.__defaultCksmType ) ).upper()
if self.cksmType and self.cksmType not in ( "ADLER32", "MD5", "SHA1" ):
self.log.warn("unknown checksum type: %s, will use default %s" % ( self.cksmType, self.__defaultCksmType ) )
self.cksmType = self.__defaultCksmType
self.log.info( "checksum test is %s" % ( { True : "enabled using %s checksum" % self.cksmType,
False : "disabled"}[self.cksmTest] ) )
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def execute( self ):
""" execution in one agent's cycle
:param self: self reference
"""
#########################################################################
# Obtain the eligible channels for submission.
self.log.info( 'Obtaining channels eligible for submission.' )
res = self.transferDB.selectChannelsForSubmission( self.maxJobsPerChannel )
if not res['OK']:
self.log.error( "Failed to retrieve channels for submission.", res['Message'] )
return S_OK()
elif not res['Value']:
self.log.info( "FTSSubmitAgent. No channels eligible for submission." )
return S_OK()
channelDicts = res['Value']
self.log.info( 'Found %s eligible channels.' % len( channelDicts ) )
#########################################################################
# Submit to all the eligible waiting channels.
i = 1
for channelDict in channelDicts:
infoStr = "\n\n##################################################################################\n\n"
infoStr = "%sStarting submission loop %s of %s\n\n" % ( infoStr, i, len( channelDicts ) )
self.log.info( infoStr )
res = self.submitTransfer( channelDict )
i += 1
return S_OK()
def submitTransfer( self, channelDict ):
""" create and submit FTS jobs based on information it gets from the DB
:param self: self reference
:param dict channelDict: dict with channel info as read from TransferDB.selectChannelsForSubmission
"""
# Create the FTSRequest object for preparing the submission
oFTSRequest = FTSRequest()
channelID = channelDict['ChannelID']
filesPerJob = channelDict['NumFiles']
#########################################################################
# Obtain the first files in the selected channel.
self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s" % channelID )
res = self.transferDB.getFilesForChannel( channelID, 2 * filesPerJob )
if not res['OK']:
errStr = 'FTSSubmitAgent.%s' % res['Message']
self.log.error( errStr )
return S_OK()
if not res['Value']:
self.log.info( "FTSSubmitAgent.submitTransfer: No files to found for channel." )
return S_OK()
filesDict = res['Value']
self.log.info( 'Obtained %s files for channel' % len( filesDict['Files'] ) )
sourceSE = filesDict['SourceSE']
oFTSRequest.setSourceSE( sourceSE )
targetSE = filesDict['TargetSE']
oFTSRequest.setTargetSE( targetSE )
self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel." % ( sourceSE,
targetSE ) )
files = filesDict['Files']
## enable/disable cksm test
oFTSRequest.setCksmTest( self.cksmTest )
if self.cksmType:
oFTSRequest.setCksmType( self.cksmType )
#########################################################################
# Populate the FTS Request with the files.
self.log.info( 'Populating the FTS request with file information' )
fileIDs = []
totalSize = 0
fileIDSizes = {}
for fileMeta in files:
lfn = fileMeta['LFN']
oFTSRequest.setLFN( lfn )
oFTSRequest.setSourceSURL( lfn, fileMeta['SourceSURL'] )
oFTSRequest.setTargetSURL( lfn, fileMeta['TargetSURL'] )
fileID = fileMeta['FileID']
fileIDs.append( fileID )
totalSize += fileMeta['Size']
fileIDSizes[fileID] = fileMeta['Size']
#########################################################################
# Submit the FTS request and retrieve the FTS GUID/Server
self.log.info( 'Submitting the FTS request' )
res = oFTSRequest.submit()
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
self.log.error( errStr )
self.log.info( 'Updating the Channel table for files to retry' )
res = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
if not res['OK']:
self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
return S_ERROR( errStr )
ftsGUID = res['Value']['ftsGUID']
ftsServer = res['Value']['ftsServer']
infoStr = """Submitted FTS Job:
FTS Guid: %s
FTS Server: %s
ChannelID: %s
SourceSE: %s
TargetSE: %s
Files: %s
""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( len( files ) ) )
self.log.info( infoStr )
## filter out skipped files
failedFiles = oFTSRequest.getFailed()
if not failedFiles["OK"]:
self.log.warn("Unable to read skipped LFNs.")
failedFiles = failedFiles["Value"] if "Value" in failedFiles else []
failedIDs = [ meta["FileID"] for meta in files if meta["LFN"] in failedFiles ]
## only submitted
fileIDs = [ fileID for fileID in fileIDs if fileID not in failedIDs ]
## sub failed from total size
totalSize -= sum( [ meta["Size"] for meta in files if meta["LFN"] in failedFiles ] )
#########################################################################
# Insert the FTS Req details and add the number of files and size
res = self.transferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
self.log.error( errStr )
return S_ERROR( errStr )
ftsReqID = res['Value']
self.log.info( 'Obtained FTS RequestID %s' % ftsReqID )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
if not res['OK']:
self.log.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
if not res['OK']:
self.log.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( fileIDs ) )
if not res['OK']:
self.log.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
if not res['OK']:
self.log.error( "Failed to set TotalSize for FTSRequest", res['Message'] )
#########################################################################
# Insert the submission event in the FTSReqLogging table
event = 'Submitted'
res = self.transferDB.addLoggingEvent( ftsReqID, event )
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
self.log.error( errStr )
#########################################################################
# Insert the FileToFTS details and remove the files from the channel
self.log.info( 'Setting the files as Executing in the Channel table' )
res = self.transferDB.setChannelFilesExecuting( channelID, fileIDs )
if not res['OK']:
self.log.error( 'Failed to update the Channel tables for files.', res['Message'] )
lfns = []
fileToFTSFileAttributes = []
for fileMeta in files:
lfn = fileMeta['LFN']
fileID = fileMeta['FileID']
lfns.append( lfn )
fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )
self.log.info( 'Populating the FileToFTS table with file information' )
res = self.transferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
if not res['OK']:
self.log.error( 'Failed to populate the FileToFTS table with files.' )
class FTSSubmitAgent( AgentModule ):
def initialize( self ):
self.TransferDB = TransferDB()
self.maxJobsPerChannel = self.am_getOption( 'MaxJobsPerChannel', 2 )
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def execute( self ):
#########################################################################
# Obtain the eligible channels for submission.
gLogger.info( 'Obtaining channels eligible for submission.' )
res = self.TransferDB.selectChannelsForSubmission( self.maxJobsPerChannel )
if not res['OK']:
gLogger.error( "Failed to retrieve channels for submission.", res['Message'] )
return S_OK()
elif not res['Value']:
gLogger.info( "FTSSubmitAgent. No channels eligible for submission." )
return S_OK()
channelDicts = res['Value']
gLogger.info( 'Found %s eligible channels.' % len( channelDicts ) )
#########################################################################
# Submit to all the eligible waiting channels.
i = 1
for channelDict in channelDicts:
infoStr = "\n\n##################################################################################\n\n"
infoStr = "%sStarting submission loop %s of %s\n\n" % ( infoStr, i, len( channelDicts ) )
gLogger.info( infoStr )
res = self.submitTransfer( channelDict )
i += 1
return S_OK()
def submitTransfer( self, channelDict ):
""" This method creates and submits FTS jobs based on information it gets from the DB
"""
# Create the FTSRequest object for preparing the submission
oFTSRequest = FTSRequest()
channelID = channelDict['ChannelID']
filesPerJob = channelDict['NumFiles']
#########################################################################
# Obtain the first files in the selected channel.
gLogger.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s" % channelID )
res = self.TransferDB.getFilesForChannel( channelID, 2 * filesPerJob )
if not res['OK']:
errStr = 'FTSSubmitAgent.%s' % res['Message']
gLogger.error( errStr )
return S_OK()
if not res['Value']:
gLogger.info( "FTSSubmitAgent.submitTransfer: No files to found for channel." )
return S_OK()
filesDict = res['Value']
gLogger.info( 'Obtained %s files for channel' % len( filesDict['Files'] ) )
sourceSE = filesDict['SourceSE']
oFTSRequest.setSourceSE( sourceSE )
targetSE = filesDict['TargetSE']
oFTSRequest.setTargetSE( targetSE )
gLogger.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel." % ( sourceSE, targetSE ) )
files = filesDict['Files']
#########################################################################
# Populate the FTS Request with the files.
gLogger.info( 'Populating the FTS request with file information' )
fileIDs = []
totalSize = 0
fileIDSizes = {}
for file in files:
lfn = file['LFN']
oFTSRequest.setLFN( lfn )
oFTSRequest.setSourceSURL( lfn, file['SourceSURL'] )
oFTSRequest.setTargetSURL( lfn, file['TargetSURL'] )
fileID = file['FileID']
fileIDs.append( fileID )
totalSize += file['Size']
fileIDSizes[fileID] = file['Size']
#########################################################################
# Submit the FTS request and retrieve the FTS GUID/Server
gLogger.info( 'Submitting the FTS request' )
res = oFTSRequest.submit()
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
gLogger.error( errStr )
gLogger.info( 'Updating the Channel table for files to retry' )
res = self.TransferDB.resetFileChannelStatus( channelID, fileIDs )
if not res['OK']:
gLogger.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
return S_ERROR( errStr )
ftsGUID = res['Value']['ftsGUID']
ftsServer = res['Value']['ftsServer']
infoStr = """Submitted FTS Job:
FTS Guid: %s
FTS Server: %s
ChannelID: %s
SourceSE: %s
TargetSE: %s
Files: %s
""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( len( files ) ) )
gLogger.info( infoStr )
#########################################################################
# Insert the FTS Req details and add the number of files and size
res = self.TransferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
gLogger.error( errStr )
return S_ERROR( errStr )
ftsReqID = res['Value']
gLogger.info( 'Obtained FTS RequestID %s' % ftsReqID )
res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
if not res['OK']:
gLogger.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
if not res['OK']:
gLogger.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( fileIDs ) )
if not res['OK']:
gLogger.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
res = self.TransferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
if not res['OK']:
gLogger.error( "Failed to set TotalSize for FTSRequest", res['Message'] )
#########################################################################
# Insert the submission event in the FTSReqLogging table
event = 'Submitted'
res = self.TransferDB.addLoggingEvent( ftsReqID, event )
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
gLogger.error( errStr )
#########################################################################
# Insert the FileToFTS details and remove the files from the channel
gLogger.info( 'Setting the files as Executing in the Channel table' )
res = self.TransferDB.setChannelFilesExecuting( channelID, fileIDs )
if not res['OK']:
gLogger.error( 'Failed to update the Channel tables for files.', res['Message'] )
lfns = []
fileToFTSFileAttributes = []
for file in files:
lfn = file['LFN']
fileID = file['FileID']
lfns.append( lfn )
fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )
gLogger.info( 'Populating the FileToFTS table with file information' )
res = self.TransferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
if not res['OK']:
gLogger.error( 'Failed to populate the FileToFTS table with files.' )
class FTSSubmitAgent(AgentModule):
def initialize(self):
self.TransferDB = TransferDB()
self.maxJobsPerChannel = self.am_getOption('MaxJobsPerChannel', 2)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def execute(self):
#########################################################################
# Obtain the eligible channels for submission.
gLogger.info('Obtaining channels eligible for submission.')
res = self.TransferDB.selectChannelsForSubmission(
self.maxJobsPerChannel)
if not res['OK']:
gLogger.error("Failed to retrieve channels for submission.",
res['Message'])
return S_OK()
elif not res['Value']:
gLogger.info(
"FTSSubmitAgent. No channels eligible for submission.")
return S_OK()
channelDicts = res['Value']
gLogger.info('Found %s eligible channels.' % len(channelDicts))
#########################################################################
# Submit to all the eligible waiting channels.
i = 1
for channelDict in channelDicts:
infoStr = "\n\n##################################################################################\n\n"
infoStr = "%sStarting submission loop %s of %s\n\n" % (
infoStr, i, len(channelDicts))
gLogger.info(infoStr)
res = self.submitTransfer(channelDict)
i += 1
return S_OK()
def submitTransfer(self, channelDict):
""" This method creates and submits FTS jobs based on information it gets from the DB
"""
# Create the FTSRequest object for preparing the submission
oFTSRequest = FTSRequest()
channelID = channelDict['ChannelID']
filesPerJob = channelDict['NumFiles']
#########################################################################
# Obtain the first files in the selected channel.
gLogger.info(
"FTSSubmitAgent.submitTransfer: Attempting to obtain files to transfer on channel %s"
% channelID)
res = self.TransferDB.getFilesForChannel(channelID, 2 * filesPerJob)
if not res['OK']:
errStr = 'FTSSubmitAgent.%s' % res['Message']
gLogger.error(errStr)
return S_OK()
if not res['Value']:
gLogger.info(
"FTSSubmitAgent.submitTransfer: No files to found for channel."
)
return S_OK()
filesDict = res['Value']
gLogger.info('Obtained %s files for channel' % len(filesDict['Files']))
sourceSE = filesDict['SourceSE']
oFTSRequest.setSourceSE(sourceSE)
targetSE = filesDict['TargetSE']
oFTSRequest.setTargetSE(targetSE)
gLogger.info(
"FTSSubmitAgent.submitTransfer: Attempting to obtain files for %s to %s channel."
% (sourceSE, targetSE))
files = filesDict['Files']
#########################################################################
# Populate the FTS Request with the files.
gLogger.info('Populating the FTS request with file information')
fileIDs = []
totalSize = 0
fileIDSizes = {}
for file in files:
lfn = file['LFN']
oFTSRequest.setLFN(lfn)
oFTSRequest.setSourceSURL(lfn, file['SourceSURL'])
oFTSRequest.setTargetSURL(lfn, file['TargetSURL'])
fileID = file['FileID']
fileIDs.append(fileID)
totalSize += file['Size']
fileIDSizes[fileID] = file['Size']
#########################################################################
# Submit the FTS request and retrieve the FTS GUID/Server
gLogger.info('Submitting the FTS request')
res = oFTSRequest.submit()
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
gLogger.error(errStr)
gLogger.info('Updating the Channel table for files to retry')
res = self.TransferDB.resetFileChannelStatus(channelID, fileIDs)
if not res['OK']:
gLogger.error(
'Failed to update the Channel table for file to retry.',
res['Message'])
return S_ERROR(errStr)
ftsGUID = res['Value']['ftsGUID']
ftsServer = res['Value']['ftsServer']
infoStr = """Submitted FTS Job:
FTS Guid: %s
FTS Server: %s
ChannelID: %s
SourceSE: %s
TargetSE: %s
Files: %s
""" % (ftsGUID, ftsServer, str(channelID), sourceSE, targetSE, str(len(files)))
gLogger.info(infoStr)
#########################################################################
# Insert the FTS Req details and add the number of files and size
res = self.TransferDB.insertFTSReq(ftsGUID, ftsServer, channelID)
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
gLogger.error(errStr)
return S_ERROR(errStr)
ftsReqID = res['Value']
gLogger.info('Obtained FTS RequestID %s' % ftsReqID)
res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'SourceSE',
sourceSE)
if not res['OK']:
gLogger.error("Failed to set SourceSE for FTSRequest",
res['Message'])
res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'TargetSE',
targetSE)
if not res['OK']:
gLogger.error("Failed to set TargetSE for FTSRequest",
res['Message'])
res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'NumberOfFiles',
len(fileIDs))
if not res['OK']:
gLogger.error("Failed to set NumberOfFiles for FTSRequest",
res['Message'])
res = self.TransferDB.setFTSReqAttribute(ftsReqID, 'TotalSize',
totalSize)
if not res['OK']:
gLogger.error("Failed to set TotalSize for FTSRequest",
res['Message'])
#########################################################################
# Insert the submission event in the FTSReqLogging table
event = 'Submitted'
res = self.TransferDB.addLoggingEvent(ftsReqID, event)
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
gLogger.error(errStr)
#########################################################################
# Insert the FileToFTS details and remove the files from the channel
gLogger.info('Setting the files as Executing in the Channel table')
res = self.TransferDB.setChannelFilesExecuting(channelID, fileIDs)
if not res['OK']:
gLogger.error('Failed to update the Channel tables for files.',
res['Message'])
lfns = []
fileToFTSFileAttributes = []
for file in files:
lfn = file['LFN']
fileID = file['FileID']
lfns.append(lfn)
fileToFTSFileAttributes.append((fileID, fileIDSizes[fileID]))
gLogger.info('Populating the FileToFTS table with file information')
res = self.TransferDB.setFTSReqFiles(ftsReqID, channelID,
fileToFTSFileAttributes)
if not res['OK']:
gLogger.error('Failed to populate the FileToFTS table with files.')
class FTSSubmitAgent( AgentModule ):
"""
.. class:: FTSSubmitAgent
This class is submitting previously scheduled files to the FTS system using helper class FTSRequest.
Files to be transferred are read from TransferDB.Channel table, only those with Status = 'Waiting'.
After submission TransferDB.Channel.Status is set to 'Executing'. The rest of state propagation is
done in FTSMonitorAgent.
An information about newly created FTS request is hold in TransferDB.FTSReq (request itself) table and
TransferDB.FileToFTS (files in request) and TransferDB.FileToCat (files to be registered, for
failover only).
"""
# # placaholder for TransferDB reference
transferDB = None
# # placeholder for max job per channel
maxJobsPerChannel = 2
# # placeholder for checksum test option flag
cksmTest = False
# # placeholder for checksum type
cksmType = ""
# # default checksum type
__defaultCksmType = "ADLER32"
def initialize( self ):
""" agent's initalisation
:param self: self reference
"""
# # save tarsnferDB handler
self.transferDB = TransferDB()
# # read config options
self.maxJobsPerChannel = self.am_getOption( 'MaxJobsPerChannel', self.maxJobsPerChannel )
self.log.info( "max jobs/channel = %s" % self.maxJobsPerChannel )
# # checksum test
self.cksmTest = bool( self.am_getOption( "ChecksumTest", False ) )
# # ckecksum type
if self.cksmTest:
self.cksmType = str( self.am_getOption( "ChecksumType", self.__defaultCksmType ) ).upper()
if self.cksmType and self.cksmType not in ( "ADLER32", "MD5", "SHA1" ):
self.log.warn( "unknown checksum type: %s, will use default %s" % ( self.cksmType, self.__defaultCksmType ) )
self.cksmType = self.__defaultCksmType
self.log.info( "checksum test is %s" % ( { True : "enabled using %s checksum" % self.cksmType,
False : "disabled"}[self.cksmTest] ) )
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
self.filesBeingStaged = {}
return S_OK()
def execute( self ):
""" execution in one agent's cycle
:param self: self reference
"""
#########################################################################
# Obtain the eligible channels for submission.
self.log.info( 'Obtaining channels eligible for submission.' )
res = self.transferDB.selectChannelsForSubmission( self.maxJobsPerChannel )
if not res['OK']:
self.log.error( "Failed to retrieve channels for submission.", res['Message'] )
return S_OK()
elif not res['Value']:
self.log.info( "FTSSubmitAgent. No channels eligible for submission." )
return S_OK()
channelDicts = res['Value']
self.log.info( 'Found %s eligible channels.' % len( channelDicts ) )
#########################################################################
# Submit to all the eligible waiting channels.
i = 1
for channelDict in channelDicts:
infoStr = "\n\n##################################################################################\n\n"
infoStr = "%sStarting submission loop %s of %s\n\n" % ( infoStr, i, len( channelDicts ) )
self.log.info( infoStr )
res = self.submitTransfer( channelDict )
i += 1
return S_OK()
def submitTransfer( self, channelDict ):
""" create and submit FTS jobs based on information it gets from the DB
:param self: self reference
:param dict channelDict: dict with channel info as read from TransferDB.selectChannelsForSubmission
"""
channelID = channelDict['ChannelID']
filesPerJob = channelDict['NumFiles']
#########################################################################
# Obtain the first files in the selected channel.
self.log.info( "FTSSubmitAgent.submitTransfer: Attempting to obtain files for channel %s: %s to %s"
% ( channelID, channelDict['Source'], channelDict['Destination'] ) )
res = self.transferDB.getFilesForChannel( channelID, 2 * filesPerJob )
if not res['OK']:
errStr = 'FTSSubmitAgent.%s' % res['Message']
self.log.error( errStr )
return S_OK()
if not res['Value']:
self.log.info( "FTSSubmitAgent.submitTransfer: No files found for channel." )
return S_OK()
filesDict = res['Value']
sourceSE = filesDict['SourceSE']
targetSE = filesDict['TargetSE']
self.log.info( 'Obtained %s files for channel %s to %s' % ( len( filesDict['Files'] ), sourceSE, targetSE ) )
if self.filesBeingStaged.get( channelID ):
self.log.info( '%d files are currently in staging status' % len( self.filesBeingStaged[channelID] ) )
# Create the FTSRequest object for preparing the submission
oFTSRequest = FTSRequest()
oFTSRequest.setSourceSE( sourceSE )
oFTSRequest.setTargetSE( targetSE )
files = filesDict['Files']
# # enable/disable cksm test
oFTSRequest.setCksmTest( self.cksmTest )
if self.cksmType:
oFTSRequest.setCksmType( self.cksmType )
#########################################################################
# Populate the FTS Request with the files.
self.log.info( 'Populating the FTS request with file information' )
fileIDs = []
totalSize = 0
fileIDSizes = {}
lfns = set()
for fileMeta in files:
lfn = fileMeta['LFN']
lfns.add( lfn )
oFTSRequest.setLFN( lfn )
oFTSRequest.setSourceSURL( lfn, fileMeta['SourceSURL'] )
oFTSRequest.setTargetSURL( lfn, fileMeta['TargetSURL'] )
if lfn in self.filesBeingStaged.get( channelID, [] ):
oFTSRequest.setStatus( lfn, 'Staging' )
fileID = fileMeta['FileID']
fileIDs.append( fileID )
totalSize += fileMeta['Size']
fileIDSizes[fileID] = fileMeta['Size']
oFTSRequest.resolveSource()
noSource = [ lfn for lfn, fileInfo in oFTSRequest.fileDict.items()
if fileInfo.get( "Status", "" ) == "Failed" and fileInfo.get( "Reason", "" ) in ( "No replica at SourceSE",
"Source file does not exist" ) ]
toReschedule = [fileMeta["FileID"] for fileMeta in files if fileMeta["LFN"] in noSource]
if toReschedule:
self.log.info( "Found %s files to reschedule" % len( toReschedule ) )
for fileID in toReschedule:
res = self.transferDB.setFileToReschedule( fileID )
if not res["OK"]:
self.log.error( "Failed to update Channel table for failed files.", res["Message"] )
elif res["Value"] == "max reschedule attempt reached":
self.log.error( "setting Channel status to 'Failed' : " % res["Value"] )
res = self.transferDB.setFileChannelStatus( channelID, fileID, 'Failed' )
if not res["OK"]:
self.log.error( "Failed to update Channel table for failed files.", res["Message"] )
#########################################################################
# Submit the FTS request and retrieve the FTS GUID/Server
self.log.info( 'Submitting the FTS request' )
res = oFTSRequest.submit()
if not res['OK']:
errStr = "FTSSubmitAgent.submit: %s" % res['Message']
self.log.error( errStr )
self.log.info( 'Updating the Channel table for files to retry' )
res = self.transferDB.resetFileChannelStatus( channelID, fileIDs )
if not res['OK']:
self.log.error( 'Failed to update the Channel table for file to retry.', res['Message'] )
return S_ERROR( errStr )
ftsGUID = res['Value']['ftsGUID']
ftsServer = res['Value']['ftsServer']
nbSubmitted = res['Value']['submittedFiles']
infoStr = """Submitted FTS Job:
FTS Guid: %s
FTS Server: %s
ChannelID: %s
SourceSE: %s
TargetSE: %s
Files: %s
""" % ( ftsGUID, ftsServer, str( channelID ), sourceSE, targetSE, str( nbSubmitted ) )
self.log.info( infoStr )
# # filter out skipped files
failedFiles = oFTSRequest.getFailed()['Value']
stagingFiles = oFTSRequest.getStaging()['Value']
# cache files being staged
self.filesBeingStaged.setdefault( channelID, set() ).update( stagingFiles )
submittedFiles = lfns.difference( failedFiles, stagingFiles )
# files being submitted are staged
self.filesBeingStaged[channelID] -= submittedFiles
failedIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in failedFiles ] )
stagingIDs = set( [ meta["FileID"] for meta in files if meta["LFN"] in stagingFiles ] )
# # only submitted
submittedIDs = set( fileIDs ) - failedIDs - stagingIDs
# # only count the submitted size
totalSize = sum( [ meta["Size"] for meta in files if meta["FileID"] in submittedIDs ] )
#########################################################################
# Insert the FTS Req details and add the number of files and size
res = self.transferDB.insertFTSReq( ftsGUID, ftsServer, channelID )
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
self.log.error( errStr )
return S_ERROR( errStr )
ftsReqID = res['Value']
self.log.info( 'Obtained FTS RequestID %s' % ftsReqID )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'SourceSE', sourceSE )
if not res['OK']:
self.log.error( "Failed to set SourceSE for FTSRequest", res['Message'] )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TargetSE', targetSE )
if not res['OK']:
self.log.error( "Failed to set TargetSE for FTSRequest", res['Message'] )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'NumberOfFiles', len( submittedIDs ) )
if not res['OK']:
self.log.error( "Failed to set NumberOfFiles for FTSRequest", res['Message'] )
res = self.transferDB.setFTSReqAttribute( ftsReqID, 'TotalSize', totalSize )
if not res['OK']:
self.log.error( "Failed to set TotalSize for FTSRequest", res['Message'] )
#########################################################################
# Insert the submission event in the FTSReqLogging table
event = 'Submitted'
res = self.transferDB.addLoggingEvent( ftsReqID, event )
if not res['OK']:
errStr = "FTSSubmitAgent.%s" % res['Message']
self.log.error( errStr )
#########################################################################
# Insert the FileToFTS details and remove the files from the channel
self.log.info( 'Setting the files as Executing in the Channel table' )
res = self.transferDB.setChannelFilesExecuting( channelID, list( submittedIDs ) )
if not res['OK']:
self.log.error( 'Failed to update the Channel tables for files.', res['Message'] )
lfns = []
fileToFTSFileAttributes = []
for fileMeta in files:
fileID = fileMeta['FileID']
# Staging is not an error case
if fileID not in stagingIDs:
lfns.append( fileMeta['LFN'] )
fileToFTSFileAttributes.append( ( fileID, fileIDSizes[fileID] ) )
self.log.info( 'Populating the FileToFTS table with file information' )
res = self.transferDB.setFTSReqFiles( ftsReqID, channelID, fileToFTSFileAttributes )
if not res['OK']:
self.log.error( 'Failed to populate the FileToFTS table with files.' )
