def initialize(self):
self.section = PathFinder.getAgentSection(AGENT_NAME)
self.RequestDB = RequestDBMySQL()
self.TransferDB = TransferDB()
self.DataLog = DataLoggingClient()
self.factory = StorageFactory()
self.rm = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def initialize(self):
self.RequestDBClient = RequestClient()
backend = self.am_getOption('Backend', '')
self.RequestDB = False
if backend == 'mysql':
from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
requestDB = RequestDBMySQL()
if requestDB._connected:
self.RequestDB = requestDB
gMonitor.registerActivity("Iteration", "Agent Loops",
"DISETForwardingAgent", "Loops/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Attempted", "Request Processed",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Successful", "Request Forward Successful",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Failed", "Request Forward Failed",
"DISETForwardingAgent", "Requests/min",
gMonitor.OP_SUM)
self.local = PathFinder.getServiceURL("RequestManagement/localURL")
if not self.local:
self.local = AgentModule.am_getOption(self, 'localURL', '')
if not self.local:
errStr = 'The RequestManagement/localURL option must be defined.'
gLogger.fatal(errStr)
return S_ERROR(errStr)
return S_OK()
def requestDBMySQL(cls):
""" RequestDBMySQL getter
:param cls: class reference
"""
if not cls.__requestDBMySQL:
cls.__requestDBMySQL = RequestDBMySQL()
return cls.__requestDBMySQL
def initialize( self ):
self.section = PathFinder.getAgentSection( AGENT_NAME )
self.RequestDB = RequestDBMySQL()
self.TransferDB = TransferDB()
self.DataLog = DataLoggingClient()
self.factory = StorageFactory()
self.rm = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def initializeRequestManagerHandler(serviceInfo):
global requestDB
csSection = PathFinder.getServiceSection( "RequestManagement/RequestManager" )
backend = gConfig.getValue('%s/Backend' % csSection)
if not backend:
fatStr = "RequestManager.initializeRequestManagerHandler: Failed to get backed for RequestDB from CS."
gLogger.fatal(fatStr)
return S_ERROR(fatStr)
gLogger.info("RequestManager.initializeRequestManagerHandler: Initialising with backend",backend)
if backend == 'file':
from DIRAC.RequestManagementSystem.DB.RequestDBFile import RequestDBFile
requestDB = RequestDBFile()
elif backend == 'mysql':
from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
requestDB = RequestDBMySQL()
else:
fatStr = "RequestManager.initializeRequestManagerHandler: Supplied backend is not supported."
gLogger.fatal(fatStr,backend)
return S_ERROR(fatStr)
return S_OK()
class ReplicationScheduler(AgentModule):
def initialize(self):
self.section = PathFinder.getAgentSection(AGENT_NAME)
self.RequestDB = RequestDBMySQL()
self.TransferDB = TransferDB()
self.DataLog = DataLoggingClient()
self.factory = StorageFactory()
self.rm = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
return S_OK()
def execute(self):
""" The main agent execution method """
# This allows dynamic changing of the throughput timescale
self.throughputTimescale = self.am_getOption('ThroughputTimescale',
3600)
self.throughputTimescale = 60 * 60 * 1
#print 'ThroughputTimescale:',self.throughputTimescale
######################################################################################
#
# Obtain information on the current state of the channel queues
#
res = self.TransferDB.getChannelQueues()
if not res['OK']:
errStr = "ReplicationScheduler._execute: Failed to get channel queues from TransferDB."
gLogger.error(errStr, res['Message'])
return S_OK()
if not res['Value']:
gLogger.info(
"ReplicationScheduler._execute: No active channels found for replication."
)
return S_OK()
channels = res['Value']
res = self.TransferDB.getChannelObservedThroughput(
self.throughputTimescale)
if not res['OK']:
errStr = "ReplicationScheduler._execute: Failed to get observed throughput from TransferDB."
gLogger.error(errStr, res['Message'])
return S_OK()
if not res['Value']:
gLogger.info(
"ReplicationScheduler._execute: No active channels found for replication."
)
return S_OK()
bandwidths = res['Value']
self.strategyHandler = StrategyHandler(bandwidths, channels,
self.section)
processedRequests = []
requestsPresent = True
while requestsPresent:
######################################################################################
#
# The first step is to obtain a transfer request from the RequestDB which should be scheduled.
#
gLogger.info(
"ReplicationScheduler._execute: Contacting RequestDB for suitable requests."
)
res = self.RequestDB.getRequest('transfer')
if not res['OK']:
gLogger.error(
"ReplicationScheduler._execute: Failed to get a request list from RequestDB.",
res['Message'])
continue
if not res['Value']:
gLogger.info(
"ReplicationScheduler._execute: No requests found in RequestDB."
)
requestsPresent = False
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
gLogger.info(
"ReplicationScheduler._execute: Obtained Request %s from RequestDB."
% (requestName))
######################################################################################
#
# The request must then be parsed to obtain the sub-requests, their attributes and files.
#
logStr = 'ReplicationScheduler._execute: Parsing Request %s.' % (
requestName)
gLogger.info(logStr)
oRequest = RequestContainer(requestString)
res = oRequest.getAttribute('RequestID')
if not res['OK']:
gLogger.error(
'ReplicationScheduler._execute: Failed to get requestID.',
res['Message'])
return S_ERROR(
'ReplicationScheduler._execute: Failed to get number of sub-requests.'
)
requestID = res['Value']
if requestID in processedRequests:
# Break the loop once we have iterated once over all requests
res = self.RequestDB.updateRequest(requestName, requestString)
if not res['OK']:
gLogger.error("Failed to update request",
"%s %s" % (requestName, res['Message']))
return S_OK()
processedRequests.append(requestID)
res = oRequest.getNumSubRequests('transfer')
if not res['OK']:
gLogger.error(
'ReplicationScheduler._execute: Failed to get number of sub-requests.',
res['Message'])
return S_ERROR(
'ReplicationScheduler._execute: Failed to get number of sub-requests.'
)
numberRequests = res['Value']
gLogger.info(
"ReplicationScheduler._execute: '%s' found with %s sub-requests."
% (requestName, numberRequests))
######################################################################################
#
# The important request attributes are the source and target SEs.
#
for ind in range(numberRequests):
gLogger.info(
"ReplicationScheduler._execute: Treating sub-request %s from '%s'."
% (ind, requestName))
attributes = oRequest.getSubRequestAttributes(
ind, 'transfer')['Value']
if attributes['Status'] != 'Waiting':
# If the sub-request is already in terminal state
gLogger.info(
"ReplicationScheduler._execute: Sub-request %s is status '%s' and not to be executed."
% (ind, attributes['Status']))
continue
sourceSE = attributes['SourceSE']
targetSE = attributes['TargetSE']
""" This section should go in the transfer request class """
if type(targetSE) in types.StringTypes:
if re.search(',', targetSE):
targetSEs = targetSE.split(',')
else:
targetSEs = [targetSE]
"""----------------------------------------------------- """
operation = attributes['Operation']
reqRepStrategy = None
if operation in self.strategyHandler.getSupportedStrategies():
reqRepStrategy = operation
######################################################################################
#
# Then obtain the file attribute of interest are the LFN and FileID
#
res = oRequest.getSubRequestFiles(ind, 'transfer')
if not res['OK']:
gLogger.error(
'ReplicationScheduler._execute: Failed to obtain sub-request files.',
res['Message'])
continue
files = res['Value']
gLogger.info(
"ReplicationScheduler._execute: Sub-request %s found with %s files."
% (ind, len(files)))
filesDict = {}
for file in files:
lfn = file['LFN']
if file['Status'] != 'Waiting':
gLogger.debug(
"ReplicationScheduler._execute: %s will not be scheduled because it is %s."
% (lfn, file['Status']))
else:
fileID = file['FileID']
filesDict[lfn] = fileID
if not filesDict:
gLogger.info(
"ReplicationScheduler._execute: No Waiting files found for request"
)
continue
notSched = len(files) - len(filesDict)
if notSched:
gLogger.info(
"ReplicationScheduler._execute: %d files found not Waiting"
% notSched)
######################################################################################
#
# Now obtain replica information for the files associated to the sub-request.
#
lfns = filesDict.keys()
gLogger.info(
"ReplicationScheduler._execute: Obtaining replica information for %d sub-request files."
% len(lfns))
res = self.rm.getCatalogReplicas(lfns)
if not res['OK']:
gLogger.error(
"ReplicationScheduler._execute: Failed to get replica information.",
res['Message'])
continue
for lfn, failure in res['Value']['Failed'].items():
gLogger.error(
"ReplicationScheduler._execute: Failed to get replicas.",
'%s: %s' % (lfn, failure))
replicas = res['Value']['Successful']
if not replicas.keys():
gLogger.error(
"ReplicationScheduler._execute: Failed to get replica information for all files."
)
continue
######################################################################################
#
# Now obtain the file sizes for the files associated to the sub-request.
#
lfns = replicas.keys()
gLogger.info(
"ReplicationScheduler._execute: Obtaining file sizes for %d sub-request files."
% len(lfns))
res = self.rm.getCatalogFileMetadata(lfns)
if not res['OK']:
gLogger.error(
"ReplicationScheduler._execute: Failed to get file size information.",
res['Message'])
continue
for lfn, failure in res['Value']['Failed'].items():
gLogger.error(
'ReplicationScheduler._execute: Failed to get file size.',
'%s: %s' % (lfn, failure))
metadata = res['Value']['Successful']
if not metadata.keys():
gLogger.error(
"ReplicationScheduler._execute: Failed to get metadata for all files."
)
continue
######################################################################################
#
# For each LFN determine the replication tree
#
for lfn in sortList(metadata.keys()):
fileSize = metadata[lfn]['Size']
lfnReps = replicas[lfn]
fileID = filesDict[lfn]
targets = []
for targetSE in targetSEs:
if targetSE in lfnReps.keys():
gLogger.debug(
"ReplicationScheduler.execute: %s already present at %s."
% (lfn, targetSE))
else:
targets.append(targetSE)
if not targets:
gLogger.info(
"ReplicationScheduler.execute: %s present at all targets."
% lfn)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
continue
if not lfnReps:
gLogger.error(
"ReplicationScheduler.execute: The file has no replicas.",
lfn)
continue
res = self.strategyHandler.determineReplicationTree(
sourceSE,
targets,
lfnReps,
fileSize,
strategy=reqRepStrategy)
if not res['OK']:
gLogger.error(
"ReplicationScheduler.execute: Failed to determine replication tree.",
res['Message'])
continue
tree = res['Value']
######################################################################################
#
# For each item in the replication tree obtain the source and target SURLS
#
for channelID, dict in tree.items():
gLogger.info(
"ReplicationScheduler.execute: processing for channel %d %s"
% (channelID, str(dict)))
hopSourceSE = dict['SourceSE']
hopDestSE = dict['DestSE']
hopAncestor = dict['Ancestor']
# Get the sourceSURL
if hopAncestor:
status = 'Waiting%s' % (hopAncestor)
res = self.obtainLFNSURL(hopSourceSE, lfn)
if not res['OK']:
errStr = res['Message']
gLogger.error(errStr)
return S_ERROR(errStr)
sourceSURL = res['Value']
else:
status = 'Waiting'
res = self.resolvePFNSURL(hopSourceSE,
lfnReps[hopSourceSE])
if not res['OK']:
sourceSURL = lfnReps[hopSourceSE]
else:
sourceSURL = res['Value']
# Get the targetSURL
res = self.obtainLFNSURL(hopDestSE, lfn)
if not res['OK']:
errStr = res['Message']
gLogger.error(errStr)
return S_ERROR(errStr)
targetSURL = res['Value']
######################################################################################
#
# For each item in the replication tree add the file to the channel
#
res = self.TransferDB.addFileToChannel(
channelID,
fileID,
hopSourceSE,
sourceSURL,
hopDestSE,
targetSURL,
fileSize,
fileStatus=status)
if not res['OK']:
errStr = res['Message']
gLogger.error(
"ReplicationScheduler._execute: Failed to add File to Channel.",
"%s %s" % (fileID, channelID))
return S_ERROR(errStr)
res = self.TransferDB.addFileRegistration(
channelID, fileID, lfn, targetSURL, hopDestSE)
if not res['OK']:
errStr = res['Message']
gLogger.error(
"ReplicationScheduler._execute: Failed to add File registration.",
"%s %s" % (fileID, channelID))
result = self.TransferDB.removeFileFromChannel(
channelID, fileID)
if not result['OK']:
errStr += result['Message']
gLogger.error(
"ReplicationScheduler._execute: Failed to remove File.",
"%s %s" % (fileID, channelID))
return S_ERROR(errStr)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Scheduled')
res = self.TransferDB.addReplicationTree(fileID, tree)
if oRequest.isSubRequestEmpty(ind, 'transfer')['Value']:
oRequest.setSubRequestStatus(ind, 'transfer', 'Scheduled')
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDB.updateRequest(requestName, requestString)
if not res['OK']:
gLogger.error(
"ReplicationScheduler._execute: Failed to update request",
"%s %s" % (requestName, res['Message']))
def obtainLFNSURL(self, targetSE, lfn):
""" Creates the targetSURL for the storage and LFN supplied
"""
res = self.factory.getStorages(targetSE, protocolList=['SRM2'])
if not res['OK']:
errStr = 'ReplicationScheduler._execute: Failed to create SRM2 storage for %s: %s. ' % (
targetSE, res['Message'])
gLogger.error(errStr)
return S_ERROR(errStr)
storageObjects = res['Value']['StorageObjects']
for storageObject in storageObjects:
res = storageObject.getCurrentURL(lfn)
if res['OK']:
return res
gLogger.error(
'ReplicationScheduler._execute: Failed to get SRM compliant storage.',
targetSE)
return S_ERROR(
'ReplicationScheduler._execute: Failed to get SRM compliant storage.'
)
def resolvePFNSURL(self, sourceSE, pfn):
""" Creates the targetSURL for the storage and LFN supplied
"""
res = self.rm.getPfnForProtocol([pfn], sourceSE)
if not res['OK']:
return res
if pfn in res['Value']['Failed'].keys():
return S_ERROR(res['Value']['Failed'][pfn])
return S_OK(res['Value']['Successful'][pfn])
class ReplicationScheduler( AgentModule ):
def initialize( self ):
self.section = PathFinder.getAgentSection( AGENT_NAME )
self.RequestDB = RequestDBMySQL()
self.TransferDB = TransferDB()
self.DataLog = DataLoggingClient()
self.factory = StorageFactory()
self.rm = ReplicaManager()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def execute( self ):
""" The main agent execution method """
# This allows dynamic changing of the throughput timescale
self.throughputTimescale = self.am_getOption( 'ThroughputTimescale', 3600 )
self.throughputTimescale = 60 * 60 * 1
#print 'ThroughputTimescale:',self.throughputTimescale
######################################################################################
#
# Obtain information on the current state of the channel queues
#
res = self.TransferDB.getChannelQueues()
if not res['OK']:
errStr = "ReplicationScheduler._execute: Failed to get channel queues from TransferDB."
gLogger.error( errStr, res['Message'] )
return S_OK()
if not res['Value']:
gLogger.info( "ReplicationScheduler._execute: No active channels found for replication." )
return S_OK()
channels = res['Value']
res = self.TransferDB.getChannelObservedThroughput( self.throughputTimescale )
if not res['OK']:
errStr = "ReplicationScheduler._execute: Failed to get observed throughput from TransferDB."
gLogger.error( errStr, res['Message'] )
return S_OK()
if not res['Value']:
gLogger.info( "ReplicationScheduler._execute: No active channels found for replication." )
return S_OK()
bandwidths = res['Value']
self.strategyHandler = StrategyHandler( bandwidths, channels, self.section )
processedRequests = []
requestsPresent = True
while requestsPresent:
######################################################################################
#
# The first step is to obtain a transfer request from the RequestDB which should be scheduled.
#
gLogger.info( "ReplicationScheduler._execute: Contacting RequestDB for suitable requests." )
res = self.RequestDB.getRequest( 'transfer' )
if not res['OK']:
gLogger.error( "ReplicationScheduler._execute: Failed to get a request list from RequestDB.", res['Message'] )
continue
if not res['Value']:
gLogger.info( "ReplicationScheduler._execute: No requests found in RequestDB." )
requestsPresent = False
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
gLogger.info( "ReplicationScheduler._execute: Obtained Request %s from RequestDB." % ( requestName ) )
######################################################################################
#
# The request must then be parsed to obtain the sub-requests, their attributes and files.
#
logStr = 'ReplicationScheduler._execute: Parsing Request %s.' % ( requestName )
gLogger.info( logStr )
oRequest = RequestContainer( requestString )
res = oRequest.getAttribute( 'RequestID' )
if not res['OK']:
gLogger.error( 'ReplicationScheduler._execute: Failed to get requestID.', res['Message'] )
return S_ERROR( 'ReplicationScheduler._execute: Failed to get number of sub-requests.' )
requestID = res['Value']
if requestID in processedRequests:
# Break the loop once we have iterated once over all requests
res = self.RequestDB.updateRequest( requestName, requestString )
if not res['OK']:
gLogger.error( "Failed to update request", "%s %s" % ( requestName, res['Message'] ) )
return S_OK()
processedRequests.append( requestID )
res = oRequest.getNumSubRequests( 'transfer' )
if not res['OK']:
gLogger.error( 'ReplicationScheduler._execute: Failed to get number of sub-requests.', res['Message'] )
return S_ERROR( 'ReplicationScheduler._execute: Failed to get number of sub-requests.' )
numberRequests = res['Value']
gLogger.info( "ReplicationScheduler._execute: '%s' found with %s sub-requests." % ( requestName, numberRequests ) )
######################################################################################
#
# The important request attributes are the source and target SEs.
#
for ind in range( numberRequests ):
gLogger.info( "ReplicationScheduler._execute: Treating sub-request %s from '%s'." % ( ind, requestName ) )
attributes = oRequest.getSubRequestAttributes( ind, 'transfer' )['Value']
if attributes['Status'] != 'Waiting':
# If the sub-request is already in terminal state
gLogger.info( "ReplicationScheduler._execute: Sub-request %s is status '%s' and not to be executed." % ( ind, attributes['Status'] ) )
continue
sourceSE = attributes['SourceSE']
targetSE = attributes['TargetSE']
""" This section should go in the transfer request class """
if type( targetSE ) in types.StringTypes:
if re.search( ',', targetSE ):
targetSEs = targetSE.split( ',' )
else:
targetSEs = [targetSE]
"""----------------------------------------------------- """
operation = attributes['Operation']
reqRepStrategy = None
if operation in self.strategyHandler.getSupportedStrategies():
reqRepStrategy = operation
######################################################################################
#
# Then obtain the file attribute of interest are the LFN and FileID
#
res = oRequest.getSubRequestFiles( ind, 'transfer' )
if not res['OK']:
gLogger.error( 'ReplicationScheduler._execute: Failed to obtain sub-request files.' , res['Message'] )
continue
files = res['Value']
gLogger.info( "ReplicationScheduler._execute: Sub-request %s found with %s files." % ( ind, len( files ) ) )
filesDict = {}
for file in files:
lfn = file['LFN']
if file['Status'] != 'Waiting':
gLogger.debug( "ReplicationScheduler._execute: %s will not be scheduled because it is %s." % ( lfn, file['Status'] ) )
else:
fileID = file['FileID']
filesDict[lfn] = fileID
if not filesDict:
gLogger.info( "ReplicationScheduler._execute: No Waiting files found for request" )
continue
notSched = len( files ) - len( filesDict )
if notSched:
gLogger.info( "ReplicationScheduler._execute: %d files found not Waiting" % notSched )
######################################################################################
#
# Now obtain replica information for the files associated to the sub-request.
#
lfns = filesDict.keys()
gLogger.info( "ReplicationScheduler._execute: Obtaining replica information for %d sub-request files." % len( lfns ) )
res = self.rm.getCatalogReplicas( lfns )
if not res['OK']:
gLogger.error( "ReplicationScheduler._execute: Failed to get replica information.", res['Message'] )
continue
for lfn, failure in res['Value']['Failed'].items():
gLogger.error( "ReplicationScheduler._execute: Failed to get replicas.", '%s: %s' % ( lfn, failure ) )
replicas = res['Value']['Successful']
if not replicas.keys():
gLogger.error( "ReplicationScheduler._execute: Failed to get replica information for all files." )
continue
######################################################################################
#
# Now obtain the file sizes for the files associated to the sub-request.
#
lfns = replicas.keys()
gLogger.info( "ReplicationScheduler._execute: Obtaining file sizes for %d sub-request files." % len( lfns ) )
res = self.rm.getCatalogFileMetadata( lfns )
if not res['OK']:
gLogger.error( "ReplicationScheduler._execute: Failed to get file size information.", res['Message'] )
continue
for lfn, failure in res['Value']['Failed'].items():
gLogger.error( 'ReplicationScheduler._execute: Failed to get file size.', '%s: %s' % ( lfn, failure ) )
metadata = res['Value']['Successful']
if not metadata.keys():
gLogger.error( "ReplicationScheduler._execute: Failed to get metadata for all files." )
continue
######################################################################################
#
# For each LFN determine the replication tree
#
for lfn in sortList( metadata.keys() ):
fileSize = metadata[lfn]['Size']
lfnReps = replicas[lfn]
fileID = filesDict[lfn]
targets = []
for targetSE in targetSEs:
if targetSE in lfnReps.keys():
gLogger.debug( "ReplicationScheduler.execute: %s already present at %s." % ( lfn, targetSE ) )
else:
targets.append( targetSE )
if not targets:
gLogger.info( "ReplicationScheduler.execute: %s present at all targets." % lfn )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
continue
if not lfnReps:
gLogger.error( "ReplicationScheduler.execute: The file has no replicas.", lfn )
continue
res = self.strategyHandler.determineReplicationTree( sourceSE, targets, lfnReps, fileSize, strategy = reqRepStrategy )
if not res['OK']:
gLogger.error( "ReplicationScheduler.execute: Failed to determine replication tree.", res['Message'] )
continue
tree = res['Value']
######################################################################################
#
# For each item in the replication tree obtain the source and target SURLS
#
for channelID, dict in tree.items():
gLogger.info( "ReplicationScheduler.execute: processing for channel %d %s" % ( channelID, str( dict ) ) )
hopSourceSE = dict['SourceSE']
hopDestSE = dict['DestSE']
hopAncestor = dict['Ancestor']
# Get the sourceSURL
if hopAncestor:
status = 'Waiting%s' % ( hopAncestor )
res = self.obtainLFNSURL( hopSourceSE, lfn )
if not res['OK']:
errStr = res['Message']
gLogger.error( errStr )
return S_ERROR( errStr )
sourceSURL = res['Value']
else:
status = 'Waiting'
res = self.resolvePFNSURL( hopSourceSE, lfnReps[hopSourceSE] )
if not res['OK']:
sourceSURL = lfnReps[hopSourceSE]
else:
sourceSURL = res['Value']
# Get the targetSURL
res = self.obtainLFNSURL( hopDestSE, lfn )
if not res['OK']:
errStr = res['Message']
gLogger.error( errStr )
return S_ERROR( errStr )
targetSURL = res['Value']
######################################################################################
#
# For each item in the replication tree add the file to the channel
#
res = self.TransferDB.addFileToChannel( channelID, fileID, hopSourceSE, sourceSURL, hopDestSE, targetSURL, fileSize, fileStatus = status )
if not res['OK']:
errStr = res['Message']
gLogger.error( "ReplicationScheduler._execute: Failed to add File to Channel." , "%s %s" % ( fileID, channelID ) )
return S_ERROR( errStr )
res = self.TransferDB.addFileRegistration( channelID, fileID, lfn, targetSURL, hopDestSE )
if not res['OK']:
errStr = res['Message']
gLogger.error( "ReplicationScheduler._execute: Failed to add File registration." , "%s %s" % ( fileID, channelID ) )
result = self.TransferDB.removeFileFromChannel( channelID, fileID )
if not result['OK']:
errStr += result['Message']
gLogger.error( "ReplicationScheduler._execute: Failed to remove File." , "%s %s" % ( fileID, channelID ) )
return S_ERROR( errStr )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Scheduled' )
res = self.TransferDB.addReplicationTree( fileID, tree )
if oRequest.isSubRequestEmpty( ind, 'transfer' )['Value']:
oRequest.setSubRequestStatus( ind, 'transfer', 'Scheduled' )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDB.updateRequest( requestName, requestString )
if not res['OK']:
gLogger.error( "ReplicationScheduler._execute: Failed to update request", "%s %s" % ( requestName, res['Message'] ) )
def obtainLFNSURL( self, targetSE, lfn ):
""" Creates the targetSURL for the storage and LFN supplied
"""
res = self.factory.getStorages( targetSE, protocolList = ['SRM2'] )
if not res['OK']:
errStr = 'ReplicationScheduler._execute: Failed to create SRM2 storage for %s: %s. ' % ( targetSE, res['Message'] )
gLogger.error( errStr )
return S_ERROR( errStr )
storageObjects = res['Value']['StorageObjects']
for storageObject in storageObjects:
res = storageObject.getCurrentURL( lfn )
if res['OK']:
return res
gLogger.error( 'ReplicationScheduler._execute: Failed to get SRM compliant storage.' , targetSE )
return S_ERROR( 'ReplicationScheduler._execute: Failed to get SRM compliant storage.' )
def resolvePFNSURL( self, sourceSE, pfn ):
""" Creates the targetSURL for the storage and LFN supplied
"""
res = self.rm.getPfnForProtocol( [pfn], sourceSE )
if not res['OK']:
return res
if pfn in res['Value']['Failed'].keys():
return S_ERROR( res['Value']['Failed'][pfn] )
return S_OK( res['Value']['Successful'][pfn] )
