class RequestTask( object ):
"""
.. class:: RequestTask
Base class for DMS 'transfer', 'removal' and 'register' Requests processing.
This class is meant to be executed as a ProcessTask inside ProcessPool.
The most important and common global DIRAC objects are created in RequestTask constructor.
This includes gLogger, gConfig, gProxyManager, S_OK and S_ERROR. The constructor also
imports a set of common modules: os, sys, re, time and everything from types module.
All other DIRAC tools and clients (i.e. RequestManager) are private in RequestTask class and will
be imported and instantiated on demand during first usage. They are accessible using proxied public methods, i.e.
if you are going to use ReplicaManager just call::
self.replicaManager().someMethod()
All currently proxied tools are::
DataLoggingClient -- self.dataLoggingClient()
ReplicaManager -- self.replicaManager()
RequestClient -- self.requestClient()
StorageFactory -- self.storageFactory()
SubLogger message handles for all levels are also proxied, so you can directly use them in your code, i.e.::
self.info("An info message")
self.debug("This will be shown only in debug")
For handling sub-request one has to register their actions handlers using :self.addOperationAction:
method. This method checks if handler is defined as a method of inherited class and then puts its
definition into internal operation dispatcher dictionary with a key of sub-request's operation name.
Each operation handler should have the signature::
def operationName( self, index, requestObj, subRequestAttrs, subRequestFiles )
where index is a sub-request counter, requestObj is a RequestContainer instance,
subRequestAttrs is a dict with sub-request attributes and subRequestFiles is a dict with
files attached to the sub-request.
Handlers shoudl always return S_OK with value of (modified or not) requestObj, S_ERROR with some
error message otherwise.
Processing of request is done automatically in self.__call__, one doesn't have to worry about changing
credentials, looping over subrequests or request finalizing -- only sub-request processing matters in
the all inherited classes.
Concerning :MonitringClient: (or better known its global instance :gMonitor:), if someone wants to send
some metric over there, she has to put in agent's code registration of activity and then in a particular
task use :RequestTask.addMark: to save monitoring data. All monitored activities are held in
:RequestTask.__monitor: dict which at the end of processing is returned from :RequestTask.__call__:.
The values are then processed and pushed to the gMonitor instance in the default callback function.
"""
## reference to ReplicaManager
__replicaManager = None
## reference to DataLoggingClient
__dataLoggingClient = None
## reference to RequestClient
__requestClient = None
## reference to StotageFactory
__storageFactory = None
## subLogger
__log = None
## request type
__requestType = None
## placeholder for request owner DB
requestOwnerDN = None
## placeholder for Request owner group
requestOwnerGroup = None
## operation dispatcher for SubRequests,
## a dictonary
## "operation" => methodToRun
##
__operationDispatcher = {}
## holder for DataManager proxy file
__dataManagerProxy = None
## monitoring dict
__monitor = {}
def __init__( self, requestString, requestName, executionOrder, jobID, configPath ):
""" c'tor
:param self: self reference
:param str requestString: XML serialised RequestContainer
:param str requestName: request name
:param list executionOrder: request execution order
:param int jobID: jobID
:param str sourceServer: request's source server
:param str configPath: path in CS for parent agent
"""
## fixtures
## python fixtures
import os, os.path, sys, time, re, types
self.makeGlobal( "os", os )
self.makeGlobal( "os.path", os.path )
self.makeGlobal( "sys", sys )
self.makeGlobal( "time", time )
self.makeGlobal( "re", re )
## export all Types from types
[ self.makeGlobal( item, getattr( types, item ) ) for item in dir(types) if "Type" in item ]
## DIRAC fixtures
from DIRAC.FrameworkSystem.Client.Logger import gLogger
self.__log = gLogger.getSubLogger( "%s/%s" % ( self.__class__.__name__, str(requestName) ) )
self.always = self.__log.always
self.notice = self.__log.notice
self.info = self.__log.info
self.debug = self.__log.debug
self.warn = self.__log.warn
self.error = self.__log.error
self.exception = self.__log.exception
self.fatal = self.__log.fatal
from DIRAC import S_OK, S_ERROR
from DIRAC.ConfigurationSystem.Client.Config import gConfig
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getGroupsWithVOMSAttribute
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
## export DIRAC global tools and functions
self.makeGlobal( "S_OK", S_OK )
self.makeGlobal( "S_ERROR", S_ERROR )
self.makeGlobal( "gLogger", gLogger )
self.makeGlobal( "gConfig", gConfig )
self.makeGlobal( "gProxyManager", gProxyManager )
self.makeGlobal( "getGroupsWithVOMSAttribute", getGroupsWithVOMSAttribute )
self.makeGlobal( "gConfigurationData", gConfigurationData )
## save request string
self.requestString = requestString
## build request object
from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer
self.requestObj = RequestContainer( init = False )
self.requestObj.parseRequest( request = self.requestString )
## save request name
self.requestName = requestName
## .. and jobID
self.jobID = jobID
## .. and execution order
self.executionOrder = executionOrder
## save config path
self.__configPath = configPath
## set requestType
self.setRequestType( gConfig.getValue( os.path.join( configPath, "RequestType" ), "" ) )
## get log level
self.__log.setLevel( gConfig.getValue( os.path.join( configPath, self.__class__.__name__, "LogLevel" ), "INFO" ) )
## clear monitoring
self.__monitor = {}
## save DataManager proxy
if "X509_USER_PROXY" in os.environ:
self.info("saving path to current proxy file")
self.__dataManagerProxy = os.environ["X509_USER_PROXY"]
else:
self.error("'X509_USER_PROXY' environment variable not set")
def dataManagerProxy( self ):
""" get dataManagerProxy file
:param self: self reference
"""
return self.__dataManagerProxy
def addMark( self, name, value = 1 ):
""" add mark to __monitor dict
:param self: self reference
:param name: mark name
:param value: value to be
"""
if name not in self.__monitor:
self.__monitor.setdefault( name, 0 )
self.__monitor[name] += value
def monitor( self ):
""" get monitoring dict
:param cls: class reference
"""
return self.__monitor
def makeGlobal( self, objName, objDef ):
""" export :objDef: to global name space using :objName: name
:param self: self reference
:param str objName: symbol name
:param mixed objDef: symbol definition
:throws: NameError if symbol of that name is already in
"""
if objName not in __builtins__:
if type( __builtins__) == type( {} ):
__builtins__[objName] = objDef
else:
setattr( __builtins__, objName, objDef )
return True
def requestType( self ):
""" get request type
:params self: self reference
"""
return self.__requestType
def setRequestType( self, requestType ):
""" set request type
:param self: self reference
"""
self.debug( "Setting requestType to %s" % str(requestType) )
self.__requestType = requestType
@classmethod
def replicaManager( cls ):
""" ReplicaManager getter
:param cls: class reference
"""
if not cls.__replicaManager:
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
cls.__replicaManager = ReplicaManager()
return cls.__replicaManager
@classmethod
def dataLoggingClient( cls ):
""" DataLoggingClient getter
:param cls: class reference
"""
if not cls.__dataLoggingClient:
from DIRAC.DataManagementSystem.Client.DataLoggingClient import DataLoggingClient
cls.__dataLoggingClient = DataLoggingClient()
return cls.__dataLoggingClient
@classmethod
def requestClient( cls ):
""" RequestClient getter
:param cls: class reference
"""
if not cls.__requestClient:
from DIRAC.RequestManagementSystem.Client.RequestClient import RequestClient
cls.__requestClient = RequestClient()
return cls.__requestClient
@classmethod
def storageFactory( cls ):
""" StorageFactory getter
:param cls: class reference
"""
if not cls.__storageFactory:
from DIRAC.Resources.Storage.StorageFactory import StorageFactory
cls.__storageFactory = StorageFactory()
return cls.__storageFactory
def changeProxy( self, ownerDN, ownerGroup ):
""" get proxy from gProxyManager, save it to file
:param self: self reference
:param str ownerDN: request owner DN
:param str ownerGroup: request owner group
:return: S_OK with name of newly created owner proxy file
"""
ownerProxy = gProxyManager.downloadVOMSProxy( str(ownerDN), str(ownerGroup) )
if not ownerProxy["OK"] or not ownerProxy["Value"]:
reason = ownerProxy["Message"] if "Message" in ownerProxy else "No valid proxy found in ProxyManager."
return S_ERROR( "Change proxy error for '%s'@'%s': %s" % ( ownerDN, ownerGroup, reason ) )
ownerProxyFile = ownerProxy["Value"].dumpAllToFile()
if not ownerProxyFile["OK"]:
return S_ERROR( ownerProxyFile["Message"] )
ownerProxyFile = ownerProxyFile["Value"]
os.environ["X509_USER_PROXY"] = ownerProxyFile
return S_OK( ownerProxyFile )
######################################################################
# operationDispatcher
@classmethod
def operationDispatcher( cls ):
""" operation dispatcher getter
:param cls: class reference
"""
return cls.__operationDispatcher
@classmethod
def addOperationAction( cls, operation, methodToRun, overwrite = True ):
""" register handler :methodToRun: for SubRequest operation :operation:
:warn: all handlers should have the same signature
:param self: self reference
:param str operation: SubRequest operation name
:param MethodType methodToRun: handler to be executed for SubRequest
:param bool overwrite: flag to overwrite handler, if already present
:return: S_OK/S_ERROR
Every action handler should return S_OK with of a structure::
{ "OK" : True,
"Value" : requestObj # that has been sent to operation handler
}
otherwise S_ERROR.
"""
if operation in cls.__operationDispatcher and not overwrite:
return S_ERROR("addOperationAction: operation for '%s' is already registered" % operation )
if type(methodToRun) is not MethodType:
return S_ERROR("addOperationAction: wrong type (%s = types.MethodType) for '%s' operation" % \
( str(type(methodToRun)), operation ) )
cls.__operationDispatcher[operation] = methodToRun
return S_OK()
def __call__( self ):
""" generic function to process one Request of a type requestType
This method could be run in a thread.
:param self: self reference
:param str requestType: request type
:return: S_OK/S_ERROR
"""
self.always("executing request %s" % self.requestName )
################################################################
## get ownerDN and ownerGroup
ownerDN = self.requestObj.getAttribute( "OwnerDN" )
if not ownerDN["OK"]:
return ownerDN
ownerDN = ownerDN["Value"]
ownerGroup = self.requestObj.getAttribute( "OwnerGroup" )
if not ownerGroup["OK"]:
return ownerGroup
ownerGroup = ownerGroup["Value"]
## save request owner
self.requestOwnerDN = ownerDN if ownerDN else ""
self.requestOwnerGroup = ownerGroup if ownerGroup else ""
#################################################################
## change proxy
ownerProxyFile = None
if ownerDN and ownerGroup:
ownerProxyFile = self.changeProxy( ownerDN, ownerGroup )
if not ownerProxyFile["OK"]:
self.error( "handleReuqest: unable to get proxy for '%s'@'%s': %s" % ( ownerDN,
ownerGroup,
ownerProxyFile["Message"] ) )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when updating request: %s" % update["Message"] )
return update
return ownerProxyFile
ownerProxyFile = ownerProxyFile["Value"]
#self.ownerProxyFile = ownerProxyFile
self.info( "Will execute request for '%s'@'%s' using proxy file %s" % ( ownerDN, ownerGroup, ownerProxyFile ) )
else:
self.info( "Will execute request for DataManager using her/his proxy")
#################################################################
## execute handlers
ret = { "OK" : False, "Message" : "" }
useServerCert = gConfig.useServerCertificate()
try:
# Execute task with the owner proxy even for contacting DIRAC services
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','false')
ret = self.handleRequest()
finally:
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','true')
## delete owner proxy
if self.__dataManagerProxy:
os.environ["X509_USER_PROXY"] = self.__dataManagerProxy
if ownerProxyFile and os.path.exists( ownerProxyFile ):
os.unlink( ownerProxyFile )
if not ret["OK"]:
self.error( "handleRequest: error during request processing: %s" % ret["Message"] )
self.error( "handleRequest: will put original request back" )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when putting back request: %s" % update["Message"] )
## return at least
return ret
def handleRequest( self ):
""" read SubRequests and ExecutionOrder, fire registered handlers upon SubRequests operations
:param self: self reference
:param dict requestDict: request dictionary as read from self.readRequest
"""
##############################################################
# here comes the processing
##############################################################
res = self.requestObj.getNumSubRequests( self.__requestType )
if not res["OK"]:
errMsg = "handleRequest: failed to obtain number of '%s' subrequests." % self.__requestType
self.error( errMsg, res["Message"] )
return S_ERROR( res["Message"] )
## for gMonitor
self.addMark( "Execute", 1 )
## process sub requests
for index in range( res["Value"] ):
self.info( "handleRequest: processing subrequest %s." % str(index) )
subRequestAttrs = self.requestObj.getSubRequestAttributes( index, self.__requestType )["Value"]
if subRequestAttrs["ExecutionOrder"]:
subExecutionOrder = int( subRequestAttrs["ExecutionOrder"] )
else:
subExecutionOrder = 0
subRequestStatus = subRequestAttrs["Status"]
if subRequestStatus != "Waiting":
self.info( "handleRequest: subrequest %s has status '%s' and is not to be executed." % ( str(index),
subRequestStatus ) )
continue
if subExecutionOrder <= self.executionOrder:
operation = subRequestAttrs["Operation"]
if operation not in self.operationDispatcher():
self.error( "handleRequest: '%s' operation not supported" % operation )
else:
self.info( "handleRequest: will execute %s '%s' subrequest" % ( str(index), operation ) )
## get files
subRequestFiles = self.requestObj.getSubRequestFiles( index, self.__requestType )["Value"]
## execute operation action
ret = self.operationDispatcher()[operation].__call__( index,
self.requestObj,
subRequestAttrs,
subRequestFiles )
################################################
## error in operation action?
if not ret["OK"]:
self.error( "handleRequest: error when handling subrequest %s: %s" % ( str(index), ret["Message"] ) )
self.requestObj.setSubRequestAttributeValue( index, self.__requestType, "Error", ret["Message"] )
else:
## update ref to requestObj
self.requestObj = ret["Value"]
## check if subrequest status == Done, disable finalisation if not
subRequestDone = self.requestObj.isSubRequestDone( index, self.__requestType )
if not subRequestDone["OK"]:
self.error( "handleRequest: unable to determine subrequest status: %s" % subRequestDone["Message"] )
else:
if not subRequestDone["Value"]:
self.warn("handleRequest: subrequest %s is not done yet" % str(index) )
################################################
# Generate the new request string after operation
newRequestString = self.requestObj.toXML()['Value']
update = self.putBackRequest( self.requestName, newRequestString )
if not update["OK"]:
self.error( "handleRequest: error when updating request: %s" % update["Message"] )
return update
## get request status
if self.jobID:
requestStatus = self.requestClient().getRequestStatus( self.requestName )
if not requestStatus["OK"]:
return requestStatus
requestStatus = requestStatus["Value"]
## finalize request if jobID is present and request status = 'Done'
self.info("handleRequest: request status is %s" % requestStatus )
if ( requestStatus["RequestStatus"] == "Done" ) and ( requestStatus["SubRequestStatus"] not in ( "Waiting", "Assigned" ) ):
self.debug("handleRequest: request is going to be finalised")
finalize = self.requestClient().finalizeRequest( self.requestName, self.jobID )
if not finalize["OK"]:
self.error("handleRequest: error in request finalization: %s" % finalize["Message"] )
return finalize
self.info("handleRequest: request is finalised")
## for gMonitor
self.addMark( "Done", 1 )
## should return S_OK with monitor dict
return S_OK( { "monitor" : self.monitor() } )
def putBackRequest( self, requestName, requestString ):
""" put request back
:param self: self reference
:param str requestName: request name
:param str requestString: XML-serilised request
:param str sourceServer: request server URL
"""
update = self.requestClient().updateRequest( requestName, requestString )
if not update["OK"]:
self.error( "putBackRequest: error when updating request: %s" % update["Message"] )
return update
return S_OK()
def executeRequest(self):
################################################
# Get a request from request DB
gMonitor.addMark("Iteration", 1)
res = self.RequestDBClient.getRequest('transfer')
if not res['OK']:
gLogger.info(
"TransferAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"TransferAgent.execute: No requests to be executed found.")
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
try:
jobID = int(res['Value']['JobID'])
except:
jobID = 0
gLogger.info("TransferAgent.execute: Obtained request %s" %
requestName)
result = self.RequestDBClient.getCurrentExecutionOrder(
requestName, sourceServer)
if result['OK']:
currentOrder = result['Value']
else:
return S_OK('Can not get the request execution order')
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('transfer')
if not res['OK']:
errStr = "TransferAgent.execute: Failed to obtain number of transfer subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("TransferAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
modified = False
for ind in range(res['Value']):
gMonitor.addMark("Execute", 1)
gLogger.info("TransferAgent.execute: Processing sub-request %s." %
ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'transfer')['Value']
if subRequestAttributes['ExecutionOrder']:
subExecutionOrder = int(subRequestAttributes['ExecutionOrder'])
else:
subExecutionOrder = 0
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'transfer')['Value']
operation = subRequestAttributes['Operation']
subRequestError = ''
################################################
# If the sub-request is a put and register operation
if operation == 'putAndRegister' or operation == 'putAndRegisterAndRemove':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSE = str(subRequestAttributes['TargetSE'])
catalog = ''
if subRequestAttributes.has_key('Catalogue'):
catalog = subRequestAttributes['Catalogue']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Put and register", 1)
lfn = str(subRequestFile['LFN'])
file = subRequestFile['PFN']
guid = subRequestFile['GUID']
addler = subRequestFile['Addler']
res = self.ReplicaManager.putAndRegister(
lfn,
file,
diracSE,
guid=guid,
checksum=addler,
catalog=catalog)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
if not res['Value']['Successful'][
lfn].has_key('put'):
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Failed to put %s to %s."
% (lfn, diracSE))
subRequestError = "Put operation failed for %s to %s" % (
lfn, diracSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put failed')
elif not res['Value']['Successful'][
lfn].has_key('register'):
gMonitor.addMark("Put successful", 1)
gMonitor.addMark(
"File registration failed", 1)
self.DataLog.addFileRecord(
lfn, 'Put', diracSE, '',
'TransferAgent')
self.DataLog.addFileRecord(
lfn, 'RegisterFail', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Successfully put %s to %s in %s seconds."
% (lfn, diracSE, res['Value']
['Successful'][lfn]['put']))
gLogger.info(
"TransferAgent.execute: Failed to register %s to %s."
% (lfn, diracSE))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Registration failed')
subRequestError = "Registration failed for %s to %s" % (
lfn, diracSE)
fileDict = res['Value']['Failed'][lfn][
'register']
registerRequestDict = {
'Attributes': {
'TargetSE':
fileDict['TargetSE'],
'Operation': 'registerFile'
},
'Files': [{
'LFN':
fileDict['LFN'],
'PFN':
fileDict['PFN'],
'Size':
fileDict['Size'],
'Addler':
fileDict['Addler'],
'GUID':
fileDict['GUID']
}]
}
gLogger.info(
"TransferAgent.execute: Setting registration request for failed file."
)
oRequest.addSubRequest(
registerRequestDict, 'register')
modified = True
else:
gMonitor.addMark("Put successful", 1)
gMonitor.addMark(
"File registration successful", 1)
self.DataLog.addFileRecord(
lfn, 'Put', diracSE, '',
'TransferAgent')
self.DataLog.addFileRecord(
lfn, 'Register', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Successfully put %s to %s in %s seconds."
% (lfn, diracSE, res['Value']
['Successful'][lfn]['put']))
gLogger.info(
"TransferAgent.execute: Successfully registered %s to %s in %s seconds."
%
(lfn, diracSE, res['Value']
['Successful'][lfn]['register']))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
modified = True
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Failed to put and register file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, diracSE,
res['Value']['Failed'][lfn]))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Complete file failure')
subRequestError = "Failed to put and register file"
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Completely failed to put and register file."
gLogger.error(errStr, res['Message'])
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'RM call failure')
subRequestError = operation + " RM call file"
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a put operation
elif operation == 'put':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Put", 1)
lfn = subRequestFile['LFN']
file = subRequestFile['PFN']
res = self.ReplicaManager.put(lfn, file, diracSE)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
gMonitor.addMark("Put successful", 1)
self.DataLog.addFileRecord(
lfn, 'Put', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Successfully put %s to %s in %s seconds."
% (lfn, diracSE,
res['Value']['Successful'][lfn]))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
modified = True
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Failed to put file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, diracSE,
res['Value']['Failed'][lfn]))
subRequestError = "Put operation failed for %s to %s" % (
lfn, diracSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put failed')
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Completely failed to put file."
gLogger.error(errStr, res['Message'])
subRequestError = "Put RM call failed for %s to %s" % (
lfn, diracSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put RM call failed')
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a replicate and register operation
elif operation == 'replicateAndRegister' or operation == 'replicateAndRegisterAndRemove':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
if sourceSE == "None":
sourceSE = ''
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Replicate and register", 1)
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicateAndRegister(
lfn, targetSE, sourceSE=sourceSE)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
if not res['Value']['Successful'][
lfn].has_key('replicate'):
gLogger.info(
"TransferAgent.execute: Failed to replicate %s to %s."
% (lfn, targetSE))
gMonitor.addMark(
"Replication failed", 1)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, "Error",
"Replication failed")
subRequestError = "Replication failed for %s to %s" % (
lfn, targetSE)
elif not res['Value']['Successful'][
lfn].has_key('register'):
gMonitor.addMark(
"Replication successful", 1)
gMonitor.addMark(
"Replica registration failed", 1)
gLogger.info(
"TransferAgent.execute: Successfully replicated %s to %s in %s seconds."
%
(lfn, targetSE, res['Value']
['Successful'][lfn]['replicate']))
gLogger.info(
"TransferAgent.execute: Failed to register %s to %s."
% (lfn, targetSE))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Registration failed')
subRequestError = "Registration failed for %s to %s" % (
lfn, targetSE)
fileDict = res['Value']['Failed'][lfn][
'register']
registerRequestDict = {
'Attributes': {
'TargetSE':
fileDict['TargetSE'],
'Operation': 'registerReplica'
},
'Files': [{
'LFN': fileDict['LFN'],
'PFN': fileDict['PFN']
}]
}
gLogger.info(
"TransferAgent.execute: Setting registration request for failed replica."
)
oRequest.addSubRequest(
registerRequestDict, 'register')
modified = True
else:
gMonitor.addMark(
"Replication successful", 1)
gMonitor.addMark(
"Replica registration successful",
1)
gLogger.info(
"TransferAgent.execute: Successfully replicated %s to %s in %s seconds."
%
(lfn, targetSE, res['Value']
['Successful'][lfn]['replicate']))
gLogger.info(
"TransferAgent.execute: Successfully registered %s to %s in %s seconds."
%
(lfn, targetSE, res['Value']
['Successful'][lfn]['register']))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
modified = True
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Failed to replicate and register file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, targetSE,
res['Value']['Failed'][lfn]))
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Completely failed to replicate and register file."
gLogger.error(errStr, res['Message'])
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'RM call failure')
subRequestError = operation + " RM call failed"
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a replicate operation
elif operation == 'replicate':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Replicate", 1)
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicate(
lfn, targetSE, sourceSE=sourceSE)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
gMonitor.addMark("Replication successful",
1)
gLogger.info(
"TransferAgent.execute: Successfully replicated %s to %s in %s seconds."
% (lfn, diracSE,
res['Value']['Successful'][lfn]))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
modified = True
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Failed to replicate file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, targetSE,
res['Value']['Failed'][lfn]))
subRequestError = "Replicate operation failed for %s to %s" % (
lfn, targetSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put failed')
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Completely failed to replicate file."
gLogger.error(errStr, res['Message'])
subRequestError = "Replicate RM call failed for %s to %s" % (
lfn, targetSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Replicate RM call failed')
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a get operation
elif operation == 'get':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
sourceSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str(subRequestFile['LFN'])
pfn = str(subRequestFile['PFN'])
got = False
if sourceSE and pfn:
res = self.ReplicaManager.getStorageFile(
pfn, sourceSE)
if res['Value']['Successful'].has_key(pfn):
got = True
else:
res = self.ReplicaManager.getFile(lfn)
if res['Value']['Successful'].has_key(lfn):
got = False
if got:
gLogger.info(
"TransferAgent.execute: Successfully got %s."
% lfn)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
modified = True
else:
errStr = "TransferAgent.execute: Failed to get file."
gLogger.error(errStr, lfn)
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is none of the above types
else:
gLogger.error(
"TransferAgent.execute: Operation not supported.",
operation)
if subRequestError:
oRequest.setSubRequestAttributeValue(
ind, 'transfer', 'Error', subRequestError)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'transfer')['Value']:
oRequest.setSubRequestStatus(ind, 'transfer', 'Done')
gMonitor.addMark("Done", 1)
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"TransferAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest(requestName, requestString,
sourceServer)
if modified and jobID:
result = self.finalizeRequest(requestName, jobID, sourceServer)
return S_OK()
class RequestTask( object ):
"""
.. class:: RequestTask
Base class for DMS 'transfer', 'removal' and 'register' Requests processing.
This class is meant to be executed as a ProcessTask inside ProcessPool.
The most important and common global DIRAC objects are created in RequestTask constructor.
This includes gLogger, gConfig, gProxyManager, S_OK and S_ERROR. The constructor also
imports a set of common modules: os, sys, re, time and everything from types module.
All other DIRAC tools and clients (i.e. RequestManager) are private in RequestTask class and will
be imported and instantiated on demand during first usage. They are accessible using proxied public methods, i.e.
if you are going to use ReplicaManager just call::
self.replicaManager().someMethod()
All currently proxied tools are::
DataLoggingClient -- self.dataLoggingClient()
ReplicaManager -- self.replicaManager()
RequestClient -- self.requestClient()
StorageFactory -- self.storageFactory()
SubLogger message handles for all levels are also proxied, so you can directly use them in your code, i.e.::
self.info("An info message")
self.debug("This will be shown only in debug")
For handling sub-request one has to register their actions handlers using :self.addOperationAction:
method. This method checks if handler is defined as a method of inherited class and then puts its
definition into internal operation dispatcher dictionary with a key of sub-request's operation name.
Each operation handler should have the signature::
def operationName( self, index, requestObj, subRequestAttrs, subRequestFiles )
where index is a sub-request counter, requestObj is a RequestContainer instance,
subRequestAttrs is a dict with sub-request attributes and subRequestFiles is a dict with
files attached to the sub-request.
Handlers shoudl always return S_OK with value of (modified or not) requestObj, S_ERROR with some
error message otherwise.
Processing of request is done automatically in self.__call__, one doesn't have to worry about changing
credentials, looping over subrequests or request finalizing -- only sub-request processing matters in
the all inherited classes.
Concerning :MonitringClient: (or better known its global instance :gMonitor:), if someone wants to send
some metric over there, she has to put in agent's code registration of activity and then in a particular
task use :RequestTask.addMark: to save monitoring data. All monitored activities are held in
:RequestTask.__monitor: dict which at the end of processing is returned from :RequestTask.__call__:.
The values are then processed and pushed to the gMonitor instance in the default callback function.
"""
## reference to ReplicaManager
__replicaManager = None
## reference to DataLoggingClient
__dataLoggingClient = None
## reference to RequestClient
__requestClient = None
## reference to StotageFactory
__storageFactory = None
## subLogger
__log = None
## request type
__requestType = None
## placeholder for request owner DB
requestOwnerDN = None
## placeholder for Request owner group
requestOwnerGroup = None
## operation dispatcher for SubRequests,
## a dictonary
## "operation" => methodToRun
##
__operationDispatcher = {}
## holder for DataManager proxy file
__dataManagerProxy = None
## monitoring dict
__monitor = {}
def __init__( self, requestString, requestName, executionOrder, jobID, configPath ):
""" c'tor
:param self: self reference
:param str requestString: XML serialised RequestContainer
:param str requestName: request name
:param list executionOrder: request execution order
:param int jobID: jobID
:param str sourceServer: request's source server
:param str configPath: path in CS for parent agent
"""
## fixtures
## python fixtures
import os, os.path, sys, time, re, types
self.makeGlobal( "os", os )
self.makeGlobal( "os.path", os.path )
self.makeGlobal( "sys", sys )
self.makeGlobal( "time", time )
self.makeGlobal( "re", re )
## export all Types from types
[ self.makeGlobal( item, getattr( types, item ) ) for item in dir(types) if "Type" in item ]
## DIRAC fixtures
from DIRAC.FrameworkSystem.Client.Logger import gLogger
self.__log = gLogger.getSubLogger( "%s/%s" % ( self.__class__.__name__, str(requestName) ) )
self.always = self.__log.always
self.notice = self.__log.notice
self.info = self.__log.info
self.debug = self.__log.debug
self.warn = self.__log.warn
self.error = self.__log.error
self.exception = self.__log.exception
self.fatal = self.__log.fatal
from DIRAC import S_OK, S_ERROR
from DIRAC.ConfigurationSystem.Client.Config import gConfig
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getGroupsWithVOMSAttribute
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
## export DIRAC global tools and functions
self.makeGlobal( "S_OK", S_OK )
self.makeGlobal( "S_ERROR", S_ERROR )
self.makeGlobal( "gLogger", gLogger )
self.makeGlobal( "gConfig", gConfig )
self.makeGlobal( "gProxyManager", gProxyManager )
self.makeGlobal( "getGroupsWithVOMSAttribute", getGroupsWithVOMSAttribute )
self.makeGlobal( "gConfigurationData", gConfigurationData )
## save request string
self.requestString = requestString
## build request object
from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer
self.requestObj = RequestContainer( init = False )
self.requestObj.parseRequest( request = self.requestString )
## save request name
self.requestName = requestName
## .. and jobID
self.jobID = jobID
## .. and execution order
self.executionOrder = executionOrder
## save config path
self.__configPath = configPath
## set requestType
self.setRequestType( gConfig.getValue( os.path.join( configPath, "RequestType" ), "" ) )
## get log level
self.__log.setLevel( gConfig.getValue( os.path.join( configPath, self.__class__.__name__, "LogLevel" ), "INFO" ) )
## clear monitoring
self.__monitor = {}
## save DataManager proxy
if "X509_USER_PROXY" in os.environ:
self.info("saving path to current proxy file")
self.__dataManagerProxy = os.environ["X509_USER_PROXY"]
else:
self.error("'X509_USER_PROXY' environment variable not set")
def dataManagerProxy( self ):
""" get dataManagerProxy file
:param self: self reference
"""
return self.__dataManagerProxy
def addMark( self, name, value = 1 ):
""" add mark to __monitor dict
:param self: self reference
:param name: mark name
:param value: value to be
"""
if name not in self.__monitor:
self.__monitor.setdefault( name, 0 )
self.__monitor[name] += value
def monitor( self ):
""" get monitoring dict
:param cls: class reference
"""
return self.__monitor
def makeGlobal( self, objName, objDef ):
""" export :objDef: to global name space using :objName: name
:param self: self reference
:param str objName: symbol name
:param mixed objDef: symbol definition
:throws: NameError if symbol of that name is already in
"""
if objName not in __builtins__:
if type( __builtins__) == type( {} ):
__builtins__[objName] = objDef
else:
setattr( __builtins__, objName, objDef )
return True
def requestType( self ):
""" get request type
:params self: self reference
"""
return self.__requestType
def setRequestType( self, requestType ):
""" set request type
:param self: self reference
"""
self.debug( "Setting requestType to %s" % str(requestType) )
self.__requestType = requestType
@classmethod
def replicaManager( cls ):
""" ReplicaManager getter
:param cls: class reference
"""
if not cls.__replicaManager:
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
cls.__replicaManager = ReplicaManager()
return cls.__replicaManager
@classmethod
def dataLoggingClient( cls ):
""" DataLoggingClient getter
:param cls: class reference
"""
if not cls.__dataLoggingClient:
from DIRAC.DataManagementSystem.Client.DataLoggingClient import DataLoggingClient
cls.__dataLoggingClient = DataLoggingClient()
return cls.__dataLoggingClient
@classmethod
def requestClient( cls ):
""" RequestClient getter
:param cls: class reference
"""
if not cls.__requestClient:
from DIRAC.Core.DISET.RPCClient import RPCClient
from DIRAC.RequestManagementSystem.Client.RequestClient import RequestClient
cls.__requestClient = RequestClient()
return cls.__requestClient
@classmethod
def storageFactory( cls ):
""" StorageFactory getter
:param cls: class reference
"""
if not cls.__storageFactory:
from DIRAC.Resources.Storage.StorageFactory import StorageFactory
cls.__storageFactory = StorageFactory()
return cls.__storageFactory
def changeProxy( self, ownerDN, ownerGroup ):
""" get proxy from gProxyManager, save it to file
:param self: self reference
:param str ownerDN: request owner DN
:param str ownerGroup: request owner group
:return: S_OK with name of newly created owner proxy file
"""
ownerProxy = gProxyManager.downloadVOMSProxy( str(ownerDN), str(ownerGroup) )
if not ownerProxy["OK"] or not ownerProxy["Value"]:
reason = ownerProxy["Message"] if "Message" in ownerProxy else "No valid proxy found in ProxyManager."
return S_ERROR( "Change proxy error for '%s'@'%s': %s" % ( ownerDN, ownerGroup, reason ) )
ownerProxyFile = ownerProxy["Value"].dumpAllToFile()
if not ownerProxyFile["OK"]:
return S_ERROR( ownerProxyFile["Message"] )
ownerProxyFile = ownerProxyFile["Value"]
os.environ["X509_USER_PROXY"] = ownerProxyFile
return S_OK( ownerProxyFile )
######################################################################
# operationDispatcher
@classmethod
def operationDispatcher( cls ):
""" operation dispatcher getter
:param cls: class reference
"""
return cls.__operationDispatcher
@classmethod
def addOperationAction( cls, operation, methodToRun, overwrite = True ):
""" register handler :methodToRun: for SubRequest operation :operation:
:warn: all handlers should have the same signature
:param self: self reference
:param str operation: SubRequest operation name
:param MethodType methodToRun: handler to be executed for SubRequest
:param bool overwrite: flag to overwrite handler, if already present
:return: S_OK/S_ERROR
Every action handler should return S_OK with of a structure::
{ "OK" : True,
"Value" : requestObj # that has been sent to operation handler
}
otherwise S_ERROR.
"""
if operation in cls.__operationDispatcher and not overwrite:
return S_ERROR("addOperationAction: operation for '%s' is already registered" % operation )
if type(methodToRun) is not MethodType:
return S_ERROR("addOperationAction: wrong type (%s = types.MethodType) for '%s' operation" % \
( str(type(methodToRun)), operation ) )
cls.__operationDispatcher[operation] = methodToRun
return S_OK()
def __call__( self ):
""" generic function to process one Request of a type requestType
This method could be run in a thread.
:param self: self reference
:param str requestType: request type
:return: S_OK/S_ERROR
"""
self.always("executing request %s" % self.requestName )
################################################################
## get ownerDN and ownerGroup
ownerDN = self.requestObj.getAttribute( "OwnerDN" )
if not ownerDN["OK"]:
return ownerDN
ownerDN = ownerDN["Value"]
ownerGroup = self.requestObj.getAttribute( "OwnerGroup" )
if not ownerGroup["OK"]:
return ownerGroup
ownerGroup = ownerGroup["Value"]
## save request owner
self.requestOwnerDN = ownerDN if ownerDN else ""
self.requestOwnerGroup = ownerGroup if ownerGroup else ""
#################################################################
## change proxy
ownerProxyFile = None
if ownerDN and ownerGroup:
ownerProxyFile = self.changeProxy( ownerDN, ownerGroup )
if not ownerProxyFile["OK"]:
self.error( "handleReuqest: unable to get proxy for '%s'@'%s': %s" % ( ownerDN,
ownerGroup,
ownerProxyFile["Message"] ) )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when updating request: %s" % update["Message"] )
return update
return ownerProxyFile
ownerProxyFile = ownerProxyFile["Value"]
#self.ownerProxyFile = ownerProxyFile
self.info( "Will execute request for '%s'@'%s' using proxy file %s" % ( ownerDN, ownerGroup, ownerProxyFile ) )
else:
self.info( "Will execute request for DataManager using her/his proxy")
#################################################################
## execute handlers
ret = { "OK" : False, "Message" : "" }
useServerCert = gConfig.useServerCertificate()
try:
# Execute task with the owner proxy even for contacting DIRAC services
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','false')
ret = self.handleRequest()
finally:
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','true')
## delete owner proxy
if self.__dataManagerProxy:
os.environ["X509_USER_PROXY"] = self.__dataManagerProxy
if ownerProxyFile and os.path.exists( ownerProxyFile ):
os.unlink( ownerProxyFile )
if not ret["OK"]:
self.error( "handleRequest: error during request processing: %s" % ret["Message"] )
self.error( "handleRequest: will put original request back" )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when putting back request: %s" % update["Message"] )
## return at least
return ret
def handleRequest( self ):
""" read SubRequests and ExecutionOrder, fire registered handlers upon SubRequests operations
:param self: self reference
:param dict requestDict: request dictionary as read from self.readRequest
"""
##############################################################
# here comes the processing
##############################################################
res = self.requestObj.getNumSubRequests( self.__requestType )
if not res["OK"]:
errMsg = "handleRequest: failed to obtain number of '%s' subrequests." % self.__requestType
self.error( errMsg, res["Message"] )
return S_ERROR( res["Message"] )
## for gMonitor
self.addMark( "Execute", 1 )
## process sub requests
for index in range( res["Value"] ):
self.info( "handleRequest: processing subrequest %s." % str(index) )
subRequestAttrs = self.requestObj.getSubRequestAttributes( index, self.__requestType )["Value"]
if subRequestAttrs["ExecutionOrder"]:
subExecutionOrder = int( subRequestAttrs["ExecutionOrder"] )
else:
subExecutionOrder = 0
subRequestStatus = subRequestAttrs["Status"]
if subRequestStatus != "Waiting":
self.info( "handleRequest: subrequest %s has status '%s' and is not to be executed." % ( str(index),
subRequestStatus ) )
continue
if subExecutionOrder <= self.executionOrder:
operation = subRequestAttrs["Operation"]
if operation not in self.operationDispatcher():
self.error( "handleRequest: '%s' operation not supported" % operation )
else:
self.info( "handleRequest: will execute %s '%s' subrequest" % ( str(index), operation ) )
## get files
subRequestFiles = self.requestObj.getSubRequestFiles( index, self.__requestType )["Value"]
## execute operation action
ret = self.operationDispatcher()[operation].__call__( index,
self.requestObj,
subRequestAttrs,
subRequestFiles )
################################################
## error in operation action?
if not ret["OK"]:
self.error( "handleRequest: error when handling subrequest %s: %s" % ( str(index), ret["Message"] ) )
self.requestObj.setSubRequestAttributeValue( index, self.__requestType, "Error", ret["Message"] )
else:
## update ref to requestObj
self.requestObj = ret["Value"]
## check if subrequest status == Done, disable finalisation if not
subRequestDone = self.requestObj.isSubRequestDone( index, self.__requestType )
if not subRequestDone["OK"]:
self.error( "handleRequest: unable to determine subrequest status: %s" % subRequestDone["Message"] )
else:
if not subRequestDone["Value"]:
self.warn("handleRequest: subrequest %s is not done yet" % str(index) )
################################################
# Generate the new request string after operation
newRequestString = self.requestObj.toXML()['Value']
update = self.putBackRequest( self.requestName, newRequestString )
if not update["OK"]:
self.error( "handleRequest: error when updating request: %s" % update["Message"] )
return update
## get request status
if self.jobID:
requestStatus = self.requestClient().getRequestStatus( self.requestName )
if not requestStatus["OK"]:
return requestStatus
requestStatus = requestStatus["Value"]
## finalize request if jobID is present and request status = 'Done'
self.info("handleRequest: request status is %s" % requestStatus )
if ( requestStatus["RequestStatus"] == "Done" ) and ( requestStatus["SubRequestStatus"] not in ( "Waiting", "Assigned" ) ):
self.debug("handleRequest: request is going to be finalised")
finalize = self.requestClient().finalizeRequest( self.requestName, self.jobID )
if not finalize["OK"]:
self.error("handleRequest: error in request finalization: %s" % finalize["Message"] )
return finalize
self.info("handleRequest: request is finalised")
## for gMonitor
self.addMark( "Done", 1 )
## should return S_OK with monitor dict
return S_OK( { "monitor" : self.monitor() } )
def putBackRequest( self, requestName, requestString ):
""" put request back
:param self: self reference
:param str requestName: request name
:param str requestString: XML-serilised request
:param str sourceServer: request server URL
"""
update = self.requestClient().updateRequest( requestName, requestString )
if not update["OK"]:
self.error( "putBackRequest: error when updating request: %s" % update["Message"] )
return update
return S_OK()
def executeRequest( self ):
################################################
# Get a request from request DB
gMonitor.addMark( "Iteration", 1 )
res = self.RequestDBClient.getRequest( 'transfer' )
if not res['OK']:
gLogger.info( "TransferAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "TransferAgent.execute: No requests to be executed found." )
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
try:
jobID = int( res['Value']['JobID'] )
except:
jobID = 0
gLogger.info( "TransferAgent.execute: Obtained request %s" % requestName )
result = self.RequestDBClient.getCurrentExecutionOrder( requestName, sourceServer )
if result['OK']:
currentOrder = result['Value']
else:
return S_OK( 'Can not get the request execution order' )
oRequest = RequestContainer( request = requestString )
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'transfer' )
if not res['OK']:
errStr = "TransferAgent.execute: Failed to obtain number of transfer subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "TransferAgent.execute: Found %s sub requests." % res['Value'] )
################################################
# For all the sub-requests in the request
modified = False
for ind in range( res['Value'] ):
gMonitor.addMark( "Execute", 1 )
gLogger.info( "TransferAgent.execute: Processing sub-request %s." % ind )
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'transfer' )['Value']
if subRequestAttributes['ExecutionOrder']:
subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
else:
subExecutionOrder = 0
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles( ind, 'transfer' )['Value']
operation = subRequestAttributes['Operation']
subRequestError = ''
################################################
# If the sub-request is a put and register operation
if operation == 'putAndRegister' or operation == 'putAndRegisterAndRemove':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
diracSE = str( subRequestAttributes['TargetSE'] )
catalog = ''
if subRequestAttributes.has_key( 'Catalogue' ):
catalog = subRequestAttributes['Catalogue']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Put and register", 1 )
lfn = str( subRequestFile['LFN'] )
file = subRequestFile['PFN']
guid = subRequestFile['GUID']
addler = subRequestFile['Addler']
res = self.ReplicaManager.putAndRegister( lfn, file, diracSE, guid = guid, checksum = addler, catalog = catalog )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
if not res['Value']['Successful'][lfn].has_key( 'put' ):
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Failed to put %s to %s." % ( lfn, diracSE ) )
subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
elif not res['Value']['Successful'][lfn].has_key( 'register' ):
gMonitor.addMark( "Put successful", 1 )
gMonitor.addMark( "File registration failed", 1 )
self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['put'] ) )
gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % ( lfn, diracSE ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Registration failed' )
subRequestError = "Registration failed for %s to %s" % ( lfn, diracSE )
fileDict = res['Value']['Failed'][lfn]['register']
registerRequestDict = {'Attributes':{'TargetSE': fileDict['TargetSE'], 'Operation':'registerFile'},
'Files':[{'LFN': fileDict['LFN'],
'PFN':fileDict['PFN'],
'Size':fileDict['Size'],
'Addler':fileDict['Addler'],
'GUID':fileDict['GUID']}]}
gLogger.info( "TransferAgent.execute: Setting registration request for failed file." )
oRequest.addSubRequest( registerRequestDict, 'register' )
modified = True
else:
gMonitor.addMark( "Put successful", 1 )
gMonitor.addMark( "File registration successful", 1 )
self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
self.DataLog.addFileRecord( lfn, 'Register', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['put'] ) )
gLogger.info( "TransferAgent.execute: Successfully registered %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['register'] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Failed to put and register file."
gLogger.error( errStr, "%s %s %s" % ( lfn, diracSE, res['Value']['Failed'][lfn] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Complete file failure' )
subRequestError = "Failed to put and register file"
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Completely failed to put and register file."
gLogger.error( errStr, res['Message'] )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'RM call failure' )
subRequestError = operation + " RM call file"
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a put operation
elif operation == 'put':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
diracSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Put", 1 )
lfn = subRequestFile['LFN']
file = subRequestFile['PFN']
res = self.ReplicaManager.put( lfn, file, diracSE )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
gMonitor.addMark( "Put successful", 1 )
self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Failed to put file."
gLogger.error( errStr, "%s %s %s" % ( lfn, diracSE, res['Value']['Failed'][lfn] ) )
subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Completely failed to put file."
gLogger.error( errStr, res['Message'] )
subRequestError = "Put RM call failed for %s to %s" % ( lfn, diracSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put RM call failed' )
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a replicate and register operation
elif operation == 'replicateAndRegister' or operation == 'replicateAndRegisterAndRemove':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
if sourceSE == "None":
sourceSE = ''
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Replicate and register", 1 )
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicateAndRegister( lfn, targetSE, sourceSE = sourceSE )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
if not res['Value']['Successful'][lfn].has_key( 'replicate' ):
gLogger.info( "TransferAgent.execute: Failed to replicate %s to %s." % ( lfn, targetSE ) )
gMonitor.addMark( "Replication failed", 1 )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,"Error", "Replication failed" )
subRequestError = "Replication failed for %s to %s" % ( lfn, targetSE )
elif not res['Value']['Successful'][lfn].has_key( 'register' ):
gMonitor.addMark( "Replication successful", 1 )
gMonitor.addMark( "Replica registration failed", 1 )
gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['replicate'] ) )
gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % ( lfn, targetSE ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Registration failed' )
subRequestError = "Registration failed for %s to %s" % ( lfn, targetSE )
fileDict = res['Value']['Failed'][lfn]['register']
registerRequestDict = {'Attributes':{'TargetSE': fileDict['TargetSE'], 'Operation':'registerReplica'}, 'Files':[{'LFN': fileDict['LFN'], 'PFN':fileDict['PFN']}]}
gLogger.info( "TransferAgent.execute: Setting registration request for failed replica." )
oRequest.addSubRequest( registerRequestDict, 'register' )
modified = True
else:
gMonitor.addMark( "Replication successful", 1 )
gMonitor.addMark( "Replica registration successful", 1 )
gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['replicate'] ) )
gLogger.info( "TransferAgent.execute: Successfully registered %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['register'] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Failed to replicate and register file."
gLogger.error( errStr, "%s %s %s" % ( lfn, targetSE, res['Value']['Failed'][lfn] ) )
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Completely failed to replicate and register file."
gLogger.error( errStr, res['Message'] )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'RM call failure' )
subRequestError = operation + " RM call failed"
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a replicate operation
elif operation == 'replicate':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Replicate", 1 )
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicate( lfn, targetSE, sourceSE = sourceSE )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
gMonitor.addMark( "Replication successful", 1 )
gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Failed to replicate file."
gLogger.error( errStr, "%s %s %s" % ( lfn, targetSE, res['Value']['Failed'][lfn] ) )
subRequestError = "Replicate operation failed for %s to %s" % ( lfn, targetSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Completely failed to replicate file."
gLogger.error( errStr, res['Message'] )
subRequestError = "Replicate RM call failed for %s to %s" % ( lfn, targetSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Replicate RM call failed' )
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a get operation
elif operation == 'get':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
sourceSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str( subRequestFile['LFN'] )
pfn = str( subRequestFile['PFN'] )
got = False
if sourceSE and pfn:
res = self.ReplicaManager.getStorageFile( pfn, sourceSE )
if res['Value']['Successful'].has_key( pfn ):
got = True
else:
res = self.ReplicaManager.getFile( lfn )
if res['Value']['Successful'].has_key( lfn ):
got = False
if got:
gLogger.info( "TransferAgent.execute: Successfully got %s." % lfn )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
errStr = "TransferAgent.execute: Failed to get file."
gLogger.error( errStr, lfn )
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is none of the above types
else:
gLogger.error( "TransferAgent.execute: Operation not supported.", operation )
if subRequestError:
oRequest.setSubRequestAttributeValue( ind, 'transfer', 'Error', subRequestError )
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty( ind, 'transfer' )['Value']:
oRequest.setSubRequestStatus( ind, 'transfer', 'Done' )
gMonitor.addMark( "Done", 1 )
################################################
# If the sub-request is already in terminal state
else:
gLogger.info( "TransferAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )
if modified and jobID:
result = self.finalizeRequest( requestName, jobID, sourceServer )
return S_OK()