def __cbRecvMsg(self, trid, msgObj):
msgName = msgObj.getName()
msgObj.setMsgClient(self)
for cb in self.__specialCallbacks['msg']:
try:
result = cb(self, msgObj)
if not isReturnStructure(result):
gLogger.error("Callback for message does not return S_OK/S_ERROR", msgObj.getName())
return S_ERROR("No response")
if not result['OK']:
return result
# If no specific callback but a generic one, return the generic one
if msgName not in self.__callbacks:
return result
except BaseException:
gLogger.exception("Exception while processing callbacks", msgObj.getName())
if msgName not in self.__callbacks:
return S_ERROR("Unexpected message")
try:
result = self.__callbacks[msgName](msgObj)
if not isReturnStructure(result):
gLogger.error("Callback for message does not return S_OK/S_ERROR", msgName)
return S_ERROR("No response")
return result
except BaseException:
gLogger.exception("Exception while processing callbacks", msgName)
return S_ERROR("No response")
def debug(self, sMsg, sVarMsg=''):
# In case of S_ERROR structure make full string representation
if self.__testLevel(self._logLevels.debug):
if isReturnStructure(sMsg):
sMsg = reprReturnErrorStructure(sMsg, full=True)
if isReturnStructure(sVarMsg):
sVarMsg = reprReturnErrorStructure(sVarMsg, full=True)
return self._sendMessage(self._logLevels.debug, sMsg, sVarMsg)
def debug(self, sMsg, sVarMsg=''):
# In case of S_ERROR structure make full string representation
if isReturnStructure(sMsg):
sMsg = reprReturnErrorStructure(sMsg, full=True)
if isReturnStructure(sVarMsg):
sVarMsg = reprReturnErrorStructure(sVarMsg, full=True)
messageObject = Message(self._systemName, self._logLevels.debug,
Time.dateTime(), sMsg, sVarMsg,
self.__discoverCallingFrame())
return self.processMessage(messageObject)
def debug( self, sMsg, sVarMsg = '' ):
# In case of S_ERROR structure make full string representation
if self.__testLevel( self._logLevels.debug ):
if isReturnStructure( sMsg ):
sMsg = reprReturnErrorStructure( sMsg, full = True )
if isReturnStructure( sVarMsg ):
sVarMsg = reprReturnErrorStructure( sVarMsg, full = True )
return self._sendMessage( self._logLevels.debug,
sMsg,
sVarMsg )
def debug( self, sMsg, sVarMsg = '' ):
# In case of S_ERROR structure make full string representation
if isReturnStructure( sMsg ):
sMsg = reprReturnErrorStructure( sMsg, full = True )
if isReturnStructure( sVarMsg ):
sVarMsg = reprReturnErrorStructure( sVarMsg, full = True )
messageObject = Message( self._systemName,
self._logLevels.debug,
Time.dateTime(),
sMsg,
sVarMsg,
self.__discoverCallingFrame() )
return self.processMessage( messageObject )
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._initMonitoring()
self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()),
max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {'serviceName': self._name,
'serviceSectionPath': PathFinder.getServiceSection(self._name),
'URL': self._cfg.getURL(),
'messageSender': MessageSender(self._name, self._msgBroker),
'validNames': self._validNames,
'csPaths': [PathFinder.getServiceSection(svcName) for svcName in self._validNames]
}
# Call static initialization function
try:
self._handler['class']._rh__initializeClass(dict(self._serviceInfoDict),
self._lockManager,
self._msgBroker,
self._monitor)
if self._handler['init']:
for initFunc in self._handler['init']:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception("Exception while calling initialization function", lException=excp)
return S_ERROR("Exception while calling initialization function: %s" % str(excp))
if not isReturnStructure(result):
return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc)
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" % (self._name, result['Message']))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result['OK']:
return result
self._actions = result['Value']
gThreadScheduler.addPeriodicTask(30, self.__reportThreadPoolContents)
return S_OK()
def _ex_initialize(cls, exeName, loadName):
cls.__properties = {
"fullName": exeName,
"loadName": loadName,
"section": PathFinder.getExecutorSection(exeName),
"loadSection": PathFinder.getExecutorSection(loadName),
"messagesProcessed": 0,
"reconnects": 0,
"setup": gConfig.getValue("/DIRAC/Setup", "Unknown"),
}
cls.__defaults = {}
cls.__defaults["MonitoringEnabled"] = True
cls.__defaults["Enabled"] = True
cls.__defaults["ControlDirectory"] = os.path.join(rootPath, "control", *exeName.split("/"))
cls.__defaults["WorkDirectory"] = os.path.join(rootPath, "work", *exeName.split("/"))
cls.__defaults["ReconnectRetries"] = 10
cls.__defaults["ReconnectSleep"] = 5
cls.__defaults["shifterProxy"] = ""
cls.__defaults["shifterProxyLocation"] = os.path.join(cls.__defaults["WorkDirectory"], ".shifterCred")
cls.__properties["shifterProxy"] = ""
cls.__properties["shifterProxyLocation"] = os.path.join(cls.__defaults["WorkDirectory"], ".shifterCred")
cls.__mindName = False
cls.__mindExtraArgs = False
cls.__freezeTime = 0
cls.__fastTrackEnabled = True
cls.log = gLogger.getSubLogger(exeName)
try:
result = cls.initialize() # pylint: disable=no-member
except Exception as excp:
gLogger.exception("Exception while initializing %s" % loadName, lException=excp)
return S_ERROR("Exception while initializing: %s" % str(excp))
if not isReturnStructure(result):
return S_ERROR("Executor %s does not return an S_OK/S_ERROR after initialization" % loadName)
return result
def initialize( self ):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % self._name )
gLogger.verbose( "Service URL is %s" % self._url )
#Load handler
result = self._loadHandlerInit()
if not result[ 'OK' ]:
return result
self._handler = result[ 'Value' ]
#Initialize lock manager
self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() )
self._initMonitoring()
self._threadPool = ThreadPool( max( 1, self._cfg.getMinThreads() ),
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool )
#Create static dict
self._serviceInfoDict = { 'serviceName' : self._name,
'serviceSectionPath' : PathFinder.getServiceSection( self._name ),
'URL' : self._cfg.getURL(),
'messageSender' : MessageSender( self._name, self._msgBroker ),
'validNames' : self._validNames,
'csPaths' : [ PathFinder.getServiceSection( svcName ) for svcName in self._validNames ]
}
#Call static initialization function
try:
self._handler[ 'class' ]._rh__initializeClass( dict( self._serviceInfoDict ),
self._lockManager,
self._msgBroker,
self._monitor )
if self._handler[ 'init' ]:
for initFunc in self._handler[ 'init' ]:
gLogger.verbose( "Executing initialization function" )
try:
result = initFunc( dict( self._serviceInfoDict ) )
except Exception as excp:
gLogger.exception( "Exception while calling initialization function", lException = excp )
return S_ERROR( "Exception while calling initialization function: %s" % str( excp ) )
if not isReturnStructure( result ):
return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc )
if not result[ 'OK' ]:
return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) )
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception( e )
gLogger.exception( errMsg )
return S_ERROR( errMsg )
#Load actions after the handler has initialized itself
result = self._loadActions()
if not result[ 'OK' ]:
return result
self._actions = result[ 'Value' ]
gThreadScheduler.addPeriodicTask( 30, self.__reportThreadPoolContents )
return S_OK()
def _rh_executeMessageCallback( self, msgObj ):
msgName = msgObj.getName()
if not self.__msgBroker.getMsgFactory().messageExists( self.__svcName, msgName ):
return S_ERROR( "Unknown message %s" % msgName )
methodName = "msg_%s" % msgName
self.__logRemoteQuery( "Message/%s" % methodName, msgObj.dumpAttrs() )
startTime = time.time()
try:
oMethod = getattr( self, methodName )
except:
return S_ERROR( "Handler function for message %s does not exist!" % msgName )
self.__lockManager.lock( methodName )
try:
try:
uReturnValue = oMethod( msgObj )
except Exception, v:
gLogger.exception( "Uncaught exception when serving message", methodName )
return S_ERROR( "Server error while serving %s: %s" % ( msgName, str( v ) ) )
finally:
self.__lockManager.unlock( methodName )
if not isReturnStructure( uReturnValue ):
gLogger.error( "Message does not return a S_OK/S_ERROR", msgName )
uReturnValue = S_ERROR( "Message %s does not return a S_OK/S_ERROR" % msgName )
self.__logRemoteQueryResponse( uReturnValue, time.time() - startTime )
return uReturnValue
def _rh_executeAction(self, proposalTuple):
"""
Execute an action.
:type proposalTuple: tuple
:param proposalTuple: Type of action to execute. First position of the tuple must be the type
of action to execute. The second position is the action itself.
"""
actionTuple = proposalTuple[1]
gLogger.debug("Executing %s:%s action" % tuple(actionTuple))
startTime = time.time()
actionType = actionTuple[0]
self.serviceInfoDict['actionTuple'] = actionTuple
try:
if actionType == "RPC":
retVal = self.__doRPC(actionTuple[1])
elif actionType == "FileTransfer":
retVal = self.__doFileTransfer(actionTuple[1])
elif actionType == "Connection":
retVal = self.__doConnection(actionTuple[1])
else:
return S_ERROR("Unknown action %s" % actionType)
except RequestHandler.ConnectionError as excp:
gLogger.error("ConnectionError", str(excp))
return S_ERROR(excp)
if not isReturnStructure(retVal):
message = "Method %s for action %s does not return a S_OK/S_ERROR!" % (actionTuple[1], actionTuple[0])
gLogger.error(message)
retVal = S_ERROR(message)
elapsedTime = time.time() - startTime
self.__logRemoteQueryResponse(retVal, elapsedTime)
result = self.__trPool.send(self.__trid, retVal) # this will delete the value from the S_OK(value)
del retVal
return S_OK([result, elapsedTime])
def _rh_executeMessageCallback(self, msgObj):
msgName = msgObj.getName()
if not self.__msgBroker.getMsgFactory().messageExists(
self.__svcName, msgName):
return S_ERROR("Unknown message %s" % msgName)
methodName = "msg_%s" % msgName
self.__logRemoteQuery("Message/%s" % methodName, msgObj.dumpAttrs())
startTime = time.time()
try:
oMethod = getattr(self, methodName)
except:
return S_ERROR("Handler function for message %s does not exist!" %
msgName)
self.__lockManager.lock(methodName)
try:
try:
uReturnValue = oMethod(msgObj)
except Exception, v:
gLogger.exception("Uncaught exception when serving message",
methodName)
return S_ERROR("Server error while serving %s: %s" %
(msgName, str(v)))
finally:
self.__lockManager.unlock(methodName)
if not isReturnStructure(uReturnValue):
gLogger.error("Message does not return a S_OK/S_ERROR", msgName)
uReturnValue = S_ERROR(
"Message %s does not return a S_OK/S_ERROR" % msgName)
self.__logRemoteQueryResponse(uReturnValue, time.time() - startTime)
return uReturnValue
def __sendTask(self, taskId, taskObj, eId, eType):
try:
result = self.exec_prepareToSend(taskId, taskObj, eId)
if not result['OK']:
return result
except Exception as excp:
gLogger.exception("Exception while executing prepareToSend: %s" % str(excp), lException=excp)
return S_ERROR("Cannot presend task")
try:
result = self.exec_serializeTask(taskObj)
except Exception as excp:
gLogger.exception("Exception while serializing task %s" % taskId, lException=excp)
return S_ERROR("Cannot serialize task %s: %s" % (taskId, str(excp)))
if not isReturnStructure(result):
raise Exception("exec_serializeTask does not return a return structure")
if not result['OK']:
return result
taskStub = result['Value']
result = self.srv_msgCreate("ProcessTask")
if not result['OK']:
return result
msgObj = result['Value']
msgObj.taskId = taskId
msgObj.taskStub = taskStub
msgObj.eType = eType
return self.srv_msgSend(eId, msgObj)
def __getNextExecutor(self, taskId):
try:
eTask = self.__tasks[taskId]
except IndexError:
msg = "Task %s was deleted prematurely while being dispatched" % taskId
self.__log.error(
"Task was deleted prematurely while being dispatched",
"%s" % taskId)
return S_ERROR(msg)
try:
result = self.__cbHolder.cbDispatch(taskId, eTask.taskObj,
tuple(eTask.pathExecuted))
except BaseException:
self.__log.exception("Exception while calling dispatch callback")
return S_ERROR("Exception while calling dispatch callback")
if not isReturnStructure(result):
errMsg = "Dispatch callback did not return a S_OK/S_ERROR structure"
self.__log.fatal(errMsg)
return S_ERROR(errMsg)
# Assign the next executor type to the task
if result['OK']:
eTask.eType = result['Value']
return result
def __sendTask( self, taskId, taskObj, eId, eType ):
try:
result = self.exec_prepareToSend( taskId, taskObj, eId )
if not result[ 'OK' ]:
return result
except Exception as excp:
gLogger.exception( "Exception while executing prepareToSend: %s" % str( excp ), lException = excp )
return S_ERROR( "Cannot presend task" )
try:
result = self.exec_serializeTask( taskObj )
except Exception as excp:
gLogger.exception( "Exception while serializing task %s" % taskId, lException = excp )
return S_ERROR( "Cannot serialize task %s: %s" % ( taskId, str( excp ) ) )
if not isReturnStructure( result ):
raise Exception( "exec_serializeTask does not return a return structure" )
if not result[ 'OK' ]:
return result
taskStub = result[ 'Value' ]
result = self.srv_msgCreate( "ProcessTask" )
if not result[ 'OK' ]:
return result
msgObj = result[ 'Value' ]
msgObj.taskId = taskId
msgObj.taskStub = taskStub
msgObj.eType = eType
return self.srv_msgSend( eId, msgObj )
def msg_TaskError(self, msgObj):
taskId = msgObj.taskId
try:
result = self.exec_deserializeTask(msgObj.taskStub)
except Exception as excp:
gLogger.exception("Exception while deserializing task",
taskId,
lException=excp)
return S_ERROR("Cannot deserialize task %s: %s" %
(taskId, str(excp)))
if not isReturnStructure(result):
raise Exception(
"exec_deserializeTask does not return a return structure")
if not result["OK"]:
return result
taskObj = result["Value"]
# TODO: Check the executor has privileges over the task
self.__eDispatch.removeTask(msgObj.taskId)
try:
self.exec_taskError(msgObj.taskId, taskObj, msgObj.errorMsg)
except Exception as excp:
gLogger.exception("Exception when processing task",
msgObj.taskId,
lException=excp)
return S_OK()
def am_initialize(self, *initArgs):
""" Common initialization for all the agents.
This is executed every time an agent (re)starts.
This is called by the AgentReactor, should not be overridden.
"""
agentName = self.am_getModuleParam('fullName')
result = self.initialize(*initArgs)
if not isReturnStructure(result):
return S_ERROR("initialize must return S_OK/S_ERROR")
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" %
(agentName, result['Message']))
mkDir(self.am_getControlDirectory())
workDirectory = self.am_getWorkDirectory()
mkDir(workDirectory)
# Set the work directory in an environment variable available to subprocesses if needed
os.environ['AGENT_WORKDIRECTORY'] = workDirectory
self.__moduleProperties['shifterProxy'] = self.am_getOption(
'shifterProxy')
if self.am_monitoringEnabled() and not self.activityMonitoring:
self.monitor.enable()
if len(self.__moduleProperties['executors']) < 1:
return S_ERROR("At least one executor method has to be defined")
if not self.am_Enabled():
return S_ERROR("Agent is disabled via the configuration")
self.log.notice("=" * 40)
self.log.notice("Loaded agent module %s" %
self.__moduleProperties['fullName'])
self.log.notice(" Site: %s" % DIRAC.siteName())
self.log.notice(" Setup: %s" % gConfig.getValue("/DIRAC/Setup"))
self.log.notice(" Base Module version: %s " % __RCSID__)
self.log.notice(" Agent version: %s" %
self.__codeProperties['version'])
self.log.notice(" DIRAC version: %s" % DIRAC.version)
self.log.notice(" DIRAC platform: %s" % DIRAC.getPlatform())
pollingTime = int(self.am_getOption('PollingTime'))
if pollingTime > 3600:
self.log.notice(" Polling time: %s hours" % (pollingTime / 3600.))
else:
self.log.notice(" Polling time: %s seconds" %
self.am_getOption('PollingTime'))
self.log.notice(" Control dir: %s" % self.am_getControlDirectory())
self.log.notice(" Work dir: %s" % self.am_getWorkDirectory())
if self.am_getOption('MaxCycles') > 0:
self.log.notice(" Cycles: %s" % self.am_getMaxCycles())
else:
self.log.notice(" Cycles: unlimited")
if self.am_getWatchdogTime() > 0:
self.log.notice(" Watchdog interval: %s" %
self.am_getWatchdogTime())
else:
self.log.notice(" Watchdog interval: disabled ")
self.log.notice("=" * 40)
self.__initialized = True
return S_OK()
def __deserialize( self, taskId, taskStub ):
try:
result = self.deserializeTask( taskStub )
except Exception as excp:
gLogger.exception( "Exception while deserializing task %s" % taskId, lException = excp )
return S_ERROR( "Cannot deserialize task %s: %s" % ( taskId, str( excp ) ) )
if not isReturnStructure( result ):
raise Exception( "deserializeTask does not return a return structure" )
return result
def __processIncomingResponse( self, trid, msg ):
#This is a message response
for requiredField in ( 'id', 'result' ):
if requiredField not in msg:
gLogger.error( "Message does not have %s" % requiredField )
return S_ERROR( "Message does not have %s" % requiredField )
if not isReturnStructure( msg[ 'result' ] ):
return S_ERROR( "Message response did not return a result structure" )
return self.__notifyCallback( msg[ 'id' ], msg[ 'result' ] )
def __deserialize(self, taskId, taskStub):
try:
result = self.deserializeTask(taskStub)
except Exception as excp:
gLogger.exception("Exception while deserializing task %s" % taskId, lException=excp)
return S_ERROR("Cannot deserialize task %s: %s" % (taskId, str(excp)))
if not isReturnStructure(result):
raise Exception("deserializeTask does not return a return structure")
return result
def __processIncomingResponse(self, trid, msg):
# This is a message response
for requiredField in ("id", "result"):
if requiredField not in msg:
gLogger.error("Message does not have %s" % requiredField)
return S_ERROR("Message does not have %s" % requiredField)
if not isReturnStructure(msg["result"]):
return S_ERROR("Message response did not return a result structure")
return self.__notifyCallback(msg["id"], msg["result"])
def am_secureCall( self, functor, args = (), name = False ):
if not name:
name = str( functor )
try:
result = functor( *args )
if not isReturnStructure( result ):
raise Exception( "%s method for %s module has to return S_OK/S_ERROR" % ( name, self.__moduleProperties[ 'fullName' ] ) )
return result
except Exception as e:
self.log.exception( "Agent exception while calling method", name )
return S_ERROR( "Exception while calling %s method: %s" % ( name, str( e ) ) )
def __processIncomingRequest(self, trid, msg):
self.__trInOutLock.acquire()
try:
rcvCB = self.__messageTransports[trid]["cbReceiveMessage"]
except KeyError:
return S_ERROR("Transport %s unknown" % trid)
finally:
self.__trInOutLock.release()
if not rcvCB:
gLogger.fatal(
"Transport %s does not have a callback defined and a message arrived!"
% trid)
return S_ERROR("No message was expected in for this transport")
# Check message has id and name
for requiredField in ["name"]:
if requiredField not in msg:
gLogger.error("Message does not have required field",
requiredField)
return S_ERROR("Message does not have %s" % requiredField)
# Load message
if "attrs" in msg:
attrs = msg["attrs"]
if not isinstance(attrs, (tuple, list)):
return S_ERROR(
"Message args has to be a tuple or a list, not %s" %
type(attrs))
else:
attrs = None
# Do we "unpack" or do we send the raw data to the callback?
if self.__useMessageObjects:
result = self.__msgFactory.createMessage(
self.__messageTransports[trid]["svcName"], msg["name"], attrs)
if not result["OK"]:
return result
msgObj = result["Value"]
else:
msgObj = DummyMessage(msg)
# Is msg ok?
if not msgObj.isOK():
return S_ERROR("Messsage is invalid")
try:
# Callback it and return response
result = rcvCB(trid, msgObj)
if not isReturnStructure(result):
return S_ERROR(
"Request function does not return a result structure")
return result
except Exception as e:
# Whoops. Show exception and return
gLogger.exception("Exception while processing message %s" %
msg["name"],
lException=e)
return S_ERROR("Exception while processing message %s: %s" %
(msg["name"], str(e)))
def __msgTaskToExecutor(self, taskId, eId, eType):
self.__tasks[taskId].sendTime = time.time()
result = self.__cbHolder.cbSendTask(taskId,
self.__tasks[taskId].taskObj, eId,
eType)
if not isReturnStructure(result):
errMsg = "Send task callback did not send back an S_OK/S_ERROR structure"
self.__log.fatal(errMsg)
raise ValueError(errMsg)
if not result["OK"]:
self.__log.error("Failed to cbSendTask", "%r" % result)
raise RuntimeError(result)
def __taskFreezeCallback(self, taskId, taskObj, eType):
try:
result = self.__cbHolder.cbTaskFreeze(taskId, taskObj, eType)
except BaseException:
self.__log.exception("Exception while calling taskFreeze callback")
return S_ERROR("Exception while calling taskFreeze callback")
if not isReturnStructure(result):
errMsg = "taskFreeze callback did not return a S_OK/S_ERROR structure"
self.__log.fatal(errMsg)
return S_ERROR(errMsg)
return result
def __taskFreezeCallback( self, taskId, taskObj, eType ):
try:
result = self.__cbHolder.cbTaskFreeze( taskId, taskObj, eType )
except:
self.__log.exception( "Exception while calling taskFreeze callback" )
return S_ERROR( "Exception while calling taskFreeze callback" )
if not isReturnStructure( result ):
errMsg = "taskFreeze callback did not return a S_OK/S_ERROR structure"
self.__log.fatal( errMsg )
return S_ERROR( errMsg )
return result
def am_initialize(self, *initArgs):
""" Common initialization for all the agents.
This is executed every time an agent (re)starts.
This is called by the AgentReactor, should not be overridden.
"""
agentName = self.am_getModuleParam('fullName')
result = self.initialize(*initArgs)
if not isReturnStructure(result):
return S_ERROR("initialize must return S_OK/S_ERROR")
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" % (agentName, result['Message']))
mkDir(self.am_getControlDirectory())
workDirectory = self.am_getWorkDirectory()
mkDir(workDirectory)
# Set the work directory in an environment variable available to subprocesses if needed
os.environ['AGENT_WORKDIRECTORY'] = workDirectory
self.__moduleProperties['shifterProxy'] = self.am_getOption('shifterProxy')
if self.am_monitoringEnabled():
self.monitor.enable()
if len(self.__moduleProperties['executors']) < 1:
return S_ERROR("At least one executor method has to be defined")
if not self.am_Enabled():
return S_ERROR("Agent is disabled via the configuration")
self.log.notice("=" * 40)
self.log.notice("Loaded agent module %s" % self.__moduleProperties['fullName'])
self.log.notice(" Site: %s" % DIRAC.siteName())
self.log.notice(" Setup: %s" % gConfig.getValue("/DIRAC/Setup"))
self.log.notice(" Base Module version: %s " % __RCSID__)
self.log.notice(" Agent version: %s" % self.__codeProperties['version'])
self.log.notice(" DIRAC version: %s" % DIRAC.version)
self.log.notice(" DIRAC platform: %s" % DIRAC.getPlatform())
pollingTime = int(self.am_getOption('PollingTime'))
if pollingTime > 3600:
self.log.notice(" Polling time: %s hours" % (pollingTime / 3600.))
else:
self.log.notice(" Polling time: %s seconds" % self.am_getOption('PollingTime'))
self.log.notice(" Control dir: %s" % self.am_getControlDirectory())
self.log.notice(" Work dir: %s" % self.am_getWorkDirectory())
if self.am_getOption('MaxCycles') > 0:
self.log.notice(" Cycles: %s" % self.am_getMaxCycles())
else:
self.log.notice(" Cycles: unlimited")
if self.am_getWatchdogTime() > 0:
self.log.notice(" Watchdog interval: %s" % self.am_getWatchdogTime())
else:
self.log.notice(" Watchdog interval: disabled ")
self.log.notice("=" * 40)
self.__initialized = True
return S_OK()
def am_secureCall(self, functor, args=(), name=False):
if not name:
name = str(functor)
try:
result = functor(*args)
if not isReturnStructure(result):
raise Exception(
"%s method for %s module has to return S_OK/S_ERROR" %
(name, self.__moduleProperties['fullName']))
return result
except Exception, e:
self.log.exception("Agent exception while calling method", name)
return S_ERROR("Exception while calling %s method: %s" %
(name, str(e)))
def __msgTaskToExecutor(self, taskId, eId, eType):
try:
self.__tasks[taskId].sendTime = time.time()
except KeyError:
return S_ERROR("Task %s has been deleted" % taskId)
try:
result = self.__cbHolder.cbSendTask(taskId, self.__tasks[taskId].taskObj, eId, eType)
except BaseException:
self.__log.exception("Exception while sending task to executor")
return S_ERROR("Exception while sending task to executor")
if isReturnStructure(result):
return result
errMsg = "Send task callback did not send back an S_OK/S_ERROR structure"
self.__log.fatal(errMsg)
return S_ERROR(errMsg)
def initialize( self ):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % self._name )
gLogger.verbose( "Service URL is %s" % self._url )
#Discover Handler
self._handlerLocation = self._discoverHandlerLocation()
if not self._handlerLocation:
return S_ERROR( "Could not find handler location for %s" % self._name )
gLogger.verbose( "Handler found at %s" % self._handlerLocation )
#Load handler
result = self._loadHandler()
if not result[ 'OK' ]:
return result
self._handler = result[ 'Value' ]
#Initialize lock manager
self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() )
#Load actions
result = self._loadActions()
if not result[ 'OK' ]:
return result
self._actions = result[ 'Value' ]
self._initMonitoring()
self._threadPool = ThreadPool( 1,
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool )
#Create static dict
self._serviceInfoDict = { 'serviceName' : self._name,
'URL' : self._cfg.getURL(),
'systemSectionPath' : self._cfg.getSystemPath(),
'serviceSectionPath' : self._cfg.getServicePath(),
'messageSender' : MessageSender( self._msgBroker )
}
#Call static initialization function
try:
if self._handler[ 'init' ]:
result = self._handler[ 'init' ]( dict( self._serviceInfoDict ) )
if not isReturnStructure( result ):
return S_ERROR( "Service initialization function must return S_OK/S_ERROR" )
if not result[ 'OK' ]:
return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) )
except Exception, e:
errMsg = "Exception while intializing %s" % self._name
gLogger.exception( errMsg )
return S_ERROR( errMsg )
def msg_TaskDone( self, msgObj ):
taskId = msgObj.taskId
try:
result = self.exec_deserializeTask( msgObj.taskStub )
except Exception as excp:
gLogger.exception( "Exception while deserializing task %s" % taskId, lException = excp )
return S_ERROR( "Cannot deserialize task %s: %s" % ( taskId, str( excp ) ) )
if not isReturnStructure( result ):
raise Exception( "exec_deserializeTask does not return a return structure" )
if not result[ 'OK' ]:
return result
taskObj = result[ 'Value' ]
result = self.__eDispatch.taskProcessed( self.srv_getTransportID(), msgObj.taskId, taskObj )
if not result[ 'OK' ]:
gLogger.error( "There was a problem processing task", "%s: %s" % ( taskId, result[ 'Message' ] ) )
return S_OK()
def msg_TaskDone(self, msgObj):
taskId = msgObj.taskId
try:
result = self.exec_deserializeTask(msgObj.taskStub)
except Exception as excp:
gLogger.exception("Exception while deserializing task %s" % taskId, lException=excp)
return S_ERROR("Cannot deserialize task %s: %s" % (taskId, str(excp)))
if not isReturnStructure(result):
raise Exception("exec_deserializeTask does not return a return structure")
if not result['OK']:
return result
taskObj = result['Value']
result = self.__eDispatch.taskProcessed(self.srv_getTransportID(), msgObj.taskId, taskObj)
if not result['OK']:
gLogger.error("There was a problem processing task", "%s: %s" % (taskId, result['Message']))
return S_OK()
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result["OK"]:
return result
self._handler = result["Value"]
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._initMonitoring()
self._threadPool = ThreadPool(1, max(0, self._cfg.getMaxThreads()), self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {
"serviceName": self._name,
"serviceSectionPath": PathFinder.getServiceSection(self._name),
"URL": self._cfg.getURL(),
"messageSender": MessageSender(self._name, self._msgBroker),
"validNames": self._validNames,
"csPaths": [PathFinder.getServiceSection(svcName) for svcName in self._validNames],
}
# Call static initialization function
try:
self._handler["class"]._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager, self._msgBroker, self._monitor
)
if self._handler["init"]:
for initFunc in self._handler["init"]:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception, excp:
gLogger.exception("Exception while calling initialization function")
return S_ERROR("Exception while calling initialization function: %s" % str(excp))
if not isReturnStructure(result):
return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc)
if not result["OK"]:
return S_ERROR("Error while initializing %s: %s" % (self._name, result["Message"]))
except Exception, e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(errMsg)
return S_ERROR(errMsg)
class ExecutorModule(object):
""" All executors should inherit from this module
"""
@classmethod
def _ex_initialize(cls, exeName, loadName):
cls.__properties = {
'fullName': exeName,
'loadName': loadName,
'section': PathFinder.getExecutorSection(exeName),
'loadSection': PathFinder.getExecutorSection(loadName),
'messagesProcessed': 0,
'reconnects': 0,
'setup': gConfig.getValue("/DIRAC/Setup", "Unknown")
}
cls.__basePath = gConfig.getValue('/LocalSite/InstancePath', rootPath)
cls.__defaults = {}
cls.__defaults['MonitoringEnabled'] = True
cls.__defaults['Enabled'] = True
cls.__defaults['ControlDirectory'] = os.path.join(
cls.__basePath, 'control', *exeName.split("/"))
cls.__defaults['WorkDirectory'] = os.path.join(cls.__basePath, 'work',
*exeName.split("/"))
cls.__defaults['ReconnectRetries'] = 10
cls.__defaults['ReconnectSleep'] = 5
cls.__defaults['shifterProxy'] = ''
cls.__defaults['shifterProxyLocation'] = os.path.join(
cls.__defaults['WorkDirectory'], '.shifterCred')
cls.__properties['shifterProxy'] = ''
cls.__properties['shifterProxyLocation'] = os.path.join(
cls.__defaults['WorkDirectory'], '.shifterCred')
cls.__mindName = False
cls.__mindExtraArgs = False
cls.__freezeTime = 0
cls.__fastTrackEnabled = True
cls.log = gLogger.getSubLogger(exeName, child=False)
try:
result = cls.initialize()
except Exception, excp:
gLogger.exception("Exception while initializing %s" % loadName)
return S_ERROR("Exception while initializing: %s" % str(excp))
if not isReturnStructure(result):
return S_ERROR(
"Executor %s does not return an S_OK/S_ERROR after initialization"
% loadName)
return result
def __msgTaskToExecutor(self, taskId, eId, eType):
try:
self.__tasks[taskId].sendTime = time.time()
except KeyError:
return S_ERROR("Task %s has been deleted" % taskId)
try:
result = self.__cbHolder.cbSendTask(taskId,
self.__tasks[taskId].taskObj,
eId, eType)
except BaseException:
self.__log.exception("Exception while sending task to executor")
return S_ERROR("Exception while sending task to executor")
if isReturnStructure(result):
return result
errMsg = "Send task callback did not send back an S_OK/S_ERROR structure"
self.__log.fatal(errMsg)
return S_ERROR(errMsg)
def __processIncomingRequest(self, trid, msg):
self.__trInOutLock.acquire()
try:
rcvCB = self.__messageTransports[trid]["cbReceiveMessage"]
except KeyError:
return S_ERROR("Transport %s unknown" % trid)
finally:
self.__trInOutLock.release()
if not rcvCB:
gLogger.fatal("Transport %s does not have a callback defined and a message arrived!" % trid)
return S_ERROR("No message was expected in for this transport")
# Check message has id and name
for requiredField in ["name"]:
if requiredField not in msg:
gLogger.error("Message does not have %s" % requiredField)
return S_ERROR("Message does not have %s" % requiredField)
# Load message
if "attrs" in msg:
attrs = msg["attrs"]
if type(attrs) not in (types.TupleType, types.ListType):
return S_ERROR("Message args has to be a tuple or a list, not %s" % type(args))
else:
attrs = None
# Do we "unpack" or do we send the raw data to the callback?
if self.__useMessageObjects:
result = self.__msgFactory.createMessage(self.__messageTransports[trid]["svcName"], msg["name"], attrs)
if not result["OK"]:
return result
msgObj = result["Value"]
else:
msgObj = DummyMessage(msg)
# Is msg ok?
if not msgObj.isOK():
return S_ERROR("Messsage is invalid")
try:
# Callback it and return response
result = rcvCB(trid, msgObj)
if not isReturnStructure(result):
return S_ERROR("Request function does not return a result structure")
return result
except Exception, e:
# Whoops. Show exception and return
gLogger.exception("Exception while processing message %s" % msg["name"])
return S_ERROR("Exception while processing message %s: %s" % (msg["name"], str(e)))
def am_initialize(self, *initArgs):
agentName = self.am_getModuleParam('fullName')
result = self.initialize(*initArgs)
if not isReturnStructure(result):
return S_ERROR("initialize must return S_OK/S_ERROR")
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" %
(agentName, result['Message']))
_checkDir(self.am_getControlDirectory())
_checkDir(self.am_getWorkDirectory())
self.__moduleProperties['shifterProxy'] = self.am_getOption(
'shifterProxy')
if self.am_monitoringEnabled():
self.monitor.enable()
if len(self.__moduleProperties['executors']) < 1:
return S_ERROR("At least one executor method has to be defined")
if not self.am_Enabled():
return S_ERROR("Agent is disabled via the configuration")
self.log.notice("=" * 40)
self.log.notice("Loaded agent module %s" %
self.__moduleProperties['fullName'])
self.log.notice(" Site: %s" % DIRAC.siteName())
self.log.notice(" Setup: %s" % gConfig.getValue("/DIRAC/Setup"))
self.log.notice(" Base Module version: %s " % __RCSID__)
self.log.notice(" Agent version: %s" %
self.__codeProperties['version'])
self.log.notice(" DIRAC version: %s" % DIRAC.version)
self.log.notice(" DIRAC platform: %s" % DIRAC.platform)
pollingTime = int(self.am_getOption('PollingTime'))
if pollingTime > 3600:
self.log.notice(" Polling time: %s hours" % (pollingTime / 3600.))
else:
self.log.notice(" Polling time: %s seconds" %
self.am_getOption('PollingTime'))
self.log.notice(" Control dir: %s" % self.am_getControlDirectory())
self.log.notice(" Work dir: %s" % self.am_getWorkDirectory())
if self.am_getOption('MaxCycles') > 0:
self.log.notice(" Cycles: %s" % self.am_getMaxCycles())
else:
self.log.notice(" Cycles: unlimited")
self.log.notice("=" * 40)
self.__initialized = True
return S_OK()
def msg_TaskError( self, msgObj ):
taskId = msgObj.taskId
try:
result = self.exec_deserializeTask( msgObj.taskStub )
except Exception as excp:
gLogger.exception( "Exception while deserializing task %s" % taskId, lException = excp )
return S_ERROR( "Cannot deserialize task %s: %s" % ( taskId, str( excp ) ) )
if not isReturnStructure( result ):
raise Exception( "exec_deserializeTask does not return a return structure" )
if not result[ 'OK' ]:
return result
taskObj = result[ 'Value' ]
#TODO: Check the executor has privileges over the task
self.__eDispatch.removeTask( msgObj.taskId )
try:
self.exec_taskError( msgObj.taskId, taskObj, msgObj.errorMsg )
except Exception as excp:
gLogger.exception( "Exception when processing task %s" % msgObj.taskId, lException = excp )
return S_OK()
def _ex_processTask(self, taskId, taskStub):
self.__properties['shifterProxy'] = self.ex_getOption('shifterProxy')
self.__freezeTime = 0
self.__fastTrackEnabled = True
self.log.verbose("Task %s: Received" % str(taskId))
result = self.__deserialize(taskId, taskStub)
if not result['OK']:
self.log.error("Can not deserialize task",
"Task %s: %s" % (str(taskId), result['Message']))
return result
taskObj = result['Value']
# Shifter proxy?
result = self.__installShifterProxy()
if not result['OK']:
return result
# Execute!
result = self.processTask(taskId, taskObj)
if not isReturnStructure(result):
raise Exception("processTask does not return a return structure")
if not result['OK']:
return result
# If there's a result, serialize it again!
if result['Value']:
taskObj = result['Value']
# Serialize again
result = self.__serialize(taskId, taskObj)
if not result['OK']:
self.log.verbose("Task %s: Cannot serialize: %s" %
(str(taskId), result['Message']))
return result
taskStub = result['Value']
# Try fast track
fastTrackType = False
if not self.__freezeTime and self.__fastTrackEnabled:
result = self.fastTrackDispatch(taskId, taskObj)
if not result['OK']:
self.log.error("FastTrackDispatch failed for job",
"%s: %s" % (taskId, result['Message']))
else:
fastTrackType = result['Value']
# EOP
return S_OK((taskStub, self.__freezeTime, fastTrackType))
def __msgTaskToExecutor( self, taskId, eId, eType ):
try:
self.__tasks[ taskId ].sendTime = time.time()
except KeyError:
return S_ERROR( "Task %s has been deleted" % taskId )
try:
result = self.__cbHolder.cbSendTask( taskId, self.__tasks[ taskId ].taskObj, eId, eType )
except:
self.__log.exception( "Exception while sending task to executor" )
return S_ERROR( "Exception while sending task to executor" )
if isReturnStructure( result ):
return result
else:
errMsg = "Send task callback did not send back an S_OK/S_ERROR structure"
self.__log.fatal( errMsg )
return S_ERROR( "Send task callback did not send back an S_OK/S_ERROR structure" )
#Seems an executor problem
self.__log.verbose( "Disconnecting executor" )
self.removeExecutor( eId )
return S_ERROR( "Exception while sending task to executor" )
def __msgTaskToExecutor( self, taskId, eId, eType ):
try:
self.__tasks[ taskId ].sendTime = time.time()
except KeyError:
return S_ERROR( "Task %s has been deleted" % taskId )
try:
result = self.__cbHolder.cbSendTask( taskId, self.__tasks[ taskId ].taskObj, eId, eType )
except:
self.__log.exception( "Exception while sending task to executor" )
return S_ERROR( "Exception while sending task to executor" )
if isReturnStructure( result ):
return result
else:
errMsg = "Send task callback did not send back an S_OK/S_ERROR structure"
self.__log.fatal( errMsg )
return S_ERROR( "Send task callback did not send back an S_OK/S_ERROR structure" )
#Seems an executor problem
self.__log.verbose( "Disconnecting executor" )
self.removeExecutor( eId )
return S_ERROR( "Exception while sending task to executor" )
def _ex_initialize( cls, exeName, loadName ):
cls.__properties = { 'fullName' : exeName,
'loadName' : loadName,
'section' : PathFinder.getExecutorSection( exeName ),
'loadSection' : PathFinder.getExecutorSection( loadName ),
'messagesProcessed' : 0,
'reconnects' : 0,
'setup' : gConfig.getValue( "/DIRAC/Setup", "Unknown" ) }
cls.__basePath = gConfig.getValue( '/LocalSite/InstancePath', rootPath )
cls.__defaults = {}
cls.__defaults[ 'MonitoringEnabled' ] = True
cls.__defaults[ 'Enabled' ] = True
cls.__defaults[ 'ControlDirectory' ] = os.path.join( cls.__basePath,
'control',
*exeName.split( "/" ) )
cls.__defaults[ 'WorkDirectory' ] = os.path.join( cls.__basePath,
'work',
*exeName.split( "/" ) )
cls.__defaults[ 'ReconnectRetries' ] = 10
cls.__defaults[ 'ReconnectSleep' ] = 5
cls.__defaults[ 'shifterProxy' ] = ''
cls.__defaults[ 'shifterProxyLocation' ] = os.path.join( cls.__defaults[ 'WorkDirectory' ],
'.shifterCred' )
cls.__properties[ 'shifterProxy' ] = ''
cls.__properties[ 'shifterProxyLocation' ] = os.path.join( cls.__defaults[ 'WorkDirectory' ],
'.shifterCred' )
cls.__mindName = False
cls.__mindExtraArgs = False
cls.__freezeTime = 0
cls.__fastTrackEnabled = True
cls.log = gLogger.getSubLogger( exeName, child = False )
try:
result = cls.initialize()
except Exception as excp:
gLogger.exception( "Exception while initializing %s" % loadName, lException = excp )
return S_ERROR( "Exception while initializing: %s" % str( excp ) )
if not isReturnStructure( result ):
return S_ERROR( "Executor %s does not return an S_OK/S_ERROR after initialization" % loadName )
return result
def _ex_processTask( self, taskId, taskStub ):
self.__properties[ 'shifterProxy' ] = self.ex_getOption( 'shifterProxy' )
self.__freezeTime = 0
self.__fastTrackEnabled = True
self.log.verbose( "Task %s: Received" % str( taskId ) )
result = self.__deserialize( taskId, taskStub )
if not result[ 'OK' ]:
self.log.error( "Can not deserialize task", "Task %s: %s" % ( str( taskId ), result[ 'Message' ] ) )
return result
taskObj = result[ 'Value' ]
#Shifter proxy?
result = self.__installShifterProxy()
if not result[ 'OK' ]:
return result
#Execute!
result = self.processTask( taskId, taskObj )
if not isReturnStructure( result ):
raise Exception( "processTask does not return a return structure" )
if not result[ 'OK' ]:
return result
#If there's a result, serialize it again!
if result[ 'Value' ]:
taskObj = result[ 'Value' ]
#Serialize again
result = self.__serialize( taskId, taskObj )
if not result[ 'OK' ]:
self.log.verbose( "Task %s: Cannot serialize: %s" % ( str( taskId ), result[ 'Message' ] ) )
return result
taskStub = result[ 'Value' ]
#Try fast track
fastTrackType = False
if not self.__freezeTime and self.__fastTrackEnabled:
result = self.fastTrackDispatch( taskId, taskObj )
if not result[ 'OK' ]:
self.log.error( "FastTrackDispatch failed for job", "%s: %s" % ( taskId, result[ 'Message' ] ) )
else:
fastTrackType = result[ 'Value' ]
#EOP
return S_OK( ( taskStub, self.__freezeTime, fastTrackType ) )
def am_initialize( self, *initArgs ):
agentName = self.am_getModuleParam( 'fullName' )
result = self.initialize( *initArgs )
if not isReturnStructure( result ):
return S_ERROR( "initialize must return S_OK/S_ERROR" )
if not result[ 'OK' ]:
return S_ERROR( "Error while initializing %s: %s" % ( agentName, result[ 'Message' ] ) )
_checkDir( self.am_getControlDirectory() )
_checkDir( self.am_getWorkDirectory() )
self.__moduleProperties[ 'shifterProxy' ] = self.am_getOption( 'shifterProxy' )
if self.am_monitoringEnabled():
self.monitor.enable()
if len( self.__moduleProperties[ 'executors' ] ) < 1:
return S_ERROR( "At least one executor method has to be defined" )
if not self.am_Enabled():
return S_ERROR( "Agent is disabled via the configuration" )
self.log.notice( "="*40 )
self.log.notice( "Loaded agent module %s" % self.__moduleProperties[ 'fullName' ] )
self.log.notice( " Site: %s" % DIRAC.siteName() )
self.log.notice( " Setup: %s" % gConfig.getValue( "/DIRAC/Setup" ) )
self.log.notice( " Base Module version: %s " % __RCSID__ )
self.log.notice( " Agent version: %s" % self.__codeProperties[ 'version' ] )
self.log.notice( " DIRAC version: %s" % DIRAC.version )
self.log.notice( " DIRAC platform: %s" % DIRAC.platform )
pollingTime = int( self.am_getOption( 'PollingTime' ) )
if pollingTime > 3600:
self.log.notice( " Polling time: %s hours" % ( pollingTime / 3600. ) )
else:
self.log.notice( " Polling time: %s seconds" % self.am_getOption( 'PollingTime' ) )
self.log.notice( " Control dir: %s" % self.am_getControlDirectory() )
self.log.notice( " Work dir: %s" % self.am_getWorkDirectory() )
if self.am_getOption( 'MaxCycles' ) > 0:
self.log.notice( " Cycles: %s" % self.am_getMaxCycles() )
else:
self.log.notice( " Cycles: unlimited" )
self.log.notice( "="*40 )
self.__initialized = True
return S_OK()
def __getNextExecutor( self, taskId ):
try:
eTask = self.__tasks[ taskId ]
except IndexError:
msg = "Task %s was deleted prematurely while being dispatched" % taskId
self.__log.error( "Task was deleted prematurely while being dispatched", "%s" % taskId )
return S_ERROR( msg )
try:
result = self.__cbHolder.cbDispatch( taskId, eTask.taskObj, tuple( eTask.pathExecuted ) )
except:
self.__log.exception( "Exception while calling dispatch callback" )
return S_ERROR( "Exception while calling dispatch callback" )
if not isReturnStructure( result ):
errMsg = "Dispatch callback did not return a S_OK/S_ERROR structure"
self.__log.fatal( errMsg )
return S_ERROR( errMsg )
#Assign the next executor type to the task
if result[ 'OK' ]:
eTask.eType = result[ 'Value' ]
return result
@classmethod
def __sendTask( self, taskId, taskObj, eId, eType ):
try:
result = self.exec_prepareToSend( taskId, taskObj, eId )
if not result[ 'OK' ]:
return result
except Exception, excp:
gLogger.exception( "Exception while executing prepareToSend: %s" % str( excp ) )
return S_ERROR( "Cannot presend task" )
try:
result = self.exec_serializeTask( taskObj )
except Exception, excp:
gLogger.exception( "Exception while serializing task %s" % taskId )
return S_ERROR( "Cannot serialize task %s: %s" % ( taskId, str( excp ) ) )
if not isReturnStructure( result ):
raise Exception( "exec_serializeTask does not return a return structure" )
if not result[ 'OK' ]:
return result
taskStub = result[ 'Value' ]
result = self.srv_msgCreate( "ProcessTask" )
if not result[ 'OK' ]:
return result
msgObj = result[ 'Value' ]
msgObj.taskId = taskId
msgObj.taskStub = taskStub
msgObj.eType = eType
return self.srv_msgSend( eId, msgObj )
@classmethod
def __execDisconnected( cls, trid ):
@classmethod
def ex_getMind( cls ):
return cls.__mindName
@classmethod
def ex_getExtraArguments( cls ):
return cls.__mindExtraArgs
def __serialize( self, taskId, taskObj ):
try:
result = self.serializeTask( taskObj )
except Exception, excp:
gLogger.exception( "Exception while serializing task %s" % taskId )
return S_ERROR( "Cannot serialize task %s: %s" % ( taskId, str( excp ) ) )
if not isReturnStructure( result ):
raise Exception( "serializeTask does not return a return structure" )
return result
def __deserialize( self, taskId, taskStub ):
try:
result = self.deserializeTask( taskStub )
except Exception, excp:
gLogger.exception( "Exception while deserializing task %s" % taskId )
return S_ERROR( "Cannot deserialize task %s: %s" % ( taskId, str( excp ) ) )
if not isReturnStructure( result ):
raise Exception( "deserializeTask does not return a return structure" )
return result
def _ex_processTask( self, taskId, taskStub ):
self.__properties[ 'shifterProxy' ] = self.ex_getOption( 'shifterProxy' )
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result["OK"]:
return result
self._handler = result["Value"]
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._threadPool = ThreadPoolExecutor(max(0,
self._cfg.getMaxThreads()))
self._msgBroker = MessageBroker("%sMSB" % self._name,
threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {
"serviceName":
self._name,
"serviceSectionPath":
PathFinder.getServiceSection(self._name),
"URL":
self._cfg.getURL(),
"messageSender":
MessageSender(self._name, self._msgBroker),
"validNames":
self._validNames,
"csPaths": [
PathFinder.getServiceSection(svcName)
for svcName in self._validNames
],
}
self.securityLogging = Operations().getValue(
"EnableSecurityLogging", True) and getServiceOption(
self._serviceInfoDict, "EnableSecurityLogging", True)
# Initialize Monitoring
# The import needs to be here because of the CS must be initialized before importing
# this class (see https://github.com/DIRACGrid/DIRAC/issues/4793)
from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter
self.activityMonitoringReporter = MonitoringReporter(
monitoringType="ServiceMonitoring")
self._initMonitoring()
# Call static initialization function
try:
self._handler["class"]._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self.activityMonitoringReporter)
if self._handler["init"]:
for initFunc in self._handler["init"]:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception(
"Exception while calling initialization function",
lException=excp)
return S_ERROR(
"Exception while calling initialization function: %s"
% str(excp))
if not isReturnStructure(result):
return S_ERROR(
"Service initialization function %s must return S_OK/S_ERROR"
% initFunc)
if not result["OK"]:
return S_ERROR("Error while initializing %s: %s" %
(self._name, result["Message"]))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
if self.activityMonitoring:
gThreadScheduler.addPeriodicTask(30, self.__reportActivity)
gThreadScheduler.addPeriodicTask(
100, self.__activityMonitoringReporting)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result["OK"]:
return result
self._actions = result["Value"]
return S_OK()
class RequestHandler(object):
class ConnectionError(Exception):
def __init__(self, msg):
self.__msg = msg
def __str__(self):
return "ConnectionError: %s" % self.__msg
def __init__(self, handlerInitDict, trid):
"""
Constructor
@type handlerInitDict: dictionary
@param handlerInitDict: Information vars for the service
@type trid: object
@param trid: Transport to use
"""
#Initially serviceInfoDict is the one base to the RequestHandler
# the one created in _rh_initializeClass
#FSM help me for I have made a complex stuff that I will forget in 5 mins :P
handlerInitDict.update(self.__srvInfoDict)
self.serviceInfoDict = handlerInitDict
self.__trid = trid
def initialize(self):
"""Initialize this instance of the handler (to be overwritten)
"""
pass
@classmethod
def _rh__initializeClass(cls, serviceInfoDict, lockManager, msgBroker,
monitor):
"""
Class initialization (not to be called by hand or overwritten!!)
@type serviceInfoDict: dictionary
@param serviceInfoDict: Information vars for the service
@type msgBroker: object
@param msgBroker: Message delivery
@type lockManager: object
@param lockManager: Lock manager to use
"""
cls.__srvInfoDict = serviceInfoDict
cls.__svcName = cls.__srvInfoDict['serviceName']
cls.__lockManager = lockManager
cls.__msgBroker = msgBroker
cls.__trPool = msgBroker.getTransportPool()
cls.__monitor = monitor
cls.log = gLogger
def getRemoteAddress(self):
"""
Get the address of the remote peer.
@return : Address of remote peer.
"""
return self.__trPool.get(self.__trid).getRemoteAddress()
def getRemoteCredentials(self):
"""
Get the credentials of the remote peer.
@return : Credentials dictionary of remote peer.
"""
return self.__trPool.get(self.__trid).getConnectingCredentials()
@classmethod
def getCSOption(cls, optionName, defaultValue=False):
"""
Get an option from the CS section of the services
@return : Value for serviceSection/optionName in the CS being defaultValue the default
"""
return cls.srv_getCSOption(optionName, defaultValue)
def _rh_executeAction(self, proposalTuple):
"""
Execute an action.
@type proposalTuple: tuple
@param proposalTuple: Type of action to execute. First position of the tuple must be the type
of action to execute. The second position is the action itself.
"""
actionTuple = proposalTuple[1]
gLogger.debug("Executing %s:%s action" % actionTuple)
startTime = time.time()
actionType = actionTuple[0]
self.serviceInfoDict['actionTuple'] = actionTuple
try:
if actionType == "RPC":
retVal = self.__doRPC(actionTuple[1])
elif actionType == "FileTransfer":
retVal = self.__doFileTransfer(actionTuple[1])
elif actionType == "Connection":
retVal = self.__doConnection(actionTuple[1])
else:
return S_ERROR("Unknown action %s" % actionType)
except RequestHandler.ConnectionError, excp:
gLogger.error("ConnectionError", str(excp))
return S_ERROR(excp)
if not isReturnStructure(retVal):
message = "Method %s for action %s does not return a S_OK/S_ERROR!" % (
actionTuple[1], actionTuple[0])
gLogger.error(message)
retVal = S_ERROR(message)
self.__logRemoteQueryResponse(retVal, time.time() - startTime)
return self.__trPool.send(self.__trid, retVal)
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
# TODO: remove ThreadPool
if useThreadPoolExecutor:
self._threadPool = ThreadPoolExecutor(
max(0, self._cfg.getMaxThreads()))
else:
self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()),
max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name,
threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {
'serviceName':
self._name,
'serviceSectionPath':
PathFinder.getServiceSection(self._name),
'URL':
self._cfg.getURL(),
'messageSender':
MessageSender(self._name, self._msgBroker),
'validNames':
self._validNames,
'csPaths': [
PathFinder.getServiceSection(svcName)
for svcName in self._validNames
]
}
# Initialize Monitoring
# This is a flag used to check whether "EnableActivityMonitoring" is enabled or not from the config file.
self.activityMonitoring = (Operations().getValue(
"EnableActivityMonitoring", False) or getServiceOption(
self._serviceInfoDict, "EnableActivityMonitoring", False))
if self.activityMonitoring:
# The import needs to be here because of the CS must be initialized before importing
# this class (see https://github.com/DIRACGrid/DIRAC/issues/4793)
from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter
self.activityMonitoringReporter = MonitoringReporter(
monitoringType="ComponentMonitoring")
gThreadScheduler.addPeriodicTask(
100, self.__activityMonitoringReporting)
elif self._standalone:
self._monitor = gMonitor
else:
self._monitor = MonitoringClient()
self._initMonitoring()
# Call static initialization function
try:
if self.activityMonitoring:
self._handler['class']._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self.activityMonitoringReporter)
else:
self._handler['class']._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self._monitor)
if self._handler['init']:
for initFunc in self._handler['init']:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception(
"Exception while calling initialization function",
lException=excp)
return S_ERROR(
"Exception while calling initialization function: %s"
% str(excp))
if not isReturnStructure(result):
return S_ERROR(
"Service initialization function %s must return S_OK/S_ERROR"
% initFunc)
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" %
(self._name, result['Message']))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result['OK']:
return result
self._actions = result['Value']
if not self.activityMonitoring:
gThreadScheduler.addPeriodicTask(30,
self.__reportThreadPoolContents)
return S_OK()