def initialize(self):
""" This replaces the standard initialize from Service
"""
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % GatewayService.GATEWAY_NAME)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
# Discover Handler
# TODO: remove later
if useThreadPoolExecutor:
self._threadPool = ThreadPoolExecutor(max(0, self._cfg.getMaxThreads()))
else:
self._threadPool = ThreadPool(1,
max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % GatewayService.GATEWAY_NAME, threadPool=self._threadPool)
self._msgBroker.useMessageObjects(False)
getGlobalMessageBroker().useMessageObjects(False)
self._msgForwarder = MessageForwarder(self._msgBroker)
return S_OK()
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._initMonitoring()
self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()),
max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {'serviceName': self._name,
'serviceSectionPath': PathFinder.getServiceSection(self._name),
'URL': self._cfg.getURL(),
'messageSender': MessageSender(self._name, self._msgBroker),
'validNames': self._validNames,
'csPaths': [PathFinder.getServiceSection(svcName) for svcName in self._validNames]
}
# Call static initialization function
try:
self._handler['class']._rh__initializeClass(dict(self._serviceInfoDict),
self._lockManager,
self._msgBroker,
self._monitor)
if self._handler['init']:
for initFunc in self._handler['init']:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception("Exception while calling initialization function", lException=excp)
return S_ERROR("Exception while calling initialization function: %s" % str(excp))
if not isReturnStructure(result):
return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc)
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" % (self._name, result['Message']))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result['OK']:
return result
self._actions = result['Value']
gThreadScheduler.addPeriodicTask(30, self.__reportThreadPoolContents)
return S_OK()
def initialize( self ):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % self._name )
gLogger.verbose( "Service URL is %s" % self._url )
#Load handler
result = self._loadHandlerInit()
if not result[ 'OK' ]:
return result
self._handler = result[ 'Value' ]
#Initialize lock manager
self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() )
self._initMonitoring()
self._threadPool = ThreadPool( max( 1, self._cfg.getMinThreads() ),
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool )
#Create static dict
self._serviceInfoDict = { 'serviceName' : self._name,
'serviceSectionPath' : PathFinder.getServiceSection( self._name ),
'URL' : self._cfg.getURL(),
'messageSender' : MessageSender( self._name, self._msgBroker ),
'validNames' : self._validNames,
'csPaths' : [ PathFinder.getServiceSection( svcName ) for svcName in self._validNames ]
}
#Call static initialization function
try:
self._handler[ 'class' ]._rh__initializeClass( dict( self._serviceInfoDict ),
self._lockManager,
self._msgBroker,
self._monitor )
if self._handler[ 'init' ]:
for initFunc in self._handler[ 'init' ]:
gLogger.verbose( "Executing initialization function" )
try:
result = initFunc( dict( self._serviceInfoDict ) )
except Exception as excp:
gLogger.exception( "Exception while calling initialization function", lException = excp )
return S_ERROR( "Exception while calling initialization function: %s" % str( excp ) )
if not isReturnStructure( result ):
return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc )
if not result[ 'OK' ]:
return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) )
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception( e )
gLogger.exception( errMsg )
return S_ERROR( errMsg )
#Load actions after the handler has initialized itself
result = self._loadActions()
if not result[ 'OK' ]:
return result
self._actions = result[ 'Value' ]
gThreadScheduler.addPeriodicTask( 30, self.__reportThreadPoolContents )
return S_OK()
def initialize( self ):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % GatewayService.GATEWAY_NAME )
gLogger.verbose( "Service URL is %s" % self._url )
#Discover Handler
self._initMonitoring()
self._threadPool = ThreadPool( 1,
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % GatewayService.GATEWAY_NAME, threadPool = self._threadPool )
self._msgBroker.useMessageObjects( False )
getGlobalMessageBroker().useMessageObjects( False )
self._msgForwarder = MessageForwarder( self._msgBroker )
return S_OK()
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % GatewayService.GATEWAY_NAME)
gLogger.verbose("Service URL is %s" % self._url)
# Discover Handler
self._initMonitoring()
self._threadPool = ThreadPool(1, max(0, self._cfg.getMaxThreads()), self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % GatewayService.GATEWAY_NAME, threadPool=self._threadPool)
self._msgBroker.useMessageObjects(False)
getGlobalMessageBroker().useMessageObjects(False)
self._msgForwarder = MessageForwarder(self._msgBroker)
return S_OK()
def initialize( self ):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % self._name )
gLogger.verbose( "Service URL is %s" % self._url )
#Discover Handler
self._handlerLocation = self._discoverHandlerLocation()
if not self._handlerLocation:
return S_ERROR( "Could not find handler location for %s" % self._name )
gLogger.verbose( "Handler found at %s" % self._handlerLocation )
#Load handler
result = self._loadHandler()
if not result[ 'OK' ]:
return result
self._handler = result[ 'Value' ]
#Initialize lock manager
self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() )
#Load actions
result = self._loadActions()
if not result[ 'OK' ]:
return result
self._actions = result[ 'Value' ]
self._initMonitoring()
self._threadPool = ThreadPool( 1,
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool )
#Create static dict
self._serviceInfoDict = { 'serviceName' : self._name,
'URL' : self._cfg.getURL(),
'systemSectionPath' : self._cfg.getSystemPath(),
'serviceSectionPath' : self._cfg.getServicePath(),
'messageSender' : MessageSender( self._msgBroker )
}
#Call static initialization function
try:
if self._handler[ 'init' ]:
result = self._handler[ 'init' ]( dict( self._serviceInfoDict ) )
if not isReturnStructure( result ):
return S_ERROR( "Service initialization function must return S_OK/S_ERROR" )
if not result[ 'OK' ]:
return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) )
except Exception, e:
errMsg = "Exception while intializing %s" % self._name
gLogger.exception( errMsg )
return S_ERROR( errMsg )
def initialize( self ):
""" This replaces the standard initialize from Service
"""
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % GatewayService.GATEWAY_NAME )
gLogger.verbose( "Service URL is %s" % self._url )
#Load handler
result = self._loadHandlerInit()
if not result[ 'OK' ]:
return result
self._handler = result[ 'Value' ]
#Discover Handler
self._threadPool = ThreadPool( 1,
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % GatewayService.GATEWAY_NAME, threadPool = self._threadPool )
self._msgBroker.useMessageObjects( False )
getGlobalMessageBroker().useMessageObjects( False )
self._msgForwarder = MessageForwarder( self._msgBroker )
return S_OK()
class Service(object):
SVC_VALID_ACTIONS = {
'RPC': 'export',
'FileTransfer': 'transfer',
'Message': 'msg',
'Connection': 'Message'
}
SVC_SECLOG_CLIENT = SecurityLogClient()
def __init__(self, serviceData):
"""
Init the variables for the service
:param serviceData: dict with modName, standalone, loadName, moduleObj, classObj. e.g.:
{'modName': 'Framework/serviceName',
'standalone': True,
'loadName': 'Framework/serviceName',
'moduleObj': <module 'serviceNameHandler' from '/home/DIRAC/FrameworkSystem/Service/serviceNameHandler.pyo'>,
'classObj': <class 'serviceNameHandler.serviceHandler'>}
Standalone is true if there is only one service started
If it's false, every service is linked to a different MonitoringClient
"""
self._svcData = serviceData
self._name = serviceData['modName']
self._startTime = Time.dateTime()
self._validNames = [serviceData['modName']]
if serviceData['loadName'] not in self._validNames:
self._validNames.append(serviceData['loadName'])
self._cfg = ServiceConfiguration(list(self._validNames))
if serviceData['standalone']:
self._monitor = gMonitor
else:
self._monitor = MonitoringClient()
self.__monitorLastStatsUpdate = time.time()
self._stats = {'queries': 0, 'connections': 0}
self._authMgr = AuthManager(
"%s/Authorization" %
PathFinder.getServiceSection(serviceData['loadName']))
self._transportPool = getGlobalTransportPool()
self.__cloneId = 0
self.__maxFD = 0
def setCloneProcessId(self, cloneId):
self.__cloneId = cloneId
self._monitor.setComponentName("%s-Clone:%s" % (self._name, cloneId))
def _isMetaAction(self, action):
referedAction = Service.SVC_VALID_ACTIONS[action]
if referedAction in Service.SVC_VALID_ACTIONS:
return referedAction
return False
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._initMonitoring()
# TODO: remove ThreadPool
if useThreadPoolExecutor:
self._threadPool = ThreadPoolExecutor(
max(0, self._cfg.getMaxThreads()))
else:
self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()),
max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name,
threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {
'serviceName':
self._name,
'serviceSectionPath':
PathFinder.getServiceSection(self._name),
'URL':
self._cfg.getURL(),
'messageSender':
MessageSender(self._name, self._msgBroker),
'validNames':
self._validNames,
'csPaths': [
PathFinder.getServiceSection(svcName)
for svcName in self._validNames
]
}
# Call static initialization function
try:
self._handler['class']._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self._monitor)
if self._handler['init']:
for initFunc in self._handler['init']:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception(
"Exception while calling initialization function",
lException=excp)
return S_ERROR(
"Exception while calling initialization function: %s"
% str(excp))
if not isReturnStructure(result):
return S_ERROR(
"Service initialization function %s must return S_OK/S_ERROR"
% initFunc)
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" %
(self._name, result['Message']))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result['OK']:
return result
self._actions = result['Value']
gThreadScheduler.addPeriodicTask(30, self.__reportThreadPoolContents)
return S_OK()
def __searchInitFunctions(self, handlerClass, currentClass=None):
if not currentClass:
currentClass = handlerClass
initFuncs = []
ancestorHasInit = False
for ancestor in currentClass.__bases__:
initFuncs += self.__searchInitFunctions(handlerClass, ancestor)
if 'initializeHandler' in dir(ancestor):
ancestorHasInit = True
if ancestorHasInit:
initFuncs.append(
super(currentClass, handlerClass).initializeHandler)
if currentClass == handlerClass and 'initializeHandler' in dir(
handlerClass):
initFuncs.append(handlerClass.initializeHandler)
return initFuncs
def _loadHandlerInit(self):
handlerClass = self._svcData['classObj']
handlerName = handlerClass.__name__
handlerInitMethods = self.__searchInitFunctions(handlerClass)
try:
handlerInitMethods.append(
getattr(self._svcData['moduleObj'],
"initialize%s" % handlerName))
except AttributeError:
gLogger.verbose(
"Not found global initialization function for service")
if handlerInitMethods:
gLogger.info("Found %s initialization methods" %
len(handlerInitMethods))
handlerInfo = {}
handlerInfo["name"] = handlerName
handlerInfo["module"] = self._svcData['moduleObj']
handlerInfo["class"] = handlerClass
handlerInfo["init"] = handlerInitMethods
return S_OK(handlerInfo)
def _loadActions(self):
handlerClass = self._handler['class']
authRules = {}
typeCheck = {}
methodsList = {}
for actionType in Service.SVC_VALID_ACTIONS:
if self._isMetaAction(actionType):
continue
authRules[actionType] = {}
typeCheck[actionType] = {}
methodsList[actionType] = []
handlerAttributeList = dir(handlerClass)
for actionType in Service.SVC_VALID_ACTIONS:
if self._isMetaAction(actionType):
continue
methodPrefix = '%s_' % Service.SVC_VALID_ACTIONS[actionType]
for attribute in handlerAttributeList:
if attribute.find(methodPrefix) != 0:
continue
exportedName = attribute[len(methodPrefix):]
methodsList[actionType].append(exportedName)
gLogger.verbose("+ Found %s method %s" %
(actionType, exportedName))
# Create lock for method
self._lockManager.createLock(
"%s/%s" % (actionType, exportedName),
self._cfg.getMaxThreadsForMethod(actionType, exportedName))
# Look for type and auth rules
if actionType == 'RPC':
typeAttr = "types_%s" % exportedName
authAttr = "auth_%s" % exportedName
else:
typeAttr = "types_%s_%s" % (
Service.SVC_VALID_ACTIONS[actionType], exportedName)
authAttr = "auth_%s_%s" % (
Service.SVC_VALID_ACTIONS[actionType], exportedName)
if typeAttr in handlerAttributeList:
obj = getattr(handlerClass, typeAttr)
gLogger.verbose("|- Found type definition %s: %s" %
(typeAttr, str(obj)))
typeCheck[actionType][exportedName] = obj
if authAttr in handlerAttributeList:
obj = getattr(handlerClass, authAttr)
gLogger.verbose("|- Found auth rules %s: %s" %
(authAttr, str(obj)))
authRules[actionType][exportedName] = obj
for actionType in Service.SVC_VALID_ACTIONS:
referedAction = self._isMetaAction(actionType)
if not referedAction:
continue
gLogger.verbose("Action %s is a meta action for %s" %
(actionType, referedAction))
authRules[actionType] = []
for method in authRules[referedAction]:
for prop in authRules[referedAction][method]:
if prop not in authRules[actionType]:
authRules[actionType].append(prop)
gLogger.verbose("Meta action %s props are %s" %
(actionType, authRules[actionType]))
return S_OK({
'methods': methodsList,
'auth': authRules,
'types': typeCheck
})
def _initMonitoring(self):
# Init extra bits of monitoring
self._monitor.setComponentType(MonitoringClient.COMPONENT_SERVICE)
self._monitor.setComponentName(self._name)
self._monitor.setComponentLocation(self._cfg.getURL())
self._monitor.initialize()
self._monitor.registerActivity("Connections", "Connections received",
"Framework", "connections",
MonitoringClient.OP_RATE)
self._monitor.registerActivity("Queries", "Queries served",
"Framework", "queries",
MonitoringClient.OP_RATE)
self._monitor.registerActivity('CPU', "CPU Usage", 'Framework',
"CPU,%", MonitoringClient.OP_MEAN, 600)
self._monitor.registerActivity('MEM', "Memory Usage", 'Framework',
'Memory,MB', MonitoringClient.OP_MEAN,
600)
self._monitor.registerActivity('PendingQueries', "Pending queries",
'Framework', 'queries',
MonitoringClient.OP_MEAN)
self._monitor.registerActivity('ActiveQueries', "Active queries",
'Framework', 'threads',
MonitoringClient.OP_MEAN)
self._monitor.registerActivity('RunningThreads', "Running threads",
'Framework', 'threads',
MonitoringClient.OP_MEAN)
self._monitor.registerActivity('MaxFD', "Max File Descriptors",
'Framework', 'fd',
MonitoringClient.OP_MEAN)
self._monitor.setComponentExtraParam('DIRACVersion', DIRAC.version)
self._monitor.setComponentExtraParam('platform', DIRAC.getPlatform())
self._monitor.setComponentExtraParam('startTime', Time.dateTime())
for prop in (("__RCSID__", "version"), ("__doc__", "description")):
try:
value = getattr(self._handler['module'], prop[0])
except Exception as e:
gLogger.exception(e)
gLogger.error("Missing property", prop[0])
value = 'unset'
self._monitor.setComponentExtraParam(prop[1], value)
for secondaryName in self._cfg.registerAlsoAs():
gLogger.info("Registering %s also as %s" %
(self._name, secondaryName))
self._validNames.append(secondaryName)
return S_OK()
def __reportThreadPoolContents(self):
# TODO: remove later
if useThreadPoolExecutor:
pendingQueries = self._threadPool._work_queue.qsize()
activeQuereies = len(self._threadPool._threads)
else:
pendingQueries = self._threadPool.pendingJobs()
activeQuereies = self._threadPool.numWorkingThreads()
self._monitor.addMark('PendingQueries', pendingQueries)
self._monitor.addMark('ActiveQueries', activeQuereies)
self._monitor.addMark('RunningThreads', threading.activeCount())
self._monitor.addMark('MaxFD', self.__maxFD)
self.__maxFD = 0
def getConfig(self):
return self._cfg
# End of initialization functions
def handleConnection(self, clientTransport):
"""
This method may be called by ServiceReactor.
The method stacks openened connection in a queue, another thread
read this queue and handle connection.
:param clientTransport: Object wich describe opened connection (PlainTransport or SSLTransport)
"""
self._stats['connections'] += 1
self._monitor.setComponentExtraParam('queries',
self._stats['connections'])
# TODO: remove later
if useThreadPoolExecutor:
self._threadPool.submit(self._processInThread, clientTransport)
else:
self._threadPool.generateJobAndQueueIt(self._processInThread,
args=(clientTransport, ))
# Threaded process function
def _processInThread(self, clientTransport):
"""
This method handles a RPC, FileTransfer or Connection.
Connection may be opened via ServiceReactor.__acceptIncomingConnection
- Do the SSL/TLS Handshake (if dips is used) and extract credentials
- Get the action called by the client
- Check if the client is authorized to perform ation
- If not, connection is closed
- Instanciate the RequestHandler (RequestHandler contain all methods callable)
(Following is not directly in this method but it describe what happen at
#Execute the action)
- Notify the client we're ready to execute the action (via _processProposal)
and call RequestHandler._rh_executeAction()
- Receive arguments/file/something else (depending on action) in the RequestHandler
- Executing the action asked by the client
:param clientTransport: Object who describe the opened connection (SSLTransport or PlainTransport)
:return: S_OK with "closeTransport" a boolean to indicate if th connection have to be closed
e.g. after RPC, closeTransport=True
"""
self.__maxFD = max(self.__maxFD, clientTransport.oSocket.fileno())
self._lockManager.lockGlobal()
try:
monReport = self.__startReportToMonitoring()
except Exception:
monReport = False
try:
# Handshake
try:
result = clientTransport.handshake()
if not result['OK']:
clientTransport.close()
return
except BaseException:
return
# Add to the transport pool
trid = self._transportPool.add(clientTransport)
if not trid:
return
# Receive and check proposal
result = self._receiveAndCheckProposal(trid)
if not result['OK']:
self._transportPool.sendAndClose(trid, result)
return
proposalTuple = result['Value']
# Instantiate handler
result = self._instantiateHandler(trid, proposalTuple)
if not result['OK']:
self._transportPool.sendAndClose(trid, result)
return
handlerObj = result['Value']
# Execute the action
result = self._processProposal(trid, proposalTuple, handlerObj)
# Close the connection if required
if result['closeTransport'] or not result['OK']:
if not result['OK']:
gLogger.error("Error processing proposal",
result['Message'])
self._transportPool.close(trid)
return result
finally:
self._lockManager.unlockGlobal()
if monReport:
self.__endReportToMonitoring(*monReport)
def _createIdentityString(self, credDict, clientTransport=None):
if 'username' in credDict:
if 'group' in credDict:
identity = "[%s:%s]" % (credDict['username'],
credDict['group'])
else:
identity = "[%s:unknown]" % credDict['username']
else:
identity = 'unknown'
if clientTransport:
addr = clientTransport.getRemoteAddress()
if addr:
addr = "{%s:%s}" % (addr[0], addr[1])
if 'DN' in credDict:
identity += "(%s)" % credDict['DN']
return identity
@staticmethod
def _deserializeProposalTuple(serializedProposal):
""" We receive the proposalTuple as a list.
Turn it into a tuple again
"""
proposalTuple = tuple(
tuple(x) if isinstance(x, list) else x for x in serializedProposal)
return proposalTuple
def _receiveAndCheckProposal(self, trid):
clientTransport = self._transportPool.get(trid)
# Get the peer credentials
credDict = clientTransport.getConnectingCredentials()
# Receive the action proposal
retVal = clientTransport.receiveData(1024)
if not retVal['OK']:
gLogger.error(
"Invalid action proposal",
"%s %s" % (self._createIdentityString(
credDict, clientTransport), retVal['Message']))
return S_ERROR("Invalid action proposal")
proposalTuple = Service._deserializeProposalTuple(retVal['Value'])
gLogger.debug("Received action from client",
"/".join(list(proposalTuple[1])))
# Check if there are extra credentials
if proposalTuple[2]:
clientTransport.setExtraCredentials(proposalTuple[2])
# Check if this is the requested service
requestedService = proposalTuple[0][0]
if requestedService not in self._validNames:
return S_ERROR("%s is not up in this server" % requestedService)
# Check if the action is valid
requestedActionType = proposalTuple[1][0]
if requestedActionType not in Service.SVC_VALID_ACTIONS:
return S_ERROR("%s is not a known action type" %
requestedActionType)
# Check if it's authorized
result = self._authorizeProposal(proposalTuple[1], trid, credDict)
if not result['OK']:
return result
# Proposal is OK
return S_OK(proposalTuple)
def _authorizeProposal(self, actionTuple, trid, credDict):
# Find CS path for the Auth rules
referedAction = self._isMetaAction(actionTuple[0])
if referedAction:
csAuthPath = "%s/Default" % actionTuple[0]
hardcodedMethodAuth = self._actions['auth'][actionTuple[0]]
else:
if actionTuple[0] == 'RPC':
csAuthPath = actionTuple[1]
else:
csAuthPath = "/".join(actionTuple)
# Find if there are hardcoded auth rules in the code
hardcodedMethodAuth = False
if actionTuple[0] in self._actions['auth']:
hardcodedRulesByType = self._actions['auth'][actionTuple[0]]
if actionTuple[0] == "FileTransfer":
methodName = actionTuple[1][0].lower() + actionTuple[1][1:]
else:
methodName = actionTuple[1]
if methodName in hardcodedRulesByType:
hardcodedMethodAuth = hardcodedRulesByType[methodName]
# Auth time!
if not self._authMgr.authQuery(csAuthPath, credDict,
hardcodedMethodAuth):
# Get the identity string
identity = self._createIdentityString(credDict)
fromHost = "unknown host"
tr = self._transportPool.get(trid)
if tr:
fromHost = '/'.join(
[str(item) for item in tr.getRemoteAddress()])
gLogger.warn(
"Unauthorized query", "to %s:%s by %s from %s" %
(self._name, "/".join(actionTuple), identity, fromHost))
result = S_ERROR(ENOAUTH, "Unauthorized query")
else:
result = S_OK()
# Security log
tr = self._transportPool.get(trid)
if not tr:
return S_ERROR("Client disconnected")
sourceAddress = tr.getRemoteAddress()
identity = self._createIdentityString(credDict)
Service.SVC_SECLOG_CLIENT.addMessage(result['OK'], sourceAddress[0],
sourceAddress[1], identity,
self._cfg.getHostname(),
self._cfg.getPort(), self._name,
"/".join(actionTuple))
return result
def _instantiateHandler(self, trid, proposalTuple=None):
"""
Generate an instance of the handler for a given service
:param int trid: transport ID
:param tuple proposalTuple: tuple describing the proposed action
:return: S_OK/S_ERROR, Value is the handler object
"""
# Generate the client params
clientParams = {'serviceStartTime': self._startTime}
if proposalTuple:
# The 4th element is the client version
clientParams['clientVersion'] = proposalTuple[3] if len(
proposalTuple) > 3 else None
clientParams['clientSetup'] = proposalTuple[0][1]
if len(proposalTuple[0]) < 3:
clientParams['clientVO'] = gConfig.getValue(
"/DIRAC/VirtualOrganization", "unknown")
else:
clientParams['clientVO'] = proposalTuple[0][2]
clientTransport = self._transportPool.get(trid)
if clientTransport:
clientParams['clientAddress'] = clientTransport.getRemoteAddress()
# Generate handler dict with per client info
handlerInitDict = dict(self._serviceInfoDict)
for key in clientParams:
handlerInitDict[key] = clientParams[key]
# Instantiate and initialize
try:
handlerInstance = self._handler['class'](handlerInitDict, trid)
handlerInstance.initialize()
except Exception as e:
gLogger.exception("Server error while loading handler: %s" %
str(e))
return S_ERROR("Server error while loading handler")
return S_OK(handlerInstance)
def _processProposal(self, trid, proposalTuple, handlerObj):
# Notify the client we're ready to execute the action
retVal = self._transportPool.send(trid, S_OK())
if not retVal['OK']:
return retVal
messageConnection = False
if proposalTuple[1] == ('Connection', 'new'):
messageConnection = True
if messageConnection:
if self._msgBroker.getNumConnections(
) > self._cfg.getMaxMessagingConnections():
result = S_ERROR(
"Maximum number of connections reached. Try later")
result['closeTransport'] = True
return result
# This is a stable connection
self._msgBroker.addTransportId(
trid,
self._name,
receiveMessageCallback=self._mbReceivedMsg,
disconnectCallback=self._mbDisconnect,
listenToConnection=False)
result = self._executeAction(trid, proposalTuple, handlerObj)
if result['OK'] and messageConnection:
self._msgBroker.listenToTransport(trid)
result = self._mbConnect(trid, handlerObj)
if not result['OK']:
self._msgBroker.removeTransport(trid)
result['closeTransport'] = not messageConnection or not result['OK']
return result
def _mbConnect(self, trid, handlerObj=None):
if not handlerObj:
result = self._instantiateHandler(trid)
if not result['OK']:
return result
handlerObj = result['Value']
return handlerObj._rh_executeConnectionCallback('connected')
def _executeAction(self, trid, proposalTuple, handlerObj):
try:
return handlerObj._rh_executeAction(proposalTuple)
except Exception as e:
gLogger.exception("Exception while executing handler action")
return S_ERROR("Server error while executing action: %s" % str(e))
def _mbReceivedMsg(self, trid, msgObj):
result = self._authorizeProposal(
('Message', msgObj.getName()), trid,
self._transportPool.get(trid).getConnectingCredentials())
if not result['OK']:
return result
result = self._instantiateHandler(trid)
if not result['OK']:
return result
handlerObj = result['Value']
return handlerObj._rh_executeMessageCallback(msgObj)
def _mbDisconnect(self, trid):
result = self._instantiateHandler(trid)
if not result['OK']:
return result
handlerObj = result['Value']
return handlerObj._rh_executeConnectionCallback('drop')
def __startReportToMonitoring(self):
self._monitor.addMark("Queries")
now = time.time()
stats = os.times()
cpuTime = stats[0] + stats[2]
if now - self.__monitorLastStatsUpdate < 0:
return (now, cpuTime)
# Send CPU consumption mark
wallClock = now - self.__monitorLastStatsUpdate
self.__monitorLastStatsUpdate = now
# Send Memory consumption mark
membytes = MemStat.VmB('VmRSS:')
if membytes:
mem = membytes / (1024. * 1024.)
self._monitor.addMark('MEM', mem)
return (now, cpuTime)
def __endReportToMonitoring(self, initialWallTime, initialCPUTime):
wallTime = time.time() - initialWallTime
stats = os.times()
cpuTime = stats[0] + stats[2] - initialCPUTime
percentage = cpuTime / wallTime * 100.
if percentage > 0:
self._monitor.addMark('CPU', percentage)
class Service( object ):
SVC_VALID_ACTIONS = { 'RPC' : 'export',
'FileTransfer': 'transfer',
'Message' : 'msg',
'Connection' : 'Message' }
SVC_SECLOG_CLIENT = SecurityLogClient()
def __init__( self, serviceData ):
self._svcData = serviceData
self._name = serviceData[ 'modName' ]
self._startTime = Time.dateTime()
self._validNames = [ serviceData[ 'modName' ] ]
if serviceData[ 'loadName' ] not in self._validNames:
self._validNames.append( serviceData[ 'loadName' ] )
self._cfg = ServiceConfiguration( list( self._validNames ) )
if serviceData[ 'standalone' ]:
self._monitor = gMonitor
else:
self._monitor = MonitoringClient()
self.__monitorLastStatsUpdate = time.time()
self._stats = { 'queries' : 0, 'connections' : 0 }
self._authMgr = AuthManager( "%s/Authorization" % PathFinder.getServiceSection( serviceData[ 'loadName' ] ) )
self._transportPool = getGlobalTransportPool()
self.__cloneId = 0
self.__maxFD = 0
def setCloneProcessId( self, cloneId ):
self.__cloneId = cloneId
self._monitor.setComponentName( "%s-Clone:%s" % ( self._name, cloneId ) )
def _isMetaAction( self, action ):
referedAction = Service.SVC_VALID_ACTIONS[ action ]
if referedAction in Service.SVC_VALID_ACTIONS:
return referedAction
return False
def initialize( self ):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % self._name )
gLogger.verbose( "Service URL is %s" % self._url )
#Load handler
result = self._loadHandlerInit()
if not result[ 'OK' ]:
return result
self._handler = result[ 'Value' ]
#Initialize lock manager
self._lockManager = LockManager( self._cfg.getMaxWaitingPetitions() )
self._initMonitoring()
self._threadPool = ThreadPool( max( 1, self._cfg.getMinThreads() ),
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % self._name, threadPool = self._threadPool )
#Create static dict
self._serviceInfoDict = { 'serviceName' : self._name,
'serviceSectionPath' : PathFinder.getServiceSection( self._name ),
'URL' : self._cfg.getURL(),
'messageSender' : MessageSender( self._name, self._msgBroker ),
'validNames' : self._validNames,
'csPaths' : [ PathFinder.getServiceSection( svcName ) for svcName in self._validNames ]
}
#Call static initialization function
try:
self._handler[ 'class' ]._rh__initializeClass( dict( self._serviceInfoDict ),
self._lockManager,
self._msgBroker,
self._monitor )
if self._handler[ 'init' ]:
for initFunc in self._handler[ 'init' ]:
gLogger.verbose( "Executing initialization function" )
try:
result = initFunc( dict( self._serviceInfoDict ) )
except Exception as excp:
gLogger.exception( "Exception while calling initialization function", lException = excp )
return S_ERROR( "Exception while calling initialization function: %s" % str( excp ) )
if not isReturnStructure( result ):
return S_ERROR( "Service initialization function %s must return S_OK/S_ERROR" % initFunc )
if not result[ 'OK' ]:
return S_ERROR( "Error while initializing %s: %s" % ( self._name, result[ 'Message' ] ) )
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception( e )
gLogger.exception( errMsg )
return S_ERROR( errMsg )
#Load actions after the handler has initialized itself
result = self._loadActions()
if not result[ 'OK' ]:
return result
self._actions = result[ 'Value' ]
gThreadScheduler.addPeriodicTask( 30, self.__reportThreadPoolContents )
return S_OK()
def __searchInitFunctions( self, handlerClass, currentClass = None ):
if not currentClass:
currentClass = handlerClass
initFuncs = []
ancestorHasInit = False
for ancestor in currentClass.__bases__:
initFuncs += self.__searchInitFunctions( handlerClass, ancestor )
if 'initializeHandler' in dir( ancestor ):
ancestorHasInit = True
if ancestorHasInit:
initFuncs.append( super( currentClass, handlerClass ).initializeHandler )
if currentClass == handlerClass and 'initializeHandler' in dir( handlerClass ):
initFuncs.append( handlerClass.initializeHandler )
return initFuncs
def _loadHandlerInit( self ):
handlerClass = self._svcData[ 'classObj' ]
handlerName = handlerClass.__name__
handlerInitMethods = self.__searchInitFunctions( handlerClass )
try:
handlerInitMethods.append( getattr( self._svcData[ 'moduleObj' ], "initialize%s" % handlerName ) )
except AttributeError:
gLogger.verbose( "Not found global initialization function for service" )
if handlerInitMethods:
gLogger.info( "Found %s initialization methods" % len( handlerInitMethods ) )
handlerInfo = {}
handlerInfo[ "name" ] = handlerName
handlerInfo[ "module" ] = self._svcData[ 'moduleObj' ]
handlerInfo[ "class" ] = handlerClass
handlerInfo[ "init" ] = handlerInitMethods
return S_OK( handlerInfo )
def _loadActions( self ):
handlerClass = self._handler[ 'class' ]
authRules = {}
typeCheck = {}
methodsList = {}
for actionType in Service.SVC_VALID_ACTIONS:
if self._isMetaAction( actionType ):
continue
authRules[ actionType ] = {}
typeCheck[ actionType ] = {}
methodsList[ actionType ] = []
handlerAttributeList = dir( handlerClass )
for actionType in Service.SVC_VALID_ACTIONS:
if self._isMetaAction( actionType ):
continue
methodPrefix = '%s_' % Service.SVC_VALID_ACTIONS[ actionType ]
for attribute in handlerAttributeList:
if attribute.find( methodPrefix ) != 0:
continue
exportedName = attribute[ len( methodPrefix ) : ]
methodsList[ actionType ].append( exportedName )
gLogger.verbose( "+ Found %s method %s" % ( actionType, exportedName ) )
#Create lock for method
self._lockManager.createLock( "%s/%s" % ( actionType, exportedName ),
self._cfg.getMaxThreadsForMethod( actionType, exportedName ) )
#Look for type and auth rules
if actionType == 'RPC':
typeAttr = "types_%s" % exportedName
authAttr = "auth_%s" % exportedName
else:
typeAttr = "types_%s_%s" % ( Service.SVC_VALID_ACTIONS[ actionType ], exportedName )
authAttr = "auth_%s_%s" % ( Service.SVC_VALID_ACTIONS[ actionType ], exportedName )
if typeAttr in handlerAttributeList:
obj = getattr( handlerClass, typeAttr )
gLogger.verbose( "|- Found type definition %s: %s" % ( typeAttr, str( obj ) ) )
typeCheck[ actionType ][ exportedName ] = obj
if authAttr in handlerAttributeList:
obj = getattr( handlerClass, authAttr )
gLogger.verbose( "|- Found auth rules %s: %s" % ( authAttr, str( obj ) ) )
authRules[ actionType ][ exportedName ] = obj
for actionType in Service.SVC_VALID_ACTIONS:
referedAction = self._isMetaAction( actionType )
if not referedAction:
continue
gLogger.verbose( "Action %s is a meta action for %s" % ( actionType, referedAction ) )
authRules[ actionType ] = []
for method in authRules[ referedAction ]:
for prop in authRules[ referedAction ][ method ]:
if prop not in authRules[ actionType ]:
authRules[ actionType ].append( prop )
gLogger.verbose( "Meta action %s props are %s" % ( actionType, authRules[ actionType ] ) )
return S_OK( { 'methods' : methodsList, 'auth' : authRules, 'types' : typeCheck } )
def _initMonitoring( self ):
#Init extra bits of monitoring
self._monitor.setComponentType( MonitoringClient.COMPONENT_SERVICE )
self._monitor.setComponentName( self._name )
self._monitor.setComponentLocation( self._cfg.getURL() )
self._monitor.initialize()
self._monitor.registerActivity( "Connections", "Connections received", "Framework", "connections", MonitoringClient.OP_RATE )
self._monitor.registerActivity( "Queries", "Queries served", "Framework", "queries", MonitoringClient.OP_RATE )
self._monitor.registerActivity( 'CPU', "CPU Usage", 'Framework', "CPU,%", MonitoringClient.OP_MEAN, 600 )
self._monitor.registerActivity( 'MEM', "Memory Usage", 'Framework', 'Memory,MB', MonitoringClient.OP_MEAN, 600 )
self._monitor.registerActivity( 'PendingQueries', "Pending queries", 'Framework', 'queries', MonitoringClient.OP_MEAN )
self._monitor.registerActivity( 'ActiveQueries', "Active queries", 'Framework', 'threads', MonitoringClient.OP_MEAN )
self._monitor.registerActivity( 'RunningThreads', "Running threads", 'Framework', 'threads', MonitoringClient.OP_MEAN )
self._monitor.registerActivity( 'MaxFD', "Max File Descriptors", 'Framework', 'fd', MonitoringClient.OP_MEAN )
self._monitor.setComponentExtraParam( 'DIRACVersion', DIRAC.version )
self._monitor.setComponentExtraParam( 'platform', DIRAC.getPlatform() )
self._monitor.setComponentExtraParam( 'startTime', Time.dateTime() )
for prop in ( ( "__RCSID__", "version" ), ( "__doc__", "description" ) ):
try:
value = getattr( self._handler[ 'module' ], prop[0] )
except Exception as e:
gLogger.exception( e )
gLogger.error( "Missing property", prop[0] )
value = 'unset'
self._monitor.setComponentExtraParam( prop[1], value )
for secondaryName in self._cfg.registerAlsoAs():
gLogger.info( "Registering %s also as %s" % ( self._name, secondaryName ) )
self._validNames.append( secondaryName )
return S_OK()
def __reportThreadPoolContents( self ):
self._monitor.addMark( 'PendingQueries', self._threadPool.pendingJobs() )
self._monitor.addMark( 'ActiveQueries', self._threadPool.numWorkingThreads() )
self._monitor.addMark( 'RunningThreads', threading.activeCount() )
self._monitor.addMark( 'MaxFD', self.__maxFD )
self.__maxFD = 0
def getConfig( self ):
return self._cfg
#End of initialization functions
def handleConnection( self, clientTransport ):
self._stats[ 'connections' ] += 1
self._monitor.setComponentExtraParam( 'queries', self._stats[ 'connections' ] )
self._threadPool.generateJobAndQueueIt( self._processInThread,
args = ( clientTransport, ) )
#Threaded process function
def _processInThread( self, clientTransport ):
self.__maxFD = max( self.__maxFD, clientTransport.oSocket.fileno() )
self._lockManager.lockGlobal()
try:
monReport = self.__startReportToMonitoring()
except Exception:
monReport = False
try:
#Handshake
try:
result = clientTransport.handshake()
if not result[ 'OK' ]:
clientTransport.close()
return
except:
return
#Add to the transport pool
trid = self._transportPool.add( clientTransport )
if not trid:
return
#Receive and check proposal
result = self._receiveAndCheckProposal( trid )
if not result[ 'OK' ]:
self._transportPool.sendAndClose( trid, result )
return
proposalTuple = result[ 'Value' ]
#Instantiate handler
result = self._instantiateHandler( trid, proposalTuple )
if not result[ 'OK' ]:
self._transportPool.sendAndClose( trid, result )
return
handlerObj = result[ 'Value' ]
#Execute the action
result = self._processProposal( trid, proposalTuple, handlerObj )
#Close the connection if required
if result[ 'closeTransport' ] or not result[ 'OK' ]:
if not result[ 'OK' ]:
gLogger.error( "Error processing proposal", result[ 'Message' ] )
self._transportPool.close( trid )
return result
finally:
self._lockManager.unlockGlobal()
if monReport:
self.__endReportToMonitoring( *monReport )
def _createIdentityString( self, credDict, clientTransport = None ):
if 'username' in credDict:
if 'group' in credDict:
identity = "[%s:%s]" % ( credDict[ 'username' ], credDict[ 'group' ] )
else:
identity = "[%s:unknown]" % credDict[ 'username' ]
else:
identity = 'unknown'
if clientTransport:
addr = clientTransport.getRemoteAddress()
if addr:
addr = "{%s:%s}" % ( addr[0], addr[1] )
if 'DN' in credDict:
identity += "(%s)" % credDict[ 'DN' ]
return identity
def _receiveAndCheckProposal( self, trid ):
clientTransport = self._transportPool.get( trid )
#Get the peer credentials
credDict = clientTransport.getConnectingCredentials()
#Receive the action proposal
retVal = clientTransport.receiveData( 1024 )
if not retVal[ 'OK' ]:
gLogger.error( "Invalid action proposal", "%s %s" % ( self._createIdentityString( credDict,
clientTransport ),
retVal[ 'Message' ] ) )
return S_ERROR( "Invalid action proposal" )
proposalTuple = retVal[ 'Value' ]
gLogger.debug( "Received action from client", "/".join( list( proposalTuple[1] ) ) )
#Check if there are extra credentials
if proposalTuple[2]:
clientTransport.setExtraCredentials( proposalTuple[2] )
#Check if this is the requested service
requestedService = proposalTuple[0][0]
if requestedService not in self._validNames:
return S_ERROR( "%s is not up in this server" % requestedService )
#Check if the action is valid
requestedActionType = proposalTuple[1][0]
if requestedActionType not in Service.SVC_VALID_ACTIONS:
return S_ERROR( "%s is not a known action type" % requestedActionType )
#Check if it's authorized
result = self._authorizeProposal( proposalTuple[1], trid, credDict )
if not result[ 'OK' ]:
return result
#Proposal is OK
return S_OK( proposalTuple )
def _authorizeProposal( self, actionTuple, trid, credDict ):
#Find CS path for the Auth rules
referedAction = self._isMetaAction( actionTuple[0] )
if referedAction:
csAuthPath = "%s/Default" % actionTuple[0]
hardcodedMethodAuth = self._actions[ 'auth' ][ actionTuple[0] ]
else:
if actionTuple[0] == 'RPC':
csAuthPath = actionTuple[1]
else:
csAuthPath = "/".join( actionTuple )
#Find if there are hardcoded auth rules in the code
hardcodedMethodAuth = False
if actionTuple[0] in self._actions[ 'auth' ]:
hardcodedRulesByType = self._actions[ 'auth' ][ actionTuple[0] ]
if actionTuple[0] == "FileTransfer":
methodName = actionTuple[1][0].lower() + actionTuple[1][1:]
else:
methodName = actionTuple[1]
if methodName in hardcodedRulesByType:
hardcodedMethodAuth = hardcodedRulesByType[ methodName ]
#Auth time!
if not self._authMgr.authQuery( csAuthPath, credDict, hardcodedMethodAuth ):
#Get the identity string
identity = self._createIdentityString( credDict )
fromHost = "unknown host"
tr = self._transportPool.get( trid )
if tr:
fromHost = '/'.join( [ str( item ) for item in tr.getRemoteAddress() ] )
gLogger.warn( "Unauthorized query", "to %s:%s by %s from %s" % ( self._name,
"/".join( actionTuple ),
identity, fromHost ) )
result = S_ERROR( "Unauthorized query" )
else:
result = S_OK()
#Security log
tr = self._transportPool.get( trid )
if not tr:
return S_ERROR( "Client disconnected" )
sourceAddress = tr.getRemoteAddress()
identity = self._createIdentityString( credDict )
Service.SVC_SECLOG_CLIENT.addMessage( result[ 'OK' ], sourceAddress[0], sourceAddress[1], identity,
self._cfg.getHostname(),
self._cfg.getPort(),
self._name, "/".join( actionTuple ) )
return result
def _instantiateHandler( self, trid, proposalTuple = None ):
"""
Generate an instance of the handler for a given service
"""
#Generate the client params
clientParams = { 'serviceStartTime' : self._startTime }
if proposalTuple:
clientParams[ 'clientSetup' ] = proposalTuple[0][1]
if len( proposalTuple[0] ) < 3:
clientParams[ 'clientVO' ] = gConfig.getValue( "/DIRAC/VirtualOrganization", "unknown" )
else:
clientParams[ 'clientVO' ] = proposalTuple[0][2]
clientTransport = self._transportPool.get( trid )
if clientTransport:
clientParams[ 'clientAddress' ] = clientTransport.getRemoteAddress()
#Generate handler dict with per client info
handlerInitDict = dict( self._serviceInfoDict )
for key in clientParams:
handlerInitDict[ key ] = clientParams[ key ]
#Instantiate and initialize
try:
handlerInstance = self._handler[ 'class' ]( handlerInitDict, trid )
handlerInstance.initialize()
except Exception as e:
gLogger.exception( "Server error while loading handler: %s" % str( e ) )
return S_ERROR( "Server error while loading handler" )
return S_OK( handlerInstance )
def _processProposal( self, trid, proposalTuple, handlerObj ):
#Notify the client we're ready to execute the action
retVal = self._transportPool.send( trid, S_OK() )
if not retVal[ 'OK' ]:
return retVal
messageConnection = False
if proposalTuple[1] == ( 'Connection', 'new' ):
messageConnection = True
if messageConnection:
if self._msgBroker.getNumConnections() > self._cfg.getMaxMessagingConnections():
result = S_ERROR( "Maximum number of connections reached. Try later" )
result[ 'closeTransport' ] = True
return result
#This is a stable connection
self._msgBroker.addTransportId( trid, self._name,
receiveMessageCallback = self._mbReceivedMsg,
disconnectCallback = self._mbDisconnect,
listenToConnection = False )
result = self._executeAction( trid, proposalTuple, handlerObj )
if result[ 'OK' ] and messageConnection:
self._msgBroker.listenToTransport( trid )
result = self._mbConnect( trid, handlerObj )
if not result[ 'OK' ]:
self._msgBroker.removeTransport( trid )
result[ 'closeTransport' ] = not messageConnection or not result[ 'OK' ]
return result
def _mbConnect( self, trid, handlerObj = None ):
if not handlerObj:
result = self._instantiateHandler( trid )
if not result[ 'OK' ]:
return result
handlerObj = result[ 'Value' ]
return handlerObj._rh_executeConnectionCallback( 'connected' )
def _executeAction( self, trid, proposalTuple, handlerObj ):
try:
return handlerObj._rh_executeAction( proposalTuple )
except Exception as e:
gLogger.exception( "Exception while executing handler action" )
return S_ERROR( "Server error while executing action: %s" % str( e ) )
def _mbReceivedMsg( self, trid, msgObj ):
result = self._authorizeProposal( ( 'Message', msgObj.getName() ),
trid,
self._transportPool.get( trid ).getConnectingCredentials() )
if not result[ 'OK' ]:
return result
result = self._instantiateHandler( trid )
if not result[ 'OK' ]:
return result
handlerObj = result[ 'Value' ]
return handlerObj._rh_executeMessageCallback( msgObj )
def _mbDisconnect( self, trid ):
result = self._instantiateHandler( trid )
if not result[ 'OK' ]:
return result
handlerObj = result[ 'Value' ]
return handlerObj._rh_executeConnectionCallback( 'drop' )
def __startReportToMonitoring( self ):
self._monitor.addMark( "Queries" )
now = time.time()
stats = os.times()
cpuTime = stats[0] + stats[2]
if now - self.__monitorLastStatsUpdate < 0:
return ( now, cpuTime )
# Send CPU consumption mark
wallClock = now - self.__monitorLastStatsUpdate
self.__monitorLastStatsUpdate = now
# Send Memory consumption mark
membytes = MemStat.VmB( 'VmRSS:' )
if membytes:
mem = membytes / ( 1024. * 1024. )
self._monitor.addMark( 'MEM', mem )
return ( now, cpuTime )
def __endReportToMonitoring( self, initialWallTime, initialCPUTime ):
wallTime = time.time() - initialWallTime
stats = os.times()
cpuTime = stats[0] + stats[2] - initialCPUTime
percentage = cpuTime / wallTime * 100.
if percentage > 0:
self._monitor.addMark( 'CPU', percentage )
class GatewayService( Service ):
GATEWAY_NAME = "Framework/Gateway"
def __init__( self ):
Service.__init__( self, GatewayService.GATEWAY_NAME )
self.__delegatedCredentials = DictCache()
self.__transferBytesLimit = 1024 * 1024 * 100
def initialize( self ):
#Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR( "Could not build service URL for %s" % GatewayService.GATEWAY_NAME )
gLogger.verbose( "Service URL is %s" % self._url )
#Discover Handler
self._initMonitoring()
self._threadPool = ThreadPool( 1,
max( 0, self._cfg.getMaxThreads() ),
self._cfg.getMaxWaitingPetitions() )
self._threadPool.daemonize()
self._msgBroker = MessageBroker( "%sMSB" % GatewayService.GATEWAY_NAME, threadPool = self._threadPool )
self._msgBroker.useMessageObjects( False )
getGlobalMessageBroker().useMessageObjects( False )
self._msgForwarder = MessageForwarder( self._msgBroker )
return S_OK()
#Threaded process function
def _processInThread( self, clientTransport ):
#Handshake
try:
clientTransport.handshake()
except:
return
#Add to the transport pool
trid = self._transportPool.add( clientTransport )
if not trid:
return
#Receive and check proposal
result = self._receiveAndCheckProposal( trid )
if not result[ 'OK' ]:
self._transportPool.sendAndClose( trid, result )
return
proposalTuple = result[ 'Value' ]
#Instantiate handler
result = self.__getClientInitArgs( trid, proposalTuple )
if not result[ 'OK' ]:
self._transportPool.sendAndClose( trid, result )
return
clientInitArgs = result[ 'Value' ]
#Execute the action
result = self._processProposal( trid, proposalTuple, clientInitArgs )
#Close the connection if required
if result[ 'closeTransport' ]:
self._transportPool.close( trid )
return result
def _receiveAndCheckProposal( self, trid ):
clientTransport = self._transportPool.get( trid )
#Get the peer credentials
credDict = clientTransport.getConnectingCredentials()
#Receive the action proposal
retVal = clientTransport.receiveData( 1024 )
if not retVal[ 'OK' ]:
gLogger.error( "Invalid action proposal", "%s %s" % ( self._createIdentityString( credDict,
clientTransport ),
retVal[ 'Message' ] ) )
return S_ERROR( "Invalid action proposal" )
proposalTuple = retVal[ 'Value' ]
gLogger.debug( "Received action from client", "/".join( list( proposalTuple[1] ) ) )
#Check if there are extra credentials
if proposalTuple[2]:
clientTransport.setExtraCredentials( proposalTuple[2] )
return S_OK( proposalTuple )
def __getClientInitArgs( self, trid, proposalTuple ):
clientTransport = self._transportPool.get( trid )
#Get the peer credentials
credDict = clientTransport.getConnectingCredentials()
if 'x509Chain' not in credDict:
return S_OK()
cKey = ( credDict[ 'DN' ],
credDict.get( 'group', False ),
credDict.get( 'extraCredentials', False ),
credDict[ 'isLimitedProxy' ] )
dP = self.__delegatedCredentials.get( cKey, 3600 )
idString = self._createIdentityString( credDict, clientTransport )
if dP:
gLogger.verbose( "Proxy for %s is cached" % idString )
return S_OK( dP )
result = self.__requestDelegation( clientTransport, credDict )
if not result[ 'OK' ]:
gLogger.warn( "Could not get proxy for %s: %s" % ( idString, result[ 'Message' ] ) )
return result
delChain = result[ 'Value' ]
delegatedChain = delChain.dumpAllToString()[ 'Value' ]
secsLeft = delChain.getRemainingSecs()[ 'Value' ] - 1
clientInitArgs = {
BaseClient.KW_SETUP : proposalTuple[0][1],
BaseClient.KW_TIMEOUT : 600,
BaseClient.KW_IGNORE_GATEWAYS : True,
BaseClient.KW_USE_CERTIFICATES : False,
BaseClient.KW_PROXY_STRING : delegatedChain
}
if BaseClient.KW_EXTRA_CREDENTIALS in credDict:
clientInitArgs[ BaseClient.KW_EXTRA_CREDENTIALS ] = credDict[ BaseClient.KW_EXTRA_CREDENTIALS ]
gLogger.warn( "Got delegated proxy for %s: %s secs left" % ( idString, secsLeft ) )
self.__delegatedCredentials.add( cKey, secsLeft, clientInitArgs )
return S_OK( clientInitArgs )
def __requestDelegation( self, clientTransport, credDict ):
peerChain = credDict[ 'x509Chain' ]
retVal = peerChain.getCertInChain()[ 'Value' ].generateProxyRequest()
if not retVal[ 'OK' ]:
return retVal
delegationRequest = retVal[ 'Value' ]
retVal = delegationRequest.dumpRequest()
if not retVal[ 'OK' ]:
retVal = S_ERROR( "Server Error: Can't generate delegation request" )
clientTransport.sendData( retVal )
return retVal
gLogger.info( "Sending delegation request for %s" % delegationRequest.getSubjectDN()[ 'Value' ] )
clientTransport.sendData( S_OK( { 'delegate' : retVal[ 'Value' ] } ) )
delegatedCertChain = clientTransport.receiveData()
delegatedChain = X509Chain( keyObj = delegationRequest.getPKey() )
retVal = delegatedChain.loadChainFromString( delegatedCertChain )
if not retVal[ 'OK' ]:
retVal = S_ERROR( "Error in receiving delegated proxy: %s" % retVal[ 'Message' ] )
clientTransport.sendData( retVal )
return retVal
return S_OK( delegatedChain )
#Msg
def _mbConnect( self, trid, clientInitArgs ):
return S_OK()
def _mbReceivedMsg( self, cliTrid, msgObj ):
return self._msgForwarder.msgFromClient( cliTrid, msgObj )
def _mbDisconnect( self, cliTrid ):
self._msgForwarder.cliDisconnect( cliTrid )
#Execute action
def _executeAction( self, trid, proposalTuple, clientInitArgs ):
clientTransport = self._transportPool.get( trid )
credDict = clientTransport.getConnectingCredentials()
targetService = proposalTuple[0][0]
actionType = proposalTuple[1][0]
actionMethod = proposalTuple[1][1]
idString = self._createIdentityString( credDict, clientTransport )
#OOkay! Lets do the magic!
retVal = clientTransport.receiveData()
if not retVal[ 'OK' ]:
gLogger.error( "Error while receiving file description", retVal[ 'Message' ] )
clientTransport.sendData( S_ERROR( "Error while receiving file description: %s" % retVal[ 'Message' ] ) )
return
if actionType == "FileTransfer":
gLogger.warn( "Received a file transfer action from %s" % idString )
clientTransport.sendData( S_OK( "Accepted" ) )
retVal = self.__forwardFileTransferCall( targetService, clientInitArgs,
actionMethod, retVal[ 'Value' ], clientTransport )
elif actionType == "RPC":
gLogger.info( "Forwarding %s/%s action to %s for %s" % ( actionType, actionMethod, targetService, idString ) )
retVal = self.__forwardRPCCall( targetService, clientInitArgs, actionMethod, retVal[ 'Value' ] )
elif actionType == "Connection" and actionMethod == "new":
gLogger.info( "Initiating a messaging connection to %s for %s" % ( targetService, idString ) )
retVal = self._msgForwarder.addClient( trid, targetService, clientInitArgs, retVal[ 'Value' ] )
else:
gLogger.warn( "Received an invalid %s/%s action from %s" % ( actionType, actionMethod, idString ) )
retVal = S_ERROR( "Unknown type of action (%s)" % actionType )
#TODO: Send back the data?
if 'rpcStub' in retVal:
retVal.pop( 'rpcStub' )
clientTransport.sendData( retVal )
return retVal
def __forwardRPCCall( self, targetService, clientInitArgs, method, params ):
if targetService == "Configuration/Server":
if method == "getCompressedDataIfNewer":
#Relay CS data directly
serviceVersion = gConfigurationData.getVersion()
retDict = { 'newestVersion' : serviceVersion }
clientVersion = params[0]
if clientVersion < serviceVersion:
retDict[ 'data' ] = gConfigurationData.getCompressedData()
return S_OK( retDict )
#Default
rpcClient = RPCClient( targetService, **clientInitArgs )
methodObj = getattr( rpcClient, method )
return methodObj( *params )
def __forwardFileTransferCall( self, targetService, clientInitArgs, method,
params, clientTransport ):
transferRelay = TransferRelay( targetService, **clientInitArgs )
transferRelay.setTransferLimit( self.__transferBytesLimit )
cliFH = FileHelper( clientTransport )
#Check file size
if method.find( "ToClient" ) > -1:
cliFH.setDirection( "send" )
elif method.find( "FromClient" ) > -1:
cliFH.setDirection( "receive" )
if not self.__ftCheckMaxTransferSize( params[2] ):
cliFH.markAsTransferred()
return S_ERROR( "Transfer size is too big" )
#Forward queries
try:
relayMethodObject = getattr( transferRelay, 'forward%s' % method )
except:
return S_ERROR( "Cannot forward unknown method %s" % method )
result = relayMethodObject( cliFH, params )
return result
def __ftCheckMaxTransferSize( self, requestedTransferSize ):
if not self.__transferBytesLimit:
return True
if not requestedTransferSize:
return True
if requestedTransferSize <= self.__transferBytesLimit:
return True
return False
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result["OK"]:
return result
self._handler = result["Value"]
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._threadPool = ThreadPoolExecutor(max(0,
self._cfg.getMaxThreads()))
self._msgBroker = MessageBroker("%sMSB" % self._name,
threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {
"serviceName":
self._name,
"serviceSectionPath":
PathFinder.getServiceSection(self._name),
"URL":
self._cfg.getURL(),
"messageSender":
MessageSender(self._name, self._msgBroker),
"validNames":
self._validNames,
"csPaths": [
PathFinder.getServiceSection(svcName)
for svcName in self._validNames
],
}
self.securityLogging = Operations().getValue(
"EnableSecurityLogging", True) and getServiceOption(
self._serviceInfoDict, "EnableSecurityLogging", True)
# Initialize Monitoring
# The import needs to be here because of the CS must be initialized before importing
# this class (see https://github.com/DIRACGrid/DIRAC/issues/4793)
from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter
self.activityMonitoringReporter = MonitoringReporter(
monitoringType="ServiceMonitoring")
self._initMonitoring()
# Call static initialization function
try:
self._handler["class"]._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self.activityMonitoringReporter)
if self._handler["init"]:
for initFunc in self._handler["init"]:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception(
"Exception while calling initialization function",
lException=excp)
return S_ERROR(
"Exception while calling initialization function: %s"
% str(excp))
if not isReturnStructure(result):
return S_ERROR(
"Service initialization function %s must return S_OK/S_ERROR"
% initFunc)
if not result["OK"]:
return S_ERROR("Error while initializing %s: %s" %
(self._name, result["Message"]))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
if self.activityMonitoring:
gThreadScheduler.addPeriodicTask(30, self.__reportActivity)
gThreadScheduler.addPeriodicTask(
100, self.__activityMonitoringReporting)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result["OK"]:
return result
self._actions = result["Value"]
return S_OK()
class Service(object):
SVC_VALID_ACTIONS = {
"RPC": "export",
"FileTransfer": "transfer",
"Message": "msg",
"Connection": "Message"
}
SVC_SECLOG_CLIENT = SecurityLogClient()
def __init__(self, serviceData):
"""
Init the variables for the service
:param serviceData: dict with modName, standalone, loadName, moduleObj, classObj. e.g.:
{'modName': 'Framework/serviceName',
'standalone': True,
'loadName': 'Framework/serviceName',
'moduleObj': <module 'serviceNameHandler' from '/home/DIRAC/FrameworkSystem/Service/serviceNameHandler.pyo'>,
'classObj': <class 'serviceNameHandler.serviceHandler'>}
"""
self._svcData = serviceData
self._name = serviceData["modName"]
self._startTime = datetime.datetime.utcnow()
self._validNames = [serviceData["modName"]]
if serviceData["loadName"] not in self._validNames:
self._validNames.append(serviceData["loadName"])
self._cfg = ServiceConfiguration(list(self._validNames))
self._standalone = serviceData["standalone"]
self.__monitorLastStatsUpdate = time.time()
self._stats = {"queries": 0, "connections": 0}
self._authMgr = AuthManager(
"%s/Authorization" %
PathFinder.getServiceSection(serviceData["loadName"]))
self._transportPool = getGlobalTransportPool()
self.__cloneId = 0
self.__maxFD = 0
self.activityMonitoring = False
# Check if monitoring is enabled
if "Monitoring" in Operations().getMonitoringBackends(
monitoringType="ServiceMonitoring"):
self.activityMonitoring = True
def setCloneProcessId(self, cloneId):
self.__cloneId = cloneId
def _isMetaAction(self, action):
referedAction = Service.SVC_VALID_ACTIONS[action]
if referedAction in Service.SVC_VALID_ACTIONS:
return referedAction
return False
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result["OK"]:
return result
self._handler = result["Value"]
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
self._threadPool = ThreadPoolExecutor(max(0,
self._cfg.getMaxThreads()))
self._msgBroker = MessageBroker("%sMSB" % self._name,
threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {
"serviceName":
self._name,
"serviceSectionPath":
PathFinder.getServiceSection(self._name),
"URL":
self._cfg.getURL(),
"messageSender":
MessageSender(self._name, self._msgBroker),
"validNames":
self._validNames,
"csPaths": [
PathFinder.getServiceSection(svcName)
for svcName in self._validNames
],
}
self.securityLogging = Operations().getValue(
"EnableSecurityLogging", True) and getServiceOption(
self._serviceInfoDict, "EnableSecurityLogging", True)
# Initialize Monitoring
# The import needs to be here because of the CS must be initialized before importing
# this class (see https://github.com/DIRACGrid/DIRAC/issues/4793)
from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter
self.activityMonitoringReporter = MonitoringReporter(
monitoringType="ServiceMonitoring")
self._initMonitoring()
# Call static initialization function
try:
self._handler["class"]._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self.activityMonitoringReporter)
if self._handler["init"]:
for initFunc in self._handler["init"]:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception(
"Exception while calling initialization function",
lException=excp)
return S_ERROR(
"Exception while calling initialization function: %s"
% str(excp))
if not isReturnStructure(result):
return S_ERROR(
"Service initialization function %s must return S_OK/S_ERROR"
% initFunc)
if not result["OK"]:
return S_ERROR("Error while initializing %s: %s" %
(self._name, result["Message"]))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
if self.activityMonitoring:
gThreadScheduler.addPeriodicTask(30, self.__reportActivity)
gThreadScheduler.addPeriodicTask(
100, self.__activityMonitoringReporting)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result["OK"]:
return result
self._actions = result["Value"]
return S_OK()
def __searchInitFunctions(self, handlerClass, currentClass=None):
if not currentClass:
currentClass = handlerClass
initFuncs = []
ancestorHasInit = False
for ancestor in currentClass.__bases__:
initFuncs += self.__searchInitFunctions(handlerClass, ancestor)
if "initializeHandler" in dir(ancestor):
ancestorHasInit = True
if ancestorHasInit:
initFuncs.append(
super(currentClass, handlerClass).initializeHandler)
if currentClass == handlerClass and "initializeHandler" in dir(
handlerClass):
initFuncs.append(handlerClass.initializeHandler)
return initFuncs
def _loadHandlerInit(self):
handlerClass = self._svcData["classObj"]
handlerName = handlerClass.__name__
handlerInitMethods = self.__searchInitFunctions(handlerClass)
try:
handlerInitMethods.append(
getattr(self._svcData["moduleObj"],
"initialize%s" % handlerName))
except AttributeError:
gLogger.verbose(
"Not found global initialization function for service")
if handlerInitMethods:
gLogger.info("Found %s initialization methods" %
len(handlerInitMethods))
handlerInfo = {}
handlerInfo["name"] = handlerName
handlerInfo["module"] = self._svcData["moduleObj"]
handlerInfo["class"] = handlerClass
handlerInfo["init"] = handlerInitMethods
return S_OK(handlerInfo)
def _loadActions(self):
handlerClass = self._handler["class"]
authRules = {}
typeCheck = {}
methodsList = {}
for actionType in Service.SVC_VALID_ACTIONS:
if self._isMetaAction(actionType):
continue
authRules[actionType] = {}
typeCheck[actionType] = {}
methodsList[actionType] = []
handlerAttributeList = dir(handlerClass)
for actionType in Service.SVC_VALID_ACTIONS:
if self._isMetaAction(actionType):
continue
methodPrefix = "%s_" % Service.SVC_VALID_ACTIONS[actionType]
for attribute in handlerAttributeList:
if attribute.find(methodPrefix) != 0:
continue
exportedName = attribute[len(methodPrefix):]
methodsList[actionType].append(exportedName)
gLogger.verbose("+ Found %s method %s" %
(actionType, exportedName))
# Create lock for method
self._lockManager.createLock(
"%s/%s" % (actionType, exportedName),
self._cfg.getMaxThreadsForMethod(actionType, exportedName))
# Look for type and auth rules
if actionType == "RPC":
typeAttr = "types_%s" % exportedName
authAttr = "auth_%s" % exportedName
else:
typeAttr = "types_%s_%s" % (
Service.SVC_VALID_ACTIONS[actionType], exportedName)
authAttr = "auth_%s_%s" % (
Service.SVC_VALID_ACTIONS[actionType], exportedName)
if typeAttr in handlerAttributeList:
obj = getattr(handlerClass, typeAttr)
gLogger.verbose("|- Found type definition %s: %s" %
(typeAttr, str(obj)))
typeCheck[actionType][exportedName] = obj
if authAttr in handlerAttributeList:
obj = getattr(handlerClass, authAttr)
gLogger.verbose("|- Found auth rules %s: %s" %
(authAttr, str(obj)))
authRules[actionType][exportedName] = obj
for actionType in Service.SVC_VALID_ACTIONS:
referedAction = self._isMetaAction(actionType)
if not referedAction:
continue
gLogger.verbose("Action %s is a meta action for %s" %
(actionType, referedAction))
authRules[actionType] = []
for method in authRules[referedAction]:
for prop in authRules[referedAction][method]:
if prop not in authRules[actionType]:
authRules[actionType].append(prop)
gLogger.verbose("Meta action %s props are %s" %
(actionType, authRules[actionType]))
return S_OK({
"methods": methodsList,
"auth": authRules,
"types": typeCheck
})
def _initMonitoring(self):
props = [("__doc__", "description")]
for prop in props:
try:
value = getattr(self._handler["module"], prop[0])
except Exception as e:
gLogger.exception(e)
gLogger.error("Missing property", prop[0])
value = "unset"
for secondaryName in self._cfg.registerAlsoAs():
gLogger.info("Registering %s also as %s" %
(self._name, secondaryName))
self._validNames.append(secondaryName)
return S_OK()
def __reportActivity(self):
initialWallTime, initialCPUTime, mem = self.__startReportToMonitoring()
pendingQueries = self._threadPool._work_queue.qsize()
activeQuereies = len(self._threadPool._threads)
percentage = self.__endReportToMonitoring(initialWallTime,
initialCPUTime)
self.activityMonitoringReporter.addRecord({
"timestamp":
int(TimeUtilities.toEpoch()),
"Host":
Network.getFQDN(),
"ServiceName":
"_".join(self._name.split("/")),
"Location":
self._cfg.getURL(),
"MemoryUsage":
mem,
"CpuPercentage":
percentage,
"PendingQueries":
pendingQueries,
"ActiveQueries":
activeQuereies,
"RunningThreads":
threading.activeCount(),
"MaxFD":
self.__maxFD,
})
self.__maxFD = 0
def getConfig(self):
return self._cfg
# End of initialization functions
def handleConnection(self, clientTransport):
"""
This method may be called by ServiceReactor.
The method stacks openened connection in a queue, another thread
read this queue and handle connection.
:param clientTransport: Object which describes opened connection (PlainTransport or SSLTransport)
"""
if not self.activityMonitoring:
self._stats["connections"] += 1
self._threadPool.submit(self._processInThread, clientTransport)
@property
def wantsThrottle(self):
"""Boolean property for if the service wants requests to stop being accepted"""
nQueued = self._threadPool._work_queue.qsize()
return nQueued > self._cfg.getMaxWaitingPetitions()
# Threaded process function
def _processInThread(self, clientTransport):
"""
This method handles a RPC, FileTransfer or Connection.
Connection may be opened via ServiceReactor.__acceptIncomingConnection
- Do the SSL/TLS Handshake (if dips is used) and extract credentials
- Get the action called by the client
- Check if the client is authorized to perform ation
- If not, connection is closed
- Instanciate the RequestHandler (RequestHandler contain all methods callable)
(Following is not directly in this method but it describe what happen at
#Execute the action)
- Notify the client we're ready to execute the action (via _processProposal)
and call RequestHandler._rh_executeAction()
- Receive arguments/file/something else (depending on action) in the RequestHandler
- Executing the action asked by the client
:param clientTransport: Object which describe the opened connection (SSLTransport or PlainTransport)
:return: S_OK with "closeTransport" a boolean to indicate if th connection have to be closed
e.g. after RPC, closeTransport=True
"""
self.__maxFD = max(self.__maxFD, clientTransport.oSocket.fileno())
self._lockManager.lockGlobal()
try:
monReport = self.__startReportToMonitoring()
except Exception:
monReport = False
try:
# Handshake
try:
result = clientTransport.handshake()
if not result["OK"]:
clientTransport.close()
return
except Exception:
return
# Add to the transport pool
trid = self._transportPool.add(clientTransport)
if not trid:
return
# Receive and check proposal
result = self._receiveAndCheckProposal(trid)
if not result["OK"]:
self._transportPool.sendAndClose(trid, result)
return
proposalTuple = result["Value"]
# Instantiate handler
result = self._instantiateHandler(trid, proposalTuple)
if not result["OK"]:
self._transportPool.sendAndClose(trid, result)
return
handlerObj = result["Value"]
# Execute the action
result = self._processProposal(trid, proposalTuple, handlerObj)
# Close the connection if required
if result["closeTransport"] or not result["OK"]:
if not result["OK"]:
gLogger.error("Error processing proposal",
result["Message"])
self._transportPool.close(trid)
return result
finally:
self._lockManager.unlockGlobal()
if monReport:
self.__endReportToMonitoring(monReport[0], monReport[1])
@staticmethod
def _createIdentityString(credDict, clientTransport=None):
if "username" in credDict:
if "group" in credDict:
identity = "[%s:%s]" % (credDict["username"],
credDict["group"])
else:
identity = "[%s:unknown]" % credDict["username"]
else:
identity = "unknown"
if clientTransport:
addr = clientTransport.getRemoteAddress()
if addr:
addr = "{%s:%s}" % (addr[0], addr[1])
if "DN" in credDict:
identity += "(%s)" % credDict["DN"]
return identity
@staticmethod
def _deserializeProposalTuple(serializedProposal):
"""We receive the proposalTuple as a list.
Turn it into a tuple again
"""
proposalTuple = tuple(
tuple(x) if isinstance(x, list) else x for x in serializedProposal)
return proposalTuple
def _receiveAndCheckProposal(self, trid):
clientTransport = self._transportPool.get(trid)
# Get the peer credentials
credDict = clientTransport.getConnectingCredentials()
# Receive the action proposal
retVal = clientTransport.receiveData(1024)
if not retVal["OK"]:
gLogger.error(
"Invalid action proposal",
"%s %s" % (self._createIdentityString(
credDict, clientTransport), retVal["Message"]),
)
return S_ERROR("Invalid action proposal")
proposalTuple = Service._deserializeProposalTuple(retVal["Value"])
gLogger.debug("Received action from client",
"/".join(list(proposalTuple[1])))
# Check if there are extra credentials
if proposalTuple[2]:
clientTransport.setExtraCredentials(proposalTuple[2])
# Check if this is the requested service
requestedService = proposalTuple[0][0]
if requestedService not in self._validNames:
return S_ERROR("%s is not up in this server" % requestedService)
# Check if the action is valid
requestedActionType = proposalTuple[1][0]
if requestedActionType not in Service.SVC_VALID_ACTIONS:
return S_ERROR("%s is not a known action type" %
requestedActionType)
# Check if it's authorized
result = self._authorizeProposal(proposalTuple[1], trid, credDict)
if not result["OK"]:
return result
# Proposal is OK
return S_OK(proposalTuple)
def _authorizeProposal(self, actionTuple, trid, credDict):
# Find CS path for the Auth rules
referedAction = self._isMetaAction(actionTuple[0])
if referedAction:
csAuthPath = "%s/Default" % actionTuple[0]
hardcodedMethodAuth = self._actions["auth"][actionTuple[0]]
else:
if actionTuple[0] == "RPC":
csAuthPath = actionTuple[1]
else:
csAuthPath = "/".join(actionTuple)
# Find if there are hardcoded auth rules in the code
hardcodedMethodAuth = False
if actionTuple[0] in self._actions["auth"]:
hardcodedRulesByType = self._actions["auth"][actionTuple[0]]
if actionTuple[0] == "FileTransfer":
methodName = actionTuple[1][0].lower() + actionTuple[1][1:]
else:
methodName = actionTuple[1]
if methodName in hardcodedRulesByType:
hardcodedMethodAuth = hardcodedRulesByType[methodName]
# Auth time!
if not self._authMgr.authQuery(csAuthPath, credDict,
hardcodedMethodAuth):
# Get the identity string
identity = self._createIdentityString(credDict)
fromHost = "unknown host"
tr = self._transportPool.get(trid)
if tr:
fromHost = "/".join(
[str(item) for item in tr.getRemoteAddress()])
gLogger.warn(
"Unauthorized query", "to %s:%s by %s from %s" %
(self._name, "/".join(actionTuple), identity, fromHost))
result = S_ERROR(ENOAUTH, "Unauthorized query")
else:
result = S_OK()
# Security log
tr = self._transportPool.get(trid)
if not tr:
return S_ERROR("Client disconnected")
sourceAddress = tr.getRemoteAddress()
identity = self._createIdentityString(credDict)
if self.securityLogging:
Service.SVC_SECLOG_CLIENT.addMessage(
result["OK"],
sourceAddress[0],
sourceAddress[1],
identity,
self._cfg.getHostname(),
self._cfg.getPort(),
self._name,
"/".join(actionTuple),
)
return result
def _instantiateHandler(self, trid, proposalTuple=None):
"""
Generate an instance of the handler for a given service
:param int trid: transport ID
:param tuple proposalTuple: tuple describing the proposed action
:return: S_OK/S_ERROR, Value is the handler object
"""
# Generate the client params
clientParams = {"serviceStartTime": self._startTime}
if proposalTuple:
# The 4th element is the client version
clientParams["clientVersion"] = proposalTuple[3] if len(
proposalTuple) > 3 else None
clientParams["clientSetup"] = proposalTuple[0][1]
if len(proposalTuple[0]) < 3:
clientParams["clientVO"] = gConfig.getValue(
"/DIRAC/VirtualOrganization", "unknown")
else:
clientParams["clientVO"] = proposalTuple[0][2]
clientTransport = self._transportPool.get(trid)
if clientTransport:
clientParams["clientAddress"] = clientTransport.getRemoteAddress()
# Generate handler dict with per client info
handlerInitDict = dict(self._serviceInfoDict)
for key in clientParams:
handlerInitDict[key] = clientParams[key]
# Instantiate and initialize
try:
handlerInstance = self._handler["class"](handlerInitDict, trid)
handlerInstance.initialize()
except Exception as e:
gLogger.exception("Server error while loading handler: %s" %
str(e))
return S_ERROR("Server error while loading handler")
return S_OK(handlerInstance)
def _processProposal(self, trid, proposalTuple, handlerObj):
# Notify the client we're ready to execute the action
retVal = self._transportPool.send(trid, S_OK())
if not retVal["OK"]:
return retVal
messageConnection = False
if proposalTuple[1] == ("Connection", "new"):
messageConnection = True
if messageConnection:
if self._msgBroker.getNumConnections(
) > self._cfg.getMaxMessagingConnections():
result = S_ERROR(
"Maximum number of connections reached. Try later")
result["closeTransport"] = True
return result
# This is a stable connection
self._msgBroker.addTransportId(
trid,
self._name,
receiveMessageCallback=self._mbReceivedMsg,
disconnectCallback=self._mbDisconnect,
listenToConnection=False,
)
result = self._executeAction(trid, proposalTuple, handlerObj)
if result["OK"] and messageConnection:
self._msgBroker.listenToTransport(trid)
result = self._mbConnect(trid, handlerObj)
if not result["OK"]:
self._msgBroker.removeTransport(trid)
result["closeTransport"] = not messageConnection or not result["OK"]
return result
def _mbConnect(self, trid, handlerObj=None):
if not handlerObj:
result = self._instantiateHandler(trid)
if not result["OK"]:
return result
handlerObj = result["Value"]
return handlerObj._rh_executeConnectionCallback("connected")
def _executeAction(self, trid, proposalTuple, handlerObj):
try:
response = handlerObj._rh_executeAction(proposalTuple)
if not response["OK"]:
return response
if self.activityMonitoring:
self.activityMonitoringReporter.addRecord({
"timestamp":
int(TimeUtilities.toEpoch()),
"Host":
Network.getFQDN(),
"ServiceName":
"_".join(self._name.split("/")),
"Location":
self._cfg.getURL(),
"ResponseTime":
response["Value"][1],
})
return response["Value"][0]
except Exception as e:
gLogger.exception("Exception while executing handler action")
return S_ERROR("Server error while executing action: %s" % str(e))
def _mbReceivedMsg(self, trid, msgObj):
result = self._authorizeProposal(
("Message", msgObj.getName()), trid,
self._transportPool.get(trid).getConnectingCredentials())
if not result["OK"]:
return result
result = self._instantiateHandler(trid)
if not result["OK"]:
return result
handlerObj = result["Value"]
response = handlerObj._rh_executeMessageCallback(msgObj)
if self.activityMonitoring and response["OK"]:
self.activityMonitoringReporter.addRecord({
"timestamp":
int(TimeUtilities.toEpoch()),
"Host":
Network.getFQDN(),
"ServiceName":
"_".join(self._name.split("/")),
"Location":
self._cfg.getURL(),
"ResponseTime":
response["Value"][1],
})
if response["OK"]:
return response["Value"][0]
else:
return response
def _mbDisconnect(self, trid):
result = self._instantiateHandler(trid)
if not result["OK"]:
return result
handlerObj = result["Value"]
return handlerObj._rh_executeConnectionCallback("drop")
def __activityMonitoringReporting(self):
"""This method is called by the ThreadScheduler as a periodic task in order to commit the collected data which
is done by the MonitoringReporter and is send to the 'ComponentMonitoring' type.
:return: True / False
"""
return self.activityMonitoringReporter.commit()
def __startReportToMonitoring(self):
now = time.time()
stats = os.times()
cpuTime = stats[0] + stats[2]
mem = None
if now - self.__monitorLastStatsUpdate < 0:
return (now, cpuTime, mem)
self.__monitorLastStatsUpdate = now
membytes = MemStat.VmB("VmRSS:")
if membytes:
mem = membytes / (1024.0 * 1024.0)
return (now, cpuTime, mem)
def __endReportToMonitoring(self, initialWallTime, initialCPUTime):
wallTime = time.time() - initialWallTime
stats = os.times()
cpuTime = stats[0] + stats[2] - initialCPUTime
percentage = cpuTime / wallTime * 100.0
return percentage
class GatewayService(Service):
"""Inherits from Service so it can (and should) be run as a DIRAC service,
but replaces several of the internal methods
"""
GATEWAY_NAME = "Framework/Gateway"
def __init__(self):
"""Initialize like a real service"""
super(GatewayService, self).__init__(
{
"modName": GatewayService.GATEWAY_NAME,
"loadName": GatewayService.GATEWAY_NAME,
"standalone": True,
"moduleObj": sys.modules[DIRAC.Core.DISET.private.GatewayService.GatewayService.__module__],
"classObj": self.__class__,
}
)
self.__delegatedCredentials = DictCache()
self.__transferBytesLimit = 1024 * 1024 * 100
# to be resolved
self._url = None
self._handler = None
self._threadPool = None
self._msgBroker = None
self._msgForwarder = None
def initialize(self):
"""This replaces the standard initialize from Service"""
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % GatewayService.GATEWAY_NAME)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result["OK"]:
return result
self._handler = result["Value"]
# Discover Handler
self._threadPool = ThreadPoolExecutor(max(0, self._cfg.getMaxThreads()))
self._msgBroker = MessageBroker("%sMSB" % GatewayService.GATEWAY_NAME, threadPool=self._threadPool)
self._msgBroker.useMessageObjects(False)
getGlobalMessageBroker().useMessageObjects(False)
self._msgForwarder = MessageForwarder(self._msgBroker)
return S_OK()
def _processInThread(self, clientTransport):
"""Threaded process function"""
# Handshake
try:
clientTransport.handshake()
except Exception:
return
# Add to the transport pool
trid = self._transportPool.add(clientTransport)
if not trid:
return
# Receive and check proposal
result = self._receiveAndCheckProposal(trid)
if not result["OK"]:
self._transportPool.sendAndClose(trid, result)
return
proposalTuple = result["Value"]
# Instantiate handler
result = self.__getClientInitArgs(trid, proposalTuple)
if not result["OK"]:
self._transportPool.sendAndClose(trid, result)
return
clientInitArgs = result["Value"]
# Execute the action
result = self._processProposal(trid, proposalTuple, clientInitArgs)
# Close the connection if required
if result["closeTransport"]:
self._transportPool.close(trid)
return result
def _receiveAndCheckProposal(self, trid):
clientTransport = self._transportPool.get(trid)
# Get the peer credentials
credDict = clientTransport.getConnectingCredentials()
# Receive the action proposal
retVal = clientTransport.receiveData(1024)
if not retVal["OK"]:
gLogger.error(
"Invalid action proposal",
"%s %s" % (self._createIdentityString(credDict, clientTransport), retVal["Message"]),
)
return S_ERROR("Invalid action proposal")
proposalTuple = retVal["Value"]
gLogger.debug("Received action from client", "/".join(list(proposalTuple[1])))
# Check if there are extra credentials
if proposalTuple[2]:
clientTransport.setExtraCredentials(proposalTuple[2])
return S_OK(proposalTuple)
def __getClientInitArgs(self, trid, proposalTuple):
clientTransport = self._transportPool.get(trid)
# Get the peer credentials
credDict = clientTransport.getConnectingCredentials()
if "x509Chain" not in credDict:
return S_OK()
cKey = (
credDict["DN"],
credDict.get("group", False),
credDict.get("extraCredentials", False),
credDict["isLimitedProxy"],
)
dP = self.__delegatedCredentials.get(cKey, 3600)
idString = self._createIdentityString(credDict, clientTransport)
if dP:
gLogger.verbose("Proxy for %s is cached" % idString)
return S_OK(dP)
result = self.__requestDelegation(clientTransport, credDict)
if not result["OK"]:
gLogger.warn("Could not get proxy for %s: %s" % (idString, result["Message"]))
return result
delChain = result["Value"]
delegatedChain = delChain.dumpAllToString()["Value"]
secsLeft = delChain.getRemainingSecs()["Value"] - 1
clientInitArgs = {
BaseClient.KW_SETUP: proposalTuple[0][1],
BaseClient.KW_TIMEOUT: 600,
BaseClient.KW_IGNORE_GATEWAYS: True,
BaseClient.KW_USE_CERTIFICATES: False,
BaseClient.KW_PROXY_STRING: delegatedChain,
}
if BaseClient.KW_EXTRA_CREDENTIALS in credDict:
clientInitArgs[BaseClient.KW_EXTRA_CREDENTIALS] = credDict[BaseClient.KW_EXTRA_CREDENTIALS]
gLogger.warn("Got delegated proxy for %s: %s secs left" % (idString, secsLeft))
self.__delegatedCredentials.add(cKey, secsLeft, clientInitArgs)
return S_OK(clientInitArgs)
def __requestDelegation(self, clientTransport, credDict):
peerChain = credDict["x509Chain"]
retVal = peerChain.getCertInChain()["Value"].generateProxyRequest()
if not retVal["OK"]:
return retVal
delegationRequest = retVal["Value"]
retVal = delegationRequest.dumpRequest()
if not retVal["OK"]:
retVal = S_ERROR("Server Error: Can't generate delegation request")
clientTransport.sendData(retVal)
return retVal
gLogger.info("Sending delegation request for %s" % delegationRequest.getSubjectDN()["Value"])
clientTransport.sendData(S_OK({"delegate": retVal["Value"]}))
delegatedCertChain = clientTransport.receiveData()
delegatedChain = X509Chain(keyObj=delegationRequest.getPKey())
retVal = delegatedChain.loadChainFromString(delegatedCertChain)
if not retVal["OK"]:
retVal = S_ERROR("Error in receiving delegated proxy: %s" % retVal["Message"])
clientTransport.sendData(retVal)
return retVal
return S_OK(delegatedChain)
# Msg
def _mbConnect(self, trid, handlerObj=None):
return S_OK()
def _mbReceivedMsg(self, cliTrid, msgObj):
return self._msgForwarder.msgFromClient(cliTrid, msgObj)
def _mbDisconnect(self, cliTrid):
self._msgForwarder.cliDisconnect(cliTrid)
# Execute action
def _executeAction(self, trid, proposalTuple, clientInitArgs):
clientTransport = self._transportPool.get(trid)
credDict = clientTransport.getConnectingCredentials()
targetService = proposalTuple[0][0]
actionType = proposalTuple[1][0]
actionMethod = proposalTuple[1][1]
idString = self._createIdentityString(credDict, clientTransport)
# OOkay! Lets do the magic!
retVal = clientTransport.receiveData()
if not retVal["OK"]:
gLogger.error("Error while receiving file description", retVal["Message"])
clientTransport.sendData(S_ERROR("Error while receiving file description: %s" % retVal["Message"]))
return
if actionType == "FileTransfer":
gLogger.warn("Received a file transfer action from %s" % idString)
clientTransport.sendData(S_OK("Accepted"))
retVal = self.__forwardFileTransferCall(
targetService, clientInitArgs, actionMethod, retVal["Value"], clientTransport
)
elif actionType == "RPC":
gLogger.info("Forwarding %s/%s action to %s for %s" % (actionType, actionMethod, targetService, idString))
retVal = self.__forwardRPCCall(targetService, clientInitArgs, actionMethod, retVal["Value"])
elif actionType == "Connection" and actionMethod == "new":
gLogger.info("Initiating a messaging connection to %s for %s" % (targetService, idString))
retVal = self._msgForwarder.addClient(trid, targetService, clientInitArgs, retVal["Value"])
else:
gLogger.warn("Received an invalid %s/%s action from %s" % (actionType, actionMethod, idString))
retVal = S_ERROR("Unknown type of action (%s)" % actionType)
# TODO: Send back the data?
if "rpcStub" in retVal:
retVal.pop("rpcStub")
clientTransport.sendData(retVal)
return retVal
def __forwardRPCCall(self, targetService, clientInitArgs, method, params):
if targetService == "Configuration/Server":
if method == "getCompressedDataIfNewer":
# Relay CS data directly
serviceVersion = gConfigurationData.getVersion()
retDict = {"newestVersion": serviceVersion}
clientVersion = params[0]
if clientVersion < serviceVersion:
retDict["data"] = gConfigurationData.getCompressedData()
return S_OK(retDict)
# Default
rpcClient = RPCClient(targetService, **clientInitArgs)
methodObj = getattr(rpcClient, method)
return methodObj(*params)
def __forwardFileTransferCall(self, targetService, clientInitArgs, method, params, clientTransport):
transferRelay = TransferRelay(targetService, **clientInitArgs)
transferRelay.setTransferLimit(self.__transferBytesLimit)
cliFH = FileHelper(clientTransport)
# Check file size
if method.find("ToClient") > -1:
cliFH.setDirection("send")
elif method.find("FromClient") > -1:
cliFH.setDirection("receive")
if not self.__ftCheckMaxTransferSize(params[2]):
cliFH.markAsTransferred()
return S_ERROR("Transfer size is too big")
# Forward queries
try:
relayMethodObject = getattr(transferRelay, "forward%s" % method)
except Exception:
return S_ERROR("Cannot forward unknown method %s" % method)
result = relayMethodObject(cliFH, params)
return result
def __ftCheckMaxTransferSize(self, requestedTransferSize):
if not self.__transferBytesLimit:
return True
if not requestedTransferSize:
return True
if requestedTransferSize <= self.__transferBytesLimit:
return True
return False
def initialize(self):
# Build the URLs
self._url = self._cfg.getURL()
if not self._url:
return S_ERROR("Could not build service URL for %s" % self._name)
gLogger.verbose("Service URL is %s" % self._url)
# Load handler
result = self._loadHandlerInit()
if not result['OK']:
return result
self._handler = result['Value']
# Initialize lock manager
self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions())
# TODO: remove ThreadPool
if useThreadPoolExecutor:
self._threadPool = ThreadPoolExecutor(
max(0, self._cfg.getMaxThreads()))
else:
self._threadPool = ThreadPool(max(1, self._cfg.getMinThreads()),
max(0, self._cfg.getMaxThreads()),
self._cfg.getMaxWaitingPetitions())
self._threadPool.daemonize()
self._msgBroker = MessageBroker("%sMSB" % self._name,
threadPool=self._threadPool)
# Create static dict
self._serviceInfoDict = {
'serviceName':
self._name,
'serviceSectionPath':
PathFinder.getServiceSection(self._name),
'URL':
self._cfg.getURL(),
'messageSender':
MessageSender(self._name, self._msgBroker),
'validNames':
self._validNames,
'csPaths': [
PathFinder.getServiceSection(svcName)
for svcName in self._validNames
]
}
# Initialize Monitoring
# This is a flag used to check whether "EnableActivityMonitoring" is enabled or not from the config file.
self.activityMonitoring = (Operations().getValue(
"EnableActivityMonitoring", False) or getServiceOption(
self._serviceInfoDict, "EnableActivityMonitoring", False))
if self.activityMonitoring:
# The import needs to be here because of the CS must be initialized before importing
# this class (see https://github.com/DIRACGrid/DIRAC/issues/4793)
from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter
self.activityMonitoringReporter = MonitoringReporter(
monitoringType="ComponentMonitoring")
gThreadScheduler.addPeriodicTask(
100, self.__activityMonitoringReporting)
elif self._standalone:
self._monitor = gMonitor
else:
self._monitor = MonitoringClient()
self._initMonitoring()
# Call static initialization function
try:
if self.activityMonitoring:
self._handler['class']._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self.activityMonitoringReporter)
else:
self._handler['class']._rh__initializeClass(
dict(self._serviceInfoDict), self._lockManager,
self._msgBroker, self._monitor)
if self._handler['init']:
for initFunc in self._handler['init']:
gLogger.verbose("Executing initialization function")
try:
result = initFunc(dict(self._serviceInfoDict))
except Exception as excp:
gLogger.exception(
"Exception while calling initialization function",
lException=excp)
return S_ERROR(
"Exception while calling initialization function: %s"
% str(excp))
if not isReturnStructure(result):
return S_ERROR(
"Service initialization function %s must return S_OK/S_ERROR"
% initFunc)
if not result['OK']:
return S_ERROR("Error while initializing %s: %s" %
(self._name, result['Message']))
except Exception as e:
errMsg = "Exception while initializing %s" % self._name
gLogger.exception(e)
gLogger.exception(errMsg)
return S_ERROR(errMsg)
# Load actions after the handler has initialized itself
result = self._loadActions()
if not result['OK']:
return result
self._actions = result['Value']
if not self.activityMonitoring:
gThreadScheduler.addPeriodicTask(30,
self.__reportThreadPoolContents)
return S_OK()
