def __init__(self, name, transportPool=None, threadPool=None):
self.__name = name
self.__messageTransports = {}
self.__msgCounter = 0
self.__msgCounterLock = threading.Lock()
self.__responseCallbacks = {}
self.__msgInTransport = {}
self.__listenPersistConn = False
self.__useMessageObjects = True
self.__callbacksLock = threading.Condition()
self.__trInOutLock = threading.Lock()
self.__msgFactory = MessageFactory()
self.__log = gLogger.getSubLogger("MSGBRK")
if not transportPool:
transportPool = getGlobalTransportPool()
self.__trPool = transportPool
if not threadPool:
threadPool = getGlobalThreadPool()
self.__threadPool = threadPool
self.__listeningForMessages = False
def __init__( self, name, transportPool = None, threadPool = None ):
self.__name = name
self.__messageTransports = {}
self.__msgCounter = 0
self.__msgCounterLock = threading.Lock()
self.__responseCallbacks = {}
self.__msgInTransport = {}
self.__listenPersistConn = False
self.__useMessageObjects = True
self.__callbacksLock = threading.Condition()
self.__trInOutLock = threading.Lock()
self.__msgFactory = MessageFactory()
self.__log = gLogger.getSubLogger( "MSGBRK" )
if not transportPool:
transportPool = getGlobalTransportPool()
self.__trPool = transportPool
if not threadPool:
threadPool = getGlobalThreadPool()
self.__threadPool = threadPool
self.__listeningForMessages = False
class MessageBroker(object):
def __init__(self, name, transportPool=None, threadPool=None):
self.__name = name
self.__messageTransports = {}
self.__msgCounter = 0
self.__msgCounterLock = threading.Lock()
self.__responseCallbacks = {}
self.__msgInTransport = {}
self.__listenPersistConn = False
self.__useMessageObjects = True
self.__callbacksLock = threading.Condition()
self.__trInOutLock = threading.Lock()
self.__msgFactory = MessageFactory()
self.__log = gLogger.getSubLogger("MSGBRK")
if not transportPool:
transportPool = getGlobalTransportPool()
self.__trPool = transportPool
if not threadPool:
threadPool = ThreadPoolExecutor(100)
self.__threadPool = threadPool
self.__listeningForMessages = False
self.__listenThread = None
def getNumConnections(self):
return len(self.__messageTransports)
def getMsgFactory(self):
return self.__msgFactory
def useMessageObjects(self, bD):
self.__useMessageObjects = bD
# Message id generation
def __generateMsgId(self):
self.__msgCounterLock.acquire()
try:
msgId = "%s:%d" % (self.__name, self.__msgCounter)
self.__msgCounter += 1
return msgId
finally:
self.__msgCounterLock.release()
def getTransportPool(self):
return self.__trPool
# Add and remove transport to/from broker
def addTransport(self, transport, *args, **kwargs):
trid = self.__trPool.add(transport)
try:
result = self.addTransportId(trid, *args, **kwargs)
except Exception as e:
gLogger.exception("Cannot add transport id", lException=e)
result = S_ERROR("Cannot add transport id")
if not result["OK"]:
self.__trPool.remove(trid)
return result
return S_OK(trid)
def addTransportId(
self,
trid,
svcName,
receiveMessageCallback=None,
disconnectCallback=None,
idleRead=False,
listenToConnection=True,
):
self.__trInOutLock.acquire()
try:
if trid in self.__messageTransports:
return S_OK()
tr = self.__trPool.get(trid)
if not tr:
return S_ERROR("No transport with id %s registered" % trid)
self.__messageTransports[trid] = {
"transport": tr,
"svcName": svcName,
"cbReceiveMessage": receiveMessageCallback,
"cbDisconnect": disconnectCallback,
"listen": listenToConnection,
"idleRead": idleRead,
}
self.__startListeningThread()
return S_OK()
finally:
self.__trInOutLock.release()
def listenToTransport(self, trid, listen=True):
self.__trInOutLock.acquire()
try:
if trid in self.__messageTransports:
self.__messageTransports[trid]["listen"] = listen
self.__startListeningThread()
finally:
self.__trInOutLock.release()
# Listen to connections
def __startListeningThread(self):
threadDead = (self.__listeningForMessages
and self.__listenThread is not None
and not self.__listenThread.is_alive())
if not self.__listeningForMessages or threadDead:
self.__listeningForMessages = True
self.__listenThread = threading.Thread(
target=self.__listenAutoReceiveConnections)
self.__listenThread.setDaemon(True)
self.__listenThread.start()
def __listenAutoReceiveConnections(self):
while self.__listeningForMessages:
self.__trInOutLock.acquire()
try:
# TODO: A single DefaultSelector instance can probably be shared by all threads
sel = selectors.DefaultSelector()
for trid in self.__messageTransports:
mt = self.__messageTransports[trid]
if not mt["listen"]:
continue
sel.register(mt["transport"].getSocket(),
selectors.EVENT_READ, trid)
if not sel.get_map():
self.__listeningForMessages = False
return
finally:
self.__trInOutLock.release()
try:
events = sel.select(timeout=1)
except Exception:
gLogger.exception(
"Exception while selecting persistent connections")
continue
for key, event in events:
if event & selectors.EVENT_READ:
trid = key.data
if trid in self.__messageTransports:
result = self.__receiveMsgDataAndQueue(trid)
if not result["OK"]:
self.removeTransport(trid)
# Process received data functions
def __receiveMsgDataAndQueue(self, trid):
# Receive
result = self.__trPool.receive(
trid,
blockAfterKeepAlive=False,
idleReceive=self.__messageTransports[trid]["idleRead"])
self.__log.debug("[trid %s] Received data: %s" % (trid, str(result)))
# If error close transport and exit
if not result["OK"]:
self.__log.debug("[trid %s] ERROR RCV DATA %s" %
(trid, result["Message"]))
gLogger.warn(
"Error while receiving message",
"from %s : %s" %
(self.__trPool.get(trid).getFormattedCredentials(),
result["Message"]),
)
return self.removeTransport(trid)
self.__threadPool.submit(self.__processIncomingData, trid, result)
return S_OK()
def __processIncomingData(self, trid, receivedResult):
# If keep alive, return OK
if receivedResult.get("keepAlive"):
return S_OK()
# If idle read return
self.__trInOutLock.acquire()
try:
idleRead = self.__messageTransports[trid]["idleRead"]
except KeyError:
return S_ERROR("Transport %s unknown" % trid)
finally:
self.__trInOutLock.release()
if idleRead:
if receivedResult["Value"]:
gLogger.fatal("OOOops. Idle read has returned data!")
return S_OK()
if not receivedResult["Value"]:
self.__log.debug("Transport %s closed connection" % trid)
return self.removeTransport(trid)
# This is a message req/resp
msg = receivedResult["Value"]
# Valid message?
if "request" not in msg:
gLogger.warn("Received data does not seem to be a message !!!!")
return self.removeTransport(trid)
# Decide if it's a response or a request
if msg["request"]:
# If message has Id return ACK to received
if "id" in msg:
self.__sendResponse(trid, msg["id"], S_OK())
# Process msg
result = self.__processIncomingRequest(trid, msg)
else:
result = self.__processIncomingResponse(trid, msg)
# If error close the transport
if not result["OK"]:
gLogger.info(
"Closing transport because of error while processing message",
result["Message"])
return self.removeTransport(trid)
return S_OK()
def __processIncomingRequest(self, trid, msg):
self.__trInOutLock.acquire()
try:
rcvCB = self.__messageTransports[trid]["cbReceiveMessage"]
except KeyError:
return S_ERROR("Transport %s unknown" % trid)
finally:
self.__trInOutLock.release()
if not rcvCB:
gLogger.fatal(
"Transport %s does not have a callback defined and a message arrived!"
% trid)
return S_ERROR("No message was expected in for this transport")
# Check message has id and name
for requiredField in ["name"]:
if requiredField not in msg:
gLogger.error("Message does not have required field",
requiredField)
return S_ERROR("Message does not have %s" % requiredField)
# Load message
if "attrs" in msg:
attrs = msg["attrs"]
if not isinstance(attrs, (tuple, list)):
return S_ERROR(
"Message args has to be a tuple or a list, not %s" %
type(attrs))
else:
attrs = None
# Do we "unpack" or do we send the raw data to the callback?
if self.__useMessageObjects:
result = self.__msgFactory.createMessage(
self.__messageTransports[trid]["svcName"], msg["name"], attrs)
if not result["OK"]:
return result
msgObj = result["Value"]
else:
msgObj = DummyMessage(msg)
# Is msg ok?
if not msgObj.isOK():
return S_ERROR("Messsage is invalid")
try:
# Callback it and return response
result = rcvCB(trid, msgObj)
if not isReturnStructure(result):
return S_ERROR(
"Request function does not return a result structure")
return result
except Exception as e:
# Whoops. Show exception and return
gLogger.exception("Exception while processing message %s" %
msg["name"],
lException=e)
return S_ERROR("Exception while processing message %s: %s" %
(msg["name"], str(e)))
def __processIncomingResponse(self, trid, msg):
# This is a message response
for requiredField in ("id", "result"):
if requiredField not in msg:
gLogger.error("Message does not have required field",
requiredField)
return S_ERROR("Message does not have %s" % requiredField)
if not isReturnStructure(msg["result"]):
return S_ERROR(
"Message response did not return a result structure")
return self.__notifyCallback(msg["id"], msg["result"])
# Sending functions
def __sendResponse(self, trid, msgId, msgResult):
msgResponse = {"request": False, "id": msgId, "result": msgResult}
self.__trPool.send(trid, S_OK(msgResponse))
def sendMessage(self, trid, msgObj):
if not msgObj.isOK():
return S_ERROR("Message is not ready to be sent")
result = self.__sendMessage(trid, msgObj)
if not result["OK"]:
self.removeTransport(trid)
return result
def __sendMessage(self, trid, msgObj):
if not self.__trPool.exists(trid):
return S_ERROR("Not transport with id %s defined for messaging" %
trid)
msg = {"request": True, "name": msgObj.getName()}
attrs = msgObj.dumpAttrs()["Value"]
msg["attrs"] = attrs
waitForAck = msgObj.getWaitForAck()
if not waitForAck:
return self.__trPool.send(trid, S_OK(msg))
msgId = self.__generateMsgId()
msg["id"] = msgId
self.__generateMessageResponse(trid, msgId)
result = self.__trPool.send(trid, S_OK(msg))
# Lock and generate and wait
self.__callbacksLock.acquire()
try:
if not result["OK"]:
# Release lock and exit
self.__clearCallback(msgId)
return result
return self.__waitForMessageResponse(msgId)
finally:
self.__callbacksLock.release()
# Callback nightmare
# Lock need to have been aquired prior to func
def __generateMessageResponse(self, trid, msgId):
self.__callbacksLock.acquire()
try:
if msgId in self.__responseCallbacks:
return self.__responseCallbacks[msgId]
if trid not in self.__msgInTransport:
self.__msgInTransport[trid] = set()
self.__msgInTransport[trid].add(msgId)
self.__responseCallbacks[msgId] = {
"creationTime": time.time(),
"trid": trid
}
return self.__responseCallbacks[msgId]
finally:
self.__callbacksLock.release()
# Lock need to have been aquired prior to func
def __waitForMessageResponse(self, msgId):
if msgId not in self.__responseCallbacks:
return S_ERROR("Invalid msg id")
respCallback = self.__responseCallbacks[msgId]
while "result" not in respCallback and time.time(
) - respCallback["creationTime"] < 30:
self.__callbacksLock.wait(30)
self.__clearCallback(msgId)
if "result" in respCallback:
return respCallback["result"]
return S_ERROR("Timeout while waiting for message ack")
def __clearCallback(self, msgId):
if msgId not in self.__responseCallbacks:
return False
trid = self.__responseCallbacks[msgId]["trid"]
self.__responseCallbacks.pop(msgId)
try:
self.__msgInTransport[trid].remove(msgId)
except KeyError:
pass
return True
# Lock need to have been aquired prior to func
def __setCallbackResult(self, msgId, result=False):
if msgId not in self.__responseCallbacks:
return False
self.__responseCallbacks[msgId]["result"] = result
return True
def __notifyCallback(self, msgId, msgResult):
self.__callbacksLock.acquire()
try:
if self.__setCallbackResult(msgId, msgResult):
self.__callbacksLock.notifyAll()
finally:
self.__callbacksLock.release()
return S_OK()
def removeTransport(self, trid, closeTransport=True):
# Delete from the message Transports
self.__trInOutLock.acquire()
try:
if trid not in self.__messageTransports:
return S_OK()
# Save the disconnect callback if it's there
if self.__messageTransports[trid]["cbDisconnect"]:
cbDisconnect = self.__messageTransports[trid]["cbDisconnect"]
else:
cbDisconnect = False
self.__messageTransports.pop(trid)
if closeTransport:
self.__trPool.close(trid)
finally:
self.__trInOutLock.release()
# Flush remaining messages
self.__callbacksLock.acquire()
try:
msgIds = False
if trid in self.__msgInTransport:
msgIds = set(self.__msgInTransport[trid])
self.__msgInTransport.pop(trid)
for msgId in msgIds:
self.__setCallbackResult(
msgId, S_ERROR("Connection closed by peer"))
self.__callbacksLock.notifyAll()
finally:
self.__callbacksLock.release()
# Queue the disconnect CB if it's there
if cbDisconnect:
self.__threadPool.submit(cbDisconnect, trid)
return S_OK()
class MessageBroker(object):
def __init__(self, name, transportPool=None, threadPool=None):
self.__name = name
self.__messageTransports = {}
self.__msgCounter = 0
self.__msgCounterLock = threading.Lock()
self.__responseCallbacks = {}
self.__msgInTransport = {}
self.__listenPersistConn = False
self.__useMessageObjects = True
self.__callbacksLock = threading.Condition()
self.__trInOutLock = threading.Lock()
self.__msgFactory = MessageFactory()
self.__log = gLogger.getSubLogger("MSGBRK")
if not transportPool:
transportPool = getGlobalTransportPool()
self.__trPool = transportPool
if not threadPool:
threadPool = getGlobalThreadPool()
self.__threadPool = threadPool
self.__listeningForMessages = False
self.__listenThread = None
def getNumConnections(self):
return len(self.__messageTransports)
def getMsgFactory(self):
return self.__msgFactory
def useMessageObjects(self, bD):
self.__useMessageObjects = bD
# Message id generation
def __generateMsgId(self):
self.__msgCounterLock.acquire()
try:
msgId = "%s:%d" % (self.__name, self.__msgCounter)
self.__msgCounter += 1
return msgId
finally:
self.__msgCounterLock.release()
def getTransportPool(self):
return self.__trPool
# Add and remove transport to/from broker
def addTransport(self, transport, *args, **kwargs):
trid = self.__trPool.add(transport)
try:
result = self.addTransportId(trid, *args, **kwargs)
except Exception as e:
gLogger.exception("Cannot add transport id", lException=e)
result = S_ERROR("Cannot add transport id")
if not result['OK']:
self.__trPool.remove(trid)
return result
return S_OK(trid)
def addTransportId(self,
trid,
svcName,
receiveMessageCallback=None,
disconnectCallback=None,
idleRead=False,
listenToConnection=True):
self.__trInOutLock.acquire()
try:
if trid in self.__messageTransports:
return S_OK()
tr = self.__trPool.get(trid)
if not tr:
return S_ERROR("No transport with id %s registered" % trid)
self.__messageTransports[trid] = {
'transport': tr,
'svcName': svcName,
'cbReceiveMessage': receiveMessageCallback,
'cbDisconnect': disconnectCallback,
'listen': listenToConnection,
'idleRead': idleRead
}
self.__startListeningThread()
return S_OK()
finally:
self.__trInOutLock.release()
def listenToTransport(self, trid, listen=True):
self.__trInOutLock.acquire()
try:
if trid in self.__messageTransports:
self.__messageTransports[trid]['listen'] = listen
self.__startListeningThread()
finally:
self.__trInOutLock.release()
# Listen to connections
def __startListeningThread(self):
threadDead = self.__listeningForMessages and self.__listenThread is not None and not self.__listenThread.isAlive(
)
if not self.__listeningForMessages or threadDead:
self.__listeningForMessages = True
self.__listenThread = threading.Thread(
target=self.__listenAutoReceiveConnections)
self.__listenThread.setDaemon(True)
self.__listenThread.start()
def __listenAutoReceiveConnections(self):
while self.__listeningForMessages:
self.__trInOutLock.acquire()
try:
sIdList = []
for trid in self.__messageTransports:
mt = self.__messageTransports[trid]
if not mt['listen']:
continue
sIdList.append((trid, mt['transport'].getSocket()))
if not sIdList:
self.__listeningForMessages = False
return
finally:
self.__trInOutLock.release()
try:
try:
inList, _outList, _exList = select.select(
[pos[1] for pos in sIdList], [], [], 1)
if len(inList) == 0:
continue
except socket.error:
time.sleep(0.001)
continue
except select.error:
time.sleep(0.001)
continue
except Exception as e:
gLogger.exception(
"Exception while selecting persistent connections",
lException=e)
continue
for sock in inList:
for iPos in range(len(sIdList)):
if sock == sIdList[iPos][1]:
trid = sIdList[iPos][0]
if trid in self.__messageTransports:
result = self.__receiveMsgDataAndQueue(trid)
if not result['OK']:
self.removeTransport(trid)
break
#Process received data functions
def __receiveMsgDataAndQueue(self, trid):
#Receive
result = self.__trPool.receive(
trid,
blockAfterKeepAlive=False,
idleReceive=self.__messageTransports[trid]['idleRead'])
self.__log.debug("[trid %s] Received data: %s" % (trid, str(result)))
#If error close transport and exit
if not result['OK']:
self.__log.debug("[trid %s] ERROR RCV DATA %s" %
(trid, result['Message']))
gLogger.warn(
"Error while receiving message", "from %s : %s" %
(self.__trPool.get(trid).getFormattedCredentials(),
result['Message']))
return self.removeTransport(trid)
self.__threadPool.generateJobAndQueueIt(self.__processIncomingData,
args=(trid, result))
return S_OK()
def __processIncomingData(self, trid, receivedResult):
#If keep alive, return OK
if 'keepAlive' in receivedResult and receivedResult['keepAlive']:
return S_OK()
#If idle read return
self.__trInOutLock.acquire()
try:
idleRead = self.__messageTransports[trid]['idleRead']
except KeyError:
return S_ERROR("Transport %s unknown" % trid)
finally:
self.__trInOutLock.release()
if idleRead:
if receivedResult['Value']:
gLogger.fatal("OOOops. Idle read has returned data!")
return S_OK()
if not receivedResult['Value']:
self.__log.debug("Transport %s closed connection" % trid)
return self.removeTransport(trid)
#This is a message req/resp
msg = receivedResult['Value']
#Valid message?
if 'request' not in msg:
gLogger.warn("Received data does not seem to be a message !!!!")
return self.removeTransport(trid)
#Decide if it's a response or a request
if msg['request']:
#If message has Id return ACK to received
if 'id' in msg:
self.__sendResponse(trid, msg['id'], S_OK())
#Process msg
result = self.__processIncomingRequest(trid, msg)
else:
result = self.__processIncomingResponse(trid, msg)
#If error close the transport
if not result['OK']:
gLogger.info(
"Closing transport because of error while processing message",
result['Message'])
return self.removeTransport(trid)
return S_OK()
def __processIncomingRequest(self, trid, msg):
self.__trInOutLock.acquire()
try:
rcvCB = self.__messageTransports[trid]['cbReceiveMessage']
except KeyError:
return S_ERROR("Transport %s unknown" % trid)
finally:
self.__trInOutLock.release()
if not rcvCB:
gLogger.fatal(
"Transport %s does not have a callback defined and a message arrived!"
% trid)
return S_ERROR("No message was expected in for this transport")
#Check message has id and name
for requiredField in ['name']:
if requiredField not in msg:
gLogger.error("Message does not have required field",
requiredField)
return S_ERROR("Message does not have %s" % requiredField)
#Load message
if 'attrs' in msg:
attrs = msg['attrs']
if not isinstance(attrs, (tuple, list)):
return S_ERROR(
"Message args has to be a tuple or a list, not %s" %
type(attrs))
else:
attrs = None
#Do we "unpack" or do we send the raw data to the callback?
if self.__useMessageObjects:
result = self.__msgFactory.createMessage(
self.__messageTransports[trid]['svcName'], msg['name'], attrs)
if not result['OK']:
return result
msgObj = result['Value']
else:
msgObj = DummyMessage(msg)
#Is msg ok?
if not msgObj.isOK():
return S_ERROR("Messsage is invalid")
try:
#Callback it and return response
result = rcvCB(trid, msgObj)
if not isReturnStructure(result):
return S_ERROR(
"Request function does not return a result structure")
return result
except Exception as e:
#Whoops. Show exception and return
gLogger.exception("Exception while processing message %s" %
msg['name'],
lException=e)
return S_ERROR("Exception while processing message %s: %s" %
(msg['name'], str(e)))
def __processIncomingResponse(self, trid, msg):
#This is a message response
for requiredField in ('id', 'result'):
if requiredField not in msg:
gLogger.error("Message does not have required field",
requiredField)
return S_ERROR("Message does not have %s" % requiredField)
if not isReturnStructure(msg['result']):
return S_ERROR(
"Message response did not return a result structure")
return self.__notifyCallback(msg['id'], msg['result'])
#Sending functions
def __sendResponse(self, trid, msgId, msgResult):
msgResponse = {'request': False, 'id': msgId, 'result': msgResult}
_result = self.__trPool.send(trid, S_OK(msgResponse))
def sendMessage(self, trid, msgObj):
if not msgObj.isOK():
return S_ERROR("Message is not ready to be sent")
result = self.__sendMessage(trid, msgObj)
if not result['OK']:
self.removeTransport(trid)
return result
def __sendMessage(self, trid, msgObj):
if not self.__trPool.exists(trid):
return S_ERROR("Not transport with id %s defined for messaging" %
trid)
msg = {'request': True, 'name': msgObj.getName()}
attrs = msgObj.dumpAttrs()['Value']
msg['attrs'] = attrs
waitForAck = msgObj.getWaitForAck()
if not waitForAck:
return self.__trPool.send(trid, S_OK(msg))
msgId = self.__generateMsgId()
msg['id'] = msgId
self.__generateMessageResponse(trid, msgId)
result = self.__trPool.send(trid, S_OK(msg))
#Lock and generate and wait
self.__callbacksLock.acquire()
try:
if not result['OK']:
#Release lock and exit
self.__clearCallback(msgId)
return result
return self.__waitForMessageResponse(msgId)
finally:
self.__callbacksLock.release()
#Callback nightmare
#Lock need to have been aquired prior to func
def __generateMessageResponse(self, trid, msgId):
self.__callbacksLock.acquire()
try:
if msgId in self.__responseCallbacks:
return self.__responseCallbacks[msgId]
if trid not in self.__msgInTransport:
self.__msgInTransport[trid] = set()
self.__msgInTransport[trid].add(msgId)
self.__responseCallbacks[msgId] = {
'creationTime': time.time(),
'trid': trid
}
return self.__responseCallbacks[msgId]
finally:
self.__callbacksLock.release()
#Lock need to have been aquired prior to func
def __waitForMessageResponse(self, msgId):
if msgId not in self.__responseCallbacks:
return S_ERROR("Invalid msg id")
respCallback = self.__responseCallbacks[msgId]
while 'result' not in respCallback and time.time(
) - respCallback['creationTime'] < 30:
self.__callbacksLock.wait(30)
self.__clearCallback(msgId)
if 'result' in respCallback:
return respCallback['result']
return S_ERROR("Timeout while waiting for message ack")
def __clearCallback(self, msgId):
if msgId not in self.__responseCallbacks:
return False
trid = self.__responseCallbacks[msgId]['trid']
self.__responseCallbacks.pop(msgId)
try:
self.__msgInTransport[trid].remove(msgId)
except KeyError:
pass
return True
#Lock need to have been aquired prior to func
def __setCallbackResult(self, msgId, result=False):
if msgId not in self.__responseCallbacks:
return False
self.__responseCallbacks[msgId]['result'] = result
return True
def __notifyCallback(self, msgId, msgResult):
self.__callbacksLock.acquire()
try:
if self.__setCallbackResult(msgId, msgResult):
self.__callbacksLock.notifyAll()
finally:
self.__callbacksLock.release()
return S_OK()
def removeTransport(self, trid, closeTransport=True):
#Delete from the message Transports
self.__trInOutLock.acquire()
try:
if trid not in self.__messageTransports:
return S_OK()
#Save the disconnect callback if it's there
if self.__messageTransports[trid]['cbDisconnect']:
cbDisconnect = self.__messageTransports[trid]['cbDisconnect']
else:
cbDisconnect = False
self.__messageTransports.pop(trid)
if closeTransport:
self.__trPool.close(trid)
finally:
self.__trInOutLock.release()
#Flush remaining messages
self.__callbacksLock.acquire()
try:
msgIds = False
if trid in self.__msgInTransport:
msgIds = set(self.__msgInTransport[trid])
self.__msgInTransport.pop(trid)
for msgId in msgIds:
self.__setCallbackResult(
msgId, S_ERROR("Connection closed by peer"))
self.__callbacksLock.notifyAll()
finally:
self.__callbacksLock.release()
#Queue the disconnect CB if it's there
if cbDisconnect:
self.__threadPool.generateJobAndQueueIt(cbDisconnect,
args=(trid, ))
return S_OK()
class MessageBroker( object ):
def __init__( self, name, transportPool = None, threadPool = None ):
self.__name = name
self.__messageTransports = {}
self.__msgCounter = 0
self.__msgCounterLock = threading.Lock()
self.__responseCallbacks = {}
self.__msgInTransport = {}
self.__listenPersistConn = False
self.__useMessageObjects = True
self.__callbacksLock = threading.Condition()
self.__trInOutLock = threading.Lock()
self.__msgFactory = MessageFactory()
self.__log = gLogger.getSubLogger( "MSGBRK" )
if not transportPool:
transportPool = getGlobalTransportPool()
self.__trPool = transportPool
if not threadPool:
threadPool = getGlobalThreadPool()
self.__threadPool = threadPool
self.__listeningForMessages = False
def getNumConnections( self ):
return len( self.__messageTransports )
def getMsgFactory( self ):
return self.__msgFactory
def useMessageObjects( self, bD ):
self.__useMessageObjects = bD
# Message id generation
def __generateMsgId( self ):
self.__msgCounterLock.acquire()
try:
msgId = "%s:%d" % ( self.__name, self.__msgCounter )
self.__msgCounter += 1
return msgId
finally:
self.__msgCounterLock.release()
def getTransportPool( self ):
return self.__trPool
# Add and remove transport to/from broker
def addTransport( self, transport, *args, **kwargs ):
trid = self.__trPool.add( transport )
try:
result = self.addTransportId( trid, *args, **kwargs )
except Exception as e:
gLogger.exception( "Cannot add transport id", lException = e )
result = S_ERROR( "Cannot add transport id" )
if not result[ 'OK' ]:
self.__trPool.remove( trid )
return result
return S_OK( trid )
def addTransportId( self, trid, svcName,
receiveMessageCallback = None, disconnectCallback = None,
idleRead = False, listenToConnection = True ):
self.__trInOutLock.acquire()
try:
if trid in self.__messageTransports:
return S_OK()
tr = self.__trPool.get( trid )
if not tr:
return S_ERROR( "No transport with id %s registered" % trid )
self.__messageTransports[ trid ] = { 'transport' : tr,
'svcName' : svcName,
'cbReceiveMessage': receiveMessageCallback,
'cbDisconnect' : disconnectCallback,
'listen' : listenToConnection,
'idleRead' : idleRead }
self.__startListeningThread()
return S_OK()
finally:
self.__trInOutLock.release()
def listenToTransport( self, trid, listen = True ):
self.__trInOutLock.acquire()
try:
if trid in self.__messageTransports:
self.__messageTransports[ trid ][ 'listen' ] = listen
self.__startListeningThread()
finally:
self.__trInOutLock.release()
# Listen to connections
def __startListeningThread( self ):
threadDead = self.__listeningForMessages and not self.__listenThread.isAlive()
if not self.__listeningForMessages or threadDead:
self.__listeningForMessages = True
self.__listenThread = threading.Thread( target = self.__listenAutoReceiveConnections )
self.__listenThread.setDaemon( True )
self.__listenThread.start()
def __listenAutoReceiveConnections( self ):
while self.__listeningForMessages:
self.__trInOutLock.acquire()
try:
sIdList = []
for trid in self.__messageTransports:
mt = self.__messageTransports[ trid ]
if not mt[ 'listen' ]:
continue
sIdList.append( ( trid, mt[ 'transport' ].getSocket() ) )
if not sIdList:
self.__listeningForMessages = False
return
finally:
self.__trInOutLock.release()
try:
try:
inList, _outList, _exList = select.select( [ pos[1] for pos in sIdList ] , [], [], 1 )
if len( inList ) == 0:
continue
except socket.error:
time.sleep( 0.001 )
continue
except select.error:
time.sleep( 0.001 )
continue
except Exception as e:
gLogger.exception( "Exception while selecting persistent connections", lException = e )
continue
for sock in inList:
for iPos in range( len( sIdList ) ):
if sock == sIdList[ iPos ][1]:
trid = sIdList[ iPos ][0]
if trid in self.__messageTransports:
result = self.__receiveMsgDataAndQueue( trid )
if not result[ 'OK' ]:
self.removeTransport( trid )
break
#Process received data functions
def __receiveMsgDataAndQueue( self, trid ):
#Receive
result = self.__trPool.receive( trid,
blockAfterKeepAlive = False,
idleReceive = self.__messageTransports[ trid ][ 'idleRead' ] )
self.__log.debug( "[trid %s] Received data: %s" % ( trid, str( result ) ) )
#If error close transport and exit
if not result[ 'OK' ]:
self.__log.debug( "[trid %s] ERROR RCV DATA %s" % ( trid, result[ 'Message' ] ) )
gLogger.warn( "Error while receiving message", "from %s : %s" % ( self.__trPool.get( trid ).getFormattedCredentials(),
result[ 'Message' ] ) )
return self.removeTransport( trid )
self.__threadPool.generateJobAndQueueIt( self.__processIncomingData,
args = ( trid, result ) )
return S_OK()
def __processIncomingData( self, trid, receivedResult ):
#If keep alive, return OK
if 'keepAlive' in receivedResult and receivedResult[ 'keepAlive' ]:
return S_OK()
#If idle read return
self.__trInOutLock.acquire()
try:
idleRead = self.__messageTransports[ trid ][ 'idleRead' ]
except KeyError:
return S_ERROR( "Transport %s unknown" % trid )
finally:
self.__trInOutLock.release()
if idleRead:
if receivedResult[ 'Value' ]:
gLogger.fatal( "OOOops. Idle read has returned data!" )
return S_OK()
if not receivedResult[ 'Value' ]:
self.__log.debug( "Transport %s closed connection" % trid )
return self.removeTransport( trid )
#This is a message req/resp
msg = receivedResult[ 'Value' ]
#Valid message?
if 'request' not in msg:
gLogger.warn( "Received data does not seem to be a message !!!!" )
return self.removeTransport( trid )
#Decide if it's a response or a request
if msg[ 'request' ]:
#If message has Id return ACK to received
if 'id' in msg:
self.__sendResponse( trid, msg[ 'id' ], S_OK() )
#Process msg
result = self.__processIncomingRequest( trid, msg )
else:
result = self.__processIncomingResponse( trid, msg )
#If error close the transport
if not result[ 'OK' ]:
gLogger.info( "Closing transport because of error while processing message", result[ 'Message' ] )
return self.removeTransport( trid )
return S_OK()
def __processIncomingRequest( self, trid, msg ):
self.__trInOutLock.acquire()
try:
rcvCB = self.__messageTransports[ trid ][ 'cbReceiveMessage' ]
except KeyError:
return S_ERROR( "Transport %s unknown" % trid )
finally:
self.__trInOutLock.release()
if not rcvCB:
gLogger.fatal( "Transport %s does not have a callback defined and a message arrived!" % trid )
return S_ERROR( "No message was expected in for this transport" )
#Check message has id and name
for requiredField in [ 'name' ]:
if requiredField not in msg:
gLogger.error( "Message does not have required field", requiredField )
return S_ERROR( "Message does not have %s" % requiredField )
#Load message
if 'attrs' in msg:
attrs = msg[ 'attrs' ]
if type( attrs ) not in( types.TupleType, types.ListType ):
return S_ERROR( "Message args has to be a tuple or a list, not %s" % type( attrs ) )
else:
attrs = None
#Do we "unpack" or do we send the raw data to the callback?
if self.__useMessageObjects:
result = self.__msgFactory.createMessage( self.__messageTransports[ trid ][ 'svcName' ], msg[ 'name' ], attrs )
if not result[ 'OK' ]:
return result
msgObj = result[ 'Value' ]
else:
msgObj = DummyMessage( msg )
#Is msg ok?
if not msgObj.isOK():
return S_ERROR( "Messsage is invalid" )
try:
#Callback it and return response
result = rcvCB( trid, msgObj )
if not isReturnStructure( result ):
return S_ERROR( "Request function does not return a result structure" )
return result
except Exception as e:
#Whoops. Show exception and return
gLogger.exception( "Exception while processing message %s" % msg[ 'name' ], lException = e )
return S_ERROR( "Exception while processing message %s: %s" % ( msg[ 'name' ], str( e ) ) )
def __processIncomingResponse( self, trid, msg ):
#This is a message response
for requiredField in ( 'id', 'result' ):
if requiredField not in msg:
gLogger.error( "Message does not have required field", requiredField )
return S_ERROR( "Message does not have %s" % requiredField )
if not isReturnStructure( msg[ 'result' ] ):
return S_ERROR( "Message response did not return a result structure" )
return self.__notifyCallback( msg[ 'id' ], msg[ 'result' ] )
#Sending functions
def __sendResponse( self, trid, msgId, msgResult ):
msgResponse = { 'request' : False, 'id' : msgId, 'result' : msgResult }
_result = self.__trPool.send( trid, S_OK( msgResponse ) )
def sendMessage( self, trid, msgObj ):
if not msgObj.isOK():
return S_ERROR( "Message is not ready to be sent" )
result = self.__sendMessage( trid, msgObj )
if not result[ 'OK' ]:
self.removeTransport( trid )
return result
def __sendMessage( self, trid, msgObj ):
if not self.__trPool.exists( trid ):
return S_ERROR( "Not transport with id %s defined for messaging" % trid )
msg = { 'request' : True, 'name' : msgObj.getName() }
attrs = msgObj.dumpAttrs()[ 'Value' ]
msg[ 'attrs' ] = attrs
waitForAck = msgObj.getWaitForAck()
if not waitForAck:
return self.__trPool.send( trid, S_OK( msg ) )
msgId = self.__generateMsgId()
msg[ 'id' ] = msgId
self.__generateMessageResponse( trid, msgId )
result = self.__trPool.send( trid, S_OK( msg ) )
#Lock and generate and wait
self.__callbacksLock.acquire()
try:
if not result[ 'OK' ]:
#Release lock and exit
self.__clearCallback( msgId )
return result
return self.__waitForMessageResponse( msgId )
finally:
self.__callbacksLock.release()
#Callback nightmare
#Lock need to have been aquired prior to func
def __generateMessageResponse( self, trid, msgId ):
self.__callbacksLock.acquire()
try:
if msgId in self.__responseCallbacks:
return self.__responseCallbacks[ msgId ]
if trid not in self.__msgInTransport:
self.__msgInTransport[ trid ] = set()
self.__msgInTransport[ trid ].add( msgId )
self.__responseCallbacks[ msgId ] = { 'creationTime' : time.time(),
'trid' : trid
}
return self.__responseCallbacks[ msgId ]
finally:
self.__callbacksLock.release()
#Lock need to have been aquired prior to func
def __waitForMessageResponse( self, msgId ):
if msgId not in self.__responseCallbacks:
return S_ERROR( "Invalid msg id" )
respCallback = self.__responseCallbacks[ msgId ]
while 'result' not in respCallback and time.time() - respCallback[ 'creationTime' ] < 30 :
self.__callbacksLock.wait( 30 )
self.__clearCallback( msgId )
if 'result' in respCallback:
return respCallback[ 'result' ]
return S_ERROR( "Timeout while waiting for message ack" )
def __clearCallback( self, msgId ):
if msgId not in self.__responseCallbacks:
return False
trid = self.__responseCallbacks[ msgId ][ 'trid' ]
self.__responseCallbacks.pop( msgId )
try:
self.__msgInTransport[ trid ].remove( msgId )
except KeyError:
pass
return True
#Lock need to have been aquired prior to func
def __setCallbackResult( self, msgId, result = False ):
if msgId not in self.__responseCallbacks:
return False
self.__responseCallbacks[ msgId ][ 'result' ] = result
return True
def __notifyCallback( self, msgId, msgResult ):
self.__callbacksLock.acquire()
try:
if self.__setCallbackResult( msgId, msgResult ):
self.__callbacksLock.notifyAll()
finally:
self.__callbacksLock.release()
return S_OK()
def removeTransport( self, trid, closeTransport = True ):
#Delete from the message Transports
self.__trInOutLock.acquire()
try:
if trid not in self.__messageTransports:
return S_OK()
#Save the disconnect callback if it's there
if self.__messageTransports[ trid ][ 'cbDisconnect' ]:
cbDisconnect = self.__messageTransports[ trid ][ 'cbDisconnect' ]
else:
cbDisconnect = False
self.__messageTransports.pop( trid )
if closeTransport:
self.__trPool.close( trid )
finally:
self.__trInOutLock.release()
#Flush remaining messages
self.__callbacksLock.acquire()
try:
msgIds = False
if trid in self.__msgInTransport:
msgIds = set( self.__msgInTransport[ trid ] )
self.__msgInTransport.pop( trid )
for msgId in msgIds:
self.__setCallbackResult( msgId, S_ERROR( "Connection closed by peer" ) )
self.__callbacksLock.notifyAll()
finally:
self.__callbacksLock.release()
#Queue the disconnect CB if it's there
if cbDisconnect:
self.__threadPool.generateJobAndQueueIt( cbDisconnect,
args = ( trid, ) )
return S_OK()