def hearbeat_loop(self):
"""
Periodically check all locally registered workers ping time. Unregister dead workers
"""
maxpinglife = timedelta(seconds=settings.HEARTBEAT_TIMEOUT +
settings.WORKER_HEARTBEAT)
unreglist = []
while True:
now = datetime.now()
for ID, nfo in self.__pingdb.iteritems():
# find outdated timeouts
to = nfo['t'] + maxpinglife
if to < now:
LOG.warning(
"Worker [%s] with id [%s] died. Unregistering." %
(nfo['s'], ID))
unreglist.append(ID)
# unregister all dead workers
while len(unreglist) > 0:
ID = unreglist.pop()
self.worker_stop(ID)
gevent.sleep(settings.WORKER_HEARTBEAT)
def loop(self):
while self.is_running:
# receive data
msgdata, addr = self.SOCK.recvfrom(4096)
# skip own broadcast messages
if addr[0] == self.own_ip:
continue
# deserialize
try:
msgdata, repreq = self.serializer.deserialize(msgdata)
except NotOurMessage:
continue
except Exception:
LOG.warning("Message from broadcast deserialisation error")
LOG.debug(
"Broken message body dump in hex (only first 1024 bytes):\n%s"
% msgdata[:1024].encode("hex"))
continue
# own broadcast from another interface
try:
if msgdata['__sid__'] == self.ID:
continue
except KeyError:
continue
# message type
try:
msg = msgdata['message']
except KeyError:
LOG.debug("Decoded message is incomplete. Message dump: %s" %
repr(msgdata))
continue
# find handler
try:
handler = self._msgdb[msg]
except KeyError:
# unknown messages are ignored silently
LOG.warning("Unknown message received [%s]" % msg)
LOG.debug("Message body dump:\n%s" % repr(msgdata))
continue
# run handler
try:
handler(msgdata)
except Exception as e:
# log exception details
excname = e.__class__.__name__
# traceback
tback = traceback.format_exc()
try:
tback = unicode(tback, "utf-8")
except:
tback = repr(tback)
# error message
errmsg = e.message
try:
errmsg = unicode(errmsg, "utf-8")
except:
errmsg = repr(errmsg)
# log & clean
LOG.error(
"Exception [%s] when processing message [%s]. Message: %s."
% (excname, msg, errmsg))
LOG.debug("Message dump: %s" % repr(msgdata))
LOG.debug(tback)
del excname, tback, errmsg
def connection_handler(self, SOCK, address):
ssid = None
while True:
try:
msgdata, resreq = _receive_and_deserialize(
SOCK, self.serializer)
except (NoData, ConnectionClosed):
return
try:
msg = msgdata['message']
except KeyError:
if resreq:
self._send_noop(SOCK)
LOG.debug("Decoded message is incomplete. Message dump: %s" %
repr(msgdata))
continue
# message SET_SESSION_ID is special message
# it never return reply and is not propagated to handlers
if msg == messages.SET_SESSION_ID:
try:
ssid = msgdata['id']
#print("conn session id" , address, ssid)
except KeyError:
pass
if resreq:
self._send_noop(SOCK)
continue
# find message handler
try:
handler, rawmsg = self._msgdb[msg]
except KeyError:
# unknown messages are ignored
if resreq:
self._send_noop(SOCK)
LOG.warning("Unknown message received [%s]" % msg)
LOG.debug("Message body dump:\n%s" % repr(msgdata))
continue
# run handler
try:
result = handler(msgdata)
except Exception as e:
result = exception_serialize(e, False)
LOG.info(
"Exception [%s] when processing message [%s]. Message: %s."
% (result['name'], msg, result['description']))
#LOG.debug("Message dump: %s" % repr(msgdata) )
#LOG.debug(result['traceback'])
if not resreq:
# if response is not required, then don't send exceptions
continue
_serialize_and_send(
SOCK,
self.serializer,
exception_serialize(e, False),
resreq=False, # response never require another response
)
continue
# response is not expected, throw result and back to loop
if not resreq:
continue
try:
# send result
if rawmsg:
_serialize_and_send(
SOCK,
self.serializer,
result,
resreq=False,
)
else:
_serialize_and_send(SOCK,
self.serializer, {
"message": messages.RESULT,
"result": result,
},
resreq=False)
except ConnectionClosed:
return