def __init__(self, topic, channel, timeout=None, concurrency=1,
message_preprocessor=None, service_name=get_random_string()):
"""Wrapper around nsqworker.ThreadWorker
"""
super(NSQHandler, self).__init__()
self.logger = self.__class__.get_logger()
self.io_loop = ioloop.IOLoop.instance()
self.topic = topic
self.channel = channel
self.locker = _locker.RedisLocker(service_name, self.logger)
self._message_preprocessor = message_preprocessor if message_preprocessor else _identity
self._persistor = MessagePersistor(self.logger)
ThreadWorker(
message_handler=self.handle_message,
exception_handler=self.handle_exception,
timeout=timeout,
concurrency=concurrency,
topic=topic, channel=channel, **kwargs
).subscribe_worker()
class NSQHandler(NSQWriter):
def __init__(self, topic, channel, timeout=None, concurrency=1,
message_preprocessor=None, service_name=get_random_string()):
"""Wrapper around nsqworker.ThreadWorker
"""
super(NSQHandler, self).__init__()
self.logger = self.__class__.get_logger()
self.io_loop = ioloop.IOLoop.instance()
self.topic = topic
self.channel = channel
self.locker = _locker.RedisLocker(service_name, self.logger)
self._message_preprocessor = message_preprocessor if message_preprocessor else _identity
self._persistor = MessagePersistor(self.logger)
ThreadWorker(
message_handler=self.handle_message,
exception_handler=self.handle_exception,
timeout=timeout,
concurrency=concurrency,
topic=topic, channel=channel, **kwargs
).subscribe_worker()
# self.routes = []
@classmethod
def get_logger(cls, name=None):
logger = logging.getLogger(name or cls.__name__)
if not logger.handlers:
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler = logging.StreamHandler(stream=sys.stdout)
handler.setFormatter(formatter)
handler.setLevel(logging.INFO)
logger.addHandler(handler)
logger.setLevel(logging.INFO)
logger.propagate = 0
return logger
@classmethod
def register_route(cls, matcher_func, handler_func):
"""Register route
"""
if getattr(cls, "routes", None) is None:
cls.routes = []
# Don't use bound methods - convert to bare function
if getattr(handler_func, "im_self", None) is not None:
handler_func = handler_func.__func__
cls.routes.append((matcher_func, handler_func))
def route_message(self, message):
"""Basic message router
Handlers for the same route will be run sequentially
type message: nsq.Message
"""
m_body = message.body
handlers = []
for matcher_func, handler_func in self.__class__.routes:
if matcher_func(m_body) is True:
handlers.append(handler_func)
if len(handlers) == 0:
self.logger.debug("No handlers found for message {}.".format(message.body))
return
event_name = "<undefined>"
jsn = None
try:
jsn = json.loads(m_body)
event_name = jsn['name']
except Exception:
pass
handler_id = gen_random_string()
self.logger.info("[{}] [Handling new event] [topic={}] [channel={}] [event={}]".format(
handler_id, self.topic, self.channel, event_name
))
for handler in handlers:
status = "OK"
route_id = gen_random_string()
if jsn is not None and self._persistor.is_persisted_message(jsn):
if self._persistor.is_route_message(jsn, self.channel, handler.__name__):
self.logger.info("[{}] Route {} in channel {} will handle persisted message".format(
route_id, handler.__name__, self.channel
))
else:
continue
self.logger.info("[{}] [START] Routing message to handler [route={}] [event={}]".format(
route_id, handler.__name__, event_name)
)
start_time = current_milli_time()
try:
handler(self, self._message_preprocessor(message))
except Exception as e:
status = "FAILED"
msg = "[{}] Handler {} failed handling message {} with error {}".format(
route_id, handler.__name__, message.body, e.message)
self.logger.error(msg)
self.handle_exception(message, e)
if self._persistor.enabled:
new = self._persistor.persist_message(self.topic, self.channel, handler.__name__, m_body, repr(e))
if new:
self.logger.info("[{}] Persisted failed message".format(route_id))
else:
self.logger.info("[{}] Updated existing failed message".format(route_id))
self.logger.info("[{}] [END] [route={}] [event={}] [status={}] [time={}] ".format(
route_id, handler.__name__, event_name, status, str(current_milli_time() - start_time))
)
self.logger.info("[{}] [DONE handling new event] [topic={}] [channel={}] [event={}]".format(
handler_id, self.topic, self.channel, event_name
))
def handle_message(self, message):
"""
Basic message handler
:type message: nsq.Message
"""
self.logger.debug("Received message: {}".format(message.body))
self.route_message(message)
self.logger.debug("Finished handling message: {}".format(message.body))
def handle_exception(self, message, e):
"""
Basic error handler
:type message: nsq.Message
:type e: Exception
"""
error = "message raised an exception: {}. Message body: {}".format(e, message.body)
self.logger.error(error)
self.logger.error(traceback.format_exc())
