def report_stats(settings):
client = stats.get_client(settings)
while True:
client.gauge('streamer.connected_clients',
len(websocket.WebSocket.instances))
client.gauge('streamer.queue_length', WORK_QUEUE.qsize())
gevent.sleep(10)
def report_stats(settings):
client = stats.get_client(settings)
while True:
client.gauge('streamer.connected_clients',
len(websocket.WebSocket.instances))
client.gauge('streamer.queue_length', WORK_QUEUE.qsize())
gevent.sleep(10)
def process_work_queue(settings, queue, session_factory=None):
"""
Process each message from the queue in turn, handling exceptions.
This is the core of the streamer: we pull messages off the work queue,
dispatching them as appropriate. The handling of each message is wrapped in
code that ensures the database session is appropriately committed and
closed between messages.
"""
if session_factory is None:
session_factory = _get_session
s = stats.get_client(settings).pipeline()
session = session_factory(settings)
topic_handlers = {
ANNOTATION_TOPIC: messages.handle_annotation_event,
USER_TOPIC: messages.handle_user_event,
}
for msg in queue:
t_total = s.timer("streamer.msg.handler_total")
t_total.start()
try:
# All access to the database in the streamer is currently
# read-only, so enforce that:
session.execute(
"SET TRANSACTION "
"ISOLATION LEVEL SERIALIZABLE "
"READ ONLY "
"DEFERRABLE"
)
if isinstance(msg, messages.Message):
with s.timer("streamer.msg.handler_message"):
messages.handle_message(msg, settings, session, topic_handlers)
elif isinstance(msg, websocket.Message):
with s.timer("streamer.msg.handler_websocket"):
websocket.handle_message(msg, session)
else:
raise UnknownMessageType(repr(msg))
except (KeyboardInterrupt, SystemExit):
session.rollback()
raise
except Exception as exc:
log.warning("Caught exception handling streamer message:", exc_info=exc)
session.rollback()
else:
session.commit()
finally:
session.close()
t_total.stop()
s.send()
def process_work_queue(settings, queue, session_factory=None):
"""
Process each message from the queue in turn, handling exceptions.
This is the core of the streamer: we pull messages off the work queue,
dispatching them as appropriate. The handling of each message is wrapped in
code that ensures the database session is appropriately committed and
closed between messages.
"""
if session_factory is None:
session_factory = _get_session
s = stats.get_client(settings).pipeline()
session = session_factory(settings)
topic_handlers = {
ANNOTATION_TOPIC: messages.handle_annotation_event,
USER_TOPIC: messages.handle_user_event,
}
for msg in queue:
t_total = s.timer("streamer.msg.handler_total")
t_total.start()
try:
# All access to the database in the streamer is currently
# read-only, so enforce that:
session.execute(
"SET TRANSACTION "
"ISOLATION LEVEL SERIALIZABLE "
"READ ONLY "
"DEFERRABLE"
)
if isinstance(msg, messages.Message):
with s.timer("streamer.msg.handler_message"):
messages.handle_message(msg, settings, session, topic_handlers)
elif isinstance(msg, websocket.Message):
with s.timer("streamer.msg.handler_websocket"):
websocket.handle_message(msg, session)
else:
raise UnknownMessageType(repr(msg))
except (KeyboardInterrupt, SystemExit):
session.rollback()
raise
except Exception as exc:
log.warning("Caught exception handling streamer message:", exc_info=exc)
session.rollback()
else:
session.commit()
finally:
session.close()
t_total.stop()
s.send()