def test_handle_message_expands_uris_in_uri_filter(expand_uri):
expand_uri.return_value = [
'http://example.com', 'http://example.com/alter',
'http://example.com/print'
]
socket = mock.Mock()
socket.filter = None
message = websocket.Message(socket=socket,
payload=json.dumps({
'filter': {
'actions': {},
'match_policy':
'include_all',
'clauses': [{
'field':
'/uri',
'operator':
'equals',
'value':
'http://example.com',
}],
}
}))
websocket.handle_message(message)
uri_filter = socket.filter.filter['clauses'][0]
uri_values = uri_filter['value']
assert len(uri_values) == 3
assert 'http://example.com' in uri_values
assert 'http://example.com/alter' in uri_values
assert 'http://example.com/print' in uri_values
def test_handle_message_clears_feature_cache():
socket = mock.Mock()
message = websocket.Message(socket=socket,
payload=json.dumps({'messageType': 'foo'}))
websocket.handle_message(message)
socket.request.feature.clear.assert_called_with()
def test_handle_message_expands_uris_in_uri_filter_with_session(expand_uri):
expand_uri.return_value = ['http://example.com',
'http://example.com/alter',
'http://example.com/print']
session = mock.sentinel.db_session
socket = mock.Mock()
socket.filter = None
message = websocket.Message(socket=socket, payload=json.dumps({
'filter': {
'actions': {},
'match_policy': 'include_all',
'clauses': [{
'field': '/uri',
'operator': 'equals',
'value': 'http://example.com',
}],
}
}))
websocket.handle_message(message, session=session)
uri_filter = socket.filter.filter['clauses'][0]
uri_values = uri_filter['value']
assert len(uri_values) == 3
assert 'http://example.com' in uri_values
assert 'http://example.com/alter' in uri_values
assert 'http://example.com/print' in uri_values
def test_handle_message_clears_feature_cache():
socket = mock.Mock()
message = websocket.Message(socket=socket, payload=json.dumps({
'messageType': 'foo'}))
websocket.handle_message(message)
socket.request.feature.clear.assert_called_with()
def test_uses_appropriate_handler_for_known_type(self, foo_handler):
"""If the type is recognised, call the relevant handler."""
socket = mock.Mock(spec_set=["close"])
message = websocket.Message(socket, payload={"type": "foo", "foo": "bar"})
websocket.handle_message(message)
foo_handler.assert_called_once_with(message, session=None)
def test_uses_unknown_handler_for_unknown_type(self, unknown_handler):
"""If the type is unknown, call the `None` handler."""
socket = mock.Mock(spec_set=["close"])
message = websocket.Message(socket, payload={"type": "donkeys", "foo": "bar"})
websocket.handle_message(message)
unknown_handler.assert_called_once_with(message, session=None)
def test_uses_unknown_handler_for_missing_type(self, unknown_handler):
"""If the type is missing, call the `None` handler."""
socket = mock.Mock(spec_set=['close'])
message = websocket.Message(socket, payload={"foo": "bar"})
websocket.handle_message(message)
unknown_handler.assert_called_once_with(message, session=None)
def test_uses_appropriate_handler_for_known_type(self, foo_handler):
"""If the type is recognised, call the relevant handler."""
socket = mock.Mock(spec_set=["close"])
message = websocket.Message(socket, payload={"type": "foo", "foo": "bar"})
websocket.handle_message(message)
foo_handler.assert_called_once_with(message, session=None)
def test_handle_message_sets_socket_client_id_for_client_id_messages():
socket = mock.Mock()
socket.client_id = None
message = websocket.Message(socket=socket, payload=json.dumps({
'messageType': 'client_id',
'value': 'abcd1234',
}))
websocket.handle_message(message)
assert socket.client_id == 'abcd1234'
def process_work_queue(settings, queue, session_factory=None):
"""
Process each message from the queue in turn, handling exceptions.
This is the core of the streamer: we pull messages off the work queue,
dispatching them as appropriate. The handling of each message is wrapped in
code that ensures the database session is appropriately committed and
closed between messages.
"""
if session_factory is None:
session_factory = _get_session
s = stats.get_client(settings).pipeline()
session = session_factory(settings)
topic_handlers = {
ANNOTATION_TOPIC: messages.handle_annotation_event,
USER_TOPIC: messages.handle_user_event,
}
for msg in queue:
t_total = s.timer("streamer.msg.handler_total")
t_total.start()
try:
# All access to the database in the streamer is currently
# read-only, so enforce that:
session.execute(
"SET TRANSACTION "
"ISOLATION LEVEL SERIALIZABLE "
"READ ONLY "
"DEFERRABLE"
)
if isinstance(msg, messages.Message):
with s.timer("streamer.msg.handler_message"):
messages.handle_message(msg, settings, session, topic_handlers)
elif isinstance(msg, websocket.Message):
with s.timer("streamer.msg.handler_websocket"):
websocket.handle_message(msg, session)
else:
raise UnknownMessageType(repr(msg))
except (KeyboardInterrupt, SystemExit):
session.rollback()
raise
except Exception as exc:
log.warning("Caught exception handling streamer message:", exc_info=exc)
session.rollback()
else:
session.commit()
finally:
session.close()
t_total.stop()
s.send()
def process_work_queue(settings, queue, session_factory=None):
"""
Process each message from the queue in turn, handling exceptions.
This is the core of the streamer: we pull messages off the work queue,
dispatching them as appropriate. The handling of each message is wrapped in
code that ensures the database session is appropriately committed and
closed between messages.
"""
if session_factory is None:
session_factory = _get_session
s = stats.get_client(settings).pipeline()
session = session_factory(settings)
topic_handlers = {
ANNOTATION_TOPIC: messages.handle_annotation_event,
USER_TOPIC: messages.handle_user_event,
}
for msg in queue:
t_total = s.timer("streamer.msg.handler_total")
t_total.start()
try:
# All access to the database in the streamer is currently
# read-only, so enforce that:
session.execute(
"SET TRANSACTION "
"ISOLATION LEVEL SERIALIZABLE "
"READ ONLY "
"DEFERRABLE"
)
if isinstance(msg, messages.Message):
with s.timer("streamer.msg.handler_message"):
messages.handle_message(msg, settings, session, topic_handlers)
elif isinstance(msg, websocket.Message):
with s.timer("streamer.msg.handler_websocket"):
websocket.handle_message(msg, session)
else:
raise UnknownMessageType(repr(msg))
except (KeyboardInterrupt, SystemExit):
session.rollback()
raise
except Exception as exc:
log.warning("Caught exception handling streamer message:", exc_info=exc)
session.rollback()
else:
session.commit()
finally:
session.close()
t_total.stop()
s.send()
def test_handle_message_sets_socket_filter_for_filter_messages():
socket = mock.Mock()
socket.filter = None
message = websocket.Message(socket=socket, payload=json.dumps({
'filter': {
'actions': {},
'match_policy': 'include_all',
'clauses': [{
'field': '/uri',
'operator': 'equals',
'value': 'http://example.com',
}],
}
}))
websocket.handle_message(message)
assert socket.filter is not None
def process_work_queue(settings, queue, session_factory=db.Session):
"""
Process each message from the queue in turn, handling exceptions.
This is the core of the streamer: we pull messages off the work queue,
dispatching them as appropriate. The handling of each message is wrapped in
code that ensures the database session is appropriately committed and
closed between messages.
"""
session = session_factory()
annotations_topic = resolve_topic(ANNOTATIONS_TOPIC, settings=settings)
user_topic = resolve_topic(USER_TOPIC, settings=settings)
topic_handlers = {
annotations_topic: nsq.handle_annotation_event,
user_topic: nsq.handle_user_event,
}
for msg in queue:
try:
# All access to the database in the streamer is currently
# read-only, so enforce that:
session.execute("SET TRANSACTION "
"ISOLATION LEVEL SERIALIZABLE "
"READ ONLY "
"DEFERRABLE")
if isinstance(msg, nsq.Message):
nsq.handle_message(msg, topic_handlers=topic_handlers)
elif isinstance(msg, websocket.Message):
websocket.handle_message(msg)
else:
raise UnknownMessageType(repr(msg))
except (KeyboardInterrupt, SystemExit):
session.rollback()
raise
except:
log.exception('Caught exception handling streamer message:')
session.rollback()
else:
session.commit()
finally:
session.close()
def process_work_queue(settings, queue, session_factory=db.Session):
"""
Process each message from the queue in turn, handling exceptions.
This is the core of the streamer: we pull messages off the work queue,
dispatching them as appropriate. The handling of each message is wrapped in
code that ensures the database session is appropriately committed and
closed between messages.
"""
session = session_factory()
annotations_topic = resolve_topic(ANNOTATIONS_TOPIC, settings=settings)
user_topic = resolve_topic(USER_TOPIC, settings=settings)
topic_handlers = {
annotations_topic: nsq.handle_annotation_event,
user_topic: nsq.handle_user_event,
}
for msg in queue:
try:
# All access to the database in the streamer is currently
# read-only, so enforce that:
session.execute("SET TRANSACTION "
"ISOLATION LEVEL SERIALIZABLE "
"READ ONLY "
"DEFERRABLE")
if isinstance(msg, nsq.Message):
nsq.handle_message(msg, topic_handlers=topic_handlers)
elif isinstance(msg, websocket.Message):
websocket.handle_message(msg)
else:
raise UnknownMessageType(repr(msg))
except (KeyboardInterrupt, SystemExit):
session.rollback()
raise
except:
log.exception('Caught exception handling streamer message:')
session.rollback()
else:
session.commit()
finally:
session.close()
def process_work_queue(registry, queue):
"""
Process each message from the queue in turn, handling exceptions.
This is the core of the streamer: we pull messages off the work queue,
dispatching them as appropriate. The handling of each message is wrapped in
code that ensures the database session is appropriately committed and
closed between messages.
"""
session = db.get_session(registry.settings)
for msg in queue:
with db.read_only_transaction(session):
if isinstance(msg, messages.Message):
messages.handle_message(msg, registry, session, TOPIC_HANDLERS)
elif isinstance(msg, websocket.Message):
websocket.handle_message(msg, session)
else:
raise UnknownMessageType(repr(msg))
def test_handle_message_expands_uris_using_passed_session(expand_uri):
expand_uri.return_value = ['http://example.com', 'http://example.org/']
session = mock.sentinel.db_session
socket = mock.Mock()
socket.filter = None
message = websocket.Message(socket=socket, payload=json.dumps({
'filter': {
'actions': {},
'match_policy': 'include_all',
'clauses': [{
'field': '/uri',
'operator': 'equals',
'value': 'http://example.com',
}],
}
}))
websocket.handle_message(message, session=session)
expand_uri.assert_called_once_with(session, 'http://example.com')