def test_get_reader(fake_reader):
req = testing.DummyRequest()
req.registry.settings.update({"nsq.reader.addresses": "foo:1234\nbar:4567"})
queue.get_reader(req, "ethics-in-games-journalism", "channel4")
fake_reader.assert_called_with(
"ethics-in-games-journalism", "channel4", nsqd_tcp_addresses=["foo:1234", "bar:4567"]
)
def test_get_reader(fake_reader):
settings = {'nsq.reader.addresses': "foo:1234\nbar:4567"}
queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4')
fake_reader.assert_called_with('ethics-in-games-journalism',
'channel4',
nsqd_tcp_addresses=['foo:1234',
'bar:4567'])
def test_get_reader(fake_reader):
req = testing.DummyRequest()
req.registry.settings.update(
{'nsq.reader.addresses': "foo:1234\nbar:4567"})
queue.get_reader(req, 'ethics-in-games-journalism', 'channel4')
fake_reader.assert_called_with('ethics-in-games-journalism',
'channel4',
nsqd_tcp_addresses=['foo:1234', 'bar:4567'])
def test_get_reader(fake_reader):
req = testing.DummyRequest()
req.registry.settings.update({
'nsq.reader.addresses': "foo:1234\nbar:4567"
})
queue.get_reader(req, 'ethics-in-games-journalism', 'channel4')
fake_reader.assert_called_with('ethics-in-games-journalism',
'channel4',
nsqd_tcp_addresses=['foo:1234',
'bar:4567'])
def test_get_reader_namespace(fake_reader):
"""
When the ``nsq.namespace`` setting is provided, `get_reader` should return
a reader that automatically prefixes the namespace onto the name of the
topic being read.
"""
req = testing.DummyRequest()
req.registry.settings.update({"nsq.namespace": "abc123"})
queue.get_reader(req, "safari", "elephants")
fake_reader.assert_called_with("abc123-safari", "elephants", nsqd_tcp_addresses=["localhost:4150"])
def test_get_reader_namespace(fake_reader):
"""
When the ``nsq.namespace`` setting is provided, `get_reader` should return
a reader that automatically prefixes the namespace onto the name of the
topic being read.
"""
settings = {'nsq.namespace': "abc123"}
queue.get_reader(settings, 'safari', 'elephants')
fake_reader.assert_called_with('abc123-safari',
'elephants',
nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader_namespace(fake_reader):
"""
When the ``nsq.namespace`` setting is provided, `get_reader` should return
a reader that automatically prefixes the namespace onto the name of the
topic being read.
"""
settings = {'nsq.namespace': "abc123"}
queue.get_reader(settings, 'safari', 'elephants')
fake_reader.assert_called_with('abc123-safari',
'elephants',
nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader_namespace(fake_reader):
"""
When the ``nsq.namespace`` setting is provided, `get_reader` should return
a reader that automatically prefixes the namespace onto the name of the
topic being read.
"""
req = testing.DummyRequest()
req.registry.settings.update({'nsq.namespace': "abc123"})
queue.get_reader(req, 'safari', 'elephants')
fake_reader.assert_called_with('abc123-safari',
'elephants',
nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader_sentry_on_giving_up_hook(fake_client):
settings = {}
sentry = fake_client()
reader = queue.get_reader(settings, 'safari', 'elephants',
sentry_client=sentry)
reader.on_giving_up.send()
sentry.captureMessage.assert_called_with(extra={'topic': 'safari'})
def test_get_reader_respects_address_settings():
settings = {'nsq.reader.addresses': "foo:1234\nbar:4567"}
reader = queue.get_reader(settings,
'ethics-in-games-journalism',
'channel4')
assert reader.kwargs['nsqd_tcp_addresses'] == ['foo:1234', 'bar:4567']
def test_get_reader_sentry_on_exception_hook(fake_client):
settings = {}
sentry = fake_client()
reader = queue.get_reader(settings, 'safari', 'elephants',
sentry_client=sentry)
reader.on_exception.send(error='An error happened')
sentry.captureException.assert_called_with(exc_info=True,
extra={'topic': 'safari'})
def test_get_reader_sentry_on_error_hook(fake_client):
settings = {}
sentry = fake_client()
reader = queue.get_reader(settings, 'safari', 'elephants',
sentry_client=sentry)
reader.on_error.send()
sentry.captureException.assert_called_with(
exc_info=(type(None), None, None),
extra={'topic': 'safari'})
def test_get_reader_default():
settings = {}
reader = queue.get_reader(settings,
'ethics-in-games-journalism',
'channel4')
assert reader.topic == 'ethics-in-games-journalism'
assert reader.channel == 'channel4'
assert reader.kwargs['nsqd_tcp_addresses'] == ['localhost:4150']
def test_get_reader_sentry_on_giving_up_hook(fake_client):
settings = {}
sentry = fake_client()
reader = queue.get_reader(settings,
'safari',
'elephants',
sentry_client=sentry)
reader.on_giving_up.send()
sentry.captureMessage.assert_called_with(extra={'topic': 'safari'})
def test_get_reader_sentry_on_exception_hook(fake_client):
settings = {}
sentry = fake_client()
reader = queue.get_reader(settings,
'safari',
'elephants',
sentry_client=sentry)
reader.on_exception.send(error='An error happened')
sentry.captureException.assert_called_with(exc_info=True,
extra={'topic': 'safari'})
def test_get_reader_uses_namespace():
"""
When the ``nsq.namespace`` setting is provided, `get_reader` should return
a reader that automatically prefixes the namespace onto the name of the
topic being read.
"""
settings = {'nsq.namespace': "abc123"}
reader = queue.get_reader(settings, 'safari', 'elephants')
assert reader.topic == 'abc123-safari'
def test_get_reader_sentry_on_error_hook(fake_client):
settings = {}
sentry = fake_client()
reader = queue.get_reader(settings,
'safari',
'elephants',
sentry_client=sentry)
reader.on_error.send()
sentry.captureException.assert_called_with(exc_info=(type(None), None,
None),
extra={'topic': 'safari'})
def test_get_reader_connects_on_error_hook_to_sentry_client():
settings = {}
client = FakeClient()
reader = queue.get_reader(settings,
'safari',
'elephants',
sentry_client=client)
error = RuntimeError("asplode!")
reader.simulate_error(error=error)
client.captureException.assert_called_with(
exc_info=(RuntimeError, error, None),
extra={'topic': 'safari', 'channel': 'elephants'})
def test_get_reader_connects_on_exception_hook_to_sentry_client():
settings = {}
client = FakeClient()
reader = queue.get_reader(settings,
'safari',
'elephants',
sentry_client=client)
reader.simulate_exception(message=FakeMessage("foobar"))
client.captureException.assert_called_with(exc_info=True,
extra={'topic': 'safari',
'channel': 'elephants',
'message': 'foobar'})
def test_get_reader_connects_on_giving_up_hook_to_sentry_client():
settings = {}
client = FakeClient()
reader = queue.get_reader(settings,
'safari',
'elephants',
sentry_client=client)
reader.simulate_giving_up(message=FakeMessage("nopeski"))
client.captureMessage.assert_called_with("Giving up on message",
extra={'topic': 'safari',
'channel': 'elephants',
'message': 'nopeski'})
def process_queue(settings, topic, handler):
"""
Configure, start, and monitor a queue reader for the specified topic.
This sets up a :py:class:`gnsq.Reader` to route messages from `topic` to
`handler`, and starts it. The reader should never return. If it does, this
fact is logged and the function returns.
"""
channel = 'stream-{}#ephemeral'.format(_random_id())
receiver = functools.partial(process_message, handler)
reader = queue.get_reader(settings, topic, channel)
reader.on_message.connect(receiver=receiver, weak=False)
reader.start(block=True)
# We should never get here. If we do, it's because a reader thread has
# prematurely quit.
log.error("queue reader for topic '%s' exited: killing reader", topic)
reader.close()
def process_queue(settings, topic, handler):
"""
Configure, start, and monitor a queue reader for the specified topic.
This sets up a :py:class:`gnsq.Reader` to route messages from `topic` to
`handler`, and starts it. The reader should never return. If it does, this
fact is logged and the function returns.
"""
channel = 'stream-{}#ephemeral'.format(_random_id())
receiver = functools.partial(process_message, handler)
reader = queue.get_reader(settings, topic, channel)
reader.on_message.connect(receiver=receiver, weak=False)
reader.start(block=True)
# We should never get here. If we do, it's because a reader thread has
# prematurely quit.
log.error("queue reader for topic '%s' exited: killing reader", topic)
reader.close()
def process_nsq_topic(settings, topic, work_queue, raise_error=True):
"""
Configure, start, and monitor a queue reader for the specified topic.
This sets up a :py:class:`gnsq.Reader` to route messages from `topic` to
the passed `work_queue`, and starts it. The reader should never return. If
it does, this function will raise an exception.
If `raise_error` is False, the function will not reraise errors from the
queue reader.
"""
channel = 'stream-{}#ephemeral'.format(_random_id())
client = h.sentry.get_client(settings)
reader = queue.get_reader(settings, topic, channel, sentry_client=client)
# The only thing queue readers do is put the incoming messages onto the
# work queue.
#
# Note that this means that any errors occurring while handling the
# messages will not result in requeues, but this is probably the best we
# can do given that these messages fan out to our WebSocket clients, and we
# can't easily know that we haven't already sent a requeued message out to
# a particular client.
def _handler(reader, message):
try:
work_queue.put(Message(topic=reader.topic, payload=message.body),
timeout=0.1)
except Full:
log.warn('Streamer work queue full! Unable to queue message from '
'NSQ having waited 0.1s: giving up.')
reader.on_message.connect(receiver=_handler, weak=False)
reader.start()
# Reraise any exceptions raised by reader greenlets
reader.join(raise_error=raise_error)
# We should never get here: if we do, it means that the reader exited
# without raising an exception, which doesn't make much sense.
if raise_error:
raise RuntimeError('Queue reader quit unexpectedly!')
def process_nsq_topic(settings, topic, work_queue, raise_error=True):
"""
Configure, start, and monitor a queue reader for the specified topic.
This sets up a :py:class:`gnsq.Reader` to route messages from `topic` to
the passed `work_queue`, and starts it. The reader should never return. If
it does, this function will raise an exception.
If `raise_error` is False, the function will not reraise errors from the
queue reader.
"""
channel = 'stream-{}#ephemeral'.format(_random_id())
client = h.sentry.get_client(settings)
reader = queue.get_reader(settings, topic, channel, sentry_client=client)
# The only thing queue readers do is put the incoming messages onto the
# work queue.
#
# Note that this means that any errors occurring while handling the
# messages will not result in requeues, but this is probably the best we
# can do given that these messages fan out to our WebSocket clients, and we
# can't easily know that we haven't already sent a requeued message out to
# a particular client.
def _handler(reader, message):
try:
work_queue.put(Message(topic=reader.topic, payload=message.body),
timeout=0.1)
except Full:
log.warn('Streamer work queue full! Unable to queue message from '
'NSQ having waited 0.1s: giving up.')
reader.on_message.connect(receiver=_handler, weak=False)
reader.start()
# Reraise any exceptions raised by reader greenlets
reader.join(raise_error=raise_error)
# We should never get here: if we do, it means that the reader exited
# without raising an exception, which doesn't make much sense.
if raise_error:
raise RuntimeError('Queue reader quit unexpectedly!')
def test_get_reader_default(fake_reader):
req = testing.DummyRequest()
queue.get_reader(req, 'ethics-in-games-journalism', 'channel4')
fake_reader.assert_called_with('ethics-in-games-journalism',
'channel4',
nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader_default(fake_reader):
req = testing.DummyRequest()
queue.get_reader(req, 'ethics-in-games-journalism', 'channel4')
fake_reader.assert_called_with('ethics-in-games-journalism',
'channel4',
nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader(fake_reader):
settings = {'nsq.reader.addresses': "foo:1234\nbar:4567"}
queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4')
fake_reader.assert_called_with('ethics-in-games-journalism',
'channel4',
nsqd_tcp_addresses=['foo:1234', 'bar:4567'])
def test_get_reader_default(fake_reader):
settings = {}
queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4')
fake_reader.assert_called_with('ethics-in-games-journalism',
'channel4',
nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader_default(fake_reader):
settings = {}
queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4')
fake_reader.assert_called_with('ethics-in-games-journalism',
'channel4',
nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader_default(fake_reader):
req = testing.DummyRequest()
queue.get_reader(req, "ethics-in-games-journalism", "channel4")
fake_reader.assert_called_with("ethics-in-games-journalism", "channel4", nsqd_tcp_addresses=["localhost:4150"])
