def test_get_reader(fake_reader): req = testing.DummyRequest() req.registry.settings.update({"nsq.reader.addresses": "foo:1234\nbar:4567"}) queue.get_reader(req, "ethics-in-games-journalism", "channel4") fake_reader.assert_called_with( "ethics-in-games-journalism", "channel4", nsqd_tcp_addresses=["foo:1234", "bar:4567"] )
def test_get_reader(fake_reader): settings = {'nsq.reader.addresses': "foo:1234\nbar:4567"} queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4') fake_reader.assert_called_with('ethics-in-games-journalism', 'channel4', nsqd_tcp_addresses=['foo:1234', 'bar:4567'])
def test_get_reader(fake_reader): req = testing.DummyRequest() req.registry.settings.update( {'nsq.reader.addresses': "foo:1234\nbar:4567"}) queue.get_reader(req, 'ethics-in-games-journalism', 'channel4') fake_reader.assert_called_with('ethics-in-games-journalism', 'channel4', nsqd_tcp_addresses=['foo:1234', 'bar:4567'])
def test_get_reader(fake_reader): req = testing.DummyRequest() req.registry.settings.update({ 'nsq.reader.addresses': "foo:1234\nbar:4567" }) queue.get_reader(req, 'ethics-in-games-journalism', 'channel4') fake_reader.assert_called_with('ethics-in-games-journalism', 'channel4', nsqd_tcp_addresses=['foo:1234', 'bar:4567'])
def test_get_reader_namespace(fake_reader): """ When the ``nsq.namespace`` setting is provided, `get_reader` should return a reader that automatically prefixes the namespace onto the name of the topic being read. """ req = testing.DummyRequest() req.registry.settings.update({"nsq.namespace": "abc123"}) queue.get_reader(req, "safari", "elephants") fake_reader.assert_called_with("abc123-safari", "elephants", nsqd_tcp_addresses=["localhost:4150"])
def test_get_reader_namespace(fake_reader): """ When the ``nsq.namespace`` setting is provided, `get_reader` should return a reader that automatically prefixes the namespace onto the name of the topic being read. """ settings = {'nsq.namespace': "abc123"} queue.get_reader(settings, 'safari', 'elephants') fake_reader.assert_called_with('abc123-safari', 'elephants', nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader_namespace(fake_reader): """ When the ``nsq.namespace`` setting is provided, `get_reader` should return a reader that automatically prefixes the namespace onto the name of the topic being read. """ req = testing.DummyRequest() req.registry.settings.update({'nsq.namespace': "abc123"}) queue.get_reader(req, 'safari', 'elephants') fake_reader.assert_called_with('abc123-safari', 'elephants', nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader_sentry_on_giving_up_hook(fake_client): settings = {} sentry = fake_client() reader = queue.get_reader(settings, 'safari', 'elephants', sentry_client=sentry) reader.on_giving_up.send() sentry.captureMessage.assert_called_with(extra={'topic': 'safari'})
def test_get_reader_respects_address_settings(): settings = {'nsq.reader.addresses': "foo:1234\nbar:4567"} reader = queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4') assert reader.kwargs['nsqd_tcp_addresses'] == ['foo:1234', 'bar:4567']
def test_get_reader_sentry_on_exception_hook(fake_client): settings = {} sentry = fake_client() reader = queue.get_reader(settings, 'safari', 'elephants', sentry_client=sentry) reader.on_exception.send(error='An error happened') sentry.captureException.assert_called_with(exc_info=True, extra={'topic': 'safari'})
def test_get_reader_sentry_on_error_hook(fake_client): settings = {} sentry = fake_client() reader = queue.get_reader(settings, 'safari', 'elephants', sentry_client=sentry) reader.on_error.send() sentry.captureException.assert_called_with( exc_info=(type(None), None, None), extra={'topic': 'safari'})
def test_get_reader_default(): settings = {} reader = queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4') assert reader.topic == 'ethics-in-games-journalism' assert reader.channel == 'channel4' assert reader.kwargs['nsqd_tcp_addresses'] == ['localhost:4150']
def test_get_reader_uses_namespace(): """ When the ``nsq.namespace`` setting is provided, `get_reader` should return a reader that automatically prefixes the namespace onto the name of the topic being read. """ settings = {'nsq.namespace': "abc123"} reader = queue.get_reader(settings, 'safari', 'elephants') assert reader.topic == 'abc123-safari'
def test_get_reader_sentry_on_error_hook(fake_client): settings = {} sentry = fake_client() reader = queue.get_reader(settings, 'safari', 'elephants', sentry_client=sentry) reader.on_error.send() sentry.captureException.assert_called_with(exc_info=(type(None), None, None), extra={'topic': 'safari'})
def test_get_reader_connects_on_error_hook_to_sentry_client(): settings = {} client = FakeClient() reader = queue.get_reader(settings, 'safari', 'elephants', sentry_client=client) error = RuntimeError("asplode!") reader.simulate_error(error=error) client.captureException.assert_called_with( exc_info=(RuntimeError, error, None), extra={'topic': 'safari', 'channel': 'elephants'})
def test_get_reader_connects_on_exception_hook_to_sentry_client(): settings = {} client = FakeClient() reader = queue.get_reader(settings, 'safari', 'elephants', sentry_client=client) reader.simulate_exception(message=FakeMessage("foobar")) client.captureException.assert_called_with(exc_info=True, extra={'topic': 'safari', 'channel': 'elephants', 'message': 'foobar'})
def test_get_reader_connects_on_giving_up_hook_to_sentry_client(): settings = {} client = FakeClient() reader = queue.get_reader(settings, 'safari', 'elephants', sentry_client=client) reader.simulate_giving_up(message=FakeMessage("nopeski")) client.captureMessage.assert_called_with("Giving up on message", extra={'topic': 'safari', 'channel': 'elephants', 'message': 'nopeski'})
def process_queue(settings, topic, handler): """ Configure, start, and monitor a queue reader for the specified topic. This sets up a :py:class:`gnsq.Reader` to route messages from `topic` to `handler`, and starts it. The reader should never return. If it does, this fact is logged and the function returns. """ channel = 'stream-{}#ephemeral'.format(_random_id()) receiver = functools.partial(process_message, handler) reader = queue.get_reader(settings, topic, channel) reader.on_message.connect(receiver=receiver, weak=False) reader.start(block=True) # We should never get here. If we do, it's because a reader thread has # prematurely quit. log.error("queue reader for topic '%s' exited: killing reader", topic) reader.close()
def process_nsq_topic(settings, topic, work_queue, raise_error=True): """ Configure, start, and monitor a queue reader for the specified topic. This sets up a :py:class:`gnsq.Reader` to route messages from `topic` to the passed `work_queue`, and starts it. The reader should never return. If it does, this function will raise an exception. If `raise_error` is False, the function will not reraise errors from the queue reader. """ channel = 'stream-{}#ephemeral'.format(_random_id()) client = h.sentry.get_client(settings) reader = queue.get_reader(settings, topic, channel, sentry_client=client) # The only thing queue readers do is put the incoming messages onto the # work queue. # # Note that this means that any errors occurring while handling the # messages will not result in requeues, but this is probably the best we # can do given that these messages fan out to our WebSocket clients, and we # can't easily know that we haven't already sent a requeued message out to # a particular client. def _handler(reader, message): try: work_queue.put(Message(topic=reader.topic, payload=message.body), timeout=0.1) except Full: log.warn('Streamer work queue full! Unable to queue message from ' 'NSQ having waited 0.1s: giving up.') reader.on_message.connect(receiver=_handler, weak=False) reader.start() # Reraise any exceptions raised by reader greenlets reader.join(raise_error=raise_error) # We should never get here: if we do, it means that the reader exited # without raising an exception, which doesn't make much sense. if raise_error: raise RuntimeError('Queue reader quit unexpectedly!')
def test_get_reader_default(fake_reader): req = testing.DummyRequest() queue.get_reader(req, 'ethics-in-games-journalism', 'channel4') fake_reader.assert_called_with('ethics-in-games-journalism', 'channel4', nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader_default(fake_reader): settings = {} queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4') fake_reader.assert_called_with('ethics-in-games-journalism', 'channel4', nsqd_tcp_addresses=['localhost:4150'])
def test_get_reader_default(fake_reader): req = testing.DummyRequest() queue.get_reader(req, "ethics-in-games-journalism", "channel4") fake_reader.assert_called_with("ethics-in-games-journalism", "channel4", nsqd_tcp_addresses=["localhost:4150"])