Beispiel #1
0
def test_get_reader(fake_reader):
    req = testing.DummyRequest()
    req.registry.settings.update({"nsq.reader.addresses": "foo:1234\nbar:4567"})
    queue.get_reader(req, "ethics-in-games-journalism", "channel4")
    fake_reader.assert_called_with(
        "ethics-in-games-journalism", "channel4", nsqd_tcp_addresses=["foo:1234", "bar:4567"]
    )
Beispiel #2
0
def test_get_reader(fake_reader):
    settings = {'nsq.reader.addresses': "foo:1234\nbar:4567"}
    queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4')
    fake_reader.assert_called_with('ethics-in-games-journalism',
                                   'channel4',
                                   nsqd_tcp_addresses=['foo:1234',
                                                       'bar:4567'])
Beispiel #3
0
def test_get_reader(fake_reader):
    req = testing.DummyRequest()
    req.registry.settings.update(
        {'nsq.reader.addresses': "foo:1234\nbar:4567"})
    queue.get_reader(req, 'ethics-in-games-journalism', 'channel4')
    fake_reader.assert_called_with('ethics-in-games-journalism',
                                   'channel4',
                                   nsqd_tcp_addresses=['foo:1234', 'bar:4567'])
Beispiel #4
0
def test_get_reader(fake_reader):
    req = testing.DummyRequest()
    req.registry.settings.update({
        'nsq.reader.addresses': "foo:1234\nbar:4567"
    })
    queue.get_reader(req, 'ethics-in-games-journalism', 'channel4')
    fake_reader.assert_called_with('ethics-in-games-journalism',
                                   'channel4',
                                   nsqd_tcp_addresses=['foo:1234',
                                                       'bar:4567'])
Beispiel #5
0
def test_get_reader_namespace(fake_reader):
    """
    When the ``nsq.namespace`` setting is provided, `get_reader` should return
    a reader that automatically prefixes the namespace onto the name of the
    topic being read.
    """
    req = testing.DummyRequest()
    req.registry.settings.update({"nsq.namespace": "abc123"})
    queue.get_reader(req, "safari", "elephants")
    fake_reader.assert_called_with("abc123-safari", "elephants", nsqd_tcp_addresses=["localhost:4150"])
Beispiel #6
0
def test_get_reader_namespace(fake_reader):
    """
    When the ``nsq.namespace`` setting is provided, `get_reader` should return
    a reader that automatically prefixes the namespace onto the name of the
    topic being read.
    """
    settings = {'nsq.namespace': "abc123"}
    queue.get_reader(settings, 'safari', 'elephants')
    fake_reader.assert_called_with('abc123-safari',
                                   'elephants',
                                   nsqd_tcp_addresses=['localhost:4150'])
Beispiel #7
0
def test_get_reader_namespace(fake_reader):
    """
    When the ``nsq.namespace`` setting is provided, `get_reader` should return
    a reader that automatically prefixes the namespace onto the name of the
    topic being read.
    """
    settings = {'nsq.namespace': "abc123"}
    queue.get_reader(settings, 'safari', 'elephants')
    fake_reader.assert_called_with('abc123-safari',
                                   'elephants',
                                   nsqd_tcp_addresses=['localhost:4150'])
Beispiel #8
0
def test_get_reader_namespace(fake_reader):
    """
    When the ``nsq.namespace`` setting is provided, `get_reader` should return
    a reader that automatically prefixes the namespace onto the name of the
    topic being read.
    """
    req = testing.DummyRequest()
    req.registry.settings.update({'nsq.namespace': "abc123"})
    queue.get_reader(req, 'safari', 'elephants')
    fake_reader.assert_called_with('abc123-safari',
                                   'elephants',
                                   nsqd_tcp_addresses=['localhost:4150'])
Beispiel #9
0
def test_get_reader_sentry_on_giving_up_hook(fake_client):
    settings = {}
    sentry = fake_client()
    reader = queue.get_reader(settings, 'safari', 'elephants',
                              sentry_client=sentry)
    reader.on_giving_up.send()

    sentry.captureMessage.assert_called_with(extra={'topic': 'safari'})
Beispiel #10
0
def test_get_reader_respects_address_settings():
    settings = {'nsq.reader.addresses': "foo:1234\nbar:4567"}

    reader = queue.get_reader(settings,
                              'ethics-in-games-journalism',
                              'channel4')

    assert reader.kwargs['nsqd_tcp_addresses'] == ['foo:1234', 'bar:4567']
Beispiel #11
0
def test_get_reader_sentry_on_exception_hook(fake_client):
    settings = {}
    sentry = fake_client()
    reader = queue.get_reader(settings, 'safari', 'elephants',
                              sentry_client=sentry)
    reader.on_exception.send(error='An error happened')

    sentry.captureException.assert_called_with(exc_info=True,
                                               extra={'topic': 'safari'})
Beispiel #12
0
def test_get_reader_sentry_on_error_hook(fake_client):
    settings = {}
    sentry = fake_client()
    reader = queue.get_reader(settings, 'safari', 'elephants',
                              sentry_client=sentry)
    reader.on_error.send()

    sentry.captureException.assert_called_with(
        exc_info=(type(None), None, None),
        extra={'topic': 'safari'})
Beispiel #13
0
def test_get_reader_default():
    settings = {}

    reader = queue.get_reader(settings,
                              'ethics-in-games-journalism',
                              'channel4')

    assert reader.topic == 'ethics-in-games-journalism'
    assert reader.channel == 'channel4'
    assert reader.kwargs['nsqd_tcp_addresses'] == ['localhost:4150']
Beispiel #14
0
def test_get_reader_sentry_on_giving_up_hook(fake_client):
    settings = {}
    sentry = fake_client()
    reader = queue.get_reader(settings,
                              'safari',
                              'elephants',
                              sentry_client=sentry)
    reader.on_giving_up.send()

    sentry.captureMessage.assert_called_with(extra={'topic': 'safari'})
Beispiel #15
0
def test_get_reader_sentry_on_exception_hook(fake_client):
    settings = {}
    sentry = fake_client()
    reader = queue.get_reader(settings,
                              'safari',
                              'elephants',
                              sentry_client=sentry)
    reader.on_exception.send(error='An error happened')

    sentry.captureException.assert_called_with(exc_info=True,
                                               extra={'topic': 'safari'})
Beispiel #16
0
def test_get_reader_uses_namespace():
    """
    When the ``nsq.namespace`` setting is provided, `get_reader` should return
    a reader that automatically prefixes the namespace onto the name of the
    topic being read.
    """
    settings = {'nsq.namespace': "abc123"}

    reader = queue.get_reader(settings, 'safari', 'elephants')

    assert reader.topic == 'abc123-safari'
Beispiel #17
0
def test_get_reader_sentry_on_error_hook(fake_client):
    settings = {}
    sentry = fake_client()
    reader = queue.get_reader(settings,
                              'safari',
                              'elephants',
                              sentry_client=sentry)
    reader.on_error.send()

    sentry.captureException.assert_called_with(exc_info=(type(None), None,
                                                         None),
                                               extra={'topic': 'safari'})
Beispiel #18
0
def test_get_reader_connects_on_error_hook_to_sentry_client():
    settings = {}
    client = FakeClient()
    reader = queue.get_reader(settings,
                              'safari',
                              'elephants',
                              sentry_client=client)

    error = RuntimeError("asplode!")
    reader.simulate_error(error=error)

    client.captureException.assert_called_with(
        exc_info=(RuntimeError, error, None),
        extra={'topic': 'safari', 'channel': 'elephants'})
Beispiel #19
0
def test_get_reader_connects_on_exception_hook_to_sentry_client():
    settings = {}
    client = FakeClient()
    reader = queue.get_reader(settings,
                              'safari',
                              'elephants',
                              sentry_client=client)

    reader.simulate_exception(message=FakeMessage("foobar"))

    client.captureException.assert_called_with(exc_info=True,
                                               extra={'topic': 'safari',
                                                      'channel': 'elephants',
                                                      'message': 'foobar'})
Beispiel #20
0
def test_get_reader_connects_on_giving_up_hook_to_sentry_client():
    settings = {}
    client = FakeClient()
    reader = queue.get_reader(settings,
                              'safari',
                              'elephants',
                              sentry_client=client)

    reader.simulate_giving_up(message=FakeMessage("nopeski"))

    client.captureMessage.assert_called_with("Giving up on message",
                                             extra={'topic': 'safari',
                                                    'channel': 'elephants',
                                                    'message': 'nopeski'})
Beispiel #21
0
def process_queue(settings, topic, handler):
    """
    Configure, start, and monitor a queue reader for the specified topic.

    This sets up a :py:class:`gnsq.Reader` to route messages from `topic` to
    `handler`, and starts it. The reader should never return. If it does, this
    fact is logged and the function returns.
    """
    channel = 'stream-{}#ephemeral'.format(_random_id())
    receiver = functools.partial(process_message, handler)
    reader = queue.get_reader(settings, topic, channel)
    reader.on_message.connect(receiver=receiver, weak=False)
    reader.start(block=True)

    # We should never get here. If we do, it's because a reader thread has
    # prematurely quit.
    log.error("queue reader for topic '%s' exited: killing reader", topic)
    reader.close()
Beispiel #22
0
def process_queue(settings, topic, handler):
    """
    Configure, start, and monitor a queue reader for the specified topic.

    This sets up a :py:class:`gnsq.Reader` to route messages from `topic` to
    `handler`, and starts it. The reader should never return. If it does, this
    fact is logged and the function returns.
    """
    channel = 'stream-{}#ephemeral'.format(_random_id())
    receiver = functools.partial(process_message, handler)
    reader = queue.get_reader(settings, topic, channel)
    reader.on_message.connect(receiver=receiver, weak=False)
    reader.start(block=True)

    # We should never get here. If we do, it's because a reader thread has
    # prematurely quit.
    log.error("queue reader for topic '%s' exited: killing reader", topic)
    reader.close()
Beispiel #23
0
def process_nsq_topic(settings, topic, work_queue, raise_error=True):
    """
    Configure, start, and monitor a queue reader for the specified topic.

    This sets up a :py:class:`gnsq.Reader` to route messages from `topic` to
    the passed `work_queue`, and starts it. The reader should never return. If
    it does, this function will raise an exception.

    If `raise_error` is False, the function will not reraise errors from the
    queue reader.
    """
    channel = 'stream-{}#ephemeral'.format(_random_id())
    client = h.sentry.get_client(settings)
    reader = queue.get_reader(settings, topic, channel, sentry_client=client)

    # The only thing queue readers do is put the incoming messages onto the
    # work queue.
    #
    # Note that this means that any errors occurring while handling the
    # messages will not result in requeues, but this is probably the best we
    # can do given that these messages fan out to our WebSocket clients, and we
    # can't easily know that we haven't already sent a requeued message out to
    # a particular client.
    def _handler(reader, message):
        try:
            work_queue.put(Message(topic=reader.topic, payload=message.body),
                           timeout=0.1)
        except Full:
            log.warn('Streamer work queue full! Unable to queue message from '
                     'NSQ having waited 0.1s: giving up.')

    reader.on_message.connect(receiver=_handler, weak=False)
    reader.start()

    # Reraise any exceptions raised by reader greenlets
    reader.join(raise_error=raise_error)

    # We should never get here: if we do, it means that the reader exited
    # without raising an exception, which doesn't make much sense.
    if raise_error:
        raise RuntimeError('Queue reader quit unexpectedly!')
Beispiel #24
0
def process_nsq_topic(settings, topic, work_queue, raise_error=True):
    """
    Configure, start, and monitor a queue reader for the specified topic.

    This sets up a :py:class:`gnsq.Reader` to route messages from `topic` to
    the passed `work_queue`, and starts it. The reader should never return. If
    it does, this function will raise an exception.

    If `raise_error` is False, the function will not reraise errors from the
    queue reader.
    """
    channel = 'stream-{}#ephemeral'.format(_random_id())
    client = h.sentry.get_client(settings)
    reader = queue.get_reader(settings, topic, channel, sentry_client=client)

    # The only thing queue readers do is put the incoming messages onto the
    # work queue.
    #
    # Note that this means that any errors occurring while handling the
    # messages will not result in requeues, but this is probably the best we
    # can do given that these messages fan out to our WebSocket clients, and we
    # can't easily know that we haven't already sent a requeued message out to
    # a particular client.
    def _handler(reader, message):
        try:
            work_queue.put(Message(topic=reader.topic, payload=message.body),
                           timeout=0.1)
        except Full:
            log.warn('Streamer work queue full! Unable to queue message from '
                     'NSQ having waited 0.1s: giving up.')

    reader.on_message.connect(receiver=_handler, weak=False)
    reader.start()

    # Reraise any exceptions raised by reader greenlets
    reader.join(raise_error=raise_error)

    # We should never get here: if we do, it means that the reader exited
    # without raising an exception, which doesn't make much sense.
    if raise_error:
        raise RuntimeError('Queue reader quit unexpectedly!')
Beispiel #25
0
def test_get_reader_default(fake_reader):
    req = testing.DummyRequest()
    queue.get_reader(req, 'ethics-in-games-journalism', 'channel4')
    fake_reader.assert_called_with('ethics-in-games-journalism',
                                   'channel4',
                                   nsqd_tcp_addresses=['localhost:4150'])
Beispiel #26
0
def test_get_reader_default(fake_reader):
    req = testing.DummyRequest()
    queue.get_reader(req, 'ethics-in-games-journalism', 'channel4')
    fake_reader.assert_called_with('ethics-in-games-journalism',
                                   'channel4',
                                   nsqd_tcp_addresses=['localhost:4150'])
Beispiel #27
0
def test_get_reader(fake_reader):
    settings = {'nsq.reader.addresses': "foo:1234\nbar:4567"}
    queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4')
    fake_reader.assert_called_with('ethics-in-games-journalism',
                                   'channel4',
                                   nsqd_tcp_addresses=['foo:1234', 'bar:4567'])
Beispiel #28
0
def test_get_reader_default(fake_reader):
    settings = {}
    queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4')
    fake_reader.assert_called_with('ethics-in-games-journalism',
                                   'channel4',
                                   nsqd_tcp_addresses=['localhost:4150'])
Beispiel #29
0
def test_get_reader_default(fake_reader):
    settings = {}
    queue.get_reader(settings, 'ethics-in-games-journalism', 'channel4')
    fake_reader.assert_called_with('ethics-in-games-journalism',
                                   'channel4',
                                   nsqd_tcp_addresses=['localhost:4150'])
Beispiel #30
0
def test_get_reader_default(fake_reader):
    req = testing.DummyRequest()
    queue.get_reader(req, "ethics-in-games-journalism", "channel4")
    fake_reader.assert_called_with("ethics-in-games-journalism", "channel4", nsqd_tcp_addresses=["localhost:4150"])