def test_add_remove_readers(self):
hub = Hub()
P = hub.poller = Mock()
read_A = Mock()
read_B = Mock()
hub.add_reader(10, read_A, 10)
hub.add_reader(File(11), read_B, 11)
P.register.assert_has_calls([
call(10, hub.READ | hub.ERR),
call(11, hub.READ | hub.ERR),
],
any_order=True)
self.assertEqual(hub.readers[10], (read_A, (10, )))
self.assertEqual(hub.readers[11], (read_B, (11, )))
hub.remove(10)
self.assertNotIn(10, hub.readers)
hub.remove(File(11))
self.assertNotIn(11, hub.readers)
P.unregister.assert_has_calls([
call(10),
call(11),
])
def test_poller_regeneration_on_access(self):
self.hub = Hub()
assert self.hub.poller
self.hub.stop()
self.hub.close()
assert self.hub._poller is None
assert self.hub.poller, 'It should be regenerated automatically!'
def test_add_writers(self):
hub = Hub()
P = hub.poller = Mock()
write_A = Mock()
write_B = Mock()
hub.add_writer(20, write_A)
hub.add_writer(File(21), write_B)
P.register.assert_has_calls([
call(20, hub.WRITE),
call(21, hub.WRITE),
],
any_order=True)
self.assertEqual(hub.writers[20], (write_A, ()))
self.assertEqual(hub.writers[21], (write_B, ()))
hub.remove(20)
self.assertNotIn(20, hub.writers)
hub.remove(File(21))
self.assertNotIn(21, hub.writers)
P.unregister.assert_has_calls([
call(20),
call(21),
])
def test_repr_active(self):
hub = Hub()
hub.readers = {1: Mock(), 2: Mock()}
hub.writers = {3: Mock(), 4: Mock()}
for value in list(hub.readers.values()) + list(hub.writers.values()):
value.__name__ = 'mock'
self.assertTrue(hub.repr_active())
def test_start_stop(self, poll):
self.hub = Hub()
poll.assert_called_with()
poller = self.hub.poller
self.hub.stop()
self.hub.close()
poller.close.assert_called_with()
def consume(self, queue, callback):
"""
对于消费者而言，需要知道两个信息：
1. 从哪个queue消费
2. 消费后回调函数
"""
self._hub = Hub()
self._conn.register_with_event_loop(self._hub)
with Consumer(self._conn, [queue], on_message=callback):
self._hub.run_forever()
def test_callback_for(self):
hub = Hub()
reader, writer = Mock(), Mock()
hub.readers = {6: reader}
hub.writers = {7: writer}
self.assertEqual(callback_for(hub, 6, READ), reader)
self.assertEqual(callback_for(hub, 7, WRITE), writer)
with self.assertRaises(KeyError):
callback_for(hub, 6, WRITE)
self.assertEqual(callback_for(hub, 6, WRITE, 'foo'), 'foo')
def test_start_stop(self, poll):
self.hub = Hub()
poll.assert_called_with()
poller = self.hub.poller
self.hub.stop()
mock_callback = Mock()
self.hub._ready = set([mock_callback])
self.hub.close()
poller.close.assert_called_with()
mock_callback.assert_called_once_with()
assert self.hub._ready == set()
class KombuClient():
"""rabbitmq client base on celery.kombu"""
def __init__(self, amqp_url):
self._conn = Connection(amqp_url)
self._hub = None
def queue_declare(self, queue):
"""
:param queue: kombu.queue obj
:return: Returns a tuple containing 3 items:
1. the name of the queue (essential for automatically-named queues),
2. message count
3. consumer count
"""
channel = self._conn.channel()
return queue.queue_declare(channel=channel)
def produce(self, message, exchange, routing_key):
"""
对于生产者而言，需要知道三个信息：
1. 发送的exchange
2. 分发的routing_key
3. 发送的消息
"""
producer = Producer(self._conn)
producer.publish(
body=message,
exchange=exchange,
routing_key=routing_key,
)
def consume(self, queue, callback):
"""
对于消费者而言，需要知道两个信息：
1. 从哪个queue消费
2. 消费后回调函数
"""
self._hub = Hub()
self._conn.register_with_event_loop(self._hub)
with Consumer(self._conn, [queue], on_message=callback):
self._hub.run_forever()
def close(self):
"""
资源释放：
1. 停止轮询
2. 关闭连接
"""
if self._hub:
self._hub.stop()
self._conn.close()
def test_repr_events(self):
hub = Hub()
hub.readers = {6: Mock(), 7: Mock(), 8: Mock()}
hub.writers = {9: Mock()}
for value in list(hub.readers.values()) + list(hub.writers.values()):
value.__name__ = 'mock'
self.assertTrue(
hub.repr_events([
(6, READ),
(7, ERR),
(8, READ | ERR),
(9, WRITE),
(10, 13213),
]))
def handle(self, queue, *args, **options):
fmt = '%(asctime)s %(name)-12s: %(levelname)-8s %(message)s'
log_level = 40 - (options['verbosity'] * 10)
logging.basicConfig(level=log_level, format=fmt)
# TODO: Ensure that failed processing does not requeue task into
# work queue
set_event_loop(Hub())
kwargs = {
'transport_options': conf.broker_transport_options,
}
with app.connection(**kwargs) as conn:
logger.info("Launching worker")
worker = Worker(conn, queues=[Queue(queue)])
worker.connect_max_retries = 1
while True:
try:
breakers.celery.call(worker.run)
except KeyboardInterrupt:
logger.info("KeyboardInterrupt, exiting. Bye!")
break
except breakers.celery.expected_errors:
rest_val = 5
logger.warning(
"Open circuit detected. "
"Sleeping for %s seconds and then will try again.",
rest_val)
time.sleep(rest_val)
def __init__(self,
app,
heartbeat=None,
on_task_message=None,
transport_driver_type=None):
hub = Hub()
(
self.obj,
self.connection,
self.consumer,
self.blueprint,
self.hub,
self.qos,
self.heartbeat,
self.clock,
) = self.args = [
Mock(name='obj'),
Mock(name='connection'),
Mock(name='consumer'),
Mock(name='blueprint'), hub,
Mock(name='qos'), heartbeat,
Mock(name='clock')
]
self.connection.supports_heartbeats = True
self.connection.get_heartbeat_interval.side_effect = (
lambda: self.heartbeat)
self.consumer.callbacks = []
self.obj.strategies = {}
self.connection.connection_errors = (socket.error, )
if transport_driver_type:
self.connection.transport.driver_type = transport_driver_type
self.hub.readers = {}
self.hub.timer = Mock(name='hub.timer')
self.hub.timer._queue = [Mock()]
self.hub.fire_timers = Mock(name='hub.fire_timers')
self.hub.fire_timers.return_value = 1.7
self.hub.poller = Mock(name='hub.poller')
self.hub.close = Mock(name='hub.close()') # asynloop calls hub.close
self.Hub = self.hub
self.blueprint.state = RUN
# need this for create_task_handler
self._consumer = _consumer = Consumer(Mock(),
timer=Mock(),
controller=Mock(),
app=app)
_consumer.on_task_message = on_task_message or []
self.obj.create_task_handler = _consumer.create_task_handler
self.on_unknown_message = self.obj.on_unknown_message = Mock(
name='on_unknown_message', )
_consumer.on_unknown_message = self.on_unknown_message
self.on_unknown_task = self.obj.on_unknown_task = Mock(
name='on_unknown_task', )
_consumer.on_unknown_task = self.on_unknown_task
self.on_invalid_task = self.obj.on_invalid_task = Mock(
name='on_invalid_task', )
_consumer.on_invalid_task = self.on_invalid_task
self.on_decode_error = self.obj.on_decode_error = Mock(
name='on_decode_error', )
_consumer.on_decode_error = self.on_decode_error
_consumer.strategies = self.obj.strategies
def test_get_set_event_loop(self):
set_event_loop(None)
assert _hub._current_loop is None
assert get_event_loop() is None
hub = Hub()
set_event_loop(hub)
assert _hub._current_loop is hub
assert get_event_loop() is hub
def test_start_stop(self, poll):
self.hub = Hub()
poll.assert_called_with()
poller = self.hub.poller
self.hub.stop()
self.hub.close()
poller.close.assert_called_with()
def test_poller_regeneration_on_access(self):
self.hub = Hub()
assert self.hub.poller
self.hub.stop()
self.hub._ready = set()
self.hub.close()
assert self.hub._poller is None
assert self.hub.poller, 'It should be regenerated automatically!'
def test_fire_timers_raises(self):
hub = Hub()
eback = Mock()
eback.side_effect = KeyError('foo')
hub.timer = Mock()
hub.scheduler = iter([(0, eback)])
with self.assertRaises(KeyError):
hub.fire_timers(propagate=(KeyError, ))
eback.side_effect = ValueError('foo')
hub.scheduler = iter([(0, eback)])
with patch('kombu.asynchronous.hub.logger') as logger:
with self.assertRaises(StopIteration):
hub.fire_timers()
self.assertTrue(logger.error.called)
def hub(request):
from kombu.asynchronous import Hub, get_event_loop, set_event_loop
_prev_hub = get_event_loop()
hub = Hub()
set_event_loop(hub)
yield hub
if _prev_hub is not None:
set_event_loop(_prev_hub)
def main():
with Connection('amqp://*****:*****@rabbitmq//') as conn:
# Can't rely on the fact that the connection is available immediately due to
# docker-compose startup delays
hub = Hub()
while True:
try:
conn.register_with_event_loop(hub)
except Exception as e:
# This should be an exponential backoff, but left as is for simplicity
time.sleep(1.)
print('Unable to connect', e)
else:
print('Success!')
break
with Consumer(conn, [write_queue], on_message=on_write):
with Consumer(conn, [read_queue],
on_message=partial(on_read, conn)):
hub.run_forever()
def test_add_raises_ValueError(self):
hub = Hub()
hub.poller = Mock(name='hub.poller')
hub.poller.register.side_effect = ValueError()
hub._discard = Mock(name='hub.discard')
with self.assertRaises(ValueError):
hub.add(2, Mock(), READ)
hub._discard.assert_called_with(2)
def test_start_stop(self, poll):
hub = Hub()
poll.assert_called_with()
poller = hub.poller
hub.stop()
hub.close()
poller.close.assert_called_with()
def test_fire_timers(self):
hub = Hub()
hub.timer = Mock()
hub.timer._queue = []
self.assertEqual(hub.fire_timers(min_delay=42.324, max_delay=32.321),
32.321)
hub.timer._queue = [1]
hub.scheduler = iter([(3.743, None)])
self.assertEqual(hub.fire_timers(), 3.743)
e1, e2, e3 = Mock(), Mock(), Mock()
entries = [e1, e2, e3]
def reset():
return [m.reset() for m in [e1, e2, e3]]
def se():
while 1:
while entries:
yield None, entries.pop()
yield 3.982, None
hub.scheduler = se()
self.assertEqual(hub.fire_timers(max_timers=10), 3.982)
for E in [e3, e2, e1]:
E.assert_called_with()
reset()
entries[:] = [Mock() for _ in range(11)]
keep = list(entries)
self.assertEqual(hub.fire_timers(max_timers=10, min_delay=1.13), 1.13)
for E in reversed(keep[1:]):
E.assert_called_with()
reset()
self.assertEqual(hub.fire_timers(max_timers=10), 3.982)
keep[0].assert_called_with()
#!/usr/bin/env python
from __future__ import absolute_import, unicode_literals
from kombu import Connection, Exchange, Queue, Producer, Consumer
from kombu.asynchronous import Hub
hub = Hub()
exchange = Exchange('asynt')
queue = Queue('asynt', exchange, 'asynt')
def send_message(conn):
producer = Producer(conn)
producer.publish('hello world', exchange=exchange, routing_key='asynt')
print('message sent')
def on_message(message):
print('received: {0!r}'.format(message.body))
message.ack()
hub.stop() # <-- exit after one message
if __name__ == '__main__':
conn = Connection('amqp://')
conn.register_with_event_loop(hub)
with Consumer(conn, [queue], on_message=on_message):
send_message(conn)
hub.run_forever()
#!/usr/bin/env python
from __future__ import annotations
from kombu import Connection, Consumer, Exchange, Producer, Queue
from kombu.asynchronous import Hub
hub = Hub()
exchange = Exchange('asynt')
queue = Queue('asynt', exchange, 'asynt')
def send_message(conn):
producer = Producer(conn)
producer.publish('hello world', exchange=exchange, routing_key='asynt')
print('message sent')
def on_message(message):
print(f'received: {message.body!r}')
message.ack()
hub.stop() # <-- exit after one message
if __name__ == '__main__':
conn = Connection('amqp://')
conn.register_with_event_loop(hub)
with Consumer(conn, [queue], on_message=on_message):
send_message(conn)
hub.run_forever()
from flask import Flask, request, Response
import time
import sys
import os
import shlex
from kombu import Connection, Exchange, Queue, Consumer
from functools import partial
from kombu import uuid
from kombu.asynchronous import Hub
from kombu import Connection, Exchange, Queue, Producer, Consumer
hub = Hub()
exchange = Exchange('supreme-octo-eureka', 'direct', durable=False)
read_queue = Queue('db.read', exchange=exchange, routing_key='db.read')
write_queue = Queue('db.write', exchange=exchange, routing_key='db.write')
class RPCClient(object):
"""A basic "RPC" client that keeps state,
and polls for the response in a blocking way.
"""
def __init__(self, connection):
self.connection = connection
self.callback_queue = Queue(uuid(), exclusive=True, auto_delete=True)
def on_response(self, message):
if message.properties['correlation_id'] == self.correlation_id:
self.response = message.payload
def call(self, key):
class test_Hub:
def setup(self):
self.hub = Hub()
def teardown(self):
self.hub.close()
def test_reset(self):
self.hub.close = Mock(name='close')
self.hub._create_poller = Mock(name='_create_poller')
self.hub.reset()
self.hub.close.assert_called_with()
self.hub._create_poller.assert_called_with()
def test__close_poller__no_poller(self):
self.hub.poller = None
self.hub._close_poller()
def test__close_poller(self):
poller = self.hub.poller = Mock(name='poller')
self.hub._close_poller()
poller.close.assert_called_with()
assert self.hub._poller is None
def test_stop(self):
self.hub.call_soon = Mock(name='call_soon')
self.hub.stop()
self.hub.call_soon.assert_called_with(_raise_stop_error)
@patch('kombu.asynchronous.hub.promise')
def test_call_soon(self, promise):
callback = Mock(name='callback')
ret = self.hub.call_soon(callback, 1, 2, 3)
promise.assert_called_with(callback, (1, 2, 3))
assert promise() in self.hub._ready
assert ret is promise()
def test_call_soon__promise_argument(self):
callback = promise(Mock(name='callback'), (1, 2, 3))
ret = self.hub.call_soon(callback)
assert ret is callback
assert ret in self.hub._ready
def test_call_later(self):
callback = Mock(name='callback')
self.hub.timer = Mock(name='hub.timer')
self.hub.call_later(10.0, callback, 1, 2)
self.hub.timer.call_after.assert_called_with(10.0, callback, (1, 2))
def test_call_at(self):
callback = Mock(name='callback')
self.hub.timer = Mock(name='hub.timer')
self.hub.call_at(21231122, callback, 1, 2)
self.hub.timer.call_at.assert_called_with(21231122, callback, (1, 2))
def test_repr(self):
assert repr(self.hub)
def test_repr_flag(self):
assert repr_flag(READ) == 'R'
assert repr_flag(WRITE) == 'W'
assert repr_flag(ERR) == '!'
assert repr_flag(READ | WRITE) == 'RW'
assert repr_flag(READ | ERR) == 'R!'
assert repr_flag(WRITE | ERR) == 'W!'
assert repr_flag(READ | WRITE | ERR) == 'RW!'
def test_repr_callback_rcb(self):
def f():
pass
assert _rcb(f) == f.__name__
assert _rcb('foo') == 'foo'
@patch('kombu.asynchronous.hub.poll')
def test_start_stop(self, poll):
self.hub = Hub()
poll.assert_called_with()
poller = self.hub.poller
self.hub.stop()
self.hub._ready = set()
self.hub.close()
poller.close.assert_called_with()
def test_poller_regeneration_on_access(self):
self.hub = Hub()
assert self.hub.poller
self.hub.stop()
self.hub._ready = set()
self.hub.close()
assert self.hub._poller is None
assert self.hub.poller, 'It should be regenerated automatically!'
def test_fire_timers(self):
self.hub.timer = Mock()
self.hub.timer._queue = []
assert self.hub.fire_timers(
min_delay=42.324, max_delay=32.321) == 32.321
self.hub.timer._queue = [1]
self.hub.scheduler = iter([(3.743, None)])
assert self.hub.fire_timers() == 3.743
e1, e2, e3 = Mock(), Mock(), Mock()
entries = [e1, e2, e3]
def reset():
return [m.reset() for m in [e1, e2, e3]]
def se():
while 1:
while entries:
yield None, entries.pop()
yield 3.982, None
self.hub.scheduler = se()
assert self.hub.fire_timers(max_timers=10) == 3.982
for E in [e3, e2, e1]:
E.assert_called_with()
reset()
entries[:] = [Mock() for _ in range(11)]
keep = list(entries)
assert self.hub.fire_timers(
max_timers=10, min_delay=1.13) == 1.13
for E in reversed(keep[1:]):
E.assert_called_with()
reset()
assert self.hub.fire_timers(max_timers=10) == 3.982
keep[0].assert_called_with()
def test_fire_timers_raises(self):
eback = Mock()
eback.side_effect = KeyError('foo')
self.hub.timer = Mock()
self.hub.scheduler = iter([(0, eback)])
with pytest.raises(KeyError):
self.hub.fire_timers(propagate=(KeyError,))
eback.side_effect = ValueError('foo')
self.hub.scheduler = iter([(0, eback)])
with patch('kombu.asynchronous.hub.logger') as logger:
with pytest.raises(StopIteration):
self.hub.fire_timers()
logger.error.assert_called()
eback.side_effect = MemoryError('foo')
self.hub.scheduler = iter([(0, eback)])
with pytest.raises(MemoryError):
self.hub.fire_timers()
eback.side_effect = OSError()
eback.side_effect.errno = errno.ENOMEM
self.hub.scheduler = iter([(0, eback)])
with pytest.raises(OSError):
self.hub.fire_timers()
eback.side_effect = OSError()
eback.side_effect.errno = errno.ENOENT
self.hub.scheduler = iter([(0, eback)])
with patch('kombu.asynchronous.hub.logger') as logger:
with pytest.raises(StopIteration):
self.hub.fire_timers()
logger.error.assert_called()
def test_add_raises_ValueError(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.poller.register.side_effect = ValueError()
self.hub._discard = Mock(name='hub.discard')
with pytest.raises(ValueError):
self.hub.add(2, Mock(), READ)
self.hub._discard.assert_called_with(2)
def test_remove_reader(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.add(2, Mock(), READ)
self.hub.add(2, Mock(), WRITE)
self.hub.remove_reader(2)
assert 2 not in self.hub.readers
assert 2 in self.hub.writers
def test_remove_reader__not_writeable(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.add(2, Mock(), READ)
self.hub.remove_reader(2)
assert 2 not in self.hub.readers
def test_remove_writer(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.add(2, Mock(), READ)
self.hub.add(2, Mock(), WRITE)
self.hub.remove_writer(2)
assert 2 in self.hub.readers
assert 2 not in self.hub.writers
def test_remove_writer__not_readable(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.add(2, Mock(), WRITE)
self.hub.remove_writer(2)
assert 2 not in self.hub.writers
def test_add__consolidate(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.add(2, Mock(), WRITE, consolidate=True)
assert 2 in self.hub.consolidate
assert self.hub.writers[2] is None
@patch('kombu.asynchronous.hub.logger')
def test_on_callback_error(self, logger):
self.hub.on_callback_error(Mock(name='callback'), KeyError())
logger.error.assert_called()
def test_loop_property(self):
self.hub._loop = None
self.hub.create_loop = Mock(name='hub.create_loop')
assert self.hub.loop is self.hub.create_loop()
assert self.hub._loop is self.hub.create_loop()
def test_run_forever(self):
self.hub.run_once = Mock(name='hub.run_once')
self.hub.run_once.side_effect = Stop()
self.hub.run_forever()
def test_run_once(self):
self.hub._loop = iter([1])
self.hub.run_once()
self.hub.run_once()
assert self.hub._loop is None
def test_repr_active(self):
self.hub.readers = {1: Mock(), 2: Mock()}
self.hub.writers = {3: Mock(), 4: Mock()}
for value in list(
self.hub.readers.values()) + list(self.hub.writers.values()):
value.__name__ = 'mock'
assert self.hub.repr_active()
def test_repr_events(self):
self.hub.readers = {6: Mock(), 7: Mock(), 8: Mock()}
self.hub.writers = {9: Mock()}
for value in list(
self.hub.readers.values()) + list(self.hub.writers.values()):
value.__name__ = 'mock'
assert self.hub.repr_events([
(6, READ),
(7, ERR),
(8, READ | ERR),
(9, WRITE),
(10, 13213),
])
def test_callback_for(self):
reader, writer = Mock(), Mock()
self.hub.readers = {6: reader}
self.hub.writers = {7: writer}
assert callback_for(self.hub, 6, READ) == reader
assert callback_for(self.hub, 7, WRITE) == writer
with pytest.raises(KeyError):
callback_for(self.hub, 6, WRITE)
assert callback_for(self.hub, 6, WRITE, 'foo') == 'foo'
def test_add_remove_readers(self):
P = self.hub.poller = Mock()
read_A = Mock()
read_B = Mock()
self.hub.add_reader(10, read_A, 10)
self.hub.add_reader(File(11), read_B, 11)
P.register.assert_has_calls([
call(10, self.hub.READ | self.hub.ERR),
call(11, self.hub.READ | self.hub.ERR),
], any_order=True)
assert self.hub.readers[10] == (read_A, (10,))
assert self.hub.readers[11] == (read_B, (11,))
self.hub.remove(10)
assert 10 not in self.hub.readers
self.hub.remove(File(11))
assert 11 not in self.hub.readers
P.unregister.assert_has_calls([
call(10), call(11),
])
def test_can_remove_unknown_fds(self):
self.hub.poller = Mock()
self.hub.remove(30)
self.hub.remove(File(301))
def test_remove__unregister_raises(self):
self.hub.poller = Mock()
self.hub.poller.unregister.side_effect = OSError()
self.hub.remove(313)
def test_add_writers(self):
P = self.hub.poller = Mock()
write_A = Mock()
write_B = Mock()
self.hub.add_writer(20, write_A)
self.hub.add_writer(File(21), write_B)
P.register.assert_has_calls([
call(20, self.hub.WRITE),
call(21, self.hub.WRITE),
], any_order=True)
assert self.hub.writers[20], (write_A == ())
assert self.hub.writers[21], (write_B == ())
self.hub.remove(20)
assert 20 not in self.hub.writers
self.hub.remove(File(21))
assert 21 not in self.hub.writers
P.unregister.assert_has_calls([
call(20), call(21),
])
def test_enter__exit(self):
P = self.hub.poller = Mock()
on_close = Mock()
self.hub.on_close.add(on_close)
try:
read_A = Mock()
read_B = Mock()
self.hub.add_reader(10, read_A)
self.hub.add_reader(File(11), read_B)
write_A = Mock()
write_B = Mock()
self.hub.add_writer(20, write_A)
self.hub.add_writer(File(21), write_B)
assert self.hub.readers
assert self.hub.writers
finally:
assert self.hub.poller
self.hub.close()
assert not self.hub.readers
assert not self.hub.writers
P.unregister.assert_has_calls([
call(10), call(11), call(20), call(21),
], any_order=True)
on_close.assert_called_with(self.hub)
def test_scheduler_property(self):
hub = Hub(timer=[1, 2, 3])
assert list(hub.scheduler), [1, 2 == 3]
def test_loop__tick_callbacks(self):
ticks = [Mock(name='cb1'), Mock(name='cb2')]
self.hub.on_tick = list(ticks)
next(self.hub.loop)
ticks[0].assert_called_once_with()
ticks[1].assert_called_once_with()
def test_loop__todo(self):
deferred = Mock(name='cb_deferred')
def defer():
self.hub.call_soon(deferred)
callbacks = [Mock(name='cb1', wraps=defer), Mock(name='cb2')]
for cb in callbacks:
self.hub.call_soon(cb)
self.hub._ready.add(None)
next(self.hub.loop)
callbacks[0].assert_called_once_with()
callbacks[1].assert_called_once_with()
deferred.assert_not_called()
def setup(self):
self.hub = Hub()
def test_enter__exit(self):
hub = Hub()
P = hub.poller = Mock()
on_close = Mock()
hub.on_close.add(on_close)
try:
read_A = Mock()
read_B = Mock()
hub.add_reader(10, read_A)
hub.add_reader(File(11), read_B)
write_A = Mock()
write_B = Mock()
hub.add_writer(20, write_A)
hub.add_writer(File(21), write_B)
self.assertTrue(hub.readers)
self.assertTrue(hub.writers)
finally:
assert hub.poller
hub.close()
self.assertFalse(hub.readers)
self.assertFalse(hub.writers)
P.unregister.assert_has_calls([
call(10),
call(11),
call(20),
call(21),
],
any_order=True)
on_close.assert_called_with(hub)
def test_scheduler_property(self):
hub = Hub(timer=[1, 2, 3])
self.assertEqual(list(hub.scheduler), [1, 2, 3])
class test_Hub:
def setup(self):
self.hub = Hub()
def teardown(self):
self.hub.close()
def test_reset(self):
self.hub.close = Mock(name='close')
self.hub._create_poller = Mock(name='_create_poller')
self.hub.reset()
self.hub.close.assert_called_with()
self.hub._create_poller.assert_called_with()
def test__close_poller__no_poller(self):
self.hub.poller = None
self.hub._close_poller()
def test__close_poller(self):
poller = self.hub.poller = Mock(name='poller')
self.hub._close_poller()
poller.close.assert_called_with()
assert self.hub._poller is None
def test_stop(self):
self.hub.call_soon = Mock(name='call_soon')
self.hub.stop()
self.hub.call_soon.assert_called_with(_raise_stop_error)
@patch('kombu.asynchronous.hub.promise')
def test_call_soon(self, promise):
callback = Mock(name='callback')
ret = self.hub.call_soon(callback, 1, 2, 3)
promise.assert_called_with(callback, (1, 2, 3))
assert promise() in self.hub._ready
assert ret is promise()
def test_call_soon__promise_argument(self):
callback = promise(Mock(name='callback'), (1, 2, 3))
ret = self.hub.call_soon(callback)
assert ret is callback
assert ret in self.hub._ready
def test_call_later(self):
callback = Mock(name='callback')
self.hub.timer = Mock(name='hub.timer')
self.hub.call_later(10.0, callback, 1, 2)
self.hub.timer.call_after.assert_called_with(10.0, callback, (1, 2))
def test_call_at(self):
callback = Mock(name='callback')
self.hub.timer = Mock(name='hub.timer')
self.hub.call_at(21231122, callback, 1, 2)
self.hub.timer.call_at.assert_called_with(21231122, callback, (1, 2))
def test_repr(self):
assert repr(self.hub)
def test_repr_flag(self):
assert repr_flag(READ) == 'R'
assert repr_flag(WRITE) == 'W'
assert repr_flag(ERR) == '!'
assert repr_flag(READ | WRITE) == 'RW'
assert repr_flag(READ | ERR) == 'R!'
assert repr_flag(WRITE | ERR) == 'W!'
assert repr_flag(READ | WRITE | ERR) == 'RW!'
def test_repr_callback_rcb(self):
def f():
pass
assert _rcb(f) == f.__name__
assert _rcb('foo') == 'foo'
@patch('kombu.asynchronous.hub.poll')
def test_start_stop(self, poll):
self.hub = Hub()
poll.assert_called_with()
poller = self.hub.poller
self.hub.stop()
mock_callback = Mock()
self.hub._ready = set([mock_callback])
self.hub.close()
poller.close.assert_called_with()
mock_callback.assert_called_once_with()
assert self.hub._ready == set()
def test_poller_regeneration_on_access(self):
self.hub = Hub()
assert self.hub.poller
self.hub.stop()
self.hub._ready = set()
self.hub.close()
assert self.hub._poller is None
assert self.hub.poller, 'It should be regenerated automatically!'
def test_fire_timers(self):
self.hub.timer = Mock()
self.hub.timer._queue = []
assert self.hub.fire_timers(min_delay=42.324,
max_delay=32.321) == 32.321
self.hub.timer._queue = [1]
self.hub.scheduler = iter([(3.743, None)])
assert self.hub.fire_timers() == 3.743
e1, e2, e3 = Mock(), Mock(), Mock()
entries = [e1, e2, e3]
def reset():
return [m.reset() for m in [e1, e2, e3]]
def se():
while 1:
while entries:
yield None, entries.pop()
yield 3.982, None
self.hub.scheduler = se()
assert self.hub.fire_timers(max_timers=10) == 3.982
for E in [e3, e2, e1]:
E.assert_called_with()
reset()
entries[:] = [Mock() for _ in range(11)]
keep = list(entries)
assert self.hub.fire_timers(max_timers=10, min_delay=1.13) == 1.13
for E in reversed(keep[1:]):
E.assert_called_with()
reset()
assert self.hub.fire_timers(max_timers=10) == 3.982
keep[0].assert_called_with()
def test_fire_timers_raises(self):
eback = Mock()
eback.side_effect = KeyError('foo')
self.hub.timer = Mock()
self.hub.scheduler = iter([(0, eback)])
with pytest.raises(KeyError):
self.hub.fire_timers(propagate=(KeyError, ))
eback.side_effect = ValueError('foo')
self.hub.scheduler = iter([(0, eback)])
with patch('kombu.asynchronous.hub.logger') as logger:
with pytest.raises(StopIteration):
self.hub.fire_timers()
logger.error.assert_called()
eback.side_effect = MemoryError('foo')
self.hub.scheduler = iter([(0, eback)])
with pytest.raises(MemoryError):
self.hub.fire_timers()
eback.side_effect = OSError()
eback.side_effect.errno = errno.ENOMEM
self.hub.scheduler = iter([(0, eback)])
with pytest.raises(OSError):
self.hub.fire_timers()
eback.side_effect = OSError()
eback.side_effect.errno = errno.ENOENT
self.hub.scheduler = iter([(0, eback)])
with patch('kombu.asynchronous.hub.logger') as logger:
with pytest.raises(StopIteration):
self.hub.fire_timers()
logger.error.assert_called()
def test_add_raises_ValueError(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.poller.register.side_effect = ValueError()
self.hub._discard = Mock(name='hub.discard')
with pytest.raises(ValueError):
self.hub.add(2, Mock(), READ)
self.hub._discard.assert_called_with(2)
def test_remove_reader(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.add(2, Mock(), READ)
self.hub.add(2, Mock(), WRITE)
self.hub.remove_reader(2)
assert 2 not in self.hub.readers
assert 2 in self.hub.writers
def test_remove_reader__not_writeable(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.add(2, Mock(), READ)
self.hub.remove_reader(2)
assert 2 not in self.hub.readers
def test_remove_writer(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.add(2, Mock(), READ)
self.hub.add(2, Mock(), WRITE)
self.hub.remove_writer(2)
assert 2 in self.hub.readers
assert 2 not in self.hub.writers
def test_remove_writer__not_readable(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.add(2, Mock(), WRITE)
self.hub.remove_writer(2)
assert 2 not in self.hub.writers
def test_add__consolidate(self):
self.hub.poller = Mock(name='hub.poller')
self.hub.add(2, Mock(), WRITE, consolidate=True)
assert 2 in self.hub.consolidate
assert self.hub.writers[2] is None
@patch('kombu.asynchronous.hub.logger')
def test_on_callback_error(self, logger):
self.hub.on_callback_error(Mock(name='callback'), KeyError())
logger.error.assert_called()
def test_loop_property(self):
self.hub._loop = None
self.hub.create_loop = Mock(name='hub.create_loop')
assert self.hub.loop is self.hub.create_loop()
assert self.hub._loop is self.hub.create_loop()
def test_run_forever(self):
self.hub.run_once = Mock(name='hub.run_once')
self.hub.run_once.side_effect = Stop()
self.hub.run_forever()
def test_run_once(self):
self.hub._loop = iter([1])
self.hub.run_once()
self.hub.run_once()
assert self.hub._loop is None
def test_repr_active(self):
self.hub.readers = {1: Mock(), 2: Mock()}
self.hub.writers = {3: Mock(), 4: Mock()}
for value in list(self.hub.readers.values()) + list(
self.hub.writers.values()):
value.__name__ = 'mock'
assert self.hub.repr_active()
def test_repr_events(self):
self.hub.readers = {6: Mock(), 7: Mock(), 8: Mock()}
self.hub.writers = {9: Mock()}
for value in list(self.hub.readers.values()) + list(
self.hub.writers.values()):
value.__name__ = 'mock'
assert self.hub.repr_events([
(6, READ),
(7, ERR),
(8, READ | ERR),
(9, WRITE),
(10, 13213),
])
def test_callback_for(self):
reader, writer = Mock(), Mock()
self.hub.readers = {6: reader}
self.hub.writers = {7: writer}
assert callback_for(self.hub, 6, READ) == reader
assert callback_for(self.hub, 7, WRITE) == writer
with pytest.raises(KeyError):
callback_for(self.hub, 6, WRITE)
assert callback_for(self.hub, 6, WRITE, 'foo') == 'foo'
def test_add_remove_readers(self):
P = self.hub.poller = Mock()
read_A = Mock()
read_B = Mock()
self.hub.add_reader(10, read_A, 10)
self.hub.add_reader(File(11), read_B, 11)
P.register.assert_has_calls([
call(10, self.hub.READ | self.hub.ERR),
call(11, self.hub.READ | self.hub.ERR),
],
any_order=True)
assert self.hub.readers[10] == (read_A, (10, ))
assert self.hub.readers[11] == (read_B, (11, ))
self.hub.remove(10)
assert 10 not in self.hub.readers
self.hub.remove(File(11))
assert 11 not in self.hub.readers
P.unregister.assert_has_calls([
call(10),
call(11),
])
def test_can_remove_unknown_fds(self):
self.hub.poller = Mock()
self.hub.remove(30)
self.hub.remove(File(301))
def test_remove__unregister_raises(self):
self.hub.poller = Mock()
self.hub.poller.unregister.side_effect = OSError()
self.hub.remove(313)
def test_add_writers(self):
P = self.hub.poller = Mock()
write_A = Mock()
write_B = Mock()
self.hub.add_writer(20, write_A)
self.hub.add_writer(File(21), write_B)
P.register.assert_has_calls([
call(20, self.hub.WRITE),
call(21, self.hub.WRITE),
],
any_order=True)
assert self.hub.writers[20], (write_A == ())
assert self.hub.writers[21], (write_B == ())
self.hub.remove(20)
assert 20 not in self.hub.writers
self.hub.remove(File(21))
assert 21 not in self.hub.writers
P.unregister.assert_has_calls([
call(20),
call(21),
])
def test_enter__exit(self):
P = self.hub.poller = Mock()
on_close = Mock()
self.hub.on_close.add(on_close)
try:
read_A = Mock()
read_B = Mock()
self.hub.add_reader(10, read_A)
self.hub.add_reader(File(11), read_B)
write_A = Mock()
write_B = Mock()
self.hub.add_writer(20, write_A)
self.hub.add_writer(File(21), write_B)
assert self.hub.readers
assert self.hub.writers
finally:
assert self.hub.poller
self.hub.close()
assert not self.hub.readers
assert not self.hub.writers
P.unregister.assert_has_calls([
call(10),
call(11),
call(20),
call(21),
],
any_order=True)
on_close.assert_called_with(self.hub)
def test_scheduler_property(self):
hub = Hub(timer=[1, 2, 3])
assert list(hub.scheduler), [1, 2 == 3]
def test_loop__tick_callbacks(self):
ticks = [Mock(name='cb1'), Mock(name='cb2')]
self.hub.on_tick = list(ticks)
next(self.hub.loop)
ticks[0].assert_called_once_with()
ticks[1].assert_called_once_with()
def test_loop__todo(self):
deferred = Mock(name='cb_deferred')
def defer():
self.hub.call_soon(deferred)
callbacks = [Mock(name='cb1', wraps=defer), Mock(name='cb2')]
for cb in callbacks:
self.hub.call_soon(cb)
self.hub._ready.add(None)
next(self.hub.loop)
callbacks[0].assert_called_once_with()
callbacks[1].assert_called_once_with()
deferred.assert_not_called()
def test_remove__unregister_raises(self):
hub = Hub()
hub.poller = Mock()
hub.poller.unregister.side_effect = OSError()
hub.remove(313)
def test_can_remove_unknown_fds(self):
hub = Hub()
hub.poller = Mock()
hub.remove(30)
hub.remove(File(301))
def setup(self):
self.hub = Hub()
def test_scheduler_property(self):
hub = Hub(timer=[1, 2, 3])
assert list(hub.scheduler), [1, 2 == 3]
