def handle(self, queue, *args, **options):
fmt = '%(asctime)s %(name)-12s: %(levelname)-8s %(message)s'
log_level = 40 - (options['verbosity'] * 10)
logging.basicConfig(level=log_level, format=fmt)
# TODO: Ensure that failed processing does not requeue task into
# work queue
set_event_loop(Hub())
kwargs = {
'transport_options': conf.broker_transport_options,
}
with app.connection(**kwargs) as conn:
logger.info("Launching worker")
worker = Worker(conn, queues=[Queue(queue)])
worker.connect_max_retries = 1
while True:
try:
breakers.celery.call(worker.run)
except KeyboardInterrupt:
logger.info("KeyboardInterrupt, exiting. Bye!")
break
except breakers.celery.expected_errors:
rest_val = 5
logger.warning(
"Open circuit detected. "
"Sleeping for %s seconds and then will try again.",
rest_val)
time.sleep(rest_val)
def test_repr_active(self):
hub = Hub()
hub.readers = {1: Mock(), 2: Mock()}
hub.writers = {3: Mock(), 4: Mock()}
for value in list(hub.readers.values()) + list(hub.writers.values()):
value.__name__ = 'mock'
self.assertTrue(hub.repr_active())
def test_add_remove_readers(self):
hub = Hub()
P = hub.poller = Mock()
read_A = Mock()
read_B = Mock()
hub.add_reader(10, read_A, 10)
hub.add_reader(File(11), read_B, 11)
P.register.assert_has_calls([
call(10, hub.READ | hub.ERR),
call(11, hub.READ | hub.ERR),
],
any_order=True)
self.assertEqual(hub.readers[10], (read_A, (10, )))
self.assertEqual(hub.readers[11], (read_B, (11, )))
hub.remove(10)
self.assertNotIn(10, hub.readers)
hub.remove(File(11))
self.assertNotIn(11, hub.readers)
P.unregister.assert_has_calls([
call(10),
call(11),
])
def test_add_raises_ValueError(self):
hub = Hub()
hub.poller = Mock(name='hub.poller')
hub.poller.register.side_effect = ValueError()
hub._discard = Mock(name='hub.discard')
hub.add(2, Mock(), READ)
hub._discard.assert_called_with(2)
def test_enter__exit(self):
hub = Hub()
P = hub.poller = Mock()
on_close = Mock()
hub.on_close.add(on_close)
try:
read_A = Mock()
read_B = Mock()
hub.add_reader(10, read_A)
hub.add_reader(File(11), read_B)
write_A = Mock()
write_B = Mock()
hub.add_writer(20, write_A)
hub.add_writer(File(21), write_B)
self.assertTrue(hub.readers)
self.assertTrue(hub.writers)
finally:
assert hub.poller
hub.close()
self.assertFalse(hub.readers)
self.assertFalse(hub.writers)
P.unregister.assert_has_calls([
call(10),
call(11),
call(20),
call(21),
],
any_order=True)
on_close.assert_called_with(hub)
def test_add_writers(self):
hub = Hub()
P = hub.poller = Mock()
write_A = Mock()
write_B = Mock()
hub.add_writer(20, write_A)
hub.add_writer(File(21), write_B)
P.register.assert_has_calls([
call(20, hub.WRITE),
call(21, hub.WRITE),
],
any_order=True)
self.assertEqual(hub.writers[20], (write_A, ()))
self.assertEqual(hub.writers[21], (write_B, ()))
hub.remove(20)
self.assertNotIn(20, hub.writers)
hub.remove(File(21))
self.assertNotIn(21, hub.writers)
P.unregister.assert_has_calls([
call(20),
call(21),
])
def __init__(self,
app,
heartbeat=None,
on_task_message=None,
transport_driver_type=None):
hub = Hub()
(
self.obj,
self.connection,
self.consumer,
self.blueprint,
self.hub,
self.qos,
self.heartbeat,
self.clock,
) = self.args = [
Mock(name='obj'),
Mock(name='connection'),
Mock(name='consumer'),
Mock(name='blueprint'), hub,
Mock(name='qos'), heartbeat,
Mock(name='clock')
]
self.connection.supports_heartbeats = True
self.connection.get_heartbeat_interval.side_effect = (
lambda: self.heartbeat)
self.consumer.callbacks = []
self.obj.strategies = {}
self.connection.connection_errors = (socket.error, )
if transport_driver_type:
self.connection.transport.driver_type = transport_driver_type
self.hub.readers = {}
self.hub.writers = {}
self.hub.consolidate = set()
self.hub.timer = Mock(name='hub.timer')
self.hub.timer._queue = [Mock()]
self.hub.fire_timers = Mock(name='hub.fire_timers')
self.hub.fire_timers.return_value = 1.7
self.hub.poller = Mock(name='hub.poller')
self.hub.close = Mock(name='hub.close()') # asynloop calls hub.close
self.Hub = self.hub
self.blueprint.state = RUN
# need this for create_task_handler
self._consumer = _consumer = Consumer(Mock(),
timer=Mock(),
controller=Mock(),
app=app)
_consumer.on_task_message = on_task_message or []
self.obj.create_task_handler = _consumer.create_task_handler
self.on_unknown_message = self.obj.on_unknown_message = Mock(
name='on_unknown_message', )
_consumer.on_unknown_message = self.on_unknown_message
self.on_unknown_task = self.obj.on_unknown_task = Mock(
name='on_unknown_task', )
_consumer.on_unknown_task = self.on_unknown_task
self.on_invalid_task = self.obj.on_invalid_task = Mock(
name='on_invalid_task', )
_consumer.on_invalid_task = self.on_invalid_task
_consumer.strategies = self.obj.strategies
def test_get_set_event_loop(self):
set_event_loop(None)
assert _hub._current_loop is None
assert get_event_loop() is None
hub = Hub()
set_event_loop(hub)
assert _hub._current_loop is hub
assert get_event_loop() is hub
def test_get_set_event_loop(self):
set_event_loop(None)
self.assertIsNone(_hub._current_loop)
self.assertIsNone(get_event_loop())
hub = Hub()
set_event_loop(hub)
self.assertIs(_hub._current_loop, hub)
self.assertIs(get_event_loop(), hub)
def test_start_stop(self, poll):
self.hub = Hub()
poll.assert_called_with()
poller = self.hub.poller
self.hub.stop()
self.hub.close()
poller.close.assert_called_with()
def test_start_stop(self, poll):
hub = Hub()
poll.assert_called_with()
poller = hub.poller
hub.stop()
hub.close()
poller.close.assert_called_with()
def hub(request):
from kombu. async import Hub, get_event_loop, set_event_loop
_prev_hub = get_event_loop()
hub = Hub()
set_event_loop(hub)
yield hub
if _prev_hub is not None:
set_event_loop(_prev_hub)
def test_callback_for(self):
hub = Hub()
reader, writer = Mock(), Mock()
hub.readers = {6: reader}
hub.writers = {7: writer}
self.assertEqual(callback_for(hub, 6, READ), reader)
self.assertEqual(callback_for(hub, 7, WRITE), writer)
with self.assertRaises(KeyError):
callback_for(hub, 6, WRITE)
self.assertEqual(callback_for(hub, 6, WRITE, 'foo'), 'foo')
def hub(request):
from kombu. async import Hub, get_event_loop, set_event_loop
_prev_hub = get_event_loop()
hub = Hub()
set_event_loop(hub)
def fin():
if _prev_hub is not None:
set_event_loop(_prev_hub)
request.addfinalizer(fin)
return hub
def test_repr_events(self):
hub = Hub()
hub.readers = {6: Mock(), 7: Mock(), 8: Mock()}
hub.writers = {9: Mock()}
for value in list(hub.readers.values()) + list(hub.writers.values()):
value.__name__ = 'mock'
self.assertTrue(hub.repr_events([
(6, READ),
(7, ERR),
(8, READ | ERR),
(9, WRITE),
(10, 13213),
]))
def test_fire_timers_raises(self):
hub = Hub()
eback = Mock()
eback.side_effect = KeyError('foo')
hub.timer = Mock()
hub.scheduler = iter([(0, eback)])
with self.assertRaises(KeyError):
hub.fire_timers(propagate=(KeyError, ))
eback.side_effect = ValueError('foo')
hub.scheduler = iter([(0, eback)])
with patch('kombu.async.hub.logger') as logger:
with self.assertRaises(StopIteration):
hub.fire_timers()
self.assertTrue(logger.error.called)
def main(arguments):
hub = Hub()
exchange = Exchange('asynt')
queue = Queue('asynt', exchange, 'asynt')
def send_message(conn):
producer = Producer(conn)
producer.publish('hello world', exchange=exchange, routing_key='asynt')
print('message sent')
def on_message(message):
print('received: {0!r}'.format(message.body))
message.ack()
hub.stop() # <-- exit after one message
conn = Connection('redis://localhost:6379')
conn.register_with_event_loop(hub)
with Consumer(conn, [queue], on_message=on_message):
send_message(conn)
hub.run_forever()
def test_fire_timers(self):
hub = Hub()
hub.timer = Mock()
hub.timer._queue = []
self.assertEqual(hub.fire_timers(min_delay=42.324, max_delay=32.321),
32.321)
hub.timer._queue = [1]
hub.scheduler = iter([(3.743, None)])
self.assertEqual(hub.fire_timers(), 3.743)
e1, e2, e3 = Mock(), Mock(), Mock()
entries = [e1, e2, e3]
def reset():
return [m.reset() for m in [e1, e2, e3]]
def se():
while 1:
while entries:
yield None, entries.pop()
yield 3.982, None
hub.scheduler = se()
self.assertEqual(hub.fire_timers(max_timers=10), 3.982)
for E in [e3, e2, e1]:
E.assert_called_with()
reset()
entries[:] = [Mock() for _ in range(11)]
keep = list(entries)
self.assertEqual(hub.fire_timers(max_timers=10, min_delay=1.13), 1.13)
for E in reversed(keep[1:]):
E.assert_called_with()
reset()
self.assertEqual(hub.fire_timers(max_timers=10), 3.982)
keep[0].assert_called_with()
def test_remove__unregister_raises(self):
hub = Hub()
hub.poller = Mock()
hub.poller.unregister.side_effect = OSError()
hub.remove(313)
def test_can_remove_unknown_fds(self):
hub = Hub()
hub.poller = Mock()
hub.remove(30)
hub.remove(File(301))
def test_scheduler_property(self):
hub = Hub(timer=[1, 2, 3])
assert list(hub.scheduler), [1, 2 == 3]
def setUp(self):
from kombu. async import Hub, get_event_loop, set_event_loop
self._prev_hub = get_event_loop()
self.hub = Hub()
set_event_loop(self.hub)
super(HubCase, self).setUp()
#!/usr/bin/env python
from kombu import Connection, Exchange, Queue, Producer, Consumer
from kombu. async import Hub
hub = Hub()
exchange = Exchange('asynt')
queue = Queue('asynt', exchange, 'asynt')
def send_message(conn):
producer = Producer(conn)
producer.publish('hello world', exchange=exchange, routing_key='asynt')
print('MESSAGE SENT')
def on_message(message):
print('RECEIVED: %r' % (message.body, ))
message.ack()
hub.stop() # <-- exit after one message
if __name__ == '__main__':
conn = Connection('amqp://')
conn.register_with_event_loop(hub)
with Consumer(conn, [queue], on_message=on_message):
send_message(conn)
hub.run_forever()
def test_scheduler_property(self):
hub = Hub(timer=[1, 2, 3])
self.assertEqual(list(hub.scheduler), [1, 2, 3])
def setup(self):
self.hub = Hub()
