def __init__(self, func, *args, **kwargs):
self.my_sem = Semaphore(0) # This is held by the thread as it runs.
self.caller_sem = None
self.dead = False
started = Event()
self.id = 5
self.ALL.append(self)
def go():
self.id = eventlet.corolocal.get_ident()
started.send(True)
self.my_sem.acquire(blocking=True, timeout=None)
try:
func(*args, **kwargs)
# except Exception as e:
# print("Exception in coroutine! %s" % e)
finally:
self.dead = True
self.caller_sem.release() # Relinquish control back to caller.
for i in range(len(self.ALL)):
if self.ALL[i].id == self.id:
del self.ALL[i]
break
true_spawn(go)
started.wait()
def test_defer_event(ctx):
from datetime import datetime, timedelta
from eventlet import sleep, spawn, with_timeout
from eventlet.event import Event
from melkman.messaging import EventBus
from melkman.scheduler import defer_event
from melkman.scheduler.worker import ScheduledMessageService
CHAN = 'test_chan'
sms = ScheduledMessageService(ctx)
sched = spawn(sms.run)
got_message = Event()
def got_message_cb(*args, **kw):
got_message.send(True)
eb = EventBus(ctx)
eb.add_listener(CHAN, got_message_cb)
now = datetime.utcnow()
wait = timedelta(seconds=2)
defer_event(now + wait, CHAN, {'foo': 'bar'}, ctx)
sleep(3)
try:
with_timeout(10, got_message.wait)
assert got_message.ready()
finally:
eb.kill()
sched.kill()
sched.wait()
def test_kill_container_with_active_workers(container_factory):
waiting = Event()
wait_forever = Event()
class Service(object):
name = 'kill-with-active-workers'
@foobar
def spam(self):
waiting.send(None)
wait_forever.wait()
container = container_factory(Service, {})
dep = get_dependency(container, EntrypointProvider)
# start the first worker, which should wait for spam_continue
container.spawn_worker(dep, (), {})
waiting.wait()
with patch('nameko.containers._log') as logger:
container.kill()
calls = logger.warning.call_args_list
assert call(
'killing active thread for %s', 'kill-with-active-workers.spam'
) in calls
def _invoke_method_implementation(self, method, this, murano_class,
context, params):
body = method.body
if not body:
return None
current_thread = eventlet.greenthread.getcurrent()
if not hasattr(current_thread, '_murano_dsl_thread_marker'):
thread_marker = current_thread._murano_dsl_thread_marker = \
uuid.uuid4().hex
else:
thread_marker = current_thread._murano_dsl_thread_marker
method_id = id(body)
this_id = this.object_id
event, marker = self._locks.get((method_id, this_id), (None, None))
if event:
if marker == thread_marker:
return self._invoke_method_implementation_gt(
body, this, params, murano_class, context)
event.wait()
event = Event()
self._locks[(method_id, this_id)] = (event, thread_marker)
gt = eventlet.spawn(self._invoke_method_implementation_gt, body,
this, params, murano_class, context,
thread_marker)
result = gt.wait()
del self._locks[(method_id, this_id)]
event.send()
return result
def rengine_side(self, appid, token, uri):
""" Handle rengine (client) GET requests """
if not self.rengine_authorization_ok(appid, token):
LOGGER.info('Rengine content request authorization fails')
abort(401, 'Authorization failed')
evt = Event()
request_id = str(uuid4())
self.request_id_events[request_id] = evt
headers = ["%s: %s" % (header, val) for (header, val) in request.headers.items()]
packet = ScpPacket.make_sfkcontent(uri, request_id, headers)
try:
self._send(packet, appid)
except Exception as e:
abort(500, str(e))
LOGGER.debug("uri %s expected" % uri)
timeout = Timeout(TIMEOUT)
try:
resp = evt.wait()
except Timeout:
del self.request_id_events[request_id]
abort(504, 'Gateway Timeout')
finally:
timeout.cancel()
LOGGER.debug("uri %s got" % uri)
return resp
def test_create_shutdown_race(self):
"""
Test the race condition where the pipeline shuts down while
`create` is still executing.
"""
created = []
destroyed = []
counter = itertools.count()
creating = Event()
def create():
creating.send(True)
eventlet.sleep()
obj = next(counter)
created.append(obj)
return obj
def destroy(obj):
destroyed.append(obj)
with ResourcePipeline(create, destroy).run():
creating.wait()
assert created == []
assert created == destroyed == list(range(1))
def create(self):
id = uuid.uuid4().hex
remove = functools.partial(self._remove_event, id)
event = Event()
self.events[id] = weakref.proxy(event, remove)
event.id = id
return event, id
def handle_request(self, request):
request.shallow = False
try:
context_data = self.server.context_data_from_headers(request)
args, kwargs = self.get_entrypoint_parameters(request)
self.check_signature(args, kwargs)
event = Event()
self.container.spawn_worker(
self, args, kwargs, context_data=context_data,
handle_result=partial(self.handle_result, event))
result = event.wait()
response = response_from_result(result)
except Exception as exc:
if (
isinstance(exc, self.expected_exceptions) or
isinstance(exc, BadRequest)
):
status_code = 400
else:
status_code = 500
error_dict = serialize(exc)
payload = u'Error: {exc_type}: {value}\n'.format(**error_dict)
response = Response(
payload,
status=status_code,
)
return response
def test_kill_container_with_active_workers(container_factory):
waiting = Event()
wait_forever = Event()
class Service(object):
name = 'kill-with-active-workers'
@foobar
def spam(self):
waiting.send(None)
wait_forever.wait()
container = container_factory(Service, {})
dep = get_extension(container, Entrypoint)
# start the first worker, which should wait for spam_continue
container.spawn_worker(dep, (), {})
waiting.wait()
with patch('nameko.containers._log') as logger:
container.kill()
assert logger.warning.call_args_list == [
call('killing %s active workers(s)', 1),
call('killing active worker for %s', ANY)
]
def __init__(self, interval, eager=False, **kwargs):
"""
Timer entrypoint. Fires every `interval` seconds or as soon as
the previous worker completes if that took longer.
The default behaviour is to wait `interval` seconds
before firing for the first time. If you want the entrypoint
to fire as soon as the service starts, pass `eager=True`.
Example::
timer = Timer.decorator
class Service(object):
name = "service"
@timer(interval=5)
def tick(self):
pass
"""
self.interval = interval
self.eager = eager
self.should_stop = Event()
self.worker_complete = Event()
self.gt = None
super(Timer, self).__init__(**kwargs)
def test_fail_fast_imap():
# A failing call...
failing_exception = Exception()
def failing_call():
raise failing_exception
# ...and an eventually successful call.
slow_call_returned = Event()
def slow_call():
sleep(5)
slow_call_returned.send() # pragma: no cover
def identity_fn(fn):
return fn()
calls = [slow_call, failing_call]
pool = GreenPool(2)
# fail_fast_imap fails as soon as the exception is raised
with pytest.raises(Exception) as raised_exc:
list(fail_fast_imap(pool, identity_fn, calls))
assert raised_exc.value == failing_exception
# The slow call won't go past the sleep as it was killed
assert not slow_call_returned.ready()
assert pool.free() == 2
class StreamsResource(object):
def __init__(self):
self._action_event = Event()
self._session_events = {}
def new(self):
new_id = str(uuid.uuid4())
self._session_events[new_id] = Event()
print self._session_events.keys()
return new_id
def sessions(self):
return self._session_events.keys()
def send_message(self, id, message):
if id:
self._session_events[id].send(message)
eventlet.sleep()
self._session_events[id] = Event()
else:
self._action_event.send(message)
eventlet.sleep
self._action_event = Event()
def get_message(self, id):
if id:
return self._session_events[id].wait()
else:
return self._action_event.wait()
def handler(sock, client):
# socket opened
# first message is either hammerlib:get_clientid:sessionid
# or hammerlib:have_clientid:sessionid:clientid
app, command, payload = parse(sock.recv(2048)) # FIXME: what about framing?
payload = payload.strip() # FIXME: dirty hack
if app != "hammerlib": return error(sock, "no handshake")
if command == "get_clientid":
clientid, sessionid = get_next_id(), payload
add_connection(sock, clientid, payload)
elif command == "have_clientid":
# assert the proper session for given clientid
sessionid, clientid = payload.split(":")
client = clients.get(clientid)
if client:
if client["sessionid"] != sessionid:
return error(sock, "bad sessionid")
else:
add_connection(sock, clientid, sessionid)
else:
return error(sock, "bad handshake")
send_message_to_client(clientid, "hammerlib", "connected", clientid)
send_message_to_app(sock, "hammerlib", "client_connected", "")
event = Event()
eventlet.spawn_n(reader, sock, event)
event.wait()
class Handler(base.Handler):
__doc__ = base.Handler.__doc__ + """
This Handler subclass is designed for use with eventlet. It spawns a
a new green thread to handle each incoming request.
"""
ConnectionClass = Connection
def __init__(self,*args,**kwds):
super(Handler,self).__init__(*args,**kwds)
# We need to count the number of inflight requests, so the
# main thread can wait for them to complete when shutting down.
self._num_inflight_requests = 0
self._all_requests_complete = None
def handle_request(self,req):
self._num_inflight_requests += 1
if self._num_inflight_requests == 1:
self._all_requests_complete = Event()
@eventlet.spawn_n
def do_handle_request():
try:
self.process_request(req)
finally:
self._num_inflight_requests -= 1
if self._num_inflight_requests == 0:
self._all_requests_complete.send()
self._all_requests_complete = None
def wait_for_completion(self):
if self._num_inflight_requests > 0:
self._all_requests_complete.wait()
def _poll(self,sockets,timeout=None):
# Don't bother trampolining if there's data available immediately.
# This also avoids calling into the eventlet hub with a timeout of
# zero, which doesn't work right (it still switches the greenthread)
(r,_,_) = zmq_poll.select(sockets,[],[],timeout=0)
if r:
return r
if timeout == 0:
return []
# Looks like we'll have to block :-(
ready = []
threads = []
res = Event()
for sock in sockets:
threads.append(eventlet.spawn(self._do_poll,sock,ready,res,timeout))
self.poll_threads.append((res,threads))
try:
res.wait()
finally:
self.poll_threads.remove((res,threads))
for t in threads:
t.kill()
try:
t.wait()
except GreenletExit:
pass
return ready
class TimerProvider(EntrypointProvider):
def __init__(self, interval, config_key):
self._default_interval = interval
self.config_key = config_key
self.should_stop = Event()
self.gt = None
def prepare(self):
interval = self._default_interval
if self.config_key:
config = self.container.config
interval = config.get(self.config_key, interval)
self.interval = interval
def start(self):
_log.debug('starting %s', self)
self.gt = self.container.spawn_managed_thread(self._run)
def stop(self):
_log.debug('stopping %s', self)
self.should_stop.send(True)
self.gt.wait()
def kill(self, exc):
_log.debug('killing %s', self)
self.gt.kill()
def _run(self):
''' Runs the interval loop.
This should not be called directly, rather the `start()` method
should be used.
'''
while not self.should_stop.ready():
start = time.time()
self.handle_timer_tick()
elapsed_time = (time.time() - start)
sleep_time = max(self.interval - elapsed_time, 0)
self._sleep_or_stop(sleep_time)
def _sleep_or_stop(self, sleep_time):
''' Sleeps for `sleep_time` seconds or until a `should_stop` event
has been fired, whichever comes first.
'''
try:
with Timeout(sleep_time):
self.should_stop.wait()
except Timeout:
# we use the timeout as a cancellable sleep
pass
def handle_timer_tick(self):
args = tuple()
kwargs = {}
self.container.spawn_worker(self, args, kwargs)
def init_events(self):
# these events correspond to the server socket
self.server_start = Event()
self.server_stop = Event()
# these events more or less correspond to the completion of the
# startup process, including forking
self.running = Event()
self.stopped = Event()
def handle_message(self, socket_id, data, context_data):
self.check_signature((socket_id,), data)
event = Event()
self.container.spawn_worker(self, (socket_id,), data,
context_data=context_data,
handle_result=partial(
self.handle_result, event))
return event.wait()
def test_prefetch_count(rabbit_manager, rabbit_config, container_factory):
class NonShared(QueueConsumer):
@property
def sharing_key(self):
return uuid.uuid4()
messages = []
class SelfishConsumer1(Consumer):
queue_consumer = NonShared()
def handle_message(self, body, message):
consumer_continue.wait()
super(SelfishConsumer1, self).handle_message(body, message)
class SelfishConsumer2(Consumer):
queue_consumer = NonShared()
def handle_message(self, body, message):
messages.append(body)
super(SelfishConsumer2, self).handle_message(body, message)
class Service(object):
name = "service"
@SelfishConsumer1.decorator(queue=ham_queue)
@SelfishConsumer2.decorator(queue=ham_queue)
def handle(self, payload):
pass
rabbit_config['max_workers'] = 1
container = container_factory(Service, rabbit_config)
container.start()
consumer_continue = Event()
# the two handlers would ordinarily take alternating messages, but are
# limited to holding one un-ACKed message. Since Handler1 never ACKs, it
# only ever gets one message, and Handler2 gets the others.
def wait_for_expected(worker_ctx, res, exc_info):
return {'m3', 'm4', 'm5'}.issubset(set(messages))
with entrypoint_waiter(container, 'handle', callback=wait_for_expected):
vhost = rabbit_config['vhost']
properties = {'content_type': 'application/data'}
for message in ('m1', 'm2', 'm3', 'm4', 'm5'):
rabbit_manager.publish(
vhost, 'spam', '', message, properties=properties
)
# we don't know which handler picked up the first message,
# but all the others should've been handled by Handler2
assert messages[-3:] == ['m3', 'm4', 'm5']
# release the waiting consumer
consumer_continue.send(None)
def start_branch(env, argv=None):
env.syncdb(interactive=False)
from cyme.branch import Branch
ready_event = Event()
CYME_INSTANCE_DIR.mkdir()
instance = Branch("127.0.0.1:%s" % (CYME_PORT, ), numc=1,
ready_event=ready_event)
instance.start()
ready_event.wait()
return instance
class MessageHandler(object):
queue = ham_queue
def __init__(self):
self.handle_message_called = Event()
def handle_message(self, body, message):
self.handle_message_called.send(message)
def wait(self):
return self.handle_message_called.wait()
def _start_consumer(self, channel):
ready = Event()
def make_consumer(ctx):
consumer = EventConsumer(channel, ctx)
self._consumers[channel] = consumer
if not ready.has_result():
ready.send(consumer)
return consumer
proc = spawn(consumer_loop, make_consumer, self.context)
consumer = ready.wait()
return consumer, proc
class Timer(object):
''' A timer object, which will call a given method repeatedly at a given
interval.
'''
def __init__(self, interval, func):
self.interval = interval
self.func = func
self.gt = None
self.should_stop = Event()
def start(self):
''' Starts the timer in a separate green thread.
Once started it may be stopped using its `stop()` method.
'''
self.gt = eventlet.spawn(self._run)
_log.debug(
'started timer for %s with %ss interval',
self.func, self.interval)
def _run(self):
''' Runs the interval loop.
This should not be called directly, rather the `start()` method
should be used.
'''
while not self.should_stop.ready():
start = time.time()
try:
self.func()
except Exception as e:
_log.exception('error in timer handler: %s', e)
sleep_time = max(self.interval - (time.time() - start), 0)
self._sleep_or_stop(sleep_time)
def _sleep_or_stop(self, sleep_time):
''' Sleeps for `sleep_time` seconds or until a `should_stop` event
has been fired, whichever comes first.
'''
try:
with Timeout(sleep_time):
self.should_stop.wait()
except Timeout:
# we use the timeout as a cancellable sleep
pass
def stop(self):
''' Gracefully stops the timer, waiting for it's timer_method
to complete if it is running.
'''
self.should_stop.send(True)
self.gt.wait()
def test_send(self):
event1 = Event()
event2 = Event()
spawn(event1.send, 'hello event1')
eventlet.Timeout(0, ValueError('interrupted'))
try:
result = event1.wait()
except ValueError:
X = object()
result = with_timeout(DELAY, event2.wait, timeout_value=X)
assert result is X, 'Nobody sent anything to event2 yet it received %r' % (result, )
def test_send_rpc_multi_message_reply_ignores_all_but_last(get_connection):
queue_declared = Event()
def response_greenthread():
with get_connection() as conn:
with conn.channel() as chan:
queue = nova.get_topic_queue(
'test_rpc', 'test', channel=chan)
queue.declare()
queue_declared.send(True)
body, msg = ifirst(
queue_iterator(queue, no_ack=True, timeout=2))
msgid, _, _, args = nova.parse_message(body)
exchange = nova.get_reply_exchange(msgid)
producer = Producer(chan, exchange=exchange, routing_key=msgid)
for _ in range(3):
msg = dict(
result='should ignore this message',
failure=None, ending=False)
producer.publish(msg)
eventlet.sleep(0.1)
msg = dict(result=args, failure=None, ending=False)
producer.publish(msg)
msg = dict(result=None, failure=None, ending=True)
producer.publish(msg)
g = eventlet.spawn_n(response_greenthread)
eventlet.sleep()
with get_connection() as conn:
ctx = context.get_admin_context()
queue_declared.wait()
resp = nova.send_rpc(
conn,
context=ctx,
exchange='test_rpc',
topic='test',
method='test_method',
args={'spam': 'shrub', },
timeout=3)
assert resp == {'spam': 'shrub', }
eventlet.sleep()
def check_greenthread_dead():
assert not g
assert_stops_raising(check_greenthread_dead)
class EventletCallback(object):
def __init__(self):
self.event = Event()
def wait(self):
with eventlet.Timeout(10):
return self.event.wait()
def success(self, result):
self.event.send(result)
def failure(self, exc):
self.event.send_exception(exc)
def test_deferred_send_receive(ctx):
from datetime import datetime, timedelta
from carrot.messaging import Consumer
from eventlet import sleep, spawn, with_timeout
from eventlet.event import Event
from eventlet.support.greenlets import GreenletExit
import logging
from melk.util.nonce import nonce_str
import sys
from melkman.context import Context
from melkman.scheduler import defer_amqp_message, cancel_deferred
from melkman.scheduler.worker import ScheduledMessageService
got_message = Event()
def got_message_cb(*args, **kw):
got_message.send(True)
def do_consume():
consumer = Consumer(ctx.broker, exchange='testx', queue='testq',
routing_key='testq', exclusive=True, durable=False)
consumer.register_callback(got_message_cb)
try:
consumer.wait(limit=1)
except StopIteration:
pass
except GreenletExit:
pass
finally:
consumer.close()
cons = spawn(do_consume)
sms = ScheduledMessageService(ctx)
sched = spawn(sms.run)
m1 = {'hello': 'world'}
now = datetime.utcnow()
wait = timedelta(seconds=2)
defer_amqp_message(now + wait, m1, 'testq', 'testx', ctx)
try:
#sleep(1)
with_timeout(10, got_message.wait)
assert got_message.ready()
finally:
sched.kill()
sched.wait()
cons.kill()
cons.wait()
def test_container_doesnt_exhaust_max_workers(container):
spam_called = Event()
spam_continue = Event()
class Service(object):
name = 'max-workers'
@foobar
def spam(self, a):
spam_called.send(a)
spam_continue.wait()
container = ServiceContainer(Service, config={MAX_WORKERS_CONFIG_KEY: 1})
dep = get_extension(container, Entrypoint)
# start the first worker, which should wait for spam_continue
container.spawn_worker(dep, ['ham'], {})
# start the next worker in a speparate thread,
# because it should block until the first one completed
gt = spawn(container.spawn_worker, dep, ['eggs'], {})
with Timeout(1):
assert spam_called.wait() == 'ham'
# if the container had spawned the second worker, we would see
# an error indicating that spam_called was fired twice, and the
# greenthread would now be dead.
assert not gt.dead
# reset the calls and allow the waiting worker to complete.
spam_called.reset()
spam_continue.send(None)
# the second worker should now run and complete
assert spam_called.wait() == 'eggs'
assert gt.dead
def rpc(self, _method, **data):
id = str(uuid.uuid4())
event = Event()
result_handlers[id] = event.send
ws_app.send(json.dumps({
'method': _method,
'data': data,
'correlation_id': id,
}))
rv = event.wait()
if rv['success']:
return rv['data']
raise deserialize(rv['error'])
def test_waiting(self):
def do_wait(q, evt):
result = q.wait()
evt.send(result)
q = coros.queue()
e1 = Event()
spawn(do_wait, q, e1)
sleep(0)
self.assertEqual(1, q.waiting())
q.send('hi')
sleep(0)
self.assertEqual(0, q.waiting())
self.assertEqual('hi', e1.wait())
self.assertEqual(0, q.waiting())
class Queue(LightQueue):
'''Create a queue object with a given maximum size.
If *maxsize* is less than zero or ``None``, the queue size is infinite.
``Queue(0)`` is a channel, that is, its :meth:`put` method always blocks
until the item is delivered. (This is unlike the standard
:class:`Stdlib_Queue`, where 0 means infinite size).
In all other respects, this Queue class resembles the standard library,
:class:`Stdlib_Queue`.
'''
def __init__(self, maxsize=None):
LightQueue.__init__(self, maxsize)
self.unfinished_tasks = 0
self._cond = Event()
def _format(self):
result = LightQueue._format(self)
if self.unfinished_tasks:
result += ' tasks=%s _cond=%s' % (self.unfinished_tasks,
self._cond)
return result
def _put(self, item):
LightQueue._put(self, item)
self._put_bookkeeping()
def _put_bookkeeping(self):
self.unfinished_tasks += 1
if self._cond.ready():
self._cond.reset()
def task_done(self):
'''Indicate that a formerly enqueued task is complete. Used by queue consumer threads.
For each :meth:`get <Queue.get>` used to fetch a task, a subsequent call to
:meth:`task_done` tells the queue that the processing on the task is complete.
If a :meth:`join` is currently blocking, it will resume when all items have been processed
(meaning that a :meth:`task_done` call was received for every item that had been
:meth:`put <Queue.put>` into the queue).
Raises a :exc:`ValueError` if called more times than there were items placed in the queue.
'''
if self.unfinished_tasks <= 0:
raise ValueError('task_done() called too many times')
self.unfinished_tasks -= 1
if self.unfinished_tasks == 0:
self._cond.send(None)
def join(self):
'''Block until all items in the queue have been gotten and processed.
The count of unfinished tasks goes up whenever an item is added to the queue.
The count goes down whenever a consumer thread calls :meth:`task_done` to indicate
that the item was retrieved and all work on it is complete. When the count of
unfinished tasks drops to zero, :meth:`join` unblocks.
'''
if self.unfinished_tasks > 0:
self._cond.wait()
class TimerProvider(EntrypointProvider):
def __init__(self, interval, config_key):
self._default_interval = interval
self.config_key = config_key
self.should_stop = Event()
self.gt = None
def prepare(self):
interval = self._default_interval
if self.config_key:
config = self.container.config
interval = config.get(self.config_key, interval)
self.interval = interval
def start(self):
_log.debug('starting %s', self)
self.gt = self.container.spawn_managed_thread(self._run)
def stop(self):
_log.debug('stopping %s', self)
self.should_stop.send(True)
self.gt.wait()
def kill(self):
_log.debug('killing %s', self)
self.gt.kill()
def _run(self):
''' Runs the interval loop.
This should not be called directly, rather the `start()` method
should be used.
'''
while not self.should_stop.ready():
start = time.time()
self.handle_timer_tick()
elapsed_time = (time.time() - start)
sleep_time = max(self.interval - elapsed_time, 0)
self._sleep_or_stop(sleep_time)
def _sleep_or_stop(self, sleep_time):
''' Sleeps for `sleep_time` seconds or until a `should_stop` event
has been fired, whichever comes first.
'''
try:
with Timeout(sleep_time):
self.should_stop.wait()
except Timeout:
# we use the timeout as a cancellable sleep
pass
def handle_timer_tick(self):
args = ()
kwargs = {}
# Note that we don't catch ContainerBeingKilled here. If that's raised,
# there is nothing for us to do anyway. The exception bubbles, and is
# caught by :meth:`Container._handle_thread_exited`, though the
# triggered `kill` is a no-op, since the container is alredy
# `_being_killed`.
self.container.spawn_worker(self, args, kwargs)
def __init__(self, interval, config_key):
self._default_interval = interval
self.config_key = config_key
self.should_stop = Event()
self.gt = None
class QueueConsumer(SharedExtension, ProviderCollector, ConsumerMixin):
def __init__(self):
self._consumers = {}
self._pending_remove_providers = {}
self._gt = None
self._starting = False
self._consumers_ready = Event()
super(QueueConsumer, self).__init__()
@property
def amqp_uri(self):
return self.container.config[AMQP_URI_CONFIG_KEY]
@property
def prefetch_count(self):
return self.container.max_workers
@property
def accept(self):
return self.container.accept
def _handle_thread_exited(self, gt):
exc = None
try:
gt.wait()
except Exception as e:
exc = e
if not self._consumers_ready.ready():
self._consumers_ready.send_exception(exc)
def setup(self):
ssl = self.container.config.get(AMQP_SSL_CONFIG_KEY)
verify_amqp_uri(self.amqp_uri, ssl=ssl)
def start(self):
if not self._starting:
self._starting = True
_log.debug('starting %s', self)
self._gt = self.container.spawn_managed_thread(self.run)
self._gt.link(self._handle_thread_exited)
try:
_log.debug('waiting for consumer ready %s', self)
self._consumers_ready.wait()
except QueueConsumerStopped:
_log.debug('consumer was stopped before it started %s', self)
except Exception as exc:
_log.debug('consumer failed to start %s (%s)', self, exc)
else:
_log.debug('started %s', self)
def stop(self):
""" Stop the queue-consumer gracefully.
Wait until the last provider has been unregistered and for
the ConsumerMixin's greenthread to exit (i.e. until all pending
messages have been acked or requeued and all consumers stopped).
"""
if not self._consumers_ready.ready():
_log.debug('stopping while consumer is starting %s', self)
stop_exc = QueueConsumerStopped()
# stopping before we have started successfully by brutally
# killing the consumer thread as we don't have a way to hook
# into the pre-consumption startup process
self._gt.kill(stop_exc)
self.wait_for_providers()
try:
_log.debug('waiting for consumer death %s', self)
self._gt.wait()
except QueueConsumerStopped:
pass
super(QueueConsumer, self).stop()
_log.debug('stopped %s', self)
def kill(self):
""" Kill the queue-consumer.
Unlike `stop()` any pending message ack or requeue-requests,
requests to remove providers, etc are lost and the consume thread is
asked to terminate as soon as possible.
"""
# greenlet has a magic attribute ``dead`` - pylint: disable=E1101
if self._gt is not None and not self._gt.dead:
# we can't just kill the thread because we have to give
# ConsumerMixin a chance to close the sockets properly.
self._providers = set()
self._pending_remove_providers = {}
self.should_stop = True
try:
self._gt.wait()
except Exception as exc:
# discard the exception since we're already being killed
_log.warn('QueueConsumer %s raised `%s` during kill', self,
exc)
super(QueueConsumer, self).kill()
_log.debug('killed %s', self)
def unregister_provider(self, provider):
if not self._consumers_ready.ready():
# we cannot handle the situation where we are starting up and
# want to remove a consumer at the same time
# TODO: With the upcomming error handling mechanism, this needs
# TODO: to be thought through again.
self._last_provider_unregistered.send()
return
removed_event = Event()
# we can only cancel a consumer from within the consumer thread
self._pending_remove_providers[provider] = removed_event
# so we will just register the consumer to be canceled
removed_event.wait()
super(QueueConsumer, self).unregister_provider(provider)
def ack_message(self, message):
# only attempt to ack if the message connection is alive;
# otherwise the message will already have been reclaimed by the broker
if message.channel.connection:
try:
message.ack()
except ConnectionError: # pragma: no cover
pass # ignore connection closing inside conditional
def requeue_message(self, message):
# only attempt to requeue if the message connection is alive;
# otherwise the message will already have been reclaimed by the broker
if message.channel.connection:
try:
message.requeue()
except ConnectionError: # pragma: no cover
pass # ignore connection closing inside conditional
def _cancel_consumers_if_requested(self):
provider_remove_events = self._pending_remove_providers.items()
self._pending_remove_providers = {}
for provider, removed_event in provider_remove_events:
consumer = self._consumers.pop(provider)
_log.debug('cancelling consumer [%s]: %s', provider, consumer)
consumer.cancel()
removed_event.send()
@property
def connection(self):
""" Provide the connection parameters for kombu's ConsumerMixin.
The `Connection` object is a declaration of connection parameters
that is lazily evaluated. It doesn't represent an established
connection to the broker at this point.
"""
heartbeat = self.container.config.get(HEARTBEAT_CONFIG_KEY,
DEFAULT_HEARTBEAT)
ssl = self.container.config.get(AMQP_SSL_CONFIG_KEY)
return Connection(self.amqp_uri, heartbeat=heartbeat, ssl=ssl)
def handle_message(self, provider, body, message):
ident = u"{}.handle_message[{}]".format(
type(provider).__name__, message.delivery_info['routing_key'])
self.container.spawn_managed_thread(partial(provider.handle_message,
body, message),
identifier=ident)
def get_consumers(self, consumer_cls, channel):
""" Kombu callback to set up consumers.
Called after any (re)connection to the broker.
"""
_log.debug('setting up consumers %s', self)
for provider in self._providers:
callbacks = [partial(self.handle_message, provider)]
consumer = consumer_cls(queues=[provider.queue],
callbacks=callbacks,
accept=self.accept)
consumer.qos(prefetch_count=self.prefetch_count)
self._consumers[provider] = consumer
return self._consumers.values()
def on_iteration(self):
""" Kombu callback for each `drain_events` loop iteration."""
self._cancel_consumers_if_requested()
if len(self._consumers) == 0:
_log.debug('requesting stop after iteration')
self.should_stop = True
def on_connection_error(self, exc, interval):
_log.warning("Error connecting to broker at {} ({}).\n"
"Retrying in {} seconds.".format(self.amqp_uri, exc,
interval))
def on_consume_ready(self, connection, channel, consumers, **kwargs):
""" Kombu callback when consumers are ready to accept messages.
Called after any (re)connection to the broker.
"""
if not self._consumers_ready.ready():
_log.debug('consumer started %s', self)
self._consumers_ready.send(None)
class Client(object):
def __init__(self):
self.results = []
self.stop = Event()
self.no_more_results = Event()
self.failure = None
self.next_lease_id = 100000
self.keys_written = set()
def get(self, key, metadata=False):
assert metadata, "Always expect get() call with metadata=True"
try:
result = self.read(key)
mod_revision = 10
if result.etcd_index != 0:
mod_revision = result.etcd_index
return [(result.value, {'mod_revision': str(mod_revision)})]
except etcdv3.KeyNotFound:
return []
def watch_once(self, key, timeout=None, **kwargs):
result = self.read(key)
mod_revision = 10
if result.etcd_index != 0:
mod_revision = result.etcd_index
return {'kv': {
'value': result.value,
'mod_revision': mod_revision
}}
def read(self, path, **kwargs):
try:
result = self.results.pop(0)
except IndexError:
if not self.no_more_results.ready():
self.no_more_results.send()
eventlet.with_timeout(5, self.stop.wait)
raise NoMoreResults()
if result.op != READ:
self.failure = "Unexpected result type for read(): %s" % result.op
raise UnexpectedResultType()
if result.exception is not None:
log.debug("Raise read exception %s",
type(result.exception).__name__)
raise result.exception
log.debug("Return read result %s", result)
return result
def put(self, key, value, lease=None):
self.write(key, value)
return True
def transaction(self, txn):
put_request = txn['success'][0]['request_put']
succeeded = self.put(_decode(put_request['key']),
_decode(put_request['value']))
return {'succeeded': succeeded}
def lease(self, ttl):
l = Lease(self.next_lease_id, self)
self.next_lease_id += 1
return l
def write(self, path, value, **kwargs):
log.debug("Write of %s to %s", value, path)
try:
result = self.results.pop(0)
except IndexError:
if not self.no_more_results.ready():
self.no_more_results.send()
eventlet.with_timeout(5, self.stop.wait)
raise NoMoreResults()
if result.op != WRITE:
self.failure = "Unexpected result type for write(): %s" % result.op
raise UnexpectedResultType()
if result.exception is not None:
log.debug("Raise write exception %s", result.exception)
raise result.exception
log.debug("Return write result")
self.keys_written.add(path)
return result
def assert_key_written(self, key):
assert(key in self.keys_written)
def add_read_exception(self, exception):
assert(isinstance(exception, Exception))
self.results.append(EtcdResult(exception=exception))
def add_read_result(self, **kwargs):
self.results.append(EtcdResult(**kwargs))
def add_write_result(self):
# Write results have no useful content.
self.results.append(EtcdResult(op=WRITE))
def add_write_exception(self, exception):
self.results.append(EtcdResult(op=WRITE, exception=exception))
def get_reply_event(self, correlation_id):
reply_event = Event()
self._reply_events[correlation_id] = reply_event
return reply_event
def __init__(self, callback=None):
self.callback = callback
self.ready = Event()
class RpcConsumer(SharedExtension, ProviderCollector):
queue_consumer = QueueConsumer()
def __init__(self):
self._unregistering_providers = set()
self._unregistered_from_queue_consumer = Event()
self.queue = None
super(RpcConsumer, self).__init__()
def setup(self):
if self.queue is None:
service_name = self.container.service_name
queue_name = RPC_QUEUE_TEMPLATE.format(service_name)
routing_key = '{}.*'.format(service_name)
exchange = get_rpc_exchange(self.container.config)
self.queue = Queue(queue_name,
exchange=exchange,
routing_key=routing_key,
durable=True)
self.queue_consumer.register_provider(self)
self._registered = True
def stop(self):
""" Stop the RpcConsumer.
The RpcConsumer ordinary unregisters from the QueueConsumer when the
last Rpc subclass unregisters from it. If no providers were registered,
we should unregister from the QueueConsumer as soon as we're asked
to stop.
"""
if not self._providers_registered:
self.queue_consumer.unregister_provider(self)
self._unregistered_from_queue_consumer.send(True)
def unregister_provider(self, provider):
""" Unregister a provider.
Blocks until this RpcConsumer is unregistered from its QueueConsumer,
which only happens when all providers have asked to unregister.
"""
self._unregistering_providers.add(provider)
remaining_providers = self._providers - self._unregistering_providers
if not remaining_providers:
_log.debug('unregistering from queueconsumer %s', self)
self.queue_consumer.unregister_provider(self)
_log.debug('unregistered from queueconsumer %s', self)
self._unregistered_from_queue_consumer.send(True)
_log.debug('waiting for unregister from queue consumer %s', self)
self._unregistered_from_queue_consumer.wait()
super(RpcConsumer, self).unregister_provider(provider)
def get_provider_for_method(self, routing_key):
service_name = self.container.service_name
for provider in self._providers:
key = '{}.{}'.format(service_name, provider.method_name)
if key == routing_key:
return provider
else:
method_name = routing_key.split(".")[-1]
raise MethodNotFound(method_name)
def handle_message(self, body, message):
routing_key = message.delivery_info['routing_key']
try:
provider = self.get_provider_for_method(routing_key)
provider.handle_message(body, message)
except Exception:
exc_info = sys.exc_info()
self.handle_result(message, None, exc_info)
def handle_result(self, message, result, exc_info):
amqp_uri = self.container.config[AMQP_URI_CONFIG_KEY]
serializer = self.container.config.get(SERIALIZER_CONFIG_KEY,
DEFAULT_SERIALIZER)
exchange = get_rpc_exchange(self.container.config)
responder = Responder(amqp_uri, exchange, serializer, message)
result, exc_info = responder.send_response(result, exc_info)
self.queue_consumer.ack_message(message)
return result, exc_info
def requeue_message(self, message):
self.queue_consumer.requeue_message(message)
def __init__(self):
self.event = Event()
def __init__(self, *args, **kwargs):
self._providers = set()
self._providers_registered = False
self._last_provider_unregistered = Event()
super(ProviderCollector, self).__init__(*args, **kwargs)
def test_handlers_do_not_block(config, container_factory, make_cometd_server,
message_maker, run_services, tracker, waiter):
""" Test that entrypoints do not block each other
"""
work_a = Event()
work_b = Event()
class Service:
name = 'example-service'
@subscribe('/topic/example-a')
def handle_event_a(self, channel, payload):
work_a.wait()
tracker.handle_event_a(channel, payload)
@subscribe('/topic/example-b')
def handle_event_b(self, channel, payload):
work_b.wait()
tracker.handle_event_b(channel, payload)
responses = [
# respond to handshake
[message_maker.make_handshake_response()],
# respond to subscribe
[
message_maker.make_subscribe_response(
subscription='/topic/example-a'),
message_maker.make_subscribe_response(
subscription='/topic/example-b'),
],
# respond to initial connect
[
message_maker.make_connect_response(
advice={'reconnect': Reconnection.retry.value}),
],
# two events to deliver
[
message_maker.make_event_delivery_message(
channel='/topic/example-a', data={'spam': 'one'}),
message_maker.make_event_delivery_message(
channel='/topic/example-b', data={'spam': 'two'}),
],
]
cometd_server = make_cometd_server(responses)
container = container_factory(Service, config)
cometd_server.start()
container.start()
try:
# both handlers are still working
assert (tracker.handle_event_a.call_args_list == [])
assert (tracker.handle_event_b.call_args_list == [])
# finish work of the second handler
work_b.send()
sleep(0.1)
# second handler is done
assert (tracker.handle_event_a.call_args_list == [])
assert (tracker.handle_event_b.call_args_list == [
call('/topic/example-b', {'spam': 'two'})
])
# finish work of the first handler
work_a.send()
sleep(0.1)
# first handler is done
assert (tracker.handle_event_a.call_args_list == [
call('/topic/example-a', {'spam': 'one'})
])
assert (tracker.handle_event_b.call_args_list == [
call('/topic/example-b', {'spam': 'two'})
])
finally:
if not work_a.ready():
work_a.send()
if not work_b.ready():
work_b.send()
waiter.wait()
container.kill()
cometd_server.stop()
def __init__(self):
self.handle_message_called = Event()
def test_prefetch_count(rabbit_manager, rabbit_config, mock_container):
container = mock_container
container.shared_extensions = {}
container.config = rabbit_config
container.max_workers = 1
container.spawn_managed_thread = spawn_managed_thread
content_type = 'application/data'
container.accept = [content_type]
class NonShared(QueueConsumer):
@property
def sharing_key(self):
return uuid.uuid4()
queue_consumer1 = NonShared().bind(container)
queue_consumer1.setup()
queue_consumer2 = NonShared().bind(container)
queue_consumer2.setup()
consumer_continue = Event()
class Handler1(object):
queue = ham_queue
def handle_message(self, body, message):
consumer_continue.wait()
queue_consumer1.ack_message(message)
messages = []
class Handler2(object):
queue = ham_queue
def handle_message(self, body, message):
messages.append(body)
queue_consumer2.ack_message(message)
handler1 = Handler1()
handler2 = Handler2()
queue_consumer1.register_provider(handler1)
queue_consumer2.register_provider(handler2)
queue_consumer1.start()
queue_consumer2.start()
vhost = rabbit_config['vhost']
# the two handlers would ordinarily take alternating messages, but are
# limited to holding 1 un-ACKed message. Since handler 1 never ACKs, it
# only ever gets message 'a'; handler 2 gets the others.
for message in ('a', 'b', 'c'):
rabbit_manager.publish(vhost,
'spam',
'',
message,
properties=dict(content_type=content_type))
# allow the waiting consumer to ack its message
consumer_continue.send(None)
assert messages == ['b', 'c']
queue_consumer1.unregister_provider(handler1)
queue_consumer2.unregister_provider(handler2)
queue_consumer1.kill()
queue_consumer2.kill()
def test_handlers_do_not_block(SlackClient, container_factory, config,
tracker):
work_1 = Event()
work_2 = Event()
class Service:
name = 'sample'
@rtm.handle_event
def handle_1(self, event):
work_1.wait()
tracker.handle_1(event)
@rtm.handle_event
def handle_2(self, event):
work_2.wait()
tracker.handle_2(event)
events = [{'spam': 'ham'}]
def rtm_read():
if events:
return [events.pop(0)]
else:
return []
SlackClient.return_value.rtm_read.side_effect = rtm_read
container = container_factory(Service, config)
container.start()
try:
# both handlers are still working
assert (tracker.handle_1.call_args_list == [])
assert (tracker.handle_2.call_args_list == [])
# finish work of the second handler
work_2.send()
sleep(0.1)
# second handler is done
assert (tracker.handle_1.call_args_list == [])
assert (tracker.handle_2.call_args_list == [call({'spam': 'ham'})])
# finish work of the first handler
work_1.send()
sleep(0.1)
# first handler is done
assert (tracker.handle_1.call_args_list == [call({'spam': 'ham'})])
assert (tracker.handle_2.call_args_list == [call({'spam': 'ham'})])
finally:
if not work_1.ready():
work_1.send()
if not work_2.ready():
work_2.send()
def on_service_start(self, *args, **kwargs):
self.launch_app(self.shell_app)
eventlet.spawn(self.translator.start)
Event().wait()
def __init__(self):
self._unregistering_providers = set()
self._unregistered_from_queue_consumer = Event()
self.queue = None
super(RpcConsumer, self).__init__()
class DAGPool(object):
"""
A DAGPool is a pool that constrains greenthreads, not by max concurrency,
but by data dependencies.
This is a way to implement general DAG dependencies. A simple dependency
tree (flowing in either direction) can straightforwardly be implemented
using recursion and (e.g.)
:meth:`GreenThread.imap() <eventlet.greenthread.GreenThread.imap>`.
What gets complicated is when a given node depends on several other nodes
as well as contributing to several other nodes.
With DAGPool, you concurrently launch all applicable greenthreads; each
will proceed as soon as it has all required inputs. The DAG is implicit in
which items are required by each greenthread.
Each greenthread is launched in a DAGPool with a key: any value that can
serve as a Python dict key. The caller also specifies an iterable of other
keys on which this greenthread depends. This iterable may be empty.
The greenthread callable must accept (key, results), where:
key
is its own key
results
is an iterable of (key, value) pairs.
A newly-launched DAGPool greenthread is entered immediately, and can
perform any necessary setup work. At some point it will iterate over the
(key, value) pairs from the passed 'results' iterable. Doing so blocks the
greenthread until a value is available for each of the keys specified in
its initial dependencies iterable. These (key, value) pairs are delivered
in chronological order, *not* the order in which they are initially
specified: each value will be delivered as soon as it becomes available.
The value returned by a DAGPool greenthread becomes the value for its
key, which unblocks any other greenthreads waiting on that key.
If a DAGPool greenthread terminates with an exception instead of returning
a value, attempting to retrieve the value raises :class:`PropagateError`,
which binds the key of the original greenthread and the original
exception. Unless the greenthread attempting to retrieve the value handles
PropagateError, that exception will in turn be wrapped in a PropagateError
of its own, and so forth. The code that ultimately handles PropagateError
can follow the chain of PropagateError.exc attributes to discover the flow
of that exception through the DAG of greenthreads.
External greenthreads may also interact with a DAGPool. See :meth:`wait_each`,
:meth:`waitall`, :meth:`post`.
It is not recommended to constrain external DAGPool producer greenthreads
in a :class:`GreenPool <eventlet.greenpool.GreenPool>`: it may be hard to
provably avoid deadlock.
.. automethod:: __init__
.. automethod:: __getitem__
"""
_Coro = collections.namedtuple("_Coro", ("greenthread", "pending"))
def __init__(self, preload={}):
"""
DAGPool can be prepopulated with an initial dict or iterable of (key,
value) pairs. These (key, value) pairs are of course immediately
available for any greenthread that depends on any of those keys.
"""
try:
# If a dict is passed, copy it. Don't risk a subsequent
# modification to passed dict affecting our internal state.
iteritems = six.iteritems(preload)
except AttributeError:
# Not a dict, just an iterable of (key, value) pairs
iteritems = preload
# Load the initial dict
self.values = dict(iteritems)
# track greenthreads
self.coros = {}
# The key to blocking greenthreads is the Event.
self.event = Event()
def waitall(self):
"""
waitall() blocks the calling greenthread until there is a value for
every DAGPool greenthread launched by :meth:`spawn`. It returns a dict
containing all :class:`preload data <DAGPool>`, all data from
:meth:`post` and all values returned by spawned greenthreads.
See also :meth:`wait`.
"""
# waitall() is an alias for compatibility with GreenPool
return self.wait()
def wait(self, keys=_MISSING):
"""
*keys* is an optional iterable of keys. If you omit the argument, it
waits for all the keys from :class:`preload data <DAGPool>`, from
:meth:`post` calls and from :meth:`spawn` calls: in other words, all
the keys of which this DAGPool is aware.
wait() blocks the calling greenthread until all of the relevant keys
have values. wait() returns a dict whose keys are the relevant keys,
and whose values come from the *preload* data, from values returned by
DAGPool greenthreads or from :meth:`post` calls.
If a DAGPool greenthread terminates with an exception, wait() will
raise :class:`PropagateError` wrapping that exception. If more than
one greenthread terminates with an exception, it is indeterminate
which one wait() will raise.
If an external greenthread posts a :class:`PropagateError` instance,
wait() will raise that PropagateError. If more than one greenthread
posts PropagateError, it is indeterminate which one wait() will raise.
See also :meth:`wait_each_success`, :meth:`wait_each_exception`.
"""
# This is mostly redundant with wait_each() functionality.
return dict(self.wait_each(keys))
def wait_each(self, keys=_MISSING):
"""
*keys* is an optional iterable of keys. If you omit the argument, it
waits for all the keys from :class:`preload data <DAGPool>`, from
:meth:`post` calls and from :meth:`spawn` calls: in other words, all
the keys of which this DAGPool is aware.
wait_each() is a generator producing (key, value) pairs as a value
becomes available for each requested key. wait_each() blocks the
calling greenthread until the next value becomes available. If the
DAGPool was prepopulated with values for any of the relevant keys, of
course those can be delivered immediately without waiting.
Delivery order is intentionally decoupled from the initial sequence of
keys: each value is delivered as soon as it becomes available. If
multiple keys are available at the same time, wait_each() delivers
each of the ready ones in arbitrary order before blocking again.
The DAGPool does not distinguish between a value returned by one of
its own greenthreads and one provided by a :meth:`post` call or *preload* data.
The wait_each() generator terminates (raises StopIteration) when all
specified keys have been delivered. Thus, typical usage might be:
::
for key, value in dagpool.wait_each(keys):
# process this ready key and value
# continue processing now that we've gotten values for all keys
By implication, if you pass wait_each() an empty iterable of keys, it
returns immediately without yielding anything.
If the value to be delivered is a :class:`PropagateError` exception object, the
generator raises that PropagateError instead of yielding it.
See also :meth:`wait_each_success`, :meth:`wait_each_exception`.
"""
# Build a local set() and then call _wait_each().
return self._wait_each(self._get_keyset_for_wait_each(keys))
def wait_each_success(self, keys=_MISSING):
"""
wait_each_success() filters results so that only success values are
yielded. In other words, unlike :meth:`wait_each`, wait_each_success()
will not raise :class:`PropagateError`. Not every provided (or
defaulted) key will necessarily be represented, though naturally the
generator will not finish until all have completed.
In all other respects, wait_each_success() behaves like :meth:`wait_each`.
"""
for key, value in self._wait_each_raw(
self._get_keyset_for_wait_each(keys)):
if not isinstance(value, PropagateError):
yield key, value
def wait_each_exception(self, keys=_MISSING):
"""
wait_each_exception() filters results so that only exceptions are
yielded. Not every provided (or defaulted) key will necessarily be
represented, though naturally the generator will not finish until
all have completed.
Unlike other DAGPool methods, wait_each_exception() simply yields
:class:`PropagateError` instances as values rather than raising them.
In all other respects, wait_each_exception() behaves like :meth:`wait_each`.
"""
for key, value in self._wait_each_raw(
self._get_keyset_for_wait_each(keys)):
if isinstance(value, PropagateError):
yield key, value
def _get_keyset_for_wait_each(self, keys):
"""
wait_each(), wait_each_success() and wait_each_exception() promise
that if you pass an iterable of keys, the method will wait for results
from those keys -- but if you omit the keys argument, the method will
wait for results from all known keys. This helper implements that
distinction, returning a set() of the relevant keys.
"""
if keys is not _MISSING:
return set(keys)
else:
# keys arg omitted -- use all the keys we know about
return set(six.iterkeys(self.coros)) | set(
six.iterkeys(self.values))
def _wait_each(self, pending):
"""
When _wait_each() encounters a value of PropagateError, it raises it.
In all other respects, _wait_each() behaves like _wait_each_raw().
"""
for key, value in self._wait_each_raw(pending):
yield key, self._value_or_raise(value)
@staticmethod
def _value_or_raise(value):
# Most methods attempting to deliver PropagateError should raise that
# instead of simply returning it.
if isinstance(value, PropagateError):
raise value
return value
def _wait_each_raw(self, pending):
"""
pending is a set() of keys for which we intend to wait. THIS SET WILL
BE DESTRUCTIVELY MODIFIED: as each key acquires a value, that key will
be removed from the passed 'pending' set.
_wait_each_raw() does not treat a PropagateError instance specially:
it will be yielded to the caller like any other value.
In all other respects, _wait_each_raw() behaves like wait_each().
"""
while True:
# Before even waiting, show caller any (key, value) pairs that
# are already available. Copy 'pending' because we want to be able
# to remove items from the original set while iterating.
for key in pending.copy():
value = self.values.get(key, _MISSING)
if value is not _MISSING:
# found one, it's no longer pending
pending.remove(key)
yield (key, value)
if not pending:
# Once we've yielded all the caller's keys, done.
break
# There are still more keys pending, so wait.
self.event.wait()
def spawn(self, key, depends, function, *args, **kwds):
"""
Launch the passed *function(key, results, ...)* as a greenthread,
passing it:
- the specified *key*
- an iterable of (key, value) pairs
- whatever other positional args or keywords you specify.
Iterating over the *results* iterable behaves like calling
:meth:`wait_each(depends) <DAGPool.wait_each>`.
Returning from *function()* behaves like
:meth:`post(key, return_value) <DAGPool.post>`.
If *function()* terminates with an exception, that exception is wrapped
in :class:`PropagateError` with the greenthread's *key* and (effectively) posted
as the value for that key. Attempting to retrieve that value will
raise that PropagateError.
Thus, if the greenthread with key 'a' terminates with an exception,
and greenthread 'b' depends on 'a', when greenthread 'b' attempts to
iterate through its *results* argument, it will encounter
PropagateError. So by default, an uncaught exception will propagate
through all the downstream dependencies.
If you pass :meth:`spawn` a key already passed to spawn() or :meth:`post`, spawn()
raises :class:`Collision`.
"""
if key in self.coros or key in self.values:
raise Collision(key)
# The order is a bit tricky. First construct the set() of keys.
pending = set(depends)
# It's important that we pass to _wait_each() the same 'pending' set()
# that we store in self.coros for this key. The generator-iterator
# returned by _wait_each() becomes the function's 'results' iterable.
newcoro = greenthread.spawn(self._wrapper, function, key,
self._wait_each(pending), *args, **kwds)
# Also capture the same (!) set in the new _Coro object for this key.
# We must be able to observe ready keys being removed from the set.
self.coros[key] = self._Coro(newcoro, pending)
def _wrapper(self, function, key, results, *args, **kwds):
"""
This wrapper runs the top-level function in a DAGPool greenthread,
posting its return value (or PropagateError) to the DAGPool.
"""
try:
# call our passed function
result = function(key, results, *args, **kwds)
except Exception as err:
# Wrap any exception it may raise in a PropagateError.
result = PropagateError(key, err)
finally:
# function() has returned (or terminated with an exception). We no
# longer need to track this greenthread in self.coros. Remove it
# first so post() won't complain about a running greenthread.
del self.coros[key]
try:
# as advertised, try to post() our return value
self.post(key, result)
except Collision:
# if we've already post()ed a result, oh well
pass
# also, in case anyone cares...
return result
def spawn_many(self, depends, function, *args, **kwds):
"""
spawn_many() accepts a single *function* whose parameters are the same
as for :meth:`spawn`.
The difference is that spawn_many() accepts a dependency dict
*depends*. A new greenthread is spawned for each key in the dict. That
dict key's value should be an iterable of other keys on which this
greenthread depends.
If the *depends* dict contains any key already passed to :meth:`spawn`
or :meth:`post`, spawn_many() raises :class:`Collision`. It is
indeterminate how many of the other keys in *depends* will have
successfully spawned greenthreads.
"""
# Iterate over 'depends' items, relying on self.spawn() not to
# context-switch so no one can modify 'depends' along the way.
for key, deps in six.iteritems(depends):
self.spawn(key, deps, function, *args, **kwds)
def kill(self, key):
"""
Kill the greenthread that was spawned with the specified *key*.
If no such greenthread was spawned, raise KeyError.
"""
# let KeyError, if any, propagate
self.coros[key].greenthread.kill()
# once killed, remove it
del self.coros[key]
def post(self, key, value, replace=False):
"""
post(key, value) stores the passed *value* for the passed *key*. It
then causes each greenthread blocked on its results iterable, or on
:meth:`wait_each(keys) <DAGPool.wait_each>`, to check for new values.
A waiting greenthread might not literally resume on every single
post() of a relevant key, but the first post() of a relevant key
ensures that it will resume eventually, and when it does it will catch
up with all relevant post() calls.
Calling post(key, value) when there is a running greenthread with that
same *key* raises :class:`Collision`. If you must post(key, value) instead of
letting the greenthread run to completion, you must first call
:meth:`kill(key) <DAGPool.kill>`.
The DAGPool implicitly post()s the return value from each of its
greenthreads. But a greenthread may explicitly post() a value for its
own key, which will cause its return value to be discarded.
Calling post(key, value, replace=False) (the default *replace*) when a
value for that key has already been posted, by any means, raises
:class:`Collision`.
Calling post(key, value, replace=True) when a value for that key has
already been posted, by any means, replaces the previously-stored
value. However, that may make it complicated to reason about the
behavior of greenthreads waiting on that key.
After a post(key, value1) followed by post(key, value2, replace=True),
it is unspecified which pending :meth:`wait_each([key...]) <DAGPool.wait_each>`
calls (or greenthreads iterating over *results* involving that key)
will observe *value1* versus *value2*. It is guaranteed that
subsequent wait_each([key...]) calls (or greenthreads spawned after
that point) will observe *value2*.
A successful call to
post(key, :class:`PropagateError(key, ExceptionSubclass) <PropagateError>`)
ensures that any subsequent attempt to retrieve that key's value will
raise that PropagateError instance.
"""
# First, check if we're trying to post() to a key with a running
# greenthread.
# A DAGPool greenthread is explicitly permitted to post() to its
# OWN key.
coro = self.coros.get(key, _MISSING)
if coro is not _MISSING and coro.greenthread is not greenthread.getcurrent(
):
# oh oh, trying to post a value for running greenthread from
# some other greenthread
raise Collision(key)
# Here, either we're posting a value for a key with no greenthread or
# we're posting from that greenthread itself.
# Has somebody already post()ed a value for this key?
# Unless replace == True, this is a problem.
if key in self.values and not replace:
raise Collision(key)
# Either we've never before posted a value for this key, or we're
# posting with replace == True.
# update our database
self.values[key] = value
# and wake up pending waiters
self.event.send()
# The comment in Event.reset() says: "it's better to create a new
# event rather than reset an old one". Okay, fine. We do want to be
# able to support new waiters, so create a new Event.
self.event = Event()
def __getitem__(self, key):
"""
__getitem__(key) (aka dagpool[key]) blocks until *key* has a value,
then delivers that value.
"""
# This is a degenerate case of wait_each(). Construct a tuple
# containing only this 'key'. wait_each() will yield exactly one (key,
# value) pair. Return just its value.
for _, value in self.wait_each((key, )):
return value
def get(self, key, default=None):
"""
get() returns the value for *key*. If *key* does not yet have a value,
get() returns *default*.
"""
return self._value_or_raise(self.values.get(key, default))
def keys(self):
"""
Return a snapshot tuple of keys for which we currently have values.
"""
# Explicitly return a copy rather than an iterator: don't assume our
# caller will finish iterating before new values are posted.
return tuple(six.iterkeys(self.values))
def items(self):
"""
Return a snapshot tuple of currently-available (key, value) pairs.
"""
# Don't assume our caller will finish iterating before new values are
# posted.
return tuple((key, self._value_or_raise(value))
for key, value in six.iteritems(self.values))
def running(self):
"""
Return number of running DAGPool greenthreads. This includes
greenthreads blocked while iterating through their *results* iterable,
that is, greenthreads waiting on values from other keys.
"""
return len(self.coros)
def running_keys(self):
"""
Return keys for running DAGPool greenthreads. This includes
greenthreads blocked while iterating through their *results* iterable,
that is, greenthreads waiting on values from other keys.
"""
# return snapshot; don't assume caller will finish iterating before we
# next modify self.coros
return tuple(six.iterkeys(self.coros))
def waiting(self):
"""
Return number of waiting DAGPool greenthreads, that is, greenthreads
still waiting on values from other keys. This explicitly does *not*
include external greenthreads waiting on :meth:`wait`,
:meth:`waitall`, :meth:`wait_each`.
"""
# n.b. if Event would provide a count of its waiters, we could say
# something about external greenthreads as well.
# The logic to determine this count is exactly the same as the general
# waiting_for() call.
return len(self.waiting_for())
# Use _MISSING instead of None as the default 'key' param so we can permit
# None as a supported key.
def waiting_for(self, key=_MISSING):
"""
waiting_for(key) returns a set() of the keys for which the DAGPool
greenthread spawned with that *key* is still waiting. If you pass a
*key* for which no greenthread was spawned, waiting_for() raises
KeyError.
waiting_for() without argument returns a dict. Its keys are the keys
of DAGPool greenthreads still waiting on one or more values. In the
returned dict, the value of each such key is the set of other keys for
which that greenthread is still waiting.
This method allows diagnosing a "hung" DAGPool. If certain
greenthreads are making no progress, it's possible that they are
waiting on keys for which there is no greenthread and no :meth:`post` data.
"""
# We may have greenthreads whose 'pending' entry indicates they're
# waiting on some keys even though values have now been posted for
# some or all of those keys, because those greenthreads have not yet
# regained control since values were posted. So make a point of
# excluding values that are now available.
available = set(six.iterkeys(self.values))
if key is not _MISSING:
# waiting_for(key) is semantically different than waiting_for().
# It's just that they both seem to want the same method name.
coro = self.coros.get(key, _MISSING)
if coro is _MISSING:
# Hmm, no running greenthread with this key. But was there
# EVER a greenthread with this key? If not, let KeyError
# propagate.
self.values[key]
# Oh good, there's a value for this key. Either the
# greenthread finished, or somebody posted a value. Just say
# the greenthread isn't waiting for anything.
return set()
else:
# coro is the _Coro for the running greenthread with the
# specified key.
return coro.pending - available
# This is a waiting_for() call, i.e. a general query rather than for a
# specific key.
# Start by iterating over (key, coro) pairs in self.coros. Generate
# (key, pending) pairs in which 'pending' is the set of keys on which
# the greenthread believes it's waiting, minus the set of keys that
# are now available. Filter out any pair in which 'pending' is empty,
# that is, that greenthread will be unblocked next time it resumes.
# Make a dict from those pairs.
return dict(
(key, pending)
for key, pending in ((key, (coro.pending - available))
for key, coro in six.iteritems(self.coros))
if pending)
def wait():
Event().wait()
def post(self, key, value, replace=False):
"""
post(key, value) stores the passed *value* for the passed *key*. It
then causes each greenthread blocked on its results iterable, or on
:meth:`wait_each(keys) <DAGPool.wait_each>`, to check for new values.
A waiting greenthread might not literally resume on every single
post() of a relevant key, but the first post() of a relevant key
ensures that it will resume eventually, and when it does it will catch
up with all relevant post() calls.
Calling post(key, value) when there is a running greenthread with that
same *key* raises :class:`Collision`. If you must post(key, value) instead of
letting the greenthread run to completion, you must first call
:meth:`kill(key) <DAGPool.kill>`.
The DAGPool implicitly post()s the return value from each of its
greenthreads. But a greenthread may explicitly post() a value for its
own key, which will cause its return value to be discarded.
Calling post(key, value, replace=False) (the default *replace*) when a
value for that key has already been posted, by any means, raises
:class:`Collision`.
Calling post(key, value, replace=True) when a value for that key has
already been posted, by any means, replaces the previously-stored
value. However, that may make it complicated to reason about the
behavior of greenthreads waiting on that key.
After a post(key, value1) followed by post(key, value2, replace=True),
it is unspecified which pending :meth:`wait_each([key...]) <DAGPool.wait_each>`
calls (or greenthreads iterating over *results* involving that key)
will observe *value1* versus *value2*. It is guaranteed that
subsequent wait_each([key...]) calls (or greenthreads spawned after
that point) will observe *value2*.
A successful call to
post(key, :class:`PropagateError(key, ExceptionSubclass) <PropagateError>`)
ensures that any subsequent attempt to retrieve that key's value will
raise that PropagateError instance.
"""
# First, check if we're trying to post() to a key with a running
# greenthread.
# A DAGPool greenthread is explicitly permitted to post() to its
# OWN key.
coro = self.coros.get(key, _MISSING)
if coro is not _MISSING and coro.greenthread is not greenthread.getcurrent(
):
# oh oh, trying to post a value for running greenthread from
# some other greenthread
raise Collision(key)
# Here, either we're posting a value for a key with no greenthread or
# we're posting from that greenthread itself.
# Has somebody already post()ed a value for this key?
# Unless replace == True, this is a problem.
if key in self.values and not replace:
raise Collision(key)
# Either we've never before posted a value for this key, or we're
# posting with replace == True.
# update our database
self.values[key] = value
# and wake up pending waiters
self.event.send()
# The comment in Event.reset() says: "it's better to create a new
# event rather than reset an old one". Okay, fine. We do want to be
# able to support new waiters, so create a new Event.
self.event = Event()
def waiter():
return Event()
def __init__(self, maxsize=None):
LightQueue.__init__(self, maxsize)
self.unfinished_tasks = 0
self._cond = Event()
def block(*args):
Event().wait()
def expect(self, n):
assert(len(self.received) == self.expected)
self.received = []
self.expected = n
self.event = Event()
return self
def test_disconnect_with_pending_reply(container_factory, rabbit_manager,
rabbit_config):
block = Event()
class ExampleService(object):
name = "exampleservice"
def hook(self):
pass # pragma: no cover
@rpc
def method(self, arg):
self.hook()
block.wait()
return arg
container = container_factory(ExampleService, rabbit_config)
container.start()
vhost = rabbit_config['vhost']
# get exampleservice's queue consumer connection while we know it's the
# only active connection
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
container_connection = connections[0]
with ServiceRpcProxy('exampleservice', rabbit_config) as proxy:
# grab the proxy's connection too, the only other connection
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
proxy_connection = [
conn for conn in connections if conn != container_connection
][0]
counter = itertools.count(start=1)
class ConnectionStillOpen(Exception):
pass
@retry(for_exceptions=ConnectionStillOpen, delay=0.2)
def wait_for_connection_close(name):
connections = get_rabbit_connections(vhost, rabbit_manager)
for conn in connections:
if conn['name'] == name:
raise ConnectionStillOpen(name) # pragma: no cover
def cb(args, kwargs, res, exc_info):
# trigger a disconnection on the second call.
# release running workers once the connection has been closed
count = next(counter)
if count == 2:
rabbit_manager.delete_connection(proxy_connection['name'])
wait_for_connection_close(proxy_connection['name'])
block.send(True)
return True
# attach a callback to `hook` so we can close the connection
# while there are requests in-flight
with wait_for_call(ExampleService, 'hook', callback=cb):
# make an async call that runs for some time
async_call = proxy.method.call_async("hello")
# make another call that will trigger the disconnection;
# expect the blocking proxy to raise when the service reconnects
with pytest.raises(RpcConnectionError):
proxy.method("hello")
# also expect the running call to raise, since the reply may have
# been sent while the queue was gone (deleted on disconnect, and
# not added until re-connect)
with pytest.raises(RpcConnectionError):
async_call.result()
# proxy should work again afterwards
assert proxy.method("hello") == "hello"
def make_virtual_socket(host, port, path='/ws'):
from websocket import WebSocketApp
result_handlers = {}
class Socket(object):
def __init__(self):
self._event_queues = defaultdict(Queue)
def get_event_queue(self, event_type):
return self._event_queues[event_type]
def wait_for_event(self, event_type):
return self.get_event_queue(event_type).get()
def rpc(self, _method, **data):
id = str(uuid.uuid4())
event = Event()
result_handlers[id] = event.send
ws_app.send(
json.dumps({
'method': _method,
'data': data,
'correlation_id': id,
}))
rv = event.wait()
if rv['success']:
return rv['data']
raise deserialize(rv['error'])
sock = Socket()
def on_message(ws, message):
msg = json.loads(message)
if msg['type'] == 'event':
sock.get_event_queue(msg['event']).put((msg['event'], msg['data']))
elif msg['type'] == 'result':
result_id = msg['correlation_id']
handler = result_handlers.pop(result_id, None)
if handler is not None:
handler(msg)
ready_event = Event()
def on_open(ws):
ready_event.send(None)
def on_error(ws, err):
ready_event.send(err)
ws_app = WebSocketApp(
'ws://%s:%d%s' % (host, port, path),
on_message=on_message,
on_open=on_open,
on_error=on_error,
)
def connect_socket():
err = ready_event.wait()
if err is not None:
raise err
return sock
return ws_app, connect_socket
class Timer(Entrypoint):
def __init__(self, interval):
"""
Timer entrypoint implementation. Fires every :attr:`self.interval`
seconds.
The implementation sleeps first, i.e. does not fire at time 0.
Example::
timer = Timer.decorator
class Service(object):
name = "service"
@timer(interval=5)
def tick(self):
pass
"""
self.interval = interval
self.should_stop = Event()
self.gt = None
def start(self):
_log.debug('starting %s', self)
self.gt = self.container.spawn_managed_thread(self._run)
def stop(self):
_log.debug('stopping %s', self)
self.should_stop.send(True)
self.gt.wait()
def kill(self):
_log.debug('killing %s', self)
self.gt.kill()
def _run(self):
""" Runs the interval loop. """
sleep_time = self.interval
while True:
# sleep for `sleep_time`, unless `should_stop` fires, in which
# case we leave the while loop and stop entirely
with Timeout(sleep_time, exception=False):
self.should_stop.wait()
break
start = time.time()
self.handle_timer_tick()
elapsed_time = (time.time() - start)
# next time, sleep however long is left of our interval, taking
# off the time we took to run
sleep_time = max(self.interval - elapsed_time, 0)
def handle_timer_tick(self):
args = ()
kwargs = {}
# Note that we don't catch ContainerBeingKilled here. If that's raised,
# there is nothing for us to do anyway. The exception bubbles, and is
# caught by :meth:`Container._handle_thread_exited`, though the
# triggered `kill` is a no-op, since the container is alredy
# `_being_killed`.
self.container.spawn_worker(self, args, kwargs)
def test_prefetch_count(rabbit_manager, rabbit_config, mock_container):
container = mock_container
container.shared_extensions = {}
container.config = rabbit_config
container.max_workers = 1
container.spawn_managed_thread = spawn_managed_thread
content_type = 'application/data'
container.accept = [content_type]
class NonShared(QueueConsumer):
@property
def sharing_key(self):
return uuid.uuid4()
queue_consumer1 = NonShared().bind(container)
queue_consumer1.setup()
queue_consumer2 = NonShared().bind(container)
queue_consumer2.setup()
consumer_continue = Event()
class Handler1(object):
queue = ham_queue
def handle_message(self, body, message):
consumer_continue.wait()
queue_consumer1.ack_message(message)
messages = []
class Handler2(object):
queue = ham_queue
def handle_message(self, body, message):
messages.append(body)
queue_consumer2.ack_message(message)
handler1 = Handler1()
handler2 = Handler2()
queue_consumer1.register_provider(handler1)
queue_consumer2.register_provider(handler2)
queue_consumer1.start()
queue_consumer2.start()
vhost = rabbit_config['vhost']
# the first consumer only has a prefetch_count of 1 and will only
# consume 1 message and wait in handler1()
rabbit_manager.publish(vhost,
'spam',
'',
'ham',
properties=dict(content_type=content_type))
# the next message will go to handler2() no matter of any prefetch_count
rabbit_manager.publish(vhost,
'spam',
'',
'eggs',
properties=dict(content_type=content_type))
# the third message is only going to handler2 because the first consumer
# has a prefetch_count of 1 and thus is unable to deal with another message
# until having ACKed the first one
rabbit_manager.publish(vhost,
'spam',
'',
'bacon',
properties=dict(content_type=content_type))
with eventlet.Timeout(TIMEOUT):
while len(messages) < 2:
eventlet.sleep()
# allow the waiting consumer to ack its message
consumer_continue.send(None)
assert messages == ['eggs', 'bacon']
queue_consumer1.unregister_provider(handler1)
queue_consumer2.unregister_provider(handler2)
queue_consumer1.kill()
queue_consumer2.kill()
def __init__(self):
self.results = []
self.stop = Event()
self.no_more_results = Event()
self.failure = None
def wait_forever():
try:
Event().wait()
except:
killed_by_error_raised.send()
raise
class QueueConsumer(SharedExtension, ProviderCollector, ConsumerMixin):
amqp_uri = None
prefetch_count = None
def __init__(self):
self._connection = None
self._consumers = {}
self._pending_messages = set()
self._pending_ack_messages = []
self._pending_requeue_messages = []
self._pending_remove_providers = {}
self._gt = None
self._starting = False
self._consumers_ready = Event()
super(QueueConsumer, self).__init__()
def _handle_thread_exited(self, gt):
exc = None
try:
gt.wait()
except Exception as e:
exc = e
if not self._consumers_ready.ready():
self._consumers_ready.send_exception(exc)
def setup(self):
self.amqp_uri = self.container.config[AMQP_URI_CONFIG_KEY]
self.accept = self.container.accept
self.prefetch_count = self.container.max_workers
verify_amqp_uri(self.amqp_uri)
def start(self):
if not self._starting:
self._starting = True
_log.debug('starting %s', self)
self._gt = self.container.spawn_managed_thread(self.run)
self._gt.link(self._handle_thread_exited)
try:
_log.debug('waiting for consumer ready %s', self)
self._consumers_ready.wait()
except QueueConsumerStopped:
_log.debug('consumer was stopped before it started %s', self)
except Exception as exc:
_log.debug('consumer failed to start %s (%s)', self, exc)
else:
_log.debug('started %s', self)
def stop(self):
""" Stop the queue-consumer gracefully.
Wait until the last provider has been unregistered and for
the ConsumerMixin's greenthread to exit (i.e. until all pending
messages have been acked or requeued and all consumers stopped).
"""
if not self._consumers_ready.ready():
_log.debug('stopping while consumer is starting %s', self)
stop_exc = QueueConsumerStopped()
# stopping before we have started successfully by brutally
# killing the consumer thread as we don't have a way to hook
# into the pre-consumption startup process
self._gt.kill(stop_exc)
self.wait_for_providers()
try:
_log.debug('waiting for consumer death %s', self)
self._gt.wait()
except QueueConsumerStopped:
pass
super(QueueConsumer, self).stop()
_log.debug('stopped %s', self)
def kill(self):
""" Kill the queue-consumer.
Unlike `stop()` any pending message ack or requeue-requests,
requests to remove providers, etc are lost and the consume thread is
asked to terminate as soon as possible.
"""
# greenlet has a magic attribute ``dead`` - pylint: disable=E1101
if self._gt is not None and not self._gt.dead:
# we can't just kill the thread because we have to give
# ConsumerMixin a chance to close the sockets properly.
self._providers = set()
self._pending_messages = set()
self._pending_ack_messages = []
self._pending_requeue_messages = []
self._pending_remove_providers = {}
self.should_stop = True
try:
self._gt.wait()
except Exception as exc:
# discard the exception since we're already being killed
_log.warn(
'QueueConsumer %s raised `%s` during kill', self, exc)
super(QueueConsumer, self).kill()
_log.debug('killed %s', self)
def unregister_provider(self, provider):
if not self._consumers_ready.ready():
# we cannot handle the situation where we are starting up and
# want to remove a consumer at the same time
# TODO: With the upcomming error handling mechanism, this needs
# TODO: to be thought through again.
self._last_provider_unregistered.send()
return
removed_event = Event()
# we can only cancel a consumer from within the consumer thread
self._pending_remove_providers[provider] = removed_event
# so we will just register the consumer to be canceled
removed_event.wait()
super(QueueConsumer, self).unregister_provider(provider)
def ack_message(self, message):
_log.debug("stashing message-ack: %s", message)
self._pending_messages.remove(message)
self._pending_ack_messages.append(message)
def requeue_message(self, message):
_log.debug("stashing message-requeue: %s", message)
self._pending_messages.remove(message)
self._pending_requeue_messages.append(message)
def _on_message(self, body, message):
_log.debug("received message: %s", message)
self._pending_messages.add(message)
def _cancel_consumers_if_requested(self):
provider_remove_events = self._pending_remove_providers.items()
self._pending_remove_providers = {}
for provider, removed_event in provider_remove_events:
consumer = self._consumers.pop(provider)
_log.debug('cancelling consumer [%s]: %s', provider, consumer)
consumer.cancel()
removed_event.send()
def _process_pending_message_acks(self):
messages = self._pending_ack_messages
if messages:
_log.debug('ack() %d processed messages', len(messages))
while messages:
msg = messages.pop()
msg.ack()
eventlet.sleep()
messages = self._pending_requeue_messages
if messages:
_log.debug('requeue() %d processed messages', len(messages))
while messages:
msg = messages.pop()
msg.requeue()
eventlet.sleep()
@property
def connection(self):
""" Kombu requirement """
if self.amqp_uri is None:
return # don't cache a connection during introspection
if self._connection is None:
heartbeat = self.container.config.get(
HEARTBEAT_CONFIG_KEY, DEFAULT_HEARTBEAT
)
self._connection = Connection(self.amqp_uri, heartbeat=heartbeat)
return self._connection
def get_consumers(self, Consumer, channel):
""" Kombu callback to set up consumers.
Called after any (re)connection to the broker.
"""
_log.debug('setting up consumers %s', self)
for provider in self._providers:
callbacks = [self._on_message, provider.handle_message]
consumer = Consumer(queues=[provider.queue], callbacks=callbacks,
accept=self.accept)
consumer.qos(prefetch_count=self.prefetch_count)
self._consumers[provider] = consumer
return self._consumers.values()
def on_iteration(self):
""" Kombu callback for each `drain_events` loop iteration."""
self._cancel_consumers_if_requested()
self._process_pending_message_acks()
num_consumers = len(self._consumers)
num_pending_messages = len(self._pending_messages)
if num_consumers + num_pending_messages == 0:
_log.debug('requesting stop after iteration')
self.should_stop = True
def on_connection_error(self, exc, interval):
_log.warn(
"Error connecting to broker at {} ({}).\n"
"Retrying in {} seconds.".format(self.amqp_uri, exc, interval))
def on_consume_ready(self, connection, channel, consumers, **kwargs):
""" Kombu callback when consumers are ready to accept messages.
Called after any (re)connection to the broker.
"""
if not self._consumers_ready.ready():
_log.debug('consumer started %s', self)
self._consumers_ready.send(None)
for provider in self._providers:
try:
callback = provider.on_consume_ready
except AttributeError:
pass
else:
callback()
def consume(self, limit=None, timeout=None, safety_interval=1, **kwargs):
""" Lifted from kombu
We switch the order of the `break` and `self.on_iteration()` to
avoid waiting on a drain_events timeout before breaking the loop.
"""
elapsed = 0
with self.consumer_context(**kwargs) as (conn, channel, consumers):
for i in limit and range(limit) or count():
self.on_iteration()
if self.should_stop:
break
try:
conn.drain_events(timeout=safety_interval)
except socket.timeout:
conn.heartbeat_check()
elapsed += safety_interval
if timeout and elapsed >= timeout:
# Excluding the following clause from coverage,
# as timeout never appears to be set - This method
# is a lift from kombu so will leave in place for now.
raise # pragma: no cover
except socket.error:
if not self.should_stop:
raise
else:
yield
elapsed = 0
