def direct_send(self, msg_id, msg):
"""Send a 'direct' message."""
timer = rpc_common.DecayingTimer(duration=60)
timer.start()
# NOTE(sileht): retry at least 60sec, after we have a good change
# that the caller is really dead too...
while True:
try:
self.publisher_send(DirectPublisher, msg_id, msg)
except self.connection.channel_errors as exc:
# NOTE(noelbk/sileht):
# If rabbit dies, the consumer can be disconnected before the
# publisher sends, and if the consumer hasn't declared the
# queue, the publisher's will send a message to an exchange
# that's not bound to a queue, and the message wll be lost.
# So we set passive=True to the publisher exchange and catch
# the 404 kombu ChannelError and retry until the exchange
# appears
if exc.code == 404 and timer.check_return() > 0:
LOG.info(
_("The exchange to reply to %s doesn't "
"exist yet, retrying...") % msg_id)
time.sleep(1)
continue
raise
return
def test_duration_callback(self):
t = common.DecayingTimer(2)
t._ends_at = time.time() - 10
callback = mock.Mock()
remaining = t.check_return(callback)
self.assertAlmostEqual(-10, remaining, 0)
callback.assert_called_once
def wait(self, msg_id, timeout):
#
# NOTE(markmc): we're waiting for a reply for msg_id to come in for on
# the reply_q, but there may be other threads also waiting for replies
# to other msg_ids
#
# Only one thread can be consuming from the queue using this connection
# and we don't want to hold open a connection per thread, so instead we
# have the first thread take responsibility for passing replies not
# intended for itself to the appropriate thread.
#
timer = rpc_common.DecayingTimer(duration=timeout)
timer.start()
final_reply = None
while True:
if self.conn_lock.acquire(False):
# Ok, we're the thread responsible for polling the connection
try:
# Check the queue to see if a previous lock-holding thread
# queued up a reply already
while True:
reply, ending, empty = self._check_queue(msg_id)
if empty:
break
if not ending:
final_reply = reply
else:
return final_reply
# Now actually poll the connection
while True:
reply, ending = self._poll_connection(msg_id, timer)
if not ending:
final_reply = reply
else:
return final_reply
finally:
self.conn_lock.release()
# We've got our reply, tell the other threads to wake up
# so that one of them will take over the responsibility for
# polling the connection
self.waiters.wake_all(msg_id)
else:
# We're going to wait for the first thread to pass us our reply
reply, ending, trylock = self._poll_queue(msg_id, timer)
if trylock:
# The first thread got its reply, let's try and take over
# the responsibility for polling
continue
if not ending:
final_reply = reply
else:
return final_reply
def iterconsume(self, limit=None, timeout=None):
"""Return an iterator that will consume from all queues/consumers."""
timer = rpc_common.DecayingTimer(duration=timeout).start()
def _raise_timeout(exc):
LOG.debug('Timed out waiting for RPC response: %s', exc)
raise rpc_common.Timeout()
def _error_callback(exc):
timer.check_return(_raise_timeout, exc)
LOG.exception(_('Failed to consume message from queue: %s'), exc)
def _consume():
# NOTE(sileht):
# maximun value choosen according the best practice from kombu:
# http://kombu.readthedocs.org/en/latest/reference/kombu.common.html#kombu.common.eventloop
poll_timeout = 1 if timeout is None else min(timeout, 1)
while True:
if self._consume_loop_stopped:
self._consume_loop_stopped = False
raise StopIteration
try:
nxt_receiver = self.session.next_receiver(
timeout=poll_timeout)
except qpid_exceptions.Empty as exc:
poll_timeout = timer.check_return(_raise_timeout,
exc,
maximum=1)
else:
break
try:
self._lookup_consumer(nxt_receiver).consume()
except Exception:
LOG.exception(_("Error processing message. " "Skipping it."))
for iteration in itertools.count(0):
if limit and iteration >= limit:
raise StopIteration
yield self.ensure(_error_callback, _consume)
def iterconsume(self, limit=None, timeout=None):
"""Return an iterator that will consume from all queues/consumers."""
timer = rpc_common.DecayingTimer(duration=timeout).start()
def _raise_timeout(exc):
LOG.debug('Timed out waiting for RPC response: %s', exc)
raise rpc_common.Timeout()
def _error_callback(exc):
self.do_consume = True
timer.check_return(_raise_timeout, exc)
LOG.exception(_('Failed to consume message from queue: %s'),
exc)
def _consume(channel):
if self.do_consume:
queues_head = self.consumers[:-1] # not fanout.
queues_tail = self.consumers[-1] # fanout
for queue in queues_head:
queue.consume(nowait=True)
queues_tail.consume(nowait=False)
self.do_consume = False
# NOTE(sileht):
# maximun value choosen according the best practice from kombu:
# http://kombu.readthedocs.org/en/latest/reference/kombu.common.html#kombu.common.eventloop
poll_timeout = 1 if timeout is None else min(timeout, 1)
while True:
if self._consume_loop_stopped:
self._consume_loop_stopped = False
raise StopIteration
try:
return self.connection.drain_events(timeout=poll_timeout)
except socket.timeout as exc:
poll_timeout = timer.check_return(_raise_timeout, exc,
maximum=1)
for iteration in itertools.count(0):
if limit and iteration >= limit:
raise StopIteration
yield self.ensure(_error_callback, _consume)
def iterconsume(self, limit=None, timeout=None):
"""Return an iterator that will consume from all queues/consumers."""
timer = rpc_common.DecayingTimer(duration=timeout)
timer.start()
def _raise_timeout(exc):
LOG.debug('Timed out waiting for RPC response: %s', exc)
raise rpc_common.Timeout()
def _error_callback(exc):
timer.check_return(_raise_timeout, exc)
LOG.exception(_('Failed to consume message from queue: %s'), exc)
self.do_consume = True
def _consume():
if self.do_consume:
queues_head = self.consumers[:-1] # not fanout.
queues_tail = self.consumers[-1] # fanout
for queue in queues_head:
queue.consume(nowait=True)
queues_tail.consume(nowait=False)
self.do_consume = False
poll_timeout = 1 if timeout is None else min(timeout, 1)
while True:
try:
return self.connection.drain_events(timeout=poll_timeout)
except socket.timeout as exc:
poll_timeout = timer.check_return(_raise_timeout,
exc,
maximum=1)
for iteration in itertools.count(0):
if limit and iteration >= limit:
raise StopIteration
yield self.ensure(_error_callback, _consume)
def test_duration_expired_no_callback(self):
t = common.DecayingTimer(2)
t._ends_at = time.time() - 10
remaining = t.check_return()
self.assertAlmostEqual(-10, remaining, 0)
def test_no_duration_but_maximun(self):
t = common.DecayingTimer()
t.start()
remaining = t.check_return(maximum=2)
self.assertEqual(2, remaining)
def test_no_duration_no_callback(self):
t = common.DecayingTimer()
t.start()
remaining = t.check_return()
self.assertEqual(None, remaining)