def test_any_two_terminal_functions_raise_error(self):
for term_a, term_b in permutations(self.terminals, 2):
proxy = AMQPProxy(self.channel, self.msg)
term_func_a = getattr(proxy, term_a)
term_func_b = getattr(proxy, term_b)
term_func_a()
self.assertRaises(Exception, term_func_b)
def func():
consumer = self._pool.get()
amqp_proxy = AMQPProxy(self._channel, msg)
def put_back(successful_greenlet):
self._logger.debug('Successful run, putting consumer back')
if not amqp_proxy.has_responded_to_message:
amqp_proxy.ack()
self._pool.put(consumer)
def recreate(failed_greenlet):
self._logger.info('Consume failed, shutting down consumer')
if not amqp_proxy.has_responded_to_message:
amqp_proxy.reject(requeue=True)
shutdown_greenlet = gevent.Greenlet(consumer.shutdown,
failed_greenlet.exception)
def create_wrapper(*args):
self._create()
shutdown_greenlet.link(create_wrapper)
shutdown_greenlet.start()
greenlet = self._gm(consumer.consume, amqp_proxy, msg)
greenlet.link_value(put_back)
greenlet.link_exception(recreate)
greenlet.start()
async def test_any_two_terminal_functions_raise_error():
(connection, channel, message) = create_mocks()
for term_a, term_b in permutations(TERMINALS, 2):
proxy = AMQPProxy(connection, channel, message)
term_func_a = getattr(proxy, term_a)
term_func_b = getattr(proxy, term_b)
await term_func_a()
with pytest.raises(Exception):
await term_func_b()
async def test_terminal_twice_raises_error():
(connection, channel, message) = create_mocks()
for terminal in TERMINALS:
proxy = AMQPProxy(connection, channel, message)
term_func = getattr(proxy, terminal)
await term_func()
with pytest.raises(Exception):
await term_func()
async def create_consumption_task(connection: TrulyRobustConnection,
consumer: Dict[Any, Any],
connection_name: str) -> None:
"""
A consumption task fulfills a specification for a consumer entry in the
consumers section of the YAML. Note that a consumption task may specify that
multiple consumers be used. The consumer pool is handled by the
consumer loop.
:param connection:
:param consumer:
:param created_queues:
:param connection_name:
:return:
"""
logger = logging.getLogger("amqp-dispatcher")
queue_name = consumer["queue"]
prefetch_count = consumer.get("prefetch_count", 1)
consumer_str = consumer.get("consumer", "")
consumer_count = consumer.get("consumer_count", 1)
consumer_class = load_consumer(consumer_str)
# Consumers can use the AMQP proxy to publish messages. This
# is a dedicated channel for that purpose
publish_channel: Channel = await connection.channel()
# We should have one channel for each group of consumers.
consume_channel: Channel = await connection.channel()
await consume_channel.set_qos(prefetch_count=prefetch_count)
queue = Queue(
connection=connection,
channel=consume_channel.channel,
name=queue_name,
durable=None,
exclusive=None,
auto_delete=None,
arguments=None,
)
# Create a pool of consumers that can be used to
# process incoming messages
consumer_pool: asyncio.Queue = asyncio.Queue(maxsize=consumer_count)
for i in range(0, consumer_count):
await consumer_pool.put(consumer_class())
logger.info("all consumers of class {0} created".format(
consumer_class.__name__))
random_generator = random.SystemRandom()
random_string = "".join(
[random_generator.choice(string.ascii_lowercase) for _ in range(10)])
consumer_tag = "{0}-[{1}]-{2}".format(connection_name, consumer_str,
random_string)
async with queue.iterator(consumer_tag=consumer_tag) as queue_iterator:
async for message in queue_iterator:
# ignore_processed=True allows us to manually acknowledge
# and reject messages without the context manager
# making the decisions for us.
processed_message: IncomingMessage = message
logger.info("a message was received with delivery tag: {0}".format(
processed_message.delivery_tag))
wrapped_message = Message(processed_message)
amqp_proxy = AMQPProxy(connection, publish_channel,
wrapped_message)
# We do not want the processing of a single message to block other available
# consumers in the pool from being able to process other incoming messages.
# Therefore, we schedule this without blocking control flow.
# The consumption coroutine is responsible for putting the
# consumer instance back on the queue when it is done.
asyncio.ensure_future(
consumption_coroutine(
consumer_pool,
amqp_proxy,
wrapped_message,
connection.consumer_completion_group,
))
def test_terminal_state_true_after_responding(self):
proxy = AMQPProxy(self.channel, self.msg)
proxy.ack()
self.assertTrue(proxy.has_responded_to_message)
def test_terminal_state_false_when_starting(self):
proxy = AMQPProxy(self.channel, self.msg)
self.assertFalse(proxy.has_responded_to_message)
def test_terminal_twice_raises_error(self):
for terminal in self.terminals:
proxy = AMQPProxy(self.channel, self.msg)
term_func = getattr(proxy, terminal)
term_func()
self.assertRaises(Exception, term_func)
def test_terminal_state_true_after_responding(self):
proxy = AMQPProxy(self.channel, self.msg)
proxy.ack()
self.assertTrue(proxy.has_responded_to_message)
async def test_terminal_state_true_after_responding():
(connection, channel, message) = create_mocks()
proxy = AMQPProxy(connection, channel, message)
await proxy.ack()
assert proxy.has_responded_to_message is True
def test_terminal_state_false_when_starting():
(connection, channel, message) = create_mocks()
proxy = AMQPProxy(connection, channel, message)
assert proxy.has_responded_to_message is False