async def _create_consumer_groups(self, streams, redis, consumer_group):
"""Ensure the consumer groups exist
This is means we have to ensure the streams exist too
"""
for stream, since in streams.items():
if not await redis.exists(stream):
# Add a noop to ensure the stream exists
# TODO: We can now use MKSTREAM, change this logic
# Documented here: https://redis.io/topics/streams-intro
await redis.xadd(stream, fields={"": ""})
try:
# Create the group (it may already exist)
await redis.xgroup_create(stream, consumer_group, latest_id=since)
except ReplyError as e:
if "BUSYGROUP" in str(e):
# Already exists
pass
else:
raise
async def consume(
self,
listen_for: List[Tuple[str, str]],
listener_name: str,
error_queue: ErrorQueueType,
since: Union[Since, Sequence[Since]] = "$",
forever=True,
) -> AsyncGenerator[List[RedisEventMessage], None]:
"""Consume events for the given APIs"""
self._sanity_check_listen_for(listen_for)
consumer_group = f"{self.service_name}-{listener_name}"
if not isinstance(since, (list, tuple)):
# Since has been specified as a single value. Normalise it into
# the value-per-listener format.
since = [since] * len(listen_for)
since = map(normalise_since_value, since)
stream_names = self._get_stream_names(listen_for)
# Keys are stream names, values as the latest ID consumed from that stream
streams = OrderedDict(zip(stream_names, since))
expected_events = {event_name for _, event_name in listen_for}
logger.debug(
LBullets(
L(
"Consuming events as consumer {} in group {} on streams",
Bold(self.consumer_name),
Bold(consumer_group),
),
items={"{} ({})".format(*v)
for v in streams.items()},
))
# Cleanup any old groups & consumers
await self._cleanup(stream_names)
# Here we use a queue to combine messages coming from both the
# fetch messages loop and the reclaim messages loop.
queue = InternalQueue(maxsize=1)
initial_reclaiming_complete = asyncio.Event()
async def consume_loop():
"""Regular event consuming. See _fetch_new_messages()"""
logger.debug(
"Will begin consuming events once the initial event reclaiming is complete"
)
await initial_reclaiming_complete.wait()
logger.debug(
"Event reclaiming is complete, beginning to consume events")
async for messages in self._fetch_new_messages(
streams, consumer_group, expected_events, forever):
await queue.put(messages)
# Wait for the queue to empty before getting trying to get another message
await queue.join()
retry_consume_loop = retry_on_redis_connection_failure(
fn=consume_loop,
retry_delay=self.consumption_restart_delay,
action="consuming events")
async def reclaim_loop():
"""
Reclaim messages which other consumers have failed to
processes in reasonable time. See _reclaim_lost_messages()
"""
while True:
logger.debug("Checking for any events which need reclaiming")
async for messages in self._reclaim_lost_messages(
stream_names, consumer_group, expected_events):
await queue.put(messages)
# Wait for the queue to empty before getting trying to get another message
await queue.join()
initial_reclaiming_complete.set()
await asyncio.sleep(self.reclaim_interval)
consume_task = None
reclaim_task = None
try:
# Run the two above coroutines in their own tasks
consume_task = asyncio.ensure_future(
queue_exception_checker(retry_consume_loop, error_queue))
reclaim_task = asyncio.ensure_future(
queue_exception_checker(reclaim_loop(), error_queue))
while True:
try:
messages = await queue.get()
logger.debug(
f"Got batch of {len(messages)} message(s). Yielding messages to Lightbus"
" client")
yield messages
logger.debug(
f"Batch of {len(messages)} message(s) was processed by Lightbus client."
" Marking as done.")
queue.task_done()
except GeneratorExit:
return
finally:
# Make sure we cleanup the tasks we created
await cancel(consume_task, reclaim_task)