async def consume_rpcs(self, apis: Sequence[Api]) -> Sequence[RpcMessage]:
# Get the name of each stream
streams = ['{}:stream'.format(api.meta.name) for api in apis]
# Get where we last left off in each stream
latest_ids = [self._latest_ids.get(stream, '$') for stream in streams]
logger.debug(LBullets(
'Consuming RPCs from', items=[
'{} ({})'.format(s, self._latest_ids.get(s, '$')) for s in streams
]
))
pool = await self.get_redis_pool()
with await pool as redis:
# TODO: Count/timeout configurable
stream_messages = await redis.xread(streams, latest_ids=latest_ids, count=10)
rpc_messages = []
for stream, message_id, fields in stream_messages:
stream = decode(stream, 'utf8')
message_id = decode(message_id, 'utf8')
decoded_fields = decode_message_fields(fields)
# See comment on events transport re updating message_id
self._latest_ids[stream] = message_id
rpc_messages.append(
RpcMessage.from_dict(decoded_fields)
)
logger.debug(LBullets(
L("⬅ Received message {} on stream {}", Bold(message_id), Bold(stream)),
items=decoded_fields
))
return rpc_messages
async def _reclaim_lost_messages(self, stream_names: List[str],
consumer_group: str,
expected_events: set):
"""Reclaim messages that other consumers in the group failed to acknowledge"""
with await self.connection_manager() as redis:
for stream in stream_names:
old_messages = await redis.xpending(
stream,
consumer_group,
"-",
"+",
count=self.reclaim_batch_size)
timeout = self.acknowledgement_timeout * 1000
for (
message_id,
consumer_name,
ms_since_last_delivery,
num_deliveries,
) in old_messages:
message_id = decode(message_id, "utf8")
consumer_name = decode(consumer_name, "utf8")
if ms_since_last_delivery > timeout:
logger.info(
L(
"Found timed out event {} in stream {}. Abandoned by {}. Attempting to reclaim...",
Bold(message_id),
Bold(stream),
Bold(consumer_name),
))
result = await redis.xclaim(stream, consumer_group,
self.consumer_name,
int(timeout), message_id)
for claimed_message_id, fields in result:
claimed_message_id = decode(claimed_message_id, "utf8")
event_message = self._fields_to_message(
fields,
expected_events,
native_id=claimed_message_id)
if not event_message:
# noop message, or message an event we don't care about
continue
logger.debug(
LBullets(
L(
"⬅ Reclaimed timed out event {} on stream {}. Abandoned by {}.",
Bold(message_id),
Bold(stream),
Bold(consumer_name),
),
items=dict(
**event_message.get_metadata(),
kwargs=event_message.get_kwargs(),
),
))
yield event_message, stream
async def _execute_node(self, pool, command, *args, **kwargs):
"""Execute redis command and returns Future waiting for the answer.
:param command str
:param pool obj
Raises:
* TypeError if any of args can not be encoded as bytes.
* ReplyError on redis '-ERR' responses.
* ProtocolError when response can not be decoded meaning connection
is broken.
"""
cmd = decode(command, 'utf-8').lower()
try:
with await pool as conn:
return await getattr(conn, cmd)(*args, **kwargs)
except ReplyError as err:
address = parse_moved_response_error(err)
if address is None:
raise
logger.debug('Got MOVED command: {}'.format(err))
self._moved_count += 1
if self._moved_count >= self.MAX_MOVED_COUNT:
await self.initialize()
pool = self.get_node(command, *args, **kwargs)
with await pool as conn:
return await getattr(conn, cmd)(*args, **kwargs)
else:
conn = await self.create_connection(address)
res = await getattr(conn, cmd)(*args, **kwargs)
conn.close()
await conn.wait_closed()
return res
def redis_steam_id_to_datetime(message_id):
message_id = decode(message_id, "utf8")
milliseconds, seq = map(int, message_id.split("-"))
# Treat the sequence value as additional microseconds to ensure correct sequencing
microseconds = (milliseconds % 1000 * 1000) + seq
dt = datetime.utcfromtimestamp(milliseconds // 1000).replace(microsecond=microseconds)
return dt
async def _execute_node(self, pool, command, *args, **kwargs):
"""Execute redis command and returns Future waiting for the answer.
:param command str
:param pool obj
Raises:
* TypeError if any of args can not be encoded as bytes.
* ReplyError on redis '-ERR' responses.
* ProtocolError when response can not be decoded meaning connection
is broken.
"""
cmd = decode(command, 'utf-8').lower()
try:
with await pool as conn:
return await getattr(conn, cmd)(*args, **kwargs)
except ReplyError as err:
address = parse_moved_response_error(err)
if address is None:
raise
logger.debug('Got MOVED command: {}'.format(err))
self._moved_count += 1
if self._moved_count >= self.MAX_MOVED_COUNT:
await self.initialize()
pool = self.get_node(command, *args, **kwargs)
with await pool as conn:
return await getattr(conn, cmd)(*args, **kwargs)
else:
conn = await self.create_connection(address)
res = await getattr(conn, cmd)(*args, **kwargs)
conn.close()
await conn.wait_closed()
return res
async def history(
self,
api_name,
event_name,
start: datetime = None,
stop: datetime = None,
start_inclusive: bool = True,
batch_size: int = 100,
) -> AsyncGenerator[EventMessage, None]:
"""Retrieve historical events for the given API
Will not have any impact on existing consumer groups.
"""
redis_start = datetime_to_redis_steam_id(start) if start else "-"
redis_stop = datetime_to_redis_steam_id(stop) if stop else "+"
if start and not start_inclusive:
redis_start = redis_stream_id_add_one(redis_start)
stream_name = self._get_stream_names([(api_name, event_name)])[0]
logger.debug(
f"Getting history for stream {stream_name} from {redis_start} ({start}) "
f"to {redis_stop} ({stop}) in batches of {batch_size}"
)
with await self.connection_manager() as redis:
messages = True
while messages:
messages = await redis.xrevrange(
stream_name, redis_stop, redis_start, count=batch_size
)
if not messages:
return
for message_id, fields in messages:
message_id = decode(message_id, "utf8")
redis_stop = redis_stream_id_subtract_one(message_id)
try:
event_message = self._fields_to_message(
fields,
expected_event_names={event_name},
stream=stream_name,
native_id=message_id,
consumer_group=None,
)
except (NoopMessage, IgnoreMessage):
logger.debug(
f"Ignoring NOOP event with ID {message_id} discovered during fetching"
" of event history"
)
else:
yield event_message
async def _consume_rpcs(self, apis: Sequence[Api]) -> Sequence[RpcMessage]:
# Get the name of each list queue
queue_keys = ["{}:rpc_queue".format(api.meta.name) for api in apis]
logger.debug(
LBullets(
"Consuming RPCs from",
items=[
"{} ({})".format(s, self._latest_ids.get(s, "$"))
for s in queue_keys
],
))
with await self.connection_manager() as redis:
try:
try:
stream, data = await redis.blpop(*queue_keys)
except RuntimeError:
# For some reason aio-redis likes to eat the CancelledError and
# turn it into a Runtime error:
# https://github.com/aio-libs/aioredis/blob/9f5964/aioredis/connection.py#L184
raise asyncio.CancelledError(
"aio-redis task was cancelled and decided it should be a RuntimeError"
)
except asyncio.CancelledError:
# We need to manually close the connection here otherwise the aioredis
# pool will emit warnings saying that this connection still has pending
# commands (i.e. the above blocking pop)
redis.close()
raise
stream = decode(stream, "utf8")
rpc_message = self.deserializer(data)
expiry_key = f"rpc_expiry_key:{rpc_message.id}"
key_deleted = await redis.delete(expiry_key)
if not key_deleted:
return []
logger.debug(
LBullets(
L("⬅ Received RPC message on stream {}", Bold(stream)),
items=dict(**rpc_message.get_metadata(),
kwargs=rpc_message.get_kwargs()),
))
return [rpc_message]
async def history(
self,
api_name,
event_name,
start: datetime = None,
stop: datetime = None,
start_inclusive: bool = True,
) -> AsyncGenerator[EventMessage, None]:
# TODO: Test
redis_start = datetime_to_redis_steam_id(start) if start else "-"
redis_stop = datetime_to_redis_steam_id(stop) if stop else "+"
if start and not start_inclusive:
redis_start = redis_stream_id_add_one(redis_start)
stream_name = self._get_stream_names([(api_name, event_name)])[0]
batch_size = 1000
logger.debug(
f"Getting history for stream {stream_name} from {redis_start} ({start}) "
f"to {redis_stop} ({stop}) in batches of {batch_size}")
with await self.connection_manager() as redis:
messages = await redis.xrange(stream_name,
redis_start,
redis_stop,
count=batch_size)
if not messages:
return
for message_id, fields in messages:
message_id = decode(message_id, "utf8")
event_message = self._fields_to_message(
fields,
expected_event_names={event_name},
stream=stream_name,
native_id=message_id,
consumer_group=None,
)
if event_message:
yield event_message
async def fetch_events(self) -> Tuple[Sequence[EventMessage], Any]:
pool = await self.get_redis_pool()
with await pool as redis:
if not self._streams:
logger.debug('Event backend has been given no events to consume. Event backend will sleep.')
self._task = asyncio.ensure_future(asyncio.sleep(3600 * 24 * 365))
else:
logger.info(LBullets(
'Consuming events from', items={
'{} ({})'.format(*v) for v in self._streams.items()
}
))
# TODO: Count/timeout
self._task = asyncio.ensure_future(
redis.xread(
streams=list(self._streams.keys()),
latest_ids=list(self._streams.values()),
count=10, # TODO: Make configurable, add timeout too
)
)
try:
stream_messages = await self._task or []
except asyncio.CancelledError as e:
if self._reload:
# Streams to listen on have changed.
# Bail out and let this method get called again,
# at which point we'll pickup the new streams.
logger.debug('Event transport reloading.')
stream_messages = []
self._reload = False
else:
raise
event_messages = []
latest_ids = {}
for stream, message_id, fields in stream_messages:
stream = decode(stream, 'utf8')
message_id = decode(message_id, 'utf8')
decoded_fields = decode_message_fields(fields)
# Keep track of which event ID we are up to. We will store these
# in consumption_complete(), once we know the events have definitely
# been consumed.
latest_ids[stream] = message_id
# Unfortunately, these is an edge-case when BOTH:
# 1. We are consuming events from 'now' (i.e. event ID '$'), the default
# 2. There is an unhandled error when processing the FIRST batch of events
# In which case, the next iteration would start again from '$', in which
# case we would loose events. Therefore 'subtract one' from the message ID
# and store that immediately. Subtracting one is imprecise, as there is a SLIM
# chance we could grab another event in the process. However, if events are
# being consumed from 'now' then the developer presumably doesn't care about
# a high level of precision.
if self._streams[stream] == '$':
self._streams[stream] = redis_stream_id_subtract_one(message_id)
event_messages.append(
EventMessage.from_dict(decoded_fields)
)
logger.debug(LBullets(
L("⬅ Received event {} on stream {}", Bold(message_id), Bold(stream)),
items=decoded_fields
))
return event_messages, latest_ids
def _decode(s, encoding):
if encoding:
return decode(s, encoding)
return s
async def _reclaim_lost_messages(
self, stream_names: List[str], consumer_group: str,
expected_events: set) -> AsyncGenerator[List[EventMessage], None]:
"""Reclaim batches of messages that other consumers in the group failed to acknowledge within a timeout.
The timeout period is specified by the `acknowledgement_timeout` option.
"""
with await self.connection_manager() as redis:
for stream in stream_names:
old_messages = True
reclaim_from = None
# Keep pulling reclaimable messages from Redis until there are none left
while old_messages:
# reclaim_from keeps track of where we are up to in our fetching
# of messages
if not reclaim_from:
# This is our first iteration, so fetch from the start of time
reclaim_from = "-"
else:
# This is a subsequent iteration. XPENDING's 'start' parameter is inclusive,
# so we need to add one to the reclaim_from value to ensure we don't get a message
# we've already seen
reclaim_from = redis_stream_id_add_one(reclaim_from)
# Fetch the next batch of messages
try:
old_messages = await redis.xpending(
stream,
consumer_group,
reclaim_from,
"+",
count=self.reclaim_batch_size)
except ReplyError as e:
if "NOGROUP" in str(e):
# Group or consumer doesn't exist yet, so stop processing for this loop.
break
else:
raise
timeout = self.acknowledgement_timeout * 1000
event_messages = []
# Try to claim each messages
for (
message_id,
consumer_name,
ms_since_last_delivery,
num_deliveries,
) in old_messages:
message_id = decode(message_id, "utf8")
consumer_name = decode(consumer_name, "utf8")
reclaim_from = message_id
# This 'if' is not strictly required as the subsequent call to xclaim
# will honor the timeout parameter. However, using this if here allows
# for more sane logging from the point of view of the user. Without it
# we would report that we were trying to claim messages which were
# clearly not timed out yet.
if ms_since_last_delivery > timeout:
logger.info(
L(
"Found timed out event {} in stream {}. Abandoned by {}."
" Attempting to reclaim...",
Bold(message_id),
Bold(stream),
Bold(consumer_name),
))
# *Try* to claim the message...
result = await redis.xclaim(
stream, consumer_group, self.consumer_name,
int(timeout), message_id)
# Parse each message we managed to claim
for _, fields in result:
# Note that sometimes we will claim a message and it will be 'nil'.
# In this case the result will be (None, {}). We therefore do not
# rely on the values we get back from XCLAIM.
# I suspect this may happen if a stream has been trimmed, thereby causing
# un-acknowledged messages to be deleted from Redis.
try:
event_message = self._fields_to_message(
fields,
expected_events,
stream=stream,
native_id=message_id,
consumer_group=consumer_group,
)
except (NoopMessage, IgnoreMessage):
# This listener doesn't need to care about this message, so acknowledge
# it and move on with our lives
logger.debug(
f"Ignoring NOOP event with ID {message_id} discovered"
" during event reclaiming")
await redis.xack(stream, consumer_group,
message_id)
continue
logger.debug(
LBullets(
L(
"⬅ Reclaimed timed out event {} on stream {}. Abandoned"
" by {}.",
Bold(message_id),
Bold(stream),
Bold(consumer_name),
),
items=dict(
**event_message.get_metadata(),
kwargs=event_message.get_kwargs(),
),
))
event_messages.append(event_message)
# And yield our batch of messages
if event_messages:
yield event_messages
async def _fetch_new_messages(
self, streams, consumer_group, expected_events,
forever) -> AsyncGenerator[List[EventMessage], None]:
"""Coroutine to consume new messages
The consumption has two stages:
1. Fetch and yield any messages this consumer is responsible for processing but has yet
to successfully process. This can happen in cases where a message was
previously consumed but not acknowledged (i.e. due to an error).
This is a one-off startup stage.
2. Wait for new messages to arrive. Yield these messages when they arrive, then
resume waiting for messages
See Also:
_reclaim_lost_messages() - Another coroutine which reclaims messages which timed out
while being processed by other consumers in this group
"""
with await self.connection_manager() as redis:
# Firstly create the consumer group if we need to
await self._create_consumer_groups(streams, redis, consumer_group)
# Get any messages that this consumer has yet to process.
# This can happen in the case where the processes died before acknowledging.
pending_messages = await redis.xread_group(
group_name=consumer_group,
consumer_name=self.consumer_name,
streams=list(streams.keys()),
# Using ID '0' indicates we want unacked pending messages
latest_ids=["0"] * len(streams),
timeout=None, # Don't block, return immediately
)
event_messages = []
for stream, message_id, fields in pending_messages:
message_id = decode(message_id, "utf8")
stream = decode(stream, "utf8")
try:
event_message = self._fields_to_message(
fields,
expected_events,
stream=stream,
native_id=message_id,
consumer_group=consumer_group,
)
except (NoopMessage, IgnoreMessage):
# This listener doesn't need to care about this message, so acknowledge
# it and move on with our lives
await redis.xack(stream, consumer_group, message_id)
continue
logger.debug(
LBullets(
L(
"⬅ Receiving pending event {} on stream {}",
Bold(message_id),
Bold(stream),
),
items=dict(**event_message.get_metadata(),
kwargs=event_message.get_kwargs()),
))
event_messages.append(event_message)
if event_messages:
yield event_messages
# We've now cleaned up any old messages that were hanging around.
# Now we get on to the main loop which blocks and waits for new messages
while True:
# Fetch some messages.
# This will block until there are some messages available
try:
stream_messages = await redis.xread_group(
group_name=consumer_group,
consumer_name=self.consumer_name,
streams=list(streams.keys()),
# Using ID '>' indicates we only want new messages which have not
# been passed to other consumers in this group
latest_ids=[">"] * len(streams),
count=self.batch_size,
)
except asyncio.CancelledError:
# We need to manually close the connection here otherwise the aioredis
# pool will emit warnings saying that this connection still has pending
# commands (i.e. the above blocking pop)
redis.close()
raise
# Handle the messages we have received
event_messages = []
for stream, message_id, fields in stream_messages:
message_id = decode(message_id, "utf8")
stream = decode(stream, "utf8")
try:
event_message = self._fields_to_message(
fields,
expected_events,
stream=stream,
native_id=message_id,
consumer_group=consumer_group,
)
except (NoopMessage, IgnoreMessage):
# This listener doesn't need to care about this message, so acknowledge
# it and move on with our lives
logger.debug(
f"Ignoring NOOP event with ID {message_id} discovered during streaming"
" of messages")
await redis.xack(stream, consumer_group, message_id)
continue
logger.debug(
LBullets(
L(
"⬅ Received new event {} on stream {}",
Bold(message_id),
Bold(stream),
),
items=dict(**event_message.get_metadata(),
kwargs=event_message.get_kwargs()),
))
event_messages.append(event_message)
if event_messages:
yield event_messages
if not forever:
return
async def _fetch_new_messages(
self, streams, consumer_group, expected_events,
forever) -> AsyncGenerator[List[EventMessage], None]:
"""Coroutine to consume new messages
The consumption has two stages:
1. Fetch and yield any messages this consumer is responsible for processing but has yet
to successfully process. This can happen in cases where a message was
previously consumed but not acknowledged (i.e. due to an error).
This is a one-off startup stage.
2. Wait for new messages to arrive. Yield these messages when they arrive, then
resume waiting for messages
See Also:
_reclaim_lost_messages() - Another coroutine which reclaims messages which timed out
while being processed by other consumers in this group
"""
with await self.connection_manager() as redis:
# Firstly create the consumer group if we need to
await self._create_consumer_groups(streams, redis, consumer_group)
# Get any messages that this consumer has yet to process.
# This can happen in the case where the processes died before acknowledging.
pending_messages = await redis.xread_group(
group_name=consumer_group,
consumer_name=self.consumer_name,
streams=list(streams.keys()),
# Using ID '0' indicates we want unacked pending messages
latest_ids=["0"] * len(streams),
timeout=None, # Don't block, return immediately
)
event_messages = []
for stream, message_id, fields in pending_messages:
message_id = decode(message_id, "utf8")
stream = decode(stream, "utf8")
event_message = self._fields_to_message(
fields,
expected_events,
stream=stream,
native_id=message_id,
consumer_group=consumer_group,
)
if not event_message:
# noop message, or message an event we don't care about
continue
logger.debug(
LBullets(
L(
"⬅ Receiving pending event {} on stream {}",
Bold(message_id),
Bold(stream),
),
items=dict(**event_message.get_metadata(),
kwargs=event_message.get_kwargs()),
))
event_messages.append(event_message)
if event_messages:
yield event_messages
# We've now cleaned up any old messages that were hanging around.
# Now we get on to the main loop which blocks and waits for new messages
while True:
# Fetch some messages.
# This will block until there are some messages available
stream_messages = await redis.xread_group(
group_name=consumer_group,
consumer_name=self.consumer_name,
streams=list(streams.keys()),
# Using ID '>' indicates we only want new messages which have not
# been passed to other consumers in this group
latest_ids=[">"] * len(streams),
count=self.batch_size,
)
# Handle the messages we have received
event_messages = []
for stream, message_id, fields in stream_messages:
message_id = decode(message_id, "utf8")
stream = decode(stream, "utf8")
event_message = self._fields_to_message(
fields,
expected_events,
stream=stream,
native_id=message_id,
consumer_group=consumer_group,
)
if not event_message:
# noop message, or message an event we don't care about
continue
logger.debug(
LBullets(
L(
"⬅ Received new event {} on stream {}",
Bold(message_id),
Bold(stream),
),
items=dict(**event_message.get_metadata(),
kwargs=event_message.get_kwargs()),
))
# NOTE: YIELD ALL MESSAGES, NOT JUST ONE
event_messages.append(event_message)
if event_messages:
yield event_messages
if not forever:
return
def _decode(s, encoding):
if encoding:
return decode(s, encoding)
return s
async def _fetch_new_messages(self, streams, consumer_group,
expected_events, forever):
with await self.connection_manager() as redis:
# Firstly create the consumer group if we need to
await self._create_consumer_groups(streams, redis, consumer_group)
# Get any messages that this consumer has yet to process.
# This can happen in the case where the processes died before acknowledging.
pending_messages = await redis.xread_group(
group_name=consumer_group,
consumer_name=self.consumer_name,
streams=list(streams.keys()),
# Using ID '0' indicates we want unacked pending messages
latest_ids=["0"] * len(streams),
timeout=None, # Don't block, return immediately
)
for stream, message_id, fields in pending_messages:
message_id = decode(message_id, "utf8")
event_message = self._fields_to_message(fields,
expected_events,
native_id=message_id)
if not event_message:
# noop message, or message an event we don't care about
continue
logger.debug(
LBullets(
L(
"⬅ Receiving pending event {} on stream {}",
Bold(message_id),
Bold(stream),
),
items=dict(**event_message.get_metadata(),
kwargs=event_message.get_kwargs()),
))
yield event_message, stream
# We've now cleaned up any old messages that were hanging around.
# Now we get on to the main loop which blocks and waits for new messages
while True:
# Fetch some messages.
# This will block until there are some messages available
stream_messages = await redis.xread_group(
group_name=consumer_group,
consumer_name=self.consumer_name,
streams=list(streams.keys()),
# Using ID '>' indicates we only want new messages which have not
# been passed to other consumers in this group
latest_ids=[">"] * len(streams),
count=self.batch_size,
)
# Handle the messages we have received
for stream, message_id, fields in stream_messages:
message_id = decode(message_id, "utf8")
event_message = self._fields_to_message(
fields, expected_events, native_id=message_id)
if not event_message:
# noop message, or message an event we don't care about
continue
logger.debug(
LBullets(
L(
"⬅ Received new event {} on stream {}",
Bold(message_id),
Bold(stream),
),
items=dict(**event_message.get_metadata(),
kwargs=event_message.get_kwargs()),
))
yield event_message, stream
if not forever:
return
async def _reclaim_lost_messages(
self, stream_names: List[str], consumer_group: str, expected_events: set
) -> AsyncGenerator[List[EventMessage], None]:
"""Reclaim batches of messages that other consumers in the group failed to acknowledge within a timeout.
The timeout period is specified by the `acknowledgement_timeout` option.
"""
with await self.connection_manager() as redis:
for stream in stream_names:
old_messages = True
reclaim_from = None
# Keep pulling reclaimable messages from Redis until there are none left
while old_messages:
# reclaim_from keeps track of where we are up to in our fetching
# of messages
if not reclaim_from:
# This is our first iteration, so fetch from the start of time
reclaim_from = "-"
else:
# This is a subsequent iteration. XPENDING's 'start' parameter is inclusive,
# so we need to add one to the reclaim_from value to ensure we don't get a message
# we've already seen
reclaim_from = redis_stream_id_add_one(reclaim_from)
# Fetch the next batch of messages
old_messages = await redis.xpending(
stream, consumer_group, reclaim_from, "+", count=self.reclaim_batch_size
)
timeout = self.acknowledgement_timeout * 1000
event_messages = []
# Try to claim each messages
for (
message_id,
consumer_name,
ms_since_last_delivery,
num_deliveries,
) in old_messages:
message_id = decode(message_id, "utf8")
consumer_name = decode(consumer_name, "utf8")
reclaim_from = message_id
# This 'if' is not strictly required as the subsequent call to xclaim
# will honor the timeout parameter. However, using this if here allows
# for more sane logging from the point of view of the user. Without it
# we would report that we were trying to claim messages which were
# clearly not timed out yet.
if ms_since_last_delivery > timeout:
logger.info(
L(
"Found timed out event {} in stream {}. Abandoned by {}. Attempting to reclaim...",
Bold(message_id),
Bold(stream),
Bold(consumer_name),
)
)
# *Try* to claim the messages...
result = await redis.xclaim(
stream, consumer_group, self.consumer_name, int(timeout), message_id
)
# Parse each message we managed to claim
for claimed_message_id, fields in result:
claimed_message_id = decode(claimed_message_id, "utf8")
event_message = self._fields_to_message(
fields,
expected_events,
stream=stream,
native_id=claimed_message_id,
consumer_group=consumer_group,
)
if not event_message:
# noop message, or message an event we don't care about
continue
logger.debug(
LBullets(
L(
"⬅ Reclaimed timed out event {} on stream {}. Abandoned by {}.",
Bold(message_id),
Bold(stream),
Bold(consumer_name),
),
items=dict(
**event_message.get_metadata(),
kwargs=event_message.get_kwargs(),
),
)
)
event_messages.append(event_message)
# And yield our batch of messages
if event_messages:
yield event_messages
