async def test_consume_events_since_datetime(
redis_event_transport: RedisEventTransport, redis_client, error_queue):
await redis_client.xadd(
"my.dummy.my_event:stream",
fields={
b"api_name": b"my.dummy",
b"event_name": b"my_event",
b"id": b"123",
b"version": b"1",
b":field": b'"1"',
},
message_id="1515000001000-0",
)
await redis_client.xadd(
"my.dummy.my_event:stream",
fields={
b"api_name": b"my.dummy",
b"event_name": b"my_event",
b"id": b"123",
b"version": b"1",
b":field": b'"2"',
},
message_id="1515000002000-0",
)
await redis_client.xadd(
"my.dummy.my_event:stream",
fields={
b"api_name": b"my.dummy",
b"event_name": b"my_event",
b"id": b"123",
b"version": b"1",
b":field": b'"3"',
},
message_id="1515000003000-0",
)
# 1515000001500-0 -> 2018-01-03T17:20:01.500Z
since_datetime = datetime(2018, 1, 3, 17, 20, 1, 500)
consumer = redis_event_transport.consume(
[("my.dummy", "my_event")],
listener_name="test",
since=since_datetime,
forever=False,
error_queue=error_queue,
)
events = []
async def co():
async for messages in consumer:
events.extend(messages)
await redis_event_transport.acknowledge(*messages)
task = asyncio.ensure_future(co())
await asyncio.sleep(0.1)
await cancel(task)
assert len(events) == 2
assert events[0].kwargs["field"] == "2"
assert events[1].kwargs["field"] == "3"
async def test_consume_events_since_id(
redis_event_transport: RedisEventTransport, redis_client, error_queue):
await redis_client.xadd(
"my.dummy.my_event:stream",
fields={
b"api_name": b"my.dummy",
b"event_name": b"my_event",
b"id": b"123",
b"version": b"1",
b":field": b'"1"',
},
message_id="1515000001000-0",
)
await redis_client.xadd(
"my.dummy.my_event:stream",
fields={
b"api_name": b"my.dummy",
b"event_name": b"my_event",
b"id": b"123",
b"version": b"1",
b":field": b'"2"',
},
message_id="1515000002000-0",
)
await redis_client.xadd(
"my.dummy.my_event:stream",
fields={
b"api_name": b"my.dummy",
b"event_name": b"my_event",
b"id": b"123",
b"version": b"1",
b":field": b'"3"',
},
message_id="1515000003000-0",
)
consumer = redis_event_transport.consume(
[("my.dummy", "my_event")],
"cg",
since="1515000001500-0",
forever=False,
error_queue=error_queue,
)
events = []
async def co():
async for messages in consumer:
events.extend(messages)
await redis_event_transport.acknowledge(*messages)
task = asyncio.ensure_future(co())
await asyncio.sleep(0.1)
await cancel(task)
messages_ids = [m.native_id for m in events if isinstance(m, EventMessage)]
assert len(messages_ids) == 2
assert len(events) == 2
assert events[0].kwargs["field"] == "2"
assert events[1].kwargs["field"] == "3"
async def test_reclaim_lost_messages_consume(loop, redis_client, redis_pool,
dummy_api):
"""Test that messages which another consumer has timed out on can be reclaimed
Unlike the above test, we call consume() here, not _reclaim_lost_messages()
"""
# Add a message
await redis_client.xadd(
"my.dummy.my_event:stream",
fields={
b"api_name": b"my.dummy",
b"event_name": b"my_event",
b"id": b"123",
b"version": b"1",
b":field": b'"value"',
},
)
# Create the consumer group
await redis_client.xgroup_create(stream="my.dummy.my_event:stream",
group_name="test_service-test_listener",
latest_id="0")
# Claim it in the name of another consumer
await redis_client.xread_group(
group_name="test_service-test_listener",
consumer_name="bad_consumer",
streams=["my.dummy.my_event:stream"],
latest_ids=[">"],
)
# Sleep a moment to fake a short timeout
await asyncio.sleep(0.1)
event_transport = RedisEventTransport(
redis_pool=redis_pool,
service_name="test_service",
consumer_name="good_consumer",
acknowledgement_timeout=0.01, # in ms, short for the sake of testing
stream_use=StreamUse.PER_EVENT,
)
consumer = event_transport.consume(
listen_for=[("my.dummy", "my_event")],
since="0",
listener_name="test_listener",
bus_client=None,
)
messages = []
async def consume():
async for messages_ in consumer:
messages.extend(messages_)
task = asyncio.ensure_future(consume())
await asyncio.sleep(0.1)
await cancel(task)
assert len(messages) == 1
async def co_consume(group_number):
event_transport = RedisEventTransport(
redis_pool=redis_pool,
service_name=f"test_service{group_number}",
consumer_name=f"test_consumer",
stream_use=StreamUse.PER_EVENT,
)
async for messages_ in event_transport.consume(
[("my.dummy", "my_event")], "test_listener", bus_client=None):
messages.append(messages_)
await event_transport.acknowledge(*messages_, bus_client=None)
async def co_consume(consumer_number):
event_transport = RedisEventTransport(
redis_pool=redis_pool,
service_name="test_service",
consumer_name=f"test_consumer{consumer_number}",
stream_use=StreamUse.PER_EVENT,
)
consumer = event_transport.consume(listen_for=[("my.dummy", "my_event")
],
listener_name="test_listener",
bus_client=None)
async for messages in consumer:
events.append(messages)
await event_transport.acknowledge(*messages, bus_client=None)
async def test_reclaim_pending_messages(redis_client, redis_pool, error_queue):
"""Test that unacked messages belonging to this consumer get reclaimed on startup"""
# Add a message
await redis_client.xadd(
"my.dummy.my_event:stream",
fields={
b"api_name": b"my.dummy",
b"event_name": b"my_event",
b"id": b"123",
b"version": b"1",
b":field": b'"value"',
},
)
# Create the consumer group
await redis_client.xgroup_create(stream="my.dummy.my_event:stream",
group_name="test_service-test_listener",
latest_id="0")
# Claim it in the name of ourselves
await redis_client.xread_group(
group_name="test_service-test_listener",
consumer_name="good_consumer",
streams=["my.dummy.my_event:stream"],
latest_ids=[">"],
)
event_transport = RedisEventTransport(
redis_pool=redis_pool,
service_name="test_service",
consumer_name="good_consumer",
stream_use=StreamUse.PER_EVENT,
)
consumer = event_transport.consume(
listen_for=[("my.dummy", "my_event")],
since="0",
listener_name="test_listener",
error_queue=error_queue,
)
messages = []
async def consume():
async for messages_ in consumer:
messages.extend(messages_)
await event_transport.acknowledge(*messages_)
task = asyncio.ensure_future(consume())
await asyncio.sleep(0.1)
await cancel(task)
assert len(messages) == 1
assert messages[0].api_name == "my.dummy"
assert messages[0].event_name == "my_event"
assert messages[0].kwargs == {"field": "value"}
assert messages[0].native_id
assert type(messages[0].native_id) == str
# Now check that redis believes the message has been consumed
total_pending, *_ = await redis_client.xpending(
stream="my.dummy.my_event:stream",
group_name="test_service-test_listener")
assert total_pending == 0