async def test_redis_checkpoint_reallocate(self):
name = "test-{}".format(str(uuid.uuid4())[0:8])
# first consumer
checkpointer_a = RedisCheckPointer(name=name, id="proc-1")
await checkpointer_a.allocate("test")
# checkpoint
await checkpointer_a.checkpoint("test", "123")
# stop on this shard
await checkpointer_a.deallocate("test")
# second consumer
checkpointer_b = RedisCheckPointer(name=name, id="proc-2")
success, sequence = await checkpointer_b.allocate("test")
self.assertTrue(success)
self.assertEquals("123", sequence)
await checkpointer_b.close()
self.assertEquals(checkpointer_b.get_all_checkpoints(), {})
await checkpointer_a.close()
async def test_producer_and_consumer_consume_multiple_shards_with_redis_checkpointer(
self,
):
stream_name = "test_{}".format(str(uuid.uuid4())[0:8])
async with Producer(
stream_name=stream_name,
endpoint_url=ENDPOINT_URL,
create_stream=stream_name,
create_stream_shards=2,
) as producer:
for i in range(0, 100):
await producer.put("test.{}".format(i))
await producer.flush()
results = []
checkpointer = RedisCheckPointer(
name="test-{}".format(str(uuid.uuid4())[0:8]), heartbeat_frequency=3
)
async with Consumer(
stream_name=stream_name,
endpoint_url=ENDPOINT_URL,
checkpointer=checkpointer,
record_limit=10,
) as consumer:
# consumer will stop if no msgs
for i in range(0, 6):
async for item in consumer:
results.append(item)
await asyncio.sleep(0.5)
self.assertEquals(100, len(results))
checkpoints = checkpointer.get_all_checkpoints()
self.assertEquals(2, len(checkpoints))
# Expect both shards to have been used/set
for item in checkpoints.values():
self.assertIsNotNone(item)