async def test_redis_checkpoint_reallocate(self):
name = "test-{}".format(str(uuid.uuid4())[0:8])
# first consumer
checkpointer_a = RedisCheckPointer(name=name, id="proc-1")
await checkpointer_a.allocate("test")
# checkpoint
await checkpointer_a.checkpoint("test", "123")
# stop on this shard
await checkpointer_a.deallocate("test")
# second consumer
checkpointer_b = RedisCheckPointer(name=name, id="proc-2")
success, sequence = await checkpointer_b.allocate("test")
self.assertTrue(success)
self.assertEquals("123", sequence)
await checkpointer_b.close()
self.assertEquals(checkpointer_b.get_all_checkpoints(), {})
await checkpointer_a.close()
async def test_producer_and_consumer_consume_multiple_shards_with_redis_checkpointer(
self,
):
stream_name = "test_{}".format(str(uuid.uuid4())[0:8])
async with Producer(
stream_name=stream_name,
endpoint_url=ENDPOINT_URL,
create_stream=stream_name,
create_stream_shards=2,
) as producer:
for i in range(0, 100):
await producer.put("test.{}".format(i))
await producer.flush()
results = []
checkpointer = RedisCheckPointer(
name="test-{}".format(str(uuid.uuid4())[0:8]), heartbeat_frequency=3
)
async with Consumer(
stream_name=stream_name,
endpoint_url=ENDPOINT_URL,
checkpointer=checkpointer,
record_limit=10,
) as consumer:
# consumer will stop if no msgs
for i in range(0, 6):
async for item in consumer:
results.append(item)
await asyncio.sleep(0.5)
self.assertEquals(100, len(results))
checkpoints = checkpointer.get_all_checkpoints()
self.assertEquals(2, len(checkpoints))
# Expect both shards to have been used/set
for item in checkpoints.values():
self.assertIsNotNone(item)
async def test_redis_checkpoint_hearbeat(self):
name = "test-{}".format(str(uuid.uuid4())[0:8])
checkpointer = RedisCheckPointer(name=name, heartbeat_frequency=0.5)
await checkpointer.allocate("test")
await checkpointer.checkpoint("test", "123")
await asyncio.sleep(1)
await checkpointer.close()
# nothing to assert
self.assertTrue(True)
async def test_redis_checkpoint_locking(self):
name = "test-{}".format(str(uuid.uuid4())[0:8])
# first consumer
checkpointer_a = RedisCheckPointer(name=name, id="proc-1")
# second consumer
checkpointer_b = RedisCheckPointer(name=name, id="proc-2")
# try to allocate the same shard
result = await asyncio.gather(
*[checkpointer_a.allocate("test"), checkpointer_b.allocate("test")]
)
result = list(sorted([x[0] for x in result]))
# Expect only one to have succeeded
self.assertEquals([False, True], result)
await checkpointer_a.close()
await checkpointer_b.close()
async def test_resharding(self, *args):
# *args pass through mock
stream_name = "test_{}".format(str(uuid.uuid4())[0:8])
# Create stream with 2x shards. Add some records
async with Producer(stream_name=stream_name,
endpoint_url=ENDPOINT_URL,
create_stream=stream_name,
create_stream_shards=2,
shard_refresh_timer=15) as producer:
for i in range(0, 50):
await producer.put("test.{}".format(i))
await producer.flush()
results = []
checkpointer = RedisCheckPointer(name="test-{}".format(
str(uuid.uuid4())[0:8]),
heartbeat_frequency=3)
async with Consumer(
stream_name=stream_name,
endpoint_url=ENDPOINT_URL,
checkpointer=checkpointer,
record_limit=5,
# Limit the queue so there records will remain in the shards
max_queue_size=5,
shard_refresh_timer=15) as consumer:
for i in range(0, 3):
async for item in consumer:
results.append(item)
await asyncio.sleep(0.5)
log.info(f"Consumed {len(results)} records")
# Start reshard operation
await producer.client.update_shard_count(
StreamName=stream_name,
TargetShardCount=4,
ScalingType='UNIFORM_SCALING')
await self.describe_stream(client=producer.client,
stream_name=stream_name)
await asyncio.sleep(1)
await self.describe_stream(client=producer.client,
stream_name=stream_name)
# Now add some more records
for i in range(50, 100):
await producer.put("test.{}".format(i))
await producer.flush()
for i in range(0, 10):
async for item in consumer:
results.append(item)
await asyncio.sleep(0.5)
log.info(f"Consumed {len(results)} records")
assert len(results) == 100
async def test_resharding(self):
stream_name = self.STREAM_NAME_SINGLE_SHARD
# Create stream with 2x shards. Add some records
async with Producer(stream_name=stream_name,
shard_refresh_timer=15) as producer:
for i in range(0, 50):
await producer.put("test.{}".format(i))
await producer.flush()
results = []
checkpointer = RedisCheckPointer(name="test-{}".format(
str(uuid.uuid4())[0:8]),
heartbeat_frequency=3)
async with Consumer(
stream_name=stream_name,
checkpointer=checkpointer,
record_limit=5,
# Limit the queue so there records will remain in the shards
max_queue_size=5,
shard_refresh_timer=15) as consumer:
for i in range(0, 3):
async for item in consumer:
results.append(item)
await asyncio.sleep(0.5)
log.info(f"Consumed {len(results)} records")
# Start reshard operation
# TODO: Producer not writing to new shards, shards are being found and checkpointed
await producer.client.update_shard_count(
StreamName=stream_name,
TargetShardCount=2,
ScalingType='UNIFORM_SCALING')
await self.describe_stream(client=producer.client,
stream_name=stream_name)
await asyncio.sleep(1)
await self.describe_stream(client=producer.client,
stream_name=stream_name)
# Now add some more records
for i in range(50, 100):
await producer.put("test.{}".format(i))
await producer.flush()
await asyncio.sleep(10)
for i in range(0, 20):
async for item in consumer:
results.append(item)
await asyncio.sleep(2)
log.info(f"Consumed {len(results)} records")
assert len(results) == 100