async def test_producer_producer_limit(self):
# Expect some throughput errors
async with Producer(
stream_name=self.STREAM_NAME_SINGLE_SHARD,
processor=StringProcessor(),
put_bandwidth_limit_per_shard=1500,
) as producer:
async with Consumer(
stream_name=self.STREAM_NAME_SINGLE_SHARD,
processor=StringProcessor(),
iterator_type="LATEST",
) as consumer:
await consumer.start_consumer()
# Wait a bit just to be sure iterator is gonna get late
await asyncio.sleep(3)
for x in range(20):
await producer.put(self.random_string(1024 * 250))
# todo: async timeout
output = []
while len(output) < 20:
async for item in consumer:
output.append(item)
self.assertEquals(len(output), 20)
self.assertTrue(producer.throughput_exceeded_count > 0)
async def test_producer_and_consumer_consume_from_start_after(self):
# Don't flush, close producer immediately to test all data is written to stream on exit.
async with Producer(
stream_name=self.stream_name,
endpoint_url=ENDPOINT_URL,
processor=StringProcessor(),
) as producer:
# Put enough data to ensure it will require more than one put
# ie test overflow behaviour
for _ in range(15):
await producer.put(self.random_string(100 * 1024))
results = []
async with Consumer(
stream_name=self.stream_name,
endpoint_url=ENDPOINT_URL,
processor=StringProcessor(),
) as consumer:
async for item in consumer:
results.append(item)
# Expect to have consumed from start as default iterator_type=TRIM_HORIZON
self.assertEquals(len(results), 15)
def test_string_processor(self):
processor = StringProcessor()
self.assertEquals(processor.max_bytes, 1024 * 25 * 40)
output = list(processor.add_item("test"))
self.assertEqual(len(output), 1)
self.assertIsInstance(output[0], OutputItem)
self.assertEqual(output[0].size, len("test"))
self.assertEqual(output[0].n, 1)
self.assertEqual(output[0].data, b"test")
self.assertFalse(processor.has_items())
async def test_producer_put_below_limit(self):
async with Producer(
stream_name=self.stream_name,
processor=StringProcessor(),
endpoint_url=ENDPOINT_URL,
) as producer:
# The maximum size of the data payload of a record before base64-encoding is up to 1 MiB.
# Limit is set in aggregators.BaseAggregator (few bytes short of 1MiB)
await producer.put(self.random_string(40 * 25 * 1024))
async def test_consumer_checkpoint(self):
checkpointer = MemoryCheckPointer(name="test")
results = []
async with Producer(
stream_name=self.STREAM_NAME_SINGLE_SHARD,
processor=StringProcessor(),
) as producer:
async with Consumer(
stream_name=self.STREAM_NAME_SINGLE_SHARD,
checkpointer=checkpointer,
processor=StringProcessor(),
iterator_type="LATEST",
) as consumer:
# Manually start
await consumer.start_consumer()
await producer.put("test")
await producer.flush()
for i in range(3):
async for item in consumer:
results.append(item)
checkpoints = checkpointer.get_all_checkpoints()
# Expect 1 as only 1 shard
self.assertEquals(1, len(checkpoints))
self.assertIsNotNone(checkpoints[list(
checkpoints.keys())[0]]["sequence"])
self.assertListEqual(results, ["test"])
def test_processor_exceed_put_limit(self):
processor = StringProcessor()
with self.assertRaises(exceptions.ExceededPutLimit):
list(processor.add_item(self.random_string(1024 * 1024 + 1)))