async def test_producer_producer_limit(self): # Expect some throughput errors async with Producer( stream_name=self.STREAM_NAME_SINGLE_SHARD, processor=StringProcessor(), put_bandwidth_limit_per_shard=1500, ) as producer: async with Consumer( stream_name=self.STREAM_NAME_SINGLE_SHARD, processor=StringProcessor(), iterator_type="LATEST", ) as consumer: await consumer.start_consumer() # Wait a bit just to be sure iterator is gonna get late await asyncio.sleep(3) for x in range(20): await producer.put(self.random_string(1024 * 250)) # todo: async timeout output = [] while len(output) < 20: async for item in consumer: output.append(item) self.assertEquals(len(output), 20) self.assertTrue(producer.throughput_exceeded_count > 0)
async def test_producer_and_consumer_consume_from_start_after(self): # Don't flush, close producer immediately to test all data is written to stream on exit. async with Producer( stream_name=self.stream_name, endpoint_url=ENDPOINT_URL, processor=StringProcessor(), ) as producer: # Put enough data to ensure it will require more than one put # ie test overflow behaviour for _ in range(15): await producer.put(self.random_string(100 * 1024)) results = [] async with Consumer( stream_name=self.stream_name, endpoint_url=ENDPOINT_URL, processor=StringProcessor(), ) as consumer: async for item in consumer: results.append(item) # Expect to have consumed from start as default iterator_type=TRIM_HORIZON self.assertEquals(len(results), 15)
def test_string_processor(self): processor = StringProcessor() self.assertEquals(processor.max_bytes, 1024 * 25 * 40) output = list(processor.add_item("test")) self.assertEqual(len(output), 1) self.assertIsInstance(output[0], OutputItem) self.assertEqual(output[0].size, len("test")) self.assertEqual(output[0].n, 1) self.assertEqual(output[0].data, b"test") self.assertFalse(processor.has_items())
async def test_producer_put_below_limit(self): async with Producer( stream_name=self.stream_name, processor=StringProcessor(), endpoint_url=ENDPOINT_URL, ) as producer: # The maximum size of the data payload of a record before base64-encoding is up to 1 MiB. # Limit is set in aggregators.BaseAggregator (few bytes short of 1MiB) await producer.put(self.random_string(40 * 25 * 1024))
async def test_consumer_checkpoint(self): checkpointer = MemoryCheckPointer(name="test") results = [] async with Producer( stream_name=self.STREAM_NAME_SINGLE_SHARD, processor=StringProcessor(), ) as producer: async with Consumer( stream_name=self.STREAM_NAME_SINGLE_SHARD, checkpointer=checkpointer, processor=StringProcessor(), iterator_type="LATEST", ) as consumer: # Manually start await consumer.start_consumer() await producer.put("test") await producer.flush() for i in range(3): async for item in consumer: results.append(item) checkpoints = checkpointer.get_all_checkpoints() # Expect 1 as only 1 shard self.assertEquals(1, len(checkpoints)) self.assertIsNotNone(checkpoints[list( checkpoints.keys())[0]]["sequence"]) self.assertListEqual(results, ["test"])
def test_processor_exceed_put_limit(self): processor = StringProcessor() with self.assertRaises(exceptions.ExceededPutLimit): list(processor.add_item(self.random_string(1024 * 1024 + 1)))