async def test_producer_and_consumer_consume_with_json_line_aggregator( self): processor = JsonLineProcessor() async with Producer(stream_name=self.stream_name, endpoint_url=ENDPOINT_URL, processor=processor) as producer: for x in range(0, 10): await producer.put({"test": x}) await producer.flush() results = [] async with Consumer( stream_name=self.stream_name, endpoint_url=ENDPOINT_URL, processor=processor, ) as consumer: async for item in consumer: results.append(item) # Expect to have consumed from start as default iterator_type=TRIM_HORIZON self.assertEqual(len(results), 10) self.assertEquals(results[0], {"test": 0}) self.assertEquals(results[-1], {"test": 9})
def test_json_line_processor(self): processor = JsonLineProcessor(max_size=25) # Expect nothing as batching self.assertEqual([], list(processor.add_item({"test": 123}))) self.assertEqual([], list(processor.add_item({"test": 456}))) self.assertTrue(processor.has_items()) output = list(processor.get_items()) self.assertEqual(len(output), 1) self.assertEqual(output[0].size, 28) self.assertEqual(output[0].n, 2) self.assertEqual(output[0].data, b'{"test": 123}\n{"test": 456}\n') self.assertListEqual( list(processor.parse(output[0].data)), [{ "test": 123 }, { "test": 456 }], ) # Expect empty now self.assertFalse(processor.has_items()) result = [] for x in range(1000): output = list( processor.add_item({"test": "test with some more data"})) if output: self.assertEqual(len(output), 1) result.append(output[0]) # Expected at least one record to be output self.assertEqual(len(result), 1) self.assertEqual(result[0].size, 25567) # expect below 25*1024=25600 self.assertEqual(result[0].n, 691) # Expect some left self.assertTrue(processor.has_items()) output = list(processor.get_items()) self.assertEqual(len(output), 1) self.assertEqual(output[0].size, 11432) self.assertEqual(output[0].n, 309) self.assertFalse(processor.has_items())