async def test_producer_and_consumer_consume_with_json_line_aggregator(
self):
processor = JsonLineProcessor()
async with Producer(stream_name=self.stream_name,
endpoint_url=ENDPOINT_URL,
processor=processor) as producer:
for x in range(0, 10):
await producer.put({"test": x})
await producer.flush()
results = []
async with Consumer(
stream_name=self.stream_name,
endpoint_url=ENDPOINT_URL,
processor=processor,
) as consumer:
async for item in consumer:
results.append(item)
# Expect to have consumed from start as default iterator_type=TRIM_HORIZON
self.assertEqual(len(results), 10)
self.assertEquals(results[0], {"test": 0})
self.assertEquals(results[-1], {"test": 9})
def test_json_line_processor(self):
processor = JsonLineProcessor(max_size=25)
# Expect nothing as batching
self.assertEqual([], list(processor.add_item({"test": 123})))
self.assertEqual([], list(processor.add_item({"test": 456})))
self.assertTrue(processor.has_items())
output = list(processor.get_items())
self.assertEqual(len(output), 1)
self.assertEqual(output[0].size, 28)
self.assertEqual(output[0].n, 2)
self.assertEqual(output[0].data, b'{"test": 123}\n{"test": 456}\n')
self.assertListEqual(
list(processor.parse(output[0].data)),
[{
"test": 123
}, {
"test": 456
}],
)
# Expect empty now
self.assertFalse(processor.has_items())
result = []
for x in range(1000):
output = list(
processor.add_item({"test": "test with some more data"}))
if output:
self.assertEqual(len(output), 1)
result.append(output[0])
# Expected at least one record to be output
self.assertEqual(len(result), 1)
self.assertEqual(result[0].size, 25567) # expect below 25*1024=25600
self.assertEqual(result[0].n, 691)
# Expect some left
self.assertTrue(processor.has_items())
output = list(processor.get_items())
self.assertEqual(len(output), 1)
self.assertEqual(output[0].size, 11432)
self.assertEqual(output[0].n, 309)
self.assertFalse(processor.has_items())