async def test_producer_and_consumer_consume_with_msgpack_aggregator(self): processor = MsgpackProcessor() async with Producer(stream_name=self.stream_name, endpoint_url=ENDPOINT_URL, processor=processor) as producer: for x in range(0, 10): await producer.put({"test": x}) await producer.flush() results = [] async with Consumer( stream_name=self.stream_name, endpoint_url=ENDPOINT_URL, processor=processor, ) as consumer: async for item in consumer: results.append(item) # Expect to have consumed from start as default iterator_type=TRIM_HORIZON self.assertEqual(len(results), 10) self.assertEquals(results[0], {"test": 0}) self.assertEquals(results[-1], {"test": 9})
def test_msgpack_processor(self): processor = MsgpackProcessor(max_size=25) # Expect nothing as batching self.assertEqual([], list(processor.add_item({"test": 123}))) self.assertEqual([], list(processor.add_item({"test": 456}))) self.assertTrue(processor.has_items()) output = list(processor.get_items()) self.assertEqual(len(output), 1) self.assertEqual(output[0].size, 22) self.assertEqual(output[0].n, 2) self.assertEqual(output[0].data, b"7:\x81\xa4test{,9:\x81\xa4test\xcd\x01\xc8,") self.assertListEqual(list(processor.parse(output[0].data)), [{ "test": 123 }, { "test": 456 }]) # Expect empty now self.assertFalse(processor.has_items()) result = [] for x in range(1000): output = list( processor.add_item({"test": "test with some more data"})) if output: self.assertEqual(len(output), 1) result.append(output[0]) # Expected at least one record to be output self.assertEqual(len(result), 1) self.assertEqual(result[0].size, 25585) # expect below 25*1024=25600 self.assertEqual(result[0].n, 731) # Expect some left self.assertTrue(processor.has_items()) output = list(processor.get_items()) self.assertEqual(len(output), 1) self.assertEqual(output[0].size, 9411) self.assertEqual(output[0].n, 269) self.assertFalse(processor.has_items())