class SendLoop(object): def __init__(self): # ev_loop self.loop = asyncio.get_event_loop() asyncio.set_event_loop(self.loop) # kafka producer self.producer = AIOKafkaProducer( loop=self.loop, bootstrap_servers='127.0.0.1:9092', value_serializer=None, max_batch_size=32768) self.start() def start(self): self.loop.run_until_complete(self.producer.start()) asyncio.ensure_future(self.send_batch_loop()) self.loop.run_forever() async def producer_start(self): await self.producer.start() async def send_one_loop(self): asyncio.ensure_future(self.send_one_loop()) await self.producer.send('test', b'test message') async def send_batch_loop(self): asyncio.ensure_future(self.send_batch_loop()) batch = self.producer.create_batch() for index in range(1024): batch.append(value=b'test message', key=None, timestamp=None) await self.producer.send_batch(batch, 'test', partition=random.choice(range(0,100)))
async def produce_msg(self): """启动生产者并发送消息""" # 定义一个生产者 producer = AIOKafkaProducer(loop=self.loop, bootstrap_servers=self.hosts) await producer.start() copied_list = self.msg_list.copy() # 每次发送消息之后, 把原列表的内容清空, 避免发送重复的消息 self.msg_list = list() try: batch = producer.create_batch() for msg in copied_list: metadata = batch.append(key=None, value=msg.encode('utf-8'), timestamp=None) if metadata is None: partitions = await producer.partitions_for(self.topic) partition = random.choice(tuple(partitions)) await producer.send_batch(batch, self.topic, partition=partition) logger.warning( f'发送 {batch.record_count()} 条消息到 topic:{self.topic}, partition:{partition}' ) batch = producer.create_batch() partitions = await producer.partitions_for(self.topic) partition = random.choice(tuple(partitions)) await producer.send_batch(batch, self.topic, partition=partition) logger.warning( f'发送 {len(copied_list)} 条消息:{copied_list} 到主题:[{self.topic}], partition:{partition}' ) finally: # 等待所有待发消息被发送出去或者过期 await producer.stop()
async def produce_rand_int(): producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092') # Get cluster layout and initial topic/partition leadership information await producer.start() try: batch = producer.create_batch() # Populate the batch. The append() method will return metadata for the # added message or None if batch is full. for i in range(2): metadata = batch.append(value=b"msg %04d" % i, key=None, timestamp=None) assert metadata is not None # Optionally close the batch to further submission. If left open, the batch # may be appended to by producer.send(). batch.close() # Add the batch to the first partition's submission queue. If this method # times out, we can say for sure that batch will never be sent. fut = await producer.send_batch(batch, "my_topic", partition=1) # Batch will either be delivered or an unrecoverable error will occur. # Cancelling this future will not cancel the send. record = await fut logger.info(record) finally: # Wait for all pending messages to be delivered or expire. await producer.stop()