class SendLoop(object):
def __init__(self):
# ev_loop
self.loop = asyncio.get_event_loop()
asyncio.set_event_loop(self.loop)
# kafka producer
self.producer = AIOKafkaProducer(
loop=self.loop,
bootstrap_servers='127.0.0.1:9092',
value_serializer=None,
max_batch_size=32768)
self.start()
def start(self):
self.loop.run_until_complete(self.producer.start())
asyncio.ensure_future(self.send_batch_loop())
self.loop.run_forever()
async def producer_start(self):
await self.producer.start()
async def send_one_loop(self):
asyncio.ensure_future(self.send_one_loop())
await self.producer.send('test', b'test message')
async def send_batch_loop(self):
asyncio.ensure_future(self.send_batch_loop())
batch = self.producer.create_batch()
for index in range(1024):
batch.append(value=b'test message', key=None, timestamp=None)
await self.producer.send_batch(batch, 'test', partition=random.choice(range(0,100)))
async def produce_msg(self):
"""启动生产者并发送消息"""
# 定义一个生产者
producer = AIOKafkaProducer(loop=self.loop,
bootstrap_servers=self.hosts)
await producer.start()
copied_list = self.msg_list.copy()
# 每次发送消息之后, 把原列表的内容清空, 避免发送重复的消息
self.msg_list = list()
try:
batch = producer.create_batch()
for msg in copied_list:
metadata = batch.append(key=None,
value=msg.encode('utf-8'),
timestamp=None)
if metadata is None:
partitions = await producer.partitions_for(self.topic)
partition = random.choice(tuple(partitions))
await producer.send_batch(batch,
self.topic,
partition=partition)
logger.warning(
f'发送 {batch.record_count()} 条消息到 topic:{self.topic}, partition:{partition}'
)
batch = producer.create_batch()
partitions = await producer.partitions_for(self.topic)
partition = random.choice(tuple(partitions))
await producer.send_batch(batch, self.topic, partition=partition)
logger.warning(
f'发送 {len(copied_list)} 条消息：{copied_list} 到主题：[{self.topic}], partition:{partition}'
)
finally:
# 等待所有待发消息被发送出去或者过期
await producer.stop()
async def produce_rand_int():
producer = AIOKafkaProducer(loop=loop, bootstrap_servers='localhost:9092')
# Get cluster layout and initial topic/partition leadership information
await producer.start()
try:
batch = producer.create_batch()
# Populate the batch. The append() method will return metadata for the
# added message or None if batch is full.
for i in range(2):
metadata = batch.append(value=b"msg %04d" % i,
key=None,
timestamp=None)
assert metadata is not None
# Optionally close the batch to further submission. If left open, the batch
# may be appended to by producer.send().
batch.close()
# Add the batch to the first partition's submission queue. If this method
# times out, we can say for sure that batch will never be sent.
fut = await producer.send_batch(batch, "my_topic", partition=1)
# Batch will either be delivered or an unrecoverable error will occur.
# Cancelling this future will not cancel the send.
record = await fut
logger.info(record)
finally:
# Wait for all pending messages to be delivered or expire.
await producer.stop()
