async def _get_kafka_messages(topic: str, start: int) -> List[ConsumerRecord]:
def _value_deserializer(value):
value = value.decode("utf-8")
try:
return json.loads(value)
except JSONDecodeError:
return ast.literal_eval(value)
loop = asyncio.get_event_loop()
consumer = AIOKafkaConsumer(
topic, value_deserializer=_value_deserializer,
loop=loop, bootstrap_servers=settings.KAFKA_SERVER,
)
await consumer.start()
try:
partitions = consumer.partitions_for_topic(topic)
tps = [TopicPartition(topic, p) for p in partitions]
offsets = await consumer.offsets_for_times({tp: start for tp in tps})
for tp, offset in offsets.items():
offset = offset.offset if offset else (await consumer.end_offsets([tp]))[tp]
consumer.seek(tp, offset)
records = await consumer.getmany(*tps, timeout_ms=1000*60)
messages = []
for tp in tps:
messages += records.get(tp, [])
logger.info(f"Got kafka messages {messages} by key {topic}")
return messages
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
async def produce_and_consume():
# Produce
producer = AIOKafkaProducer(bootstrap_servers='localhost:9093',
security_protocol="SSL",
ssl_context=context)
await producer.start()
try:
msg = await producer.send_and_wait('my_topic',
b"Super Message",
partition=0)
finally:
await producer.stop()
consumer = AIOKafkaConsumer("my_topic",
bootstrap_servers='localhost:9093',
security_protocol="SSL",
ssl_context=context)
await consumer.start()
try:
consumer.seek(TopicPartition('my_topic', 0), msg.offset)
fetch_msg = await consumer.getone()
finally:
await consumer.stop()
print("Success", msg, fetch_msg)
async def seek_to_offset(consumer: AIOKafkaConsumer, topic: str, start: int = -1):
"""
Seek to the last message in topic.
"""
partition_number, offset = -1, -1
# Loop through partitions and find the latest offset
for p in consumer.partitions_for_topic(topic):
tp = TopicPartition(topic, p)
committed = await consumer.committed(tp)
await consumer.seek_to_end(tp)
last_offset = await consumer.position(tp)
# print("topic: {} partition: {} committed: {} last: {}".format(topic, p, committed, last_offset))
if offset < last_offset:
offset = last_offset
partition_number = p
tp = TopicPartition(topic, partition_number)
consumer.seek(tp, offset - start)
async def _pull(self, partitions):
logging.info(f"thread:{self.name} pull start... ")
try:
consumer = AIOKafkaConsumer(loop=asyncio.get_running_loop(),
bootstrap_servers=self.queue_brokers,
value_deserializer=_value_serializer,
enable_auto_commit=True)
await consumer.start()
consumer.assign(list(partitions.keys()))
for k in partitions:
offset = partitions[k]
if offset > 0:
consumer.seek(k, offset)
async for msg in consumer:
if self._running is False:
logging.info(f"thread:{self.name} stop running")
break
acr_meta = await self._process(msg)
if acr_meta is not None:
logging.debug(
f"thread:{self.name} pull msg {msg.topic}, {msg.partition}"
)
await self.acr_queue.put(acr_meta)
except asyncio.CancelledError:
logging.info(f"thread:{self.name} pull stop")
except:
logging.error(
f"thread:{self.name} exception stop {traceback.format_exc()}")
logging.info(f"thread:{self.name} over... ")
async def produce_and_consume(loop):
# Produce
producer = AIOKafkaProducer(
loop=loop, bootstrap_servers='localhost:9093',
security_protocol="SSL", ssl_context=context)
await producer.start()
try:
msg = await producer.send_and_wait(
'my_topic', b"Super Message", partition=0)
finally:
await producer.stop()
consumer = AIOKafkaConsumer(
"my_topic", loop=loop, bootstrap_servers='localhost:9093',
security_protocol="SSL", ssl_context=context)
await consumer.start()
try:
consumer.seek(TopicPartition('my_topic', 0), msg.offset)
fetch_msg = await consumer.getone()
finally:
await consumer.stop()
print("Success", msg, fetch_msg)
