def make_consumer(cls, loop, broker_addr, group_id, sasl_opts={}):
"""
Creates and connects Kafka consumer to the broker
:param loop:
:param broker_addr:
:return:
"""
logger.debug('Creating instance of kafka consumer')
if not sasl_opts:
consumer = AIOKafkaConsumer(loop=loop,
bootstrap_servers=broker_addr,
group_id=group_id,
session_timeout_ms=60000)
else:
consumer = AIOKafkaConsumer(
loop=loop,
bootstrap_servers=broker_addr,
group_id=group_id,
session_timeout_ms=60000,
sasl_mechanism='PLAIN',
sasl_plain_username=sasl_opts['username'],
sasl_plain_password=sasl_opts['password'],
security_protocol='SASL_SSL',
ssl_context=ssl.SSLContext(ssl.PROTOCOL_TLSv1_2))
logger.info('Connected consumer to kafka on {}'.format(broker_addr))
return consumer
async def onJoin(self):
loop = asyncio.get_event_loop()
for handler in self.handlers:
# initialize handler
handler_instance = handler()
handler_instance.set_session(self)
if hasattr(handler_instance, 'init'):
await handler_instance.init()
if hasattr(handler_instance, 'on_event'):
self.log.debug("subscribing to topic %s",
handler_instance.subscribe_topic)
# Used with base handler defined subscribe_topic
if handler_instance.subscribe_topic is not None:
consumer = AIOKafkaConsumer(
handler_instance.subscribe_topic,
bootstrap_servers=self.transport_host,
loop=loop)
await consumer.start()
self.log.debug("subscribed to topic: %s",
handler_instance.subscribe_topic)
try:
async for msg in consumer:
await handler_instance.on_event(msg.value)
finally:
await consumer.stop()
else:
# Used with config.json defined topics
if self.subscribed_topics is not None:
consumer = AIOKafkaConsumer(
bootstrap_servers=self.transport_host,
loop=loop,
group_id='my-group')
await consumer.start()
# Subscribe to all topics
for topic in self.subscribed_topics:
consumer.subscribe(topic)
try:
async for msg in consumer:
value = msg.value.decode()
await handler_instance.on_event(value)
except Exception as error:
self.log.error("Consumer error. %s", error)
await asyncio.sleep(0)
if hasattr(handler_instance, 'worker'):
while True:
try:
await handler_instance.worker()
except Exception as error:
self.log.error("Operation failed. %s", error)
traceback.print_exc(file=sys.stdout)
continue
async def kafka_consumer(
kafka_servers: str,
kafka_topic: str,
queue: asyncio.Queue[Response],
*,
deserializer: Optional[Callable] = None,
kafka_ssl_cafile: str = None,
kafka_ssl_certfile: str = None,
kafka_ssl_keyfile: str = None,
) -> None:
"""
kafka_consumer reads data from kafka and send it to a queue
"""
loop = asyncio.get_event_loop()
kafka_kwargs = {
"loop": loop,
"bootstrap_servers": kafka_servers,
"client_id": "client-storage",
"group_id": "my-group",
"enable_auto_commit": True,
"auto_commit_interval_ms": 1000, # Autocommit every second
"auto_offset_reset": "earliest", # start from beginning
"value_deserializer": deserializer,
}
if not kafka_ssl_cafile:
consumer = AIOKafkaConsumer(kafka_topic, **kafka_kwargs)
else:
context = create_ssl_context(
cafile=kafka_ssl_cafile,
certfile=kafka_ssl_certfile,
keyfile=kafka_ssl_keyfile,
)
consumer = AIOKafkaConsumer(
kafka_topic,
security_protocol="SSL",
ssl_context=context,
**kafka_kwargs,
)
await consumer.start()
try:
# Consume messages
async for msg in consumer:
if msg.value is not None:
logger.debug(f"Message received: {msg.value} at {msg.timestamp}")
try:
asyncio.get_event_loop().call_soon_threadsafe(
queue.put_nowait, msg.value
)
except asyncio.QueueFull as err:
logger.error("queue is full cannot send a response - {}", err)
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
async def consume():
consumer = AIOKafkaConsumer(
'my_topic',
loop=loop,
bootstrap_servers='localhost:9092',
group_id="my-group")
# Get cluster layout and join group `my-group`
await consumer.start()
try:
# Consume messages
msg = await consumer.getone()
logger.info(msg)
logger.info(f'msg.offset = {msg.offset}') # Unique msg autoincrement ID in this topic-partition.
logger.info(f'msg.value = {msg.value}')
tp = TopicPartition(msg.topic, msg.partition)
position = await consumer.position(tp)
# Position is the next fetched offset
assert position == msg.offset + 1
committed = await consumer.committed(tp)
logger.info(f'committed = {committed}')
# print(committed)
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
async def main ():
await init()
where_clause = '''
?submeter rdf:type/rdfs:subClassOf* brick:Meter .
?mainmeter rdf:type/rdfs:subClassOf* brick:Meter .
?mainmeter brick:feedsElectricity ?submeter .
?mainmeter rdf:label ?label .
'''
result = await query('?label', where_clause)
print(json.dumps(result, sort_keys=True, indent=4, separators=(',', ': ')))
print('Found %d matches' % len(result['results']))
topic = result['results'][0][0]
print('Using "%s"' % topic)
consumer = AIOKafkaConsumer(topic, loop=loop, bootstrap_servers=broker)
await consumer.start()
async for msg in consumer:
value = json.loads(msg.value.decode('utf-8'))
print('%d <- %s' % (value, topic))
await consumer.stop()
await finalize()
async def consume(self, ):
consumer = AIOKafkaConsumer(
'start-analysis',
loop=self.loop,
bootstrap_servers=os.environ.get(
'brokerurl'), #dev 3.22.100.122:9092 #prod 13.59.52.241:9092
group_id="1")
# Get cluster layout and join group `my-group`
await consumer.start()
try:
# Consume messages
async for msg in consumer:
data = json.loads(msg.value)
print(data)
#this is logic for clear analysis - should be moved elsewhere
if data['messageId'] == 'clear_analysis':
print(data)
self.deltable.generate_names(
data['params']['cameraId'],
data['params']['iotCoreTopic'])
#message id to drop all these tables
elif data['messageId'] == 'delete_table':
self.drop.generate_tables(data['params']['cameraId'],
data['params']['iotCoreTopic'])
else:
# with open("sample.json", "w") as outfile:
# json.dump(data, outfile)
self.q.put(msg.value)
self.db_q.put(msg.value)
finally:
#Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
async def fetch(to, _from, value, timeout_ms=600 * 1000, loop=loop):
id = str(uuid.uuid4())
consumer = AIOKafkaConsumer(_from, loop=loop, bootstrap_servers=kafka_host)
await consumer.start()
await asyncio.sleep(0.5)
await push(to, value, id)
try:
end_time = time.time() + timeout_ms / 1000
while time.time() <= end_time:
result = await consumer.getmany(timeout_ms=timeout_ms)
for messages in result.values():
for msg in messages:
key = decode_key(msg.key)
if key == id:
await consumer.stop()
return msgpack.unpackb(msg.value, raw=False)
finally:
await consumer.stop()
raise KafkaTransportError("Fetch timeout")
async def produce_and_consume():
# Produce
producer = AIOKafkaProducer(bootstrap_servers='localhost:9093',
security_protocol="SSL",
ssl_context=context)
await producer.start()
try:
msg = await producer.send_and_wait('my_topic',
b"Super Message",
partition=0)
finally:
await producer.stop()
consumer = AIOKafkaConsumer("my_topic",
bootstrap_servers='localhost:9093',
security_protocol="SSL",
ssl_context=context)
await consumer.start()
try:
consumer.seek(TopicPartition('my_topic', 0), msg.offset)
fetch_msg = await consumer.getone()
finally:
await consumer.stop()
print("Success", msg, fetch_msg)
def __init__(self, target, kafka_config):
self.consumer = AIOKafkaConsumer(
kafka_config['topic'],
bootstrap_servers=kafka_config['bootstrap_server'],
group_id=kafka_config['group_id'],
auto_offset_reset='latest')
self.target = target
async def consume(loop):
"""
根据配置文件创建mysql的连接池和kafka的消费者
从kafka中读取数据，反序列化protobuf，通过orm写入mysql数据中
"""
await orm.create_pool(**configs.db, loop=loop)
consumer = AIOKafkaConsumer(
*configs.kafka.topic,
loop=loop,
bootstrap_servers=configs.kafka.bootstrap_servers,
group_id=configs.kafka.group_id,
enable_auto_commit=True,
auto_offset_reset="earliest")
await consumer.start()
try:
batch = []
async for msg in consumer:
logging.info('CONSUMED: %s, %s, %s' %
(msg.topic, msg.partition, msg.offset))
batch.append(msg)
if len(batch) == 100:
await save2db(batch)
batch = []
finally:
await consumer.stop()
def __init__(
self,
hass,
ip_address,
port,
topic,
security_protocol,
username,
password,
):
"""Initialize."""
self._encoder = DateTimeJSONEncoder()
self._hass = hass
ssl_context = ssl_util.client_context()
self._consumer = AIOKafkaConsumer(
topic,
loop=hass.loop,
bootstrap_servers=f"{ip_address}:{port}",
security_protocol=security_protocol,
ssl_context=ssl_context,
sasl_mechanism="PLAIN",
sasl_plain_username=username,
sasl_plain_password=password,
group_id="hassio",
)
self._topic = topic
async def ingest(self, queue: Queue) -> None:
consumer = AIOKafkaConsumer(
self.topic,
group_id=self.group,
auto_offset_reset='earliest',
bootstrap_servers=self.servers,
heartbeat_interval_ms=self.heartbeat_interval_ms,
session_timeout_ms=self.session_timeout_ms,
loop=get_event_loop(),
)
await consumer.start()
self.log.info(f'Monitoring {self.topic} topic for messages...')
async for message in consumer:
msg = json.loads(message.value)
if msg.get('_is_payload'):
# This message is a payload that was placed on the queue
# from the kafka-queue archiver plugin
extra_data = msg['_payload_meta']
extra_data['request_meta'] = msg['_request_meta']
meta = PayloadMeta(extra_data=extra_data)
payload = Payload(content=b64decode(msg['_content']), payload_meta=meta)
await queue.put(payload)
else:
await queue.put(msg)
async def consume_messages() -> None:
"""Listen to Kafka topic and fetch messages.
Connects to Kafka server, consumes a topic and schedules a task for
processing the message.
:return None
"""
logger.info('Connecting to Kafka server...')
logger.info('Client configuration:')
logger.info('\tserver: %s', SERVER)
logger.info('\ttopic: %s', TOPIC)
logger.info('\tgroup_id: %s', GROUP_ID)
logger.info('\tclient_id: %s', CLIENT_ID)
consumer = AIOKafkaConsumer(
TOPIC,
loop=MAIN_LOOP,
client_id=CLIENT_ID,
group_id=GROUP_ID,
bootstrap_servers=SERVER
)
# Get cluster layout, subscribe to group
await consumer.start()
logger.info('Consumer subscribed and active!')
# Start consuming messages
try:
async for msg in consumer:
logger.debug('Received message: %s', str(msg))
MAIN_LOOP.create_task(process_message(msg))
finally:
await consumer.stop()
def __init__(self,
kafka_servers='localhost:9092',
in_topic='aiokafkarpc_in',
out_topic='aiokafkarpc_out',
out_partitions=(0, ),
translation_table=[],
*,
loop):
self.call = CallObj(self._call_wrapper)
self._topic_in = in_topic
self._loop = loop
self._waiters = {}
self._out_topic = out_topic
self._out_partitions = out_partitions
default, ext_hook = get_msgpack_hooks(translation_table)
self.__consumer = AIOKafkaConsumer(
self._out_topic,
loop=loop,
bootstrap_servers=kafka_servers,
group_id=None,
key_deserializer=lambda x: x.decode("utf-8"),
value_deserializer=lambda x: msgpack.unpackb(
x, ext_hook=ext_hook, encoding="utf-8"))
self.__producer = AIOKafkaProducer(
bootstrap_servers=kafka_servers,
loop=loop,
key_serializer=lambda x: x.encode("utf-8"),
value_serializer=lambda x: msgpack.packb(x, default=default))
async def main(loop):
### 初始化消費者
consumer = AIOKafkaConsumer(
topicName,
group_id='hs_server',
loop=loop,
auto_offset_reset = 'earliest',
bootstrap_servers = bootstrap_servers,
api_version="auto",
metadata_max_age_ms=(30 * 1000),
)
### 連接
await consumer.start()
### 獲取消息
data = await consumer.getmany()
print(data)
if data:
for tp, message in data.items():
print('tp>>>', tp)
print(tp.topic)
print('message>>', message)
for msg in message:
print(msg)
print(msg.timestamp, msg.offset)
v = msg.value
print(v)
### 斷開
await consumer.stop()
async def __consume(self, loop) -> _QueueInternalResult:
def deserializer(serialized):
return json.loads(serialized)
try:
consumer = AIOKafkaConsumer(
self._config.optional_param["topic_name"],
loop=loop,
group_id="youyaku_ai_group",
# isolation_level="read_committed",
bootstrap_servers=self._config.get_url(),
value_deserializer=deserializer,
auto_offset_reset="earliest",
enable_auto_commit=False,
)
await consumer.start()
# messageのpositionとoffset（終端）を確認し、データがなければ、空のデータを返す
# TODO : 1パーティションの対応のみなので、パーティションが複数の対応が必要
partition = list(consumer.assignment())[0]
position = await consumer.position(partition=partition)
offset_dict = await consumer.end_offsets(partitions=[partition])
end = offset_dict[partition]
if position == end:
return _QueueInternalResult(result=[], e=None)
# データを一つ取得
data = await consumer.getone()
messages = [data.value]
await consumer.commit()
except Exception as e:
return _QueueInternalResult(result=None, e=e)
finally:
await consumer.stop()
return _QueueInternalResult(result=messages, e=None)
def asyncio_sources_thread(event_loop): # pragma: no cover
"""
Configure Sources listener thread function to run the asyncio event loop.
Args:
event_loop: Asyncio event loop.
Returns:
None
"""
try:
cost_management_type_id = SourcesHTTPClient(Config.SOURCES_FAKE_HEADER).\
get_cost_management_application_type_id()
load_process_queue()
while True:
consumer = AIOKafkaConsumer(
Config.SOURCES_TOPIC,
loop=event_loop, bootstrap_servers=Config.SOURCES_KAFKA_ADDRESS, group_id='hccm-sources'
)
event_loop.create_task(listen_for_messages(consumer, cost_management_type_id, PENDING_PROCESS_QUEUE))
event_loop.create_task(process_messages(PENDING_PROCESS_QUEUE))
event_loop.create_task(synchronize_sources(PROCESS_QUEUE, cost_management_type_id))
event_loop.run_forever()
except SourcesIntegrationError as error:
err_msg = f'Kafka Connection Failure: {str(error)}. Reconnecting...'
LOG.error(err_msg)
time.sleep(Config.RETRY_SECONDS)
except SourcesHTTPClientError as error:
LOG.error(f'Unable to connect to Sources REST API. Check configuration and restart server... Error: {error}')
exit(0)
except KeyboardInterrupt:
exit(0)
async def produce_and_consume():
# Produce
producer = AIOKafkaProducer(loop=asyncio.get_event_loop(),
bootstrap_servers=CONFLUENT_KAFKA_BROKER,
security_protocol='SASL_SSL',
ssl_context=ssl_context,
sasl_mechanism='PLAIN',
sasl_plain_password='******',
sasl_plain_username='******')
await producer.start()
try:
msg = await producer.send_and_wait('test',
b"Super Message",
partition=0)
finally:
await producer.stop()
consumer = AIOKafkaConsumer('test',
loop=asyncio.get_event_loop(),
bootstrap_servers=CONFLUENT_KAFKA_BROKER,
security_protocol='SASL_SSL',
ssl_context=ssl_context,
sasl_mechanism='PLAIN',
sasl_plain_password='******',
sasl_plain_username='******')
await consumer.start()
try:
async for msg in consumer:
print("consumed: ", msg.topic, msg.partition, msg.offset, msg.key,
msg.value, msg.timestamp)
finally:
await consumer.stop()
async def listen_for_order_book_snapshots(self,
ev_loop: asyncio.BaseEventLoop,
output: asyncio.Queue):
"""
Listens to real-time order book snapshot messages from Radar Relay.
"""
while True:
try:
consumer: AIOKafkaConsumer = AIOKafkaConsumer(
self.SNAPSHOT_TOPIC_NAME,
loop=ev_loop,
bootstrap_servers=conf.kafka_2["bootstrap_servers"])
await consumer.start()
partition: TopicPartition = list(consumer.assignment())[0]
await consumer.seek_to_end(partition)
while True:
response: Dict[
TopicPartition,
List[ConsumerRecord]] = await consumer.getmany(
partition, timeout_ms=1000)
if partition in response:
for record in response[partition]:
output.put_nowait(
self.order_book_class.
snapshot_message_from_kafka(record))
except asyncio.CancelledError:
raise
except:
self.logger().error("Unknown error. Retrying after 5 seconds.",
exc_info=True)
await asyncio.sleep(5.0)
async def consume():
consumer = AIOKafkaConsumer(
'test_producer',
group_id="test_group",
loop=loop,
bootstrap_servers='localhost:9092',
auto_offset_reset="earliest",
)
# Get cluster layout
await consumer.start()
try:
# Consume messages
async for msg in consumer:
kafka_output.append({
'key':
msg.key if msg.key is None else msg.key.decode('utf8'),
'value':
msg.value
if msg.value is None else msg.value.decode('utf8')
})
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
async def _listen(
servers=servers,
group=group,
topics=topics,
json=json,
wrap=wrap,
interval=interval,
):
if self._consumer is None:
self._consumer = AIOKafkaConsumer(*topics,
bootstrap_servers=servers,
group_id=group,
**consumer_kwargs)
# Get cluster layout and join group `my-group`
await self._consumer.start()
async for msg in self._consumer:
# Consume messages
# msg.topic, msg.partition, msg.offset, msg.key, msg.value, msg.timestamp
if json:
msg.value = JSON.loads(msg.value)
if wrap:
msg.value = [msg.value]
yield msg
# Will leave consumer group; perform autocommit if enabled.
await self._consumer.stop()
async def check_websites(run_forever: bool = True):
"""Listen to the event stream for any website check events.
:param run_forever: Set the `False` to exit after all the events were processed.
"""
# Setup the Kafka consumer.
consumer = AIOKafkaConsumer(
config.CHECK_WEBSITES_TOPIC,
bootstrap_servers=config.KAFKA_URI,
security_protocol='SSL',
ssl_context=config.get_kafka_security_context(),
group_id='check_websites_group'
)
producer = AIOKafkaProducer(
bootstrap_servers=config.KAFKA_URI,
security_protocol='SSL',
ssl_context=config.get_kafka_security_context()
)
async with consumer, producer:
while True:
async for event in consumer:
# Get the website status.
website = Website(**json.loads(event.value.decode()))
website_status = await check_website_status(website)
# Write the status to the write status event stream.
await producer.send(config.WRITE_WEBSITE_STATUS_TOPIC, website_status.json().encode())
if not run_forever:
break
async def consume_urls(app):
urls_consumer = AIOKafkaConsumer(
'screenshots',
loop=app.get('loop'),
bootstrap_servers=BOOTSTRAP_SERVERS, #["10.199.13.36:9091", "10.199.13.37:9092", "10.199.13.38:9093"],
auto_offset_reset='earliest',
value_deserializer=lambda m: json.loads(m.decode('utf8')),
consumer_timeout_ms=60000*60,
group_id='screenshots_recever_1'
)
await urls_consumer.start()
try:
async for message in urls_consumer:
real_time = '{0:%Y-%m-%d %H:%M:%S}'.format(
datetime.datetime.fromtimestamp((message.timestamp // 1000) - 10)
)
task_id = message.value.get('task_id', '')
url = unquote(message.value.get('url', ''))
screen_id = message.value.get('screen_id', 'empty_id')
await app['to_process'].put({
'value': message.value,
'task_id':task_id,
'real_time':real_time,
'url':url,
'screen_id':screen_id,
'timestamp': message.timestamp
})
except:
app['logger'].debug('urls_consumer exception')
finally:
app['logger'].debug('urls_consumer stopped')
await urls_consumer.stop()
def __init__(self, topic, group_id=GROUP_ID, bootstrap_servers=BOOTSTRAP_SERVERS, **kwargs):
self.loop = asyncio.get_event_loop()
if isinstance(topic, str):
topic = [topic]
consumer = AIOKafkaConsumer(*topic, loop=self.loop, bootstrap_servers=bootstrap_servers, group_id=group_id,
auto_offset_reset='latest', session_timeout_ms=30000, **kwargs)
super(MQReader, self).__init__(consumer, 'AIOKafka Consumer')
async def bench_simple(self):
topic = self._topic
loop = asyncio.get_event_loop()
consumer = AIOKafkaConsumer(topic,
group_id="test_group",
auto_offset_reset="earliest",
enable_auto_commit=False,
bootstrap_servers=self._bootstrap_servers,
loop=loop)
await consumer.start()
# We start from after producer connect
reporter_task = loop.create_task(self._stats_report(loop.time()))
try:
total_msgs = 0
while True:
msg_set = await consumer.getmany(timeout_ms=1000)
if not msg_set:
break
for msgs in msg_set.values():
len_msgs = len(msgs)
self._stats[-1]['count'] += len_msgs
total_msgs += len_msgs
if total_msgs > self._num:
break
except asyncio.CancelledError:
pass
finally:
await consumer.stop()
reporter_task.cancel()
await reporter_task
def asyncio_worker_thread(loop): # pragma: no cover
"""
Worker thread function to run the asyncio event loop.
Args:
None
Returns:
None
"""
count = 0
try:
while True:
consumer = AIOKafkaConsumer(
HCCM_TOPIC,
loop=EVENT_LOOP,
bootstrap_servers=Config.INSIGHTS_KAFKA_ADDRESS,
group_id="hccm-group")
loop.create_task(process_messages())
try:
loop.run_until_complete(listen_for_messages(consumer))
except KafkaMsgHandlerError as err:
LOG.info("Kafka connection failure. Error: %s", str(err))
backoff(count, Config.INSIGHTS_KAFKA_CONN_RETRY_MAX)
count += 1
LOG.info("Attempting to reconnect")
except KeyboardInterrupt:
exit(0)
async def consume():
logger.info(
f"Creating MongoDB connection: "
f"DB {settings.MONGO_DB_NAME} | COLLECTION {settings.MONGO_COLLECTION}"
)
db_client = motor.motor_asyncio.AsyncIOMotorClient(settings.MONGO_DB_CONN)
db = db_client[settings.MONGO_DB_NAME]
collection = db[settings.MONGO_COLLECTION]
logger.info("Creating consumer...")
consumer = AIOKafkaConsumer(settings.KAFKA_TOPIC,
bootstrap_servers=settings.KAFKA_CONN,
group_id="my-group")
logger.info(f"Get cluster layout and join group `my-group`...")
await consumer.start()
logger.info(
f"Started consuming for {settings.KAFKA_CONN} [{settings.KAFKA_TOPIC}]..."
)
try:
# Consume messages
async for msg in consumer:
logger.info(f"consumed:\n"
f" topic: {msg.topic}\n"
f" partition: {msg.partition}\n"
f" offset: {msg.offset}\n"
f" key: {msg.key}\n"
f" value: {msg.value}\n"
f" timestamp: {msg.timestamp}\n"
f" headers: {msg.headers}\n")
msg = json.loads(msg.value.decode("utf-8"))
await update_playlist(collection, playlist_id=msg["id"])
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
async def _get_kafka_messages(topic: str, start: int) -> List[ConsumerRecord]:
def _value_deserializer(value):
value = value.decode("utf-8")
try:
return json.loads(value)
except JSONDecodeError:
return ast.literal_eval(value)
loop = asyncio.get_event_loop()
consumer = AIOKafkaConsumer(
topic, value_deserializer=_value_deserializer,
loop=loop, bootstrap_servers=settings.KAFKA_SERVER,
)
await consumer.start()
try:
partitions = consumer.partitions_for_topic(topic)
tps = [TopicPartition(topic, p) for p in partitions]
offsets = await consumer.offsets_for_times({tp: start for tp in tps})
for tp, offset in offsets.items():
offset = offset.offset if offset else (await consumer.end_offsets([tp]))[tp]
consumer.seek(tp, offset)
records = await consumer.getmany(*tps, timeout_ms=1000*60)
messages = []
for tp in tps:
messages += records.get(tp, [])
logger.info(f"Got kafka messages {messages} by key {topic}")
return messages
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
async def _consumer_benchmark(brokers, topic, num_messages, msg_size, num_runs):
loop = asyncio.get_event_loop()
consumer = AIOKafkaConsumer(
topic, group_id=str(uuid.uuid1()),
auto_offset_reset="earliest",
enable_auto_commit=False,
loop=loop
)
await consumer.start()
print("Starting benchmark for AIOKafka Consumer.")
run_times = []
try:
for _ in range(num_runs):
run_start_time = time.time()
await _consume(consumer, num_messages)
run_time_taken = time.time() - run_start_time
run_times.append(run_time_taken)
except asyncio.CancelledError:
pass
finally:
await consumer.stop()
utils.print_results(
"AIOKafka Consumer", run_times, num_messages, msg_size
)
async def main():
data_array_size = 10
try:
consumer = AIOKafkaConsumer(
'noaa-json-us-az',
loop=loop,
bootstrap_servers='localhost:9092',
group_id="alert-group-v4",
auto_offset_reset="earliest" # earliest or latest
)
# Get cluster layout and join group
await consumer.start()
producer = AIOKafkaProducer(loop=loop,
bootstrap_servers='localhost:9092')
# Get cluster layout and initial topic/partition leadership information
await producer.start()
while True:
data_array = await consume(consumer, data_array_size)
await produce(producer, process(data_array))
# sleep(0.2)
finally:
await consumer.stop()
await producer.stop()
