async def consumer_main(*, loop, topic_name, consumer_settings,
schema_registry_url):
"""Main asyncio-based function for the single-topic consumer.
"""
logger = structlog.get_logger(__name__)
topic_name = topic_name.replace('_', '-').lower()
async with aiohttp.ClientSession() as httpsession:
schema, schema_id = await get_schema(
topic_name + '-value',
httpsession,
schema_registry_url)
# Start up the Kafka consumer
consumer = aiokafka.AIOKafkaConsumer(loop=loop, **consumer_settings)
# Main loop for consuming messages
try:
await consumer.start()
# Subscribe to all topics in the experiment
consumer.subscribe([topic_name])
logger.info(f'Started consumer for topic {topic_name}')
while True:
async for message in consumer:
value_fh = BytesIO(message.value)
value_fh.seek(0)
value = fastavro.schemaless_reader(
value_fh,
schema)
logger.info("Received message", message=value)
finally:
consumer.stop()
def _create_worker_consumer(
self,
transport: 'Transport',
loop: asyncio.AbstractEventLoop) -> aiokafka.AIOKafkaConsumer:
isolation_level: str = 'read_uncommitted'
conf = self.app.conf
if self.consumer.in_transaction:
isolation_level = 'read_committed'
self._assignor = self.app.assignor
auth_settings = credentials_to_aiokafka_auth(
conf.broker_credentials, conf.ssl_context)
max_poll_interval = conf.broker_max_poll_interval or 0
return aiokafka.AIOKafkaConsumer(
loop=loop,
client_id=conf.broker_client_id,
group_id=conf.id,
bootstrap_servers=server_list(
transport.url, transport.default_port),
partition_assignment_strategy=[self._assignor],
enable_auto_commit=False,
auto_offset_reset=conf.consumer_auto_offset_reset,
max_poll_records=conf.broker_max_poll_records,
max_poll_interval_ms=int(max_poll_interval * 1000.0),
max_partition_fetch_bytes=conf.consumer_max_fetch_size,
fetch_max_wait_ms=1500,
request_timeout_ms=int(conf.broker_request_timeout * 1000.0),
check_crcs=conf.broker_check_crcs,
session_timeout_ms=int(conf.broker_session_timeout * 1000.0),
heartbeat_interval_ms=int(conf.broker_heartbeat_interval * 1000.0),
isolation_level=isolation_level,
traced_from_parent_span=self.traced_from_parent_span,
start_rebalancing_span=self.start_rebalancing_span,
start_coordinator_span=self.start_coordinator_span,
**auth_settings,
)
def _create_worker_consumer(
self, transport: 'Transport',
loop: asyncio.AbstractEventLoop) -> aiokafka.AIOKafkaConsumer:
isolation_level: str = 'read_uncommitted'
conf = self.app.conf
if self.consumer.in_transaction:
isolation_level = 'read_committed'
self._assignor = self.app.assignor
return aiokafka.AIOKafkaConsumer(
loop=loop,
client_id=conf.broker_client_id,
group_id=conf.id,
bootstrap_servers=server_list(transport.url,
transport.default_port),
partition_assignment_strategy=[self._assignor],
enable_auto_commit=False,
auto_offset_reset=conf.consumer_auto_offset_reset,
max_poll_records=conf.broker_max_poll_records,
max_partition_fetch_bytes=conf.consumer_max_fetch_size,
fetch_max_wait_ms=1500,
request_timeout_ms=int(conf.broker_request_timeout * 1000.0),
check_crcs=conf.broker_check_crcs,
session_timeout_ms=int(conf.broker_session_timeout * 1000.0),
heartbeat_interval_ms=int(conf.broker_heartbeat_interval * 1000.0),
security_protocol="SSL" if conf.ssl_context else "PLAINTEXT",
ssl_context=conf.ssl_context,
isolation_level=isolation_level,
)
async def history(request: web.Request):
"""
Get historic data from the given topic
get params:
- start: the start timestamp as milliseconds since 00:00:00 Thursday, 1 January 1970
- end: (optinoal) the end timestamp as milliseconds since 00:00:00 Thursday, 1 January 1970
"""
topic = request.match_info['id']
start = request.query['start']
end = request.query['end'] if 'end' in request.query else 9999999999999
if not isinstance(start, int) or not isinstance(end, int):
raise web.HTTPBadRequest()
if topic not in request.app['topics']:
raise web.HTTPNotFound()
consumer = aiokafka.AIOKafkaConsumer(
topic,
loop=asyncio.get_event_loop(),
api_version='2.2.0',
bootstrap_servers=request.app['settings'].KAFKA_SERVER)
await consumer.start()
topic_partition = aiokafka.TopicPartition(topic, 0)
offsets = await consumer.offsets_for_times({topic_partition: start})
offset = offsets[topic_partition].offset
consumer.seek(topic_partition, offset)
data = topic.encode('utf-8')
try:
async for msg in consumer:
if msg.timestamp > end:
break
data += msg.value
finally:
await consumer.stop()
return web.Response(body=data)
async def kafka_consumer_factory(topic, config):
if config["ssl_context"]:
config = dict(config,
ssl_context=create_ssl_context(**config["ssl_context"]))
consumer = aiokafka.AIOKafkaConsumer(topic, **config)
await consumer.start()
return consumer
async def main(source_topic, sink_topic):
consumer = aiokafka.AIOKafkaConsumer(
group_id="events_cons_explain",
enable_auto_commit=False,
auto_offset_reset="earliest",
)
await consumer.start()
consumer.subscribe([source_topic])
producer = aiokafka.AIOKafkaProducer()
await producer.start()
mem = defaultdict(list)
async def handle_msg(msg):
val = mem[msg.key.decode()]
new_val = int(msg.value.decode())
if (
new_val not in val
): # make idempotent; see the time gap between setting this and the consumer.commit
val.append(new_val)
print(f"Out value {msg.key.decode()} -> {shortlist(val)}")
await producer.send(sink_topic,
value=json.dumps(val).encode(),
key=msg.key)
try:
async for msg in consumer:
print(f"In value {msg.key.decode()} -> {msg.value.decode()}")
await handle_msg(msg)
await consumer.commit()
finally:
await consumer.stop()
await producer.stop()
def create_consumer(self, *topics, **kwargs):
consumer = aiokafka.AIOKafkaConsumer(
*topics,
loop=self.Loop,
bootstrap_servers=self.get_bootstrap_servers(),
enable_auto_commit=False,
**kwargs)
return consumer
def __init__(self, group_id, mem_unique_id, auto_offset_reset, topic):
self._consumer = aiokafka.AIOKafkaConsumer(
group_id=group_id,
enable_auto_commit=False,
auto_offset_reset=auto_offset_reset)
self._topic = topic
self._mem = Mem(group_id_id=mem_unique_id)
self._lock = asyncio.Lock()
self._rbl = RebalanceListener(self._lock, self._mem)
def _setup_connection(self) -> aiokafka.AIOKafkaConsumer:
if self.settings.kafka_ssl_auth:
ssl_context = create_ssl_context(
cafile="init/kafka/ca.pem",
certfile="init/kafka/service.cert",
keyfile="init/kafka/service.key",
)
connection = aiokafka.AIOKafkaConsumer(
*self.settings.metrics_topics.split(","),
bootstrap_servers=self.settings.bootstrap_servers,
ssl_context=ssl_context,
security_protocol="SSL",
)
else:
connection = aiokafka.AIOKafkaConsumer(
*self.settings.metrics_topics.split(","),
bootstrap_servers=self.settings.bootstrap_servers,
)
return connection
def consumer(queue, bootstrap_servers, group_id, name="reader", loop=loop):
"""
consumer returns a wrapped kafka consumer that will reconnect.
"""
return ReconnectingClient(
aiokafka.AIOKafkaConsumer(
queue, bootstrap_servers=bootstrap_servers, group_id=group_id, loop=loop
),
name,
)
async def consume(self, attempts_count=100, initial_timeout=2, max_timeout=120):
loop = asyncio.get_running_loop()
topics = self.dispatcher.received_topics
running_task = None
start_consumer = True
timeout = initial_timeout
consumer = aiokafka.AIOKafkaConsumer(
*topics,
loop=loop,
bootstrap_servers=self.url,
group_id=self.group_id,
enable_auto_commit=False,
auto_offset_reset=self.auto_offset_reset,
**self._aiokafka_options,
)
try:
while True:
try:
if start_consumer:
await consumer.start()
start_consumer = False
timeout = initial_timeout
logger.info("Kafka Consumer started")
async for msg in consumer:
for _ in range(attempts_count):
try:
running_task = asyncio.create_task(
self.handle_msg(msg, consumer)
)
# Must be waited here, otherwise handling will run in
# parallel and will incorrectly commit offsets for not
# finished tasks
await asyncio.shield(running_task)
running_task = None
timeout = initial_timeout
break
except asyncio.CancelledError:
if running_task:
await asyncio.wait({running_task}, timeout=timeout)
raise
except Exception as e:
logger.exception(e)
await asyncio.sleep(timeout)
else:
raise AttemptsFinished()
except (gaierror, KafkaConnectionError):
start_consumer = True
await consumer.stop()
await asyncio.sleep(timeout)
if timeout < max_timeout:
timeout *= 2
finally:
await consumer.stop()
async def __run(self) -> None:
consumer = aiokafka.AIOKafkaConsumer(
bootstrap_servers=cast(List[str], self._app._kafka_servers),
loop=asyncio.get_event_loop(),
group_id=self._subscription.group,
api_version=self._app._kafka_api_version,
**self._app._kafka_settings or {},
)
pattern = fnmatch.translate(self._app.topic_mng.get_topic_id(self._subscription.stream_id))
listener = CustomConsumerRebalanceListener(consumer, self._app, self._subscription.group)
consumer.subscribe(pattern=pattern, listener=listener)
await consumer.start()
msg_handler = _raw_msg_handler
sig = inspect.signature(self._subscription.func)
param_name = [k for k in sig.parameters.keys()][0]
annotation = sig.parameters[param_name].annotation
if annotation and annotation != sig.empty:
if annotation == bytes:
msg_handler = _bytes_msg_handler # type: ignore
elif annotation == aiokafka.structs.ConsumerRecord:
msg_handler = _record_msg_handler # type: ignore
else:
msg_handler = partial(_pydantic_msg_handler, annotation) # type: ignore
await self.emit("started", subscription_consumer=self)
try:
# Consume messages
async for record in consumer:
try:
logger.info(f"Handling msg: {record}")
msg_data = orjson.loads(record.value)
it = iter(sig.parameters.keys())
kwargs: Dict[str, Any] = {next(it): msg_handler(record, msg_data)}
for key in it:
if key == "schema":
kwargs["schema"] = msg_data["schema"]
elif key == "record":
kwargs["record"] = record
await self._subscription.func(**kwargs)
except UnhandledMessage:
# how should we handle this? Right now, fail hard
logger.warning(f"Could not process msg: {record}", exc_info=True)
finally:
await self.emit("message", record=record)
finally:
try:
await consumer.commit()
except Exception:
logger.info("Could not commit current offsets", exc_info=True)
try:
await consumer.stop()
except Exception:
logger.warning("Could not properly stop consumer", exc_info=True)
async def consume_for_simple_topics(*, loop, httpsession, consumer_settings,
schema_registry_url, topic_name):
consumer_settings.update({
'group_id': topic_name,
'client_id': f'{topic_name}-0' # always only one consumer per topic
})
logger = structlog.get_logger(__name__).bind(
role='consumer',
group=consumer_settings['group_id'],
client_id=consumer_settings['client_id'])
logger.info(f'Getting schemas for topic {topic_name}')
registry_headers = {'Accept': 'application/vnd.schemaregistry.v1+json'}
schema_uri = URITemplate(schema_registry_url +
'/subjects{/subject}/versions/latest')
# Get schemas
r = await httpsession.get(schema_uri.expand(subject=f'{topic_name}-key'),
headers=registry_headers)
data = await r.json()
r = await httpsession.get(schema_uri.expand(subject=f'{topic_name}-value'),
headers=registry_headers)
data = await r.json()
value_schema = fastavro.parse_schema(json.loads(data['schema']))
# Start up the Kafka consumer
consumer = aiokafka.AIOKafkaConsumer(loop=loop, **consumer_settings)
# Main loop for consuming messages
try:
await consumer.start()
# Subscribe to all topics in the experiment
consumer.subscribe([topic_name])
logger.info(f'Started consumer for topic {topic_name}')
while True:
async for message in consumer:
value_fh = BytesIO(message.value)
value_fh.seek(0)
value = fastavro.schemaless_reader(value_fh, value_schema)
now = datetime.datetime.now(datetime.timezone.utc)
latency = now - value['timestamp']
latency_millisec = \
latency.seconds * 1000 + latency.microseconds / 1000
CONSUMED.inc() # increment prometheus consumption counter
LATENCY.observe(latency_millisec / 1000)
LATENCY_SUMMARY.observe(latency_millisec / 1000)
logger.debug('latency',
latency_millisec=latency_millisec,
topic=message.topic)
finally:
consumer.stop()
async def spectrogram(request: web.Request):
"""
Create spectogram from data from the given topic
post params:
- start: the start timestamp as milliseconds since 00:00:00 Thursday, 1 January 1970
- end: (optional) the end timestamp as milliseconds since 00:00:00 Thursday, 1 January 1970
- channel: Id of channel in topic that should make the plot
- projectId: Id of the project where plot should be saved
"""
post = await request.post()
datasource_id = request.match_info['id']
datasource = request.app['datasources'].get_source(datasource_id)
input_byte_format = datasource.input_byte_format
topic = datasource.topic
channel_id = int(post.get('channel_id'))
start = int(post.get('start'))
end = int(post.get('end'))
if not isinstance(start, int) or not isinstance(end, int):
print("bad request")
raise web.HTTPBadRequest()
if topic not in request.app['topics']:
print("no topic found")
raise web.HTTPNotFound()
consumer = aiokafka.AIOKafkaConsumer(
topic,
loop=asyncio.get_event_loop(),
api_version='2.2.0',
bootstrap_servers=request.app['settings'].KAFKA_SERVER)
await consumer.start()
topic_partition = aiokafka.TopicPartition(topic, 0)
offsets = await consumer.offsets_for_times({topic_partition: start})
if offsets is None:
raise web.HTTPNotFound()
offset = offsets[topic_partition].offset
consumer.seek(topic_partition, offset)
data = []
try:
async for msg in consumer:
if msg.timestamp > end:
break
for i, message in enumerate(
struct.iter_unpack(input_byte_format, msg.value)):
data.append(message[1 + channel_id])
finally:
await consumer.stop()
if len(data) == 0:
raise web.HTTPBadRequest()
resp = make_spectrogram(data, (end - start) / 1000)
return web.json_response(resp, dumps=dumps)
def _create_client_consumer(
self, app: AppT,
transport: 'Transport') -> aiokafka.AIOKafkaConsumer:
return aiokafka.AIOKafkaConsumer(
loop=self.loop,
client_id=app.conf.broker_client_id,
bootstrap_servers=server_list(transport.url,
transport.default_port),
enable_auto_commit=True,
auto_offset_reset='earliest',
check_crcs=app.conf.broker_check_crcs,
)
def create_consumer(self, *topics, **kwargs):
consumer = aiokafka.AIOKafkaConsumer(
*topics,
loop=self.Loop,
bootstrap_servers=self.get_bootstrap_servers(),
enable_auto_commit=False,
security_protocol=self.Config.get('security_protocol'),
sasl_mechanism=self.Config.get('sasl_mechanism'),
sasl_plain_username=self.Config.get('sasl_plain_username') or None,
sasl_plain_password=self.Config.get('sasl_plain_password') or None,
**kwargs)
return consumer
async def create_consumer_instance(topics, loop, config={}):
print('Creating Kafka consumer')
if Kafka._consumer is None:
if 'bootstrap_servers' not in config:
raise Exception('No bootstrap servers have been configured')
consumer = aiokafka.AIOKafkaConsumer(
loop=loop,
bootstrap_servers=config['bootstrap_servers'],
value_deserializer=deserializer)
await consumer.start()
consumer.subscribe(topics)
Kafka._consumer = consumer
return Kafka._consumer
def consumer_factory(self, group_id: str) -> aiokafka.AIOKafkaConsumer:
return aiokafka.AIOKafkaConsumer(
bootstrap_servers=cast(List[str], self._kafka_servers),
loop=asyncio.get_event_loop(),
group_id=group_id,
api_version=self._kafka_api_version,
enable_auto_commit=False,
**{
k: v
for k, v in self.kafka_settings.items()
if k in _aiokafka_consumer_settings
},
)
def _create_client_consumer(
self, app: AppT,
transport: 'Transport') -> aiokafka.AIOKafkaConsumer:
return aiokafka.AIOKafkaConsumer(
loop=self.loop,
client_id=app.conf.broker_client_id,
bootstrap_servers=server_list(transport.url,
transport.default_port),
enable_auto_commit=True,
auto_offset_reset='earliest',
check_crcs=app.conf.broker_check_crcs,
security_protocol="SSL" if app.conf.ssl_context else "PLAINTEXT",
ssl_context=app.conf.ssl_context,
)
async def fft_from_source(request: web.Request):
datasource_id = request.match_info['id']
datasource = request.app['datasources'].get_source(datasource_id)
input_byte_format = datasource.input_byte_format
metadata = len(datasource.input_names) - len(datasource.output_names)
topic = datasource.topic
channel_id = int(request.match_info['channel_id'])
start = int(request.match_info['start'])
end = int(request.match_info['end'])
sample_spacing = float(request.match_info['sample_spacing'])
print(isinstance(start, int), isinstance(end, int))
if not isinstance(start, int) or not isinstance(end, int):
print("bad request")
raise web.HTTPBadRequest()
if topic not in request.app['topics']:
print("no topic found")
raise web.HTTPNotFound()
consumer = aiokafka.AIOKafkaConsumer(
topic,
loop=asyncio.get_event_loop(),
api_version='2.2.0',
bootstrap_servers=request.app['settings'].KAFKA_SERVER)
await consumer.start()
topic_partition = aiokafka.TopicPartition(topic, 0)
offsets = await consumer.offsets_for_times({topic_partition: start})
if offsets is None:
print("offsets is none")
raise web.HTTPNotFound()
offset = offsets[topic_partition].offset
consumer.seek(topic_partition, offset)
data = []
try:
async for msg in consumer:
if msg.timestamp > end:
break
for i, message in enumerate(
struct.iter_unpack(input_byte_format, msg.value)):
# print("i and value", i, message)
# print("value: ", message[metadata + channel_id])
data.append(message[metadata + channel_id])
finally:
await consumer.stop()
print("data: ", data)
if len(data) == 0:
print("no data (len == 0)")
raise web.HTTPBadRequest()
resp = get_fft(data, sample_spacing)
print("resp", resp)
return web.json_response(resp, dumps=dumps)
async def consume():
consumer = aiokafka.AIOKafkaConsumer("welcomerMain",
loop=loop,
bootstrap_servers="localhost:9092")
await consumer.start()
print("Started consuming")
try:
# Consume messages
async for msg in consumer:
data = msgpack.unpackb(msg.value)
print(json.dumps(data))
# print("consumed: ", msg.topic, msg.partition, msg.offset,
# msg.key, msg.value, msg.timestamp)
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
def __init__(self,
vk,
*topics: typing.Tuple[str],
group_id: str,
bootstrap_servers: str = "localhost"):
if aiokafka:
self._vk = vk
self.consumer = aiokafka.AIOKafkaConsumer(
*topics,
loop=vk.loop,
bootstrap_servers=bootstrap_servers,
group_id=group_id)
else:
raise RuntimeWarning(
"Please install aiokafka (pip install aiokafka) for use this extension"
)
def _create_client_consumer(
self, transport: 'Transport',
loop: asyncio.AbstractEventLoop) -> aiokafka.AIOKafkaConsumer:
conf = self.app.conf
return aiokafka.AIOKafkaConsumer(
loop=loop,
client_id=conf.broker_client_id,
bootstrap_servers=server_list(transport.url,
transport.default_port),
request_timeout_ms=int(conf.broker_request_timeout * 1000.0),
enable_auto_commit=True,
max_poll_records=conf.broker_max_poll_records,
auto_offset_reset=conf.consumer_auto_offset_reset,
check_crcs=conf.broker_check_crcs,
security_protocol="SSL" if conf.ssl_context else "PLAINTEXT",
ssl_context=conf.ssl_context,
)
async def initialize():
log.debug("Initializing the kafka consumer....")
global consumer
global loop
group_id = f'{KAFKA_CONSUMER_GROUP_PREFIX}-{randint(0, 10000)}'
log.debug(f'Initializing KafkaConsumer for topic {KAFKA_TOPIC}'
'group_id {group_id} and using '
'bootstrap servers {KAFKA_BOOTSTRAP_SERVERS}')
consumer = aiokafka.AIOKafkaConsumer(
KAFKA_TOPIC,
loop=loop,
bootstrap_servers=KAFKA_BOOTSTRAP_SERVERS,
group_id=group_id)
# get cluster layout and join group
await consumer.start()
partitions: Set[TopicPartition] = consumer.assignment()
nr_partitions = len(partitions)
if nr_partitions != 1:
log.warning(
f'Found {nr_partitions} partitions for topic {KAFKA_TOPIC}. Expecting '
f'only one, remaining partitions will be ignored!')
for tp in partitions:
# get the log_end_offset
end_offset_dict = await consumer.end_offsets([tp])
end_offset = end_offset_dict[tp]
if end_offset == 0:
log.warning(
f'Topic ({KAFKA_TOPIC}) has no messages (log_end_offset: '
f'{end_offset}), skipping initialization ...')
return
# ACtual code is above one, if there should be no messages in a graceful shutdown
# mode so there is not much other than start consumner thread to start consuming
log.debug(
f'Found log_end_offset: {end_offset} seeking to {end_offset-1}')
consumer.seek(tp, end_offset - 1)
msg = await consumer.getone()
log.info(f'Initializing API with data from msg: {msg}')
return
async def consume_from_kafka(app: web.Application):
"""The function responsible for delivering data to the connected clients."""
consumer = aiokafka.AIOKafkaConsumer(
loop=asyncio.get_event_loop(),
bootstrap_servers=app['settings'].KAFKA_SERVER)
try:
while True:
try:
await consumer.start()
break
except KafkaConnectionError:
logger.exception(
'Could not connect to Kafka server, retrying in 30 seconds'
)
await asyncio.sleep(30)
logger.info('Connected to Kafka server')
consumer.subscribe(pattern='.*')
while True:
try:
messages: Dict[
aiokafka.TopicPartition,
List[aiokafka.ConsumerRecord]] = await consumer.getmany(
timeout_ms=1000)
for topic, topic_messages in messages.items():
for subscriber in app['subscribers'][topic.topic]:
# print("subscriber", topic.topic, subscriber)
await subscriber.receive(
topic.topic,
b''.join(message.value
for message in topic_messages))
except KafkaConnectionError as e:
logger.exception(
'Lost connection to kafka server, waiting 10 seconds before retrying'
)
await asyncio.sleep(10)
except ConnectionAbortedError:
logger.warning(
'Got a connection aborted error when trying to send to a websocket'
)
except asyncio.CancelledError:
pass
except:
logger.exception('Exception in client consumer process')
finally:
await consumer.stop()
def _create_client_consumer(
self, transport: 'Transport',
loop: asyncio.AbstractEventLoop) -> aiokafka.AIOKafkaConsumer:
conf = self.app.conf
auth_settings = credentials_to_aiokafka_auth(conf.broker_credentials,
conf.ssl_context)
return aiokafka.AIOKafkaConsumer(
loop=loop,
client_id=conf.broker_client_id,
bootstrap_servers=server_list(transport.url,
transport.default_port),
request_timeout_ms=int(conf.broker_request_timeout * 1000.0),
enable_auto_commit=True,
max_poll_records=conf.broker_max_poll_records,
auto_offset_reset=conf.consumer_auto_offset_reset,
check_crcs=conf.broker_check_crcs,
**auth_settings,
)
async def consume(out_chan: ac.Chan) -> None:
loop = asyncio.get_running_loop()
consumer = aiokafka.AIOKafkaConsumer('salutations',
loop=loop,
bootstrap_servers='kafka:9092',
group_id="salutated",
auto_offset_reset="earliest")
logging.info("Starting consumer")
await consumer.start()
try:
logging.info("Consumer started")
async for msg in consumer:
out_chan.put((msg.key, msg.value))
except asyncio.CancelledError:
logging.info("Consumer cancelled")
finally:
logging.info("Stopping consumer")
await consumer.stop()
async def initialize():
loop = asyncio.get_event_loop()
global consumer
group_id = f'{KAFKA_CONSUMER_GROUP_PREFIX}-{randint(0, 10000)}'
log.debug(
f'Initializing KafkaConsumer for topic {KAFKA_TOPIC}, group_id {group_id}'
f' and using bootstrap servers {KAFKA_BOOTSTRAP_SERVERS}')
consumer = aiokafka.AIOKafkaConsumer(
KAFKA_TOPIC,
loop=loop,
bootstrap_servers=KAFKA_BOOTSTRAP_SERVERS,
group_id=group_id)
# get cluster layout and join group
await consumer.start()
partitions: Set[TopicPartition] = consumer.assignment()
nr_partitions = len(partitions)
if nr_partitions != 1:
log.warning(
f'Found {nr_partitions} partitions for topic {KAFKA_TOPIC}. Expecting '
f'only one, remaining partitions will be ignored!')
for tp in partitions:
# get the log_end_offset
end_offset_dict = await consumer.end_offsets([tp])
end_offset = end_offset_dict[tp]
if end_offset == 0:
log.warning(
f'Topic ({KAFKA_TOPIC}) has no messages (log_end_offset: '
f'{end_offset}), skipping initialization ...')
return
log.debug(
f'Found log_end_offset: {end_offset} seeking to {end_offset-1}')
consumer.seek(tp, end_offset - 1)
msg = await consumer.getone()
log.info(f'Initializing API with data from msg: {msg}')
# update the API state
_update_state(msg)
return
def __init__(self, app, pipeline, connection, id=None, config=None): super().__init__(app, pipeline, id=id, config=config) topics = re.split(r'\s*,\s*', self.Config['topic']) self._group_id = self.Config['group_id'] if len(self._group_id) == 0: self._group_id = None self.Connection = pipeline.locate_connection(app, connection) self.Consumer = aiokafka.AIOKafkaConsumer( *topics, loop = app.Loop, bootstrap_servers = self.Connection.get_bootstrap_servers(), client_id = self.Config['client_id'], group_id = self._group_id, max_partition_fetch_bytes = int(self.Config['max_partition_fetch_bytes']), auto_offset_reset = self.Config['auto_offset_reset'], api_version = self.Config['api_version'], enable_auto_commit=False )
async def main():
loop = asyncio.get_event_loop()
consumer = aiokafka.AIOKafkaConsumer(
"my_topic4",
loop=loop,
bootstrap_servers="128.0.255.10:9092",
# Consumer must be in a group to commit
group_id="test",
# Autocommit every second
auto_commit_interval_ms=1000,
# If committed offset not found, start from begin
auto_offset_reset="earliest")
await consumer.start()
try:
async for msg in consumer:
print("{}:{:d}:{:d}: key={} value={} timestamp_ms={}".format(
msg.topic, msg.partition, msg.offset, msg.key, msg.value,
msg.timestamp))
finally:
await consumer.stop()