def consume_topic(broker, topic, start_from_oldest=False, truncate=False):
consumer = KafkaConsumer({
"bootstrap.servers":
broker,
"group.id":
f"get-topic-{time.time_ns()}",
"auto.offset.reset":
"earliest" if start_from_oldest else "latest",
})
metadata = consumer.list_topics(topic)
if topic not in metadata.topics:
raise Exception("Topic does not exist")
topic_partitions = [
TopicPartition(topic, p) for p in metadata.topics[topic].partitions
]
consumer.assign(topic_partitions)
while True:
msg = consumer.poll(0.0)
if msg:
value = msg.value()[0:100] if truncate else msg.value()
print(f"Timestamp: {msg.timestamp()[1]}\n{value}")
time.sleep(0.1)
def __createConsumer(self):
if self.__shouldRun():
config = {
'metadata.broker.list': ','.join(self.brokers),
'group.id': self.trigger,
'default.topic.config': {
'auto.offset.reset': 'latest'
},
'enable.auto.commit': False,
'api.version.request': True
}
if self.isMessageHub:
# append Message Hub specific config
config.update({
'ssl.ca.location': '/etc/ssl/certs/',
'sasl.mechanisms': 'PLAIN',
'sasl.username': self.username,
'sasl.password': self.password,
'security.protocol': 'sasl_ssl'
})
consumer = KafkaConsumer(config)
consumer.subscribe([self.topic], self.__on_assign,
self.__on_revoke)
logging.info("[{}] Now listening in order to fire trigger".format(
self.trigger))
return consumer
def __init__(self, kafka_client_config):
print("=" * 50)
print("Printing Kafka_Confluent kwargs...")
import pprint
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(kafka_client_config)
print("=" * 50)
self.broker = kafka_client_config["broker"]
self.producer_params = kafka_client_config["producer_params"]
self.consumer_1_params = kafka_client_config["consumer_1_params"]
self.consumer_2_params = kafka_client_config["consumer_2_params"]
self.producer_topic = kafka_client_config.get('producer_topic')
self.consumer_1_topic = kafka_client_config.get('consumer_1_topic')
self.consumer_2_topic = kafka_client_config.get('consumer_2_topic')
self.producer = None
self.consumer_1 = None
self.consumer_2 = None
# Create Producer
if (self.producer_topic):
self.producer_params['bootstrap.servers'] = kafka_client_config[
"broker"]
self.producer = KafkaProducer(self.producer_params)
print("Producer created successfully...")
# Create Consumer 1
if (self.consumer_1_topic):
self.consumer_1_params['bootstrap.servers'] = kafka_client_config[
"broker"]
self.consumer_1 = KafkaConsumer(self.consumer_1_params)
self.consumer_1.subscribe([self.consumer_1_topic])
self.consumer_1.poll(timeout=0.01)
print("Consumer 1 created successfully...")
# Create Consumer 2
if (self.consumer_2_topic):
self.consumer_2_params['bootstrap.servers'] = kafka_client_config[
"broker"]
self.consumer_2 = KafkaConsumer(self.consumer_2_params)
self.consumer_2.subscribe([self.consumer_2_topic])
self.consumer_2.poll(timeout=0.01)
print("Consumer 1 created successfully...")
def get_topics(broker):
consumer = KafkaConsumer({
"bootstrap.servers": broker,
"group.id": f"get-topic-{time.time_ns()}",
"auto.offset.reset": "latest",
})
metadata = consumer.list_topics()
for n, v in metadata.topics.items():
print(f"{n} {len(v.partitions)}")
def __init__(self,
broker,
schema_registry,
topic=None,
logging_enabled=False,
group_id=None,
auto_commit=True):
"""
Initialiser for Confluent Consumer using AvroConsumer.
Each consumer can only be subscribed to one topic
Parameters
----------
broker: str
The URL of the broker (example: 'localhost:9092')
schema_registry: str
The URL of the confluent Schema Registry endpoint (example: 'http://localhost:8081')
topic: str
The topic to subscribe too
logger: Logger object, Optional
The logger object which will be used to log messages if provided
groupId: str, Optional
An optional groupId which can be used to loadbalance consumers default is "asgard"
"""
if group_id is None:
new_hash = hashlib.sha1()
new_hash.update(str(time.time()).encode("utf-8"))
group_id = new_hash.hexdigest()
self.__consumer = AvroConsumer({
"bootstrap.servers": broker,
"group.id": group_id,
"schema.registry.url": schema_registry,
"enable.auto.commit": auto_commit
})
self.__consumer_non_avro = KafkaConsumer({
"bootstrap.servers":
broker,
"group.id":
group_id + "0",
"enable.auto.commit":
auto_commit
})
self.auto_commit = auto_commit
if not auto_commit:
self.consumed_messages = PriorityQueue()
if not topic is None:
self.subscribe_to_topic(topic)
else:
self.topic = None
if logging_enabled:
self.logger = logging.getLogger(__name__)
else:
self.logger = None
def __init__(self,
broker,
schema_registry,
topic,
logging_enabled=False,
groupId="asgardConsumerGroup",
autocommit=True):
"""
Initialiser for Confluent Consumer using AvroConsumer.
Each consumer can only be subscribed to one topic
Parameters
----------
broker: str
The URL of the broker (example: 'localhost:9092')
schema_registry: str
The URL of the confluent Schema Registry endpoint (example: 'http://localhost:8081')
topic: str
The topic to subscribe too
logger: Logger object, Optional
The logger object which will be used to log messages if provided
groupId: str, Optional
An optional groupId which can be used to loadbalance consumers default is "asgard"
"""
"""self.__consumer = AvroConsumer(
{
"bootstrap.servers": broker,
"group.id": groupId,
"schema.registry.url": schema_registry,
"enable.auto.commit": autocommit
}
)"""
self.__consumer = KafkaConsumer({
"bootstrap.servers": broker,
"group.id": groupId,
"enable.auto.commit": autocommit,
"auto.offset.reset": "latest"
})
self.autocommit = autocommit
if not autocommit:
self.consumed_messages = SimpleQueue()
self.__consumer.subscribe([topic])
if logging_enabled:
self.logger = logging.getLogger(__name__)
else:
self.logger = None
def __init__(self, bootstrap_servers, topics, logger=None):
# setup logger
self.logger = logger or logging.getLogger(__name__)
if logger == None:
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
# connect to kafka cluster after setting kafka consumer's config
kc_config = {
"bootstrap.servers": bootstrap_servers,
"on_commit": self.on_commit,
"group.id": GROUP_ID
}
self.consumer = KafkaConsumer(kc_config)
self.logger.info("Consumer configured")
# subscribe to topic
self.consumer.subscribe(topics, on_assign=self.on_assignment)
self.num_msgs = 0 # counter
def __init__(self, config, topics):
self.client = KafkaConsumer(config)
self.client.subscribe(topics)
def configure(self) -> None:
self.kafka_consumer = KafkaConsumer(self._config)
