def consume_topic(broker, topic, start_from_oldest=False, truncate=False): consumer = KafkaConsumer({ "bootstrap.servers": broker, "group.id": f"get-topic-{time.time_ns()}", "auto.offset.reset": "earliest" if start_from_oldest else "latest", }) metadata = consumer.list_topics(topic) if topic not in metadata.topics: raise Exception("Topic does not exist") topic_partitions = [ TopicPartition(topic, p) for p in metadata.topics[topic].partitions ] consumer.assign(topic_partitions) while True: msg = consumer.poll(0.0) if msg: value = msg.value()[0:100] if truncate else msg.value() print(f"Timestamp: {msg.timestamp()[1]}\n{value}") time.sleep(0.1)
def __createConsumer(self): if self.__shouldRun(): config = { 'metadata.broker.list': ','.join(self.brokers), 'group.id': self.trigger, 'default.topic.config': { 'auto.offset.reset': 'latest' }, 'enable.auto.commit': False, 'api.version.request': True } if self.isMessageHub: # append Message Hub specific config config.update({ 'ssl.ca.location': '/etc/ssl/certs/', 'sasl.mechanisms': 'PLAIN', 'sasl.username': self.username, 'sasl.password': self.password, 'security.protocol': 'sasl_ssl' }) consumer = KafkaConsumer(config) consumer.subscribe([self.topic], self.__on_assign, self.__on_revoke) logging.info("[{}] Now listening in order to fire trigger".format( self.trigger)) return consumer
def __init__(self, kafka_client_config): print("=" * 50) print("Printing Kafka_Confluent kwargs...") import pprint pp = pprint.PrettyPrinter(indent=4) pp.pprint(kafka_client_config) print("=" * 50) self.broker = kafka_client_config["broker"] self.producer_params = kafka_client_config["producer_params"] self.consumer_1_params = kafka_client_config["consumer_1_params"] self.consumer_2_params = kafka_client_config["consumer_2_params"] self.producer_topic = kafka_client_config.get('producer_topic') self.consumer_1_topic = kafka_client_config.get('consumer_1_topic') self.consumer_2_topic = kafka_client_config.get('consumer_2_topic') self.producer = None self.consumer_1 = None self.consumer_2 = None # Create Producer if (self.producer_topic): self.producer_params['bootstrap.servers'] = kafka_client_config[ "broker"] self.producer = KafkaProducer(self.producer_params) print("Producer created successfully...") # Create Consumer 1 if (self.consumer_1_topic): self.consumer_1_params['bootstrap.servers'] = kafka_client_config[ "broker"] self.consumer_1 = KafkaConsumer(self.consumer_1_params) self.consumer_1.subscribe([self.consumer_1_topic]) self.consumer_1.poll(timeout=0.01) print("Consumer 1 created successfully...") # Create Consumer 2 if (self.consumer_2_topic): self.consumer_2_params['bootstrap.servers'] = kafka_client_config[ "broker"] self.consumer_2 = KafkaConsumer(self.consumer_2_params) self.consumer_2.subscribe([self.consumer_2_topic]) self.consumer_2.poll(timeout=0.01) print("Consumer 1 created successfully...")
def get_topics(broker): consumer = KafkaConsumer({ "bootstrap.servers": broker, "group.id": f"get-topic-{time.time_ns()}", "auto.offset.reset": "latest", }) metadata = consumer.list_topics() for n, v in metadata.topics.items(): print(f"{n} {len(v.partitions)}")
def __init__(self, broker, schema_registry, topic=None, logging_enabled=False, group_id=None, auto_commit=True): """ Initialiser for Confluent Consumer using AvroConsumer. Each consumer can only be subscribed to one topic Parameters ---------- broker: str The URL of the broker (example: 'localhost:9092') schema_registry: str The URL of the confluent Schema Registry endpoint (example: 'http://localhost:8081') topic: str The topic to subscribe too logger: Logger object, Optional The logger object which will be used to log messages if provided groupId: str, Optional An optional groupId which can be used to loadbalance consumers default is "asgard" """ if group_id is None: new_hash = hashlib.sha1() new_hash.update(str(time.time()).encode("utf-8")) group_id = new_hash.hexdigest() self.__consumer = AvroConsumer({ "bootstrap.servers": broker, "group.id": group_id, "schema.registry.url": schema_registry, "enable.auto.commit": auto_commit }) self.__consumer_non_avro = KafkaConsumer({ "bootstrap.servers": broker, "group.id": group_id + "0", "enable.auto.commit": auto_commit }) self.auto_commit = auto_commit if not auto_commit: self.consumed_messages = PriorityQueue() if not topic is None: self.subscribe_to_topic(topic) else: self.topic = None if logging_enabled: self.logger = logging.getLogger(__name__) else: self.logger = None
def __init__(self, broker, schema_registry, topic, logging_enabled=False, groupId="asgardConsumerGroup", autocommit=True): """ Initialiser for Confluent Consumer using AvroConsumer. Each consumer can only be subscribed to one topic Parameters ---------- broker: str The URL of the broker (example: 'localhost:9092') schema_registry: str The URL of the confluent Schema Registry endpoint (example: 'http://localhost:8081') topic: str The topic to subscribe too logger: Logger object, Optional The logger object which will be used to log messages if provided groupId: str, Optional An optional groupId which can be used to loadbalance consumers default is "asgard" """ """self.__consumer = AvroConsumer( { "bootstrap.servers": broker, "group.id": groupId, "schema.registry.url": schema_registry, "enable.auto.commit": autocommit } )""" self.__consumer = KafkaConsumer({ "bootstrap.servers": broker, "group.id": groupId, "enable.auto.commit": autocommit, "auto.offset.reset": "latest" }) self.autocommit = autocommit if not autocommit: self.consumed_messages = SimpleQueue() self.__consumer.subscribe([topic]) if logging_enabled: self.logger = logging.getLogger(__name__) else: self.logger = None
def __init__(self, bootstrap_servers, topics, logger=None): # setup logger self.logger = logger or logging.getLogger(__name__) if logger == None: self.logger.setLevel(logging.INFO) self.logger.addHandler(logging.StreamHandler()) # connect to kafka cluster after setting kafka consumer's config kc_config = { "bootstrap.servers": bootstrap_servers, "on_commit": self.on_commit, "group.id": GROUP_ID } self.consumer = KafkaConsumer(kc_config) self.logger.info("Consumer configured") # subscribe to topic self.consumer.subscribe(topics, on_assign=self.on_assignment) self.num_msgs = 0 # counter
def __init__(self, config, topics): self.client = KafkaConsumer(config) self.client.subscribe(topics)
def configure(self) -> None: self.kafka_consumer = KafkaConsumer(self._config)