def main(args): topic = args.topic protobuf_deserializer = ProtobufDeserializer(user_pb2.User) string_deserializer = StringDeserializer('utf_8') consumer_conf = {'bootstrap.servers': args.bootstrap_servers, 'key.deserializer': string_deserializer, 'value.deserializer': protobuf_deserializer, 'group.id': args.group, 'auto.offset.reset': "earliest"} consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([topic]) while True: try: # SIGINT can't be handled when polling, limit timeout to 1 second. msg = consumer.poll(1.0) if msg is None: continue user = msg.value() if user is not None: print("User record {}: name: {}\n" "\tfavorite_number: {}\n" "\tfavorite_color: {}\n" .format(msg.key(), user.name, user.favorite_color, user.favorite_number)) except KeyboardInterrupt: break consumer.close()
def kafpubsub(args): publisher = pubsub.PublisherClient() project_id = args.project kafka_topic = args.topic pubsub_topic = f'projects/{project_id}/topics/{kafka_topic}' try: publisher.create_topic(pubsub_topic) except AlreadyExists: pass # I don't need an error if topic already created. consumer_conf = { 'bootstrap.servers': args.bootstrap_server, 'group.id': args.group_id, 'auto.offset.reset': args.auto_offset_reset } consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([kafka_topic]) logging.info( f'Publish Kafka ({args.bootstrap_server}) values to pubsub...') while True: try: # SIGINT can't be handled when polling, limit timeout to 1 second. msg = consumer.poll(1.0) if msg is None: continue logging.debug(f'> {datetime.today()} | {msg.key()}\n') publisher.publish(pubsub_topic, msg.value()) except KeyboardInterrupt: break consumer.close()
def main(args): topic = args.topic protobuf_deserializer = ProtobufDeserializer(user_pb2.User) string_deserializer = StringDeserializer('utf_8') consumer_conf = {'bootstrap.servers': args.bootstrap_servers, 'key.deserializer': string_deserializer, 'value.deserializer': protobuf_deserializer, 'group.id': args.group, 'auto.offset.reset': "earliest"} consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([topic]) while True: try: msg = consumer.poll(1.0) if msg is None: continue user = msg.value() if user is not None: print(f"User record {msg.key()}:\n name: {user.name}\n" f"\tfavorite_number: {user.favorite_color}\n" f"\tfavorite_color: {user.favorite_number}\n") except KeyboardInterrupt: break consumer.close()
def receive(): json_deserializer = JSONDeserializer(USER_SCHEMA, from_dict=dict_to_user) string_deserializer = StringDeserializer('utf_8') consumer_conf = { 'bootstrap.servers': 'localhost:9092', 'key.deserializer': string_deserializer, 'value.deserializer': json_deserializer, 'group.id': 'django-kafka', 'auto.offset.reset': "earliest" } consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([USER_TOPIC]) """ The idea is to start the Kafka consumer when the message is sent to the Kafka producer. Resulting in two queues: Task Queue and Message/Content Queue. Multi-threading might be an overkill for a simple application, hence the for loop (Temporary). """ for x in range(200): try: msg = consumer.poll(timeout=5.0) if msg is not None: user = msg.value() if user is not None: print("User record {}: username: {}\n" "\tdata: {}\n".format(msg.key(), user.username, user.data)) except Exception as e: print('An exception occurred: {}'.format(e)) logging.error(traceback.format_exc())
def run_consumer(shutdown_flag, clients, lock): print("Starting Kafka Consumer.") schema_registry_client = SchemaRegistryClient( {"url": "http://localhost:8081"}) deserializer = AvroDeserializer(schema_registry_client) config = { "bootstrap.servers": "localhost:9092", "group.id": "dashboard-demo", "value.deserializer": deserializer } consumer = DeserializingConsumer(config) consumer.subscribe(["DASHBOARD"]) while not shutdown_flag.done(): msg = consumer.poll(0.2) if msg is None: print("Waiting...") elif msg.error(): print(f"ERROR: {msg.error()}") else: value = msg.value() formatted = simplejson.dumps(value) print(f"Sending {formatted} to {clients}") with lock: websockets.broadcast(clients, formatted) print("Closing Kafka Consumer") consumer.close()
def consume(self, count: int): consumer = DeserializingConsumer({ 'bootstrap.servers': self.brokers, 'key.deserializer': StringDeserializer('utf_8'), 'value.deserializer': self._make_deserializer(), 'group.id': self.group, 'auto.offset.reset': "earliest" }) consumer.subscribe([self.topic]) self.logger.info("Consuming %d %s records from topic %s with group %s", count, self.schema_type.name, self.topic, self.group) while self.consumed < count: msg = consumer.poll(1) if msg is None: continue payload = msg.value() self.logger.debug("Consumed %d at %d", payload.val, msg.offset()) assert payload.val == self.consumed self.consumed += 1 consumer.close()
def _make_consumer(self) -> DeserializingConsumer: schema_registry_client = SchemaRegistryClient( {"url": self.config["schema_registry"]}) key_deserializer = AvroDeserializer(schema_registry_client) value_deserializer = AvroDeserializer(schema_registry_client) config = { "bootstrap.servers": ",".join(self.config["bootstrap_servers"]), "key.deserializer": key_deserializer, "value.deserializer": value_deserializer, "enable.auto.commit": False, "enable.partition.eof": True, "group.id": self.config["group_id"], "default.topic.config": { "auto.offset.reset": "earliest" }, **self.config["kafka_opts"], } hash_sensitive_values = self.config["hash_sensitive_values"] consumer = DeserializingConsumer(config) hidden_config = hide_sensitive_values( config, hash_sensitive_values=hash_sensitive_values) logger.info( f"AvroConsumer created with config: {pformat(hidden_config, indent=2)}" ) # noinspection PyArgumentList consumer.subscribe(self.config["topics"], on_assign=self._on_assign, on_revoke=self._on_revoke) return consumer
def _consume(self, on_consume): if isinstance(on_consume, types.FunctionType): callback = on_consume else: callback_cls = on_consume() callback = callback_cls.on_message consumer = DeserializingConsumer(self.kafka_config) consumer.subscribe([self.topic]) q = Queue(maxsize=self.num_threads) msg = None while True: try: # Check if we should rate limit msg = consumer.poll(1) if msg is None: continue if msg.error(): logger.error( f'Worker for topic {self.topic} error: {msg.error()}') continue q.put(msg) t = threading.Thread( target=_process_msg, args=(q, consumer, callback, self.topic), ) t.start() except Exception as err: logger.error( f'Worker for topic {self.topic} terminated: {err}') logger.error(msg) consumer.close() break
def main(args): topic = args.topic schema_str = """ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "User", "description": "A Confluent Kafka Python User", "type": "object", "properties": { "name": { "description": "User's name", "type": "string" }, "favorite_number": { "description": "User's favorite number", "type": "number", "exclusiveMinimum": 0 }, "favorite_color": { "description": "User's favorite color", "type": "string" } }, "required": [ "name", "favorite_number", "favorite_color" ] } """ json_deserializer = JSONDeserializer(schema_str, from_dict=dict_to_user) string_deserializer = StringDeserializer('utf_8') consumer_conf = { 'bootstrap.servers': args.bootstrap_servers, 'key.deserializer': string_deserializer, 'value.deserializer': json_deserializer, 'group.id': args.group, 'auto.offset.reset': "earliest" } consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([topic]) while True: try: # SIGINT can't be handled when polling, limit timeout to 1 second. msg = consumer.poll(1.0) if msg is None: continue user = msg.value() if user is not None: print("User record {}: name: {}\n" "\tfavorite_number: {}\n" "\tfavorite_color: {}\n".format(msg.key(), user.name, user.favorite_color, user.favorite_number)) except KeyboardInterrupt: break consumer.close()
def main(args): topic = args.topic outputtopic = args.outputtopic schema_str = EventSchema schema_enriched_event_str = EnrichedEventSchema sr_conf = {'url': args.schema_registry} schema_registry_client = SchemaRegistryClient(sr_conf) avro_deserializer = AvroDeserializer(schema_str, schema_registry_client) string_deserializer = StringDeserializer('utf_8') avro_serializer = AvroSerializer(schema_enriched_event_str, schema_registry_client) consumer_conf = { 'bootstrap.servers': args.bootstrap_servers, 'key.deserializer': string_deserializer, 'value.deserializer': avro_deserializer, 'group.id': args.group + str(random.Random()), 'auto.offset.reset': "latest" } consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([topic]) cluster = Cluster([args.host]) session = cluster.connect("datascience") session.row_factory = dict_factory producer_conf = { 'bootstrap.servers': args.bootstrap_servers, 'key.serializer': StringSerializer('utf_8'), 'value.serializer': avro_serializer } producer = SerializingProducer(producer_conf) while True: try: # SIGINT can't be handled when polling, limit timeout to 1 second. start = time.time() msg = consumer.poll(1.0) if msg is None: continue evt = msg.value() enrich(evt, session, producer, outputtopic) except Exception: print('Exception', sys.exc_info()[0]) continue consumer.close()
def _init_consumer(topics: List[str], config: Dict) -> Consumer: """config must contain: `bootstrap.servers` 'group.id' but may contain every other kafka setting as well """ assert "bootstrap.servers" in config.keys() assert "group.id" in config.keys() consumer = DeserializingConsumer(config) consumer.subscribe(topics) return consumer
def main(): schema_registry_client = SchemaRegistryClient({'url': SCHEMA_REGISTRY_URL}) avro_deserializer = AvroDeserializer( schema_registry_client=schema_registry_client) string_deserializer = StringDeserializer('utf_8') consumer_conf = { 'bootstrap.servers': BOOTSTRAP_SERVERS, 'key.deserializer': string_deserializer, 'max.poll.interval.ms': MAX_POLL_INTERVAL_MS, 'value.deserializer': avro_deserializer, 'group.id': CONSUMER_GROUP } consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([TOPIC]) while True: try: msg = consumer.poll(1.0) if msg is None: continue msg_value = msg.value() if msg_value is not None: try: measurements = list(dict(msg_value).get("measurements")) measurements_df = pd.DataFrame(measurements) groups = measurements_df.groupby("tenant") for _, group in groups: tenant = group.iloc[0]['tenant'] device_registry = DeviceRegistry( tenant, AIRQO_BASE_URL) group_measurements = list( group.to_dict(orient="records")) for i in range(0, len(group_measurements), int(REQUEST_BODY_SIZE)): measurements_list = group_measurements[ i:i + int(REQUEST_BODY_SIZE)] device_registry.insert_events(measurements_list) except Exception as ex: print(ex) except KeyboardInterrupt: break consumer.close()
def main(args): topic = args.topic schema_str = """ { "namespace": "confluent.io.examples.serialization.avro", "name": "User", "type": "record", "fields": [ {"name": "name", "type": "string"}, {"name": "favorite_number", "type": "int"}, {"name": "favorite_color", "type": "string"} ] } """ sr_conf = {'url': args.schema_registry} schema_registry_client = SchemaRegistryClient(sr_conf) avro_deserializer = AvroDeserializer(schema_str, schema_registry_client, dict_to_user) string_deserializer = StringDeserializer('utf_8') consumer_conf = {'bootstrap.servers': args.bootstrap_servers, 'key.deserializer': string_deserializer, 'value.deserializer': avro_deserializer, 'group.id': args.group, 'auto.offset.reset': "earliest"} consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([topic]) while True: try: # SIGINT can't be handled when polling, limit timeout to 1 second. msg = consumer.poll(1.0) if msg is None: continue user = msg.value() if user is not None: print("User record {}: name: {}\n" "\tfavorite_number: {}\n" "\tfavorite_color: {}\n" .format(msg.key(), user.name, user.favorite_color, user.favorite_number)) except KeyboardInterrupt: break consumer.close()
def main(): top = 20 consumer = DeserializingConsumer({ 'bootstrap.servers': os.environ['KAFKA_BROKERS'], 'security.protocol': 'SASL_SSL', 'sasl.mechanism': 'SCRAM-SHA-512', 'sasl.password': os.environ['KAFKA_PASS'], 'sasl.username': os.environ['KAFKA_USER'], 'ssl.ca.location': '/usr/local/share/ca-certificates/Yandex/YandexCA.crt', 'group.id': 'group1', 'key.deserializer': StringDeserializer(), 'value.deserializer': LongDeserializer(), }) consumer.subscribe(['streams-wordcount-output']) try: frequencies = [] while True: msg = consumer.poll(1.0) if msg is None: if frequencies: print('==============================================') print(f'Current list of top {top} most frequent words:') frequencies = sorted(frequencies, key=lambda x: x[1], reverse=True) for frequency in frequencies[0:top]: print(f'{frequency[0]}: {frequency[1]}') frequencies.clear() continue elif msg.error(): print('error: {}'.format(msg.error())) else: frequencies.append((msg.key(), msg.value())) except KeyboardInterrupt: pass finally: consumer.close()
def main(args): topic = args.topic key_schema_str = open('schema/KeySchema.avsc', "r").read() value_schema_str = open('schema/ValueSchema.avsc', "r").read() sr_conf = {'url': args.schema_registry} schema_registry_client = SchemaRegistryClient(sr_conf) avro_key_deserializer = AvroDeserializer(key_schema_str, schema_registry_client, dict_to_user_quote_key) avro_value_deserializer = AvroDeserializer(value_schema_str, schema_registry_client, dict_to_user_quote_value) consumer_conf = { 'bootstrap.servers': args.bootstrap_servers, 'key.deserializer': avro_key_deserializer, 'value.deserializer': avro_value_deserializer, 'group.id': args.group, 'auto.offset.reset': "earliest" } consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([topic]) while True: try: # SIGINT can't be handled when polling, limit timeout to 1 second. msg = consumer.poll(1.0) if msg is None: continue user_quote = msg.value() if user_quote is not None: print("User {} Quote record: product_id: {}\n" "\tquoted_price: {}\n" "\tquoted_quantity: {}\n" "\tuser_note: {}\n".format(msg.key().user_id, user_quote.product_id, user_quote.quoted_price, user_quote.quoted_quantity, user_quote.user_note)) except KeyboardInterrupt: break consumer.close()
def main(): string_deserializer = StringDeserializer('utf_8') conf = { 'bootstrap.servers': 'localhost:9092', 'group.id': 'bitcoin_group', 'key.deserializer': string_deserializer, 'value.deserializer': string_deserializer, 'session.timeout.ms': 6000, 'fetch.wait.max.ms': 5000, 'auto.offset.reset': 'smallest', 'enable.auto.commit': 'false', 'fetch.min.bytes': 307200 } consumer = DeserializingConsumer(conf) consumer.subscribe(['bitcoin-transaction']) messages = [] try: while True: msg = consumer.poll(timeout=1000) if msg is None: continue if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: # End of partition event sys.stderr.write( '%% %s [%d] reached end at offset %d\n' % (msg.topic(), msg.partition(), msg.offset())) elif msg.error(): raise KafkaException(msg.error()) else: obj = json.loads(msg.value()) transaction = dict_to_transaction(obj) messages.append(transaction) if len(messages) > 100: messages = sorted(messages, key=lambda x: x.price, reverse=True)[0:10] print(messages) consumer.commit(asynchronous=False) except KeyboardInterrupt: sys.stderr.write('%% Aborted by user\n') finally: # Close down consumer to commit final offsets. consumer.close()
def run_consumer(container_manager): schema_registry_conf = {'url': config['kafka']['schema_registry']} schema_registry_client = SchemaRegistryClient(schema_registry_conf) avro_deserializer = AvroDeserializer(schemas.run_record_schema, schema_registry_client) string_deserializer = StringDeserializer('utf_8') conf = { 'bootstrap.servers': config['kafka']['servers'], 'key.deserializer': string_deserializer, 'value.deserializer': avro_deserializer, 'group.id': "runs-consumers", 'auto.offset.reset': 'earliest', 'enable.auto.commit': 'false' } consumer = DeserializingConsumer(conf) print('[+] Listening for incoming runs') try: consumer_topics = [config['kafka']['runs-topic']] consumer.subscribe(consumer_topics) while True: try: msg = consumer.poll(timeout=1.0) if msg is None: continue if msg.error(): raise KafkaException(msg.error()) else: print('[-] Run initialization') print(msg.value()) consumer.commit(asynchronous=False) # handlers.handle_run_execution(container_manager, msg.value()) threading.Thread(target=handlers.handle_run_execution, args=(container_manager, msg.value())).start() except ConsumeError as e: print( f'[Exception] error_code: {e.code()} message: {e.message()} exception: {e}' ) finally: consumer.close()
def main(args): topic = args.topic schema_str = MetricSchema sr_conf = {'url': args.schema_registry} schema_registry_client = SchemaRegistryClient(sr_conf) avro_deserializer = AvroDeserializer(schema_str, schema_registry_client) string_deserializer = StringDeserializer('utf_8') consumer_conf = { 'bootstrap.servers': args.bootstrap_servers, 'key.deserializer': string_deserializer, 'value.deserializer': avro_deserializer, 'group.id': args.group, 'auto.offset.reset': "earliest" } consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([topic]) client = InfluxDBClient(host=args.host_influx, port=8086, username='******', password='******') while True: try: # SIGINT can't be handled when polling, limit timeout to 1 second. msg = consumer.poll(1.0) if msg is None: continue timespent = msg.value() if timespent is not None: print("time ==>", timespent) print(timespent["metricName"]) print(timespent["time"]) client.switch_database('datascience') json_body = [{ "measurement": "metric", "fields": { "name": timespent["metricName"], "value": timespent["time"] } }] client.write_points(json_body) except KeyboardInterrupt: break consumer.close()
class Consumer: def __init__(self, bootstrap_servers: str, topic: str, group: str, callback: Callable[[Message], None], value_deserializer=None, poll_timeout: float = 1.0, config=None): consumer_config = { "bootstrap.servers": bootstrap_servers, "group.id": group, "value.deserializer": value_deserializer } if config: consumer_config.update(config) self.consumer = DeserializingConsumer(consumer_config) self.topic = topic self.callback = callback self.poll_timeout = poll_timeout def start(self): logger.info("Starting Kafka consumer") self.consumer.subscribe([self.topic]) while True: message = self.consumer.poll(self.poll_timeout) if message is None: continue if message.error(): print(f"Consumer error: {message.error()}") continue self.callback(message) def close(self): logger.info("Closing Kafka consumer") self.consumer.close()
def create_consumer(self, registry_client): """ Subscribes to topic defined in configs and creates a consumer to deserialize messages from topic :param registry_client: SchemaRegistryClient object get this from register_client() :return: DeserializingConsumer object """ metadata_schema = None topic = None if self.metadata_type == "COLLECTION": metadata_schema = registry_client.get_latest_version( self.collection_topic + '-value').schema.schema_str topic = self.collection_topic if self.metadata_type == "GRANULE": metadata_schema = registry_client.get_latest_version( self.granule_topic + '-value').schema.schema_str topic = self.granule_topic metadata_deserializer = AvroDeserializer(metadata_schema, registry_client) consumer_conf = {'bootstrap.servers': self.brokers} if self.security: consumer_conf['security.protocol'] = 'SSL' consumer_conf['ssl.ca.location'] = self.conf['security']['caLoc'] consumer_conf['ssl.key.location'] = self.conf['security']['keyLoc'] consumer_conf['ssl.certificate.location'] = self.conf['security'][ 'certLoc'] meta_consumer_conf = consumer_conf meta_consumer_conf['key.deserializer'] = StringDeserializer('utf-8') meta_consumer_conf['value.deserializer'] = metadata_deserializer meta_consumer_conf['group.id'] = self.group_id meta_consumer_conf['auto.offset.reset'] = self.auto_offset_reset metadata_consumer = DeserializingConsumer(meta_consumer_conf) metadata_consumer.subscribe([topic]) return metadata_consumer
def consume(): reusableConsumer = DeserializingConsumer(getConfigs()) reusableConsumer.subscribe(["myprototopic"]) while (True): try: msg = reusableConsumer.poll(0.1) if msg is None: continue else: key = msg.key() value = msg.value() print("Received msg name: {}, fav food: {}, times eaten: {}". format(value.name, value.favoriteFood, value.timesEaten)) except KeyboardInterrupt: break print("Closing Consumer") reusableConsumer.close()
def receive_record(args): """ Receives Record using a DeserializingConsumer & AvroDeserializer """ topics = [args.topic.rstrip()] schema_registry_config = {'url': args.schema_registry} schema_registry_client = SchemaRegistryClient(schema_registry_config) avro_deserializer = AvroDeserializer(schema_registry_client, DATA_SCHEMA, dict_to_data) string_deserializer = StringDeserializer('utf_8') consumer_config = { 'bootstrap.servers': args.bootstrap_servers, 'key.deserializer': string_deserializer, 'value.deserializer': avro_deserializer, 'group.id': args.group, 'auto.offset.reset': 'earliest' } consumer = DeserializingConsumer(consumer_config) consumer.subscribe(topics) print(f'Consuming data records from topic(s) {topics}. ^C to exit.') while True: try: # SIGINT can't be handled when polling, limit timeout to 1 second. msg = consumer.poll(10.0) if msg is None: print('\t---Waiting. . .') continue data = msg.value() if data is not None: print(f'Data record {msg.key()}:\n' f'\tValues: {data}') except KeyboardInterrupt: break print('\nClosing consumer.') consumer.close()
def plain_avro_consumer(running_cluster_config: Dict[str, str], topic_and_partitions: Tuple[str, int]): topic_id, _ = topic_and_partitions schema_registry_client = SchemaRegistryClient( {"url": running_cluster_config["schema-registry"]}) key_deserializer = AvroDeserializer(schema_registry_client) value_deserializer = AvroDeserializer(schema_registry_client) config = { "bootstrap.servers": running_cluster_config["broker"], "group.id": f"{topic_id}_consumer", "key.deserializer": key_deserializer, "value.deserializer": value_deserializer, "enable.partition.eof": False, "default.topic.config": { "auto.offset.reset": "earliest" }, "allow.auto.create.topics": True, } consumer = DeserializingConsumer(config) consumer.subscribe([topic_id]) consumer.list_topics() return consumer
def main(args): topic = args.topic schema_registry_conf = {'url': args.schema_registry} schema_registry_client = SchemaRegistryClient(schema_registry_conf) schema_obj = schema_registry_client.get_latest_version( subject_name='example_serde_json-value') json_deserializer = JSONDeserializer(schema_obj.schema.schema_str, from_dict=dict_to_user) string_deserializer = StringDeserializer('utf_8') consumer_conf = { 'bootstrap.servers': args.bootstrap_servers, 'key.deserializer': string_deserializer, 'value.deserializer': json_deserializer, 'group.id': args.group, 'auto.offset.reset': "earliest" } consumer = DeserializingConsumer(consumer_conf) consumer.subscribe([topic]) while True: try: msg = consumer.poll(1.0) if msg is None: continue user = msg.value() if user is not None: print(f"User record {msg.key()}:\n name: {user.name}\n" f"\tfavorite_number: {user.favorite_color}\n" f"\tfavorite_color: {user.favorite_number}\n") except KeyboardInterrupt: break consumer.close()
from confluent_kafka import DeserializingConsumer if __name__ == '__main__': consumer_conf = { 'bootstrap.servers': 'localhost:9092', 'group.id': 'kafka-client', 'auto.offset.reset': "earliest" } consumer = DeserializingConsumer(consumer_conf) consumer.subscribe(['dbserver1.inventory.customers']) print('Kafka Client Listening...') while True: try: # SIGINT can't be handled when polling, limit timeout to 1 second. msg = consumer.poll(1.0) if msg is None: continue print(msg.value()) print() except KeyboardInterrupt: break consumer.close()
def consume(consumer: DeserializingConsumer, timeout) -> iter: while True: # Waiting for message until timeout reached if there is no message. # If message exists, message will be returned. message = consumer.poll(timeout) # print('[kafka] polling...') if message is None: continue if message.error(): print('Consumer error: {}'.format(message.error())) continue yield message if __name__ == '__main__': c = DeserializingConsumer({ 'bootstrap.servers': 'localhost:9092', 'group.id': 'my-consumer-group', 'auto.offset.reset': 'earliest', 'key.deserializer': IntegerDeserializer() }) c.subscribe(topics=['sample-topic']) try: for msg in consume(c, timeout=1.0): print(f'{msg.key()},{msg.value().decode("utf-8")}') finally: print('Consumer will be closed') c.close()
class KafkaAvroConsumer: def __init__(self, consumer_name, value_schema, topic_name = "kafka-avro-producer", groupID = 'KafkaAvroConsumer', autocommit = True): # Consumer name for logging purposes self.logging_prefix = '['+ consumer_name + '][KafkaAvroConsumer]' # Schema Registry configuration self.schema_registry_conf = EventBackboneConfig.getSchemaRegistryConf() # Schema Registry Client self.schema_registry_client = SchemaRegistryClient(self.schema_registry_conf) # Key Deserializer self.key_deserializer = StringDeserializer('utf_8') # Get Schema for the value self.schema_id_value = self.schema_registry_client.get_latest_version(topic_name + "-value").schema_id # print('The Schema ID for the value is: {}'.format(self.schema_id_value)) self.value_schema = self.schema_registry_client.get_schema(self.schema_id_value).schema_str print(self.logging_prefix + ' - Value Subject: {}'.format(topic_name)) print(self.logging_prefix + ' - Value Schema:') print(self.logging_prefix + ' - -------------\n') print(self.logging_prefix + ' - ' + self.value_schema + '\n') # Value Deserializer # Presenting the schema to the Avro Deserializer is needed at the moment. In the future it might change # https://github.com/confluentinc/confluent-kafka-python/issues/834 self.value_deserializer = AvroDeserializer(self.value_schema,self.schema_registry_client) # Get the consumer configuration self.consumer_conf = EventBackboneConfig.getConsumerConfiguration(groupID, autocommit, self.key_deserializer, self.value_deserializer) # Create the consumer self.consumer = DeserializingConsumer(self.consumer_conf) # Print consumer configuration EventBackboneConfig.printConsumerConfiguration(self.logging_prefix,self.consumer_conf,self.schema_registry_conf['url']) # Subscribe to the topic self.consumer.subscribe([topic_name]) def traceResponse(self, msg): print(self.logging_prefix + ' - New event received\n\tTopic: {}\n\tPartition: {}\n\tOffset: {}\n\tkey: {}\n\tvalue: {}\n' .format(msg.topic(), msg.partition(), msg.offset(), msg.key(), msg.value())) # Polls for next event def pollNextEvent(self): # Poll for messages msg = self.consumer.poll(timeout=POLL_TIMEOUT) anEvent = {} # Validate the returned message if msg is None: print(self.logging_prefix + ' - [INFO] - No new messages on the topic') return None elif msg.error(): if ("PARTITION_EOF" in msg.error()): print(self.logging_prefix + ' - [INFO] - End of partition') else: print(self.logging_prefix + ' - [ERROR] - Consumer error: {}'.format(msg.error())) return None else: # Print the message self.traceResponse(msg) return msg.value() # Polls for the next event but returns the raw event def pollNextRawEvent(self): records = self.consumer.poll(timeout=POLL_TIMEOUT) if records is None: return None if records.error(): # Stop reading if we find end of partition in the error message if ("PARTITION_EOF" in records.error()): return None else: print(self.logging_prefix + ' - [ERROR] - Consumer error: {}'.format(records.error())) return None else: self.traceResponse(records) return records def commitEvent(self,event): self.consumer.commit(event) def close(self): self.consumer.close()
from confluent_kafka import DeserializingConsumer from confluent_kafka.serialization import StringDeserializer # from confluent_kafka.avro.serializer import SerializerError string_serialiser = StringDeserializer('utf_8') consumer_conf = { 'bootstrap.servers': 'localhost:9092', 'group.id': "groupid1234", 'key.deserializer': string_serialiser, 'value.deserializer': string_serialiser } consumer = DeserializingConsumer(consumer_conf) consumer.subscribe(['test-topic-profileus1']) while True: try: msg = consumer.poll(10) print("poll returned {}".format(msg)) except Exception as e: print("Message deserialization failed for {}: {}".format(msg, e)) break if msg is None: continue if msg.error(): print("AvroConsumer error: {}".format(msg.error()))
# Initialize the consumer consumer_conf = { 'bootstrap.servers': conf['hops']['url'] + ':' + conf['kafka']['port'], 'security.protocol': 'SSL', 'ssl.ca.location': conf['project']['ca_file'], 'ssl.certificate.location': conf['project']['certificate_file'], 'ssl.key.location': conf['project']['key_file'], 'ssl.key.password': conf['project']['key_password'], 'key.deserializer': string_deserializer, 'value.deserializer': string_deserializer, 'group.id': conf['kafka']['consumer']['group_id'], 'auto.offset.reset': conf['kafka']['consumer']['auto_offset_reset'], } consumer = DeserializingConsumer(consumer_conf) # Subscribe to a topic consumer.subscribe([conf['kafka']['topic']]) # Get the token, URL and light for Home Assistant API token = conf['hass']['token'] url = conf['hass']['url'] light = conf['hass']['light'] headers = { "Authorization": "Bearer " + token, "content-type": "application/json", } # Load and print state of light response = requests.get(url + "states/" + light, headers=headers) print(response.text)
consumer_conf = { 'bootstrap.servers': conf['bootstrap.servers'], 'sasl.mechanisms': conf['sasl.mechanisms'], 'security.protocol': conf['security.protocol'], 'sasl.username': conf['sasl.username'], 'sasl.password': conf['sasl.password'], 'key.deserializer': name_avro_deserializer, 'value.deserializer': count_avro_deserializer, 'group.id': 'python_example_group_2', 'auto.offset.reset': 'earliest' } consumer = DeserializingConsumer(consumer_conf) # Subscribe to topic consumer.subscribe([topic]) # Process messages total_count = 0 while True: try: msg = consumer.poll(1.0) if msg is None: # No message available within timeout. # Initial message consumption may take up to # `session.timeout.ms` for the consumer group to # rebalance and start consuming print("Waiting for message or event/error in poll()") continue elif msg.error(): print('error: {}'.format(msg.error()))