class KafkaListener(): def __init__(self, topic, group_id, advertised_listeners, message_handler): self.consumer = KafkaConsumer( topic, group_id=group_id, bootstrap_servers=advertised_listeners ) self.message_handler = message_handler logging.info('initializing kafka listener') thread = threading.Thread(target=self.run, args=()) thread.daemon = True thread.start() def is_connected(self): return self.consumer.bootstrap_connected() def run(self): for msg in self.consumer: try: json_message = json.loads(msg.value) self.message_handler( url=json_message['url'], html_element_id=json_message['html_element_id'], filename=json_message['output_file'] ) except Exception: logging.error('cannot parse message')
def consume_messages(self): try: kafka_consumer = KafkaConsumer( self.topic_name, bootstrap_servers=self.bootstrap_servers) print(kafka_consumer.bootstrap_connected()) if kafka_consumer is not None: for message in kafka_consumer: # message value and key are raw bytes -- decode if necessary! # e.g., for unicode: `message.value.decode('utf-8')` print(message) # print("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition, # message.offset, message.key, # message.value)) except Exception as ex: print("Failure in consume messages", ex)
from kafka import KafkaConsumer from kafka.errors import NoBrokersAvailable import json import time import sys try: consumer = KafkaConsumer( 'test-docker', group_id='my-group-1', bootstrap_servers='kafka:9092', value_deserializer=lambda m: json.loads(m.decode('utf-8'))) except NoBrokersAvailable: sys.exit('broker not available (yet?)') if not consumer.bootstrap_connected(): sys.exit('not connected, restarting...') try: print('consuming') for message in consumer: # message value and key are raw bytes -- decode if necessary! # e.g., for unicode: `message.value.decode('utf-8')` print("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition, message.offset, message.key, message.value)) print('finished batch') time.sleep(1) except Exception as e: print(repr(e)) consumer.unsubscribe()
from kafka import KafkaConsumer from setup import topicName # To consume latest messages and auto-commit offsets consumer = KafkaConsumer(topicName, group_id='group_1', bootstrap_servers=['localhost:9092']) if consumer.bootstrap_connected() == False: raise Exception('You arent connected to the kafka server') for message in consumer: # message value and key are raw bytes -- decode if necessary! # e.g., for unicode: `message.value.decode('utf-8')` print("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition, message.offset, message.key, message.value)) #If we need more thn one consumer # consumer1 = KafkaConsumer('my-topic', # group_id='my-group', # bootstrap_servers='my.server.com') # consumer2 = KafkaConsumer('my-topic', # group_id='my-group', # bootstrap_servers='my.server.com')
if __name__ == '__main__': print("Starting up: Task 3 ---- Kafka Service Event Consumer") sleep(1) try: # create the Producer instance service_consumer = KafkaConsumer( bootstrap_servers=kafka_server, auto_offset_reset='earliest', enable_auto_commit=True, auto_commit_interval_ms=500, group_id='my-group', value_deserializer=lambda x: json.loads(x.decode('utf-8'))) # subscribe to topic service_consumer.subscribe(topics=kafka_event_topic) print("Consumer connected: ", service_consumer.bootstrap_connected()) ## Attempting connection to the Postgres database # read connection parameters params = config() # connect to the PostgreSQL server print('Connecting to the PostgreSQL database...') conn = psycopg2.connect(**params) # create a psycopg2 cursor that can execute queries cursor = conn.cursor() except KeyboardInterrupt: print("Keyboard Interrupt ") except (Exception, psycopg2.DatabaseError) as error: print(error)
from kafka import KafkaConsumer import json import io topic = 'electric' key_deserializer = 'org.apache.kafka.connect.storage.StringConverter' value_deserializer = lambda m: json.loads(m.decode('ascii')) group_id = 'electric-group' consumer = KafkaConsumer(topic, group_id='consumer-grp', value_deserializer=value_deserializer) print("Consumer connected : ", consumer.bootstrap_connected()) i = 0 for msg in consumer: print(i, " Message is : ", msg) i = i + 1
from kafka import KafkaConsumer # continuous loop var = 1 while var == 1: # initialize consumer to given topic and broker consumer = KafkaConsumer('test', bootstrap_servers='localhost:9092') # verify connection print('Connected') print(consumer.bootstrap_connected()) # loop and print messages for msg in consumer: print('...received') print(msg)
def monitor(config_name: str, kafka_bootstrap: str, retry_count: int = 10): """Monitor Kafka topics and write to designated alarm process variables. Args: config_name (str): Name of configuration kafka_bootstrap (str): Name of kafka bootstrap server retry_count (int): Number of connection retries before logging a failure """ while True: while retry_count: try: consumer = KafkaConsumer( config_name, bootstrap_servers=[kafka_bootstrap], key_deserializer=lambda x: x.decode('utf-8')) retry_count = 0 except KeyboardInterrupt: print("Shutting down") sys.exit(0) except: print("No consumers available.") print("Retrying...") retry_count -= 1 if consumer is None: print("Unable to connect to Kafka bootstrap server.") sys.exit(0) elif consumer.bootstrap_connected(): # initialize so can seek to beginning to get latest compacted state while not consumer._client.poll(): continue consumer.seek_to_beginning() for message in consumer: try: if "config:/" in message.key: val = json.loads(message.value.decode('utf-8')) pv = message.key.split("/")[-1] # if enabled, write alarm print(f"writing alarm {pv}FP") if val.get("enabled") is not None: # will be false epics.caput(f"{pv}FP", 1) else: epics.caput(f"{pv}FP", 0) assert 1 == 5 except KeyboardInterrupt: print("Shutting down...") sys.exit(0) except Exception as e: print(e) else: print("Unable to connect to Kafka bootstrap server.") sys.exit(0)