class KafkaConnector(object): def __init__(self, host_name, host_port): self.client = KafkaClient(host_name + ":" + host_port) self.producer = SimpleProducer(self.client) def create_topic(self, topic_name): topic_exists = self.client.has_metadata_for_topic(topic_name) if not topic_exists: self.client.ensure_topic_exists(topic_name) def send_message(self, topic_name, message): self.producer.send_messages(topic_name, message) def register_consumer(self, callback, parse_json, topic_group, topic_name): consumer = SimpleConsumer(self.client, topic_group, topic_name) consumer_thread = ConsumerThread(consumer, callback, parse_json) consumer_thread.start() def blocking_consumer(self, message_consume_function, parse_json, topic_group, topic_name): print "starting blocking consumer with topic group %s and topic name %s" % (topic_group, topic_name) consumer = SimpleConsumer(self.client, topic_group, topic_name) consumer.seek(0,2) for message in consumer: message = parse_json(message) print "=============" + str(message) + "============" message_consume_function(message) print "called message consume function"
def wait_for_kafka_topic(hostport, topic, timeout=60): """Wait for a Kafka topic to become available.""" start = time.time() client = KafkaClient(hostport, client_id=b'dummy', timeout=1) while not client.has_metadata_for_topic(topic): if time.time() - start > timeout: raise Exception('timeout reached waiting for topic') time.sleep(0.1) client.load_metadata_for_topics()
class KafkaConnector(object): def __init__(self, host_name, host_port): self.client = KafkaClient(host_name + ":" + host_port) self.producer = SimpleProducer(self.client) def create_topic(self, topic_name): topic_exists = self.client.has_metadata_for_topic(topic_name) if not topic_exists: self.client.ensure_topic_exists(topic_name) def send_message(self, topic_name, message): self.producer.send_messages(topic_name, message) def register_consumer(self, callback, parse_json, topic_group, topic_name): consumer = SimpleConsumer(self.client, topic_group, topic_name, max_buffer_size=None) consumer_thread = ConsumerThread(consumer, callback, parse_json) print "Starting new subscriber for topic " + topic_name + ' with group ' + topic_group consumer_thread.start()