def __init__(self, app_instance, kafka_host_name, kafka_host_port): self.app_instance = app_instance self.SPLASH_TOPIC_OUTPUT = self.SPLASH_TOPIC + "-output" self.SPLASH_TOPIC_GROUP = self.SPLASH_TOPIC + "-group" self.kafka_connector = KafkaConnector(kafka_host_name, kafka_host_port) print("creating topic" + self.SPLASH_TOPIC_INPUT) self.kafka_connector.create_topic(self.SPLASH_TOPIC_INPUT) print("creating topic" + self.SPLASH_TOPIC_OUTPUT) self.kafka_connector.create_topic(self.SPLASH_TOPIC_OUTPUT) print("Splash Broker started")
class BrokerService(object): SPLASH_TOPIC = 'splash' SPLASH_TOPIC_INPUT = SPLASH_TOPIC + "-input" def __init__(self, app_instance, kafka_host_name, kafka_host_port): self.app_instance = app_instance self.SPLASH_TOPIC_OUTPUT = self.SPLASH_TOPIC + "-output" self.SPLASH_TOPIC_GROUP = self.SPLASH_TOPIC + "-group" self.kafka_connector = KafkaConnector(kafka_host_name, kafka_host_port) print("creating topic" + self.SPLASH_TOPIC_INPUT) self.kafka_connector.create_topic(self.SPLASH_TOPIC_INPUT) print("creating topic" + self.SPLASH_TOPIC_OUTPUT) self.kafka_connector.create_topic(self.SPLASH_TOPIC_OUTPUT) print("Splash Broker started") def add_message(self, message): # add more headers to the message message['source'] = self.app_instance message['callbackQueue'] = self.SPLASH_TOPIC_OUTPUT message['timestamp'] = time.time() message['strTimestamp'] = strftime("%Y-%m-%d %H:%M:%S", gmtime()) json_message = json.dumps(message) self.kafka_connector.send_message(self.SPLASH_TOPIC_INPUT, json_message) def output_message(self, message): # add more headers to the message message['source'] = self.app_instance message['timestamp'] = time.time() message['strTimestamp'] = strftime("%Y-%m-%d %H:%M:%S", gmtime()) json_message = json.dumps(message) try: self.kafka_connector.send_message(self.SPLASH_TOPIC_OUTPUT, json_message) print json_message except: traceback.print_exc() def read_topic_from_splash_queue(self, callback): self.kafka_connector.register_consumer(callback, self.parse_json, self.SPLASH_TOPIC_GROUP, self.SPLASH_TOPIC_INPUT) def read_topic_from_splash_queue_blocking(self, message_function): self.kafka_connector.blocking_consumer(message_function, self.parse_json, self.SPLASH_TOPIC_GROUP, self.SPLASH_TOPIC_INPUT) # OffsetAndMessage(offset=0, message=Message(magic=0, attributes=0, key=None, value='sssss')) def parse_json(self, message): payload = message[1][3] obj = json.loads(payload) obj['offset'] = message[0] return obj
__author__ = 'tomas' from kafkaclient import KafkaConnector from time import gmtime, strftime, time if __name__ == "__main__": host_port = "9092" host_name = "localhost" topic_group = "test-group" topic_name = 'testa4' message = 'python integration test ' + strftime("%Y-%m-%d %H:%M:%S", gmtime()) kafkaclient = KafkaConnector(host_name, host_port) # putIfAbsent topic # kafkaclient.create_topic(topic_name) # produce kafkaclient.send_message(topic_name, message) print "Finish sending %s" % (message) # consume kafkaclient.register_consumer("function-todo", topic_group, topic_name) print "Finish registering " time.sleep(60) print "Finish all"