def __init__(self, app_instance, kafka_host_name, kafka_host_port):
self.app_instance = app_instance
self.SPLASH_TOPIC_OUTPUT = self.SPLASH_TOPIC + "-output"
self.SPLASH_TOPIC_GROUP = self.SPLASH_TOPIC + "-group"
self.kafka_connector = KafkaConnector(kafka_host_name, kafka_host_port)
print("creating topic" + self.SPLASH_TOPIC_INPUT)
self.kafka_connector.create_topic(self.SPLASH_TOPIC_INPUT)
print("creating topic" + self.SPLASH_TOPIC_OUTPUT)
self.kafka_connector.create_topic(self.SPLASH_TOPIC_OUTPUT)
print("Splash Broker started")
class BrokerService(object):
SPLASH_TOPIC = 'splash'
SPLASH_TOPIC_INPUT = SPLASH_TOPIC + "-input"
def __init__(self, app_instance, kafka_host_name, kafka_host_port):
self.app_instance = app_instance
self.SPLASH_TOPIC_OUTPUT = self.SPLASH_TOPIC + "-output"
self.SPLASH_TOPIC_GROUP = self.SPLASH_TOPIC + "-group"
self.kafka_connector = KafkaConnector(kafka_host_name, kafka_host_port)
print("creating topic" + self.SPLASH_TOPIC_INPUT)
self.kafka_connector.create_topic(self.SPLASH_TOPIC_INPUT)
print("creating topic" + self.SPLASH_TOPIC_OUTPUT)
self.kafka_connector.create_topic(self.SPLASH_TOPIC_OUTPUT)
print("Splash Broker started")
def add_message(self, message):
# add more headers to the message
message['source'] = self.app_instance
message['callbackQueue'] = self.SPLASH_TOPIC_OUTPUT
message['timestamp'] = time.time()
message['strTimestamp'] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
json_message = json.dumps(message)
self.kafka_connector.send_message(self.SPLASH_TOPIC_INPUT, json_message)
def output_message(self, message):
# add more headers to the message
message['source'] = self.app_instance
message['timestamp'] = time.time()
message['strTimestamp'] = strftime("%Y-%m-%d %H:%M:%S", gmtime())
json_message = json.dumps(message)
try:
self.kafka_connector.send_message(self.SPLASH_TOPIC_OUTPUT, json_message)
print json_message
except:
traceback.print_exc()
def read_topic_from_splash_queue(self, callback):
self.kafka_connector.register_consumer(callback, self.parse_json, self.SPLASH_TOPIC_GROUP, self.SPLASH_TOPIC_INPUT)
def read_topic_from_splash_queue_blocking(self, message_function):
self.kafka_connector.blocking_consumer(message_function, self.parse_json, self.SPLASH_TOPIC_GROUP, self.SPLASH_TOPIC_INPUT)
# OffsetAndMessage(offset=0, message=Message(magic=0, attributes=0, key=None, value='sssss'))
def parse_json(self, message):
payload = message[1][3]
obj = json.loads(payload)
obj['offset'] = message[0]
return obj
__author__ = 'tomas'
from kafkaclient import KafkaConnector
from time import gmtime, strftime, time
if __name__ == "__main__":
host_port = "9092"
host_name = "localhost"
topic_group = "test-group"
topic_name = 'testa4'
message = 'python integration test ' + strftime("%Y-%m-%d %H:%M:%S",
gmtime())
kafkaclient = KafkaConnector(host_name, host_port)
# putIfAbsent topic
# kafkaclient.create_topic(topic_name)
# produce
kafkaclient.send_message(topic_name, message)
print "Finish sending %s" % (message)
# consume
kafkaclient.register_consumer("function-todo", topic_group, topic_name)
print "Finish registering "
time.sleep(60)
print "Finish all"
