def __init__(self, api, kafka_host='localhost:9092', stream_config={}):
super(tweepy.StreamListener, self).__init__()
self.api = api
self.stream_config = stream_config
print('bootstrap_servers:', kafka_host)
self.producer = KafkaProducer(bootstrap_servers=kafka_host)
# Add Kafka topics
topic = self.stream_config.get('kafka_topic')
if topic:
client = KafkaClient(bootstrap_servers=kafka_host)
client.add_topic(topic)
class KafkaConnector(object):
def __init__(self, host_name, host_port):
self.client = KafkaClient(bootstrap_servers=host_name + ":" + host_port, receive_buffer_bytes=8*1024*1024)
self.producer = KafkaProducer(bootstrap_servers=host_name + ":" + host_port, receive_buffer_bytes=8*1024*1024)
def create_topic(self, topic_name):
topics = self.client.cluster.topics()
if topic_name not in topics:
self.client.add_topic(topic_name)
def send_message(self, topic_name, message):
# self.producer.send_messages(topic_name, message)
self.producer.send(topic_name, message)
def register_consumer(self, callback, parse_json, topic_group, topic_name):
consumer = KafkaConsumer(topic_name)
consumer_thread = ConsumerThread(consumer, callback, parse_json)
print "Starting new subscriber for topic " + topic_name + ' with group ' + topic_group
consumer_thread.start()
