def __init__(self, api, kafka_host='localhost:9092', stream_config={}):
self.api = api
self.stream_config = stream_config
super(tweepy.StreamListener, self).__init__()
print('bootstrap_servers:', kafka_host)
self.producer = KafkaProducer(bootstrap_servers=kafka_host)
# Add Kafka topics
topic = self.stream_config.get('kafka_topic')
if topic:
client = KafkaClient(kafka_host)
client.add_topic(topic)
def generator(file_name="transform_data/data.csv"):
producer = KafkaProducer(
bootstrap_servers='localhost:9092',
value_serializer=lambda m: dumps(m).encode('utf-8'))
line = [0]
with open(file_name, "r") as text:
while True:
prev_step = line[0]
try:
line = list(map(float, text.readline().split(",")))
if float(round(line[0], 1)) == 0:
producer.send('dashboard1', value=message(parameter, line))
producer.send('clickhouse1',
value=message(parameter, line))
else:
time.sleep((line[0] - prev_step) / 10)
producer.send('dashboard1', value=message(parameter, line))
producer.send('clickhouse1',
value=message(parameter, line))
except ValueError:
break
if __name__ == '__main__':
client = KafkaClient(bootstrap_servers='localhost:9092')
client.add_topic('dashboard1')
client.add_topic('clickhouse1')
client.close()
generator()
from kafka import KafkaClient
client = KafkaClient(bootstrap_servers=['localhost:9092'])
if client.bootstrap_connected == 'False':
raise Exception('You arent connected to the kafka server')
topicName = 'TwitterKafka'
client.add_topic(topicName)
# print(f"Created Topic called {topicName}")
from kafka import KafkaProducer
from kafka import KafkaClient
client = KafkaClient(
bootstrap_servers=
'b-1.st1612-cluster-1.e0djg5.c8.kafka.us-east-1.amazonaws.com:9092')
client.add_topic('AWSKafkaTutorialTopic')
producer = KafkaProducer(
bootstrap_servers=
'b-1.st1612-cluster-1.e0djg5.c8.kafka.us-east-1.amazonaws.com:9092')
print(producer.send('AWSKafkaTutorialTopic', b'Hello, World!'))
producer.send('AWSKafkaTutorialTopic',
key=b'message-two',
value=b'This is Kafka-Python')