# init consumer
consumer = KafkaConsumer(source_topic_name,
bootstrap_servers=['localhost:9092'],
auto_offset_reset='earliest',
enable_auto_commit=True,
group_id=consumer_group_id,
value_deserializer=lambda x: loads(x.decode('utf-8')))
# init producer
producer = KafkaProducer(bootstrap_servers=['localhost:9092'],
value_serializer=lambda x: dumps(x).encode('utf-8'),
api_version=(0, 10, 1))
# init sentiment analyzer
sa = SentimentAnalyzer()
tokenizer = sa.token()
# start consuming
for message in consumer:
# overwrite message with its value and preprocess text
message = message.value.copy()
# extract hashtags
hashtags = []
if len(message['hashtags']) != 0:
for hashtag_data in message['hashtags']:
hashtags.append(hashtag_data["text"])
# overwrite hashtags data structure with plain hashtags text
message["hashtags"] = hashtags
