# overwrite hashtags data structure with plain hashtags text
message["hashtags"] = hashtags
# get product and company info
message["company"], message[
"product"] = get_associated_company_and_product(message['text'])
# preprocess text data
try:
tokenized_text = sa.preprocess(message['text'])
except:
tokenized_text = []
# make predictions
try:
message['sentiment'], message['confidence'] = sa.predict(
tokenized_text, tokenizer)
except: # to prevent there may be other bugs we did not imagine
message['sentiment'], message['confidence'] = ('Neutral', 0.5)
# for identiable tweets, save analyzed tweets back to kafka in a separate topic
if message["company"] != "none" and message["company"] != "mix":
print("==============================================================")
print(message)
producer.send(sink_topic_name, value=message)
else:
print("==============================================================")
print("Product match not found.")
