def kafka_stream_analysis_loop(self):
"""
Endless loop that reads from the Kafka stream and does sentiment
analysis of each message. Aggregates the results into averages for
each thread.
"""
consumer = kafka_helper.get_kafka_consumer(topic='chatter')
for message in consumer:
print("%s:%d:%d: key=%s value=%s" %
(message.topic, message.partition, message.offset,
message.key, message.value))
blob = TextBlob(message.value['Body'])
# Keep running averages of the most polarizing threads (in the future, we could also keep track
# of posters and parentIds).
feed_item_id = message.value['FeedItemId']
if feed_item_id not in self.feed_item_stats:
self.feed_item_stats[feed_item_id] = dict(total_polarity=0,
num_times=0)
current_stat = self.feed_item_stats[feed_item_id]
current_stat['total_polarity'] += blob.sentiment.polarity
current_stat['num_times'] += 1
current_stat['average'] = current_stat[
'total_polarity'] / current_stat['num_times']
def __init__(self):
logging.info('Started consuming\n')
self.use_db = False
try: #heroku
self.consumer = kafka_helper.get_kafka_consumer(topic=TOPIC)
logging.info('RUNNING CONSUMER ON HEROKU\n')
except Exception: #locally on docker
consumer = KafkaConsumer(
'test',
bootstrap_servers='localhost:9092',
value_deserializer=lambda m: json.loads(m.decode('utf-8')))
self.consumer = consumer
logging.info(
'RUNNING CONSUMER LOCAL - so not using kafka_helper\n')
if 'DATABASE_URL' in os.environ:
self.use_db = True
import psycopg2
DATABASE_URL = os.environ['DATABASE_URL']
try:
self.conn = psycopg2.connect(DATABASE_URL, sslmode='require')
self.cur = self.conn.cursor()
logging.info("Connected to the database")
except:
logging.info("I am unable to connect to the database")
from flask import Flask
# from flask import render_template
import os
import kafka_helper
import json
import asyncio
import websockets
# app = Flask(__name__)
# @app.route("/")
# def index():
# return render_template("index.html", port=os.environ["WS_PORT"])
topic = "{}temp".format(os.environ["KAFKA_PREFIX"])
consumer = kafka_helper.get_kafka_consumer(topic=topic)
print("Connected")
async def echo(websocket, path):
# async for message in websocket:
for message in consumer:
print(message)
await websocket.send(json.dumps(message.value))
asyncio.get_event_loop().run_until_complete(
websockets.serve(echo, 'localhost', '8765'))
asyncio.get_event_loop().run_forever()
def testKafkaHelperRCV():
import kafka_helper
consumer = kafka_helper.get_kafka_consumer(topic='ple2')
for message in consumer:
logger.debug(message)