def update_graph_live(value, n_intervals):
mongo = MongoWrapper()
neg_sentiment, neutral_sentiment, pos_sentiment = mongo.get_polarity_tweets_of_stock(
value)
negs = neg_sentiment.count()
poss = pos_sentiment.count()
neuu = neutral_sentiment.count()
tots = negs + poss + neuu
posPercentage = (poss / (tots)) * 100
neuPercentage = (neuu / (tots)) * 100
negPercentage = (negs / (tots)) * 100
fig = ({
'data': [{
'type': 'pie',
'labels': ['Positive', 'Neutral', 'Negative'],
'values': [posPercentage, neuPercentage, negPercentage],
'marker': {
'colors': ['#32C85A', '#4C93B1', '#FA4632'],
# 'values': [posPercentage, negPercentage],
# 'marker': {'colors': ['#32C85A', '#FA4632'],
},
}],
'layout': {
'title': {
'text': "Twitter Sentiment for {}\n".format(value),
},
}
})
return fig
def update_graph_live(value):
mongo = MongoWrapper()
getTweets = mongo.get_tweets_with_lat_long('Facebook')
allLatitude = getTweets['Latitude']
allLongitude = getTweets['Longitude']
allSentiment = getTweets['Sentiment_Value']
bigGraph = dcc.Graph(
style={'height': '800px'},
figure={
'data': [{
'type': 'scattergeo',
'locationmode': 'USA-states',
'lon': allLongitude,
'lat': allLatitude,
'text': allSentiment,
'mode': 'markers',
'marker': {
'size': 8,
'opacity': 0.8,
'reversescale': True,
'autocolorscale': False,
'symbol': 'circle',
'line': {
'width': 1,
'color': 'rgba(102, 102, 102)'
},
'colorscale': scl,
'cmin': -1,
'color': allSentiment,
'cmax': 1,
'colorbar': {
'title': "Polarity Scale"
}
}
}],
'layout': {
'title': {
'text': 'Tweet locations with sentiment ratings',
},
'font': {
'size': 15,
},
'geo': {
# 'scope':'usa',
# 'projection':dict( 'type'='albers usa' ),
'showland': True,
'landcolor': "rgb(250, 250, 250)",
'subunitcolor': "rgb(217, 217, 217)",
'countrycolor': "rgb(217, 217, 217)",
'countrywidth': 0.5,
'subunitwidth': 0.5
},
}
})
return (bigGraph)
def update_graph_live(value):
mongo = MongoWrapper()
neg_sentiment, neutral_sentiment, pos_sentiment = mongo.get_polarity_tweets_of_stock(value)
print(neg_sentiment)
negs = neg_sentiment.count()
poss = pos_sentiment.count()
neuu = neutral_sentiment.count()
tots = negs+poss+neuu
posPercentage = (poss/(tots))*100
# neuPercentage = (neuu/(tots))*100
negPercentage = (negs/(tots))*100
# print(posPercentage)
# print(neuPercentage)
# print(negPercentage)
# print(tots)
bigGraph = dcc.Graph(
animate=False,
figure={
'data': [{
'type': 'pie',
'labels': ['Positive', 'Neutral' ,'Negative'],
# 'values': [posPercentage, neuPercentage, negPercentage],
# 'marker': {'colors': ['#32C85A', '#4C93B1', '#FA4632'],
'values': [posPercentage, negPercentage],
'marker': {'colors': ['#32C85A', '#FA4632'],
},
}],
'layout':{
'title':"Twitter Sentiment for {}\n".format(value)
}
}
),
afterText = html.Div('Total Tweets pulled for searchword: {}.\n'.format(tots))
return (bigGraph, afterText)
def test_db_insert(self):
CONSUMER_KEY = '1pmAQV2KBiItz3OsGWIrAPQrv'
CONSUMER_SECRET = 'oPIkrswP7eJh1gwaDiyOY8meYbcMTMhEsKnG5sdtethQSxSlMB'
ACCESS_TOKEN = '985993112-hRyi1aAGNzOGy0jan6WxC5UiEedZsCezpl1WxVKF'
ACCESS_SECRET = 'LrbZTey5eVpeRtOwOpdAMj0XVztupHF3vkqze4yBcdq7h'
def twitter_setup():
try:
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)
api = tweepy.API(auth,
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
return api
except:
print("Error: Authentication Failed")
pullTweets = twitter_setup()
tweets = pullTweets.search(q=["Facebook"],
count=200,
tweet_mode="extended")
mongo = MongoWrapper()
mongo.insert_tweet_into_db(tweets, "Facebook")
def mainFunction(value):
mongo = MongoWrapper()
neg_sentiment, neutral_sentiment, pos_sentiment = mongo.get_polarity_tweets_of_stock(
value)
negs = neg_sentiment.count()
poss = pos_sentiment.count()
neuu = neutral_sentiment.count()
tots = negs + poss + neuu
posPercentage = (poss / (tots)) * 100
neuPercentage = (neuu / (tots)) * 100
negPercentage = (negs / (tots)) * 100
# print(posPercentage, neuPercentage, negPercentage)
return dcc.Graph(
figure={
'data': [{
'type': 'pie',
'labels': ['Positive', 'Neutral', 'Negative'],
'values': [posPercentage, neuPercentage, negPercentage],
'marker': {
'colors': ['#32C85A', '#4C93B1', '#FA4632'],
# 'values': [posPercentage, negPercentage],
# 'marker': {'colors': ['#32C85A', '#FA4632'],
},
}],
'layout': {
'title': {
'text': "Twitter Sentiment for {}\n".format(value),
},
}
}), html.Div(
children='Total Tweets pulled for searchword: {}.\n'.format(tots)),
from WatchDogs_MongoWrapper import MongoWrapper
if __name__ == "__main__":
mongo = MongoWrapper()
logger = mongo.get_logger('Tweets Stats')
logger.info('# of companies in Companies in the database: ' +
str(mongo.print_statistics('Stocks')))
logger.info('# of companies in Tweets in the database: ' +
str(mongo.print_statistics('Tweets')))
def test_get_tweets_with_lat_long(self):
mongo = MongoWrapper()
test = mongo.get_tweets_with_lat_long('Google')
print(test)
def test_filter(self):
start = datetime.datetime(2019, 4, 8, 20, 16, 6)
end = datetime.datetime(2019, 4, 8, 20, 16, 10)
mng = MongoWrapper()
x = mng.filter_docs(start, end)
self.assertEqual(x.count(), 2)
def test_get_stocks_polarity(self):
mongo = MongoWrapper()
print(mongo.get_polarity_tweets_of_stock("3D Systems Corporation"))
def test_lat_long(self):
mongo = MongoWrapper()
print(mongo.get_lat_long('Facebook'))
import pymongo
from WatchDogs_MongoWrapper import MongoWrapper
if __name__ == "__main__":
mongo_wrapper = MongoWrapper()
# mongo_wrapper.del_docs_in_collection('Stocks')
comapny_file = open('companies.txt', 'r')
mongo_wrapper.load_companies(comapny_file)
def test_stocks_def(self):
mongo_wrapper = MongoWrapper()
test = mongo_wrapper.get_tweets_of_stock("3D Systems Corporation")
self.assertGreater(test.count(), 0)
import plotly
import plotly.graph_objs as go
from WatchDogs_MongoWrapper import MongoWrapper
from dash.dependencies import Input, Output
import pandas as pd
df = pd.read_csv('/Users/iankresyman/Desktop/2011_february_us_airport_traffic2.csv')
df.head()
df['text'] = df['airport'] + '' + df['city'] + ', ' + df['state'] + '' + 'Arrivals: ' + df['cnt'].astype(str)
scl = [ [0,"rgb(39,174,96)"],[0.35,"rgb(46,204,113)"],[0.5,"rgb(241,196,15)"],\
[0.6,"rgb(243,156,18)"],[0.7,"rgb(231,76,60)"],[1,"rgb(192,57,43)"] ]
mongo = MongoWrapper()
negCoord, neuCoord, posCoord = mongo.get_lat_long('Facebook')
getTweets = mongo.get_tweets_with_lat_long('Facebook')
allLatitude = getTweets['Latitude']
allLongitude = getTweets['Longitude']
allSentiment = getTweets['Sentiment_Value']
print('\n')
# print(negCoord[0])
# df1 = pd.DataFrame()
# df2 = pd.DataFrame()
# df3 = pd.DataFrame()
from WatchDogs_MongoWrapper import MongoWrapper
if __name__ == "__main__":
mongo_wrapper = MongoWrapper()
test = mongo_wrapper.get_tweets_of_stock("3D Systems Corporation")
for x in test:
print(x)
import tweepy
from WatchDogs_MongoWrapper import MongoWrapper
CONSUMER_KEY = "lQu7vXPH6hMNQ92LLVOFD5K8b"
CONSUMER_SECRET = "NwoUxJgXZkA6Fm0IJpNC6S0bavZy3YVcblKrQQbna7R4lwj39X"
ACCESS_TOKEN = "1170146904-NfXboXT6cjI8zcThUiixJp1AztyJH4Hw8wdkDwj"
ACCESS_SECRET = "rpRprMkactMVuKiJsESZGjEUmbixPABB3NjIKOxymJT7K"
def twitter_setup():
try:
auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_TOKEN, ACCESS_SECRET)
api = tweepy.API(auth,
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
return api
except:
print("Error: Authentication Failed")
if __name__ == "__main__":
pullTweets = twitter_setup()
tweets = pullTweets.search(q=["Boeing"], count=200, tweet_mode="extended")
mongo = MongoWrapper()
mongo.get_polarity_tweets_of_stock("Aramark")
import datetime
from WatchDogs_MongoWrapper import MongoWrapper
if __name__ == "__main__":
start = datetime.datetime(2019, 4, 8, 20, 16, 6)
end = datetime.datetime(2019, 4, 8, 20, 16, 10)
mng = MongoWrapper()
x = mng.filter_docs(start, end)
for each_x in x:
print(each_x)
elif number == 1:
consumer_api_key = twitter_credentials.consumer_api_key
consumer_api_secret_key = twitter_credentials.consumer_api_secret_key
access_token = twitter_credentials.access_token
access_token_secret = twitter_credentials.access_token_secret
auth = OAuthHandler(consumer_api_key, consumer_api_secret_key)
auth.set_access_token(access_token, access_token_secret)
return auth
arr = ["like", "football", "love", "car"]
threads = []
i = 0
mng = MongoWrapper()
stock_companies = []
for stock in mng.get_all_stocks():
stock_companies.append(stock['Company'])
print(stock_companies)
for a in stock_companies:
auth = getAuth()
thread = threading.Thread(target=multiple_producer_thread, args=(auth, a))
thread.start()
threads.append(thread)
for thread in threads:
thread.join()
'''
def update_graph_live(value):
mongo = MongoWrapper()
getTweets = mongo.get_tweets_with_lat_long(value)
allLatitude = getTweets['Latitude']
allLongitude = getTweets['Longitude']
allSentiment = getTweets['Sentiment_Value']
tots = allSentiment.count()
scl = [ [0,"rgb(39,174,96)"],[0.35,"rgb(46,204,113)"],[0.5,"rgb(241,196,15)"],\
[0.6,"rgb(243,156,18)"],[0.7,"rgb(231,76,60)"],[1,"rgb(192,57,43)"] ]
bigGraph = dcc.Graph(
style={'height': '800px'},
figure={
'data': [{
'type': 'scattergeo',
'locationmode': 'USA-states',
'lon': allLongitude,
'lat': allLatitude,
'text': allSentiment,
'mode': 'markers',
'marker': {
'size': 8,
'opacity': 0.8,
'reversescale': True,
'autocolorscale': False,
'symbol': 'circle',
'line': {
'width': 1,
'color': 'rgba(102, 102, 102)'
},
'colorscale': scl,
'cmin': -1,
'color': allSentiment,
'cmax': 1,
'colorbar': {
'title': "Polarity Scale",
'thickness': 20,
'titleside': "right",
# 'outlinecolor' : "rgba(68, 68, 68, 0)",
'ticks': "outside",
'ticklen': 3,
# 'showticksuffix' : "last",
# 'ticksuffix' : " inches",
'dtick': 0.1
}
}
}],
'layout': {
'title': "Twitter Sentiment for {}\n".format(value),
'font': {
'size': 15,
},
'geo': {
# 'scope':'usa',
# 'projection':dict( 'type'='albers usa' ),
'showland': True,
'landcolor': "rgb(250, 250, 250)",
'subunitcolor': "rgb(217, 217, 217)",
'countrycolor': "rgb(217, 217, 217)",
'countrywidth': 0.5,
'subunitwidth': 1,
'showsubunits': True,
'showcountries': True,
'showcoastlines': True,
'coastlinecolor': "rgb(155, 155, 155)",
'showframe': True,
'framecolor': "rgb(155, 155, 155)"
# 'showocean':True
# 'showlakes':True
},
}
}), html.Div(
children='Total Tweets pulled for searchword: {}.\n'.format(tots)),
return (bigGraph)
from kafka import KafkaConsumer
from json import loads
from WatchDogs_MongoWrapper import MongoWrapper
from WatchDogs_RedisWrapper import RedisWrapper
from constants import Constants
"""
Mongo_Group: The consumer is Mongo
Redis_Group: The consumer is Redis
"""
if __name__ == "__main__":
Constants = Constants()
r = RedisWrapper(Constants.decrypt_key)
mng = MongoWrapper(Constants.decrypt_key)
test_logger = mng.get_logger('Kafka DB Populator')
consumer = KafkaConsumer(
'mongoredis',
bootstrap_servers=['34.83.10.248:9092'],
auto_offset_reset='earliest',
enable_auto_commit=True,
group_id='MongoRedis',
value_deserializer=lambda x: loads(x.decode('utf-8')))
# r.redis_flush_all()
# test_logger.info('Redis Cache Flushed')
#### Pull all Companies and update the cache first
for each_company in mng.get_all_stocks():
stock_name = each_company['Company']
r.redis_update_json('get_tweets_with_lat_long/', stock_name)
r.redis_update_json('get_polarity_tweets_of_stock/', stock_name)
class RedisWrapper():
def __init__(self, decrypt_key):
self.redicclient = redis.StrictRedis(host='35.236.16.13', port=6379, db=0, decode_responses=True)
self.mng = MongoWrapper(decrypt_key)
def get_logger(self, logger_name):
return self.mng.get_logger(logger_name)
def redis_update_json(self, api_string, key):
if api_string == 'get_tweets_with_lat_long/':
json_data = self.mng.get_tweets_with_lat_long(key)
self.redicclient.execute_command('JSON.SET', api_string+key, '.', json_data)
elif api_string == 'get_polarity_tweets_of_stock/':
json_data = self.mng.get_polarity_tweets_of_stock(key)
self.redicclient.execute_command('JSON.SET', api_string + key, '.', json_data)
def redis_insert_tweet(self, key, tweet):
"""
Either insert a single tweet or multiple tweets and this def will update the redis cache accordingly
:param key:
:param tweets:
:return:
"""
try:
lat_long_list = tweet['Geo']['coordinates']
has_lat_long = True
except:
has_lat_long = False
lat_long_list = ['None', 'None']
if has_lat_long:
sentiment_value = float(tweet["Sentiment_Value"])
full_text = tweet["Text"]
root_json_path = {}
root_json_path["Latitude"] = lat_long_list[0]
root_json_path["Longitude"] = lat_long_list[1]
root_json_path["Tweet_Text"] = full_text
root_json_path["Sentiment_Value"] = sentiment_value
api_string = 'get_tweets_with_lat_long/'
self.redicclient.execute_command('JSON.ARRAPPEND', api_string+key, '.', json.dumps(root_json_path))
sentiment_polarity = int(tweet["Sentiment_Polarity"])
full_text = tweet["Text"]
root_json_path = {}
root_json_path["Latitude"] = lat_long_list[0]
root_json_path["Longitude"] = lat_long_list[1]
root_json_path["Tweet_Text"] = full_text
root_json_path["Sentiment_Polarity"] = sentiment_polarity
api_string = 'get_polarity_tweets_of_stock/'
if sentiment_polarity == -1:
root_path = '.Negative_Tweets'
elif sentiment_polarity == 0:
root_path = '.Neutral_Tweets'
elif sentiment_polarity == 1:
root_path = '.Positive_Tweets'
self.redicclient.execute_command('JSON.ARRAPPEND', api_string + key, root_path, json.dumps(root_json_path))
def redis_get_json(self, api_string, key):
return self.redicclient.execute_command('JSON.GET', api_string+key)
def redis_flush_all(self):
"""
Danger. This flushes the whole DB. Careful
:return:
"""
self.redicclient.flushdb()
def __init__(self, decrypt_key):
self.redicclient = redis.StrictRedis(host='35.236.16.13', port=6379, db=0, decode_responses=True)
self.mng = MongoWrapper(decrypt_key)
from WatchDogs_MongoWrapper import MongoWrapper
if __name__ == "__main__":
mongo = MongoWrapper()
logger = mongo.get_logger('Tweets Stats')
logger.info('# of companies in Companies in the database: '+str(mongo.print_statistics('Stocks')))
logger.info('# of companies in Tweets in the database: '+str(mongo.print_statistics('Tweets')))
import pymongo
from WatchDogs_MongoWrapper import MongoWrapper
if __name__ == "__main__":
mongo_wrapper = MongoWrapper()
mongo_wrapper.del_docs_in_collection('Tweets')
