def setUp(self):
TweetFeels._db_factory = (lambda db: MagicMock())
TweetFeels._auth_factory = (lambda cred: MagicMock())
TweetFeels._listener_factory = (lambda ctrl: MagicMock())
TweetFeels._stream_factory = (lambda auth, listener: MagicMock())
self.tweets_data_path = 'test/sample.json'
self.tweets = [
{'created_at': 'Sun Feb 19 09:14:18 +0000 2017',
'id_str': '833394296418082817',
'text': 'Tweetfeels is tremendous! Believe me. I know.',
'user': {'followers_count': '100', 'friends_count': '200',
'location':None}
}, # sentiment value = 0
{'created_at': 'Sun Feb 21 18:14:19 +0000 2017',
'id_str': '833394296418082818',
'text': 'Fake news. Sad!',
'user': {'followers_count': '100', 'friends_count': '200',
'location':None}
}, # sentiment value = -0.7351
{'created_at': 'Sun Feb 21 19:14:20 +0000 2017',
'id_str': '833394296418082819',
'text': 'I hate it.',
'user': {'followers_count': '100', 'friends_count': '200',
'location':None}
} # sentiment value = -0.5719
]
self.mock_feels = TweetFeels('abcd')
self.feels_db = TweetData(file='./test/db.sqlite')
self.mock_feels._feels = self.feels_db
self.mock_tweets = [Tweet(t) for t in self.tweets]
for t in self.mock_tweets:
self.feels_db.insert_tweet(t)
self.mock_feels.clear_buffer()
def test_sentiment(self):
mock_feels = TweetFeels("abcd")
mock_feels._feels.tweets_since = MagicMock(return_value=[])
mock_feels._sentiment = Sentiment(0.5, 0, 0, 0)
mock_feels._latest_calc = datetime(2017, 1, 1, 0, 0, 0)
mock_feels._feels.start = datetime(2017, 1, 1, 0, 0, 0)
mock_feels._feels.end = datetime(2017, 1, 1, 0, 0, 0)
self.assertEqual(mock_feels.sentiment.value, 0.5)
def test_start(self):
mock_feels = TweetFeels("abcd")
mock_feels.tracking = []
mock_feels.start(selfupdate=0)
mock_feels._stream.filter.assert_not_called()
mock_feels.tracking = ['tsla']
mock_feels.start(selfupdate=0)
mock_feels._stream.filter.assert_called_once()
def result():
if request.method == 'POST':
consumer_key = twitterkey.get(twitterkey.id == 1234).CONSUMERKEY
consumer_secret = twitterkey.get(twitterkey.id == 1234).CONSUMERSECRET
access_token = twitterkey.get(twitterkey.id == 1234).ACCESSTOKEN
access_token_secret = twitterkey.get(
twitterkey.id == 1234).ACCESSTOKENSECRET
login_credentials = [
consumer_key, consumer_secret, access_token, access_token_secret
]
feels = TweetFeels(login_credentials, tracking=['trump'])
t_end = time.time() + 60
k = ""
while time.time() < t_end:
try:
feels.start()
time.sleep(4)
k = feels.sentiment.value
except:
time.sleep(4)
k = feels.sentiment.value
pubnub.publish().channel("kaja").message({'sentiment-value': k})
feels.stop()
return render_template('result.html')
def test_buffer(self):
mock_feels = TweetFeels('abcd')
mock_feels.buffer_limit = 5
feels_db = TweetData(file='sample.sqlite')
mock_feels._feels = feels_db
with open(self.tweets_data_path) as tweets_file:
lines = list(filter(None, (line.rstrip() for line in tweets_file)))
for line in lines[0:3]:
t = Tweet(json.loads(line))
mock_feels.on_data(t)
self.assertEqual(len(mock_feels._tweet_buffer), 3)
for line in lines[3:6]:
t = Tweet(json.loads(line))
mock_feels.on_data(t)
time.sleep(1) #this waits for items to finish popping off the buffer
self.assertEqual(len(mock_feels._tweet_buffer), 0)
dfs = [df for df in mock_feels._feels.all]
self.assertEqual(len(dfs[0]), 6)
os.remove('sample.sqlite')
consumer_secret = "pfOKhAgpRpxdxu2SoY9xybKvHxu5m2uIolpg10WfWbi3NBTQDm"
login = [consumer_key, consumer_secret, access_token, access_token_secret]
tracking_keys = [
generate_string_fragments('beto o\'rourke') + ['@BetoORourke'],
list(set(generate_string_fragments('ted cruz')) - set(['ted', 'Ted'])) + ['@tedcruz']
]
for idx in range(0, len(tracking_keys)):
print(f'Searching for {tracking_keys[idx]} in stream {idx}')
tracking_names = ['Beto O\'Rourke', 'Ted Cruz']
db_names = ['{0}.sqlite'.format(slugify(name)) for name in tracking_names]
feels = [TweetFeels(login, tracking=tracking, db=dbname) for tracking, dbname in zip(tracking_keys, db_names)]
p = figure(sizing_mode='stretch_both')
p.legend.location = "top_left"
colors = itertools.cycle(palette)
rs = [p.line([], [], color=color, line_width=2, legend=name) for name, color in zip(tracking_names, colors)]
# Horizontal line
hline = Span(location=0, dimension='width', line_color='black', line_width=3)
pos_box = BoxAnnotation(left=-sys.maxsize, right=sys.maxsize, top=1, bottom=0, fill_color='green', fill_alpha=0.1)
neg_box = BoxAnnotation(left=-sys.maxsize, right=sys.maxsize, top=0, bottom=-1, fill_color='red', fill_alpha=0.1)
p.renderers.extend([hline, pos_box, neg_box])
dss = [r.data_source for r in rs]
def main():
login = oath()
#bottom code doesn't work
'''
for company in companies:
launch(company, login)
'''
company = 'apple'
dbname = company + '.sqlite'
feels = TweetFeels(login, tracking=keywords[company], db=dbname)
t = Thread(target=print_sentiments, args=[feels, company])
company = 'microsoft'
dbname = company + '.sqlite'
feels2 = TweetFeels(login, tracking=keywords[company], db=dbname)
t2 = Thread(target=print_sentiments, args=[feels2, company])
company = 'google'
dbname = company + '.sqlite'
feels3 = TweetFeels(login, tracking=keywords[company], db=dbname)
t3 = Thread(target=print_sentiments, args=[feels3, company])
company = 'facebook'
dbname = company + '.sqlite'
feels4 = TweetFeels(login, tracking=keywords[company], db=dbname)
t4 = Thread(target=print_sentiments, args=[feels4, company])
company = 'amazon'
dbname = company + '.sqlite'
feels5 = TweetFeels(login, tracking=keywords[company], db=dbname)
t5 = Thread(target=print_sentiments, args=[feels5, company])
feels.start()
feels2.start()
feels3.start()
feels4.start()
feels5.start()
t.start()
t2.start()
t3.start()
t4.start()
t5.start()
def launch(company, login):
dbname = company + '.sqlite'
feels = TweetFeels(login, tracking=keywords[company], db=dbname)
t = Thread(target=print_sentiments, args=[feels, company])
feels.start()
t.start()
access_token = "548017247-WGOckwwa33tD4a49G2Sf2ubExZM9qiF7awSePnic"
access_token_secret = "rwmOnGksWhKEVIRoKtXilcUEZmnB6SxQ8wFT7CoW2FeTo"
consumer_key = "2UQPiHEaCsx5L96KlTUVISweq"
consumer_secret = "pfOKhAgpRpxdxu2SoY9xybKvHxu5m2uIolpg10WfWbi3NBTQDm"
login = [consumer_key, consumer_secret, access_token, access_token_secret]
def print_feels(feels=None, seconds=10):
time.sleep(seconds)
print(f'[{time.ctime()}] Sentiment Score: {feels.sentiment.value}')
return False
tesla_feels = TweetFeels(login,
tracking=['tesla', 'tsla', 'gigafactory', 'elonmusk'],
db='tesla.sqlite')
tesla_feels.start(seconds=120)
time.sleep(125)
import pandas as pd
from datetime import timedelta, datetime
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
data1 = {
s.end: s.value
for s in tesla_feels.sentiments(delta_time=timedelta(minutes=15),
nans=True)
}
data2 = {
def test_on_data(self):
mock_feels = TweetFeels("abcd")
mock_feels.buffer_limit = 0
data = {'filter_level': 'low', 'text': 'test data'}
mock_feels.on_data(data)
mock_feels._feels.insert_tweet.assert_called_once()
# test filtering levels
mock_feels2 = TweetFeels("abcd")
mock_feels2._filter_level = 'medium'
mock_feels2.on_data(data)
mock_feels2._feels.insert_tweet.assert_not_called()
# test buffer limit. no inserts until we are over limit
mock_feels2.buffer_limit = 2
mock_feels2.filter_level = 'low'
mock_feels2.on_data(data)
mock_feels2._feels.insert_tweet.assert_not_called()
mock_feels2.on_data(data)
mock_feels2.on_data(data)
mock_feels._feels.insert_tweet.assert_called_once()
def test_stop(self):
mock_feels = TweetFeels("abcd")
mock_feels.stop()
mock_feels._stream.disconnect.assert_called_once()
class Test_Feels(unittest.TestCase):
def setUp(self):
TweetFeels._db_factory = (lambda db: MagicMock())
TweetFeels._auth_factory = (lambda cred: MagicMock())
TweetFeels._listener_factory = (lambda ctrl: MagicMock())
TweetFeels._stream_factory = (lambda auth, listener: MagicMock())
self.tweets_data_path = 'test/sample.json'
self.tweets = [
{'created_at': 'Sun Feb 19 09:14:18 +0000 2017',
'id_str': '833394296418082817',
'text': 'Tweetfeels is tremendous! Believe me. I know.',
'user': {'followers_count': '100', 'friends_count': '200',
'location':None}
}, # sentiment value = 0
{'created_at': 'Sun Feb 21 18:14:19 +0000 2017',
'id_str': '833394296418082818',
'text': 'Fake news. Sad!',
'user': {'followers_count': '100', 'friends_count': '200',
'location':None}
}, # sentiment value = -0.7351
{'created_at': 'Sun Feb 21 19:14:20 +0000 2017',
'id_str': '833394296418082819',
'text': 'I hate it.',
'user': {'followers_count': '100', 'friends_count': '200',
'location':None}
} # sentiment value = -0.5719
]
self.mock_feels = TweetFeels('abcd')
self.feels_db = TweetData(file='./test/db.sqlite')
self.mock_feels._feels = self.feels_db
self.mock_tweets = [Tweet(t) for t in self.tweets]
for t in self.mock_tweets:
self.feels_db.insert_tweet(t)
self.mock_feels.clear_buffer()
def tearDown(self):
os.remove('./test/db.sqlite')
def test_start(self):
mock_feels = TweetFeels("abcd")
mock_feels.tracking = []
mock_feels.start(selfupdate=0)
mock_feels._stream.filter.assert_not_called()
mock_feels.tracking = ['tsla']
mock_feels.start(selfupdate=0)
mock_feels._stream.filter.assert_called_once()
def test_stop(self):
mock_feels = TweetFeels("abcd")
mock_feels.stop()
mock_feels._stream.disconnect.assert_called_once()
def test_on_data(self):
mock_feels = TweetFeels("abcd")
mock_feels.buffer_limit = 0
data = {'filter_level': 'low', 'text': 'test data'}
mock_feels.on_data(data)
mock_feels._feels.insert_tweet.assert_called_once()
# test filtering levels
mock_feels2 = TweetFeels("abcd")
mock_feels2._filter_level = 'medium'
mock_feels2.on_data(data)
mock_feels2._feels.insert_tweet.assert_not_called()
# test buffer limit. no inserts until we are over limit
mock_feels2.buffer_limit = 2
mock_feels2.filter_level = 'low'
mock_feels2.on_data(data)
mock_feels2._feels.insert_tweet.assert_not_called()
mock_feels2.on_data(data)
mock_feels2.on_data(data)
mock_feels._feels.insert_tweet.assert_called_once()
def test_sentiment(self):
mock_feels = TweetFeels("abcd")
mock_feels._feels.tweets_since = MagicMock(return_value=[])
mock_feels._sentiment = Sentiment(0.5, 0, 0, 0)
mock_feels._latest_calc = datetime(2017, 1, 1, 0, 0, 0)
mock_feels._feels.start = datetime(2017, 1, 1, 0, 0, 0)
mock_feels._feels.end = datetime(2017, 1, 1, 0, 0, 0)
self.assertEqual(mock_feels.sentiment.value, 0.5)
def test_buffer(self):
mock_feels = TweetFeels('abcd')
mock_feels.buffer_limit = 5
feels_db = TweetData(file='sample.sqlite')
mock_feels._feels = feels_db
with open(self.tweets_data_path) as tweets_file:
lines = list(filter(None, (line.rstrip() for line in tweets_file)))
for line in lines[0:3]:
t = Tweet(json.loads(line))
mock_feels.on_data(t)
self.assertEqual(len(mock_feels._tweet_buffer), 3)
for line in lines[3:6]:
t = Tweet(json.loads(line))
mock_feels.on_data(t)
time.sleep(1) #this waits for items to finish popping off the buffer
self.assertEqual(len(mock_feels._tweet_buffer), 0)
dfs = [df for df in mock_feels._feels.all]
self.assertEqual(len(dfs[0]), 6)
os.remove('sample.sqlite')
def test_sentiment_comprehensive(self):
sentiment = 0.0
for t in self.mock_tweets:
if t['sentiment']!=0:
sentiment = 0.99*sentiment + 0.01*t['sentiment']
# calc = 0*0.99**2 + 0.01*0.99*-0.7531 + 0.01*-0.5719
# = -0.01299649
self.mock_feels._latest_calc = self.mock_feels._feels.start
self.assertTrue(np.isclose(self.mock_feels.sentiment.value, sentiment))
# first observation is at 2017-2-19 19:14:18 and we are using default
# 60 second bins, therefore the observation at 2017-2-21 19:14:20 will
# never get saved but will always be recalculated.
self.assertEqual(self.mock_feels._latest_calc,
datetime(2017, 2, 21, 19, 14, 0))
# repeat the calculation, nothing changes
self.assertTrue(np.isclose(self.mock_feels.sentiment.value, sentiment))
self.assertEqual(self.mock_feels._latest_calc,
datetime(2017, 2, 21, 19, 14, 0))
self.assertEqual(self.mock_feels.sentiment.start,
self.mock_feels._latest_calc)
def test_sentiment_factor(self):
sentiment = 0.0
self.mock_feels.factor = 0.75
for t in self.mock_tweets:
if t['sentiment']!=0:
sentiment = 0.75*sentiment + 0.25*t['sentiment']
# calc = 0*0.75**2 + 0.25*0.75*-0.7531 + 0.25*-0.5719
# = -0.28418125
mock_sentiment = self.mock_feels.sentiment.value
self.assertTrue(np.isclose(mock_sentiment, sentiment))
def test_sentiment_binsize(self):
T = self.mock_tweets
A = T[1]['sentiment']
B = T[2]['sentiment']
sentiment = 0.75*0 + 0.25*(A+B)/2
self.mock_feels.factor = 0.75
self.mock_feels.binsize = timedelta(days=2.5)
mock_sentiment = self.mock_feels.sentiment.value
self.assertTrue(np.isclose(mock_sentiment, sentiment))
def test_nans(self):
sentiments = self.mock_feels.sentiments(
delta_time=timedelta(hours=24), nans=True)
s = next(sentiments)
self.assertEqual(s.value, 0)
s = next(sentiments)
self.assertTrue(np.isnan(s.value)) # can return nans
# does not affect current sentiment
self.assertEqual(self.mock_feels._sentiment.value, 0)
s = next(sentiments)
self.assertTrue(s.value<0)
def test_sentiments(self):
start = datetime(2017, 2, 19, 0, 0, 0)
dt = timedelta(minutes=30)
sentiment = self.mock_feels.sentiments(strt=start, delta_time=dt)
self.assertTrue(np.isclose(next(sentiment).value, 0))
self.assertTrue(np.isclose(next(sentiment).value, -0.007351))
self.assertTrue(np.isclose(next(sentiment).value, -0.01299649))
for s in sentiment:
print(s)
# we are starting at 2017-2-19 19:00:00 and using bins with length 30
# minutes, therefore our latest calc will be just prior to the final
# observation.
self.assertEqual(self.mock_feels._latest_calc,
datetime(2017, 2, 21, 19, 0, 0))
class twitterkey(BaseModel):
id = IntegerField(primary_key=True)
CONSUMERKEY = CharField(max_length=150)
CONSUMERSECRET = CharField(max_length=150)
ACCESSTOKEN = CharField(max_length=150)
ACCESSTOKENSECRET = CharField(max_length=150)
consumer_key = twitterkey.get(twitterkey.id == 1234).CONSUMERKEY
consumer_secret = twitterkey.get(twitterkey.id == 1234).CONSUMERSECRET
access_token = twitterkey.get(twitterkey.id == 1234).ACCESSTOKEN
access_token_secret = twitterkey.get(twitterkey.id == 1234).ACCESSTOKENSECRET
login = [consumer_key, consumer_secret, access_token, access_token_secret]
def print_feels(seconds=10, feels):
while go_on:
time.sleep(seconds)
print(
f'[{time.ctime()}] Sentiment Score: {trump_feels.sentiment.value}')
go_on = True
t = Thread(target=print_feels)
trump_feels = TweetFeels(login, tracking=['trump'])
trump_feels.start()
t.start()
print(trump_feels.sentiment.value)
#trump_feels.sentiment.value