def test_sliding_company_score(redis_client, foo_company):
' should update average on every tweet received '
db.write_tweet(redis_client, foo_company, MockTweet(id=100), 0.1)
db.write_tweet(redis_client, foo_company, MockTweet(id=101), 0.2)
db.set_company_score(redis_client, foo_company, 0.15)
api = MockApi(
MockTweet(id=102, text='0.3'),
MockTweet(id=103, text='-0.4'),
MockTweet(id=104, text='0.4'),
MockTweet(id=105, text='-0.5')
)
tweets_processed = 0
for res in sync_tweets(api, redis_client, foo_company, max_tweets=4):
if tweets_processed == 0:
# [100, 101, 102]
assert db.get_tweets_timeline(redis_client, foo_company, withscores=False) == [b'100', b'101', b'102']
expected_score = (0.1 + 0.2 + 0.3) / 3
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
elif tweets_processed == 1:
assert db.get_tweets_timeline(redis_client, foo_company, withscores=False) == [b'100', b'101', b'102', b'103']
expected_score = (0.1 + 0.2 + 0.3 - 0.4) / 4
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
elif tweets_processed == 2:
assert db.get_tweets_timeline(redis_client, foo_company, withscores=False) == [b'101', b'102', b'103', b'104']
expected_score = (0.2 + 0.3 - 0.4 + 0.4) / 4
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
tweets_processed += 1
assert db.get_tweets_timeline(redis_client, foo_company, withscores=False) == [b'102', b'103', b'104', b'105']
expected_score = (0.3 + 0.4 - 0.4 - 0.5) / 4
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
def test_rem_tweet(redis_client, foo_company):
tweet1 = MockTweet(id=100)
tweet2 = MockTweet(id=101)
db.write_tweet(redis_client, foo_company, tweet1, tweet_score=0.8)
db.write_tweet(redis_client, foo_company, tweet2, tweet_score=0.8)
assert db.get_number_of_tweets(redis_client, foo_company) == 2
assert len(db.get_tweets_timeline(redis_client, foo_company)) == 2
db.remove_tweets(redis_client, foo_company, 100, 101)
assert db.get_number_of_tweets(redis_client, foo_company) == 0
assert len(db.get_tweets_timeline(redis_client, foo_company)) == 0
def test_remove_tweet(redis_client, foo_company):
' Calling db.remove_tweets will correctly remove a tweet '
tweet = MockTweet(id=100)
db.write_tweet(redis_client, foo_company, tweet, tweet_score=-0.8)
db.remove_tweets(redis_client, foo_company, 100)
assert len(db.get_best_tweets(redis_client, foo_company)) == 0
assert len(db.get_tweets_timeline(redis_client, foo_company)) == 0
def test_write_tweet(redis_client, foo_company):
tweet = MockTweet(id=100)
db.write_tweet(redis_client, foo_company, tweet, tweet_score=0.8)
timeline = db.get_tweets_timeline(redis_client, foo_company)
assert db.get_tweet_score(redis_client, foo_company, b'100') == 0.8
assert db.get_number_of_tweets(redis_client, foo_company) == 1
assert len(timeline) == 1
assert timeline[0][0] == b'100' # tweet id is used as a key
assert timeline[0][1] == 100 # and also used as score
def sync_tweets(api, redis_client, account, limit=inf, max_tweets=inf):
''' will fetch max of `limit` tweets from Twitter mentioning `@account`,
analyze them and write to db '''
since_id = db.get_max_tweet_id(redis_client, account)
print('Started syncing tweets mentioning ', account, 'starting from ',
since_id)
tweets_processed = 0
for tweet in twitter.get_tweets_about_company(api,
account,
since_id=since_id):
if tweets_processed >= limit:
break
score = get_sentiment_score(tweet.text, tweet.entities)
if score == 0.0:
continue
with redis_client.pipeline() as pipe:
pipe.watch('tweetsabout:{0}'.format(account))
n_tweets = db.get_number_of_tweets(pipe, account)
current_score = db.get_company_score(redis_client, account)
tweets_to_remove = db.get_tweets_timeline(
pipe, account, 1,
withscores=False) if n_tweets >= max_tweets else []
removed_tweets_score = sum([
db.get_tweet_score(pipe, account, tweet_id)
for tweet_id in tweets_to_remove
])
pipe.multi()
# batch-process insert operation
db.write_tweet(pipe, account, tweet, score)
db.remove_tweets(pipe, account, *tweets_to_remove)
# sliding re-calculation of the score
new_score = (
(current_score * n_tweets - removed_tweets_score + score) /
(n_tweets + 1 - len(tweets_to_remove)))
db.set_company_score(pipe, account, new_score)
pipe.execute()
tweets_processed += 1
yield {
'account': account,
'tweet': tweet,
'score': score,
'removed': tweets_to_remove
}
def test_max_tweets(redis_client, foo_company):
' should remove tweets if `max_tweets` was reached '
db.write_tweet(redis_client, foo_company, MockTweet(id=100), 0.1)
db.write_tweet(redis_client, foo_company, MockTweet(id=101), 0.2)
db.set_company_score(redis_client, foo_company, 0.15)
api = MockApi(
MockTweet(id=102, text='0.3'),
MockTweet(id=103, text='0.4')
)
for res in sync_tweets(api, redis_client, foo_company, max_tweets=3):
pass
assert db.get_number_of_tweets(redis_client, foo_company) == 3
expected_score = (0.2 + 0.3 + 0.4) / 3
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
assert db.get_tweets_timeline(redis_client, foo_company, withscores=False) == [b'101', b'102', b'103']
def get_tweets(account):
return db.get_tweets_timeline(r, account)
import redis
from lib import db
import sys
r = redis.StrictRedis(host='redis', decode_responses=True)
for company in db.get_companies(r):
try:
print('company', company, 'has', db.get_number_of_tweets(r, company),
'tweets')
print('removing unneccessary')
for tweet_id in db.get_tweets_timeline(r, company, withscores=False):
tweet_score = db.get_tweet_score(r, company, tweet_id)
if (tweet_score == 0.0):
print('removing', tweet_id, tweet_score)
db.remove_tweets(r, company, tweet_id)
print('company', company, 'has', db.get_number_of_tweets(r, company),
'tweets')
print('score before', db.get_company_score(r, company))
db.update_company_overall_score(r, company)
print('score after', db.get_company_score(r, company))
except Exception:
r.delete('companyscore:{0}'.format(company))
pass