def test_sliding_company_score(redis_client, foo_company):
' should update average on every tweet received '
db.write_tweet(redis_client, foo_company, MockTweet(id=100), 0.1)
db.write_tweet(redis_client, foo_company, MockTweet(id=101), 0.2)
db.set_company_score(redis_client, foo_company, 0.15)
api = MockApi(
MockTweet(id=102, text='0.3'),
MockTweet(id=103, text='-0.4'),
MockTweet(id=104, text='0.4'),
MockTweet(id=105, text='-0.5')
)
tweets_processed = 0
for res in sync_tweets(api, redis_client, foo_company, max_tweets=4):
if tweets_processed == 0:
# [100, 101, 102]
assert db.get_tweets_timeline(redis_client, foo_company, withscores=False) == [b'100', b'101', b'102']
expected_score = (0.1 + 0.2 + 0.3) / 3
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
elif tweets_processed == 1:
assert db.get_tweets_timeline(redis_client, foo_company, withscores=False) == [b'100', b'101', b'102', b'103']
expected_score = (0.1 + 0.2 + 0.3 - 0.4) / 4
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
elif tweets_processed == 2:
assert db.get_tweets_timeline(redis_client, foo_company, withscores=False) == [b'101', b'102', b'103', b'104']
expected_score = (0.2 + 0.3 - 0.4 + 0.4) / 4
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
tweets_processed += 1
assert db.get_tweets_timeline(redis_client, foo_company, withscores=False) == [b'102', b'103', b'104', b'105']
expected_score = (0.3 + 0.4 - 0.4 - 0.5) / 4
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
def sync_tweets(api, redis_client, account, limit=inf, max_tweets=inf):
''' will fetch max of `limit` tweets from Twitter mentioning `@account`,
analyze them and write to db '''
since_id = db.get_max_tweet_id(redis_client, account)
print('Started syncing tweets mentioning ', account, 'starting from ',
since_id)
tweets_processed = 0
for tweet in twitter.get_tweets_about_company(api,
account,
since_id=since_id):
if tweets_processed >= limit:
break
score = get_sentiment_score(tweet.text, tweet.entities)
if score == 0.0:
continue
with redis_client.pipeline() as pipe:
pipe.watch('tweetsabout:{0}'.format(account))
n_tweets = db.get_number_of_tweets(pipe, account)
current_score = db.get_company_score(redis_client, account)
tweets_to_remove = db.get_tweets_timeline(
pipe, account, 1,
withscores=False) if n_tweets >= max_tweets else []
removed_tweets_score = sum([
db.get_tweet_score(pipe, account, tweet_id)
for tweet_id in tweets_to_remove
])
pipe.multi()
# batch-process insert operation
db.write_tweet(pipe, account, tweet, score)
db.remove_tweets(pipe, account, *tweets_to_remove)
# sliding re-calculation of the score
new_score = (
(current_score * n_tweets - removed_tweets_score + score) /
(n_tweets + 1 - len(tweets_to_remove)))
db.set_company_score(pipe, account, new_score)
pipe.execute()
tweets_processed += 1
yield {
'account': account,
'tweet': tweet,
'score': score,
'removed': tweets_to_remove
}
def test_max_tweets(redis_client, foo_company):
' should remove tweets if `max_tweets` was reached '
db.write_tweet(redis_client, foo_company, MockTweet(id=100), 0.1)
db.write_tweet(redis_client, foo_company, MockTweet(id=101), 0.2)
db.set_company_score(redis_client, foo_company, 0.15)
api = MockApi(
MockTweet(id=102, text='0.3'),
MockTweet(id=103, text='0.4')
)
for res in sync_tweets(api, redis_client, foo_company, max_tweets=3):
pass
assert db.get_number_of_tweets(redis_client, foo_company) == 3
expected_score = (0.2 + 0.3 + 0.4) / 3
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
assert db.get_tweets_timeline(redis_client, foo_company, withscores=False) == [b'101', b'102', b'103']
def test_prior_tweets(redis_client, foo_company):
' should update score correctly if some tweets existed prior to syncing '
db.write_tweet(redis_client, foo_company, MockTweet(id=100), 0.1)
db.write_tweet(redis_client, foo_company, MockTweet(id=101), 0.2)
db.set_company_score(redis_client, foo_company, 0.15)
assert db.get_number_of_tweets(redis_client, foo_company) == 2
api = MockApi(
MockTweet(id=102, text='0.3'),
MockTweet(id=103, text='0.4')
)
for res in sync_tweets(api, redis_client, foo_company):
pass
assert db.get_number_of_tweets(redis_client, foo_company) == 4
expected_score = (0.1 + 0.2 + 0.3 + 0.4) / 4
assert db.get_company_score(redis_client, foo_company) == approx(expected_score)
def test_company_score(redis_client, foo_company):
' Should be able to get/set company score '
db.set_company_score(redis_client, foo_company, 0.25)
assert db.get_company_score(redis_client, foo_company) == 0.25