def test_get_user_newsfeeds(self):
newsfeed_ids = []
for i in range(3):
tweet = self.create_tweet(self.dongxie)
newsfeed = self.create_newsfeed(self.linghu, tweet)
newsfeed_ids.append(newsfeed.id)
newsfeed_ids = newsfeed_ids[::-1]
RedisClient.clear()
conn = RedisClient.get_connection()
# cache miss
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
# cache hit
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
# cache updated
tweet = self.create_tweet(self.linghu)
new_newsfeed = self.create_newsfeed(self.linghu, tweet)
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
newsfeed_ids.insert(0, new_newsfeed.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
def test_fanout_main_task(self):
tweet = self.create_tweet(self.ray, 'tweet 1')
self.create_friendship(self.lux, self.ray)
msg = fanout_newsfeeds_main_task(tweet.id, self.ray.id)
self.assertEqual(
msg, '1 newsfeeds are going to fanout, 1 batches created.')
self.assertEqual(1 + 1, NewsFeed.objects.count())
cached_list = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual(len(cached_list), 1)
for i in range(2):
user = self.create_user('user{}'.format(i))
self.create_friendship(user, self.ray)
tweet = self.create_tweet(self.ray, 'tweet 2')
msg = fanout_newsfeeds_main_task(tweet.id, self.ray.id)
self.assertEqual(
msg, '3 newsfeeds are going to fanout, 1 batches created.')
self.assertEqual(4 + 2, NewsFeed.objects.count())
cached_list = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual(len(cached_list), 2)
user = self.create_user('another user')
self.create_friendship(user, self.ray)
tweet = self.create_tweet(self.ray, 'tweet 3')
msg = fanout_newsfeeds_main_task(tweet.id, self.ray.id)
self.assertEqual(
msg, '4 newsfeeds are going to fanout, 2 batches created.')
self.assertEqual(8 + 3, NewsFeed.objects.count())
cached_list = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual(len(cached_list), 3)
cached_list = NewsFeedService.get_cached_newsfeeds(self.lux.id)
self.assertEqual(len(cached_list), 3)
def test_get_user_newsfeeds(self):
newsfeed_timestamps = []
for i in range(3):
tweet = self.create_tweet(self.dongxie)
newsfeed = self.create_newsfeed(self.linghu, tweet)
newsfeed_timestamps.append(newsfeed.created_at)
newsfeed_timestamps = newsfeed_timestamps[::-1]
# cache miss
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.created_at for f in newsfeeds],
newsfeed_timestamps)
# cache hit
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.created_at for f in newsfeeds],
newsfeed_timestamps)
# cache updated
tweet = self.create_tweet(self.linghu)
new_newsfeed = self.create_newsfeed(self.linghu, tweet)
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
newsfeed_timestamps.insert(0, new_newsfeed.created_at)
self.assertEqual([f.created_at for f in newsfeeds],
newsfeed_timestamps)
def test_fanout_main_task(self):
tweet = self.create_tweet(self.linghu, 'tweet 1')
self.create_friendship(self.dongxie, self.linghu)
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
msg = fanout_newsfeeds_main_task(tweet.id, tweet.timestamp,
self.linghu.id)
self.assertEqual(1 + 1,
len(HBaseNewsFeed.filter(prefix=(None, None))))
else:
msg = fanout_newsfeeds_main_task(tweet.id, tweet.created_at,
self.linghu.id)
self.assertEqual(1 + 1, NewsFeed.objects.count())
self.assertEqual(msg,
'1 newsfeeds going to fanout, 1 batches created.')
cached_list = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_list), 1)
for i in range(2):
user = self.create_user('user{}'.format(i))
self.create_friendship(user, self.linghu)
tweet = self.create_tweet(self.linghu, 'tweet 2')
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
msg = fanout_newsfeeds_main_task(tweet.id, tweet.timestamp,
self.linghu.id)
self.assertEqual(4 + 2,
len(HBaseNewsFeed.filter(prefix=(None, None))))
else:
msg = fanout_newsfeeds_main_task(tweet.id, tweet.created_at,
self.linghu.id)
self.assertEqual(4 + 2, NewsFeed.objects.count())
self.assertEqual(msg,
'3 newsfeeds going to fanout, 1 batches created.')
cached_list = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_list), 2)
user = self.create_user('another user')
self.create_friendship(user, self.linghu)
tweet = self.create_tweet(self.linghu, 'tweet 3')
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
msg = fanout_newsfeeds_main_task(tweet.id, tweet.timestamp,
self.linghu.id)
self.assertEqual(8 + 3,
len(HBaseNewsFeed.filter(prefix=(None, None))))
else:
msg = fanout_newsfeeds_main_task(tweet.id, tweet.created_at,
self.linghu.id)
self.assertEqual(8 + 3, NewsFeed.objects.count())
self.assertEqual(msg,
'4 newsfeeds going to fanout, 2 batches created.')
cached_list = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_list), 3)
cached_list = NewsFeedService.get_cached_newsfeeds(self.dongxie.id)
self.assertEqual(len(cached_list), 3)
def list(self, request):
cached_newsfeeds = NewsFeedService.get_cached_newsfeeds(request.user.id)
page = self.paginator.paginate_cached_list(cached_newsfeeds, request)
if page is None:
queryset = NewsFeed.objects.filter(user=request.user)
page = self.paginate_queryset(queryset)
serializer = NewsFeedSerializer(
page,
context={'request': request},
many=True,
)
return self.get_paginated_response(serializer.data)
def test_get_user_newsfeeds(self):
newsfeed_ids = []
for i in range(5):
tweet = self.create_tweet(self.user2)
newsfeed = self.create_newsfeed(self.user1, tweet)
newsfeed_ids.append(newsfeed.id)
newsfeed_ids = newsfeed_ids[::-1]
#cache miss
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.user1.id)
self.assertEqual([nf.id for nf in newsfeeds], newsfeed_ids)
#cache hit
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.user1.id)
self.assertEqual([nf.id for nf in newsfeeds], newsfeed_ids)
tweet = self.create_tweet(self.user1)
new_newsfeed = self.create_newsfeed(self.user1, tweet)
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.user1)
newsfeed_ids.insert(0, new_newsfeed.id)
self.assertEqual([nf.id for nf in newsfeeds], newsfeed_ids)
def test_create_new_newsfeed_before_get_cached_newsfeeds(self):
feed1 = self.create_newsfeed(self.ray, self.create_tweet(self.ray))
RedisClient.clear()
conn = RedisClient.get_connection()
key = USER_NEWSFEEDS_PATTERN.format(user_id=self.ray.id)
self.assertEqual(conn.exists(key), False)
feed2 = self.create_newsfeed(self.ray, self.create_tweet(self.ray))
self.assertEqual(conn.exists(key), True)
feeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual([f.id for f in feeds], [feed2.id, feed1.id])
def test_get_user_newsfeeds(self):
newsfeed_ids = []
for i in range(3):
tweet = self.create_tweet(self.lux)
newsfeed = self.create_newsfeed(self.ray, tweet)
newsfeed_ids.append(newsfeed.id)
newsfeed_ids = newsfeed_ids[::-1]
# cache miss
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
# cache hit
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
# cache updated
tweet = self.create_tweet(self.ray)
new_newsfeed = self.create_newsfeed(self.ray, tweet)
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
newsfeed_ids.insert(0, new_newsfeed.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
def list(self, request):
cached_newsfeeds = NewsFeedService.get_cached_newsfeeds(request.user.id)
page = self.paginator.paginate_cached_list(cached_newsfeeds, request)
if page is None:
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
page = self.paginator.paginate_hbase(HBaseNewsFeed, (request.user.id,), request)
else:
queryset = NewsFeed.objects.filter(user=request.user)
page = self.paginate_queryset(queryset)
serializer = NewsFeedSerializer(
page,
context={'request': request},
many=True,
)
return self.get_paginated_response(serializer.data)
def test_redis_list_limit(self):
list_limit = settings.REDIS_LIST_LENGTH_LIMIT
page_size = 20
users = [self.create_user('user{}'.format(i)) for i in range(5)]
newsfeeds = []
for i in range(list_limit + page_size):
tweet = self.create_tweet(user=users[i % 5],
content='feed{}'.format(i))
feed = self.create_newsfeed(self.linghu, tweet)
newsfeeds.append(feed)
newsfeeds = newsfeeds[::-1]
# only cached list_limit objects
cached_newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_newsfeeds), list_limit)
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
count = len(HBaseNewsFeed.filter(prefix=(self.linghu.id, None)))
else:
count = NewsFeed.objects.filter(user=self.linghu).count()
self.assertEqual(count, list_limit + page_size)
results = self._paginate_to_get_newsfeeds(self.linghu_client)
self.assertEqual(len(results), list_limit + page_size)
for i in range(list_limit + page_size):
self.assertEqual(newsfeeds[i].created_at, results[i]['created_at'])
# a followed user create a new tweet
self.create_friendship(self.linghu, self.dongxie)
new_tweet = self.create_tweet(self.dongxie, 'a new tweet')
NewsFeedService.fanout_to_followers(new_tweet)
def _test_newsfeeds_after_new_feed_pushed():
results = self._paginate_to_get_newsfeeds(self.linghu_client)
self.assertEqual(len(results), list_limit + page_size + 1)
self.assertEqual(results[0]['tweet']['id'], new_tweet.id)
for i in range(list_limit + page_size):
self.assertEqual(newsfeeds[i].created_at,
results[i + 1]['created_at'])
_test_newsfeeds_after_new_feed_pushed()
# cache expired
self.clear_cache()
_test_newsfeeds_after_new_feed_pushed()
def list(self, request):
cached_newsfeeds = NewsFeedService.get_cached_newsfeeds(request.user.id)
page = self.paginator.paginate_cached_list(cached_newsfeeds, request)
# page is None means the data we get from cache is not enough to satisfy the request
# so we instead retrieve directly from db
if page is None:
if GateKeeper.is_switched_on('switch_newsfeed_to_hbase'):
page = self.paginator.paginate_hbase(HBaseNewsFeed, (request.user.id,), request)
else:
queryset = NewsFeed.objects.filter(user=request.user)
page = self.paginate_queryset(queryset)
serializer = NewsFeedSerializer(
page,
context={'request': request},
many=True,
)
return self.get_paginated_response(serializer.data)
def test_redis_list_limit(self):
list_limit = settings.REDIS_LIST_LENGTH_LIMIT
page_size = 20
users = [self.create_user('user{}'.format(i)) for i in range(5)]
newsfeeds = []
for i in range(list_limit + page_size):
tweet = self.create_tweet(user=users[i % 5],
content='feed{}'.format(i))
feed = self.create_newsfeed(self.ray, tweet)
newsfeeds.append(feed)
newsfeeds = newsfeeds[::-1]
# only cached list_limit objects
cached_newsfeeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual(len(cached_newsfeeds), list_limit)
queryset = NewsFeed.objects.filter(user=self.ray)
self.assertEqual(queryset.count(), list_limit + page_size)
results = self._paginate_to_get_newsfeeds(self.ray_client)
self.assertEqual(len(results), list_limit + page_size)
for i in range(list_limit + page_size):
self.assertEqual(newsfeeds[i].id, results[i]['id'])
# a followed user create a new tweet
self.create_friendship(self.ray, self.diana)
new_tweet = self.create_tweet(self.diana, 'a new tweet')
NewsFeedService.fanout_to_followers(new_tweet)
def _test_newsfeeds_after_new_feed_pushed():
results = self._paginate_to_get_newsfeeds(self.ray_client)
self.assertEqual(len(results), list_limit + page_size + 1)
self.assertEqual(results[0]['tweet']['id'], new_tweet.id)
for i in range(list_limit + page_size):
self.assertEqual(newsfeeds[i].id, results[i + 1]['id'])
_test_newsfeeds_after_new_feed_pushed()
# cache expired
self.clear_cache()
_test_newsfeeds_after_new_feed_pushed()
