def test_fanout_main_task(self):
tweet = self.create_tweet(self.linghu, 'tweet 1')
self.create_friendship(self.dongxie, self.linghu)
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
msg = fanout_newsfeeds_main_task(tweet.id, tweet.timestamp,
self.linghu.id)
self.assertEqual(1 + 1,
len(HBaseNewsFeed.filter(prefix=(None, None))))
else:
msg = fanout_newsfeeds_main_task(tweet.id, tweet.created_at,
self.linghu.id)
self.assertEqual(1 + 1, NewsFeed.objects.count())
self.assertEqual(msg,
'1 newsfeeds going to fanout, 1 batches created.')
cached_list = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_list), 1)
for i in range(2):
user = self.create_user('user{}'.format(i))
self.create_friendship(user, self.linghu)
tweet = self.create_tweet(self.linghu, 'tweet 2')
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
msg = fanout_newsfeeds_main_task(tweet.id, tweet.timestamp,
self.linghu.id)
self.assertEqual(4 + 2,
len(HBaseNewsFeed.filter(prefix=(None, None))))
else:
msg = fanout_newsfeeds_main_task(tweet.id, tweet.created_at,
self.linghu.id)
self.assertEqual(4 + 2, NewsFeed.objects.count())
self.assertEqual(msg,
'3 newsfeeds going to fanout, 1 batches created.')
cached_list = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_list), 2)
user = self.create_user('another user')
self.create_friendship(user, self.linghu)
tweet = self.create_tweet(self.linghu, 'tweet 3')
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
msg = fanout_newsfeeds_main_task(tweet.id, tweet.timestamp,
self.linghu.id)
self.assertEqual(8 + 3,
len(HBaseNewsFeed.filter(prefix=(None, None))))
else:
msg = fanout_newsfeeds_main_task(tweet.id, tweet.created_at,
self.linghu.id)
self.assertEqual(8 + 3, NewsFeed.objects.count())
self.assertEqual(msg,
'4 newsfeeds going to fanout, 2 batches created.')
cached_list = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_list), 3)
cached_list = NewsFeedService.get_cached_newsfeeds(self.dongxie.id)
self.assertEqual(len(cached_list), 3)
def _lazy_load(limit):
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
return HBaseNewsFeed.filter(prefix=(user_id, None),
limit=limit,
reverse=True)
return NewsFeed.objects.filter(
user_id=user_id).order_by('-created_at')[:limit]
def batch_create(cls, batch_params):
# 错误的方法
# 不可以将数据库操作放在 for 循环里面，效率会非常低
# for follower in FriendshipService.get_followers(tweet.user):
# NewsFeed.objects.create(
# user=follower,
# tweet=tweet,
# )
# 正确的方法：使用 bulk_create，会把 insert 语句合成一条
# newsfeeds = [
# NewsFeed(user_id=follower_id, tweet_id=tweet_id)
# for follower_id in follower_ids
# ]
# NewsFeed.objects.bulk_create(newsfeeds)
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
newsfeeds = HBaseNewsFeed.batch_create(batch_params)
else:
newsfeeds = [NewsFeed(**params) for params in batch_params]
NewsFeed.objects.bulk_create(newsfeeds)
# bulk create 不会触发 post_save 的 signal，所以需要手动 push 到 cache 里
for newsfeed in newsfeeds:
NewsFeedService.push_newsfeed_to_cache(newsfeed)
return newsfeeds
def count(cls, user_id=None):
# for test only
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
return len(HBaseNewsFeed.filter(prefix=(user_id, )))
if user_id is None:
return NewsFeed.objects.count()
return NewsFeed.objects.filter(user_id=user_id).count()
def create(cls, **kwargs):
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
newsfeed = HBaseNewsFeed.create(**kwargs)
# 需要手动触发 cache 更改，因为没有 listener 监听 hbase create
cls.push_newsfeed_to_cache(newsfeed)
else:
newsfeed = NewsFeed.objects.create(**kwargs)
return newsfeed
def batch_create(cls, batch_params):
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
newsfeeds = HBaseNewsFeed.batch_create(batch_params)
else:
newsfeeds = [NewsFeed(**params) for params in batch_params]
NewsFeed.objects.bulk_create(newsfeeds)
# bulk create 不会触发 post_save 的 signal，所以需要手动 push 到 cache 里
for newsfeed in newsfeeds:
NewsFeedService.push_newsfeed_to_cache(newsfeed)
return newsfeeds
def test_redis_list_limit(self):
list_limit = settings.REDIS_LIST_LENGTH_LIMIT
page_size = 20
users = [self.create_user('user{}'.format(i)) for i in range(5)]
newsfeeds = []
for i in range(list_limit + page_size):
tweet = self.create_tweet(user=users[i % 5],
content='feed{}'.format(i))
feed = self.create_newsfeed(self.linghu, tweet)
newsfeeds.append(feed)
newsfeeds = newsfeeds[::-1]
# only cached list_limit objects
cached_newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_newsfeeds), list_limit)
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
count = len(HBaseNewsFeed.filter(prefix=(self.linghu.id, None)))
else:
count = NewsFeed.objects.filter(user=self.linghu).count()
self.assertEqual(count, list_limit + page_size)
results = self._paginate_to_get_newsfeeds(self.linghu_client)
self.assertEqual(len(results), list_limit + page_size)
for i in range(list_limit + page_size):
self.assertEqual(newsfeeds[i].created_at, results[i]['created_at'])
# a followed user create a new tweet
self.create_friendship(self.linghu, self.dongxie)
new_tweet = self.create_tweet(self.dongxie, 'a new tweet')
NewsFeedService.fanout_to_followers(new_tweet)
def _test_newsfeeds_after_new_feed_pushed():
results = self._paginate_to_get_newsfeeds(self.linghu_client)
self.assertEqual(len(results), list_limit + page_size + 1)
self.assertEqual(results[0]['tweet']['id'], new_tweet.id)
for i in range(list_limit + page_size):
self.assertEqual(newsfeeds[i].created_at,
results[i + 1]['created_at'])
_test_newsfeeds_after_new_feed_pushed()
# cache expired
self.clear_cache()
_test_newsfeeds_after_new_feed_pushed()