def test_get_user_newsfeeds(self):
newsfeed_timestamps = []
for i in range(3):
tweet = self.create_tweet(self.dongxie)
newsfeed = self.create_newsfeed(self.linghu, tweet)
newsfeed_timestamps.append(newsfeed.created_at)
newsfeed_timestamps = newsfeed_timestamps[::-1]
# cache miss
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.created_at for f in newsfeeds],
newsfeed_timestamps)
# cache hit
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.created_at for f in newsfeeds],
newsfeed_timestamps)
# cache updated
tweet = self.create_tweet(self.linghu)
new_newsfeed = self.create_newsfeed(self.linghu, tweet)
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
newsfeed_timestamps.insert(0, new_newsfeed.created_at)
self.assertEqual([f.created_at for f in newsfeeds],
newsfeed_timestamps)
def push_newsfeed_to_cache(sender, instance, created, **kwargs):
# 只有新建时才push， update时并不push
if not created:
return
from newsfeeds.services import NewsFeedService
NewsFeedService.push_newsfeed_to_cache(instance)
def test_get_user_newsfeeds(self):
newsfeed_ids = []
for i in range(3):
tweet = self.create_tweet(self.dongxie)
newsfeed = self.create_newsfeed(self.linghu, tweet)
newsfeed_ids.append(newsfeed.id)
newsfeed_ids = newsfeed_ids[::-1]
RedisClient.clear()
conn = RedisClient.get_connection()
# cache miss
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
# cache hit
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
# cache updated
tweet = self.create_tweet(self.linghu)
new_newsfeed = self.create_newsfeed(self.linghu, tweet)
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
newsfeed_ids.insert(0, new_newsfeed.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
def follow(self, request, pk):
#check if follow user exists
self.get_object()
if Friendship.objects.filter(from_user=request.user,to_user=pk):
return Response({
"success": True,
"duplicate": True,
}, HTTP_201_CREATED)
if not User.objects.filter(id=pk).exists():
logging.error(pk)
logging.error(User.objects.filter(id=pk).exists())
return Response({
"success": False,
}, HTTP_400_BAD_REQUEST)
serializer = FriendshipSerializerForCreate(data={
'from_user_id': request.user.id,
'to_user_id': pk,
})
if not serializer.is_valid():
return Response({
"success": False,
"message": "Invalid inputs.",
"error": serializer.errors,
}, HTTP_400_BAD_REQUEST)
friendship = serializer.save()
#populate newsfeed timeline for new friendship
NewsFeedService.populate_newsfeed_for_friendship(friendship)
return Response(
FriendshipSerializerForCreate(friendship).data,
status=HTTP_201_CREATED
)
def test_fanout_main_task(self):
tweet = self.create_tweet(self.ray, 'tweet 1')
self.create_friendship(self.lux, self.ray)
msg = fanout_newsfeeds_main_task(tweet.id, self.ray.id)
self.assertEqual(
msg, '1 newsfeeds are going to fanout, 1 batches created.')
self.assertEqual(1 + 1, NewsFeed.objects.count())
cached_list = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual(len(cached_list), 1)
for i in range(2):
user = self.create_user('user{}'.format(i))
self.create_friendship(user, self.ray)
tweet = self.create_tweet(self.ray, 'tweet 2')
msg = fanout_newsfeeds_main_task(tweet.id, self.ray.id)
self.assertEqual(
msg, '3 newsfeeds are going to fanout, 1 batches created.')
self.assertEqual(4 + 2, NewsFeed.objects.count())
cached_list = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual(len(cached_list), 2)
user = self.create_user('another user')
self.create_friendship(user, self.ray)
tweet = self.create_tweet(self.ray, 'tweet 3')
msg = fanout_newsfeeds_main_task(tweet.id, self.ray.id)
self.assertEqual(
msg, '4 newsfeeds are going to fanout, 2 batches created.')
self.assertEqual(8 + 3, NewsFeed.objects.count())
cached_list = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual(len(cached_list), 3)
cached_list = NewsFeedService.get_cached_newsfeeds(self.lux.id)
self.assertEqual(len(cached_list), 3)
def test_fanout_main_task(self):
tweet = self.create_tweet(self.linghu, 'tweet 1')
self.create_friendship(self.dongxie, self.linghu)
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
msg = fanout_newsfeeds_main_task(tweet.id, tweet.timestamp,
self.linghu.id)
self.assertEqual(1 + 1,
len(HBaseNewsFeed.filter(prefix=(None, None))))
else:
msg = fanout_newsfeeds_main_task(tweet.id, tweet.created_at,
self.linghu.id)
self.assertEqual(1 + 1, NewsFeed.objects.count())
self.assertEqual(msg,
'1 newsfeeds going to fanout, 1 batches created.')
cached_list = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_list), 1)
for i in range(2):
user = self.create_user('user{}'.format(i))
self.create_friendship(user, self.linghu)
tweet = self.create_tweet(self.linghu, 'tweet 2')
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
msg = fanout_newsfeeds_main_task(tweet.id, tweet.timestamp,
self.linghu.id)
self.assertEqual(4 + 2,
len(HBaseNewsFeed.filter(prefix=(None, None))))
else:
msg = fanout_newsfeeds_main_task(tweet.id, tweet.created_at,
self.linghu.id)
self.assertEqual(4 + 2, NewsFeed.objects.count())
self.assertEqual(msg,
'3 newsfeeds going to fanout, 1 batches created.')
cached_list = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_list), 2)
user = self.create_user('another user')
self.create_friendship(user, self.linghu)
tweet = self.create_tweet(self.linghu, 'tweet 3')
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
msg = fanout_newsfeeds_main_task(tweet.id, tweet.timestamp,
self.linghu.id)
self.assertEqual(8 + 3,
len(HBaseNewsFeed.filter(prefix=(None, None))))
else:
msg = fanout_newsfeeds_main_task(tweet.id, tweet.created_at,
self.linghu.id)
self.assertEqual(8 + 3, NewsFeed.objects.count())
self.assertEqual(msg,
'4 newsfeeds going to fanout, 2 batches created.')
cached_list = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_list), 3)
cached_list = NewsFeedService.get_cached_newsfeeds(self.dongxie.id)
self.assertEqual(len(cached_list), 3)
def fanout_newsfeeds_batch_task(tweet_id, follower_ids):
from newsfeeds.services import NewsFeedService
newsfeeds = [
NewsFeed(user_id=follower_id, tweet_id=tweet_id)
for follower_id in follower_ids
]
NewsFeed.objects.bulk_create(newsfeeds)
for newsfeed in newsfeeds:
NewsFeedService.push_newsfeed_to_cache(newsfeed)
return "{} newsfeeds created".format(len(newsfeeds))
def test_fanout_main_task(self):
tweet1 = self.create_tweet(user=self.user1)
self.create_friendship(from_user=self.user2, to_user=self.user1)
msg = fanout_to_followers_main_task(tweet_id=tweet1.id,
tweet_user_id=self.user1.id)
self.assertEqual(msg,
"1 newsfeeds are going to be fanout, " \
"1 batches created, " \
"batch size 3"
)
self.assertEqual(1 + 1, NewsFeed.objects.count())
cached_list = NewsFeedService.load_newsfeeds_through_cache(
user_id=self.user2.id)
self.assertEqual(1, len(cached_list))
# multiple users fanout
for i in range(10):
user = self.create_user(username='******'.format(i))
self.create_friendship(from_user=user, to_user=self.user1)
tweet2 = self.create_tweet(user=self.user1)
msg = fanout_to_followers_main_task(
tweet_id=tweet2.id,
tweet_user_id=self.user1.id,
)
self.assertEqual(msg,
"11 newsfeeds are going to be fanout, " \
"4 batches created, " \
"batch size 3"
)
self.assertEqual(2 + 12, NewsFeed.objects.count())
cached_list = NewsFeedService.load_newsfeeds_through_cache(
user_id=self.user2.id)
self.assertEqual(2, len(cached_list))
# cache invalidation
self.clear_cache()
user = self.create_user(username='******')
self.create_friendship(from_user=user, to_user=self.user1)
tweet3 = self.create_tweet(user=self.user1)
msg = fanout_to_followers_main_task(
tweet_id=tweet3.id,
tweet_user_id=self.user1.id,
)
self.assertEqual(
msg,
"12 newsfeeds are going to be fanout, " \
"4 batches created, " \
"batch size 3"
)
self.assertEqual(3 + 24, NewsFeed.objects.count())
cached_list = NewsFeedService.load_newsfeeds_through_cache(
user_id=self.user2.id)
self.assertEqual(3, len(cached_list))
def fanout_newsfeeds_batch_task(tweet_id, follower_ids):
from newsfeeds.services import NewsFeedService
newsfeeds = [
NewsFeed(user_id=follower_id, tweet_id=tweet_id)
for follower_id in follower_ids
# notice here the tweet.user is the creator of tweet
]
NewsFeed.objects.bulk_create(newsfeeds)
# bulk create will not invoke post_save signal, manually push newsfeed into cache
for newsfeed in newsfeeds:
NewsFeedService.push_newsfeed_to_cache(newsfeed)
return "{} newsfeeds created".format(len(newsfeeds))
def create(self, request):
serializer = TweetCreateSerializer(data=request.data,
context={'request': request})
if not serializer.is_valid():
return Response(
{
'success': False,
'message': 'Please check input',
'errors': serializer.errors,
},
status=400)
tweet = serializer.save()
NewsFeedService.fanout_to_followers(tweet)
return Response(TweetSerializer(tweet).data, status=201)
def create(self, request):
serializer = TweetSerializerForCreate(
data=request.data,
context={'request': request},
)
if not serializer.is_valid():
return Response({
"success": False,
"message": "Please check input.",
"errors": serializer.errors,
}, status=400)
# save will call create method in TweetSerializerForCreate
tweet = serializer.save()
NewsFeedService.fanout_to_followers(tweet)
return Response(TweetSerializer(tweet).data, status=201)
def create_newsfeed(self, user, tweet):
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
created_at = tweet.timestamp
else:
created_at = tweet.created_at
return NewsFeedService.create(user_id=user.id,
tweet_id=tweet.id,
created_at=created_at)
def test_redis_list_limit(self):
list_limit = settings.REDIS_LIST_LENGTH_LIMIT
page_size = 20
users = [self.create_user('user{}'.format(i)) for i in range(5)]
newsfeeds = []
for i in range(list_limit + page_size):
tweet = self.create_tweet(user=users[i % 5],
content='feed{}'.format(i))
feed = self.create_newsfeed(self.linghu, tweet)
newsfeeds.append(feed)
newsfeeds = newsfeeds[::-1]
# only cached list_limit objects
cached_newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual(len(cached_newsfeeds), list_limit)
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
count = len(HBaseNewsFeed.filter(prefix=(self.linghu.id, None)))
else:
count = NewsFeed.objects.filter(user=self.linghu).count()
self.assertEqual(count, list_limit + page_size)
results = self._paginate_to_get_newsfeeds(self.linghu_client)
self.assertEqual(len(results), list_limit + page_size)
for i in range(list_limit + page_size):
self.assertEqual(newsfeeds[i].created_at, results[i]['created_at'])
# a followed user create a new tweet
self.create_friendship(self.linghu, self.dongxie)
new_tweet = self.create_tweet(self.dongxie, 'a new tweet')
NewsFeedService.fanout_to_followers(new_tweet)
def _test_newsfeeds_after_new_feed_pushed():
results = self._paginate_to_get_newsfeeds(self.linghu_client)
self.assertEqual(len(results), list_limit + page_size + 1)
self.assertEqual(results[0]['tweet']['id'], new_tweet.id)
for i in range(list_limit + page_size):
self.assertEqual(newsfeeds[i].created_at,
results[i + 1]['created_at'])
_test_newsfeeds_after_new_feed_pushed()
# cache expired
self.clear_cache()
_test_newsfeeds_after_new_feed_pushed()
def unfollow(self, request, pk):
unfollow_user = self.get_object()
if request.user.id == unfollow_user.id:
return Response({
"success": False,
"message": "You cannot unfollow yourself."
}, status=HTTP_400_BAD_REQUEST)
friendship = Friendship.objects.filter(from_user=request.user,to_user_id=pk)
#update timeline for the deleted following
if friendship:
NewsFeedService.update_newsfeed_for_unfollow(friendship[0])
deleted, _ = friendship.delete()
return Response({
"success": True,
"deleted": deleted
}, status=HTTP_200_OK)
def fanout_newsfeeds_batch_task(tweet_id, created_at, follower_ids):
# import 写在里面避免循环依赖
from newsfeeds.services import NewsFeedService
batch_params = [
{'user_id': follower_id, 'created_at': created_at, 'tweet_id': tweet_id}
for follower_id in follower_ids
]
newsfeeds = NewsFeedService.batch_create(batch_params)
return "{} newsfeeds created".format(len(newsfeeds))
def create(self, request, *args, **kwargs):
serializer = TweetSerializerForCreate(
data=request.data,
context={'request': request},
)
if not serializer.is_valid():
return Response(
{
'success': False,
'message': "please check input",
'errors': serializer.errors,
},
status=400)
tweet = serializer.save()
NewsFeedService.fanout_to_followers(tweet)
serializer = TweetSerializer(tweet, context={'request': request})
return Response(serializer.data, status=201)
def fanout_newsfeeds_batch_task(tweet_id, follower_ids):
# import 写在里面避免循环依赖
from newsfeeds.services import NewsFeedService
# 错误的方法：将数据库操作放在 for 循环里面，效率会非常低 ->
# for follower_id in follower_ids:
# NewsFeed.objects.create(user_id=follower_id, tweet_id=tweet_id)
# 正确的方法：使用 bulk_create，会把 insert 语句合成一条 ->
newsfeeds = [
NewsFeed(user_id=follower_id, tweet_id=tweet_id)
for follower_id in follower_ids
]
NewsFeed.objects.bulk_create(newsfeeds)
# bulk create 不会触发 post_save 的 signal，所以需要手动 push 到 cache 里
for newsfeed in newsfeeds:
NewsFeedService.push_newsfeed_to_cache(newsfeed)
return "{} newsfeeds created".format(len(newsfeeds))
def create(self, request):
serialzier = TweetSerializerForCreate(
data=request.data,
context={'request': request},
)
if not serialzier.is_valid():
return Response(
{
"success": False,
"message": "Please check input.",
"errors": serialzier.errors,
},
status=HTTP_400_BAD_REQUEST)
tweet = serialzier.save()
#fan out to followers
NewsFeedService.fan_out_to_followers(tweet)
return Response(TweetSerializer(tweet).data, status=HTTP_201_CREATED)
def create(self, request):
"""
重载 create 方法，因为需要默认用当前登录用户作为 tweet.user
"""
serializer = TweetSerializerForCreate(
data=request.data,
context={'request': request},
)
if not serializer.is_valid():
return Response({
'success': False,
'message': "Please check input",
'errors': serializer.errors,
}, status=400)
# save() will call create method in TweetCreateSerializer
tweet = serializer.save()
NewsFeedService.fanout_to_followers(tweet)
serializer = TweetSerializer(tweet, context={'request': request})
return Response(serializer.data, status=201)
def create_newsfeed(self, user, tweet):
# 默认ORM created_at: 2021-11-11 11:11:11.123456 iso format
# HBaseModel: timestamp format: 16位int
# 兼容 iso 格式和 int 格式的时间戳
if GateKeeper.is_switch_on('switch_newsfeed_to_hbase'):
created_at = tweet.timestamp
else:
created_at = tweet.created_at
return NewsFeedService.create(user_id=user.id,
tweet_id=tweet.id,
created_at=created_at)
def fanout_newsfeeds_batch_task(tweet_id, created_at, follower_ids):
# we put import inside to avoid circular dependency
from newsfeeds.services import NewsFeedService
batch_params = [
{'user_id': follower_id, 'created_at': created_at, 'tweet_id': tweet_id}
for follower_id in follower_ids
]
newsfeeds = NewsFeedService.batch_create(batch_params)
return "{} newsfeeds created".format(len(newsfeeds))
def create(self, request, *args, **kwargs):
"""
重载 create 方法，因为需要默认用当前登录用户作为 tweet.user
"""
serializer = TweetCreateSerializer(
data=request.data,
context={'request': request},
)
# use default validator, like min max length
if not serializer.is_valid():
return Response(
{
'success': False,
'message': "Please check input",
'errors': serializer.errors,
},
status=400)
tweet = serializer.save()
NewsFeedService.fanout_to_followers(tweet)
return Response(TweetSerializer(tweet).data, status=201)
def list(self, request):
cached_newsfeeds = NewsFeedService.get_cached_newsfeeds(request.user.id)
page = self.paginator.paginate_cached_list(cached_newsfeeds, request)
if page is None:
queryset = NewsFeed.objects.filter(user=request.user)
page = self.paginate_queryset(queryset)
serializer = NewsFeedSerializer(
page,
context={'request': request},
many=True,
)
return self.get_paginated_response(serializer.data)
def test_get_user_newsfeeds(self):
newsfeed_ids = []
for i in range(5):
tweet = self.create_tweet(self.user2)
newsfeed = self.create_newsfeed(self.user1, tweet)
newsfeed_ids.append(newsfeed.id)
newsfeed_ids = newsfeed_ids[::-1]
#cache miss
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.user1.id)
self.assertEqual([nf.id for nf in newsfeeds], newsfeed_ids)
#cache hit
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.user1.id)
self.assertEqual([nf.id for nf in newsfeeds], newsfeed_ids)
tweet = self.create_tweet(self.user1)
new_newsfeed = self.create_newsfeed(self.user1, tweet)
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.user1)
newsfeed_ids.insert(0, new_newsfeed.id)
self.assertEqual([nf.id for nf in newsfeeds], newsfeed_ids)
def create(self, request, *args, **kwargs):
"""
overload create function, to default the login user as a tweet.user
"""
serializer = TweetSerializerForCreate(
data=request.data,
context={'request': request},
)
if not serializer.is_valid():
return Response(
{
'success': False,
'message': 'Please check input',
'errors': serializer.errors,
},
status=400)
tweet = serializer.save()
NewsFeedService.fanout_to_followers(tweet)
serializer = TweetSerializer(tweet, context={'request': request})
return Response(serializer.data, status=201)
def create(self, request):
serializer = TweetSerializerForCreate(
data=request.data,
context={'request': request},
)
if not serializer.is_valid():
return Response({
"success": False,
"message": "Please check input",
"errors": serializer.errors,
}, status=400)
# otherwise we have a tweet instance here
# save() will call create method in TweetSerializerForCreate
tweet = serializer.save()
NewsFeedService.fanout_to_followers(tweet)
# Here I use TweetSerializer to show whats in the tweet
# reminder: It is different from TweetSerializerForCreate
return Response(
TweetSerializer(tweet, context={'request': request}).data,
status=201,
)
def test_redis_list_limit(self):
list_limit = settings.REDIS_LIST_LENGTH_LIMIT
page_size = 20
users = [self.create_user('user{}'.format(i)) for i in range(5)]
newsfeeds = []
for i in range(list_limit + page_size):
tweet = self.create_tweet(user=users[i % 5],
content='feed{}'.format(i))
feed = self.create_newsfeed(self.ray, tweet)
newsfeeds.append(feed)
newsfeeds = newsfeeds[::-1]
# only cached list_limit objects
cached_newsfeeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual(len(cached_newsfeeds), list_limit)
queryset = NewsFeed.objects.filter(user=self.ray)
self.assertEqual(queryset.count(), list_limit + page_size)
results = self._paginate_to_get_newsfeeds(self.ray_client)
self.assertEqual(len(results), list_limit + page_size)
for i in range(list_limit + page_size):
self.assertEqual(newsfeeds[i].id, results[i]['id'])
# a followed user create a new tweet
self.create_friendship(self.ray, self.diana)
new_tweet = self.create_tweet(self.diana, 'a new tweet')
NewsFeedService.fanout_to_followers(new_tweet)
def _test_newsfeeds_after_new_feed_pushed():
results = self._paginate_to_get_newsfeeds(self.ray_client)
self.assertEqual(len(results), list_limit + page_size + 1)
self.assertEqual(results[0]['tweet']['id'], new_tweet.id)
for i in range(list_limit + page_size):
self.assertEqual(newsfeeds[i].id, results[i + 1]['id'])
_test_newsfeeds_after_new_feed_pushed()
# cache expired
self.clear_cache()
_test_newsfeeds_after_new_feed_pushed()
def test_get_user_newsfeeds(self):
newsfeed_ids = []
for i in range(3):
tweet = self.create_tweet(self.lux)
newsfeed = self.create_newsfeed(self.ray, tweet)
newsfeed_ids.append(newsfeed.id)
newsfeed_ids = newsfeed_ids[::-1]
# cache miss
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
# cache hit
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
# cache updated
tweet = self.create_tweet(self.ray)
new_newsfeed = self.create_newsfeed(self.ray, tweet)
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
newsfeed_ids.insert(0, new_newsfeed.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
def fanout_newsfeeds_batch_task(tweet_id, follower_ids):
# import written inside of method to avoid reference loop.
from newsfeeds.services import NewsFeedService
# wrong way:
# can not put db operation in for loop. efficiency is super low and slow
# for follower in FriendshipService.get_followers(tweet.user):
# NewsFeed.objects.create(
# user=follower,
# tweet=tweet,
# )
# correct way: use bulk_create, then insert once
newsfeeds = [
NewsFeed(user_id=follower_id, tweet_id=tweet_id)
for follower_id in follower_ids
]
NewsFeed.objects.bulk_create(newsfeeds)
# bulk create doesn't emit post_save signal, so, need to push each newsfeed to cache manually
for newsfeed in newsfeeds:
NewsFeedService.push_newsfeed_to_cache(newsfeed)
return '{} newsfeeds created'.format(len(newsfeeds))
def test_create_new_newsfeed_before_get_cached_newsfeeds(self):
feed1 = self.create_newsfeed(self.ray, self.create_tweet(self.ray))
RedisClient.clear()
conn = RedisClient.get_connection()
key = USER_NEWSFEEDS_PATTERN.format(user_id=self.ray.id)
self.assertEqual(conn.exists(key), False)
feed2 = self.create_newsfeed(self.ray, self.create_tweet(self.ray))
self.assertEqual(conn.exists(key), True)
feeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual([f.id for f in feeds], [feed2.id, feed1.id])
