def push_objects(cls, key, obj, queryset):
conn = RedisClient.get_connection()
if not conn.exists(key):
cls._load_object_to_cache(key, queryset)
return
serialized_data = DjangoModelSerializer.serialize(obj)
conn.lpush(key, serialized_data)
def load_objects(cls,
key,
lazy_load_objects,
serializer=DjangoModelSerializer):
# 最多只 cache REDIS_LIST_LENGTH_LIMIT 那么多个 objects
# 超过这个限制的 objects，就去数据库里读取。一般这个限制会比较大，比如 200
# 因此翻页翻到 200 的用户访问量会比较少，从数据库读取也不是大问题
# 在load和push就截断，提高效率
conn = RedisClient.get_connection()
# 如果在 cache 里存在，则直接拿出来，然后返回
# cache hit
if conn.exists(key):
serialized_list = conn.lrange(key, 0, -1)
objects = []
for serialized_data in serialized_list:
deserialized_obj = serializer.deserialize(serialized_data)
objects.append(deserialized_obj)
return objects
# 如果cache里没有，就把所有的objects写入cache
# cache miss
# 最多只 cache REDIS_LIST_LENGTH_LIMIT 那么多个 objects
# 超过这个限制的 objects，就去数据库里读取。一般这个限制会比较大，比如 1000
# 因此翻页翻到 1000 的用户访问量会比较少，从数据库读取也不是大问题
objects = lazy_load_objects(settings.REDIS_LIST_LENGTH_LIMIT)
cls._load_objects_to_cache(key, objects, serializer)
# 转换为 list 的原因是保持返回类型的统一，因为存在 redis 里的数据是 list 的形式
return list(objects)
def test_get_user_newsfeeds(self):
newsfeed_ids = []
for i in range(3):
tweet = self.create_tweet(self.dongxie)
newsfeed = self.create_newsfeed(self.linghu, tweet)
newsfeed_ids.append(newsfeed.id)
newsfeed_ids = newsfeed_ids[::-1]
RedisClient.clear()
conn = RedisClient.get_connection()
# cache miss
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
# cache hit
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
# cache updated
tweet = self.create_tweet(self.linghu)
new_newsfeed = self.create_newsfeed(self.linghu, tweet)
newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
newsfeed_ids.insert(0, new_newsfeed.id)
self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
def decr_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
if not conn.exists(key):
conn.set(key, getattr(obj, attr))
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
return getattr(obj, attr)
return conn.decr(key)
def push_object(cls, key, obj, queryset):
conn = RedisClient.get_connection()
if not conn.exists(key):
cls._load_objects_to_cache(key, queryset)
return
serialized_data = DjangoModelSerializer.serialize(obj)
conn.lpush(key, serialized_data)
conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def testRedisClient(self):
conn = RedisClient.get_connection()
# push values to Redis List
for i in range(3):
conn.lpush('testkey', f'{i}')
self.assertEqual(conn.lrange('testkey', 0, -1), [b'2', b'1', b'0'])
self.clear_cache()
self.assertEqual(conn.lrange('testkey', 0, -1), [])
def incr_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
if conn.exists(key):
return conn.incr(key)
obj.refresh_from_db()
conn.set(key, getattr(obj, attr))
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
return getattr(obj, attr)
def push_object(cls, key, obj, queryset):
conn = RedisClient.get_connection()
if not conn.exists(key):
# if key doesn't exist in cache, read from db
# don't use push single object to cache
cls._load_objects_to_cache(key, queryset)
return
serialized_data = DjangoModelSerializer.serialize(obj)
conn.lpush(key, serialized_data)
conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def get_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
count = conn.get(key)
if count is not None:
return int(count)
obj.refresh_from_db()
count = getattr(obj, attr)
conn.set(key, count)
return count
def _load_objects_to_cache(cls, key, objects):
conn = RedisClient.get_connection()
serialized_list = []
for obj in objects:
serialized_data = DjangoModelSerializer.serialize(obj)
serialized_list.append(serialized_data)
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def get(cls, gk_name):
conn = RedisClient.get_connection()
name = f'gatekeeper:{gk_name}'
if not conn.exists(name):
return {'percent': 0, 'description': ''}
redis_hash = conn.hgetall(name)
return {
'percent': int(redis_hash.get(b'percent', 0)),
'description': str(redis_hash.get(b'description', '')),
}
def test_redis_client(self):
conn = RedisClient.get_connection()
conn.lpush('redis_key', 1)
conn.lpush('redis_key', 2)
cached_list = conn.lrange('redis_key', 0, -1)
self.assertEqual(cached_list, [b'2', b'1'])
RedisClient.clear()
cached_list = conn.lrange('redis_key', 0, -1)
self.assertEqual(cached_list, [])
def decr_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
if not conn.exists(key):
# back fill cache from db
# 不执行 -1 操作， 因为必须保证调用 incr_count 之前 obj.attr 已经+1 过了
obj.refresh_from_db()
conn.set(key, getattr(obj, attr))
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
return getattr(obj, attr)
return conn.decr(key)
def test_cache_tweet_in_redis(self):
tweet = self.create_tweet(self.linghu)
conn = RedisClient.get_connection()
serialized_data = DjangoModelSerializer.serialize(tweet)
conn.set(f'tweet:{tweet.id}', serialized_data)
data = conn.get(f'tweet:not_exists')
self.assertEqual(data, None)
data = conn.get(f'tweet:{tweet.id}')
cached_tweet = DjangoModelSerializer.deserialize(data)
self.assertEqual(tweet, cached_tweet)
def test_cache_tweet_in_redis(self):
tweet = self.tweets[0]
conn = RedisClient.get_connection()
serialized_data = DjangoModelSerializer.serialize(tweet)
conn.set(f'tweet:{tweet.id}', serialized_data)
data=conn.get('tweet:bogus')
self.assertEqual(data, None)
data = conn.get(f'tweet:{tweet.id}')
cached_tweet = DjangoModelSerializer.deserialize(data)
self.assertEqual(cached_tweet, tweet)
def push_object(cls, key, obj, queryset):
queryset = queryset[:settings.REDIS_LIST_LENGTH_LIMIT]
conn = RedisClient.get_connection()
if not conn.exists(key):
# 如果key不存在，直接从数据库里面load
# 就不走单个push的方式加到cache里面了
cls._load_objects_to_cache(key, queryset)
return
serialized_data = DjangoModelSerializer.serialize(obj)
conn.lpush(key, serialized_data)
conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def test_create_new_tweet_before_get_cached_tweets(self):
tweet1 = self.create_tweet(self.user1, 'test tweet')
RedisClient.clear()
conn = RedisClient.get_connection()
key = USER_TWEET_PATTERN.format(user_id=self.user1.id)
self.assertEqual(conn.exists(key), False)
tweet2 = self.create_tweet(self.user1, 'another tweet')
self.assertEqual(conn.exists(key), True)
tweets = TweetService.get_cached_tweets(self.user1)
self.assertEqual([tweet.id for tweet in tweets], [tweet2.id, tweet1.id])
def test_redis_client(self):
conn = RedisClient.get_connection()
conn.lpush('redis_key', 1)
conn.lpush('redis_key', 2)
# from index 0 to the last element (-1)
cached_list = conn.lrange('redis_key', 0, -1)
# elements without deserialization are saved as strings so we need a prefix b (byte)
self.assertEqual(cached_list, [b'2', b'1'])
RedisClient.clear()
cached_list = conn.lrange('redis_key', 0, -1)
self.assertEqual(cached_list, [])
def test_create_tweet_before_get_cached_tweets(self):
tweet1 = self.create_tweet(user=self.user1)
RedisClient.clear()
conn = RedisClient.get_connection()
name = USER_TWEET_PATTERN.format(user_id=self.user1.id)
self.assertFalse(conn.exists(name))
tweet2 = self.create_tweet(user=self.user1)
self.assertTrue(conn.exists(name))
tweets = TweetService.load_tweets_through_cache(user_id=self.user1.id)
self.assertEqual([t.id for t in tweets], [tweet2.id, tweet1.id])
def test_create_new_newsfeed_before_get_cached_newsfeeds(self):
feed1 = self.create_newsfeed(self.ray, self.create_tweet(self.ray))
RedisClient.clear()
conn = RedisClient.get_connection()
key = USER_NEWSFEEDS_PATTERN.format(user_id=self.ray.id)
self.assertEqual(conn.exists(key), False)
feed2 = self.create_newsfeed(self.ray, self.create_tweet(self.ray))
self.assertEqual(conn.exists(key), True)
feeds = NewsFeedService.get_cached_newsfeeds(self.ray.id)
self.assertEqual([f.id for f in feeds], [feed2.id, feed1.id])
def incr_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
if conn.exists(key):
return conn.incr(key)
# back fill cache from db
# no +1 operation because we need to make sure
# obj.attr has been increased before incr_count is called
obj.refresh_from_db()
conn.set(key, getattr(obj, attr))
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
return getattr(obj, attr)
def load_objects(cls, key, queryset):
conn = RedisClient.get_connection()
if conn.exists(key):
serialized_list = conn.lrange(key, 0, -1)
objects = []
for data in serialized_list:
obj = DjangoModelSerializer.deserialize(data)
objects.append(obj)
return objects
cls._load_object_to_cache(key, queryset)
return list(queryset)
def _load_objects_to_cache(cls, key, objects, serializer):
conn = RedisClient.get_connection()
serialized_list = []
# cache REDIS_LIST_LENGTH_LIMIT
# if more than this number, go to DB to fetch
for obj in objects:
serialized_data = serializer.serialize(obj)
serialized_list.append(serialized_data)
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def _load_objects_to_cache(cls, key, objects, serializer):
conn = RedisClient.get_connection()
serialized_list = []
for obj in objects:
serialized_data = serializer.serialize(obj)
serialized_list.append(serialized_data)
# 注意 N+1 queries问题：不要写在上面循环里一个一个rpush，每次rpush都是一次redis访问
# 而是先构建好list，再rpush整个list
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def test_create_newsfeed_before_get_cached_newsfeeds(self):
feed1 = self.create_newsfeed(user=self.user2,
tweet=self.create_tweet(user=self.user1))
self.clear_cache()
conn = RedisClient.get_connection()
name = USER_NEWSFEED_PATTERN.format(user_id=self.user2.id)
self.assertFalse(conn.exists(name))
feed2 = self.create_newsfeed(user=self.user2,
tweet=self.create_tweet(user=self.user3))
self.assertTrue(conn.exists(name))
feeds = NewsFeedService.load_newsfeeds_through_cache(self.user2.id)
self.assertEqual([f.id for f in feeds], [feed2.id, feed1.id])
def _load_objects_to_cache(cls, key, objects):
conn = RedisClient.get_connection()
serialized_list = []
# 最多只 cache REDIS_LIST_LENGTH_LIMIT 那么多个 objects
# 超过这个限制的 objects，就去数据库里读取。一般这个限制会比较大，比如 1000
# 因此翻页翻到 1000 的用户访问量会比较少，从数据库读取也不是大问题
for obj in objects[:settings.REDIS_LIST_LENGTH_LIMIT]:
serialized_data = DjangoModelSerializer.serialize(obj)
serialized_list.append(serialized_data)
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def _load_objects_to_cache(cls, key, objects):
conn = RedisClient.get_connection()
serialized_list = []
# allowing to cache at most REDIS_LIST_LENGTH_LIMIT number of objects
# when exceeding the limit, fetch data in DB instead
# since the limit if often large, it's a edge case for user to read that many number of items directly
for obj in objects[:settings.REDIS_LIST_LENGTH_LIMIT]:
serialized_data = DjangoModelSerializer.serialize(obj)
serialized_list.append(serialized_data)
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def incr_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
if conn.exists(key):
return conn.incr(key)
# back fill cache from db
# don't execute +1 operation, due to before call incr_count method,
# obj.attr already +1 in db.
if not conn.exists(key):
obj.refresh_from_db()
conn.set(key, getattr(obj, attr))
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
return getattr(obj, attr)
def load_objects(cls, key, queryset):
conn = RedisClient.get_connection()
# 如果在 cache 里存在，则直接拿出来，然后返回
if conn.exists(key):
serialized_list = conn.lrange(key, 0, -1)
objects = []
for serialized_data in serialized_list:
deserialized_obj = DjangoModelSerializer.deserialize(serialized_data)
objects.append(deserialized_obj)
return objects
# 转换为 list 的原因是保持返回类型的统一，因为存在 redis 里的数据是 list 的形式
cls._load_objects_to_cache(key, queryset)
return list(queryset)
def _load_objects_to_cache(cls, key, objects):
conn = RedisClient.get_connection()
serialized_list = []
for obj in objects[:settings.REDIS_LIST_LENGTH_LIMIT]:
# it can only read REDIS_LIST_LENGTH_LIMIT number of objects
# if the number of objects is over the limitation, read from db
# usually, the limitation number is big, such as 1000
# due to user normally not page down more than 1000 objects,
# it is rarely need to read data from db
serialized_data = DjangoModelSerializer.serialize(obj)
serialized_list.append(serialized_data)
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
