def push_objects(cls, key, obj, queryset): conn = RedisClient.get_connection() if not conn.exists(key): cls._load_object_to_cache(key, queryset) return serialized_data = DjangoModelSerializer.serialize(obj) conn.lpush(key, serialized_data)
def load_objects(cls, key, lazy_load_objects, serializer=DjangoModelSerializer): # 最多只 cache REDIS_LIST_LENGTH_LIMIT 那么多个 objects # 超过这个限制的 objects,就去数据库里读取。一般这个限制会比较大,比如 200 # 因此翻页翻到 200 的用户访问量会比较少,从数据库读取也不是大问题 # 在load和push就截断,提高效率 conn = RedisClient.get_connection() # 如果在 cache 里存在,则直接拿出来,然后返回 # cache hit if conn.exists(key): serialized_list = conn.lrange(key, 0, -1) objects = [] for serialized_data in serialized_list: deserialized_obj = serializer.deserialize(serialized_data) objects.append(deserialized_obj) return objects # 如果cache里没有,就把所有的objects写入cache # cache miss # 最多只 cache REDIS_LIST_LENGTH_LIMIT 那么多个 objects # 超过这个限制的 objects,就去数据库里读取。一般这个限制会比较大,比如 1000 # 因此翻页翻到 1000 的用户访问量会比较少,从数据库读取也不是大问题 objects = lazy_load_objects(settings.REDIS_LIST_LENGTH_LIMIT) cls._load_objects_to_cache(key, objects, serializer) # 转换为 list 的原因是保持返回类型的统一,因为存在 redis 里的数据是 list 的形式 return list(objects)
def test_get_user_newsfeeds(self): newsfeed_ids = [] for i in range(3): tweet = self.create_tweet(self.dongxie) newsfeed = self.create_newsfeed(self.linghu, tweet) newsfeed_ids.append(newsfeed.id) newsfeed_ids = newsfeed_ids[::-1] RedisClient.clear() conn = RedisClient.get_connection() # cache miss newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id) self.assertEqual([f.id for f in newsfeeds], newsfeed_ids) # cache hit newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id) self.assertEqual([f.id for f in newsfeeds], newsfeed_ids) # cache updated tweet = self.create_tweet(self.linghu) new_newsfeed = self.create_newsfeed(self.linghu, tweet) newsfeeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id) newsfeed_ids.insert(0, new_newsfeed.id) self.assertEqual([f.id for f in newsfeeds], newsfeed_ids)
def decr_count(cls, obj, attr): conn = RedisClient.get_connection() key = cls.get_count_key(obj, attr) if not conn.exists(key): conn.set(key, getattr(obj, attr)) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME) return getattr(obj, attr) return conn.decr(key)
def push_object(cls, key, obj, queryset): conn = RedisClient.get_connection() if not conn.exists(key): cls._load_objects_to_cache(key, queryset) return serialized_data = DjangoModelSerializer.serialize(obj) conn.lpush(key, serialized_data) conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def testRedisClient(self): conn = RedisClient.get_connection() # push values to Redis List for i in range(3): conn.lpush('testkey', f'{i}') self.assertEqual(conn.lrange('testkey', 0, -1), [b'2', b'1', b'0']) self.clear_cache() self.assertEqual(conn.lrange('testkey', 0, -1), [])
def incr_count(cls, obj, attr): conn = RedisClient.get_connection() key = cls.get_count_key(obj, attr) if conn.exists(key): return conn.incr(key) obj.refresh_from_db() conn.set(key, getattr(obj, attr)) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME) return getattr(obj, attr)
def push_object(cls, key, obj, queryset): conn = RedisClient.get_connection() if not conn.exists(key): # if key doesn't exist in cache, read from db # don't use push single object to cache cls._load_objects_to_cache(key, queryset) return serialized_data = DjangoModelSerializer.serialize(obj) conn.lpush(key, serialized_data) conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def get_count(cls, obj, attr): conn = RedisClient.get_connection() key = cls.get_count_key(obj, attr) count = conn.get(key) if count is not None: return int(count) obj.refresh_from_db() count = getattr(obj, attr) conn.set(key, count) return count
def _load_objects_to_cache(cls, key, objects): conn = RedisClient.get_connection() serialized_list = [] for obj in objects: serialized_data = DjangoModelSerializer.serialize(obj) serialized_list.append(serialized_data) if serialized_list: conn.rpush(key, *serialized_list) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def get(cls, gk_name): conn = RedisClient.get_connection() name = f'gatekeeper:{gk_name}' if not conn.exists(name): return {'percent': 0, 'description': ''} redis_hash = conn.hgetall(name) return { 'percent': int(redis_hash.get(b'percent', 0)), 'description': str(redis_hash.get(b'description', '')), }
def test_redis_client(self): conn = RedisClient.get_connection() conn.lpush('redis_key', 1) conn.lpush('redis_key', 2) cached_list = conn.lrange('redis_key', 0, -1) self.assertEqual(cached_list, [b'2', b'1']) RedisClient.clear() cached_list = conn.lrange('redis_key', 0, -1) self.assertEqual(cached_list, [])
def decr_count(cls, obj, attr): conn = RedisClient.get_connection() key = cls.get_count_key(obj, attr) if not conn.exists(key): # back fill cache from db # 不执行 -1 操作, 因为必须保证调用 incr_count 之前 obj.attr 已经+1 过了 obj.refresh_from_db() conn.set(key, getattr(obj, attr)) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME) return getattr(obj, attr) return conn.decr(key)
def test_cache_tweet_in_redis(self): tweet = self.create_tweet(self.linghu) conn = RedisClient.get_connection() serialized_data = DjangoModelSerializer.serialize(tweet) conn.set(f'tweet:{tweet.id}', serialized_data) data = conn.get(f'tweet:not_exists') self.assertEqual(data, None) data = conn.get(f'tweet:{tweet.id}') cached_tweet = DjangoModelSerializer.deserialize(data) self.assertEqual(tweet, cached_tweet)
def test_cache_tweet_in_redis(self): tweet = self.tweets[0] conn = RedisClient.get_connection() serialized_data = DjangoModelSerializer.serialize(tweet) conn.set(f'tweet:{tweet.id}', serialized_data) data=conn.get('tweet:bogus') self.assertEqual(data, None) data = conn.get(f'tweet:{tweet.id}') cached_tweet = DjangoModelSerializer.deserialize(data) self.assertEqual(cached_tweet, tweet)
def push_object(cls, key, obj, queryset): queryset = queryset[:settings.REDIS_LIST_LENGTH_LIMIT] conn = RedisClient.get_connection() if not conn.exists(key): # 如果key不存在,直接从数据库里面load # 就不走单个push的方式加到cache里面了 cls._load_objects_to_cache(key, queryset) return serialized_data = DjangoModelSerializer.serialize(obj) conn.lpush(key, serialized_data) conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def test_create_new_tweet_before_get_cached_tweets(self): tweet1 = self.create_tweet(self.user1, 'test tweet') RedisClient.clear() conn = RedisClient.get_connection() key = USER_TWEET_PATTERN.format(user_id=self.user1.id) self.assertEqual(conn.exists(key), False) tweet2 = self.create_tweet(self.user1, 'another tweet') self.assertEqual(conn.exists(key), True) tweets = TweetService.get_cached_tweets(self.user1) self.assertEqual([tweet.id for tweet in tweets], [tweet2.id, tweet1.id])
def test_redis_client(self): conn = RedisClient.get_connection() conn.lpush('redis_key', 1) conn.lpush('redis_key', 2) # from index 0 to the last element (-1) cached_list = conn.lrange('redis_key', 0, -1) # elements without deserialization are saved as strings so we need a prefix b (byte) self.assertEqual(cached_list, [b'2', b'1']) RedisClient.clear() cached_list = conn.lrange('redis_key', 0, -1) self.assertEqual(cached_list, [])
def test_create_tweet_before_get_cached_tweets(self): tweet1 = self.create_tweet(user=self.user1) RedisClient.clear() conn = RedisClient.get_connection() name = USER_TWEET_PATTERN.format(user_id=self.user1.id) self.assertFalse(conn.exists(name)) tweet2 = self.create_tweet(user=self.user1) self.assertTrue(conn.exists(name)) tweets = TweetService.load_tweets_through_cache(user_id=self.user1.id) self.assertEqual([t.id for t in tweets], [tweet2.id, tweet1.id])
def test_create_new_newsfeed_before_get_cached_newsfeeds(self): feed1 = self.create_newsfeed(self.ray, self.create_tweet(self.ray)) RedisClient.clear() conn = RedisClient.get_connection() key = USER_NEWSFEEDS_PATTERN.format(user_id=self.ray.id) self.assertEqual(conn.exists(key), False) feed2 = self.create_newsfeed(self.ray, self.create_tweet(self.ray)) self.assertEqual(conn.exists(key), True) feeds = NewsFeedService.get_cached_newsfeeds(self.ray.id) self.assertEqual([f.id for f in feeds], [feed2.id, feed1.id])
def incr_count(cls, obj, attr): conn = RedisClient.get_connection() key = cls.get_count_key(obj, attr) if conn.exists(key): return conn.incr(key) # back fill cache from db # no +1 operation because we need to make sure # obj.attr has been increased before incr_count is called obj.refresh_from_db() conn.set(key, getattr(obj, attr)) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME) return getattr(obj, attr)
def load_objects(cls, key, queryset): conn = RedisClient.get_connection() if conn.exists(key): serialized_list = conn.lrange(key, 0, -1) objects = [] for data in serialized_list: obj = DjangoModelSerializer.deserialize(data) objects.append(obj) return objects cls._load_object_to_cache(key, queryset) return list(queryset)
def _load_objects_to_cache(cls, key, objects, serializer): conn = RedisClient.get_connection() serialized_list = [] # cache REDIS_LIST_LENGTH_LIMIT # if more than this number, go to DB to fetch for obj in objects: serialized_data = serializer.serialize(obj) serialized_list.append(serialized_data) if serialized_list: conn.rpush(key, *serialized_list) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def _load_objects_to_cache(cls, key, objects, serializer): conn = RedisClient.get_connection() serialized_list = [] for obj in objects: serialized_data = serializer.serialize(obj) serialized_list.append(serialized_data) # 注意 N+1 queries问题:不要写在上面循环里一个一个rpush,每次rpush都是一次redis访问 # 而是先构建好list,再rpush整个list if serialized_list: conn.rpush(key, *serialized_list) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def test_create_newsfeed_before_get_cached_newsfeeds(self): feed1 = self.create_newsfeed(user=self.user2, tweet=self.create_tweet(user=self.user1)) self.clear_cache() conn = RedisClient.get_connection() name = USER_NEWSFEED_PATTERN.format(user_id=self.user2.id) self.assertFalse(conn.exists(name)) feed2 = self.create_newsfeed(user=self.user2, tweet=self.create_tweet(user=self.user3)) self.assertTrue(conn.exists(name)) feeds = NewsFeedService.load_newsfeeds_through_cache(self.user2.id) self.assertEqual([f.id for f in feeds], [feed2.id, feed1.id])
def _load_objects_to_cache(cls, key, objects): conn = RedisClient.get_connection() serialized_list = [] # 最多只 cache REDIS_LIST_LENGTH_LIMIT 那么多个 objects # 超过这个限制的 objects,就去数据库里读取。一般这个限制会比较大,比如 1000 # 因此翻页翻到 1000 的用户访问量会比较少,从数据库读取也不是大问题 for obj in objects[:settings.REDIS_LIST_LENGTH_LIMIT]: serialized_data = DjangoModelSerializer.serialize(obj) serialized_list.append(serialized_data) if serialized_list: conn.rpush(key, *serialized_list) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def _load_objects_to_cache(cls, key, objects): conn = RedisClient.get_connection() serialized_list = [] # allowing to cache at most REDIS_LIST_LENGTH_LIMIT number of objects # when exceeding the limit, fetch data in DB instead # since the limit if often large, it's a edge case for user to read that many number of items directly for obj in objects[:settings.REDIS_LIST_LENGTH_LIMIT]: serialized_data = DjangoModelSerializer.serialize(obj) serialized_list.append(serialized_data) if serialized_list: conn.rpush(key, *serialized_list) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def incr_count(cls, obj, attr): conn = RedisClient.get_connection() key = cls.get_count_key(obj, attr) if conn.exists(key): return conn.incr(key) # back fill cache from db # don't execute +1 operation, due to before call incr_count method, # obj.attr already +1 in db. if not conn.exists(key): obj.refresh_from_db() conn.set(key, getattr(obj, attr)) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME) return getattr(obj, attr)
def load_objects(cls, key, queryset): conn = RedisClient.get_connection() # 如果在 cache 里存在,则直接拿出来,然后返回 if conn.exists(key): serialized_list = conn.lrange(key, 0, -1) objects = [] for serialized_data in serialized_list: deserialized_obj = DjangoModelSerializer.deserialize(serialized_data) objects.append(deserialized_obj) return objects # 转换为 list 的原因是保持返回类型的统一,因为存在 redis 里的数据是 list 的形式 cls._load_objects_to_cache(key, queryset) return list(queryset)
def _load_objects_to_cache(cls, key, objects): conn = RedisClient.get_connection() serialized_list = [] for obj in objects[:settings.REDIS_LIST_LENGTH_LIMIT]: # it can only read REDIS_LIST_LENGTH_LIMIT number of objects # if the number of objects is over the limitation, read from db # usually, the limitation number is big, such as 1000 # due to user normally not page down more than 1000 objects, # it is rarely need to read data from db serialized_data = DjangoModelSerializer.serialize(obj) serialized_list.append(serialized_data) if serialized_list: conn.rpush(key, *serialized_list) conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)