def incr_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
if conn.exists(key):
return conn.incr(key)
obj.refresh_from_db()
conn.set(key, getattr(obj, attr))
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
return getattr(obj, attr)
def testRedisClient(self):
conn = RedisClient.get_connection()
# push values to Redis List
for i in range(3):
conn.lpush('testkey', f'{i}')
self.assertEqual(conn.lrange('testkey', 0, -1), [b'2', b'1', b'0'])
self.clear_cache()
self.assertEqual(conn.lrange('testkey', 0, -1), [])
def push_object(cls, key, obj, queryset):
queryset = queryset[:settings.REDIS_LIST_LENGTH_LIMIT]
conn = RedisClient.get_connection()
if not conn.exists(key):
# if key is not in cache, load from db
cls._load_objects_to_cache(key, queryset)
return
serialized_data = DjangoModelSerializer.serialize(obj)
conn.lpush(key, serialized_data)
conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def push_object(cls, key, obj, queryset):
conn = RedisClient.get_connection()
if not conn.exists(key):
# 如果 key 不存在，直接从数据库里 load
# 就不走单个 push 的方式加到 cache 里了
cls._load_objects_to_cache(key, queryset)
return
serialized_data = DjangoModelSerializer.serialize(obj)
conn.lpush(key, serialized_data)
conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def _load_objects_to_cache(cls, key, objects):
conn = RedisClient.get_connection()
serialized_list = []
for obj in objects:
serialized_data = DjangoModelSerializer.serialize(obj)
serialized_list.append(serialized_data)
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def push_object(cls, key, obj, queryset):
conn = RedisClient.get_connection()
if not conn.exists(key):
# if key doesn't exist in cache, read from db
# don't use push single object to cache
cls._load_objects_to_cache(key, queryset)
return
serialized_data = DjangoModelSerializer.serialize(obj)
conn.lpush(key, serialized_data)
conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def get(cls, gk_name):
conn = RedisClient.get_connection()
name = f'gatekeeper:{gk_name}'
if not conn.exists(name):
return {'percent': 0, 'description': ''}
redis_hash = conn.hgetall(name)
return {
'percent': int(redis_hash.get(b'percent', 0)),
'description': str(redis_hash.get(b'description', '')),
}
def get_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
count = conn.get(key)
if count is not None:
return int(count)
obj.refresh_from_db()
count = getattr(obj, attr)
conn.set(key, count)
return count
def push_object(cls, key, obj, queryset):
conn = RedisClient.get_connection()
# cache miss
if not conn.exists(key):
cls._load_objects_to_cache(key, queryset)
return
# cache hit
serialized_data = DjangoModelSerializer.serialize(obj)
conn.lpush(key, serialized_data)
conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def setUp(self):
self.clear_cache()
RedisClient.clear()
self.user1_client = APIClient()
self.user1 = self.create_user(username='******')
self.user1_client.force_authenticate(user=self.user1)
self.user1_tweets = [
self.create_tweet(self.user1, i)
for i in range(3) # 3 tweets for user_1
]
self.user2_client = APIClient()
self.user2 = self.create_user(username='******')
self.user2_client.force_authenticate(user=self.user2)
self.user2_tweets = [
self.create_tweet(self.user2, i)
for i in range(2) # 2 tweets for user_2
]
self.user3, self.user3_client = self.create_user_and_client(
username='******')
def decr_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
if not conn.exists(key):
# back fill cache from db
# 不执行 -1 操作， 因为必须保证调用 incr_count 之前 obj.attr 已经+1 过了
obj.refresh_from_db()
conn.set(key, getattr(obj, attr))
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
return getattr(obj, attr)
return conn.decr(key)
def test_cache_tweet_in_redis(self):
tweet = self.tweets[0]
conn = RedisClient.get_connection()
serialized_data = DjangoModelSerializer.serialize(tweet)
conn.set(f'tweet:{tweet.id}', serialized_data)
data=conn.get('tweet:bogus')
self.assertEqual(data, None)
data = conn.get(f'tweet:{tweet.id}')
cached_tweet = DjangoModelSerializer.deserialize(data)
self.assertEqual(cached_tweet, tweet)
def test_cache_tweet_in_redis(self):
tweet = self.create_tweet(self.linghu)
conn = RedisClient.get_connection()
serialized_data = DjangoModelSerializer.serialize(tweet)
conn.set(f'tweet:{tweet.id}', serialized_data)
data = conn.get(f'tweet:not_exists')
self.assertEqual(data, None)
data = conn.get(f'tweet:{tweet.id}')
cached_tweet = DjangoModelSerializer.deserialize(data)
self.assertEqual(tweet, cached_tweet)
def push_object(cls, key, obj, queryset):
queryset = queryset[:settings.REDIS_LIST_LENGTH_LIMIT]
conn = RedisClient.get_connection()
if not conn.exists(key):
# 如果 key 不存在，直接从数据库里 load
# 就不走单个 push 的方式加到 cache 里了
cls._load_objects_to_cache(key, queryset)
return
serialized_data = DjangoModelSerializer.serialize(obj)
conn.lpush(key, serialized_data)
# to ensure after push the new object, we stay within the length
conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
def incr_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
if conn.exists(key):
return conn.incr(key)
# back fill cache from db
# no +1 operation because we need to make sure
# obj.attr has been increased before incr_count is called
obj.refresh_from_db()
conn.set(key, getattr(obj, attr))
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
return getattr(obj, attr)
def test_create_new_newsfeed_before_get_cached_newsfeeds(self):
feed1 = self.create_newsfeed(self.linghu, self.create_tweet(self.linghu))
self.clear_cache()
conn = RedisClient.get_connection()
key = USER_NEWSFEEDS_PATTERN.format(user_id=self.linghu.id)
self.assertEqual(conn.exists(key), False)
feed2 = self.create_newsfeed(self.linghu, self.create_tweet(self.linghu))
self.assertEqual(conn.exists(key), True)
feeds = NewsFeedService.get_cached_newsfeeds(self.linghu.id)
self.assertEqual([f.id for f in feeds], [feed2.id, feed1.id])
def load_objects(cls, key, queryset):
conn = RedisClient.get_connection()
if conn.exists(key):
serialized_list = conn.lrange(key, 0, -1)
objects = []
for data in serialized_list:
obj = DjangoModelSerializer.deserialize(data)
objects.append(obj)
return objects
cls._load_object_to_cache(key, queryset)
return list(queryset)
def _load_objects_to_cache(cls, key, objects, serializer):
conn = RedisClient.get_connection()
serialized_list = []
# cache REDIS_LIST_LENGTH_LIMIT
# if more than this number, go to DB to fetch
for obj in objects:
serialized_data = serializer.serialize(obj)
serialized_list.append(serialized_data)
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def incr_count(cls, obj, attr):
conn = RedisClient.get_connection()
key = cls.get_count_key(obj, attr)
if conn.exists(key):
return conn.incr(key)
# back fill cache from db
# don't execute +1 operation, due to before call incr_count method,
# obj.attr already +1 in db.
if not conn.exists(key):
obj.refresh_from_db()
conn.set(key, getattr(obj, attr))
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
return getattr(obj, attr)
def _load_objects_to_cache(cls, key, objects, serializer):
conn = RedisClient.get_connection()
serialized_list = []
for obj in objects:
serialized_data = serializer.serialize(obj)
serialized_list.append(serialized_data)
# 注意 N+1 queries问题：不要写在上面循环里一个一个rpush，每次rpush都是一次redis访问
# 而是先构建好list，再rpush整个list
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def test_create_newsfeed_before_get_cached_newsfeeds(self):
feed1 = self.create_newsfeed(user=self.user2,
tweet=self.create_tweet(user=self.user1))
self.clear_cache()
conn = RedisClient.get_connection()
name = USER_NEWSFEED_PATTERN.format(user_id=self.user2.id)
self.assertFalse(conn.exists(name))
feed2 = self.create_newsfeed(user=self.user2,
tweet=self.create_tweet(user=self.user3))
self.assertTrue(conn.exists(name))
feeds = NewsFeedService.load_newsfeeds_through_cache(self.user2.id)
self.assertEqual([f.id for f in feeds], [feed2.id, feed1.id])
def _load_objects_to_cache(cls, key, objects):
conn = RedisClient.get_connection()
serialized_list = []
# 最多只 cache REDIS_LIST_LENGTH_LIMIT 那么多个 objects
# 超过这个限制的 objects，就去数据库里读取。一般这个限制会比较大，比如 1000
# 因此翻页翻到 1000 的用户访问量会比较少，从数据库读取也不是大问题
for obj in objects[:settings.REDIS_LIST_LENGTH_LIMIT]:
serialized_data = DjangoModelSerializer.serialize(obj)
serialized_list.append(serialized_data)
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def _load_objects_to_cache(cls, key, objects):
conn = RedisClient.get_connection()
serialized_list = []
# allowing to cache at most REDIS_LIST_LENGTH_LIMIT number of objects
# when exceeding the limit, fetch data in DB instead
# since the limit if often large, it's a edge case for user to read that many number of items directly
for obj in objects[:settings.REDIS_LIST_LENGTH_LIMIT]:
serialized_data = DjangoModelSerializer.serialize(obj)
serialized_list.append(serialized_data)
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def test_get_user_tweets(self):
tweet_ids = []
for i in range(3):
tweet = self.create_tweet(self.linghu, 'tweet {}'.format(i))
tweet_ids.append(tweet.id)
tweet_ids = tweet_ids[::-1]
RedisClient.clear()
conn = RedisClient.get_connection()
# cache miss
tweets = TweetService.get_cached_tweets(self.linghu.id)
self.assertEqual([t.id for t in tweets], tweet_ids)
# cache hit
tweets = TweetService.get_cached_tweets(self.linghu.id)
self.assertEqual([t.id for t in tweets], tweet_ids)
# cache updated
new_tweet = self.create_tweet(self.linghu, 'new tweet')
tweets = TweetService.get_cached_tweets(self.linghu.id)
tweet_ids.insert(0, new_tweet.id)
self.assertEqual([t.id for t in tweets], tweet_ids)
def load_objects(cls, key, queryset):
conn = RedisClient.get_connection()
# 如果在 cache 里存在，则直接拿出来，然后返回
if conn.exists(key):
serialized_list = conn.lrange(key, 0, -1)
objects = []
for serialized_data in serialized_list:
deserialized_obj = DjangoModelSerializer.deserialize(serialized_data)
objects.append(deserialized_obj)
return objects
# 转换为 list 的原因是保持返回类型的统一，因为存在 redis 里的数据是 list 的形式
cls._load_objects_to_cache(key, queryset)
return list(queryset)
def _load_objects_to_cache(cls, key, objects):
conn = RedisClient.get_connection()
serialized_list = []
for obj in objects[:settings.REDIS_LIST_LENGTH_LIMIT]:
# it can only read REDIS_LIST_LENGTH_LIMIT number of objects
# if the number of objects is over the limitation, read from db
# usually, the limitation number is big, such as 1000
# due to user normally not page down more than 1000 objects,
# it is rarely need to read data from db
serialized_data = DjangoModelSerializer.serialize(obj)
serialized_list.append(serialized_data)
if serialized_list:
conn.rpush(key, *serialized_list)
conn.expire(key, settings.REDIS_KEY_EXPIRE_TIME)
def main():
psql = models.Model.set_parameters(**s.POSTGRES)
redis = RedisClient(**s.REDIS)
initializer = dict(relay=object(),
push=object(),
session=object(),
auth=AuthManager(redis))
application = tornado.web.Application([
(r'^/session/(?P<device_id>\d+)/(?P<number>[0-9\+]{3,15})$',
MainHandler, initializer)
])
server = tornado.httpserver.HTTPServer(application)
server.listen(s.SERVER_PORT, s.SERVER_HOST)
server.start()
Log.d(TAG, "OpenRedPhoneServer is run")
def load_objects(cls, key, queryset):
conn = RedisClient.get_connection()
# if data is in cache, get from cache and return it
if conn.exists(key):
serialized_list = conn.lrange(key, 0, -1)
objects = []
for serialized_data in serialized_list:
deserialized_obj = DjangoModelSerializer.deserialize(
serialized_data)
objects.append(deserialized_obj)
return objects
cls._load_objects_to_cache(key, queryset)
# the value is the redis is list type. so convert the queryset to list
return list(queryset)
def push_object(cls, key, obj, lazy_load_objects):
conn = RedisClient.get_connection()
if isinstance(obj, HBaseModel):
serializer = HBaseModelSerializer
else:
serializer = DjangoModelSerializer
# 如果在 cache 里存在，直接把 obj 放在 list 的最前面，然后 trim 一下长度
if conn.exists(key):
serialized_data = serializer.serialize(obj)
conn.lpush(key, serialized_data)
conn.ltrim(key, 0, settings.REDIS_LIST_LENGTH_LIMIT - 1)
return
# 如果 key 不存在，直接从数据库里 load, 回填到cache
# 就不走单个 push 的方式加到 cache 里了
objects = lazy_load_objects(settings.REDIS_LIST_LENGTH_LIMIT)
cls._load_objects_to_cache(key, objects, serializer)
def load_objects(cls, key, queryset):
queryset = queryset[:settings.REDIS_LIST_LENGTH_LIMIT]
conn = RedisClient.get_connection()
# if key is in cache, get and return it
if conn.exists(key):
serialized_list = conn.lrange(key, 0, -1)
objects = []
for serialized_data in serialized_list:
deserialized_obj = DjangoModelSerializer.deserialize(
serialized_data)
objects.append(deserialized_obj)
return objects
cls._load_objects_to_cache(key, queryset)
# data is stored in list in redis, so keep data type same
return list(queryset)
