def get_redis(self): ''' Only load the redis connection if we use it ''' if self._redis is None: self._redis = get_redis_connection() return self._redis
def __init__(self, key_data, redis=None): #write the key self.key_data = key_data self.key = self.key_format % key_data #handy when using fallback to other data sources self.source = 'redis' #the redis connection self.redis = redis or get_redis_connection()
def _fanout_task(self, user, following_group, operation, *args, **kwargs): ''' This bit of the fan-out is normally called via an Async task ''' connection = get_redis_connection() feeds = [] user_dict = User.objects.get_cached_users(following_group) with connection.map() as redis: for following_id in following_group: user = user_dict[following_id] profile = user.get_profile() feed = profile.get_feed(redis=redis) feeds.append(feed) operation(feed, *args, **kwargs) return feeds
def test_remove_love(self): from entity.models import Love thessa = User.objects.get(pk=13) profile = thessa.get_profile() follower_ids = profile.cached_follower_ids()[:100] love = Love.objects.all()[:1][0] connection = get_redis_connection() # divide the followers in groups of 10000 follower_groups = chunks(follower_ids, 10000) for follower_group in follower_groups: # now, for these 10000 items pipeline/thread away with connection.map() as redis: activity = love.create_activity() for follower_id in follower_group: feed = LoveFeed(follower_id, redis=redis) feed.remove(activity)
def test_add_love(self): from entity.models import Love thessa = User.objects.get(pk=13) profile = thessa.get_profile() follower_ids = profile.cached_follower_ids()[:100] love = Love.objects.all()[:1][0] connection = get_redis_connection() # divide the followers in groups of 10000 follower_groups = chunks(follower_ids, 10000) for follower_group in follower_groups: # now, for these 10000 items pipeline/thread away with connection.map() as redis: activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world')) for follower_id in follower_group: feed = LoveFeed(follower_id, redis=redis) feed.add(activity)
def _fanout_task(self, user, following_group, operation, max_length=None, *args, **kwargs): """ This bit of the fan-out is normally called via an Async task this shouldnt do any db queries whatsoever """ from feedly.feeds.love_feed import DatabaseFallbackLoveFeed connection = get_redis_connection() feed_class = DatabaseFallbackLoveFeed feeds = [] # set the default to 24 * 150 if max_length is None: max_length = 24 * 150 with connection.map() as redis: for following_id in following_group: feed = feed_class(following_id, max_length=max_length, redis=redis) feeds.append(feed) operation(feed, *args, **kwargs) return feeds
def _fanout(self, user, operation, *args, **kwargs): """ Generic functionality for running an operation on all of your follower's feeds It takes the following ids and distributes them per FANOUT_CHUNKS """ follower_groups = self.get_follower_groups(user) feeds = [] for (follower_group, max_length) in follower_groups: # now, for these items pipeline/thread away via an async task from feedly.tasks import fanout_love fanout_love.delay(self, user, follower_group, operation, max_length=max_length, *args, **kwargs) # reset the feeds to get out of the distributed mode connection = get_redis_connection() for feed in feeds: feed.redis = connection return feeds
def test_zremrangebyrank(self): redis = get_redis_connection() key = 'test' # start out fresh redis.delete(key) redis.zadd(key, 'a', 1) redis.zadd(key, 'b', 2) redis.zadd(key, 'c', 3) redis.zadd(key, 'd', 4) redis.zadd(key, 'e', 5) expected_results = [('a', 1.0), ('b', 2.0), ('c', 3.0), ( 'd', 4.0), ('e', 5.0)] results = redis.zrange(key, 0, -1, withscores=True) self.assertEqual(results, expected_results) results = redis.zrange(key, 0, -4, withscores=True) # now the idea is to only keep 3,4,5 max_length = 3 end = (max_length * -1) - 1 redis.zremrangebyrank(key, 0, end) expected_results = [('c', 3.0), ('d', 4.0), ('e', 5.0)] results = redis.zrange(key, 0, -1, withscores=True) self.assertEqual(results, expected_results)
def test_zremrangebyrank(self): redis = get_redis_connection() key = 'test' # start out fresh redis.delete(key) redis.zadd(key, 'a', 1) redis.zadd(key, 'b', 2) redis.zadd(key, 'c', 3) redis.zadd(key, 'd', 4) redis.zadd(key, 'e', 5) expected_results = [('a', 1.0), ('b', 2.0), ('c', 3.0), ('d', 4.0), ('e', 5.0)] results = redis.zrange(key, 0, -1, withscores=True) self.assertEqual(results, expected_results) results = redis.zrange(key, 0, -4, withscores=True) # now the idea is to only keep 3,4,5 max_length = 3 end = (max_length * -1) - 1 redis.zremrangebyrank(key, 0, end) expected_results = [('c', 3.0), ('d', 4.0), ('e', 5.0)] results = redis.zrange(key, 0, -1, withscores=True) self.assertEqual(results, expected_results)