def get_redis(self):
'''
Only load the redis connection if we use it
'''
if self._redis is None:
self._redis = get_redis_connection()
return self._redis
def __init__(self, key_data, redis=None):
#write the key
self.key_data = key_data
self.key = self.key_format % key_data
#handy when using fallback to other data sources
self.source = 'redis'
#the redis connection
self.redis = redis or get_redis_connection()
def _fanout_task(self, user, following_group, operation, *args, **kwargs):
'''
This bit of the fan-out is normally called via an Async task
'''
connection = get_redis_connection()
feeds = []
user_dict = User.objects.get_cached_users(following_group)
with connection.map() as redis:
for following_id in following_group:
user = user_dict[following_id]
profile = user.get_profile()
feed = profile.get_feed(redis=redis)
feeds.append(feed)
operation(feed, *args, **kwargs)
return feeds
def test_remove_love(self):
from entity.models import Love
thessa = User.objects.get(pk=13)
profile = thessa.get_profile()
follower_ids = profile.cached_follower_ids()[:100]
love = Love.objects.all()[:1][0]
connection = get_redis_connection()
# divide the followers in groups of 10000
follower_groups = chunks(follower_ids, 10000)
for follower_group in follower_groups:
# now, for these 10000 items pipeline/thread away
with connection.map() as redis:
activity = love.create_activity()
for follower_id in follower_group:
feed = LoveFeed(follower_id, redis=redis)
feed.remove(activity)
def test_add_love(self):
from entity.models import Love
thessa = User.objects.get(pk=13)
profile = thessa.get_profile()
follower_ids = profile.cached_follower_ids()[:100]
love = Love.objects.all()[:1][0]
connection = get_redis_connection()
# divide the followers in groups of 10000
follower_groups = chunks(follower_ids, 10000)
for follower_group in follower_groups:
# now, for these 10000 items pipeline/thread away
with connection.map() as redis:
activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
for follower_id in follower_group:
feed = LoveFeed(follower_id, redis=redis)
feed.add(activity)
def test_remove_love(self):
from entity.models import Love
thessa = User.objects.get(pk=13)
profile = thessa.get_profile()
follower_ids = profile.cached_follower_ids()[:100]
love = Love.objects.all()[:1][0]
connection = get_redis_connection()
# divide the followers in groups of 10000
follower_groups = chunks(follower_ids, 10000)
for follower_group in follower_groups:
# now, for these 10000 items pipeline/thread away
with connection.map() as redis:
activity = love.create_activity()
for follower_id in follower_group:
feed = LoveFeed(follower_id, redis=redis)
feed.remove(activity)
def _fanout_task(self, user, following_group, operation, max_length=None, *args, **kwargs):
"""
This bit of the fan-out is normally called via an Async task
this shouldnt do any db queries whatsoever
"""
from feedly.feeds.love_feed import DatabaseFallbackLoveFeed
connection = get_redis_connection()
feed_class = DatabaseFallbackLoveFeed
feeds = []
# set the default to 24 * 150
if max_length is None:
max_length = 24 * 150
with connection.map() as redis:
for following_id in following_group:
feed = feed_class(following_id, max_length=max_length, redis=redis)
feeds.append(feed)
operation(feed, *args, **kwargs)
return feeds
def _fanout(self, user, operation, *args, **kwargs):
"""
Generic functionality for running an operation on all of your
follower's feeds
It takes the following ids and distributes them per FANOUT_CHUNKS
"""
follower_groups = self.get_follower_groups(user)
feeds = []
for (follower_group, max_length) in follower_groups:
# now, for these items pipeline/thread away via an async task
from feedly.tasks import fanout_love
fanout_love.delay(self, user, follower_group, operation, max_length=max_length, *args, **kwargs)
# reset the feeds to get out of the distributed mode
connection = get_redis_connection()
for feed in feeds:
feed.redis = connection
return feeds
def test_add_love(self):
from entity.models import Love
thessa = User.objects.get(pk=13)
profile = thessa.get_profile()
follower_ids = profile.cached_follower_ids()[:100]
love = Love.objects.all()[:1][0]
connection = get_redis_connection()
# divide the followers in groups of 10000
follower_groups = chunks(follower_ids, 10000)
for follower_group in follower_groups:
# now, for these 10000 items pipeline/thread away
with connection.map() as redis:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
for follower_id in follower_group:
feed = LoveFeed(follower_id, redis=redis)
feed.add(activity)
def test_zremrangebyrank(self):
redis = get_redis_connection()
key = 'test'
# start out fresh
redis.delete(key)
redis.zadd(key, 'a', 1)
redis.zadd(key, 'b', 2)
redis.zadd(key, 'c', 3)
redis.zadd(key, 'd', 4)
redis.zadd(key, 'e', 5)
expected_results = [('a', 1.0), ('b', 2.0), ('c', 3.0), (
'd', 4.0), ('e', 5.0)]
results = redis.zrange(key, 0, -1, withscores=True)
self.assertEqual(results, expected_results)
results = redis.zrange(key, 0, -4, withscores=True)
# now the idea is to only keep 3,4,5
max_length = 3
end = (max_length * -1) - 1
redis.zremrangebyrank(key, 0, end)
expected_results = [('c', 3.0), ('d', 4.0), ('e', 5.0)]
results = redis.zrange(key, 0, -1, withscores=True)
self.assertEqual(results, expected_results)
def test_zremrangebyrank(self):
redis = get_redis_connection()
key = 'test'
# start out fresh
redis.delete(key)
redis.zadd(key, 'a', 1)
redis.zadd(key, 'b', 2)
redis.zadd(key, 'c', 3)
redis.zadd(key, 'd', 4)
redis.zadd(key, 'e', 5)
expected_results = [('a', 1.0), ('b', 2.0), ('c', 3.0), ('d', 4.0),
('e', 5.0)]
results = redis.zrange(key, 0, -1, withscores=True)
self.assertEqual(results, expected_results)
results = redis.zrange(key, 0, -4, withscores=True)
# now the idea is to only keep 3,4,5
max_length = 3
end = (max_length * -1) - 1
redis.zremrangebyrank(key, 0, end)
expected_results = [('c', 3.0), ('d', 4.0), ('e', 5.0)]
results = redis.zrange(key, 0, -1, withscores=True)
self.assertEqual(results, expected_results)
