def index(request, template='feedly/index.html'):
context = RequestContext(request)
#HACK for django <1.3 beta compatibility
if 'STATIC_URL' not in context and 'MEDIA_URL' in context:
context['STATIC_URL'] = context['MEDIA_URL']
sample_size = int(request.GET.get('sample_size', 1000))
context['sample_size'] = sample_size
lucky_users = random.sample(xrange(10**6), sample_size) + [13]
users_dict = User.objects.get_cached_users(lucky_users)
buckets = [
0, 24, 1 * 24, 3 * 24, 10 * 24, 30 * 24, 50 * 24, 100 * 24, 150 * 24,
1000 * 24
]
bucket_dict = dict([(b, 0) for b in buckets])
count_dict = {}
#retrieve all the counts in one pipelined request(s)
with get_redis_connection().map() as redis:
for user_id in users_dict:
feed = LoveFeed(user_id, redis=redis)
count = feed.count()
count_dict[user_id] = count
#divide into buckets using bisect left
for user_id, count in count_dict.items():
bucket_index = bisect.bisect_left(buckets, count)
bucket = buckets[bucket_index]
bucket_dict[bucket] += 1
bucket_stats = bucket_dict.items()
bucket_stats.sort(key=lambda x: x[0])
context['bucket_stats'] = bucket_stats
return render_to_response(template, context)
def monitor(request, template='feedly/monitor.html'):
context = RequestContext(request)
#HACK for django <1.3 beta compatibility
if 'STATIC_URL' not in context and 'MEDIA_URL' in context:
context['STATIC_URL'] = context['MEDIA_URL']
sample_size = int(request.GET.get('sample_size', 2))
lucky_users = random.sample(xrange(10 ** 6), sample_size) + [13]
users_dict = get_user_model().objects.get_cached_users(lucky_users)
#retrieve all the counts in one pipelined request(s)
count_dict = {}
with get_redis_connection().map() as redis:
for user_id in users_dict:
feed = LoveFeed(user_id, redis=redis)
count = feed.count()
count_dict[user_id] = count
for user_id, count in count_dict.items():
profile = users_dict[user_id].get_profile()
redis_count = int(count_dict[user_id])
db_max = redis_count + 10
db_count = profile._following_loves().count()
print profile, db_count, long(redis_count)
return render_to_response(template, context)
def test_small_feed_instance(self):
loves = Love.objects.all()[:5]
feed = LoveFeed(13, max_length=2)
for love in loves:
activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
feed.add(activity)
self.assertEqual(feed.count(), feed.max_length)
def index(request, template='feedly/index.html'):
context = RequestContext(request)
#HACK for django <1.3 beta compatibility
if 'STATIC_URL' not in context and 'MEDIA_URL' in context:
context['STATIC_URL'] = context['MEDIA_URL']
sample_size = int(request.GET.get('sample_size', 1000))
context['sample_size'] = sample_size
lucky_users = random.sample(xrange(10 ** 6), sample_size) + [13]
users_dict = get_user_model().objects.get_cached_users(lucky_users)
buckets = [0, 24, 1 * 24, 3 * 24, 10 * 24, 30 * 24, 50 * 24, 100 *
24, 150 * 24, 1000 * 24]
bucket_dict = dict([(b, 0) for b in buckets])
count_dict = {}
#retrieve all the counts in one pipelined request(s)
with get_redis_connection().map() as redis:
for user_id in users_dict:
feed = LoveFeed(user_id, redis=redis)
count = feed.count()
count_dict[user_id] = count
#divide into buckets using bisect left
for user_id, count in count_dict.items():
bucket_index = bisect.bisect_left(buckets, count)
bucket = buckets[bucket_index]
bucket_dict[bucket] += 1
bucket_stats = bucket_dict.items()
bucket_stats.sort(key=lambda x: x[0])
context['bucket_stats'] = bucket_stats
return render_to_response(template, context)
def monitor(request, template='feedly/monitor.html'):
context = RequestContext(request)
#HACK for django <1.3 beta compatibility
if 'STATIC_URL' not in context and 'MEDIA_URL' in context:
context['STATIC_URL'] = context['MEDIA_URL']
sample_size = int(request.GET.get('sample_size', 2))
lucky_users = random.sample(xrange(10**6), sample_size) + [13]
users_dict = User.objects.get_cached_users(lucky_users)
#retrieve all the counts in one pipelined request(s)
count_dict = {}
with get_redis_connection().map() as redis:
for user_id in users_dict:
feed = LoveFeed(user_id, redis=redis)
count = feed.count()
count_dict[user_id] = count
for user_id, count in count_dict.items():
profile = users_dict[user_id].get_profile()
redis_count = int(count_dict[user_id])
db_max = redis_count + 10
db_count = profile._following_loves().count()
print profile, db_count, long(redis_count)
return render_to_response(template, context)
def test_follow(self):
from user.models import Follow
follow = Follow.objects.all()[:1][0]
feed = LoveFeed(follow.user_id)
target_loves = follow.target.get_profile().loves()[:500]
for love in target_loves:
activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
feed.add(activity)
feed_loves = feed[:20]
def test_small_feed_instance(self):
loves = Love.objects.all()[:5]
feed = LoveFeed(13, max_length=2)
for love in loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
feed.add(activity)
self.assertEqual(feed.count(), feed.max_length)
def test_removed_love(self):
'''
Replicates the following scenario
- The user loves an item
- Its pushed on a feed
- The item is set to inactive, removing the love from the database
- The redis cache is cleared
- Love Item cache reads will return None
- The feed should return one result less
'''
# start with adding some data
loves = Love.objects.all()[:10]
feed = LoveFeed(13)
# slow version
activities = []
feed.delete()
for love in loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
activities.append(activity)
feed.add(activity)
assert feed.contains(activity)
# close the feed
feed.finish()
feed_loves = feed[:20]
#assert isinstance(feed_loves[-1], FeedEndMarker)
#assert len(feed_loves) == 11
# now for the scenario that the item is not there
removed_love = feed_loves[2]
removed_id = removed_love.serialization_id
# Fake that the data is None
old_get_many = feed.item_cache.get_many
def wrap_get_many(fields):
result = old_get_many(fields)
if removed_id in result:
result[removed_id] = None
return result
feed.item_cache.get_many = wrap_get_many
# verify we return None
self.assertEqual(feed.item_cache.get(removed_id), None)
empty_result = {removed_id: None}
self.assertEqual(feed.item_cache.get_many([removed_id]), empty_result)
feed_loves = feed[:20]
self.assertEqual(feed.source, 'redis')
found_activity_ids = [a.serialization_id for a in feed_loves]
assert removed_id not in found_activity_ids
self.assertEqual(len(feed_loves), 10)
def test_follow(self):
from user.models import Follow
follow = Follow.objects.all()[:1][0]
feed = LoveFeed(follow.user_id)
target_loves = follow.target.get_profile().loves()[:500]
for love in target_loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
feed.add(activity)
feed_loves = feed[:20]
def add_love(self, love):
'''
Fanout to all your followers
This is really write intensive
Reads are super light though
'''
activity = self.create_love_activity(love)
# only write the love to the cache once
from feedly.feeds.love_feed import LoveFeed
LoveFeed.set_item_cache(activity)
feeds = self._fanout(love.user, add_operation, activity=activity)
return feeds
def test_remove_love(self):
from entity.models import Love
thessa = User.objects.get(pk=13)
profile = thessa.get_profile()
follower_ids = profile.cached_follower_ids()[:100]
love = Love.objects.all()[:1][0]
connection = get_redis_connection()
# divide the followers in groups of 10000
follower_groups = chunks(follower_ids, 10000)
for follower_group in follower_groups:
# now, for these 10000 items pipeline/thread away
with connection.map() as redis:
activity = love.create_activity()
for follower_id in follower_group:
feed = LoveFeed(follower_id, redis=redis)
feed.remove(activity)
def test_add_love(self):
from entity.models import Love
thessa = User.objects.get(pk=13)
profile = thessa.get_profile()
follower_ids = profile.cached_follower_ids()[:100]
love = Love.objects.all()[:1][0]
connection = get_redis_connection()
# divide the followers in groups of 10000
follower_groups = chunks(follower_ids, 10000)
for follower_group in follower_groups:
# now, for these 10000 items pipeline/thread away
with connection.map() as redis:
activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
for follower_id in follower_group:
feed = LoveFeed(follower_id, redis=redis)
feed.add(activity)
def test_remove_love(self):
from entity.models import Love
thessa = User.objects.get(pk=13)
profile = thessa.get_profile()
follower_ids = profile.cached_follower_ids()[:100]
love = Love.objects.all()[:1][0]
connection = get_redis_connection()
# divide the followers in groups of 10000
follower_groups = chunks(follower_ids, 10000)
for follower_group in follower_groups:
# now, for these 10000 items pipeline/thread away
with connection.map() as redis:
activity = love.create_activity()
for follower_id in follower_group:
feed = LoveFeed(follower_id, redis=redis)
feed.remove(activity)
def test_follow(self):
follow = Follow.objects.filter(user=self.bogus_user)[:1][0]
# reset the feed
feed = LoveFeed(follow.user_id)
feed.delete()
# do a follow
feedly = LoveFeedly()
feed = feedly.follow(follow)
# see if we got the new loves
target_loves = follow.target.get_profile().loves()[:10]
for love in target_loves:
activity = feedly.create_love_activity(love)
assert feed.contains(activity)
# check if we correctly broadcasted
feedly.unfollow(follow)
feed_count = feed.count()
feed_results = feed[:20]
self.assertEqual(feed_results, [])
def test_removed_love(self):
'''
Replicates the following scenario
- The user loves an item
- Its pushed on a feed
- The item is set to inactive, removing the love from the database
- The redis cache is cleared
- Love Item cache reads will return None
- The feed should return one result less
'''
# start with adding some data
loves = Love.objects.all()[:10]
feed = LoveFeed(13)
# slow version
activities = []
feed.delete()
for love in loves:
activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
activities.append(activity)
feed.add(activity)
assert feed.contains(activity)
# close the feed
feed.finish()
feed_loves = feed[:20]
#assert isinstance(feed_loves[-1], FeedEndMarker)
#assert len(feed_loves) == 11
# now for the scenario that the item is not there
removed_love = feed_loves[2]
removed_id = removed_love.serialization_id
# Fake that the data is None
old_get_many = feed.item_cache.get_many
def wrap_get_many(fields):
result = old_get_many(fields)
if removed_id in result:
result[removed_id] = None
return result
feed.item_cache.get_many = wrap_get_many
# verify we return None
self.assertEqual(feed.item_cache.get(removed_id), None)
empty_result = {removed_id: None}
self.assertEqual(feed.item_cache.get_many([removed_id]), empty_result)
feed_loves = feed[:20]
self.assertEqual(feed.source, 'redis')
found_activity_ids = [a.serialization_id for a in feed_loves]
assert removed_id not in found_activity_ids
self.assertEqual(len(feed_loves), 10)
def test_add_love(self):
from entity.models import Love
thessa = User.objects.get(pk=13)
profile = thessa.get_profile()
follower_ids = profile.cached_follower_ids()[:100]
love = Love.objects.all()[:1][0]
connection = get_redis_connection()
# divide the followers in groups of 10000
follower_groups = chunks(follower_ids, 10000)
for follower_group in follower_groups:
# now, for these 10000 items pipeline/thread away
with connection.map() as redis:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
for follower_id in follower_group:
feed = LoveFeed(follower_id, redis=redis)
feed.add(activity)
def test_follow(self):
follow = Follow.objects.filter(user=self.bogus_user)[:1][0]
# reset the feed
feed = LoveFeed(follow.user_id)
feed.delete()
# do a follow
feedly = LoveFeedly()
feed = feedly.follow(follow)
# see if we got the new loves
target_loves = follow.target.get_profile().loves()[:10]
for love in target_loves:
activity = feedly.create_love_activity(love)
assert feed.contains(activity)
# check if we correctly broadcasted
feedly.unfollow(follow)
feed_count = feed.count()
feed_results = feed[:20]
self.assertEqual(feed_results, [])
def test_simple_add_love(self):
loves = Love.objects.all()[:10]
feed = LoveFeed(13)
# slow version
activities = []
feed.delete()
for love in loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
activities.append(activity)
feed.add(activity)
assert feed.contains(activity)
# close the feed
feed.finish()
feed_loves = feed[:20]
assert isinstance(feed_loves[-1], FeedEndMarker)
assert len(feed_loves) == 11
for activity in feed_loves:
assert activity
# now the fast version
feed.delete()
feed.add_many(activities)
for activity in activities:
assert feed.contains(activity)
def test_simple_remove_love(self):
from entity.models import Love
target_loves = Love.objects.all()[:10]
feed = LoveFeed(13)
feed.delete()
# slow implementation
activities = []
for love in target_loves:
# remove the items by key (id)
activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
activities.append(activity)
feed.remove(activity)
feed.add_many(activities)
for activity in activities:
assert feed.contains(activity)
feed.remove_many(activities)
assert feed.count() == 0
feed_loves = feed[:20]
def test_simple_add_love(self):
loves = Love.objects.all()[:10]
feed = LoveFeed(13)
# slow version
activities = []
feed.delete()
for love in loves:
activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
activities.append(activity)
feed.add(activity)
assert feed.contains(activity)
# close the feed
feed.finish()
feed_loves = feed[:20]
assert isinstance(feed_loves[-1], FeedEndMarker)
assert len(feed_loves) == 11
for activity in feed_loves:
assert activity
# now the fast version
feed.delete()
feed.add_many(activities)
for activity in activities:
assert feed.contains(activity)
def test_count(self):
loves = Love.objects.all()[:10]
feed = LoveFeed(13)
feed.finish()
count_lazy = feed.count()
count = int(count_lazy)
def test_count(self):
loves = Love.objects.all()[:10]
feed = LoveFeed(13)
feed.finish()
count_lazy = feed.count()
count = int(count_lazy)
def test_simple_remove_love(self):
from entity.models import Love
target_loves = Love.objects.all()[:10]
feed = LoveFeed(13)
feed.delete()
# slow implementation
activities = []
for love in target_loves:
# remove the items by key (id)
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
activities.append(activity)
feed.remove(activity)
feed.add_many(activities)
for activity in activities:
assert feed.contains(activity)
feed.remove_many(activities)
assert feed.count() == 0
feed_loves = feed[:20]