コード例 #1
0
ファイル: views.py プロジェクト: mahdiyar/Feedly
def index(request, template='feedly/index.html'):
    context = RequestContext(request)
    #HACK for django <1.3 beta compatibility
    if 'STATIC_URL' not in context and 'MEDIA_URL' in context:
        context['STATIC_URL'] = context['MEDIA_URL']
    sample_size = int(request.GET.get('sample_size', 1000))
    context['sample_size'] = sample_size
    lucky_users = random.sample(xrange(10**6), sample_size) + [13]
    users_dict = User.objects.get_cached_users(lucky_users)
    buckets = [
        0, 24, 1 * 24, 3 * 24, 10 * 24, 30 * 24, 50 * 24, 100 * 24, 150 * 24,
        1000 * 24
    ]
    bucket_dict = dict([(b, 0) for b in buckets])
    count_dict = {}

    #retrieve all the counts in one pipelined request(s)
    with get_redis_connection().map() as redis:
        for user_id in users_dict:
            feed = LoveFeed(user_id, redis=redis)
            count = feed.count()
            count_dict[user_id] = count

    #divide into buckets using bisect left
    for user_id, count in count_dict.items():
        bucket_index = bisect.bisect_left(buckets, count)
        bucket = buckets[bucket_index]
        bucket_dict[bucket] += 1
    bucket_stats = bucket_dict.items()
    bucket_stats.sort(key=lambda x: x[0])
    context['bucket_stats'] = bucket_stats

    return render_to_response(template, context)
コード例 #2
0
ファイル: views.py プロジェクト: etel/Feedly
def monitor(request, template='feedly/monitor.html'):
    context = RequestContext(request)
    #HACK for django <1.3 beta compatibility
    if 'STATIC_URL' not in context and 'MEDIA_URL' in context:
        context['STATIC_URL'] = context['MEDIA_URL']
    sample_size = int(request.GET.get('sample_size', 2))
    lucky_users = random.sample(xrange(10 ** 6), sample_size) + [13]
    users_dict = get_user_model().objects.get_cached_users(lucky_users)

    #retrieve all the counts in one pipelined request(s)
    count_dict = {}
    with get_redis_connection().map() as redis:
        for user_id in users_dict:
            feed = LoveFeed(user_id, redis=redis)
            count = feed.count()
            count_dict[user_id] = count

    for user_id, count in count_dict.items():
        profile = users_dict[user_id].get_profile()
        redis_count = int(count_dict[user_id])
        db_max = redis_count + 10
        db_count = profile._following_loves().count()
        print profile, db_count, long(redis_count)

    return render_to_response(template, context)
コード例 #3
0
ファイル: tests.py プロジェクト: shaj3/Feedly
 def test_small_feed_instance(self):
     loves = Love.objects.all()[:5]
     feed = LoveFeed(13, max_length=2)
     for love in loves:
         activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
         feed.add(activity)
     self.assertEqual(feed.count(), feed.max_length)
コード例 #4
0
ファイル: views.py プロジェクト: etel/Feedly
def index(request, template='feedly/index.html'):
    context = RequestContext(request)
    #HACK for django <1.3 beta compatibility
    if 'STATIC_URL' not in context and 'MEDIA_URL' in context:
        context['STATIC_URL'] = context['MEDIA_URL']
    sample_size = int(request.GET.get('sample_size', 1000))
    context['sample_size'] = sample_size
    lucky_users = random.sample(xrange(10 ** 6), sample_size) + [13]
    users_dict = get_user_model().objects.get_cached_users(lucky_users)
    buckets = [0, 24, 1 * 24, 3 * 24, 10 * 24, 30 * 24, 50 * 24, 100 *
               24, 150 * 24, 1000 * 24]
    bucket_dict = dict([(b, 0) for b in buckets])
    count_dict = {}

    #retrieve all the counts in one pipelined request(s)
    with get_redis_connection().map() as redis:
        for user_id in users_dict:
            feed = LoveFeed(user_id, redis=redis)
            count = feed.count()
            count_dict[user_id] = count

    #divide into buckets using bisect left
    for user_id, count in count_dict.items():
        bucket_index = bisect.bisect_left(buckets, count)
        bucket = buckets[bucket_index]
        bucket_dict[bucket] += 1
    bucket_stats = bucket_dict.items()
    bucket_stats.sort(key=lambda x: x[0])
    context['bucket_stats'] = bucket_stats

    return render_to_response(template, context)
コード例 #5
0
ファイル: views.py プロジェクト: mahdiyar/Feedly
def monitor(request, template='feedly/monitor.html'):
    context = RequestContext(request)
    #HACK for django <1.3 beta compatibility
    if 'STATIC_URL' not in context and 'MEDIA_URL' in context:
        context['STATIC_URL'] = context['MEDIA_URL']
    sample_size = int(request.GET.get('sample_size', 2))
    lucky_users = random.sample(xrange(10**6), sample_size) + [13]
    users_dict = User.objects.get_cached_users(lucky_users)

    #retrieve all the counts in one pipelined request(s)
    count_dict = {}
    with get_redis_connection().map() as redis:
        for user_id in users_dict:
            feed = LoveFeed(user_id, redis=redis)
            count = feed.count()
            count_dict[user_id] = count

    for user_id, count in count_dict.items():
        profile = users_dict[user_id].get_profile()
        redis_count = int(count_dict[user_id])
        db_max = redis_count + 10
        db_count = profile._following_loves().count()
        print profile, db_count, long(redis_count)

    return render_to_response(template, context)
コード例 #6
0
ファイル: tests.py プロジェクト: shaj3/Feedly
    def test_follow(self):
        from user.models import Follow
        follow = Follow.objects.all()[:1][0]
        feed = LoveFeed(follow.user_id)
        target_loves = follow.target.get_profile().loves()[:500]
        for love in target_loves:
            activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
            feed.add(activity)

        feed_loves = feed[:20]
コード例 #7
0
ファイル: tests.py プロジェクト: mahdiyar/Feedly
 def test_small_feed_instance(self):
     loves = Love.objects.all()[:5]
     feed = LoveFeed(13, max_length=2)
     for love in loves:
         activity = Activity(love.user,
                             LoveVerb,
                             love,
                             love.user,
                             time=love.created_at,
                             extra_context=dict(hello='world'))
         feed.add(activity)
     self.assertEqual(feed.count(), feed.max_length)
コード例 #8
0
ファイル: tests.py プロジェクト: mahdiyar/Feedly
    def test_removed_love(self):
        '''
        Replicates the following scenario
        - The user loves an item
        - Its pushed on a feed
        - The item is set to inactive, removing the love from the database
        - The redis cache is cleared
        - Love Item cache reads will return None
        - The feed should return one result less
        '''
        # start with adding some data
        loves = Love.objects.all()[:10]
        feed = LoveFeed(13)
        # slow version
        activities = []
        feed.delete()
        for love in loves:
            activity = Activity(love.user,
                                LoveVerb,
                                love,
                                love.user,
                                time=love.created_at,
                                extra_context=dict(hello='world'))
            activities.append(activity)
            feed.add(activity)
            assert feed.contains(activity)
        # close the feed
        feed.finish()
        feed_loves = feed[:20]

        #assert isinstance(feed_loves[-1], FeedEndMarker)
        #assert len(feed_loves) == 11

        # now for the scenario that the item is not there
        removed_love = feed_loves[2]
        removed_id = removed_love.serialization_id
        # Fake that the data is None
        old_get_many = feed.item_cache.get_many

        def wrap_get_many(fields):
            result = old_get_many(fields)
            if removed_id in result:
                result[removed_id] = None
            return result

        feed.item_cache.get_many = wrap_get_many
        # verify we return None
        self.assertEqual(feed.item_cache.get(removed_id), None)
        empty_result = {removed_id: None}
        self.assertEqual(feed.item_cache.get_many([removed_id]), empty_result)

        feed_loves = feed[:20]
        self.assertEqual(feed.source, 'redis')
        found_activity_ids = [a.serialization_id for a in feed_loves]
        assert removed_id not in found_activity_ids
        self.assertEqual(len(feed_loves), 10)
コード例 #9
0
ファイル: tests.py プロジェクト: mahdiyar/Feedly
    def test_follow(self):
        from user.models import Follow
        follow = Follow.objects.all()[:1][0]
        feed = LoveFeed(follow.user_id)
        target_loves = follow.target.get_profile().loves()[:500]
        for love in target_loves:
            activity = Activity(love.user,
                                LoveVerb,
                                love,
                                love.user,
                                time=love.created_at,
                                extra_context=dict(hello='world'))
            feed.add(activity)

        feed_loves = feed[:20]
コード例 #10
0
ファイル: love_feedly.py プロジェクト: etel/Feedly
    def add_love(self, love):
        '''
        Fanout to all your followers

        This is really write intensive
        Reads are super light though
        '''
        activity = self.create_love_activity(love)

        # only write the love to the cache once
        from feedly.feeds.love_feed import LoveFeed
        LoveFeed.set_item_cache(activity)

        feeds = self._fanout(love.user, add_operation, activity=activity)
        return feeds
コード例 #11
0
ファイル: tests.py プロジェクト: shaj3/Feedly
    def test_remove_love(self):
        from entity.models import Love
        thessa = User.objects.get(pk=13)
        profile = thessa.get_profile()
        follower_ids = profile.cached_follower_ids()[:100]
        love = Love.objects.all()[:1][0]
        connection = get_redis_connection()

        # divide the followers in groups of 10000
        follower_groups = chunks(follower_ids, 10000)
        for follower_group in follower_groups:
            # now, for these 10000 items pipeline/thread away
            with connection.map() as redis:
                activity = love.create_activity()
                for follower_id in follower_group:
                    feed = LoveFeed(follower_id, redis=redis)
                    feed.remove(activity)
コード例 #12
0
ファイル: tests.py プロジェクト: shaj3/Feedly
    def test_add_love(self):
        from entity.models import Love
        thessa = User.objects.get(pk=13)
        profile = thessa.get_profile()
        follower_ids = profile.cached_follower_ids()[:100]
        love = Love.objects.all()[:1][0]
        connection = get_redis_connection()

        # divide the followers in groups of 10000
        follower_groups = chunks(follower_ids, 10000)
        for follower_group in follower_groups:
            # now, for these 10000 items pipeline/thread away
            with connection.map() as redis:
                activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
                for follower_id in follower_group:
                    feed = LoveFeed(follower_id, redis=redis)
                    feed.add(activity)
コード例 #13
0
ファイル: tests.py プロジェクト: mahdiyar/Feedly
    def test_remove_love(self):
        from entity.models import Love
        thessa = User.objects.get(pk=13)
        profile = thessa.get_profile()
        follower_ids = profile.cached_follower_ids()[:100]
        love = Love.objects.all()[:1][0]
        connection = get_redis_connection()

        # divide the followers in groups of 10000
        follower_groups = chunks(follower_ids, 10000)
        for follower_group in follower_groups:
            # now, for these 10000 items pipeline/thread away
            with connection.map() as redis:
                activity = love.create_activity()
                for follower_id in follower_group:
                    feed = LoveFeed(follower_id, redis=redis)
                    feed.remove(activity)
コード例 #14
0
ファイル: tests.py プロジェクト: shaj3/Feedly
    def test_follow(self):
        follow = Follow.objects.filter(user=self.bogus_user)[:1][0]
        # reset the feed
        feed = LoveFeed(follow.user_id)
        feed.delete()
        # do a follow
        feedly = LoveFeedly()
        feed = feedly.follow(follow)
        # see if we got the new loves
        target_loves = follow.target.get_profile().loves()[:10]
        for love in target_loves:
            activity = feedly.create_love_activity(love)
            assert feed.contains(activity)

        # check if we correctly broadcasted
        feedly.unfollow(follow)
        feed_count = feed.count()
        feed_results = feed[:20]
        self.assertEqual(feed_results, [])
コード例 #15
0
ファイル: tests.py プロジェクト: etel/Feedly
    def test_removed_love(self):
        '''
        Replicates the following scenario
        - The user loves an item
        - Its pushed on a feed
        - The item is set to inactive, removing the love from the database
        - The redis cache is cleared
        - Love Item cache reads will return None
        - The feed should return one result less
        '''
        # start with adding some data
        loves = Love.objects.all()[:10]
        feed = LoveFeed(13)
        # slow version
        activities = []
        feed.delete()
        for love in loves:
            activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
            activities.append(activity)
            feed.add(activity)
            assert feed.contains(activity)
        # close the feed
        feed.finish()
        feed_loves = feed[:20]

        #assert isinstance(feed_loves[-1], FeedEndMarker)
        #assert len(feed_loves) == 11

        # now for the scenario that the item is not there
        removed_love = feed_loves[2]
        removed_id = removed_love.serialization_id
        # Fake that the data is None
        old_get_many = feed.item_cache.get_many

        def wrap_get_many(fields):
            result = old_get_many(fields)
            if removed_id in result:
                result[removed_id] = None
            return result

        feed.item_cache.get_many = wrap_get_many
        # verify we return None
        self.assertEqual(feed.item_cache.get(removed_id), None)
        empty_result = {removed_id: None}
        self.assertEqual(feed.item_cache.get_many([removed_id]), empty_result)

        feed_loves = feed[:20]
        self.assertEqual(feed.source, 'redis')
        found_activity_ids = [a.serialization_id for a in feed_loves]
        assert removed_id not in found_activity_ids
        self.assertEqual(len(feed_loves), 10)
コード例 #16
0
ファイル: tests.py プロジェクト: mahdiyar/Feedly
    def test_add_love(self):
        from entity.models import Love
        thessa = User.objects.get(pk=13)
        profile = thessa.get_profile()
        follower_ids = profile.cached_follower_ids()[:100]
        love = Love.objects.all()[:1][0]
        connection = get_redis_connection()

        # divide the followers in groups of 10000
        follower_groups = chunks(follower_ids, 10000)
        for follower_group in follower_groups:
            # now, for these 10000 items pipeline/thread away
            with connection.map() as redis:
                activity = Activity(love.user,
                                    LoveVerb,
                                    love,
                                    love.user,
                                    time=love.created_at,
                                    extra_context=dict(hello='world'))
                for follower_id in follower_group:
                    feed = LoveFeed(follower_id, redis=redis)
                    feed.add(activity)
コード例 #17
0
ファイル: tests.py プロジェクト: mahdiyar/Feedly
    def test_follow(self):
        follow = Follow.objects.filter(user=self.bogus_user)[:1][0]
        # reset the feed
        feed = LoveFeed(follow.user_id)
        feed.delete()
        # do a follow
        feedly = LoveFeedly()
        feed = feedly.follow(follow)
        # see if we got the new loves
        target_loves = follow.target.get_profile().loves()[:10]
        for love in target_loves:
            activity = feedly.create_love_activity(love)
            assert feed.contains(activity)

        # check if we correctly broadcasted
        feedly.unfollow(follow)
        feed_count = feed.count()
        feed_results = feed[:20]
        self.assertEqual(feed_results, [])
コード例 #18
0
ファイル: tests.py プロジェクト: mahdiyar/Feedly
 def test_simple_add_love(self):
     loves = Love.objects.all()[:10]
     feed = LoveFeed(13)
     # slow version
     activities = []
     feed.delete()
     for love in loves:
         activity = Activity(love.user,
                             LoveVerb,
                             love,
                             love.user,
                             time=love.created_at,
                             extra_context=dict(hello='world'))
         activities.append(activity)
         feed.add(activity)
         assert feed.contains(activity)
     # close the feed
     feed.finish()
     feed_loves = feed[:20]
     assert isinstance(feed_loves[-1], FeedEndMarker)
     assert len(feed_loves) == 11
     for activity in feed_loves:
         assert activity
     # now the fast version
     feed.delete()
     feed.add_many(activities)
     for activity in activities:
         assert feed.contains(activity)
コード例 #19
0
ファイル: tests.py プロジェクト: shaj3/Feedly
    def test_simple_remove_love(self):
        from entity.models import Love
        target_loves = Love.objects.all()[:10]
        feed = LoveFeed(13)
        feed.delete()
        # slow implementation
        activities = []
        for love in target_loves:
            # remove the items by key (id)
            activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
            activities.append(activity)
            feed.remove(activity)

        feed.add_many(activities)
        for activity in activities:
            assert feed.contains(activity)
        feed.remove_many(activities)
        assert feed.count() == 0

        feed_loves = feed[:20]
コード例 #20
0
ファイル: tests.py プロジェクト: shaj3/Feedly
 def test_simple_add_love(self):
     loves = Love.objects.all()[:10]
     feed = LoveFeed(13)
     # slow version
     activities = []
     feed.delete()
     for love in loves:
         activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world'))
         activities.append(activity)
         feed.add(activity)
         assert feed.contains(activity)
     # close the feed
     feed.finish()
     feed_loves = feed[:20]
     assert isinstance(feed_loves[-1], FeedEndMarker)
     assert len(feed_loves) == 11
     for activity in feed_loves:
         assert activity
     # now the fast version
     feed.delete()
     feed.add_many(activities)
     for activity in activities:
         assert feed.contains(activity)
コード例 #21
0
ファイル: tests.py プロジェクト: shaj3/Feedly
 def test_count(self):
     loves = Love.objects.all()[:10]
     feed = LoveFeed(13)
     feed.finish()
     count_lazy = feed.count()
     count = int(count_lazy)
コード例 #22
0
ファイル: tests.py プロジェクト: mahdiyar/Feedly
 def test_count(self):
     loves = Love.objects.all()[:10]
     feed = LoveFeed(13)
     feed.finish()
     count_lazy = feed.count()
     count = int(count_lazy)
コード例 #23
0
ファイル: tests.py プロジェクト: mahdiyar/Feedly
    def test_simple_remove_love(self):
        from entity.models import Love
        target_loves = Love.objects.all()[:10]
        feed = LoveFeed(13)
        feed.delete()
        # slow implementation
        activities = []
        for love in target_loves:
            # remove the items by key (id)
            activity = Activity(love.user,
                                LoveVerb,
                                love,
                                love.user,
                                time=love.created_at,
                                extra_context=dict(hello='world'))
            activities.append(activity)
            feed.remove(activity)

        feed.add_many(activities)
        for activity in activities:
            assert feed.contains(activity)
        feed.remove_many(activities)
        assert feed.count() == 0

        feed_loves = feed[:20]