def _test_activity_serializer(self, serializer): def test_activity(activity, name=None): serialized_activity = serializer.dumps(activity) deserialized = serializer.loads(serialized_activity) self.assertActivityEqual(activity, deserialized) # example with target activity = Activity(13, LoveVerb, 2000, target=15, time=datetime.datetime.now()) test_activity(activity, 'target_no_context') # example with target and extra context activity = Activity(13, LoveVerb, 2000, target=15, time=datetime.datetime.now(), extra_context=dict(hello='world')) test_activity(activity, 'target_and_context') # example with no target and extra context activity = Activity(13, LoveVerb, 2000, time=datetime.datetime.now(), extra_context=dict(hello='world')) test_activity(activity, 'no_target_and_context') # example with no target and no extra context activity = Activity(13, LoveVerb, 2000, time=datetime.datetime.now()) test_activity(activity, 'no_target_and_no_context')
def test_dehydrated_activity(self): activity_object = Pin(id=1) activity = Activity(1, LoveVerb, activity_object) dehydrated = activity.get_dehydrated() self.assertTrue(isinstance(dehydrated, DehydratedActivity)) self.assertEquals( dehydrated.serialization_id, activity.serialization_id)
def test_dehydrated_activity(self): activity_object = Pin(id=1) activity = Activity(1, LoveVerb, activity_object) dehydrated = activity.get_dehydrated() self.assertTrue(isinstance(dehydrated, DehydratedActivity)) self.assertEquals(dehydrated.serialization_id, activity.serialization_id)
def test_aggregated_feed(self): loves = Love.objects.all()[:10] feed = AggregatedFeed(13) # slow version activities = [] feed.delete() for love in loves: activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world')) activities.append(activity) feed.add(activity) assert feed.contains(activity) # so we have something to compare to aggregator = RecentVerbAggregator() aggregated_activities = aggregator.aggregate(activities) # check the feed feed_loves = feed[:20] self.assertEqual(len(aggregated_activities), len(feed_loves)) # now the fast version feed.delete() self.assertEqual(int(feed.count()), 0) feed.add_many(activities) for activity in activities: assert feed.contains(activity)
def test_simple_add_love(self): loves = Love.objects.all()[:10] feed = LoveFeed(13) # slow version activities = [] feed.delete() for love in loves: activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world')) activities.append(activity) feed.add(activity) assert feed.contains(activity) # close the feed feed.finish() feed_loves = feed[:20] assert isinstance(feed_loves[-1], FeedEndMarker) assert len(feed_loves) == 11 for activity in feed_loves: assert activity # now the fast version feed.delete() feed.add_many(activities) for activity in activities: assert feed.contains(activity)
def test_feed_trim(self): class SmallLoveFeed(LoveFeed): max_length = 5 loves = Love.objects.all()[:10] feed = SmallLoveFeed(13) # slow version activities = [] feed.delete() for love in loves: activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world')) activities.append(activity) feed.add(activity) # close the feed feed.finish() feed_loves = feed[:20] assert len(feed_loves) == feed.max_length for activity in feed_loves: assert activity # now the fast version feed.delete() feed.add_many(activities)
def test_simple_remove_love(self): from entity.models import Love target_loves = Love.objects.all()[:10] feed = LoveFeed(13) feed.delete() # slow implementation activities = [] for love in target_loves: # remove the items by key (id) activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world')) activities.append(activity) feed.remove(activity) feed.add_many(activities) for activity in activities: assert feed.contains(activity) feed.remove_many(activities) assert feed.count() == 0 feed_loves = feed[:20]
def loads(self, serialized_activity): activity_kwargs = serialized_activity.__dict__.copy() activity_kwargs.pop('key') activity_kwargs['verb'] = get_verb_by_id(activity_kwargs['verb']) activity_kwargs['extra_context'] = pickle.loads( activity_kwargs['extra_context']) return Activity(**activity_kwargs)
def loads(self, serialized_activity): # handle the FeedEndMarker if serialized_activity == FEED_END: activity = FeedEndMarker() else: parts = serialized_activity.split(',') # convert these to ids actor_id, verb_id, object_id, target_id, entity_id = map( int, parts[:5]) activity_datetime = epoch_to_datetime(float(parts[5])) pickle_string = parts[6] if not target_id: target_id = None verb = get_verb_by_id(verb_id) extra_context = {} if pickle_string: extra_context = pickle.loads(str(pickle_string)) if entity_id: extra_context['entity_id'] = entity_id activity = Activity(actor_id, verb, object_id, target_id, time=activity_datetime, extra_context=extra_context) return activity
def generate_aggregated_activities(self, diff=0): aggregator = RecentVerbAggregator() activities = [] for x in range(1, 20 + diff): activity = Activity(x, LoveVerb, Pin(id=x)) activities.append(activity) aggregated_activities = aggregator.aggregate(activities) return aggregated_activities
def test_removed_love(self): ''' Replicates the following scenario - The user loves an item - Its pushed on a feed - The item is set to inactive, removing the love from the database - The redis cache is cleared - Love Item cache reads will return None - The feed should return one result less ''' # start with adding some data loves = Love.objects.all()[:10] feed = LoveFeed(13) # slow version activities = [] feed.delete() for love in loves: activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world')) activities.append(activity) feed.add(activity) assert feed.contains(activity) # close the feed feed.finish() feed_loves = feed[:20] #assert isinstance(feed_loves[-1], FeedEndMarker) #assert len(feed_loves) == 11 # now for the scenario that the item is not there removed_love = feed_loves[2] removed_id = removed_love.serialization_id # Fake that the data is None old_get_many = feed.item_cache.get_many def wrap_get_many(fields): result = old_get_many(fields) if removed_id in result: result[removed_id] = None return result feed.item_cache.get_many = wrap_get_many # verify we return None self.assertEqual(feed.item_cache.get(removed_id), None) empty_result = {removed_id: None} self.assertEqual(feed.item_cache.get_many([removed_id]), empty_result) feed_loves = feed[:20] self.assertEqual(feed.source, 'redis') found_activity_ids = [a.serialization_id for a in feed_loves] assert removed_id not in found_activity_ids self.assertEqual(len(feed_loves), 10)
def create_activity(self): from feedly.activity import Activity from core.verbs import Pin as PinVerb activity = Activity(self.user_id, PinVerb, self.id, self.influencer_id, time=make_naive(self.created_at, pytz.utc), extra_context=dict(item_id=self.item.id)) return activity
def loads(self, serialized_activity): # TODO: convert cqlengine model to feedly Activity using public API activity_kwargs = { k: getattr(serialized_activity, k) for k in serialized_activity.__dict__['_values'].keys() } activity_kwargs.pop('activity_id') activity_kwargs.pop('feed_id') activity_kwargs['verb'] = get_verb_by_id(int(serialized_activity.verb)) activity_kwargs['extra_context'] = pickle.loads( activity_kwargs['extra_context']) return Activity(**activity_kwargs)
def test_small_feed_instance(self): loves = Love.objects.all()[:5] feed = LoveFeed(13, max_length=2) for love in loves: activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world')) feed.add(activity) self.assertEqual(feed.count(), feed.max_length)
def test_follow(self): from user.models import Follow follow = Follow.objects.all()[:1][0] feed = LoveFeed(follow.user_id) target_loves = follow.target.get_profile().loves()[:500] for love in target_loves: activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world')) feed.add(activity) feed_loves = feed[:20]
def loads(self, serialized_activity): if serialized_activity == FEED_END: activity = FeedEndMarker() else: parts = serialized_activity.split(',') actor_id, verb_id, object_id, target_id = map(int, parts[:4]) if not target_id: target_id = None pickle_string = parts[4] verb = get_verb_by_id(verb_id) extra_context = {} if pickle_string: extra_context = pickle.loads(pickle_string) activity = Activity(actor_id, verb, object_id, target=target_id, extra_context=extra_context) return activity
def test_aggregated_remove(self): activity_object = Pin(id=1) activities = [] for x in range(1, 101): activity = Activity(x, LoveVerb, activity_object) activities.append(activity) aggregator = RecentVerbAggregator() aggregated_activities = aggregator.aggregate(activities) aggregated = aggregated_activities[0] for activity in activities: try: aggregated.remove(activity) except (ActivityNotFound, ValueError): pass self.assertEqual(len(aggregated.activities), 1) self.assertEqual(aggregated.activity_count, 72)
def loads(self, serialized_activity): parts = serialized_activity.split('|') # convert these to ids actor_id, verb_id, object_id, target_id = map( int, parts[:4]) activity_datetime = epoch_to_datetime(float(parts[4])) pickle_string = str(parts[5]) if not target_id: target_id = None verb = get_verb_by_id(verb_id) extra_context = {} if pickle_string: extra_context = pickle.loads(pickle_string) activity = Activity(actor_id, verb, object_id, target_id, time=activity_datetime, extra_context=extra_context) return activity
def test_notification_feed(self): loves = Love.objects.all()[:10] feed = NotificationFeed(13) # slow version activities = [] feed.delete() for love in loves: activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world')) activities.append(activity) feed.add(activity) assert feed.contains(activity) # so we have something to compare to aggregator = RecentVerbAggregator() aggregated_activities = aggregator.aggregate(activities) # check the feed feed_loves = feed[:20] self.assertEqual(len(aggregated_activities), len(feed_loves)) # now the fast version feed.delete() self.assertEqual(int(feed.count()), 0) feed.add_many(activities) for activity in activities: assert feed.contains(activity) # test if we aggregated correctly self.assertEqual(feed.count_unseen(), len(aggregated_activities)) # verify if we denormalize correctly self.assertEqual(feed.count_unseen(), feed.get_denormalized_count()) # sanity check self.assertNotEqual(feed.count_unseen(), 0) # test marking as seen or read feed.mark_all(seen=True) # verify that the new count is 0 self.assertEqual(feed.count_unseen(), 0) # verify if we denormalize correctly self.assertEqual(feed.count_unseen(), feed.get_denormalized_count())
def test_aggregated_properties(self): activities = [] for x in range(1, 101): activity_object = Pin(id=x) activity = Activity(x, LoveVerb, activity_object) activities.append(activity) aggregator = RecentVerbAggregator() aggregated_activities = aggregator.aggregate(activities) aggregated = aggregated_activities[0] self.assertEqual(aggregated.verbs, [LoveVerb]) self.assertEqual(aggregated.verb, LoveVerb) self.assertEqual(aggregated.actor_count, 100) self.assertEqual(aggregated.minimized_activities, 85) self.assertEqual(aggregated.other_actor_count, 98) self.assertEqual(aggregated.activity_count, 100) self.assertEqual(aggregated.object_ids, range(86, 101)) # the other ones should be dropped self.assertEqual(aggregated.actor_ids, range(86, 101)) self.assertEqual(aggregated.is_seen(), False) self.assertEqual(aggregated.is_read(), False)
def test_add_love(self): from entity.models import Love thessa = User.objects.get(pk=13) profile = thessa.get_profile() follower_ids = profile.cached_follower_ids()[:100] love = Love.objects.all()[:1][0] connection = get_redis_connection() # divide the followers in groups of 10000 follower_groups = chunks(follower_ids, 10000) for follower_group in follower_groups: # now, for these 10000 items pipeline/thread away with connection.map() as redis: activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(hello='world')) for follower_id in follower_group: feed = LoveFeed(follower_id, redis=redis) feed.add(activity)
def test_actor_count(self): love = Love.objects.all()[:1][0] feed = NotificationFeed(13) # setup the activities, all in the same aggregated activity activities = [] feed.delete() for x in range(150): activity = Activity(love.user, LoveVerb, love, love.user, time=love.created_at, extra_context=dict(x=x)) activities.append(activity) # now the fast insert self.assertEqual(int(feed.count()), 0) feed.add_many(activities) self.assertEqual(int(feed.count()), 1) aggregated_activity = feed[:1][0] # test our Guesstimate self.assertEqual(aggregated_activity.minimized_activities, 51) self.assertEqual(aggregated_activity.actor_count, 52)
def test_serialization_length(self): activity_object = Pin(id=1) activity = Activity(1, LoveVerb, activity_object) assert len(str(activity.serialization_id)) == 26
def test_serialization_overflow_check_role_id(self): activity_object = Pin(id=1) Verb = type('Overflow', (LoveVerb, ), {'id': 9999}) activity = Activity(1, Verb, activity_object) with self.assertRaises(TypeError): activity.serialization_id
def test_serialization_overflow_check_object_id(self): activity_object = Pin(id=10**10) activity = Activity(1, LoveVerb, activity_object) with self.assertRaises(TypeError): activity.serialization_id
def test_serialization_type(self): activity_object = Pin(id=1) activity = Activity(1, LoveVerb, activity_object) assert isinstance(activity.serialization_id, (int, long, float))
def test_compare_apple_and_oranges(self): activity_object = Pin(id=1) activity = Activity(1, LoveVerb, activity_object) with self.assertRaises(ValueError): activity == activity_object
def test_contains_extraneous_object(self): activity = AggregatedActivity(1, [Activity(1, LoveVerb, Pin(id=1))]) with self.assertRaises(ValueError): activity.contains(Pin(id=1))
def test_contains(self): activity = Activity(1, LoveVerb, Pin(id=1)) aggregated = AggregatedActivity(1, [activity]) self.assertTrue(aggregated.contains(activity))
def test_compare_apple_and_oranges(self): activity = AggregatedActivity(1, [Activity(1, LoveVerb, Pin(id=1))]) with self.assertRaises(ValueError): activity == Pin(id=1)
def test_duplicated_activities(self): activity = Activity(1, LoveVerb, Pin(id=1)) aggregated = AggregatedActivity(1, [activity]) with self.assertRaises(DuplicateActivityException): aggregated.append(activity)