def _test_activity_serializer(self, serializer):
def test_activity(activity, name=None):
serialized_activity = serializer.dumps(activity)
deserialized = serializer.loads(serialized_activity)
self.assertActivityEqual(activity, deserialized)
# example with target
activity = Activity(13,
LoveVerb,
2000,
target=15,
time=datetime.datetime.now())
test_activity(activity, 'target_no_context')
# example with target and extra context
activity = Activity(13,
LoveVerb,
2000,
target=15,
time=datetime.datetime.now(),
extra_context=dict(hello='world'))
test_activity(activity, 'target_and_context')
# example with no target and extra context
activity = Activity(13,
LoveVerb,
2000,
time=datetime.datetime.now(),
extra_context=dict(hello='world'))
test_activity(activity, 'no_target_and_context')
# example with no target and no extra context
activity = Activity(13, LoveVerb, 2000, time=datetime.datetime.now())
test_activity(activity, 'no_target_and_no_context')
def test_dehydrated_activity(self):
activity_object = Pin(id=1)
activity = Activity(1, LoveVerb, activity_object)
dehydrated = activity.get_dehydrated()
self.assertTrue(isinstance(dehydrated, DehydratedActivity))
self.assertEquals(
dehydrated.serialization_id, activity.serialization_id)
def test_dehydrated_activity(self):
activity_object = Pin(id=1)
activity = Activity(1, LoveVerb, activity_object)
dehydrated = activity.get_dehydrated()
self.assertTrue(isinstance(dehydrated, DehydratedActivity))
self.assertEquals(dehydrated.serialization_id,
activity.serialization_id)
def test_aggregated_feed(self):
loves = Love.objects.all()[:10]
feed = AggregatedFeed(13)
# slow version
activities = []
feed.delete()
for love in loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
activities.append(activity)
feed.add(activity)
assert feed.contains(activity)
# so we have something to compare to
aggregator = RecentVerbAggregator()
aggregated_activities = aggregator.aggregate(activities)
# check the feed
feed_loves = feed[:20]
self.assertEqual(len(aggregated_activities), len(feed_loves))
# now the fast version
feed.delete()
self.assertEqual(int(feed.count()), 0)
feed.add_many(activities)
for activity in activities:
assert feed.contains(activity)
def test_simple_add_love(self):
loves = Love.objects.all()[:10]
feed = LoveFeed(13)
# slow version
activities = []
feed.delete()
for love in loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
activities.append(activity)
feed.add(activity)
assert feed.contains(activity)
# close the feed
feed.finish()
feed_loves = feed[:20]
assert isinstance(feed_loves[-1], FeedEndMarker)
assert len(feed_loves) == 11
for activity in feed_loves:
assert activity
# now the fast version
feed.delete()
feed.add_many(activities)
for activity in activities:
assert feed.contains(activity)
def test_feed_trim(self):
class SmallLoveFeed(LoveFeed):
max_length = 5
loves = Love.objects.all()[:10]
feed = SmallLoveFeed(13)
# slow version
activities = []
feed.delete()
for love in loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
activities.append(activity)
feed.add(activity)
# close the feed
feed.finish()
feed_loves = feed[:20]
assert len(feed_loves) == feed.max_length
for activity in feed_loves:
assert activity
# now the fast version
feed.delete()
feed.add_many(activities)
def test_simple_remove_love(self):
from entity.models import Love
target_loves = Love.objects.all()[:10]
feed = LoveFeed(13)
feed.delete()
# slow implementation
activities = []
for love in target_loves:
# remove the items by key (id)
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
activities.append(activity)
feed.remove(activity)
feed.add_many(activities)
for activity in activities:
assert feed.contains(activity)
feed.remove_many(activities)
assert feed.count() == 0
feed_loves = feed[:20]
def loads(self, serialized_activity):
activity_kwargs = serialized_activity.__dict__.copy()
activity_kwargs.pop('key')
activity_kwargs['verb'] = get_verb_by_id(activity_kwargs['verb'])
activity_kwargs['extra_context'] = pickle.loads(
activity_kwargs['extra_context'])
return Activity(**activity_kwargs)
def loads(self, serialized_activity):
# handle the FeedEndMarker
if serialized_activity == FEED_END:
activity = FeedEndMarker()
else:
parts = serialized_activity.split(',')
# convert these to ids
actor_id, verb_id, object_id, target_id, entity_id = map(
int, parts[:5])
activity_datetime = epoch_to_datetime(float(parts[5]))
pickle_string = parts[6]
if not target_id:
target_id = None
verb = get_verb_by_id(verb_id)
extra_context = {}
if pickle_string:
extra_context = pickle.loads(str(pickle_string))
if entity_id:
extra_context['entity_id'] = entity_id
activity = Activity(actor_id,
verb,
object_id,
target_id,
time=activity_datetime,
extra_context=extra_context)
return activity
def generate_aggregated_activities(self, diff=0):
aggregator = RecentVerbAggregator()
activities = []
for x in range(1, 20 + diff):
activity = Activity(x, LoveVerb, Pin(id=x))
activities.append(activity)
aggregated_activities = aggregator.aggregate(activities)
return aggregated_activities
def test_removed_love(self):
'''
Replicates the following scenario
- The user loves an item
- Its pushed on a feed
- The item is set to inactive, removing the love from the database
- The redis cache is cleared
- Love Item cache reads will return None
- The feed should return one result less
'''
# start with adding some data
loves = Love.objects.all()[:10]
feed = LoveFeed(13)
# slow version
activities = []
feed.delete()
for love in loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
activities.append(activity)
feed.add(activity)
assert feed.contains(activity)
# close the feed
feed.finish()
feed_loves = feed[:20]
#assert isinstance(feed_loves[-1], FeedEndMarker)
#assert len(feed_loves) == 11
# now for the scenario that the item is not there
removed_love = feed_loves[2]
removed_id = removed_love.serialization_id
# Fake that the data is None
old_get_many = feed.item_cache.get_many
def wrap_get_many(fields):
result = old_get_many(fields)
if removed_id in result:
result[removed_id] = None
return result
feed.item_cache.get_many = wrap_get_many
# verify we return None
self.assertEqual(feed.item_cache.get(removed_id), None)
empty_result = {removed_id: None}
self.assertEqual(feed.item_cache.get_many([removed_id]), empty_result)
feed_loves = feed[:20]
self.assertEqual(feed.source, 'redis')
found_activity_ids = [a.serialization_id for a in feed_loves]
assert removed_id not in found_activity_ids
self.assertEqual(len(feed_loves), 10)
def create_activity(self):
from feedly.activity import Activity
from core.verbs import Pin as PinVerb
activity = Activity(self.user_id,
PinVerb,
self.id,
self.influencer_id,
time=make_naive(self.created_at, pytz.utc),
extra_context=dict(item_id=self.item.id))
return activity
def loads(self, serialized_activity):
# TODO: convert cqlengine model to feedly Activity using public API
activity_kwargs = {
k: getattr(serialized_activity, k)
for k in serialized_activity.__dict__['_values'].keys()
}
activity_kwargs.pop('activity_id')
activity_kwargs.pop('feed_id')
activity_kwargs['verb'] = get_verb_by_id(int(serialized_activity.verb))
activity_kwargs['extra_context'] = pickle.loads(
activity_kwargs['extra_context'])
return Activity(**activity_kwargs)
def test_small_feed_instance(self):
loves = Love.objects.all()[:5]
feed = LoveFeed(13, max_length=2)
for love in loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
feed.add(activity)
self.assertEqual(feed.count(), feed.max_length)
def test_follow(self):
from user.models import Follow
follow = Follow.objects.all()[:1][0]
feed = LoveFeed(follow.user_id)
target_loves = follow.target.get_profile().loves()[:500]
for love in target_loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
feed.add(activity)
feed_loves = feed[:20]
def loads(self, serialized_activity):
if serialized_activity == FEED_END:
activity = FeedEndMarker()
else:
parts = serialized_activity.split(',')
actor_id, verb_id, object_id, target_id = map(int, parts[:4])
if not target_id:
target_id = None
pickle_string = parts[4]
verb = get_verb_by_id(verb_id)
extra_context = {}
if pickle_string:
extra_context = pickle.loads(pickle_string)
activity = Activity(actor_id, verb, object_id,
target=target_id, extra_context=extra_context)
return activity
def test_aggregated_remove(self):
activity_object = Pin(id=1)
activities = []
for x in range(1, 101):
activity = Activity(x, LoveVerb, activity_object)
activities.append(activity)
aggregator = RecentVerbAggregator()
aggregated_activities = aggregator.aggregate(activities)
aggregated = aggregated_activities[0]
for activity in activities:
try:
aggregated.remove(activity)
except (ActivityNotFound, ValueError):
pass
self.assertEqual(len(aggregated.activities), 1)
self.assertEqual(aggregated.activity_count, 72)
def loads(self, serialized_activity):
parts = serialized_activity.split('|')
# convert these to ids
actor_id, verb_id, object_id, target_id = map(
int, parts[:4])
activity_datetime = epoch_to_datetime(float(parts[4]))
pickle_string = str(parts[5])
if not target_id:
target_id = None
verb = get_verb_by_id(verb_id)
extra_context = {}
if pickle_string:
extra_context = pickle.loads(pickle_string)
activity = Activity(actor_id, verb, object_id, target_id,
time=activity_datetime, extra_context=extra_context)
return activity
def test_notification_feed(self):
loves = Love.objects.all()[:10]
feed = NotificationFeed(13)
# slow version
activities = []
feed.delete()
for love in loves:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
activities.append(activity)
feed.add(activity)
assert feed.contains(activity)
# so we have something to compare to
aggregator = RecentVerbAggregator()
aggregated_activities = aggregator.aggregate(activities)
# check the feed
feed_loves = feed[:20]
self.assertEqual(len(aggregated_activities), len(feed_loves))
# now the fast version
feed.delete()
self.assertEqual(int(feed.count()), 0)
feed.add_many(activities)
for activity in activities:
assert feed.contains(activity)
# test if we aggregated correctly
self.assertEqual(feed.count_unseen(), len(aggregated_activities))
# verify if we denormalize correctly
self.assertEqual(feed.count_unseen(), feed.get_denormalized_count())
# sanity check
self.assertNotEqual(feed.count_unseen(), 0)
# test marking as seen or read
feed.mark_all(seen=True)
# verify that the new count is 0
self.assertEqual(feed.count_unseen(), 0)
# verify if we denormalize correctly
self.assertEqual(feed.count_unseen(), feed.get_denormalized_count())
def test_aggregated_properties(self):
activities = []
for x in range(1, 101):
activity_object = Pin(id=x)
activity = Activity(x, LoveVerb, activity_object)
activities.append(activity)
aggregator = RecentVerbAggregator()
aggregated_activities = aggregator.aggregate(activities)
aggregated = aggregated_activities[0]
self.assertEqual(aggregated.verbs, [LoveVerb])
self.assertEqual(aggregated.verb, LoveVerb)
self.assertEqual(aggregated.actor_count, 100)
self.assertEqual(aggregated.minimized_activities, 85)
self.assertEqual(aggregated.other_actor_count, 98)
self.assertEqual(aggregated.activity_count, 100)
self.assertEqual(aggregated.object_ids, range(86, 101))
# the other ones should be dropped
self.assertEqual(aggregated.actor_ids, range(86, 101))
self.assertEqual(aggregated.is_seen(), False)
self.assertEqual(aggregated.is_read(), False)
def test_add_love(self):
from entity.models import Love
thessa = User.objects.get(pk=13)
profile = thessa.get_profile()
follower_ids = profile.cached_follower_ids()[:100]
love = Love.objects.all()[:1][0]
connection = get_redis_connection()
# divide the followers in groups of 10000
follower_groups = chunks(follower_ids, 10000)
for follower_group in follower_groups:
# now, for these 10000 items pipeline/thread away
with connection.map() as redis:
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(hello='world'))
for follower_id in follower_group:
feed = LoveFeed(follower_id, redis=redis)
feed.add(activity)
def test_actor_count(self):
love = Love.objects.all()[:1][0]
feed = NotificationFeed(13)
# setup the activities, all in the same aggregated activity
activities = []
feed.delete()
for x in range(150):
activity = Activity(love.user,
LoveVerb,
love,
love.user,
time=love.created_at,
extra_context=dict(x=x))
activities.append(activity)
# now the fast insert
self.assertEqual(int(feed.count()), 0)
feed.add_many(activities)
self.assertEqual(int(feed.count()), 1)
aggregated_activity = feed[:1][0]
# test our Guesstimate
self.assertEqual(aggregated_activity.minimized_activities, 51)
self.assertEqual(aggregated_activity.actor_count, 52)
def test_serialization_length(self):
activity_object = Pin(id=1)
activity = Activity(1, LoveVerb, activity_object)
assert len(str(activity.serialization_id)) == 26
def test_serialization_overflow_check_role_id(self):
activity_object = Pin(id=1)
Verb = type('Overflow', (LoveVerb, ), {'id': 9999})
activity = Activity(1, Verb, activity_object)
with self.assertRaises(TypeError):
activity.serialization_id
def test_serialization_overflow_check_object_id(self):
activity_object = Pin(id=10**10)
activity = Activity(1, LoveVerb, activity_object)
with self.assertRaises(TypeError):
activity.serialization_id
def test_serialization_type(self):
activity_object = Pin(id=1)
activity = Activity(1, LoveVerb, activity_object)
assert isinstance(activity.serialization_id, (int, long, float))
def test_compare_apple_and_oranges(self):
activity_object = Pin(id=1)
activity = Activity(1, LoveVerb, activity_object)
with self.assertRaises(ValueError):
activity == activity_object
def test_contains_extraneous_object(self):
activity = AggregatedActivity(1, [Activity(1, LoveVerb, Pin(id=1))])
with self.assertRaises(ValueError):
activity.contains(Pin(id=1))
def test_contains(self):
activity = Activity(1, LoveVerb, Pin(id=1))
aggregated = AggregatedActivity(1, [activity])
self.assertTrue(aggregated.contains(activity))
def test_compare_apple_and_oranges(self):
activity = AggregatedActivity(1, [Activity(1, LoveVerb, Pin(id=1))])
with self.assertRaises(ValueError):
activity == Pin(id=1)
def test_duplicated_activities(self):
activity = Activity(1, LoveVerb, Pin(id=1))
aggregated = AggregatedActivity(1, [activity])
with self.assertRaises(DuplicateActivityException):
aggregated.append(activity)