def test_db_fallback(self): cache = self.get_structure() # hack to make sure our queries work feed = DatabaseFallbackLoveFeed(self.bogus_user.id) feed.delete() # test the basic scenario if we have no data results = feed[:10] self.assertNotEqual(results, []) self.assertEqual(feed.source, 'db') # the cache stores a mapping between id and serialized activities # this reduces memory usage when writing the same love activity # to 300.000 profiles cache_count = cache.count() self.assertNotEqual(cache_count, 0) # now to test the db fallback keys = cache.keys() to_remove = keys[:3] redis_results = cache.get_many(to_remove) # now we test removing the redis results, the cache should fallback # to the database cache.delete_many(to_remove) # verify that the delete actually worked self.assertNotEqual(cache.count(), cache_count) # now proceed to lookup the missing keys # this should hit the database and return the same results db_results = cache.get_many(to_remove) self.assertEqual(redis_results, db_results) db_results = cache.get_many(to_remove)
def test_db_fallback(self): cache = self.get_structure() #hack to make sure our queries work feed = DatabaseFallbackLoveFeed(self.bogus_user.id) feed.delete() #test the basic scenario if we have no data results = feed[:10] self.assertNotEqual(results, []) self.assertEqual(feed.source, 'db') cache_count = cache.count() self.assertNotEqual(cache_count, 0) #now to test the db fallback keys = cache.keys() to_remove = keys[:3] redis_results = cache.get_many(to_remove) cache.delete_many(to_remove) self.assertNotEqual(cache.count(), cache_count) #now proceed to lookup the missing keys db_results = cache.get_many(to_remove) #these should still load from the db self.assertEqual(redis_results, db_results) db_results = cache.get_many(to_remove)
def test_filtering(self): # test the pagination feed = DatabaseFallbackLoveFeed(self.bogus_user.id) feed.delete() results = feed[:5] self.assertNotEqual(results, []) self.assertEqual(feed.source, 'db') one, two, three = feed[:3] assert one.object_id > two.object_id > three.object_id self.assertEqual(feed.source, 'redis') # we are sorted descending, this should get the first item feed.pk__gte = gte = two.object_id feed._set_filter() should_be_one = feed[:1][0] self.assertEqual(feed.source, 'redis') self.assertActivityEqual(should_be_one, one, name='should be one') # we are sorted descending, this should give the third item feed.pk__gte = None feed.pk__lte = lte = two.object_id - 1 feed._set_filter() results = feed[:1] should_be_three = results[0] self.assertEqual(feed.source, 'redis') self.assertActivityEqual(should_be_three, three, name='should be three')
def test_finish_marker_creation(self): # The user's feed is empty at the moment feed = DatabaseFallbackLoveFeed(self.bogus_user.id) feed.delete() results = feed[:100] self.assertEqual(results, []) self.assertEqual(feed.source, 'db') # now try reading the data only from redis results = feed[:100] self.assertEqual(feed.source, 'redis') # the finish marker should be there though self.assertEqual(feed.count(), 1)
def test_filtering(self): # test the pagination feed = DatabaseFallbackLoveFeed(self.bogus_user.id) feed.delete() results = feed[:5] self.assertNotEqual(results, []) self.assertEqual(feed.source, 'db') one, two, three = feed[:3] assert one.object_id > two.object_id > three.object_id self.assertEqual(feed.source, 'redis') # we are sorted descending, this should get the first item feed.pk__gte = gte = two.object_id feed._set_filter() should_be_one = feed[:1][0] self.assertEqual(feed.source, 'redis') self.assertActivityEqual(should_be_one, one, name='should be one') # we are sorted descending, this should give the third item feed.pk__gte = None feed.pk__lte = lte = two.object_id - 1 feed._set_filter() results = feed[:1] should_be_three = results[0] self.assertEqual(feed.source, 'redis') self.assertActivityEqual( should_be_three, three, name='should be three')
def test_enrichment(self): # hack to make sure our queries work feed = DatabaseFallbackLoveFeed(self.bogus_user.id) feed.delete() results = feed[:5] self.assertNotEqual(results, []) self.assertEqual(feed.source, 'db') results = feed[:5] self.assertEqual(feed.source, 'redis') # load the users and entities in batch # Transform to love objects for result in convert_activities_to_loves(results): assert isinstance(result, Love) assert isinstance(result.user, User) assert isinstance(result.entity, Entity) assert result.created_at, 'created_at is not defined'
def test_follow_many_trim(self): follows = Follow.objects.filter(user=self.bogus_user)[:5] follow = follows[0] # reset the feed feed = DatabaseFallbackLoveFeed(follow.user_id) feed.delete() # do a follow feedly = LoveFeedly() max_loves = 3 feedly.follow_many(follows, async=False, max_loves=max_loves) # we should only have 3 items in the feed feed_count = feed.count() self.assertEqual(feed_count, max_loves) # but we should fallback to the database feed_results = feed[:20] self.assertEqual(len(feed_results), 20)
def test_double_finish(self): # validate that finish called twice acts as expected feed = DatabaseFallbackLoveFeed(self.bogus_user.id) feed.delete() feed.finish() feed.finish() self.assertEqual(feed.count(), 1)
def test_small_feed_instance(self): for desired_max_length in range(3, 5): feed = DatabaseFallbackLoveFeed( self.bogus_user.id, max_length=desired_max_length) feed.delete() # test the basic scenario if we have no data results = feed[:desired_max_length] results = feed[:desired_max_length] # this should come from redis, since its smaller than the desired max length self.assertEqual(feed.source, 'redis') self.assertEqual(len(results), desired_max_length) self.assertEqual(feed.max_length, desired_max_length) self.assertEqual(feed.count(), desired_max_length) # these will have to come from the db results = feed[:desired_max_length + 2] self.assertEqual(feed.source, 'db') results = feed[:desired_max_length + 2] self.assertEqual(feed.source, 'db')
def test_small_feed_instance(self): for desired_max_length in range(3, 5): feed = DatabaseFallbackLoveFeed(self.bogus_user.id, max_length=desired_max_length) feed.delete() # test the basic scenario if we have no data results = feed[:desired_max_length] results = feed[:desired_max_length] # this should come from redis, since its smaller than the desired max length self.assertEqual(feed.source, 'redis') self.assertEqual(len(results), desired_max_length) self.assertEqual(feed.max_length, desired_max_length) self.assertEqual(feed.count(), desired_max_length) # these will have to come from the db results = feed[:desired_max_length + 2] self.assertEqual(feed.source, 'db') results = feed[:desired_max_length + 2] self.assertEqual(feed.source, 'db')
def test_empty_redis(self): # hack to make sure our queries work feed = DatabaseFallbackLoveFeed(self.bogus_user.id) feed.delete() # test the basic scenario if we have no data results = feed[:1] self.assertNotEqual(results, []) self.assertEqual(feed.source, 'db') results = feed[:1] self.assertEqual(feed.source, 'redis') # reset and test a finished empty list, this shouldnt return anything feed.delete() feed.finish() results = feed[:1] self.assertEqual(results, []) # try again past the first page feed.delete() results = feed[:1] results = feed[:2] self.assertEqual(len(results), 2) self.assertEqual(feed.source, 'db')