def test_double_finish(self):
# validate that finish called twice acts as expected
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
feed.finish()
feed.finish()
self.assertEqual(feed.count(), 1)
def test_db_fallback(self):
cache = self.get_structure()
# hack to make sure our queries work
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
# test the basic scenario if we have no data
results = feed[:10]
self.assertNotEqual(results, [])
self.assertEqual(feed.source, 'db')
# the cache stores a mapping between id and serialized activities
# this reduces memory usage when writing the same love activity
# to 300.000 profiles
cache_count = cache.count()
self.assertNotEqual(cache_count, 0)
# now to test the db fallback
keys = cache.keys()
to_remove = keys[:3]
redis_results = cache.get_many(to_remove)
# now we test removing the redis results, the cache should fallback
# to the database
cache.delete_many(to_remove)
# verify that the delete actually worked
self.assertNotEqual(cache.count(), cache_count)
# now proceed to lookup the missing keys
# this should hit the database and return the same results
db_results = cache.get_many(to_remove)
self.assertEqual(redis_results, db_results)
db_results = cache.get_many(to_remove)
def test_filtering(self):
# test the pagination
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
results = feed[:5]
self.assertNotEqual(results, [])
self.assertEqual(feed.source, 'db')
one, two, three = feed[:3]
assert one.object_id > two.object_id > three.object_id
self.assertEqual(feed.source, 'redis')
# we are sorted descending, this should get the first item
feed.pk__gte = gte = two.object_id
feed._set_filter()
should_be_one = feed[:1][0]
self.assertEqual(feed.source, 'redis')
self.assertActivityEqual(should_be_one, one, name='should be one')
# we are sorted descending, this should give the third item
feed.pk__gte = None
feed.pk__lte = lte = two.object_id - 1
feed._set_filter()
results = feed[:1]
should_be_three = results[0]
self.assertEqual(feed.source, 'redis')
self.assertActivityEqual(
should_be_three, three, name='should be three')
def test_db_fallback(self):
cache = self.get_structure()
#hack to make sure our queries work
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
#test the basic scenario if we have no data
results = feed[:10]
self.assertNotEqual(results, [])
self.assertEqual(feed.source, 'db')
cache_count = cache.count()
self.assertNotEqual(cache_count, 0)
#now to test the db fallback
keys = cache.keys()
to_remove = keys[:3]
redis_results = cache.get_many(to_remove)
cache.delete_many(to_remove)
self.assertNotEqual(cache.count(), cache_count)
#now proceed to lookup the missing keys
db_results = cache.get_many(to_remove)
#these should still load from the db
self.assertEqual(redis_results, db_results)
db_results = cache.get_many(to_remove)
def test_double_finish(self):
# validate that finish called twice acts as expected
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
feed.finish()
feed.finish()
self.assertEqual(feed.count(), 1)
def test_db_fallback(self):
cache = self.get_structure()
# hack to make sure our queries work
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
# test the basic scenario if we have no data
results = feed[:10]
self.assertNotEqual(results, [])
self.assertEqual(feed.source, 'db')
# the cache stores a mapping between id and serialized activities
# this reduces memory usage when writing the same love activity
# to 300.000 profiles
cache_count = cache.count()
self.assertNotEqual(cache_count, 0)
# now to test the db fallback
keys = cache.keys()
to_remove = keys[:3]
redis_results = cache.get_many(to_remove)
# now we test removing the redis results, the cache should fallback
# to the database
cache.delete_many(to_remove)
# verify that the delete actually worked
self.assertNotEqual(cache.count(), cache_count)
# now proceed to lookup the missing keys
# this should hit the database and return the same results
db_results = cache.get_many(to_remove)
self.assertEqual(redis_results, db_results)
db_results = cache.get_many(to_remove)
def test_filtering(self):
# test the pagination
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
results = feed[:5]
self.assertNotEqual(results, [])
self.assertEqual(feed.source, 'db')
one, two, three = feed[:3]
assert one.object_id > two.object_id > three.object_id
self.assertEqual(feed.source, 'redis')
# we are sorted descending, this should get the first item
feed.pk__gte = gte = two.object_id
feed._set_filter()
should_be_one = feed[:1][0]
self.assertEqual(feed.source, 'redis')
self.assertActivityEqual(should_be_one, one, name='should be one')
# we are sorted descending, this should give the third item
feed.pk__gte = None
feed.pk__lte = lte = two.object_id - 1
feed._set_filter()
results = feed[:1]
should_be_three = results[0]
self.assertEqual(feed.source, 'redis')
self.assertActivityEqual(should_be_three,
three,
name='should be three')
def test_finish_marker_creation(self):
# The user's feed is empty at the moment
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
results = feed[:100]
self.assertEqual(results, [])
self.assertEqual(feed.source, 'db')
# now try reading the data only from redis
results = feed[:100]
self.assertEqual(feed.source, 'redis')
# the finish marker should be there though
self.assertEqual(feed.count(), 1)
def test_finish_marker_creation(self):
# The user's feed is empty at the moment
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
results = feed[:100]
self.assertEqual(results, [])
self.assertEqual(feed.source, 'db')
# now try reading the data only from redis
results = feed[:100]
self.assertEqual(feed.source, 'redis')
# the finish marker should be there though
self.assertEqual(feed.count(), 1)
def test_enrichment(self):
# hack to make sure our queries work
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
results = feed[:5]
self.assertNotEqual(results, [])
self.assertEqual(feed.source, 'db')
results = feed[:5]
self.assertEqual(feed.source, 'redis')
# load the users and entities in batch
# Transform to love objects
for result in convert_activities_to_loves(results):
assert isinstance(result, Love)
assert isinstance(result.user, User)
assert isinstance(result.entity, Entity)
assert result.created_at, 'created_at is not defined'
def test_follow_many_trim(self):
follows = Follow.objects.filter(user=self.bogus_user)[:5]
follow = follows[0]
# reset the feed
feed = DatabaseFallbackLoveFeed(follow.user_id)
feed.delete()
# do a follow
feedly = LoveFeedly()
max_loves = 3
feedly.follow_many(follows, async=False, max_loves=max_loves)
# we should only have 3 items in the feed
feed_count = feed.count()
self.assertEqual(feed_count, max_loves)
# but we should fallback to the database
feed_results = feed[:20]
self.assertEqual(len(feed_results), 20)
def test_follow_many_trim(self):
follows = Follow.objects.filter(user=self.bogus_user)[:5]
follow = follows[0]
# reset the feed
feed = DatabaseFallbackLoveFeed(follow.user_id)
feed.delete()
# do a follow
feedly = LoveFeedly()
max_loves = 3
feedly.follow_many(follows, async=False, max_loves=max_loves)
# we should only have 3 items in the feed
feed_count = feed.count()
self.assertEqual(feed_count, max_loves)
# but we should fallback to the database
feed_results = feed[:20]
self.assertEqual(len(feed_results), 20)
def test_enrichment(self):
# hack to make sure our queries work
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
results = feed[:5]
self.assertNotEqual(results, [])
self.assertEqual(feed.source, 'db')
results = feed[:5]
self.assertEqual(feed.source, 'redis')
# load the users and entities in batch
# Transform to love objects
for result in convert_activities_to_loves(results):
assert isinstance(result, Love)
assert isinstance(result.user, User)
assert isinstance(result.entity, Entity)
assert result.created_at, 'created_at is not defined'
def test_small_feed_instance(self):
for desired_max_length in range(3, 5):
feed = DatabaseFallbackLoveFeed(
self.bogus_user.id, max_length=desired_max_length)
feed.delete()
# test the basic scenario if we have no data
results = feed[:desired_max_length]
results = feed[:desired_max_length]
# this should come from redis, since its smaller than the desired max length
self.assertEqual(feed.source, 'redis')
self.assertEqual(len(results), desired_max_length)
self.assertEqual(feed.max_length, desired_max_length)
self.assertEqual(feed.count(), desired_max_length)
# these will have to come from the db
results = feed[:desired_max_length + 2]
self.assertEqual(feed.source, 'db')
results = feed[:desired_max_length + 2]
self.assertEqual(feed.source, 'db')
def test_small_feed_instance(self):
for desired_max_length in range(3, 5):
feed = DatabaseFallbackLoveFeed(self.bogus_user.id,
max_length=desired_max_length)
feed.delete()
# test the basic scenario if we have no data
results = feed[:desired_max_length]
results = feed[:desired_max_length]
# this should come from redis, since its smaller than the desired max length
self.assertEqual(feed.source, 'redis')
self.assertEqual(len(results), desired_max_length)
self.assertEqual(feed.max_length, desired_max_length)
self.assertEqual(feed.count(), desired_max_length)
# these will have to come from the db
results = feed[:desired_max_length + 2]
self.assertEqual(feed.source, 'db')
results = feed[:desired_max_length + 2]
self.assertEqual(feed.source, 'db')
def test_empty_redis(self):
# hack to make sure our queries work
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
# test the basic scenario if we have no data
results = feed[:1]
self.assertNotEqual(results, [])
self.assertEqual(feed.source, 'db')
results = feed[:1]
self.assertEqual(feed.source, 'redis')
# reset and test a finished empty list, this shouldnt return anything
feed.delete()
feed.finish()
results = feed[:1]
self.assertEqual(results, [])
# try again past the first page
feed.delete()
results = feed[:1]
results = feed[:2]
self.assertEqual(len(results), 2)
self.assertEqual(feed.source, 'db')
def test_empty_redis(self):
# hack to make sure our queries work
feed = DatabaseFallbackLoveFeed(self.bogus_user.id)
feed.delete()
# test the basic scenario if we have no data
results = feed[:1]
self.assertNotEqual(results, [])
self.assertEqual(feed.source, 'db')
results = feed[:1]
self.assertEqual(feed.source, 'redis')
# reset and test a finished empty list, this shouldnt return anything
feed.delete()
feed.finish()
results = feed[:1]
self.assertEqual(results, [])
# try again past the first page
feed.delete()
results = feed[:1]
results = feed[:2]
self.assertEqual(len(results), 2)
self.assertEqual(feed.source, 'db')