def setUp(self):
# NOTE: we need real web pages, else the absolutization won't work or
# will find duplicates and tests will fail for a real-life reason.
self.article1 = Article(title='test1',
url='http://blog.1flow.io/post/'
'59410536612/1flow-blog-has-moved').save()
self.feed = Feed(name='1flow test feed',
url='http://blog.1flow.io/rss').save()
self.article1.update(add_to_set__feeds=self.feed)
self.article1.reload()
# User & Reads creation
for index in xrange(1, 2):
username = '******' % index
du = DjangoUser.objects.create(username=username,
email='*****@*****.**' % username)
# PG post_save() signal already created the MongoDB user.
u = du.mongo
Read(user=u, article=self.article1).save()
Subscription(user=u, feed=self.feed).save()
for index in xrange(2, 5):
username = '******' % index
du = DjangoUser.objects.create(username=username,
email='*****@*****.**' % username)
def setUp(self):
# NOTE: we need real web pages, else the absolutization won't work or
# will find duplicates and tests will fail for a real-life reason.
self.article1 = Article(title='test1',
url='http://blog.1flow.io/post/'
'59410536612/1flow-blog-has-moved').save()
self.feed = Feed(name='1flow test feed',
url='http://blog.1flow.io/rss').save()
self.article1.update(add_to_set__feeds=self.feed)
self.article1.reload()
# User & Reads creation
for index in xrange(1, 2):
username = '******' % index
du = DjangoUser.objects.create(username=username,
email='*****@*****.**' % username)
# PG post_save() signal already created the MongoDB user.
u = du.mongo
Read(user=u, article=self.article1).save()
Subscription(user=u, feed=self.feed).save()
for index in xrange(2, 5):
username = '******' % index
du = DjangoUser.objects.create(username=username,
email='*****@*****.**' % username)
def setUp(self):
# NOTE: we need real web pages, else the absolutization won't work or
# will find duplicates and tests will fail for a real-life reason.
self.article1 = Article(title='test1',
url='http://blog.1flow.io/post/'
'59410536612/1flow-blog-has-moved').save()
self.article2 = Article(title='test2',
url='http://obi.1flow.io/fr/').save()
self.article3 = Article(title='test3',
url='http://obi.1flow.io/en/').save()
# User & Reads creation
for index in xrange(1, 6):
username = '******' % index
du = DjangoUser.objects.create(username=username,
email='*****@*****.**' % username)
# NOTE: the mongoDB user is created automatically. If you
# try to create one it will fail with duplicate index error.
u = du.mongo
Read(user=u, article=self.article1).save()
for index in xrange(6, 11):
username = '******' % index
du = DjangoUser.objects.create(username=username,
email='*****@*****.**' % username)
u = du.mongo
Read(user=u, article=self.article2).save()
# Feeds creation
for index in xrange(1, 6):
f = Feed(name='test feed #%s' % index,
url='http://test-feed%s.com' % index).save()
self.article1.update(add_to_set__feeds=f)
self.article1.reload()
for index in xrange(6, 11):
f = Feed(name='test feed #%s' % index,
url='http://test-feed%s.com' % index).save()
self.article2.update(add_to_set__feeds=f)
self.article2.reload()
def tearDown(self):
Article.drop_collection()
Feed.drop_collection()
def test_feeds_creation(self):
# .setUp() creates one already.
self.assertEquals(Feed._get_collection().count(), 1)
feed, created = Feed.create_feeds_from_url(u'http://ntoll.org/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://ntoll.org/rss.xml')
self.assertEquals(Feed._get_collection().count(), 2)
# Via the Home Page
feed, created = Feed.create_feeds_from_url(u'http://www.zdnet.fr/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
# Via the RSS listing page
feed, created = Feed.create_feeds_from_url(u'http://www.zdnet.fr/services/rss/')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
# Via the first RSS (raw)
feed, created = Feed.create_feeds_from_url(u'http://www.zdnet.fr/feeds/rss/')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
feed, created = Feed.create_feeds_from_url(u'http://www.atlantico.fr/')[0] # NOQA
self.assertTrue(created)
self.assertEquals(feed.url, u'http://www.atlantico.fr/rss.xml')
self.assertEquals(Feed._get_collection().count(), 4)
feed, created = Feed.create_feeds_from_url(u'http://wordpress.org/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://wordpress.org/news/feed/')
self.assertEquals(Feed._get_collection().count(), 5)
# Not created again, even from an article which has the comment feed.
feed, created = Feed.create_feeds_from_url(u'http://ntoll.org/article/build-a-drogulus')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://ntoll.org/rss.xml')
self.assertEquals(Feed._get_collection().count(), 5)
# This one has been created in .setUp()
feed, created = Feed.create_feeds_from_url(u'http://blog.1flow.io/')[0]
self.assertFalse(created)
self.assertEquals(feed.url, u'http://blog.1flow.io/rss')
self.assertEquals(Feed._get_collection().count(), 5)
# No RSS in main page
self.assertRaises(Exception, Feed.create_feeds_from_url,
u'http://www.bbc.co.uk/')
self.assertEquals(Feed._get_collection().count(), 5)
# This one has no RSS anywhere, it won't create anything
self.assertRaises(Exception, Feed.create_feeds_from_url,
u'http://www.tumblr.com/blog/1flowio')
self.assertEquals(Feed._get_collection().count(), 5)
def tearDown(self):
Subscription.drop_collection()
Feed.drop_collection()
Read.drop_collection()
Article.drop_collection()
User.drop_collection()
class FeedsTest(TestCase):
def setUp(self):
# NOTE: we need real web pages, else the absolutization won't work or
# will find duplicates and tests will fail for a real-life reason.
self.article1 = Article(title='test1',
url='http://blog.1flow.io/post/'
'59410536612/1flow-blog-has-moved').save()
self.feed = Feed(name='1flow test feed',
url='http://blog.1flow.io/rss').save()
self.article1.update(add_to_set__feeds=self.feed)
self.article1.reload()
# User & Reads creation
for index in xrange(1, 2):
username = '******' % index
du = DjangoUser.objects.create(username=username,
email='*****@*****.**' % username)
# PG post_save() signal already created the MongoDB user.
u = du.mongo
Read(user=u, article=self.article1).save()
Subscription(user=u, feed=self.feed).save()
for index in xrange(2, 5):
username = '******' % index
du = DjangoUser.objects.create(username=username,
email='*****@*****.**' % username)
def tearDown(self):
Subscription.drop_collection()
Feed.drop_collection()
Read.drop_collection()
Article.drop_collection()
User.drop_collection()
def test_close(self):
closed_reason = u'closed for tests'
self.feed.close(closed_reason)
self.assertTrue(self.feed.closed)
self.assertEquals(self.feed.closed_reason, closed_reason)
self.assertFalse(self.feed.date_closed is None)
global_feeds_checker()
self.assertEquals(len(mail.outbox), 1)
self.assertTrue(u'Reminder: 1 feed(s) closed in last'
in mail.outbox[0].subject)
self.assertTrue(unicode(self.feed) in mail.outbox[0].body)
#self.assertEqual( mail.outbox[0].to, [ "*****@*****.**" ] )
#self.assertTrue( "*****@*****.**" in mail.outbox[0].to )
def test_feeds_creation(self):
# .setUp() creates one already.
self.assertEquals(Feed._get_collection().count(), 1)
feed, created = Feed.create_feeds_from_url(u'http://ntoll.org/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://ntoll.org/rss.xml')
self.assertEquals(Feed._get_collection().count(), 2)
# Via the Home Page
feed, created = Feed.create_feeds_from_url(u'http://www.zdnet.fr/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
# Via the RSS listing page
feed, created = Feed.create_feeds_from_url(u'http://www.zdnet.fr/services/rss/')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
# Via the first RSS (raw)
feed, created = Feed.create_feeds_from_url(u'http://www.zdnet.fr/feeds/rss/')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
feed, created = Feed.create_feeds_from_url(u'http://www.atlantico.fr/')[0] # NOQA
self.assertTrue(created)
self.assertEquals(feed.url, u'http://www.atlantico.fr/rss.xml')
self.assertEquals(Feed._get_collection().count(), 4)
feed, created = Feed.create_feeds_from_url(u'http://wordpress.org/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://wordpress.org/news/feed/')
self.assertEquals(Feed._get_collection().count(), 5)
# Not created again, even from an article which has the comment feed.
feed, created = Feed.create_feeds_from_url(u'http://ntoll.org/article/build-a-drogulus')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://ntoll.org/rss.xml')
self.assertEquals(Feed._get_collection().count(), 5)
# This one has been created in .setUp()
feed, created = Feed.create_feeds_from_url(u'http://blog.1flow.io/')[0]
self.assertFalse(created)
self.assertEquals(feed.url, u'http://blog.1flow.io/rss')
self.assertEquals(Feed._get_collection().count(), 5)
# No RSS in main page
self.assertRaises(Exception, Feed.create_feeds_from_url,
u'http://www.bbc.co.uk/')
self.assertEquals(Feed._get_collection().count(), 5)
# This one has no RSS anywhere, it won't create anything
self.assertRaises(Exception, Feed.create_feeds_from_url,
u'http://www.tumblr.com/blog/1flowio')
self.assertEquals(Feed._get_collection().count(), 5)
DjangoUser = get_user_model()
LOGGER = logging.getLogger(__file__)
# Use the test database not to pollute the production/development one.
RedisStatsCounter.REDIS = TEST_REDIS
TEST_REDIS.flushdb()
connect_mongodb_testsuite()
# Empty the database before starting in case an old test failed to tearDown().
Article.drop_collection()
Read.drop_collection()
User.drop_collection()
Group.drop_collection()
Feed.drop_collection()
Tag.drop_collection()
Folder.drop_collection()
WebSite.drop_collection()
Author.drop_collection()
class ThrottleIntervalTest(TestCase):
def test_lower_interval_with_etag_or_modified(self):
t = Feed.throttle_fetch_interval
some_news = 10
no_dupe = 0
no_mutual = 0
def tearDown(self):
Article.drop_collection()
Feed.drop_collection()
def test_feeds_creation(self):
# .setUp() creates one already.
self.assertEquals(Feed._get_collection().count(), 1)
feed, created = Feed.create_feeds_from_url(u'http://ntoll.org/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://ntoll.org/rss.xml')
self.assertEquals(Feed._get_collection().count(), 2)
# Via the Home Page
feed, created = Feed.create_feeds_from_url(u'http://www.zdnet.fr/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
# Via the RSS listing page
feed, created = Feed.create_feeds_from_url(
u'http://www.zdnet.fr/services/rss/')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
# Via the first RSS (raw)
feed, created = Feed.create_feeds_from_url(
u'http://www.zdnet.fr/feeds/rss/')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
feed, created = Feed.create_feeds_from_url(
u'http://www.atlantico.fr/')[0] # NOQA
self.assertTrue(created)
self.assertEquals(feed.url, u'http://www.atlantico.fr/rss.xml')
self.assertEquals(Feed._get_collection().count(), 4)
feed, created = Feed.create_feeds_from_url(u'http://wordpress.org/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://wordpress.org/news/feed/')
self.assertEquals(Feed._get_collection().count(), 5)
# Not created again, even from an article which has the comment feed.
feed, created = Feed.create_feeds_from_url(
u'http://ntoll.org/article/build-a-drogulus')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://ntoll.org/rss.xml')
self.assertEquals(Feed._get_collection().count(), 5)
# This one has been created in .setUp()
feed, created = Feed.create_feeds_from_url(u'http://blog.1flow.io/')[0]
self.assertFalse(created)
self.assertEquals(feed.url, u'http://blog.1flow.io/rss')
self.assertEquals(Feed._get_collection().count(), 5)
# No RSS in main page
self.assertRaises(Exception, Feed.create_feeds_from_url,
u'http://www.bbc.co.uk/')
self.assertEquals(Feed._get_collection().count(), 5)
# This one has no RSS anywhere, it won't create anything
self.assertRaises(Exception, Feed.create_feeds_from_url,
u'http://www.tumblr.com/blog/1flowio')
self.assertEquals(Feed._get_collection().count(), 5)
def tearDown(self):
Subscription.drop_collection()
Feed.drop_collection()
Read.drop_collection()
Article.drop_collection()
User.drop_collection()
class FeedsTest(TestCase):
def setUp(self):
# NOTE: we need real web pages, else the absolutization won't work or
# will find duplicates and tests will fail for a real-life reason.
self.article1 = Article(title='test1',
url='http://blog.1flow.io/post/'
'59410536612/1flow-blog-has-moved').save()
self.feed = Feed(name='1flow test feed',
url='http://blog.1flow.io/rss').save()
self.article1.update(add_to_set__feeds=self.feed)
self.article1.reload()
# User & Reads creation
for index in xrange(1, 2):
username = '******' % index
du = DjangoUser.objects.create(username=username,
email='*****@*****.**' % username)
# PG post_save() signal already created the MongoDB user.
u = du.mongo
Read(user=u, article=self.article1).save()
Subscription(user=u, feed=self.feed).save()
for index in xrange(2, 5):
username = '******' % index
du = DjangoUser.objects.create(username=username,
email='*****@*****.**' % username)
def tearDown(self):
Subscription.drop_collection()
Feed.drop_collection()
Read.drop_collection()
Article.drop_collection()
User.drop_collection()
def test_close(self):
closed_reason = u'closed for tests'
self.feed.close(closed_reason)
self.assertTrue(self.feed.closed)
self.assertEquals(self.feed.closed_reason, closed_reason)
self.assertFalse(self.feed.date_closed is None)
global_feeds_checker()
self.assertEquals(len(mail.outbox), 1)
self.assertTrue(
u'Reminder: 1 feed(s) closed in last' in mail.outbox[0].subject)
self.assertTrue(unicode(self.feed) in mail.outbox[0].body)
#self.assertEqual( mail.outbox[0].to, [ "*****@*****.**" ] )
#self.assertTrue( "*****@*****.**" in mail.outbox[0].to )
def test_feeds_creation(self):
# .setUp() creates one already.
self.assertEquals(Feed._get_collection().count(), 1)
feed, created = Feed.create_feeds_from_url(u'http://ntoll.org/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://ntoll.org/rss.xml')
self.assertEquals(Feed._get_collection().count(), 2)
# Via the Home Page
feed, created = Feed.create_feeds_from_url(u'http://www.zdnet.fr/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
# Via the RSS listing page
feed, created = Feed.create_feeds_from_url(
u'http://www.zdnet.fr/services/rss/')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
# Via the first RSS (raw)
feed, created = Feed.create_feeds_from_url(
u'http://www.zdnet.fr/feeds/rss/')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://www.zdnet.fr/feeds/rss/')
self.assertEquals(Feed._get_collection().count(), 3)
feed, created = Feed.create_feeds_from_url(
u'http://www.atlantico.fr/')[0] # NOQA
self.assertTrue(created)
self.assertEquals(feed.url, u'http://www.atlantico.fr/rss.xml')
self.assertEquals(Feed._get_collection().count(), 4)
feed, created = Feed.create_feeds_from_url(u'http://wordpress.org/')[0]
self.assertTrue(created)
self.assertEquals(feed.url, u'http://wordpress.org/news/feed/')
self.assertEquals(Feed._get_collection().count(), 5)
# Not created again, even from an article which has the comment feed.
feed, created = Feed.create_feeds_from_url(
u'http://ntoll.org/article/build-a-drogulus')[0] # NOQA
self.assertFalse(created)
self.assertEquals(feed.url, u'http://ntoll.org/rss.xml')
self.assertEquals(Feed._get_collection().count(), 5)
# This one has been created in .setUp()
feed, created = Feed.create_feeds_from_url(u'http://blog.1flow.io/')[0]
self.assertFalse(created)
self.assertEquals(feed.url, u'http://blog.1flow.io/rss')
self.assertEquals(Feed._get_collection().count(), 5)
# No RSS in main page
self.assertRaises(Exception, Feed.create_feeds_from_url,
u'http://www.bbc.co.uk/')
self.assertEquals(Feed._get_collection().count(), 5)
# This one has no RSS anywhere, it won't create anything
self.assertRaises(Exception, Feed.create_feeds_from_url,
u'http://www.tumblr.com/blog/1flowio')
self.assertEquals(Feed._get_collection().count(), 5)
def test_closed_feeds_are_never_good(self):
""" This test addresses Github #10.
It is very simple, but the `.good_feeds` query is quite complex.
"""
self.assertTrue(len(Feed.good_feeds) == 1)
closed_reason = u'closed for tests'
self.feed.close(closed_reason)
self.assertTrue(len(Feed.good_feeds) == 0)
DjangoUser = get_user_model()
LOGGER = logging.getLogger(__file__)
# Use the test database not to pollute the production/development one.
RedisStatsCounter.REDIS = TEST_REDIS
TEST_REDIS.flushdb()
connect_mongodb_testsuite()
# Empty the database before starting in case an old test failed to tearDown().
Article.drop_collection()
Read.drop_collection()
User.drop_collection()
Group.drop_collection()
Feed.drop_collection()
Tag.drop_collection()
Folder.drop_collection()
WebSite.drop_collection()
Author.drop_collection()
class ThrottleIntervalTest(TestCase):
def test_lower_interval_with_etag_or_modified(self):
t = Feed.throttle_fetch_interval
some_news = 10
no_dupe = 0
no_mutual = 0
def feed_distribution_by_last_fetch():
start_time = pytime.time()
#open_feeds = Feed.objects(Q(closed=False) | Q(closed__exists=False))
open_feeds_count = Feed.objects.filter(closed__ne=True).count()
lower_value = None
loop_count = 0
fetched_feeds = 0
delta_lengths = (
timedelta(seconds=config.FEED_FETCH_DEFAULT_INTERVAL / 6),
timedelta(seconds=config.FEED_FETCH_DEFAULT_INTERVAL / 2),
timedelta(seconds=config.FEED_FETCH_DEFAULT_INTERVAL),
timedelta(seconds=config.FEED_FETCH_DEFAULT_INTERVAL * 2),
timedelta(seconds=config.FEED_FETCH_DEFAULT_INTERVAL * 6),
timedelta(seconds=config.FEED_FETCH_DEFAULT_INTERVAL * 12),
timedelta(days=1),
timedelta(days=2),
timedelta(days=3),
timedelta(days=4),
timedelta(days=5),
timedelta(days=6),
timedelta(days=7),
timedelta(days=10),
None
)
results = {}
for delta in delta_lengths:
upper_value = (now() - delta) if delta else None
if lower_value is None:
kwargs = {'last_fetch__gt': upper_value}
elif upper_value is None:
kwargs = {'last_fetch__lte': lower_value}
else:
kwargs = {'last_fetch__lte': lower_value,
'last_fetch__gt': upper_value}
feeds = Feed.objects(**kwargs)
count = feeds.count()
percent = float(count * 100.0 / open_feeds_count)
avg_fi = sum(f.fetch_interval for f in feeds) * 1.0 / (count or 1.0)
results[loop_count] = [
feeds,
count,
percent,
lower_value,
upper_value,
avg_fi,
]
fetched_feeds += count
lower_value = upper_value
loop_count += 1
results['meta'] = {'fetched_feeds': fetched_feeds,
'open_feeds_count': open_feeds_count,
'duration': pytime.time() - start_time,
'loop_count': loop_count}
return results
