def collect():
"""
Looks for a source which has not been
updated in at least an hour
and fetches new articles for it.
"""
# Get a feed which has not yet been updated
# and is not currently being updated.
feeds = Feed.query.filter(Feed.updated_at < datetime.utcnow() - timedelta(hours=1), ~Feed.updating).all()
if feeds:
feed = random.choice(feeds)
# "Claim" this feed,
# so other workers won't pick it.
feed.updating = True
db.session.commit()
try:
collector.collect(feed)
feed.updated_at = datetime.utcnow()
except Exception:
logger.exception('Exception while collecting for feed {0}'.format(feed.ext_url))
raise
finally:
feed.updating = False
db.session.commit()
def test_collect_error(self):
self.assertEquals(self.source.errors, 0)
self.mock_articles.side_effect = feed.SAXException('', None)
collector.collect()
self.assertEquals(self.source.errors, 1)
def test_collect_error(self):
self.assertEquals(self.source.errors, 0)
self.mock_articles.side_effect = feed.SAXException('', None)
collector.collect()
self.assertEquals(self.source.errors, 1)
def test_collect_ignores_existing(self):
self.mock_articles.return_value = [
Article(title='Foo', published=datetime.utcnow(), url='foo.com')
]
collector.collect()
collector.collect()
self.assertEquals(Article.query.count(), 1)
def test_collect(self):
self.mock_articles.return_value = [
Article(title='Foo', published=datetime.utcnow(), url='foo.com')
]
self.assertEquals(Article.query.count(), 0)
collector.collect()
self.assertEquals(Article.query.count(), 1)
article = Article.query.first()
self.assertEquals(article.title, 'Foo')
def test_collect_ignores_existing(self):
self.mock_articles.return_value = [
Article(
title='Foo',
published=datetime.utcnow(),
url='foo.com'
)
]
collector.collect()
collector.collect()
self.assertEquals(Article.query.count(), 1)
def test_collect(self):
self.mock_articles.return_value = [
Article(
title='Foo',
published=datetime.utcnow(),
url='foo.com'
)
]
self.assertEquals(Article.query.count(), 0)
collector.collect()
self.assertEquals(Article.query.count(), 1)
article = Article.query.first()
self.assertEquals(article.title, 'Foo')
def test_collect_error(self):
self.patch_articles()
self.assertEquals(self.feed.errors, 0)
self.mock_articles.side_effect = [collector.SAXException('', None)]
articles = collector.collect(self.feed)
self.assertEquals(self.feed.errors, 1)
def test_collect(self):
self.assertEquals(Article.query.count(), 0)
self.patch_articles()
articles = collector.collect(self.feed)
self.assertEquals(Article.query.count(), 1)
article = Article.query.first()
self.assertEquals(article.title, 'Foo')
def test_collect_error(self):
self.patch_articles()
self.assertEquals(self.feed.errors, 0)
self.mock_articles.side_effect = [collector.SAXException('', None)]
articles = collector.collect(self.feed)
self.assertEquals(self.feed.errors, 1)
def test_collect(self):
self.assertEquals(Article.query.count(), 0)
self.patch_articles()
articles = collector.collect(self.feed)
self.assertEquals(Article.query.count(), 1)
article = Article.query.first()
self.assertEquals(article.title, 'Foo')