def collect(): """ Looks for a source which has not been updated in at least an hour and fetches new articles for it. """ # Get a feed which has not yet been updated # and is not currently being updated. feeds = Feed.query.filter(Feed.updated_at < datetime.utcnow() - timedelta(hours=1), ~Feed.updating).all() if feeds: feed = random.choice(feeds) # "Claim" this feed, # so other workers won't pick it. feed.updating = True db.session.commit() try: collector.collect(feed) feed.updated_at = datetime.utcnow() except Exception: logger.exception('Exception while collecting for feed {0}'.format(feed.ext_url)) raise finally: feed.updating = False db.session.commit()
def test_collect_error(self): self.assertEquals(self.source.errors, 0) self.mock_articles.side_effect = feed.SAXException('', None) collector.collect() self.assertEquals(self.source.errors, 1)
def test_collect_ignores_existing(self): self.mock_articles.return_value = [ Article(title='Foo', published=datetime.utcnow(), url='foo.com') ] collector.collect() collector.collect() self.assertEquals(Article.query.count(), 1)
def test_collect(self): self.mock_articles.return_value = [ Article(title='Foo', published=datetime.utcnow(), url='foo.com') ] self.assertEquals(Article.query.count(), 0) collector.collect() self.assertEquals(Article.query.count(), 1) article = Article.query.first() self.assertEquals(article.title, 'Foo')
def test_collect_ignores_existing(self): self.mock_articles.return_value = [ Article( title='Foo', published=datetime.utcnow(), url='foo.com' ) ] collector.collect() collector.collect() self.assertEquals(Article.query.count(), 1)
def test_collect(self): self.mock_articles.return_value = [ Article( title='Foo', published=datetime.utcnow(), url='foo.com' ) ] self.assertEquals(Article.query.count(), 0) collector.collect() self.assertEquals(Article.query.count(), 1) article = Article.query.first() self.assertEquals(article.title, 'Foo')
def test_collect_error(self): self.patch_articles() self.assertEquals(self.feed.errors, 0) self.mock_articles.side_effect = [collector.SAXException('', None)] articles = collector.collect(self.feed) self.assertEquals(self.feed.errors, 1)
def test_collect(self): self.assertEquals(Article.query.count(), 0) self.patch_articles() articles = collector.collect(self.feed) self.assertEquals(Article.query.count(), 1) article = Article.query.first() self.assertEquals(article.title, 'Foo')