async def parse_feed(user, feed):
"""
Fetch a feed.
Update the feed and return the articles.
"""
parsed_feed = None
up_feed = {}
articles = []
resp = None
#with (await sem):
try:
logger.info('Retrieving feed {}'.format(feed.link))
resp = await jarr_get(feed.link, timeout=5)
except Exception as e:
logger.info('Problem when reading feed {}'.format(feed.link))
return
finally:
if None is resp:
return
try:
content = io.BytesIO(resp.content)
parsed_feed = feedparser.parse(content)
except Exception as e:
up_feed['last_error'] = str(e)
up_feed['error_count'] = feed.error_count + 1
logger.exception("error when parsing feed: " + str(e))
finally:
up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal())
if parsed_feed is None:
try:
FeedController().update({'id': feed.id}, up_feed)
except Exception as e:
logger.exception('something bad here: ' + str(e))
return
if not is_parsing_ok(parsed_feed):
up_feed['last_error'] = str(parsed_feed['bozo_exception'])
up_feed['error_count'] = feed.error_count + 1
FeedController().update({'id': feed.id}, up_feed)
return
if parsed_feed['entries'] != []:
articles = parsed_feed['entries']
up_feed['error_count'] = 0
up_feed['last_error'] = ""
# Feed information
try:
construct_feed_from(feed.link, parsed_feed).update(up_feed)
except:
logger.exception('error when constructing feed: {}'.format(feed.link))
if feed.title and 'title' in up_feed:
# do not override the title set by the user
del up_feed['title']
FeedController().update({'id': feed.id}, up_feed)
return articles
def bookmarklet():
feed_contr = FeedController(current_user.id)
url = (request.args if request.method == 'GET' else request.form)\
.get('url', None)
if not url:
flash(gettext("Couldn't add feed: url missing."), "error")
raise BadRequest("url is missing")
feed_exists = list(feed_contr.read(__or__={'link': url, 'site_link': url}))
if feed_exists:
flash(gettext("Couldn't add feed: feed already exists."), "warning")
return redirect(url_for('feed.form', feed_id=feed_exists[0].id))
try:
feed = construct_feed_from(url)
except requests.exceptions.ConnectionError:
flash(gettext("Impossible to connect to the address: {}.".format(url)),
"danger")
return redirect(url_for('home'))
except Exception:
logger.exception('something bad happened when fetching %r', url)
return redirect(url_for('home'))
if not feed.get('link'):
feed['enabled'] = False
flash(
gettext("Couldn't find a feed url, you'll need to find a Atom or"
" RSS link manually and reactivate this feed"), 'warning')
feed = feed_contr.create(**feed)
flash(gettext('Feed was successfully created.'), 'success')
if feed.enabled and conf.CRAWLING_METHOD == "default":
misc_utils.fetch(current_user.id, feed.id)
flash(gettext("Downloading articles for the new feed..."), 'info')
return redirect(url_for('feed.form', feed_id=feed.id))
def bookmarklet():
feed_contr = FeedController(current_user.id)
url = (request.args if request.method == 'GET' else request.form)\
.get('url', None)
if not url:
flash(gettext("Couldn't add feed: url missing."), "error")
raise BadRequest("url is missing")
feed_exists = list(feed_contr.read(__or__={'link': url, 'site_link': url}))
if feed_exists:
flash(gettext("Couldn't add feed: feed already exists."),
"warning")
return redirect(url_for('feed.form', feed_id=feed_exists[0].id))
try:
feed = construct_feed_from(url)
except requests.exceptions.ConnectionError:
flash(gettext("Impossible to connect to the address: {}.".format(url)),
"danger")
return redirect(url_for('home'))
except Exception:
logger.exception('something bad happened when fetching %r', url)
return redirect(url_for('home'))
if not feed.get('link'):
feed['enabled'] = False
flash(gettext("Couldn't find a feed url, you'll need to find a Atom or"
" RSS link manually and reactivate this feed"),
'warning')
feed = feed_contr.create(**feed)
flash(gettext('Feed was successfully created.'), 'success')
if feed.enabled and conf.CRAWLING_METHOD == "classic":
misc_utils.fetch(current_user.id, feed.id)
flash(gettext("Downloading articles for the new feed..."), 'info')
return redirect(url_for('feed.form', feed_id=feed.id))
async def parse_feed(user, feed):
"""
Fetch a feed.
Update the feed and return the articles.
"""
parsed_feed = None
up_feed = {}
articles = []
with (await sem):
try:
parsed_feed = await get(feed.link)
except Exception as e:
up_feed['last_error'] = str(e)
up_feed['error_count'] = feed.error_count + 1
logger.exception("error when parsing feed: " + str(e))
finally:
up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal())
if parsed_feed is None:
try:
FeedController().update({'id': feed.id}, up_feed)
except Exception as e:
logger.exception('something bad here: ' + str(e))
return
if not is_parsing_ok(parsed_feed):
up_feed['last_error'] = str(parsed_feed['bozo_exception'])
up_feed['error_count'] = feed.error_count + 1
FeedController().update({'id': feed.id}, up_feed)
return
if parsed_feed['entries'] != []:
articles = parsed_feed['entries']
up_feed['error_count'] = 0
up_feed['last_error'] = ""
# Feed information
construct_feed_from(feed.link, parsed_feed).update(up_feed)
if feed.title and 'title' in up_feed:
# do not override the title set by the user
del up_feed['title']
FeedController().update({'id': feed.id}, up_feed)
return articles
async def parse_feed(user, feed):
"""
Fetch a feed.
Update the feed and return the articles.
"""
parsed_feed = None
up_feed = {}
articles = []
with (await sem):
try:
parsed_feed = await get(feed.link)
except Exception as e:
up_feed['last_error'] = str(e)
up_feed['error_count'] = feed.error_count + 1
logger.exception("error when parsing feed: " + str(e))
finally:
up_feed['last_retrieved'] = datetime.now(dateutil.tz.tzlocal())
if parsed_feed is None:
try:
FeedController().update({'id': feed.id}, up_feed)
except Exception as e:
logger.exception('something bad here: ' + str(e))
return
if not is_parsing_ok(parsed_feed):
up_feed['last_error'] = str(parsed_feed['bozo_exception'])
up_feed['error_count'] = feed.error_count + 1
FeedController().update({'id': feed.id}, up_feed)
return
if parsed_feed['entries'] != []:
articles = parsed_feed['entries']
up_feed['error_count'] = 0
up_feed['last_error'] = ""
# Feed informations
construct_feed_from(feed.link, parsed_feed).update(up_feed)
if feed.title and 'title' in up_feed:
# do not override the title set by the user
del up_feed['title']
FeedController().update({'id': feed.id}, up_feed)
return articles
def callback(self, response):
"""Will process the result from the challenge, creating missing article
and updating the feed"""
article_created = False
if response.result().status_code != 204:
results = response.result().json()
logger.debug(
'%r %r - %d entries were not matched '
'and will be created', self.feed['id'], self.feed['title'],
len(results))
for id_to_create in results:
article_created = True
entry = construct_article(
self.entries[tuple(sorted(id_to_create.items()))],
self.feed)
logger.info('%r %r - creating %r for %r - %r', self.feed['id'],
self.feed['title'], entry['title'],
entry['user_id'], id_to_create)
self.query_pyagg('post', 'article', entry)
logger.debug('%r %r - updating feed etag %r last_mod %r',
self.feed['id'], self.feed['title'],
self.headers.get('etag', ''),
self.headers.get('last-modified', ''))
up_feed = {
'error_count':
0,
'last_error':
None,
'etag':
self.headers.get('etag', ''),
'last_modified':
self.headers.get('last-modified',
strftime('%a, %d %b %Y %X %Z', gmtime()))
}
fresh_feed = construct_feed_from(url=self.feed['link'],
fp_parsed=self.parsed_feed)
for key in ('description', 'site_link', 'icon_url'):
if fresh_feed.get(key) and fresh_feed[key] != self.feed.get(key):
up_feed[key] = fresh_feed[key]
if not self.feed.get('title'):
up_feed['title'] = fresh_feed.get('title', '')
up_feed['user_id'] = self.feed['user_id']
# re-getting that feed earlier since new entries appeared
if article_created:
up_feed['last_retrieved'] \
= (datetime.now() - timedelta(minutes=45)).isoformat()
diff_keys = {
key
for key in up_feed if up_feed[key] != self.feed.get(key)
}
if not diff_keys:
return # no change in the feed, no update
if not article_created and diff_keys == {'last_modified', 'etag'}:
return # meaningless if no new article has been published
logger.info(
'%r %r - pushing feed attrs %r', self.feed['id'],
self.feed['title'], {
key: "%s -> %s" % (up_feed[key], self.feed.get(key))
for key in up_feed if up_feed[key] != self.feed.get(key)
})
self.query_pyagg('put', 'feed/%d' % self.feed['id'], up_feed)
