commit_to_feed(feed, article)
log("%s: %s/%s: Stored %s, reference to %s (%s)" % \
(feed.key.name, feed.group.name, feed.name, article.uid, url, document.headers['content-type']))
return
# Document parsing.
try:
article_content = parser.extract_body(document.text)
summary = parser.summarise(article_content)
except Exception, e:
log("%s: %s: Error parsing %s: %s" %
(feed.key.name, feed.group.name, url, e.message))
return
if not title:
title = parser.extract_title(document.text)
# Initial article object
article = Article(url=url, title=title, summary=summary)
if not "://" in article.url:
article.url = "http://" + article.url
# Determine whether to store the full content or a compressed copy
if not app.config['COMPRESS_ARTICLES']:
article.content = article_content
else:
article.ccontent = snappy.compress(
article_content.encode("utf-8", "ignore"))
article.compressed = True
log("%s: %s/%s: Stored %s, reference to %s (%s)" % \
(feed.key.name, feed.group.name, feed.name, article.uid, url, document.headers['content-type']))
return
# Document parsing.
try:
article_content = parser.extract_body(document.text)
summary = parser.summarise(article_content)
except Exception, e:
log("%s: %s: Error parsing %s: %s" %
(feed.key.name, feed.group.name, url, e.message))
return
# Ensure a title and disregard dupes
if not title:
title = parser.extract_title(document.text)
if app.config['NO_DUPLICATE_TITLES']:
if Article.query.filter(
and_(Article.title == title, Article.key == feed.key)).first():
return
# Initial article object
article = Article(url=url, title=title, summary=summary)
# Determine whether to store the full content or a compressed copy
if not app.config['COMPRESS_ARTICLES']:
article.content = article_content
else:
article.ccontent = snappy.compress(
article_content.encode("utf-8", "ignore"))
article.url = "http://" + article.url
commit_to_feed(feed, article)
log("%s: %s/%s: Stored %s, reference to %s (%s)" % \
(feed.key.name, feed.group.name, feed.name, article.uid, url, document.headers['content-type']))
return
# Document parsing.
try:
article_content = parser.extract_body(document.text)
summary = parser.summarise(article_content)
except Exception, e:
log("%s: %s: Error parsing %s: %s" % (feed.key.name, feed.group.name, url, e.message))
return
if not title:
title = parser.extract_title(document.text)
# Initial article object
article = Article(
url=url,
title=title,
summary=summary
)
if not "://" in article.url:
article.url = "http://" + article.url
# Determine whether to store the full content or a compressed copy
if not app.config['COMPRESS_ARTICLES']:
article.content=article_content
else:
commit_to_feed(feed, article)
log("%s: %s/%s: Stored %s, reference to %s (%s)" % \
(feed.key.name, feed.group.name, feed.name, article.uid, url, document.headers['content-type']))
return
# Document parsing.
try:
article_content = parser.extract_body(document.text)
summary = parser.summarise(article_content)
except Exception, e:
log("%s: %s: Error parsing %s: %s" % (feed.key.name, feed.group.name, url, e.message))
return
# Ensure a title and disregard dupes
if not title:
title = parser.extract_title(document.text)
if app.config['NO_DUPLICATE_TITLES']:
if Article.query.filter(
and_(Article.title == title, Article.key == feed.key)
).first():
return
# Initial article object
article = Article(
url=url,
title=title,
summary=summary
)
# Determine whether to store the full content or a compressed copy