def fetch_full_text(article):
from rdr.modules.feeds.models import ArticleFullText
full_text = ArticleFullText.query.filter(ArticleFullText.article_id == article.id).first()
if not full_text:
from rdr.modules.feeds.full_text import FullTextProvider
from rdr.modules.feeds.search import ArticleSearchIndex
provider = FullTextProvider.provider()
full_text_result = provider.fetch_article_full_text(article)
if full_text_result:
full_text = ArticleFullText(article_id=article.id,
text=html.nl2p(full_text_result.safe_text),
image_src=full_text_result.primary_image)
if not article.preview_image_src:
article.preview_image_src = full_text_result.primary_image
else:
full_text = ArticleFullText(article_id=article.id,
text=html.nl2p(article.preview_text),
image_src=article.preview_image_src)
db.session.add(full_text)
db.session.commit()
search_provider = ArticleSearchIndex(article)
search_provider.create_full_text_index(full_text)
return full_text
def sync(self):
feed = self.feed
extractor = FeedExtractor(feed.channel_url)
extractor.parse(etag=feed.last_etag_header, modified=feed.last_modified_header)
article_models = []
if extractor.is_modified:
for article in extractor.articles:
try:
if article.published_date:
from rdr.components.timezone import convert_to_local
published_date = convert_to_local(article.published_date)
else:
raise Exception('Empty published date')
if feed.last_update and published_date < feed.last_update:
continue
text = article.safe_text
title = article.title
import hashlib
s1 = hashlib.sha1()
check_string = (title + ' | ' + published_date.strftime('%Y-%m-%d %H:%M:%S')) \
.encode('utf-8')
s1.update(check_string)
hash_ = s1.hexdigest()
saved_article = Article.query.filter((Article.feed_id == feed.id) & (Article.hash == hash_)).first()
if saved_article is not None:
continue
fetched_date = DateTime.now()
article_model = Article(title=article.title,
article_url=article.url,
feed_id=feed.id,
preview_text=html.nl2br(text),
active=True,
published=published_date,
fetched=fetched_date,
hash=hash_)
image_url = article.primary_image_url
if image_url:
if http.check_is_not_local_url(image_url) and http.check_is_absolute_url(image_url):
article_model.preview_image_src = image_url
article_models.append(article_model)
db.session.add(article_model)
db.session.commit()
if self.is_add_to_search_index:
from rdr.modules.feeds.search import ArticleSearchIndex
search_provider = ArticleSearchIndex(article_model)
search_provider.create_index()
except Exception as e:
app.logger.exception(e)
if not feed.active:
feed.active = True
feed.last_etag_header = extractor.etag_header
feed.last_modified_header = extractor.modified_header
feed.last_update = DateTime.now()
db.session.commit()
return article_models