def fetch_full_text(article): from rdr.modules.feeds.models import ArticleFullText full_text = ArticleFullText.query.filter(ArticleFullText.article_id == article.id).first() if not full_text: from rdr.modules.feeds.full_text import FullTextProvider from rdr.modules.feeds.search import ArticleSearchIndex provider = FullTextProvider.provider() full_text_result = provider.fetch_article_full_text(article) if full_text_result: full_text = ArticleFullText(article_id=article.id, text=html.nl2p(full_text_result.safe_text), image_src=full_text_result.primary_image) if not article.preview_image_src: article.preview_image_src = full_text_result.primary_image else: full_text = ArticleFullText(article_id=article.id, text=html.nl2p(article.preview_text), image_src=article.preview_image_src) db.session.add(full_text) db.session.commit() search_provider = ArticleSearchIndex(article) search_provider.create_full_text_index(full_text) return full_text
def sync(self): feed = self.feed extractor = FeedExtractor(feed.channel_url) extractor.parse(etag=feed.last_etag_header, modified=feed.last_modified_header) article_models = [] if extractor.is_modified: for article in extractor.articles: try: if article.published_date: from rdr.components.timezone import convert_to_local published_date = convert_to_local(article.published_date) else: raise Exception('Empty published date') if feed.last_update and published_date < feed.last_update: continue text = article.safe_text title = article.title import hashlib s1 = hashlib.sha1() check_string = (title + ' | ' + published_date.strftime('%Y-%m-%d %H:%M:%S')) \ .encode('utf-8') s1.update(check_string) hash_ = s1.hexdigest() saved_article = Article.query.filter((Article.feed_id == feed.id) & (Article.hash == hash_)).first() if saved_article is not None: continue fetched_date = DateTime.now() article_model = Article(title=article.title, article_url=article.url, feed_id=feed.id, preview_text=html.nl2br(text), active=True, published=published_date, fetched=fetched_date, hash=hash_) image_url = article.primary_image_url if image_url: if http.check_is_not_local_url(image_url) and http.check_is_absolute_url(image_url): article_model.preview_image_src = image_url article_models.append(article_model) db.session.add(article_model) db.session.commit() if self.is_add_to_search_index: from rdr.modules.feeds.search import ArticleSearchIndex search_provider = ArticleSearchIndex(article_model) search_provider.create_index() except Exception as e: app.logger.exception(e) if not feed.active: feed.active = True feed.last_etag_header = extractor.etag_header feed.last_modified_header = extractor.modified_header feed.last_update = DateTime.now() db.session.commit() return article_models