Esempio n. 1
0
def fetch_full_text(article):
    from rdr.modules.feeds.models import ArticleFullText
    full_text = ArticleFullText.query.filter(ArticleFullText.article_id == article.id).first()
    if not full_text:
        from rdr.modules.feeds.full_text import FullTextProvider
        from rdr.modules.feeds.search import ArticleSearchIndex


        provider = FullTextProvider.provider()
        full_text_result = provider.fetch_article_full_text(article)
        if full_text_result:
            full_text = ArticleFullText(article_id=article.id,
                                        text=html.nl2p(full_text_result.safe_text),
                                        image_src=full_text_result.primary_image)
            if not article.preview_image_src:
                article.preview_image_src = full_text_result.primary_image
        else:
            full_text = ArticleFullText(article_id=article.id,
                                        text=html.nl2p(article.preview_text),
                                        image_src=article.preview_image_src)
        db.session.add(full_text)
        db.session.commit()

        search_provider = ArticleSearchIndex(article)
        search_provider.create_full_text_index(full_text)
    return full_text
Esempio n. 2
0
    def sync(self):
        feed = self.feed
        extractor = FeedExtractor(feed.channel_url)
        extractor.parse(etag=feed.last_etag_header, modified=feed.last_modified_header)
        article_models = []
        if extractor.is_modified:
            for article in extractor.articles:
                try:
                    if article.published_date:
                        from rdr.components.timezone import convert_to_local
                        published_date = convert_to_local(article.published_date)
                    else:
                        raise Exception('Empty published date')
                    if feed.last_update and published_date < feed.last_update:
                        continue
                    text = article.safe_text
                    title = article.title
                    import hashlib
                    s1 = hashlib.sha1()
                    check_string = (title + ' | ' + published_date.strftime('%Y-%m-%d %H:%M:%S')) \
                        .encode('utf-8')
                    s1.update(check_string)
                    hash_ = s1.hexdigest()
                    saved_article = Article.query.filter((Article.feed_id == feed.id) & (Article.hash == hash_)).first()
                    if saved_article is not None:
                        continue
                    fetched_date = DateTime.now()
                    article_model = Article(title=article.title,
                                            article_url=article.url,
                                            feed_id=feed.id,
                                            preview_text=html.nl2br(text),
                                            active=True,
                                            published=published_date,
                                            fetched=fetched_date,
                                            hash=hash_)
                    image_url = article.primary_image_url
                    if image_url:
                        if http.check_is_not_local_url(image_url) and http.check_is_absolute_url(image_url):
                            article_model.preview_image_src = image_url
                    article_models.append(article_model)
                    db.session.add(article_model)
                    db.session.commit()
                    if self.is_add_to_search_index:
                        from rdr.modules.feeds.search import ArticleSearchIndex
                        search_provider = ArticleSearchIndex(article_model)
                        search_provider.create_index()
                except Exception as e:
                    app.logger.exception(e)

            if not feed.active:
                feed.active = True
            feed.last_etag_header = extractor.etag_header
            feed.last_modified_header = extractor.modified_header
            feed.last_update = DateTime.now()
            db.session.commit()

        return article_models