def popular(): """ Return the most popular feeds for the last nb_days days. """ # try to get the 'recent' popular websites, created after # 'not_created_before' # ie: not_added_before = date_last_added_feed - nb_days try: nb_days = int(request.args.get("nb_days", 365)) except ValueError: nb_days = 10000 last_added_feed = ( FeedController().read().order_by(desc("created_date")).limit(1).all() ) if last_added_feed: date_last_added_feed = last_added_feed[0].created_date else: date_last_added_feed = datetime.now() not_added_before = date_last_added_feed - timedelta(days=nb_days) filters = {} filters["created_date__gt"] = not_added_before filters["private"] = False filters["error_count__lt"] = application.config["DEFAULT_MAX_ERROR"] feeds = FeedController().count_by_link(**filters) sorted_feeds = sorted(list(feeds.items()), key=operator.itemgetter(1), reverse=True) return render_template("popular.html", popular=sorted_feeds)
async def parse_feed(user, feed): """ Fetch a feed. Update the feed and return the articles. """ parsed_feed = None up_feed = {} articles = [] resp = None # with (await sem): try: logger.info("Retrieving feed {}".format(feed.link)) resp = newspipe_get(feed.link, timeout=5) except Exception: logger.info("Problem when reading feed {}".format(feed.link)) return finally: if None is resp: return try: content = io.BytesIO(resp.content) parsed_feed = feedparser.parse(content) except Exception as e: up_feed["last_error"] = str(e) up_feed["error_count"] = feed.error_count + 1 logger.exception("error when parsing feed: " + str(e)) finally: up_feed["last_retrieved"] = datetime.now(dateutil.tz.tzlocal()) if parsed_feed is None: try: FeedController().update({"id": feed.id}, up_feed) except Exception as e: logger.exception("something bad here: " + str(e)) return if not is_parsing_ok(parsed_feed): up_feed["last_error"] = str(parsed_feed["bozo_exception"]) up_feed["error_count"] = feed.error_count + 1 FeedController().update({"id": feed.id}, up_feed) return if parsed_feed["entries"] != []: articles = parsed_feed["entries"] up_feed["error_count"] = 0 up_feed["last_error"] = "" # Feed information try: up_feed.update(construct_feed_from(feed.link, parsed_feed)) except: logger.exception("error when constructing feed: {}".format(feed.link)) if feed.title and "title" in up_feed: # do not override the title set by the user del up_feed["title"] try: FeedController().update({"id": feed.id}, up_feed) except: logger.exception("error when updating feed: {}".format(feed.link)) return articles
def form(category_id=None): action = gettext("Add a category") head_titles = [action] form = CategoryForm() form.set_feed_choices(FeedController(current_user.id).read()) if category_id is None: return render_template( "edit_category.html", action=action, head_titles=head_titles, form=form, ) category = CategoryController(current_user.id).get(id=category_id) action = gettext("Edit category") head_titles = [action] if category.name: head_titles.append(category.name) form = CategoryForm(obj=category) form.set_feed_choices(FeedController(current_user.id).read()) form.feeds.data = [feed.id for feed in category.feeds] return render_template( "edit_category.html", action=action, head_titles=head_titles, category=category, form=form, )
def bookmarklet(): feed_contr = FeedController(current_user.id) url = (request.args if request.method == "GET" else request.form).get( "url", None) if not url: flash(gettext("Couldn't add feed: url missing."), "error") raise BadRequest("url is missing") feed_exists = list(feed_contr.read(__or__={"link": url, "site_link": url})) if feed_exists: flash(gettext("Couldn't add feed: feed already exists."), "warning") return redirect(url_for("feed.form", feed_id=feed_exists[0].id)) try: feed = construct_feed_from(url) except requests.exceptions.ConnectionError: flash(gettext("Impossible to connect to the address: {}.".format(url)), "danger") return redirect(url_for("home")) except Exception: logger.exception("something bad happened when fetching %r", url) return redirect(url_for("home")) if not feed.get("link"): feed["enabled"] = False flash( gettext("Couldn't find a feed url, you'll need to find a Atom or" " RSS link manually and reactivate this feed"), "warning", ) feed = feed_contr.create(**feed) flash(gettext("Feed was successfully created."), "success") if feed.enabled and application.config["CRAWLING_METHOD"] == "default": misc_utils.fetch(current_user.id, feed.id) flash(gettext("Downloading articles for the new feed..."), "info") return redirect(url_for("feed.form", feed_id=feed.id))
def reset_errors(feed_id): feed_contr = FeedController(current_user.id) feed = feed_contr.get(id=feed_id) feed_contr.update({"id": feed_id}, {"error_count": 0, "last_error": ""}) flash( gettext("Feed %(feed_title)r successfully updated.", feed_title=feed.title), "success", ) return redirect(request.referrer or url_for("home"))
def delete(feed_id=None): feed_contr = FeedController(current_user.id) feed = feed_contr.get(id=feed_id) feed_contr.delete(feed_id) flash( gettext("Feed %(feed_title)s successfully deleted.", feed_title=feed.title), "success", ) return redirect(url_for("home"))
def form(feed_id=None): action = gettext("Add a feed") categories = CategoryController(current_user.id).read() head_titles = [action] if feed_id is None: form = AddFeedForm() form.set_category_choices(categories) return render_template("edit_feed.html", action=action, head_titles=head_titles, form=form) feed = FeedController(current_user.id).get(id=feed_id) form = AddFeedForm(obj=feed) form.set_category_choices(categories) action = gettext("Edit feed") head_titles = [action] if feed.title: head_titles.append(feed.title) return render_template( "edit_feed.html", action=action, head_titles=head_titles, categories=categories, form=form, feed=feed, )
def export(): """ Export feeds to OPML. """ include_disabled = request.args.get("includedisabled", "") == "on" include_private = request.args.get("includeprivate", "") == "on" include_exceeded_error_count = (request.args.get( "includeexceedederrorcount", "") == "on") filter = {} if not include_disabled: filter["enabled"] = True if not include_private: filter["private"] = False if not include_exceeded_error_count: filter["error_count__lt"] = application.config["DEFAULT_MAX_ERROR"] user = UserController(current_user.id).get(id=current_user.id) feeds = FeedController(current_user.id).read(**filter) categories = { cat.id: cat.dump() for cat in CategoryController(user.id).read() } response = make_response( render_template( "opml.xml", user=user, feeds=feeds, categories=categories, now=datetime.now(), )) response.headers["Content-Type"] = "application/xml" response.headers["Content-Disposition"] = "attachment; filename=feeds.opml" return response
def _articles_to_json(articles, fd_hash=None): now, locale = datetime.now(), get_locale() fd_hash = { feed.id: { "title": feed.title, "icon_url": url_for("icon.icon", url=feed.icon_url) if feed.icon_url else None, } for feed in FeedController(current_user.id).read() } return { "articles": [ { "title": art.title, "liked": art.like, "read": art.readed, "article_id": art.id, "selected": False, "feed_id": art.feed_id, "category_id": art.category_id or 0, "feed_title": fd_hash[art.feed_id]["title"] if fd_hash else None, "icon_url": fd_hash[art.feed_id]["icon_url"] if fd_hash else None, "date": format_datetime(localize(art.date), locale=locale), "rel_date": format_timedelta( art.date - now, threshold=1.1, add_direction=True, locale=locale ), } for art in articles.limit(1000) ] }
async def retrieve_feed(queue, users, feed_id=None): """ Launch the processus. """ for user in users: logger.info("Starting to retrieve feeds for {}".format(user.nickname)) filters = {} filters["user_id"] = user.id if feed_id is not None: filters["id"] = feed_id filters["enabled"] = True filters["error_count__lt"] = application.config["DEFAULT_MAX_ERROR"] filters["last_retrieved__lt"] = datetime.now() - timedelta( minutes=application.config["FEED_REFRESH_INTERVAL"] ) feeds = FeedController().read(**filters).all() if feeds == []: logger.info("No feed to retrieve for {}".format(user.nickname)) for feed in feeds: articles = await parse_feed(user, feed) await queue.put((user, feed, articles)) await queue.put(None)
def feed_pub(feed_id=None): """ Presents details of a pubic feed if the profile of the owner is also public. """ feed = FeedController(None).get(id=feed_id) if feed.private or not feed.user.is_public_profile: return render_template("errors/404.html"), 404 return feed_view(feed_id, None)
def feeds(): "Lists the subscribed feeds in a table." art_contr = ArticleController(current_user.id) return render_template( "feeds.html", feeds=FeedController(current_user.id).read().order_by("title"), unread_article_count=art_contr.count_by_feed(readed=False), article_count=art_contr.count_by_feed(), )
def inactives(): """ List of inactive feeds. """ nb_days = int(request.args.get("nb_days", 365)) inactives = FeedController(current_user.id).get_inactives(nb_days) return render_template("inactives.html", inactives=inactives, nb_days=nb_days)
def process_form(category_id=None): form = CategoryForm() form.set_feed_choices(FeedController(current_user.id).read()) cat_contr = CategoryController(current_user.id) feed_contr = FeedController(current_user.id) if not form.validate(): return render_template("edit_category.html", form=form) existing_cats = list(cat_contr.read(name=form.name.data)) if existing_cats and category_id is None: flash(gettext("Couldn't add category: already exists."), "warning") return redirect( url_for("category.form", category_id=existing_cats[0].id)) # Edit an existing category category_attr = {"name": form.name.data} if category_id is not None: cat_contr.update({"id": category_id}, category_attr) # Update the relation with feeds for feed_id in form.feeds.data: feed_contr.update({"id": feed_id}, {"category_id": category_id}) for feed in current_user.feeds: if feed.category_id == category_id and feed.id not in form.feeds.data: feed_contr.update({"id": feed.id}, {"category_id": None}) flash( gettext( "Category %(cat_name)r successfully updated.", cat_name=category_attr["name"], ), "success", ) return redirect(url_for("category.form", category_id=category_id)) # Create a new category new_category = cat_contr.create(**category_attr) # Update the relation with feeds for feed_id in form.feeds.data: feed_contr.update({"id": feed_id}, {"category_id": new_category.id}) flash( gettext( "Category %(category_name)r successfully created.", category_name=new_category.name, ), "success", ) return redirect(url_for("category.form", category_id=new_category.id))
def list_(): "Lists the subscribed feeds in a table." art_contr = ArticleController(current_user.id) return render_template( "categories.html", categories=list( CategoryController(current_user.id).read().order_by("name")), feeds_count=FeedController(current_user.id).count_by_category(), unread_article_count=art_contr.count_by_category(readed=False), article_count=art_contr.count_by_category(), )
def duplicates(feed_id): """ Return duplicates article for a feed. """ feed, duplicates = FeedController(current_user.id).get_duplicates(feed_id) if len(duplicates) == 0: flash( gettext('No duplicates in the feed "{}".').format(feed.title), "info") return redirect(url_for("home")) return render_template("duplicates.html", duplicates=duplicates, feed=feed)
def update(self, filters, attrs): user_id = attrs.get("user_id", self.user_id) if "feed_id" in attrs: feed = FeedController().get(id=attrs["feed_id"]) assert feed.user_id == user_id, "no right on feed %r" % feed.id attrs["category_id"] = feed.category_id if attrs.get("category_id"): cat = CategoryController().get(id=attrs["category_id"]) assert self.user_id is None or cat.user_id == user_id, ( "no right on cat %r" % cat.id) return super().update(filters, attrs)
def management(): """ Display the management page. """ if request.method == "POST": if None != request.files.get("opmlfile", None): # Import an OPML file data = request.files.get("opmlfile", None) if not misc_utils.allowed_file(data.filename): flash(gettext("File not allowed."), "danger") else: try: nb = import_opml(current_user.nickname, data.read()) if application.config["CRAWLING_METHOD"] == "classic": misc_utils.fetch(current_user.id, None) flash( str(nb) + " " + gettext("feeds imported."), "success") flash(gettext("Downloading articles..."), "info") except: flash(gettext("Impossible to import the new feeds."), "danger") elif None != request.files.get("jsonfile", None): # Import an account data = request.files.get("jsonfile", None) if not misc_utils.allowed_file(data.filename): flash(gettext("File not allowed."), "danger") else: try: nb = import_json(current_user.nickname, data.read()) flash(gettext("Account imported."), "success") except: flash(gettext("Impossible to import the account."), "danger") else: flash(gettext("File not allowed."), "danger") nb_feeds = FeedController(current_user.id).read().count() art_contr = ArticleController(current_user.id) nb_articles = art_contr.read().count() nb_unread_articles = art_contr.read(readed=False).count() nb_categories = CategoryController(current_user.id).read().count() nb_bookmarks = BookmarkController(current_user.id).read().count() return render_template( "management.html", user=current_user, nb_feeds=nb_feeds, nb_articles=nb_articles, nb_unread_articles=nb_unread_articles, nb_categories=nb_categories, nb_bookmarks=nb_bookmarks, )
def get_article(article_id, parse=False): locale = get_locale() contr = ArticleController(current_user.id) article = contr.get(id=article_id) if not article.readed: article["readed"] = True contr.update({"id": article_id}, {"readed": True}) article["category_id"] = article.category_id or 0 feed = FeedController(current_user.id).get(id=article.feed_id) article["icon_url"] = ( url_for("icon.icon", url=feed.icon_url) if feed.icon_url else None ) article["date"] = format_datetime(localize(article.date), locale=locale) return article
def create(self, **attrs): # handling special denorm for article rights assert "feed_id" in attrs, "must provide feed_id when creating article" feed = FeedController(attrs.get("user_id", self.user_id)).get(id=attrs["feed_id"]) if "user_id" in attrs: assert feed.user_id == attrs["user_id"] or self.user_id is None, ( "no right on feed %r" % feed.id) attrs["user_id"], attrs["category_id"] = feed.user_id, feed.category_id skipped, read, liked = process_filters(feed.filters, attrs) if skipped: return None article = super().create(**attrs) return article
def process_form(feed_id=None): form = AddFeedForm() feed_contr = FeedController(current_user.id) form.set_category_choices(CategoryController(current_user.id).read()) if not form.validate(): return render_template("edit_feed.html", form=form) existing_feeds = list(feed_contr.read(link=form.link.data)) if existing_feeds and feed_id is None: flash(gettext("Couldn't add feed: feed already exists."), "warning") return redirect(url_for("feed.form", feed_id=existing_feeds[0].id)) # Edit an existing feed feed_attr = { "title": form.title.data, "enabled": form.enabled.data, "link": form.link.data, "site_link": form.site_link.data, "filters": [], "category_id": form.category_id.data, "private": form.private.data, } if not feed_attr["category_id"] or feed_attr["category_id"] == "0": del feed_attr["category_id"] for filter_attr in ("type", "pattern", "action on", "action"): for i, value in enumerate( request.form.getlist(filter_attr.replace(" ", "_"))): if i >= len(feed_attr["filters"]): feed_attr["filters"].append({}) feed_attr["filters"][i][filter_attr] = value if feed_id is not None: feed_contr.update({"id": feed_id}, feed_attr) flash( gettext( "Feed %(feed_title)r successfully updated.", feed_title=feed_attr["title"], ), "success", ) return redirect(url_for("feed.form", feed_id=feed_id)) # Create a new feed new_feed = feed_contr.create(**feed_attr) flash( gettext("Feed %(feed_title)r successfully created.", feed_title=new_feed.title), "success", ) if application.config["CRAWLING_METHOD"] == "default": misc_utils.fetch(current_user.id, new_feed.id) flash(gettext("Downloading articles for the new feed..."), "info") return redirect(url_for("feed.form", feed_id=new_feed.id))
def feed_view(feed_id=None, user_id=None): feed = FeedController(user_id).get(id=feed_id) category = None if feed.category_id: category = CategoryController(user_id).get(id=feed.category_id) filters = {} filters["feed_id"] = feed_id articles = ArticleController(user_id).read_light(**filters) # Server-side pagination page, per_page, offset = get_page_args(per_page_parameter="per_page") pagination = Pagination( page=page, total=articles.count(), css_framework="bootstrap4", search=False, record_name="articles", per_page=per_page, ) today = datetime.now() try: last_article = articles[0].date first_article = articles[-1].date delta = last_article - first_article average = round(float(articles.count()) / abs(delta.days), 2) except Exception: last_article = datetime.fromtimestamp(0) first_article = datetime.fromtimestamp(0) delta = last_article - first_article average = 0 elapsed = today - last_article return render_template( "feed.html", head_titles=[utils.clear_string(feed.title)], feed=feed, category=category, articles=articles.offset(offset).limit(per_page), pagination=pagination, first_post_date=first_article, end_post_date=last_article, average=average, delta=delta, elapsed=elapsed, )
def user_stream(per_page, nickname=None): """ Display the stream of a user (list of articles of public feed). """ user_contr = UserController() user = user_contr.get(nickname=nickname) if not user.is_public_profile: if current_user.is_authenticated and current_user.id == user.id: flash(gettext("You must set your profile to public."), "info") return redirect(url_for("user.profile")) category_id = int(request.args.get("category_id", 0)) category = CategoryController().read(id=category_id).first() # Load the public feeds filters = {} filters["private"] = False if category_id: filters["category_id"] = category_id feeds = FeedController().read(**filters).all() # Re-initializes the filters to load the articles filters = {} filters["feed_id__in"] = [feed.id for feed in feeds] if category: filters["category_id"] = category_id articles = ArticleController(user.id).read_ordered(**filters) # Server-side pagination page, per_page, offset = get_page_args(per_page_parameter="per_page") pagination = Pagination( page=page, total=articles.count(), css_framework="bootstrap4", search=False, record_name="articles", per_page=per_page, ) return render_template( "user_stream.html", user=user, articles=articles.offset(offset).limit(per_page), category=category, pagination=pagination, )
def profile_public(nickname=None): """ Display the public profile of the user. """ category_id = int(request.args.get("category_id", 0)) user_contr = UserController() user = user_contr.get(nickname=nickname) if not user.is_public_profile: if current_user.is_authenticated and current_user.id == user.id: flash(gettext("You must set your profile to public."), "info") return redirect(url_for("user.profile")) filters = {} filters["private"] = False if category_id: filters["category_id"] = category_id feeds = FeedController(user.id).read(**filters) return render_template("profile_public.html", user=user, feeds=feeds, selected_category_id=category_id)
def process_form(feed_id=None): form = AddFeedForm() feed_contr = FeedController(current_user.id) form.set_category_choices(CategoryController(current_user.id).read()) if not form.validate(): return render_template("edit_feed.html", form=form) existing_feeds = list( feed_contr.read(link=form.link.data, site_link=form.site_link.data)) if existing_feeds and feed_id is None: flash(gettext("Couldn't add feed: feed already exists."), "warning") return redirect(url_for("feed.form", feed_id=existing_feeds[0].id)) feed_attr = { "title": form.title.data, "enabled": form.enabled.data, "link": form.link.data, "site_link": form.site_link.data, "filters": [], "category_id": form.category_id.data, "private": form.private.data, } if not feed_attr["category_id"] or feed_attr["category_id"] == "0": del feed_attr["category_id"] for filter_attr in ("type", "pattern", "action on", "action"): for i, value in enumerate( request.form.getlist(filter_attr.replace(" ", "_"))): if i >= len(feed_attr["filters"]): feed_attr["filters"].append({}) feed_attr["filters"][i][filter_attr] = value if feed_id is not None: # Edit an existing feed feed_contr.update({"id": feed_id}, feed_attr) flash( gettext( "Feed %(feed_title)r successfully updated.", feed_title=feed_attr["title"], ), "success", ) return redirect(url_for("feed.form", feed_id=feed_id)) # Create a new feed url = form.site_link.data if form.site_link.data else form.link.data try: feed = construct_feed_from(url) except requests.exceptions.ConnectionError: flash(gettext("Impossible to connect to the address: {}.".format(url)), "danger") return redirect(url_for("home")) except Exception: logger.exception("something bad happened when fetching %r", url) return redirect(url_for("home")) del feed_attr["link"] del feed_attr["site_link"] # remove keys with empty strings feed_attr = {k: v for k, v in feed_attr.items() if v != ""} feed.update(feed_attr) new_feed = feed_contr.create(**feed) flash( gettext("Feed %(feed_title)r successfully created.", feed_title=new_feed.title), "success", ) if application.config["CRAWLING_METHOD"] == "default": misc_utils.fetch(current_user.id, new_feed.id) flash(gettext("Downloading articles for the new feed..."), "info") return redirect(url_for("feed.form", feed_id=new_feed.id))
def home(): """Displays the home page of the connected user. """ filters = _get_filters(request.args) art_contr = ArticleController(current_user.id) unread = art_contr.count_by_feed(readed=False) nb_unread = art_contr.read_light(readed=False).count() feeds = { feed.id: feed for feed in sorted( current_user.feeds, key=lambda x: x.title.lower(), reverse=False ) } filter_ = request.args.get("filter_", "unread") feed_id = int(request.args.get("feed", 0)) liked = int(request.args.get("liked", 0)) == 1 limit = request.args.get("limit", 1000) query = request.args.get("query", "") search_title = request.args.get("search_title", "off") search_content = request.args.get("search_content", "off") if filter_ in ["read", "unread"]: filters["readed"] = filter_ == "read" if feed_id: filters["feed_id"] = feed_id if liked: filters["like"] = int(liked) == 1 articles = art_contr.read_ordered(**filters) if limit != "all": limit = int(limit) articles = articles.limit(limit) in_error = { feed.id: feed.error_count for feed in FeedController(current_user.id).read(error_count__gt=0).all() } def gen_url( filter_=filter_, limit=limit, feed=feed_id, liked=liked, query=query, search_title=search_title, search_content=search_content, ): return ( "?filter_=%s&limit=%s&feed=%d&liked=%s&query=%s&search_title=%s&search_content=%s" % ( filter_, limit, feed, 1 if liked else 0, query, search_title, search_content, ) ) return render_template( "home.html", nb_unread=nb_unread, gen_url=gen_url, feed_id=feed_id, filter_=filter_, limit=limit, feeds=feeds, liked=liked, unread=dict(unread), articles=articles.all(), in_error=in_error, )