def popular():
"""
Return the most popular feeds for the last nb_days days.
"""
# try to get the 'recent' popular websites, created after
# 'not_created_before'
# ie: not_added_before = date_last_added_feed - nb_days
try:
nb_days = int(request.args.get("nb_days", 365))
except ValueError:
nb_days = 10000
last_added_feed = (
FeedController().read().order_by(desc("created_date")).limit(1).all()
)
if last_added_feed:
date_last_added_feed = last_added_feed[0].created_date
else:
date_last_added_feed = datetime.now()
not_added_before = date_last_added_feed - timedelta(days=nb_days)
filters = {}
filters["created_date__gt"] = not_added_before
filters["private"] = False
filters["error_count__lt"] = application.config["DEFAULT_MAX_ERROR"]
feeds = FeedController().count_by_link(**filters)
sorted_feeds = sorted(list(feeds.items()), key=operator.itemgetter(1), reverse=True)
return render_template("popular.html", popular=sorted_feeds)
async def parse_feed(user, feed):
"""
Fetch a feed.
Update the feed and return the articles.
"""
parsed_feed = None
up_feed = {}
articles = []
resp = None
# with (await sem):
try:
logger.info("Retrieving feed {}".format(feed.link))
resp = newspipe_get(feed.link, timeout=5)
except Exception:
logger.info("Problem when reading feed {}".format(feed.link))
return
finally:
if None is resp:
return
try:
content = io.BytesIO(resp.content)
parsed_feed = feedparser.parse(content)
except Exception as e:
up_feed["last_error"] = str(e)
up_feed["error_count"] = feed.error_count + 1
logger.exception("error when parsing feed: " + str(e))
finally:
up_feed["last_retrieved"] = datetime.now(dateutil.tz.tzlocal())
if parsed_feed is None:
try:
FeedController().update({"id": feed.id}, up_feed)
except Exception as e:
logger.exception("something bad here: " + str(e))
return
if not is_parsing_ok(parsed_feed):
up_feed["last_error"] = str(parsed_feed["bozo_exception"])
up_feed["error_count"] = feed.error_count + 1
FeedController().update({"id": feed.id}, up_feed)
return
if parsed_feed["entries"] != []:
articles = parsed_feed["entries"]
up_feed["error_count"] = 0
up_feed["last_error"] = ""
# Feed information
try:
up_feed.update(construct_feed_from(feed.link, parsed_feed))
except:
logger.exception("error when constructing feed: {}".format(feed.link))
if feed.title and "title" in up_feed:
# do not override the title set by the user
del up_feed["title"]
try:
FeedController().update({"id": feed.id}, up_feed)
except:
logger.exception("error when updating feed: {}".format(feed.link))
return articles
def form(category_id=None):
action = gettext("Add a category")
head_titles = [action]
form = CategoryForm()
form.set_feed_choices(FeedController(current_user.id).read())
if category_id is None:
return render_template(
"edit_category.html",
action=action,
head_titles=head_titles,
form=form,
)
category = CategoryController(current_user.id).get(id=category_id)
action = gettext("Edit category")
head_titles = [action]
if category.name:
head_titles.append(category.name)
form = CategoryForm(obj=category)
form.set_feed_choices(FeedController(current_user.id).read())
form.feeds.data = [feed.id for feed in category.feeds]
return render_template(
"edit_category.html",
action=action,
head_titles=head_titles,
category=category,
form=form,
)
def bookmarklet():
feed_contr = FeedController(current_user.id)
url = (request.args if request.method == "GET" else request.form).get(
"url", None)
if not url:
flash(gettext("Couldn't add feed: url missing."), "error")
raise BadRequest("url is missing")
feed_exists = list(feed_contr.read(__or__={"link": url, "site_link": url}))
if feed_exists:
flash(gettext("Couldn't add feed: feed already exists."), "warning")
return redirect(url_for("feed.form", feed_id=feed_exists[0].id))
try:
feed = construct_feed_from(url)
except requests.exceptions.ConnectionError:
flash(gettext("Impossible to connect to the address: {}.".format(url)),
"danger")
return redirect(url_for("home"))
except Exception:
logger.exception("something bad happened when fetching %r", url)
return redirect(url_for("home"))
if not feed.get("link"):
feed["enabled"] = False
flash(
gettext("Couldn't find a feed url, you'll need to find a Atom or"
" RSS link manually and reactivate this feed"),
"warning",
)
feed = feed_contr.create(**feed)
flash(gettext("Feed was successfully created."), "success")
if feed.enabled and application.config["CRAWLING_METHOD"] == "default":
misc_utils.fetch(current_user.id, feed.id)
flash(gettext("Downloading articles for the new feed..."), "info")
return redirect(url_for("feed.form", feed_id=feed.id))
def reset_errors(feed_id):
feed_contr = FeedController(current_user.id)
feed = feed_contr.get(id=feed_id)
feed_contr.update({"id": feed_id}, {"error_count": 0, "last_error": ""})
flash(
gettext("Feed %(feed_title)r successfully updated.",
feed_title=feed.title),
"success",
)
return redirect(request.referrer or url_for("home"))
def delete(feed_id=None):
feed_contr = FeedController(current_user.id)
feed = feed_contr.get(id=feed_id)
feed_contr.delete(feed_id)
flash(
gettext("Feed %(feed_title)s successfully deleted.",
feed_title=feed.title),
"success",
)
return redirect(url_for("home"))
def form(feed_id=None):
action = gettext("Add a feed")
categories = CategoryController(current_user.id).read()
head_titles = [action]
if feed_id is None:
form = AddFeedForm()
form.set_category_choices(categories)
return render_template("edit_feed.html",
action=action,
head_titles=head_titles,
form=form)
feed = FeedController(current_user.id).get(id=feed_id)
form = AddFeedForm(obj=feed)
form.set_category_choices(categories)
action = gettext("Edit feed")
head_titles = [action]
if feed.title:
head_titles.append(feed.title)
return render_template(
"edit_feed.html",
action=action,
head_titles=head_titles,
categories=categories,
form=form,
feed=feed,
)
def export():
"""
Export feeds to OPML.
"""
include_disabled = request.args.get("includedisabled", "") == "on"
include_private = request.args.get("includeprivate", "") == "on"
include_exceeded_error_count = (request.args.get(
"includeexceedederrorcount", "") == "on")
filter = {}
if not include_disabled:
filter["enabled"] = True
if not include_private:
filter["private"] = False
if not include_exceeded_error_count:
filter["error_count__lt"] = application.config["DEFAULT_MAX_ERROR"]
user = UserController(current_user.id).get(id=current_user.id)
feeds = FeedController(current_user.id).read(**filter)
categories = {
cat.id: cat.dump()
for cat in CategoryController(user.id).read()
}
response = make_response(
render_template(
"opml.xml",
user=user,
feeds=feeds,
categories=categories,
now=datetime.now(),
))
response.headers["Content-Type"] = "application/xml"
response.headers["Content-Disposition"] = "attachment; filename=feeds.opml"
return response
def _articles_to_json(articles, fd_hash=None):
now, locale = datetime.now(), get_locale()
fd_hash = {
feed.id: {
"title": feed.title,
"icon_url": url_for("icon.icon", url=feed.icon_url)
if feed.icon_url
else None,
}
for feed in FeedController(current_user.id).read()
}
return {
"articles": [
{
"title": art.title,
"liked": art.like,
"read": art.readed,
"article_id": art.id,
"selected": False,
"feed_id": art.feed_id,
"category_id": art.category_id or 0,
"feed_title": fd_hash[art.feed_id]["title"] if fd_hash else None,
"icon_url": fd_hash[art.feed_id]["icon_url"] if fd_hash else None,
"date": format_datetime(localize(art.date), locale=locale),
"rel_date": format_timedelta(
art.date - now, threshold=1.1, add_direction=True, locale=locale
),
}
for art in articles.limit(1000)
]
}
async def retrieve_feed(queue, users, feed_id=None):
"""
Launch the processus.
"""
for user in users:
logger.info("Starting to retrieve feeds for {}".format(user.nickname))
filters = {}
filters["user_id"] = user.id
if feed_id is not None:
filters["id"] = feed_id
filters["enabled"] = True
filters["error_count__lt"] = application.config["DEFAULT_MAX_ERROR"]
filters["last_retrieved__lt"] = datetime.now() - timedelta(
minutes=application.config["FEED_REFRESH_INTERVAL"]
)
feeds = FeedController().read(**filters).all()
if feeds == []:
logger.info("No feed to retrieve for {}".format(user.nickname))
for feed in feeds:
articles = await parse_feed(user, feed)
await queue.put((user, feed, articles))
await queue.put(None)
def feed_pub(feed_id=None):
"""
Presents details of a pubic feed if the profile of the owner is also
public.
"""
feed = FeedController(None).get(id=feed_id)
if feed.private or not feed.user.is_public_profile:
return render_template("errors/404.html"), 404
return feed_view(feed_id, None)
def feeds():
"Lists the subscribed feeds in a table."
art_contr = ArticleController(current_user.id)
return render_template(
"feeds.html",
feeds=FeedController(current_user.id).read().order_by("title"),
unread_article_count=art_contr.count_by_feed(readed=False),
article_count=art_contr.count_by_feed(),
)
def inactives():
"""
List of inactive feeds.
"""
nb_days = int(request.args.get("nb_days", 365))
inactives = FeedController(current_user.id).get_inactives(nb_days)
return render_template("inactives.html",
inactives=inactives,
nb_days=nb_days)
def process_form(category_id=None):
form = CategoryForm()
form.set_feed_choices(FeedController(current_user.id).read())
cat_contr = CategoryController(current_user.id)
feed_contr = FeedController(current_user.id)
if not form.validate():
return render_template("edit_category.html", form=form)
existing_cats = list(cat_contr.read(name=form.name.data))
if existing_cats and category_id is None:
flash(gettext("Couldn't add category: already exists."), "warning")
return redirect(
url_for("category.form", category_id=existing_cats[0].id))
# Edit an existing category
category_attr = {"name": form.name.data}
if category_id is not None:
cat_contr.update({"id": category_id}, category_attr)
# Update the relation with feeds
for feed_id in form.feeds.data:
feed_contr.update({"id": feed_id}, {"category_id": category_id})
for feed in current_user.feeds:
if feed.category_id == category_id and feed.id not in form.feeds.data:
feed_contr.update({"id": feed.id}, {"category_id": None})
flash(
gettext(
"Category %(cat_name)r successfully updated.",
cat_name=category_attr["name"],
),
"success",
)
return redirect(url_for("category.form", category_id=category_id))
# Create a new category
new_category = cat_contr.create(**category_attr)
# Update the relation with feeds
for feed_id in form.feeds.data:
feed_contr.update({"id": feed_id}, {"category_id": new_category.id})
flash(
gettext(
"Category %(category_name)r successfully created.",
category_name=new_category.name,
),
"success",
)
return redirect(url_for("category.form", category_id=new_category.id))
def list_():
"Lists the subscribed feeds in a table."
art_contr = ArticleController(current_user.id)
return render_template(
"categories.html",
categories=list(
CategoryController(current_user.id).read().order_by("name")),
feeds_count=FeedController(current_user.id).count_by_category(),
unread_article_count=art_contr.count_by_category(readed=False),
article_count=art_contr.count_by_category(),
)
def duplicates(feed_id):
"""
Return duplicates article for a feed.
"""
feed, duplicates = FeedController(current_user.id).get_duplicates(feed_id)
if len(duplicates) == 0:
flash(
gettext('No duplicates in the feed "{}".').format(feed.title),
"info")
return redirect(url_for("home"))
return render_template("duplicates.html", duplicates=duplicates, feed=feed)
def update(self, filters, attrs):
user_id = attrs.get("user_id", self.user_id)
if "feed_id" in attrs:
feed = FeedController().get(id=attrs["feed_id"])
assert feed.user_id == user_id, "no right on feed %r" % feed.id
attrs["category_id"] = feed.category_id
if attrs.get("category_id"):
cat = CategoryController().get(id=attrs["category_id"])
assert self.user_id is None or cat.user_id == user_id, (
"no right on cat %r" % cat.id)
return super().update(filters, attrs)
def management():
"""
Display the management page.
"""
if request.method == "POST":
if None != request.files.get("opmlfile", None):
# Import an OPML file
data = request.files.get("opmlfile", None)
if not misc_utils.allowed_file(data.filename):
flash(gettext("File not allowed."), "danger")
else:
try:
nb = import_opml(current_user.nickname, data.read())
if application.config["CRAWLING_METHOD"] == "classic":
misc_utils.fetch(current_user.id, None)
flash(
str(nb) + " " + gettext("feeds imported."),
"success")
flash(gettext("Downloading articles..."), "info")
except:
flash(gettext("Impossible to import the new feeds."),
"danger")
elif None != request.files.get("jsonfile", None):
# Import an account
data = request.files.get("jsonfile", None)
if not misc_utils.allowed_file(data.filename):
flash(gettext("File not allowed."), "danger")
else:
try:
nb = import_json(current_user.nickname, data.read())
flash(gettext("Account imported."), "success")
except:
flash(gettext("Impossible to import the account."),
"danger")
else:
flash(gettext("File not allowed."), "danger")
nb_feeds = FeedController(current_user.id).read().count()
art_contr = ArticleController(current_user.id)
nb_articles = art_contr.read().count()
nb_unread_articles = art_contr.read(readed=False).count()
nb_categories = CategoryController(current_user.id).read().count()
nb_bookmarks = BookmarkController(current_user.id).read().count()
return render_template(
"management.html",
user=current_user,
nb_feeds=nb_feeds,
nb_articles=nb_articles,
nb_unread_articles=nb_unread_articles,
nb_categories=nb_categories,
nb_bookmarks=nb_bookmarks,
)
def get_article(article_id, parse=False):
locale = get_locale()
contr = ArticleController(current_user.id)
article = contr.get(id=article_id)
if not article.readed:
article["readed"] = True
contr.update({"id": article_id}, {"readed": True})
article["category_id"] = article.category_id or 0
feed = FeedController(current_user.id).get(id=article.feed_id)
article["icon_url"] = (
url_for("icon.icon", url=feed.icon_url) if feed.icon_url else None
)
article["date"] = format_datetime(localize(article.date), locale=locale)
return article
def create(self, **attrs):
# handling special denorm for article rights
assert "feed_id" in attrs, "must provide feed_id when creating article"
feed = FeedController(attrs.get("user_id",
self.user_id)).get(id=attrs["feed_id"])
if "user_id" in attrs:
assert feed.user_id == attrs["user_id"] or self.user_id is None, (
"no right on feed %r" % feed.id)
attrs["user_id"], attrs["category_id"] = feed.user_id, feed.category_id
skipped, read, liked = process_filters(feed.filters, attrs)
if skipped:
return None
article = super().create(**attrs)
return article
def process_form(feed_id=None):
form = AddFeedForm()
feed_contr = FeedController(current_user.id)
form.set_category_choices(CategoryController(current_user.id).read())
if not form.validate():
return render_template("edit_feed.html", form=form)
existing_feeds = list(feed_contr.read(link=form.link.data))
if existing_feeds and feed_id is None:
flash(gettext("Couldn't add feed: feed already exists."), "warning")
return redirect(url_for("feed.form", feed_id=existing_feeds[0].id))
# Edit an existing feed
feed_attr = {
"title": form.title.data,
"enabled": form.enabled.data,
"link": form.link.data,
"site_link": form.site_link.data,
"filters": [],
"category_id": form.category_id.data,
"private": form.private.data,
}
if not feed_attr["category_id"] or feed_attr["category_id"] == "0":
del feed_attr["category_id"]
for filter_attr in ("type", "pattern", "action on", "action"):
for i, value in enumerate(
request.form.getlist(filter_attr.replace(" ", "_"))):
if i >= len(feed_attr["filters"]):
feed_attr["filters"].append({})
feed_attr["filters"][i][filter_attr] = value
if feed_id is not None:
feed_contr.update({"id": feed_id}, feed_attr)
flash(
gettext(
"Feed %(feed_title)r successfully updated.",
feed_title=feed_attr["title"],
),
"success",
)
return redirect(url_for("feed.form", feed_id=feed_id))
# Create a new feed
new_feed = feed_contr.create(**feed_attr)
flash(
gettext("Feed %(feed_title)r successfully created.",
feed_title=new_feed.title),
"success",
)
if application.config["CRAWLING_METHOD"] == "default":
misc_utils.fetch(current_user.id, new_feed.id)
flash(gettext("Downloading articles for the new feed..."), "info")
return redirect(url_for("feed.form", feed_id=new_feed.id))
def feed_view(feed_id=None, user_id=None):
feed = FeedController(user_id).get(id=feed_id)
category = None
if feed.category_id:
category = CategoryController(user_id).get(id=feed.category_id)
filters = {}
filters["feed_id"] = feed_id
articles = ArticleController(user_id).read_light(**filters)
# Server-side pagination
page, per_page, offset = get_page_args(per_page_parameter="per_page")
pagination = Pagination(
page=page,
total=articles.count(),
css_framework="bootstrap4",
search=False,
record_name="articles",
per_page=per_page,
)
today = datetime.now()
try:
last_article = articles[0].date
first_article = articles[-1].date
delta = last_article - first_article
average = round(float(articles.count()) / abs(delta.days), 2)
except Exception:
last_article = datetime.fromtimestamp(0)
first_article = datetime.fromtimestamp(0)
delta = last_article - first_article
average = 0
elapsed = today - last_article
return render_template(
"feed.html",
head_titles=[utils.clear_string(feed.title)],
feed=feed,
category=category,
articles=articles.offset(offset).limit(per_page),
pagination=pagination,
first_post_date=first_article,
end_post_date=last_article,
average=average,
delta=delta,
elapsed=elapsed,
)
def user_stream(per_page, nickname=None):
"""
Display the stream of a user (list of articles of public feed).
"""
user_contr = UserController()
user = user_contr.get(nickname=nickname)
if not user.is_public_profile:
if current_user.is_authenticated and current_user.id == user.id:
flash(gettext("You must set your profile to public."), "info")
return redirect(url_for("user.profile"))
category_id = int(request.args.get("category_id", 0))
category = CategoryController().read(id=category_id).first()
# Load the public feeds
filters = {}
filters["private"] = False
if category_id:
filters["category_id"] = category_id
feeds = FeedController().read(**filters).all()
# Re-initializes the filters to load the articles
filters = {}
filters["feed_id__in"] = [feed.id for feed in feeds]
if category:
filters["category_id"] = category_id
articles = ArticleController(user.id).read_ordered(**filters)
# Server-side pagination
page, per_page, offset = get_page_args(per_page_parameter="per_page")
pagination = Pagination(
page=page,
total=articles.count(),
css_framework="bootstrap4",
search=False,
record_name="articles",
per_page=per_page,
)
return render_template(
"user_stream.html",
user=user,
articles=articles.offset(offset).limit(per_page),
category=category,
pagination=pagination,
)
def profile_public(nickname=None):
"""
Display the public profile of the user.
"""
category_id = int(request.args.get("category_id", 0))
user_contr = UserController()
user = user_contr.get(nickname=nickname)
if not user.is_public_profile:
if current_user.is_authenticated and current_user.id == user.id:
flash(gettext("You must set your profile to public."), "info")
return redirect(url_for("user.profile"))
filters = {}
filters["private"] = False
if category_id:
filters["category_id"] = category_id
feeds = FeedController(user.id).read(**filters)
return render_template("profile_public.html",
user=user,
feeds=feeds,
selected_category_id=category_id)
def process_form(feed_id=None):
form = AddFeedForm()
feed_contr = FeedController(current_user.id)
form.set_category_choices(CategoryController(current_user.id).read())
if not form.validate():
return render_template("edit_feed.html", form=form)
existing_feeds = list(
feed_contr.read(link=form.link.data, site_link=form.site_link.data))
if existing_feeds and feed_id is None:
flash(gettext("Couldn't add feed: feed already exists."), "warning")
return redirect(url_for("feed.form", feed_id=existing_feeds[0].id))
feed_attr = {
"title": form.title.data,
"enabled": form.enabled.data,
"link": form.link.data,
"site_link": form.site_link.data,
"filters": [],
"category_id": form.category_id.data,
"private": form.private.data,
}
if not feed_attr["category_id"] or feed_attr["category_id"] == "0":
del feed_attr["category_id"]
for filter_attr in ("type", "pattern", "action on", "action"):
for i, value in enumerate(
request.form.getlist(filter_attr.replace(" ", "_"))):
if i >= len(feed_attr["filters"]):
feed_attr["filters"].append({})
feed_attr["filters"][i][filter_attr] = value
if feed_id is not None:
# Edit an existing feed
feed_contr.update({"id": feed_id}, feed_attr)
flash(
gettext(
"Feed %(feed_title)r successfully updated.",
feed_title=feed_attr["title"],
),
"success",
)
return redirect(url_for("feed.form", feed_id=feed_id))
# Create a new feed
url = form.site_link.data if form.site_link.data else form.link.data
try:
feed = construct_feed_from(url)
except requests.exceptions.ConnectionError:
flash(gettext("Impossible to connect to the address: {}.".format(url)),
"danger")
return redirect(url_for("home"))
except Exception:
logger.exception("something bad happened when fetching %r", url)
return redirect(url_for("home"))
del feed_attr["link"]
del feed_attr["site_link"]
# remove keys with empty strings
feed_attr = {k: v for k, v in feed_attr.items() if v != ""}
feed.update(feed_attr)
new_feed = feed_contr.create(**feed)
flash(
gettext("Feed %(feed_title)r successfully created.",
feed_title=new_feed.title),
"success",
)
if application.config["CRAWLING_METHOD"] == "default":
misc_utils.fetch(current_user.id, new_feed.id)
flash(gettext("Downloading articles for the new feed..."), "info")
return redirect(url_for("feed.form", feed_id=new_feed.id))
def home():
"""Displays the home page of the connected user.
"""
filters = _get_filters(request.args)
art_contr = ArticleController(current_user.id)
unread = art_contr.count_by_feed(readed=False)
nb_unread = art_contr.read_light(readed=False).count()
feeds = {
feed.id: feed
for feed in sorted(
current_user.feeds, key=lambda x: x.title.lower(), reverse=False
)
}
filter_ = request.args.get("filter_", "unread")
feed_id = int(request.args.get("feed", 0))
liked = int(request.args.get("liked", 0)) == 1
limit = request.args.get("limit", 1000)
query = request.args.get("query", "")
search_title = request.args.get("search_title", "off")
search_content = request.args.get("search_content", "off")
if filter_ in ["read", "unread"]:
filters["readed"] = filter_ == "read"
if feed_id:
filters["feed_id"] = feed_id
if liked:
filters["like"] = int(liked) == 1
articles = art_contr.read_ordered(**filters)
if limit != "all":
limit = int(limit)
articles = articles.limit(limit)
in_error = {
feed.id: feed.error_count
for feed in FeedController(current_user.id).read(error_count__gt=0).all()
}
def gen_url(
filter_=filter_,
limit=limit,
feed=feed_id,
liked=liked,
query=query,
search_title=search_title,
search_content=search_content,
):
return (
"?filter_=%s&limit=%s&feed=%d&liked=%s&query=%s&search_title=%s&search_content=%s"
% (
filter_,
limit,
feed,
1 if liked else 0,
query,
search_title,
search_content,
)
)
return render_template(
"home.html",
nb_unread=nb_unread,
gen_url=gen_url,
feed_id=feed_id,
filter_=filter_,
limit=limit,
feeds=feeds,
liked=liked,
unread=dict(unread),
articles=articles.all(),
in_error=in_error,
)
