def test_article_get_unread(self):
self.assertEquals({
1: 3,
2: 3,
3: 3
},
ArticleController(2).count_by_feed(readed=False))
self.assertEquals({
4: 3,
5: 3,
6: 3
},
ArticleController(3).count_by_feed(readed=False))
def history(year=None, month=None):
cntr, artcles = ArticleController(current_user.id).get_history(year, month)
return render_template('history.html',
articles_counter=cntr,
articles=artcles,
year=year,
month=month)
def populate_db():
db_create()
ucontr = UserController()
ccontr = CategoryController()
fcontr = FeedController()
acontr = ArticleController()
ccontr = CategoryController()
user1, user2 = [
ucontr.create(login=name, email="*****@*****.**" % name, password=name)
for name in ["user1", "user2"]
]
article_total = 0
for user in (user1, user2):
for i in range(3):
cat_id = None
if i:
cat_id = ccontr.create(user_id=user.id,
name="category%d" % i).id
feed = fcontr.create(link="feed%d" % i,
user_id=user.id,
category_id=cat_id,
title="%s feed%d" % (user.login, i))
for j in range(3):
entry = "%s %s article%d" % (user.login, feed.title, j)
article_total += 1
acontr.create(entry_id=entry,
link='http://test.te/%d' % article_total,
feed_id=feed.id,
user_id=user.id,
category_id=cat_id,
title=entry,
content="content %d" % article_total)
def delete(article_id=None):
"Delete an article from the database."
article = ArticleController(current_user.id).delete(article_id)
flash(
gettext('Article %(article_title)s deleted',
article_title=article.title), 'success')
return redirect(url_for('home'))
def feeds():
"Lists the subscribed feeds in a table."
art_contr = ArticleController(current_user.id)
return render_template('feeds.html',
feeds=FeedController(current_user.id).read(),
unread_article_count=art_contr.count_by_feed(readed=False),
article_count=art_contr.count_by_feed())
def get_menu():
categories_order = [0]
categories = {0: {'name': 'No category', 'id': 0}}
for cat in CategoryController(g.user.id).read().order_by('name'):
categories_order.append(cat.id)
categories[cat.id] = cat.dump()
unread = ArticleController(g.user.id).count_by_feed(readed=False)
for cat_id in categories:
categories[cat_id]['unread'] = 0
categories[cat_id]['feeds'] = []
feeds = {feed.id: feed.dump() for feed in FeedController(g.user.id).read()}
for feed_id, feed in feeds.items():
feed['created_stamp'] = timegm(feed['created_date'].timetuple()) * 1000
feed['last_stamp'] = timegm(feed['last_retrieved'].timetuple()) * 1000
feed['category_id'] = feed['category_id'] or 0
feed['unread'] = unread.get(feed['id'], 0)
if not feed['filters']:
feed['filters'] = []
if feed.get('icon_url'):
feed['icon_url'] = url_for('icon.icon', url=feed['icon_url'])
categories[feed['category_id']]['unread'] += feed['unread']
categories[feed['category_id']]['feeds'].append(feed_id)
return jsonify(
**{
'feeds': feeds,
'categories': categories,
'categories_order': categories_order,
'crawling_method': conf.CRAWLING_METHOD,
'max_error': conf.DEFAULT_MAX_ERROR,
'error_threshold': conf.ERROR_THRESHOLD,
'is_admin': g.user.is_admin(),
'all_unread_count': sum(unread.values())
})
def home():
"""Displays the home page of the connected user.
"""
art_contr = ArticleController(current_user.id)
unread = art_contr.count_by_feed(readed=False)
nb_unread = art_contr.read_light(readed=False).count()
feeds = {
feed.id: feed
for feed in sorted(
current_user.feeds, key=lambda x: x.title.lower(), reverse=False
)
}
filter_ = request.args.get("filter_", "unread")
feed_id = int(request.args.get("feed", 0))
liked = int(request.args.get("liked", 0)) == 1
limit = request.args.get("limit", 1000)
filters = {}
if filter_ in ["read", "unread"]:
filters["readed"] = filter_ == "read"
if feed_id:
filters["feed_id"] = feed_id
if liked:
filters["like"] = int(liked) == 1
articles = art_contr.read_ordered(**filters)
if limit != "all":
limit = int(limit)
articles = articles.limit(limit)
in_error = {
feed.id: feed.error_count
for feed in FeedController(current_user.id).read(error_count__gt=0).all()
}
def gen_url(filter_=filter_, limit=limit, feed=feed_id, liked=liked):
return "?filter_=%s&limit=%s&feed=%d&liked=%s" % (
filter_,
limit,
feed,
1 if liked else 0,
)
return render_template(
"home.html",
nb_unread=nb_unread,
gen_url=gen_url,
feed_id=feed_id,
filter_=filter_,
limit=limit,
feeds=feeds,
liked=liked,
unread=dict(unread),
articles=articles.all(),
in_error=in_error,
)
def get_article(article_id, parse=False):
locale = get_locale()
contr = ArticleController(current_user.id)
article = contr.get(id=article_id)
if not article.readed:
article['readed'] = True
contr.update({'id': article_id}, {'readed': True})
article['category_id'] = article.category_id or 0
feed = FeedController(current_user.id).get(id=article.feed_id)
article['icon_url'] = url_for('icon.icon', url=feed.icon_url) \
if feed.icon_url else None
readability_available = bool(current_user.readability_key
or conf.PLUGINS_READABILITY_KEY)
article['date'] = format_datetime(localize(article.date), locale=locale)
article['readability_available'] = readability_available
if parse or (not article.readability_parsed and feed.readability_auto_parse
and readability_available):
try:
new_content = readability.parse(
article.link, current_user.readability_key
or conf.PLUGINS_READABILITY_KEY)
except Exception as error:
flash("Readability failed with %r" % error, "error")
article['readability_parsed'] = False
else:
article['readability_parsed'] = True
article['content'] = clean_urls(new_content, article['link'])
new_attr = {'readability_parsed': True, 'content': new_content}
contr.update({'id': article['id']}, new_attr)
return article
def article(article_id=None):
"""
Presents an article.
"""
article = ArticleController(current_user.id).get(id=article_id)
return render_template('article.html',
head_titles=[clear_string(article.title)],
article=article)
def test_feed_rights(self):
cat = CategoryController(2).read()[0].dump()
self.assertTrue(3,
ArticleController().read(category_id=cat['id']).count())
self.assertTrue(3,
FeedController().read(category_id=cat['id']).count())
self._test_controller_rights(cat,
UserController().get(id=cat['user_id']))
def test_feed_and_article_deletion(self):
ccontr = CategoryController(2)
cat = ccontr.read()[0].dump()
ccontr.delete(cat['id'])
self.assertEquals(0,
ArticleController().read(category_id=cat['id']).count())
self.assertEquals(0,
FeedController().read(category_id=cat['id']).count())
def like(article_id=None):
"""
Mark or unmark an article as favorites.
"""
art_contr = ArticleController(current_user.id)
article = art_contr.get(id=article_id)
art_contr = art_contr.update({'id': article_id},
{'like': not article.like})
return redirect(redirect_url())
def list_():
"Lists the subscribed feeds in a table."
art_contr = ArticleController(current_user.id)
return render_template(
'categories.html',
categories=list(
CategoryController(current_user.id).read().order_by('name')),
feeds_count=FeedController(current_user.id).count_by_category(),
unread_article_count=art_contr.count_by_category(readed=False),
article_count=art_contr.count_by_category())
def get_article(article_id):
contr = ArticleController(g.user.id)
article = contr.get(id=article_id).dump()
if not article['readed']:
contr.update({'id': article_id}, {'readed': True})
article['category_id'] = article['category_id'] or 0
feed = FeedController(g.user.id).get(id=article['feed_id'])
article['icon_url'] = url_for('icon.icon', url=feed.icon_url) \
if feed.icon_url else None
return jsonify(**article)
def delete(article_id=None):
"""
Delete an article from the database.
"""
article = ArticleController(current_user.id).delete(article_id)
flash(
gettext("Article %(article_title)s deleted",
article_title=article.title),
"success",
)
return redirect(url_for("home"))
def article_pub(article_id=None):
"""
Presents an article of a public feed if the profile of the owner is also
public.
"""
article = ArticleController().get(id=article_id)
if article.source.private or not article.source.user.is_public_profile:
return render_template('errors/404.html'), 404
return render_template('article_pub.html',
head_titles=[clear_string(article.title)],
article=article)
def article(article_id=None):
"""
Presents an article.
"""
art_contr = ArticleController(current_user.id)
article = art_contr.get(id=article_id)
if not article.readed:
art_contr.update({"id": article.id}, {"readed": True})
return render_template("article.html",
head_titles=[clear_string(article.title)],
article=article)
def update(action, feed_id=None):
readed = action == 'read'
filters = {'readed__ne': readed}
nb_days = request.args.get('nb_days', 0, type=int)
if nb_days != 0:
filters['date__lt'] = datetime.now() - timedelta(days=nb_days)
if feed_id:
filters['feed_id'] = feed_id
ArticleController(current_user.id).update(filters, {'readed': readed})
flash(gettext('Feed successfully updated.'), 'success')
return redirect(request.referrer or url_for('home'))
def update(action, feed_id=None):
readed = action == "read"
filters = {"readed__ne": readed}
nb_days = request.args.get("nb_days", 0, type=int)
if nb_days != 0:
filters["date__lt"] = datetime.now() - timedelta(days=nb_days)
if feed_id:
filters["feed_id"] = feed_id
ArticleController(current_user.id).update(filters, {"readed": readed})
flash(gettext("Feed successfully updated."), "success")
return redirect(request.referrer or url_for("home"))
async def insert_database(user, feed):
articles = await parse_feed(user, feed)
if None is articles:
return []
logger.debug('inserting articles for {}'.format(feed.title))
logger.info("Database insertion...")
new_articles = []
art_contr = ArticleController(user.id)
for article in articles:
existing_article_req = art_contr.read(feed_id=feed.id,
**extract_id(article))
exist = existing_article_req.count() != 0
if exist:
# if the article has been already retrieved, we only update
# the content or the title
logger.debug("Article %r (%r) already in the database.",
article['title'], article['link'])
existing_article = existing_article_req.first()
new_updated_date = None
try:
new_updated_date = dateutil.parser.parse(article['updated'])
except Exception as e:
print(e) #new_updated_date = existing_article.date
if None is existing_article.updated_date:
existing_article.updated_date = new_updated_date.replace(
tzinfo=None)
if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \
new_updated_date.strftime('%Y-%m-%dT%H:%M:%S'):
existing_article.updated_date = \
new_updated_date.replace(tzinfo=None)
if existing_article.title != article['title']:
existing_article.title = article['title']
content = get_article_content(article)
if existing_article.content != content:
existing_article.content = content
existing_article.readed = False
art_contr.update({'entry_id': existing_article.entry_id},
existing_article.dump())
continue
article = construct_article(article, feed)
try:
new_articles.append(art_contr.create(**article))
logger.info("New article % (%r) added.", article['title'],
article['link'])
except Exception:
logger.exception("Error when inserting article in database:")
continue
return new_articles
def test_article_challange_method(self):
self.assertEquals(0, len(list(ArticleController().challenge(
[{'id': art.id} for art in ArticleController(3).read()]))))
self.assertEquals(9, len(list(ArticleController(2).challenge(
[{'id': art.id} for art in ArticleController(3).read()]))))
self.assertEquals(9, len(list(ArticleController(2).challenge(
[{'entry_id': art.id} for art in ArticleController(3).read()]
))))
def get_middle_panel():
filters = _get_filters(request.args)
art_contr = ArticleController(g.user.id)
fd_hash = {
feed.id: {
'title':
feed.title,
'icon_url':
url_for('icon.icon', url=feed.icon_url) if feed.icon_url else None
}
for feed in FeedController(g.user.id).read()
}
articles = art_contr.read(**filters).order_by(Article.date.desc())
return _articles_to_json(articles, fd_hash)
def management():
"""
Display the management page.
"""
if request.method == 'POST':
if None != request.files.get('opmlfile', None):
# Import an OPML file
data = request.files.get('opmlfile', None)
if not misc_utils.allowed_file(data.filename):
flash(gettext('File not allowed.'), 'danger')
else:
try:
nb = import_opml(current_user.nickname, data.read())
if conf.CRAWLING_METHOD == "classic":
misc_utils.fetch(current_user.id, None)
flash(
str(nb) + ' ' + gettext('feeds imported.'),
"success")
flash(gettext("Downloading articles..."), 'info')
except:
flash(gettext("Impossible to import the new feeds."),
"danger")
elif None != request.files.get('jsonfile', None):
# Import an account
data = request.files.get('jsonfile', None)
if not misc_utils.allowed_file(data.filename):
flash(gettext('File not allowed.'), 'danger')
else:
try:
nb = import_json(current_user.nickname, data.read())
flash(gettext('Account imported.'), "success")
except:
flash(gettext("Impossible to import the account."),
"danger")
else:
flash(gettext('File not allowed.'), 'danger')
nb_feeds = FeedController(current_user.id).read().count()
art_contr = ArticleController(current_user.id)
nb_articles = art_contr.read().count()
nb_unread_articles = art_contr.read(readed=False).count()
nb_categories = CategoryController(current_user.id).read().count()
nb_bookmarks = BookmarkController(current_user.id).read().count()
return render_template('management.html',
user=current_user,
nb_feeds=nb_feeds,
nb_articles=nb_articles,
nb_unread_articles=nb_unread_articles,
nb_categories=nb_categories,
nb_bookmarks=nb_bookmarks)
async def insert_database(user, feed):
articles = await parse_feed(user, feed)
if None is articles:
return []
logger.info('Inserting articles for {}'.format(feed.title))
new_articles = []
art_contr = ArticleController(user.id)
for article in articles:
new_article = await construct_article(article, feed)
try:
existing_article_req = art_contr.read(feed_id=feed.id,
entry_id=extract_id(article))
except Exception as e:
logger.exception("existing_article_req: " + str(e))
continue
exist = existing_article_req.count() != 0
if exist:
continue
# if the article has been already retrieved, we only update
# the content or the title
logger.info('Article already in the database: {}'. \
format(article['link']))
existing_article = existing_article_req.first()
if new_article['date'].replace(tzinfo=None) != \
existing_article.date:
existing_article.date = new_article['date']
existing_article.updated_date = new_article['date']
if existing_article.title != new_article['title']:
existing_article.title = new_article['title']
content = get_article_content(article)
if existing_article.content != content:
existing_article.content = content
existing_article.readed = False
art_contr.update({'entry_id': existing_article.entry_id},
existing_article.dump())
logger.info('Article updated: {}'.format(article['link']))
continue
# insertion of the new article
try:
new_articles.append(art_contr.create(**new_article))
logger.info('New article added: {}'.format(new_article['link']))
except Exception:
logger.exception('Error when inserting article in database:')
continue
return new_articles
def get_article(article_id, parse=False):
locale = get_locale()
contr = ArticleController(current_user.id)
article = contr.get(id=article_id)
if not article.readed:
article["readed"] = True
contr.update({"id": article_id}, {"readed": True})
article["category_id"] = article.category_id or 0
feed = FeedController(current_user.id).get(id=article.feed_id)
article["icon_url"] = (
url_for("icon.icon", url=feed.icon_url) if feed.icon_url else None
)
article["date"] = format_datetime(localize(article.date), locale=locale)
return article
def feed_view(feed_id=None, user_id=None):
feed = FeedController(user_id).get(id=feed_id)
word_size = 6
category = None
if feed.category_id:
category = CategoryController(user_id).get(id=feed.category_id)
filters = {}
filters["feed_id"] = feed_id
articles = ArticleController(user_id).read_light(**filters)
# Server-side pagination
page, per_page, offset = get_page_args(per_page_parameter="per_page")
pagination = Pagination(
page=page,
total=articles.count(),
css_framework="bootstrap3",
search=False,
record_name="articles",
per_page=per_page,
)
today = datetime.now()
try:
last_article = articles[0].date
first_article = articles[-1].date
delta = last_article - first_article
average = round(float(articles.count()) / abs(delta.days), 2)
except Exception as e:
last_article = datetime.fromtimestamp(0)
first_article = datetime.fromtimestamp(0)
delta = last_article - first_article
average = 0
elapsed = today - last_article
return render_template(
"feed.html",
head_titles=[utils.clear_string(feed.title)],
feed=feed,
category=category,
articles=articles.offset(offset).limit(per_page),
pagination=pagination,
first_post_date=first_article,
end_post_date=last_article,
average=average,
delta=delta,
elapsed=elapsed,
)
def user_stream(per_page, nickname=None):
"""
Display the stream of a user (list of articles of public feed).
"""
user_contr = UserController()
user = user_contr.get(nickname=nickname)
if not user.is_public_profile:
if current_user.is_authenticated and current_user.id == user.id:
flash(gettext("You must set your profile to public."), "info")
return redirect(url_for("user.profile"))
category_id = int(request.args.get("category_id", 0))
category = CategoryController().read(id=category_id).first()
# Load the public feeds
filters = {}
filters["private"] = False
if category_id:
filters["category_id"] = category_id
feeds = FeedController().read(**filters).all()
# Re-initializes the filters to load the articles
filters = {}
filters["feed_id__in"] = [feed.id for feed in feeds]
if category:
filters["category_id"] = category_id
articles = ArticleController(user.id).read_light(**filters)
# Server-side pagination
page, per_page, offset = get_page_args(per_page_parameter="per_page")
pagination = Pagination(
page=page,
total=articles.count(),
css_framework="bootstrap3",
search=False,
record_name="articles",
per_page=per_page,
)
return render_template(
"user_stream.html",
user=user,
articles=articles.offset(offset).limit(per_page),
category=category,
pagination=pagination,
)
def user(user_id=None):
"""
See information about a user (stations, etc.).
"""
user = UserController().get(id=user_id)
if user is not None:
article_contr = ArticleController(user_id)
return render_template(
'/admin/user.html',
user=user,
feeds=FeedController().read(user_id=user_id).order_by('title'),
article_count=article_contr.count_by_feed(),
unread_article_count=article_contr.count_by_feed(readed=False))
else:
flash(gettext('This user does not exist.'), 'warn')
return redirect(redirect_url())
async def insert_articles(queue, nḅ_producers=1):
"""Consumer coroutines.
"""
nb_producers_done = 0
while True:
item = await queue.get()
if item is None:
nb_producers_done += 1
if nb_producers_done == nḅ_producers:
print('All producers done.')
print('Process finished.')
break
continue
user, feed, articles = item
if None is articles:
logger.info('None')
articles = []
logger.info('Inserting articles for {}'.format(feed.link))
art_contr = ArticleController(user.id)
for article in articles:
new_article = await construct_article(article, feed)
try:
existing_article_req = art_contr.read(
user_id=user.id,
feed_id=feed.id,
entry_id=extract_id(article))
except Exception as e:
logger.exception("existing_article_req: " + str(e))
continue
exist = existing_article_req.count() != 0
if exist:
continue
# insertion of the new article
try:
art_contr.create(**new_article)
logger.info('New article added: {}'.format(new_article['link']))
except Exception:
logger.exception('Error when inserting article in database.')
continue
def expire():
"""
Delete articles older than the given number of weeks.
"""
current_time = datetime.utcnow()
weeks_ago = current_time - timedelta(int(request.args.get('weeks', 10)))
art_contr = ArticleController(current_user.id)
query = art_contr.read(__or__={
'date__lt': weeks_ago,
'retrieved_date__lt': weeks_ago
})
count = query.count()
query.delete()
db.session.commit()
flash(gettext('%(count)d articles deleted', count=count), 'info')
return redirect(redirect_url())
