def management():
"""
Display the management page.
"""
if request.method == 'POST':
if None != request.files.get('opmlfile', None):
# Import an OPML file
data = request.files.get('opmlfile', None)
if not misc_utils.allowed_file(data.filename):
flash(gettext('File not allowed.'), 'danger')
else:
try:
nb = import_opml(current_user.nickname, data.read())
if conf.CRAWLING_METHOD == "classic":
misc_utils.fetch(current_user.id, None)
flash(
str(nb) + ' ' + gettext('feeds imported.'),
"success")
flash(gettext("Downloading articles..."), 'info')
except:
flash(gettext("Impossible to import the new feeds."),
"danger")
elif None != request.files.get('jsonfile', None):
# Import an account
data = request.files.get('jsonfile', None)
if not misc_utils.allowed_file(data.filename):
flash(gettext('File not allowed.'), 'danger')
else:
try:
nb = import_json(current_user.nickname, data.read())
flash(gettext('Account imported.'), "success")
except:
flash(gettext("Impossible to import the account."),
"danger")
else:
flash(gettext('File not allowed.'), 'danger')
nb_feeds = FeedController(current_user.id).read().count()
art_contr = ArticleController(current_user.id)
nb_articles = art_contr.read().count()
nb_unread_articles = art_contr.read(readed=False).count()
nb_categories = CategoryController(current_user.id).read().count()
nb_bookmarks = BookmarkController(current_user.id).read().count()
return render_template('management.html',
user=current_user,
nb_feeds=nb_feeds,
nb_articles=nb_articles,
nb_unread_articles=nb_unread_articles,
nb_categories=nb_categories,
nb_bookmarks=nb_bookmarks)
def management():
"""
Display the management page.
"""
if request.method == 'POST':
if None != request.files.get('opmlfile', None):
# Import an OPML file
data = request.files.get('opmlfile', None)
if not misc_utils.allowed_file(data.filename):
flash(gettext('File not allowed.'), 'danger')
else:
try:
nb = import_opml(current_user.nickname, data.read())
if conf.CRAWLING_METHOD == "classic":
misc_utils.fetch(current_user.id, None)
flash(str(nb) + ' ' + gettext('feeds imported.'),
"success")
flash(gettext("Downloading articles..."), 'info')
except:
flash(gettext("Impossible to import the new feeds."),
"danger")
elif None != request.files.get('jsonfile', None):
# Import an account
data = request.files.get('jsonfile', None)
if not misc_utils.allowed_file(data.filename):
flash(gettext('File not allowed.'), 'danger')
else:
try:
nb = import_json(current_user.nickname, data.read())
flash(gettext('Account imported.'), "success")
except:
flash(gettext("Impossible to import the account."),
"danger")
else:
flash(gettext('File not allowed.'), 'danger')
nb_feeds = FeedController(current_user.id).read().count()
art_contr = ArticleController(current_user.id)
nb_articles = art_contr.read().count()
nb_unread_articles = art_contr.read(readed=False).count()
nb_categories = CategoryController(current_user.id).read().count()
nb_bookmarks = BookmarkController(current_user.id).read().count()
return render_template('management.html', user=current_user,
nb_feeds=nb_feeds, nb_articles=nb_articles,
nb_unread_articles=nb_unread_articles,
nb_categories=nb_categories,
nb_bookmarks=nb_bookmarks)
def get_middle_panel():
filters = _get_filters(request.args)
art_contr = ArticleController(g.user.id)
fd_hash = {feed.id: {'title': feed.title,
'icon_url': url_for('icon.icon', url=feed.icon_url)
if feed.icon_url else None}
for feed in FeedController(g.user.id).read()}
articles = art_contr.read(**filters).order_by(Article.date.desc())
return _articles_to_json(articles, fd_hash)
def management():
"""
Display the management page.
"""
if request.method == "POST":
if None != request.files.get("opmlfile", None):
# Import an OPML file
data = request.files.get("opmlfile", None)
if not utils.allowed_file(data.filename):
flash(gettext("File not allowed."), "danger")
else:
try:
nb = utils.import_opml(g.user.email, data.read())
if conf.CRAWLING_METHOD == "classic":
utils.fetch(g.user.email, None)
flash(str(nb) + " " + gettext("feeds imported."), "success")
flash(gettext("Downloading articles..."), "info")
except:
flash(gettext("Impossible to import the new feeds."), "danger")
elif None != request.files.get("jsonfile", None):
# Import an account
data = request.files.get("jsonfile", None)
if not utils.allowed_file(data.filename):
flash(gettext("File not allowed."), "danger")
else:
try:
nb = utils.import_json(g.user.email, data.read())
flash(gettext("Account imported."), "success")
except:
flash(gettext("Impossible to import the account."), "danger")
else:
flash(gettext("File not allowed."), "danger")
nb_feeds = FeedController(g.user.id).read().count()
art_contr = ArticleController(g.user.id)
nb_articles = art_contr.read().count()
nb_unread_articles = art_contr.read(readed=False).count()
return render_template(
"management.html",
user=g.user,
nb_feeds=nb_feeds,
nb_articles=nb_articles,
nb_unread_articles=nb_unread_articles,
)
async def insert_database(user, feed):
articles = await parse_feed(user, feed)
if None is articles:
return []
logger.debug('inserting articles for {}'.format(feed.title))
logger.info("Database insertion...")
new_articles = []
art_contr = ArticleController(user.id)
for article in articles:
existing_article_req = art_contr.read(feed_id=feed.id,
**extract_id(article))
exist = existing_article_req.count() != 0
if exist:
# if the article has been already retrieved, we only update
# the content or the title
logger.debug("Article %r (%r) already in the database.",
article['title'], article['link'])
existing_article = existing_article_req.first()
new_updated_date = None
try:
new_updated_date = dateutil.parser.parse(article['updated'])
except Exception as e:
print(e) #new_updated_date = existing_article.date
if None is existing_article.updated_date:
existing_article.updated_date = new_updated_date.replace(
tzinfo=None)
if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \
new_updated_date.strftime('%Y-%m-%dT%H:%M:%S'):
existing_article.updated_date = \
new_updated_date.replace(tzinfo=None)
if existing_article.title != article['title']:
existing_article.title = article['title']
content = get_article_content(article)
if existing_article.content != content:
existing_article.content = content
existing_article.readed = False
art_contr.update({'entry_id': existing_article.entry_id},
existing_article.dump())
continue
article = construct_article(article, feed)
try:
new_articles.append(art_contr.create(**article))
logger.info("New article % (%r) added.", article['title'],
article['link'])
except Exception:
logger.exception("Error when inserting article in database:")
continue
return new_articles
async def insert_database(user, feed):
articles = await parse_feed(user, feed)
if None is articles:
return []
logger.info('Inserting articles for {}'.format(feed.title))
new_articles = []
art_contr = ArticleController(user.id)
for article in articles:
new_article = await construct_article(article, feed)
try:
existing_article_req = art_contr.read(feed_id=feed.id,
entry_id=extract_id(article))
except Exception as e:
logger.exception("existing_article_req: " + str(e))
continue
exist = existing_article_req.count() != 0
if exist:
continue
# if the article has been already retrieved, we only update
# the content or the title
logger.info('Article already in the database: {}'. \
format(article['link']))
existing_article = existing_article_req.first()
if new_article['date'].replace(tzinfo=None) != \
existing_article.date:
existing_article.date = new_article['date']
existing_article.updated_date = new_article['date']
if existing_article.title != new_article['title']:
existing_article.title = new_article['title']
content = get_article_content(article)
if existing_article.content != content:
existing_article.content = content
existing_article.readed = False
art_contr.update({'entry_id': existing_article.entry_id},
existing_article.dump())
logger.info('Article updated: {}'.format(article['link']))
continue
# insertion of the new article
try:
new_articles.append(art_contr.create(**new_article))
logger.info('New article added: {}'.format(new_article['link']))
except Exception:
logger.exception('Error when inserting article in database:')
continue
return new_articles
async def insert_database(user, feed):
articles = await parse_feed(user, feed)
if None is articles:
return []
logger.debug('inserting articles for {}'.format(feed.title))
logger.info("Database insertion...")
new_articles = []
art_contr = ArticleController(user.id)
for article in articles:
existing_article_req = art_contr.read(feed_id=feed.id,
**extract_id(article))
exist = existing_article_req.count() != 0
if exist:
# if the article has been already retrieved, we only update
# the content or the title
logger.debug("Article %r (%r) already in the database.",
article['title'], article['link'])
existing_article = existing_article_req.first()
new_updated_date = None
try:
new_updated_date = dateutil.parser.parse(article['updated'])
except Exception as e:
print(e)#new_updated_date = existing_article.date
if None is existing_article.updated_date:
existing_article.updated_date = new_updated_date.replace(tzinfo=None)
if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \
new_updated_date.strftime('%Y-%m-%dT%H:%M:%S'):
existing_article.updated_date = \
new_updated_date.replace(tzinfo=None)
if existing_article.title != article['title']:
existing_article.title = article['title']
content = get_article_content(article)
if existing_article.content != content:
existing_article.content = content
existing_article.readed = False
art_contr.update({'entry_id': existing_article.entry_id},
existing_article.dump())
continue
article = construct_article(article, feed)
try:
new_articles.append(art_contr.create(**article))
logger.info("New article % (%r) added.",
article['title'], article['link'])
except Exception:
logger.exception("Error when inserting article in database:")
continue
return new_articles
def expire():
"""
Delete articles older than the given number of weeks.
"""
current_time = datetime.utcnow()
weeks_ago = current_time - timedelta(int(request.args.get('weeks', 10)))
art_contr = ArticleController(g.user.id)
query = art_contr.read(__or__={'date__lt': weeks_ago,
'retrieved_date__lt': weeks_ago})
count = query.count()
query.delete()
flash(gettext('%(count)d articles deleted', count=count), 'info')
return redirect(redirect_url())
def get_middle_panel():
filters = _get_filters(request.args)
art_contr = ArticleController(g.user.id)
fd_hash = {
feed.id: {
'title':
feed.title,
'icon_url':
url_for('icon.icon', url=feed.icon_url) if feed.icon_url else None
}
for feed in FeedController(g.user.id).read()
}
articles = art_contr.read(**filters).order_by(Article.date.desc())
return _articles_to_json(articles, fd_hash)
async def insert_database(user, feed):
articles = await parse_feed(user, feed)
if None is articles:
return []
logger.info('Inserting articles for {}'.format(feed.title))
new_articles = []
art_contr = ArticleController(user.id)
for article in articles:
new_article = await construct_article(article, feed)
try:
existing_article_req = art_contr.read(feed_id=feed.id,
entry_id=extract_id(article))
except Exception as e:
logger.exception("existing_article_req: " + str(e))
continue
exist = existing_article_req.count() != 0
if exist:
# if the article has been already retrieved, we only update
# the content or the title
logger.info('Article already in the database: {}'. \
format(article['link']))
existing_article = existing_article_req.first()
if new_article['date'].replace(tzinfo=None) != \
existing_article.date:
existing_article.date = new_article['date']
existing_article.updated_date = new_article['date']
if existing_article.title != new_article['title']:
existing_article.title = new_article['title']
content = get_article_content(article)
if existing_article.content != content:
existing_article.content = content
existing_article.readed = False
art_contr.update({'entry_id': existing_article.entry_id},
existing_article.dump())
logger.info('Article updated: {}'.format(article['link']))
continue
# insertion of the new article
try:
new_articles.append(art_contr.create(**new_article))
logger.info('New article added: {}'.format(new_article['link']))
except Exception:
logger.exception('Error when inserting article in database:')
continue
return new_articles
def expire():
"""
Delete articles older than the given number of weeks.
"""
current_time = datetime.utcnow()
weeks_ago = current_time - timedelta(int(request.args.get('weeks', 10)))
art_contr = ArticleController(current_user.id)
query = art_contr.read(__or__={
'date__lt': weeks_ago,
'retrieved_date__lt': weeks_ago
})
count = query.count()
query.delete()
db.session.commit()
flash(gettext('%(count)d articles deleted', count=count), 'info')
return redirect(redirect_url())
async def insert_articles(queue, nḅ_producers=1):
"""Consumer coroutines.
"""
nb_producers_done = 0
while True:
item = await queue.get()
if item is None:
nb_producers_done += 1
if nb_producers_done == nḅ_producers:
print('All producers done.')
print('Process finished.')
break
continue
user, feed, articles = item
if None is articles:
logger.info('None')
articles = []
logger.info('Inserting articles for {}'.format(feed.link))
art_contr = ArticleController(user.id)
for article in articles:
new_article = await construct_article(article, feed)
try:
existing_article_req = art_contr.read(
user_id=user.id,
feed_id=feed.id,
entry_id=extract_id(article))
except Exception as e:
logger.exception("existing_article_req: " + str(e))
continue
exist = existing_article_req.count() != 0
if exist:
continue
# insertion of the new article
try:
art_contr.create(**new_article)
logger.info('New article added: {}'.format(new_article['link']))
except Exception:
logger.exception('Error when inserting article in database.')
continue
async def insert_database(user, feed):
articles = await parse_feed(user, feed)
if None is articles:
return []
logger.debug('inserting articles for {}'.format(feed.title))
logger.info("Database insertion...")
new_articles = []
art_contr = ArticleController(user.id)
for article in articles:
existing_article_req = art_contr.read(feed_id=feed.id,
**extract_id(article))
exist = existing_article_req.count() != 0
if exist:
existing_article = existing_article_req.first()
is_updated = False
logger.debug("Article %r (%r) already in the database.",
article['title'], article['link'])
content = get_article_content(article)
if existing_article.title != article['title']:
existing_article.title = article['title']
is_updated = True
if existing_article.content != content:
existing_article.content = content
existing_article.readed = False
is_updated = True
if is_updated:
art_contr.update({'entry_id': existing_article.entry_id},
existing_article.dump())
continue
article = construct_article(article, feed.dump())
try:
new_articles.append(art_contr.create(**article))
logger.info("New article % (%r) added.",
article['title'], article['link'])
except Exception:
logger.exception("Error when inserting article in database:")
continue
return new_articles
async def insert_database(user, feed):
articles = await parse_feed(user, feed)
if None is articles:
return []
logger.debug('inserting articles for {}'.format(feed.title))
logger.info("Database insertion...")
new_articles = []
art_contr = ArticleController(user.id)
for article in articles:
existing_article_req = art_contr.read(feed_id=feed.id,
**extract_id(article))
exist = existing_article_req.count() != 0
if exist:
existing_article = existing_article_req.first()
is_updated = False
logger.debug("Article %r (%r) already in the database.",
article['title'], article['link'])
content = get_article_content(article)
if existing_article.title != article['title']:
existing_article.title = article['title']
is_updated = True
if existing_article.content != content:
existing_article.content = content
existing_article.readed = False
is_updated = True
if is_updated:
art_contr.update({'entry_id': existing_article.entry_id},
existing_article.dump())
continue
article = construct_article(article, feed.dump())
try:
new_articles.append(art_contr.create(**article))
logger.info("New article % (%r) added.", article['title'],
article['link'])
except Exception:
logger.exception("Error when inserting article in database:")
continue
return new_articles
