def management(): """ Display the management page. """ if request.method == 'POST': if None != request.files.get('opmlfile', None): # Import an OPML file data = request.files.get('opmlfile', None) if not misc_utils.allowed_file(data.filename): flash(gettext('File not allowed.'), 'danger') else: try: nb = import_opml(current_user.nickname, data.read()) if conf.CRAWLING_METHOD == "classic": misc_utils.fetch(current_user.id, None) flash( str(nb) + ' ' + gettext('feeds imported.'), "success") flash(gettext("Downloading articles..."), 'info') except: flash(gettext("Impossible to import the new feeds."), "danger") elif None != request.files.get('jsonfile', None): # Import an account data = request.files.get('jsonfile', None) if not misc_utils.allowed_file(data.filename): flash(gettext('File not allowed.'), 'danger') else: try: nb = import_json(current_user.nickname, data.read()) flash(gettext('Account imported.'), "success") except: flash(gettext("Impossible to import the account."), "danger") else: flash(gettext('File not allowed.'), 'danger') nb_feeds = FeedController(current_user.id).read().count() art_contr = ArticleController(current_user.id) nb_articles = art_contr.read().count() nb_unread_articles = art_contr.read(readed=False).count() nb_categories = CategoryController(current_user.id).read().count() nb_bookmarks = BookmarkController(current_user.id).read().count() return render_template('management.html', user=current_user, nb_feeds=nb_feeds, nb_articles=nb_articles, nb_unread_articles=nb_unread_articles, nb_categories=nb_categories, nb_bookmarks=nb_bookmarks)
def management(): """ Display the management page. """ if request.method == 'POST': if None != request.files.get('opmlfile', None): # Import an OPML file data = request.files.get('opmlfile', None) if not misc_utils.allowed_file(data.filename): flash(gettext('File not allowed.'), 'danger') else: try: nb = import_opml(current_user.nickname, data.read()) if conf.CRAWLING_METHOD == "classic": misc_utils.fetch(current_user.id, None) flash(str(nb) + ' ' + gettext('feeds imported.'), "success") flash(gettext("Downloading articles..."), 'info') except: flash(gettext("Impossible to import the new feeds."), "danger") elif None != request.files.get('jsonfile', None): # Import an account data = request.files.get('jsonfile', None) if not misc_utils.allowed_file(data.filename): flash(gettext('File not allowed.'), 'danger') else: try: nb = import_json(current_user.nickname, data.read()) flash(gettext('Account imported.'), "success") except: flash(gettext("Impossible to import the account."), "danger") else: flash(gettext('File not allowed.'), 'danger') nb_feeds = FeedController(current_user.id).read().count() art_contr = ArticleController(current_user.id) nb_articles = art_contr.read().count() nb_unread_articles = art_contr.read(readed=False).count() nb_categories = CategoryController(current_user.id).read().count() nb_bookmarks = BookmarkController(current_user.id).read().count() return render_template('management.html', user=current_user, nb_feeds=nb_feeds, nb_articles=nb_articles, nb_unread_articles=nb_unread_articles, nb_categories=nb_categories, nb_bookmarks=nb_bookmarks)
def get_middle_panel(): filters = _get_filters(request.args) art_contr = ArticleController(g.user.id) fd_hash = {feed.id: {'title': feed.title, 'icon_url': url_for('icon.icon', url=feed.icon_url) if feed.icon_url else None} for feed in FeedController(g.user.id).read()} articles = art_contr.read(**filters).order_by(Article.date.desc()) return _articles_to_json(articles, fd_hash)
def management(): """ Display the management page. """ if request.method == "POST": if None != request.files.get("opmlfile", None): # Import an OPML file data = request.files.get("opmlfile", None) if not utils.allowed_file(data.filename): flash(gettext("File not allowed."), "danger") else: try: nb = utils.import_opml(g.user.email, data.read()) if conf.CRAWLING_METHOD == "classic": utils.fetch(g.user.email, None) flash(str(nb) + " " + gettext("feeds imported."), "success") flash(gettext("Downloading articles..."), "info") except: flash(gettext("Impossible to import the new feeds."), "danger") elif None != request.files.get("jsonfile", None): # Import an account data = request.files.get("jsonfile", None) if not utils.allowed_file(data.filename): flash(gettext("File not allowed."), "danger") else: try: nb = utils.import_json(g.user.email, data.read()) flash(gettext("Account imported."), "success") except: flash(gettext("Impossible to import the account."), "danger") else: flash(gettext("File not allowed."), "danger") nb_feeds = FeedController(g.user.id).read().count() art_contr = ArticleController(g.user.id) nb_articles = art_contr.read().count() nb_unread_articles = art_contr.read(readed=False).count() return render_template( "management.html", user=g.user, nb_feeds=nb_feeds, nb_articles=nb_articles, nb_unread_articles=nb_unread_articles, )
async def insert_database(user, feed): articles = await parse_feed(user, feed) if None is articles: return [] logger.debug('inserting articles for {}'.format(feed.title)) logger.info("Database insertion...") new_articles = [] art_contr = ArticleController(user.id) for article in articles: existing_article_req = art_contr.read(feed_id=feed.id, **extract_id(article)) exist = existing_article_req.count() != 0 if exist: # if the article has been already retrieved, we only update # the content or the title logger.debug("Article %r (%r) already in the database.", article['title'], article['link']) existing_article = existing_article_req.first() new_updated_date = None try: new_updated_date = dateutil.parser.parse(article['updated']) except Exception as e: print(e) #new_updated_date = existing_article.date if None is existing_article.updated_date: existing_article.updated_date = new_updated_date.replace( tzinfo=None) if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \ new_updated_date.strftime('%Y-%m-%dT%H:%M:%S'): existing_article.updated_date = \ new_updated_date.replace(tzinfo=None) if existing_article.title != article['title']: existing_article.title = article['title'] content = get_article_content(article) if existing_article.content != content: existing_article.content = content existing_article.readed = False art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) continue article = construct_article(article, feed) try: new_articles.append(art_contr.create(**article)) logger.info("New article % (%r) added.", article['title'], article['link']) except Exception: logger.exception("Error when inserting article in database:") continue return new_articles
async def insert_database(user, feed): articles = await parse_feed(user, feed) if None is articles: return [] logger.info('Inserting articles for {}'.format(feed.title)) new_articles = [] art_contr = ArticleController(user.id) for article in articles: new_article = await construct_article(article, feed) try: existing_article_req = art_contr.read(feed_id=feed.id, entry_id=extract_id(article)) except Exception as e: logger.exception("existing_article_req: " + str(e)) continue exist = existing_article_req.count() != 0 if exist: continue # if the article has been already retrieved, we only update # the content or the title logger.info('Article already in the database: {}'. \ format(article['link'])) existing_article = existing_article_req.first() if new_article['date'].replace(tzinfo=None) != \ existing_article.date: existing_article.date = new_article['date'] existing_article.updated_date = new_article['date'] if existing_article.title != new_article['title']: existing_article.title = new_article['title'] content = get_article_content(article) if existing_article.content != content: existing_article.content = content existing_article.readed = False art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) logger.info('Article updated: {}'.format(article['link'])) continue # insertion of the new article try: new_articles.append(art_contr.create(**new_article)) logger.info('New article added: {}'.format(new_article['link'])) except Exception: logger.exception('Error when inserting article in database:') continue return new_articles
async def insert_database(user, feed): articles = await parse_feed(user, feed) if None is articles: return [] logger.debug('inserting articles for {}'.format(feed.title)) logger.info("Database insertion...") new_articles = [] art_contr = ArticleController(user.id) for article in articles: existing_article_req = art_contr.read(feed_id=feed.id, **extract_id(article)) exist = existing_article_req.count() != 0 if exist: # if the article has been already retrieved, we only update # the content or the title logger.debug("Article %r (%r) already in the database.", article['title'], article['link']) existing_article = existing_article_req.first() new_updated_date = None try: new_updated_date = dateutil.parser.parse(article['updated']) except Exception as e: print(e)#new_updated_date = existing_article.date if None is existing_article.updated_date: existing_article.updated_date = new_updated_date.replace(tzinfo=None) if existing_article.updated_date.strftime('%Y-%m-%dT%H:%M:%S') != \ new_updated_date.strftime('%Y-%m-%dT%H:%M:%S'): existing_article.updated_date = \ new_updated_date.replace(tzinfo=None) if existing_article.title != article['title']: existing_article.title = article['title'] content = get_article_content(article) if existing_article.content != content: existing_article.content = content existing_article.readed = False art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) continue article = construct_article(article, feed) try: new_articles.append(art_contr.create(**article)) logger.info("New article % (%r) added.", article['title'], article['link']) except Exception: logger.exception("Error when inserting article in database:") continue return new_articles
def expire(): """ Delete articles older than the given number of weeks. """ current_time = datetime.utcnow() weeks_ago = current_time - timedelta(int(request.args.get('weeks', 10))) art_contr = ArticleController(g.user.id) query = art_contr.read(__or__={'date__lt': weeks_ago, 'retrieved_date__lt': weeks_ago}) count = query.count() query.delete() flash(gettext('%(count)d articles deleted', count=count), 'info') return redirect(redirect_url())
def get_middle_panel(): filters = _get_filters(request.args) art_contr = ArticleController(g.user.id) fd_hash = { feed.id: { 'title': feed.title, 'icon_url': url_for('icon.icon', url=feed.icon_url) if feed.icon_url else None } for feed in FeedController(g.user.id).read() } articles = art_contr.read(**filters).order_by(Article.date.desc()) return _articles_to_json(articles, fd_hash)
async def insert_database(user, feed): articles = await parse_feed(user, feed) if None is articles: return [] logger.info('Inserting articles for {}'.format(feed.title)) new_articles = [] art_contr = ArticleController(user.id) for article in articles: new_article = await construct_article(article, feed) try: existing_article_req = art_contr.read(feed_id=feed.id, entry_id=extract_id(article)) except Exception as e: logger.exception("existing_article_req: " + str(e)) continue exist = existing_article_req.count() != 0 if exist: # if the article has been already retrieved, we only update # the content or the title logger.info('Article already in the database: {}'. \ format(article['link'])) existing_article = existing_article_req.first() if new_article['date'].replace(tzinfo=None) != \ existing_article.date: existing_article.date = new_article['date'] existing_article.updated_date = new_article['date'] if existing_article.title != new_article['title']: existing_article.title = new_article['title'] content = get_article_content(article) if existing_article.content != content: existing_article.content = content existing_article.readed = False art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) logger.info('Article updated: {}'.format(article['link'])) continue # insertion of the new article try: new_articles.append(art_contr.create(**new_article)) logger.info('New article added: {}'.format(new_article['link'])) except Exception: logger.exception('Error when inserting article in database:') continue return new_articles
def expire(): """ Delete articles older than the given number of weeks. """ current_time = datetime.utcnow() weeks_ago = current_time - timedelta(int(request.args.get('weeks', 10))) art_contr = ArticleController(current_user.id) query = art_contr.read(__or__={ 'date__lt': weeks_ago, 'retrieved_date__lt': weeks_ago }) count = query.count() query.delete() db.session.commit() flash(gettext('%(count)d articles deleted', count=count), 'info') return redirect(redirect_url())
async def insert_articles(queue, nḅ_producers=1): """Consumer coroutines. """ nb_producers_done = 0 while True: item = await queue.get() if item is None: nb_producers_done += 1 if nb_producers_done == nḅ_producers: print('All producers done.') print('Process finished.') break continue user, feed, articles = item if None is articles: logger.info('None') articles = [] logger.info('Inserting articles for {}'.format(feed.link)) art_contr = ArticleController(user.id) for article in articles: new_article = await construct_article(article, feed) try: existing_article_req = art_contr.read( user_id=user.id, feed_id=feed.id, entry_id=extract_id(article)) except Exception as e: logger.exception("existing_article_req: " + str(e)) continue exist = existing_article_req.count() != 0 if exist: continue # insertion of the new article try: art_contr.create(**new_article) logger.info('New article added: {}'.format(new_article['link'])) except Exception: logger.exception('Error when inserting article in database.') continue
async def insert_database(user, feed): articles = await parse_feed(user, feed) if None is articles: return [] logger.debug('inserting articles for {}'.format(feed.title)) logger.info("Database insertion...") new_articles = [] art_contr = ArticleController(user.id) for article in articles: existing_article_req = art_contr.read(feed_id=feed.id, **extract_id(article)) exist = existing_article_req.count() != 0 if exist: existing_article = existing_article_req.first() is_updated = False logger.debug("Article %r (%r) already in the database.", article['title'], article['link']) content = get_article_content(article) if existing_article.title != article['title']: existing_article.title = article['title'] is_updated = True if existing_article.content != content: existing_article.content = content existing_article.readed = False is_updated = True if is_updated: art_contr.update({'entry_id': existing_article.entry_id}, existing_article.dump()) continue article = construct_article(article, feed.dump()) try: new_articles.append(art_contr.create(**article)) logger.info("New article % (%r) added.", article['title'], article['link']) except Exception: logger.exception("Error when inserting article in database:") continue return new_articles