def reset_feeds():
contr = FeedController()
step = timedelta(seconds=3600 / contr.read().count())
now = datetime.utcnow()
for i, feed in enumerate(contr.read()
.order_by(contr._db_cls.last_retrieved)):
contr.update({'id': feed.id},
{'etag': '', 'last_modified': '',
'last_retrieved': now - i * step})
def reset_feeds():
contr = FeedController()
step = timedelta(seconds=3600 / contr.read().count())
now = datetime.utcnow()
for i, feed in enumerate(contr.read().order_by(
contr._db_cls.last_retrieved)):
contr.update({'id': feed.id}, {
'etag': '',
'last_modified': '',
'last_retrieved': now - i * step
})
def bookmarklet():
feed_contr = FeedController(current_user.id)
url = (request.args if request.method == 'GET' else request.form)\
.get('url', None)
if not url:
flash(gettext("Couldn't add feed: url missing."), "error")
raise BadRequest("url is missing")
feed_exists = list(feed_contr.read(__or__={'link': url, 'site_link': url}))
if feed_exists:
flash(gettext("Couldn't add feed: feed already exists."), "warning")
return redirect(url_for('feed.form', feed_id=feed_exists[0].id))
try:
feed = construct_feed_from(url)
except requests.exceptions.ConnectionError:
flash(gettext("Impossible to connect to the address: {}.".format(url)),
"danger")
return redirect(url_for('home'))
except Exception:
logger.exception('something bad happened when fetching %r', url)
return redirect(url_for('home'))
if not feed.get('link'):
feed['enabled'] = False
flash(
gettext("Couldn't find a feed url, you'll need to find a Atom or"
" RSS link manually and reactivate this feed"), 'warning')
feed = feed_contr.create(**feed)
flash(gettext('Feed was successfully created.'), 'success')
if feed.enabled and conf.CRAWLING_METHOD == "default":
misc_utils.fetch(current_user.id, feed.id)
flash(gettext("Downloading articles for the new feed..."), 'info')
return redirect(url_for('feed.form', feed_id=feed.id))
def bookmarklet():
feed_contr = FeedController(g.user.id)
url = (request.args if request.method == 'GET' else request.form)\
.get('url', None)
if not url:
flash(gettext("Couldn't add feed: url missing."), "error")
raise BadRequest("url is missing")
feed_exists = list(feed_contr.read(__or__={'link': url, 'site_link': url}))
if feed_exists:
flash(gettext("Couldn't add feed: feed already exists."),
"warning")
return redirect(url_for('feed.form', feed_id=feed_exists[0].id))
try:
feed = construct_feed_from(url)
except requests.exceptions.ConnectionError:
flash(gettext("Impossible to connect to the address: {}.".format(url)),
"danger")
return redirect(url_for('home'))
except Exception:
logger.exception('something bad happened when fetching %r', url)
return redirect(url_for('home'))
if not feed.get('link'):
feed['enabled'] = False
flash(gettext("Couldn't find a feed url, you'll need to find a Atom or"
" RSS link manually and reactivate this feed"),
'warning')
feed = feed_contr.create(**feed)
flash(gettext('Feed was successfully created.'), 'success')
if feed.enabled and conf.CRAWLING_METHOD == "classic":
utils.fetch(g.user.id, feed.id)
flash(gettext("Downloading articles for the new feed..."), 'info')
return redirect(url_for('feed.form', feed_id=feed.id))
def process_form(feed_id=None):
form = AddFeedForm()
feed_contr = FeedController(current_user.id)
form.set_category_choices(CategoryController(current_user.id).read())
if not form.validate():
return render_template("edit_feed.html", form=form)
existing_feeds = list(feed_contr.read(link=form.link.data))
if existing_feeds and feed_id is None:
flash(gettext("Couldn't add feed: feed already exists."), "warning")
return redirect(url_for("feed.form", feed_id=existing_feeds[0].id))
# Edit an existing feed
feed_attr = {
"title": form.title.data,
"enabled": form.enabled.data,
"link": form.link.data,
"site_link": form.site_link.data,
"filters": [],
"category_id": form.category_id.data,
"private": form.private.data,
}
if not feed_attr["category_id"] or feed_attr["category_id"] == "0":
del feed_attr["category_id"]
for filter_attr in ("type", "pattern", "action on", "action"):
for i, value in enumerate(
request.form.getlist(filter_attr.replace(" ", "_"))):
if i >= len(feed_attr["filters"]):
feed_attr["filters"].append({})
feed_attr["filters"][i][filter_attr] = value
if feed_id is not None:
feed_contr.update({"id": feed_id}, feed_attr)
flash(
gettext(
"Feed %(feed_title)r successfully updated.",
feed_title=feed_attr["title"],
),
"success",
)
return redirect(url_for("feed.form", feed_id=feed_id))
# Create a new feed
new_feed = feed_contr.create(**feed_attr)
flash(
gettext("Feed %(feed_title)r successfully created.",
feed_title=new_feed.title),
"success",
)
if conf.CRAWLING_METHOD == "default":
misc_utils.fetch(current_user.id, new_feed.id)
flash(gettext("Downloading articles for the new feed..."), "info")
return redirect(url_for("feed.form", feed_id=new_feed.id))
def opml_import():
if request.files.get('opmlfile', None) is None:
flash(gettext('Got no file'), 'warning')
return redirect(url_for('user.profile'))
data = request.files.get('opmlfile', None)
try:
subscriptions = opml.from_string(data.read())
except:
flash(gettext("Couldn't parse file"), 'danger')
return redirect(request.referrer)
ccontr = CategoryController(current_user.id)
fcontr = FeedController(current_user.id)
created_count, existing_count, failed_count = 0, 0, 0
categories = {cat.name: cat.id for cat in ccontr.read()}
for line in subscriptions:
try:
link = line.xmlUrl
except Exception:
failed_count += 1
continue
# don't import twice
if fcontr.read(link=link).count():
existing_count += 1
continue
# handling categories
cat_id = None
category = getattr(line, 'category', None)
if category:
if category not in categories:
new_category = ccontr.create(name=category)
categories[new_category.name] = new_category.id
cat_id = categories[category]
fcontr.create(title=getattr(line, 'text', None),
category_id=cat_id,
description=getattr(line, 'description', None),
link=link,
site_link=getattr(line, 'htmlUrl', None))
created_count += 1
flash(
gettext(
"Created %(created)d feed ! (%(failed)d import failed, "
"%(existing)d were already existing)",
created=created_count,
failed=failed_count,
existing=existing_count), "info")
return redirect(url_for('user.profile'))
def process_form(feed_id=None):
form = AddFeedForm()
feed_contr = FeedController(current_user.id)
form.set_category_choices(CategoryController(current_user.id).read())
if not form.validate():
return render_template('edit_feed.html', form=form)
existing_feeds = list(feed_contr.read(link=form.link.data))
if existing_feeds and feed_id is None:
flash(gettext("Couldn't add feed: feed already exists."), "warning")
return redirect(url_for('feed.form', feed_id=existing_feeds[0].id))
# Edit an existing feed
feed_attr = {
'title': form.title.data,
'enabled': form.enabled.data,
'link': form.link.data,
'site_link': form.site_link.data,
'filters': [],
'category_id': form.category_id.data,
'private': form.private.data
}
if not feed_attr['category_id'] or feed_attr['category_id'] == '0':
del feed_attr['category_id']
for filter_attr in ('type', 'pattern', 'action on', 'action'):
for i, value in enumerate(
request.form.getlist(filter_attr.replace(' ', '_'))):
if i >= len(feed_attr['filters']):
feed_attr['filters'].append({})
feed_attr['filters'][i][filter_attr] = value
if feed_id is not None:
feed_contr.update({'id': feed_id}, feed_attr)
flash(
gettext('Feed %(feed_title)r successfully updated.',
feed_title=feed_attr['title']), 'success')
return redirect(url_for('feed.form', feed_id=feed_id))
# Create a new feed
new_feed = feed_contr.create(**feed_attr)
flash(
gettext('Feed %(feed_title)r successfully created.',
feed_title=new_feed.title), 'success')
if conf.CRAWLING_METHOD == "default":
misc_utils.fetch(current_user.id, new_feed.id)
flash(gettext("Downloading articles for the new feed..."), 'info')
return redirect(url_for('feed.form', feed_id=new_feed.id))
def opml_import():
if request.files.get('opmlfile', None) is None:
flash(gettext('Got no file'), 'warning')
return redirect(url_for('user.profile'))
data = request.files.get('opmlfile', None)
try:
subscriptions = opml.from_string(data.read())
except:
flash(gettext("Couldn't parse file"), 'danger')
return redirect(request.referrer)
ccontr = CategoryController(current_user.id)
fcontr = FeedController(current_user.id)
created_count, existing_count, failed_count = 0, 0, 0
categories = {cat.name: cat.id for cat in ccontr.read()}
for line in subscriptions:
try:
link = line.xmlUrl
except Exception:
failed_count += 1
continue
# don't import twice
if fcontr.read(link=link).count():
existing_count += 1
continue
# handling categories
cat_id = None
category = getattr(line, 'category', None)
if category:
if category not in categories:
new_category = ccontr.create(name=category)
categories[new_category.name] = new_category.id
cat_id = categories[category]
fcontr.create(title=getattr(line, 'text', None), category_id=cat_id,
description=getattr(line, 'description', None),
link=link, site_link=getattr(line, 'htmlUrl', None))
created_count += 1
flash(gettext("Created %(created)d feed ! (%(failed)d import failed, "
"%(existing)d were already existing)",
created=created_count, failed=failed_count,
existing=existing_count), "info")
return redirect(url_for('user.profile'))
def config(self):
print("graph_title JARR - Articles adding rate")
print("graph_vlabel Articles per sec")
print("articles.label Overall rate")
print("articles.type DERIVE")
print("articles.min 0")
fcontr = FeedController(ignore_context=True)
last_conn_max = datetime.utcnow() - timedelta(days=30)
for id_ in fcontr.read()\
.join(User).filter(User.is_active == True,
User.last_connection >= last_conn_max)\
.with_entities(fcontr._db_cls.user_id)\
.distinct().order_by('feed_user_id'):
id_ = id_[0]
print("articles_user_%s.label Rate for user %s" % (id_, id_))
print("articles_user_%s.type DERIVE" % id_)
print("articles_user_%s.min 0" % id_)
print("graph_category web")
print("graph_scale yes")
def process_form(feed_id=None):
form = AddFeedForm()
feed_contr = FeedController(current_user.id)
form.set_category_choices(CategoryController(current_user.id).read())
if not form.validate():
return render_template('edit_feed.html', form=form)
existing_feeds = list(feed_contr.read(link=form.link.data))
if existing_feeds and feed_id is None:
flash(gettext("Couldn't add feed: feed already exists."), "warning")
return redirect(url_for('feed.form', feed_id=existing_feeds[0].id))
# Edit an existing feed
feed_attr = {'title': form.title.data, 'enabled': form.enabled.data,
'link': form.link.data, 'site_link': form.site_link.data,
'filters': [], 'category_id': form.category_id.data,
'private': form.private.data}
if not feed_attr['category_id'] or feed_attr['category_id'] == '0':
del feed_attr['category_id']
for filter_attr in ('type', 'pattern', 'action on', 'action'):
for i, value in enumerate(
request.form.getlist(filter_attr.replace(' ', '_'))):
if i >= len(feed_attr['filters']):
feed_attr['filters'].append({})
feed_attr['filters'][i][filter_attr] = value
if feed_id is not None:
feed_contr.update({'id': feed_id}, feed_attr)
flash(gettext('Feed %(feed_title)r successfully updated.',
feed_title=feed_attr['title']), 'success')
return redirect(url_for('feed.form', feed_id=feed_id))
# Create a new feed
new_feed = feed_contr.create(**feed_attr)
flash(gettext('Feed %(feed_title)r successfully created.',
feed_title=new_feed.title), 'success')
if conf.CRAWLING_METHOD == "classic":
misc_utils.fetch(current_user.id, new_feed.id)
flash(gettext("Downloading articles for the new feed..."), 'info')
return redirect(url_for('feed.form', feed_id=new_feed.id))
def execute(self):
delta = timedelta(minutes=LATE_AFTER + FETCH_RATE + 1)
fcontr = FeedController(ignore_context=True)
print("feeds.value %d" % len(list(fcontr.list_late(delta, limit=0))))
print("feeds_total.value %d" % fcontr.read().count())
def test_create_using_filters(self):
feed_ctr = FeedController(2)
feed1 = feed_ctr.read()[0].dump()
feed2 = feed_ctr.read()[1].dump()
feed3 = feed_ctr.read()[2].dump()
feed_ctr.update({'id': feed1['id']},
{'filters': [{"type": "simple match",
"pattern": "no see pattern",
"action on": "match",
"action": "mark as read"}]})
feed_ctr.update({'id': feed3['id']},
{'filters': [{"type": "regex",
"pattern": ".*(pattern1|pattern2).*",
"action on": "no match",
"action": "mark as favorite"},
{"type": "simple match",
"pattern": "no see pattern",
"action on": "match",
"action": "mark as read"}]})
art1 = ArticleController(2).create(
entry_id="thisisnotatest",
feed_id=feed1['id'],
title="garbage no see pattern garbage",
content="doesn't matter",
link="doesn't matter either")
art2 = ArticleController(2).create(
entry_id="thisisnotatesteither",
feed_id=feed1['id'],
title="garbage see pattern garbage",
content="doesn't matter2",
link="doesn't matter either2")
art3 = ArticleController(2).create(
entry_id="thisisnotatest",
user_id=2,
feed_id=feed2['id'],
title="garbage no see pattern garbage",
content="doesn't matter",
link="doesn't matter either")
art4 = ArticleController(2).create(
entry_id="thisisnotatesteither",
user_id=2,
feed_id=feed2['id'],
title="garbage see pattern garbage",
content="doesn't matter2",
link="doesn't matter either2")
art5 = ArticleController(2).create(
entry_id="thisisnotatest",
feed_id=feed3['id'],
title="garbage pattern1 garbage",
content="doesn't matter",
link="doesn't matter either")
art6 = ArticleController(2).create(
entry_id="thisisnotatesteither",
feed_id=feed3['id'],
title="garbage pattern2 garbage",
content="doesn't matter2",
link="doesn't matter either2")
art7 = ArticleController(2).create(
entry_id="thisisnotatesteither",
feed_id=feed3['id'],
title="garbage no see pattern3 garbage",
content="doesn't matter3",
link="doesn't matter either3")
art8 = ArticleController(2).create(
entry_id="thisisnotatesteither",
feed_id=feed3['id'],
title="garbage pattern4 garbage",
content="doesn't matter4",
link="doesn't matter either4")
self.assertTrue(art1.readed)
self.assertFalse(art1.like)
self.assertFalse(art2.readed)
self.assertFalse(art2.like)
self.assertFalse(art3.readed)
self.assertFalse(art3.like)
self.assertFalse(art4.readed)
self.assertFalse(art4.like)
self.assertFalse(art5.readed)
self.assertFalse(art5.like)
self.assertFalse(art6.readed)
self.assertFalse(art6.like)
self.assertTrue(art7.readed)
self.assertTrue(art7.like)
self.assertFalse(art8.readed)
self.assertTrue(art8.like)
def test_create_using_filters(self):
feed_ctr = FeedController(2)
feed1 = feed_ctr.read()[0].dump()
feed2 = feed_ctr.read()[1].dump()
feed3 = feed_ctr.read()[2].dump()
feed_ctr.update({'id': feed1['id']},
{'filters': [{"type": "simple match",
"pattern": "no see pattern",
"action on": "match",
"action": "mark as read"}]})
feed_ctr.update({'id': feed3['id']},
{'filters': [{"type": "regex",
"pattern": ".*(pattern1|pattern2).*",
"action on": "no match",
"action": "mark as favorite"},
{"type": "simple match",
"pattern": "no see pattern",
"action on": "match",
"action": "mark as read"}]})
art1 = ArticleController(2).create(
entry_id="thisisnotatest",
feed_id=feed1['id'],
title="garbage no see pattern garbage",
content="doesn't matter",
link="doesn't matter either")
art2 = ArticleController(2).create(
entry_id="thisisnotatesteither",
feed_id=feed1['id'],
title="garbage see pattern garbage",
content="doesn't matter2",
link="doesn't matter either2")
art3 = ArticleController(2).create(
entry_id="thisisnotatest",
user_id=2,
feed_id=feed2['id'],
title="garbage no see pattern garbage",
content="doesn't matter",
link="doesn't matter either")
art4 = ArticleController(2).create(
entry_id="thisisnotatesteither",
user_id=2,
feed_id=feed2['id'],
title="garbage see pattern garbage",
content="doesn't matter2",
link="doesn't matter either2")
art5 = ArticleController(2).create(
entry_id="thisisnotatest",
feed_id=feed3['id'],
title="garbage pattern1 garbage",
content="doesn't matter",
link="doesn't matter either")
art6 = ArticleController(2).create(
entry_id="thisisnotatesteither",
feed_id=feed3['id'],
title="garbage pattern2 garbage",
content="doesn't matter2",
link="doesn't matter either2")
art7 = ArticleController(2).create(
entry_id="thisisnotatesteither",
feed_id=feed3['id'],
title="garbage no see pattern3 garbage",
content="doesn't matter3",
link="doesn't matter either3")
art8 = ArticleController(2).create(
entry_id="thisisnotatesteither",
feed_id=feed3['id'],
title="garbage pattern4 garbage",
content="doesn't matter4",
link="doesn't matter either4")
self.assertTrue(art1.readed)
self.assertFalse(art1.like)
self.assertFalse(art2.readed)
self.assertFalse(art2.like)
self.assertFalse(art3.readed)
self.assertFalse(art3.like)
self.assertFalse(art4.readed)
self.assertFalse(art4.like)
self.assertFalse(art5.readed)
self.assertFalse(art5.like)
self.assertFalse(art6.readed)
self.assertFalse(art6.like)
self.assertTrue(art7.readed)
self.assertTrue(art7.like)
self.assertFalse(art8.readed)
self.assertTrue(art8.like)
def test_feed_article_deletion(self):
feed_ctr = FeedController(2)
feed = feed_ctr.read()[0].dump()
feed_ctr.delete(feed['id'])
self.assertFalse(0,
ArticleController().read(feed_id=feed['id']).count())