def test_fetchable(self):
fctrl = FeedController()
total = fctrl.read().count()
unix = datetime(1970, 1, 1).replace(tzinfo=timezone.utc)
count = 0
for fd in fctrl.list_late():
count += 1
self.assertEqual(unix, fd.last_retrieved)
self.assertEqual(unix, fd.expires)
self.assertEqual(total, count)
fetchables = fctrl.list_fetchable()
now = utc_now()
for fd in fetchables:
self.assert_in_range(now - timedelta(seconds=1), fd.last_retrieved,
now)
self.assertEqual(unix, fd.expires)
self.assert_late_count(
0, "no late feed to report because all just fetched")
fctrl.update({}, {'expires': unix})
now = utc_now()
for fd in fctrl.read(): # expires should be corrected
self.assert_in_range(
now + timedelta(seconds=conf.feed.min_expires - 1), fd.expires,
now + timedelta(seconds=conf.feed.min_expires + 1))
lr_not_matter = timedelta(seconds=conf.feed.min_expires + 10)
self.update_all_no_ctrl(expires=utc_now() - timedelta(seconds=1),
last_retrieved=utc_now() - lr_not_matter)
self.assert_late_count(total, "all feed just expired")
self.update_all_no_ctrl(expires=utc_now() + timedelta(seconds=1))
self.assert_late_count(
0, "all feed will expire in a second, none are expired")
def test_adding_to_cluster_by_link(self):
ccontr = ClusterController()
cluster = ccontr.read().first()
ccontr.update({'id': cluster.id}, {
'read': True,
'read_reason': 'marked'
})
cluster = ccontr.get(id=cluster.id)
self.assertTrue(cluster.read)
article = cluster.articles[0]
articles_count = len(cluster.articles)
fcontr = FeedController(cluster.user_id)
acontr = ArticleController(cluster.user_id)
fcontr.update({'id': article.feed_id}, {'cluster_wake_up': True})
feed = fcontr.read(id__ne=article.feed_id).first()
update_on_all_objs(articles=[article],
feeds=[feed],
cluster_enabled=True)
self._clone_article(acontr, article, feed)
ccontr.clusterize_pending_articles()
cluster = ccontr.get(id=cluster.id)
self.assertEqual(articles_count + 1, len(cluster.articles))
self.assertFalse(cluster.read)
def _test_unread_on_cluster(self, read_reason):
ccontr = ClusterController()
fcontr = FeedController()
cluster = ccontr.read().first()
clusterizer = Clusterizer()
self.assertFalse(clusterizer.get_config(cluster, 'cluster_enabled'))
self.assertTrue(clusterizer.get_config(cluster, 'cluster_wake_up'))
ccontr.update({'id': cluster.id}, {
'read': True,
'read_reason': read_reason
})
target_feed = fcontr.read(id__ne=cluster.main_article.feed_id,
user_id=cluster.user_id).first()
clusterizer = Clusterizer()
self.assertFalse(clusterizer.get_config(target_feed,
'cluster_enabled'))
fcontr.update(
{'id__in': [f.id for f in cluster.feeds] + [target_feed.id]}, {
'cluster_wake_up': True,
'cluster_enabled': True
})
clusterizer = Clusterizer()
self.assertTrue(clusterizer.get_config(cluster, 'cluster_enabled'))
target_feed = fcontr.read(id__ne=cluster.main_article.feed_id,
user_id=cluster.user_id).first()
article = self._clone_article(ArticleController(),
cluster.main_article, target_feed)
clusterizer = Clusterizer()
self.assertTrue(clusterizer.get_config(article, 'cluster_wake_up'))
ClusterController(cluster.user_id).clusterize_pending_articles()
self.assertEqual(2, len(article.cluster.articles))
self.assertInCluster(article, cluster)
return ccontr.get(id=cluster.id)
def put(feed_id):
"""Update an existing feed."""
fctrl = FeedController(current_identity.id)
attrs = parse_meaningful_params(parser_edit)
changed = fctrl.update({'id': feed_id}, attrs)
if not changed:
fctrl.assert_right_ok(feed_id)
return None, 204
def reset_feeds():
"""Will reschedule all active feeds to be fetched in the next two hours"""
fcontr = FeedController(ignore_context=True)
now = utc_now()
feeds = [
feed[0]
for feed in fcontr.get_active_feed().with_entities(fcontr._db_cls.id)
]
step = timedelta(seconds=conf.feed.max_expires / len(feeds))
for i, feed_id in enumerate(feeds):
fcontr.update(
{'id': feed_id}, {
'etag': '',
'last_modified': '',
'last_retrieved': datetime(1970, 1, 1, tzinfo=timezone.utc),
'expires': now + i * step
})
def feed_cleaner(feed_id):
logger.warning("Feed cleaner - start => %s", feed_id)
WORKER_BATCH.labels(worker_type='delete').observe(1)
fctrl = FeedController()
result = fctrl.update({'id': feed_id, 'status': FeedStatus.to_delete},
{'status': FeedStatus.deleting})
if not result:
logger.error('feed %r seems locked, not doing anything', feed_id)
return
try:
logger.warning("Deleting feed %r", feed_id)
fctrl.delete(feed_id)
except Exception:
logger.exception('something went wrong when deleting feeds %r',
feed_id)
fctrl.update({'id': feed_id}, {'status': FeedStatus.to_delete})
raise
finally:
REDIS_CONN.delete(JARR_FEED_DEL_KEY)
def test_time(self):
naive = dateutil.parser.parse('2016-11-17T16:18:02.727802')
aware = dateutil.parser.parse('2016-11-17T16:18:02.727802+00:00')
aware2 = dateutil.parser.parse('2016-11-17T16:18:02.727802+12:00')
fctrl = FeedController()
fctrl.update({'id': 1}, {'last_retrieved': naive})
self.assertEqual(fctrl.read(id=1).first().last_retrieved, aware)
fctrl.update({'id': 1}, {'last_retrieved': aware})
self.assertEqual(fctrl.read(id=1).first().last_retrieved, aware)
fctrl.update({'id': 1}, {'last_retrieved': aware2})
self.assertEqual(fctrl.read(id=1).first().last_retrieved, aware2)
self.assertEqual(
fctrl.read(id=1).first().last_retrieved,
aware - timedelta(hours=12))
def _test_create_using_filters(self):
# FIXME wait redo filters
feed_ctr = FeedController(USER_ID)
acontr = ArticleController(USER_ID)
feed1, feed2, feed3 = [f for f in feed_ctr.read()][0:3]
feed_ctr.update({'id': feed3.id}, {
'cluster_enabled':
True,
'filters': [{
"type": "regex",
"pattern": ".*(pattern1|pattern2).*",
"action on": "no match",
"action": "mark as favorite"
}, {
"type": "simple match",
"pattern": "pattern3",
"action on": "match",
"action": "mark as read"
}]
})
feed_ctr.update({'id': feed1.id}, {
'filters': [{
"type": "simple match",
"pattern": "pattern3",
"action on": "match",
"action": "mark as read"
}]
})
feed_ctr.update({'id': feed2.id}, {
'filters': [{
"type": "tag match",
"pattern": "pattern4",
"action on": "match",
"action": "skipped"
}, {
"type": "tag contains",
"pattern": "pattern5",
"action on": "match",
"action": "skipped"
}]
})
art1 = acontr.create(entry_id="will be read and faved 1",
feed_id=feed1.id,
title="garbage pattern1 pattern3 garbage",
content="doesn't matter",
link="cluster1")
art2 = acontr.create(entry_id="will be ignored 2",
feed_id=feed1.id,
title="garbage see pattern garbage",
content="doesn't matter2",
link="is ignored 2")
art3 = acontr.create(entry_id="will be read 3",
user_id=2,
feed_id=feed2.id,
title="garbage pattern3 garbage",
content="doesn't matter",
link="doesn't matter either3")
art4 = acontr.create(entry_id="will be ignored 4",
user_id=2,
feed_id=feed2.id,
title="garbage see pattern garbage",
content="doesn't matter2",
link="doesn't matter either4")
art5 = acontr.create(entry_id="will be faved 5",
feed_id=feed3.id,
title="garbage anti-attern3 garbage",
content="doesn't matter",
link="cluster1")
art6 = acontr.create(entry_id="will be faved 6",
feed_id=feed3.id,
title="garbage pattern1 garbage",
content="doesn't matter2",
link="doesn't matter 6")
art7 = acontr.create(entry_id="will be read 7",
feed_id=feed3.id,
title="garbage pattern3 garbage",
content="doesn't matter3",
link="doesn't matter either7")
art8 = acontr.create(entry_id="will be ignored",
feed_id=feed3.id,
title="garbage pattern4 garbage",
content="doesn't matter4-matter4_matter4",
lang='fa_ke',
link="doesn't matter either8")
art9 = acontr.create(entry_id="unique9",
feed_id=feed2.id,
title="garbage",
tags=['garbage', 'pattern4'],
content="doesn't matterç",
link="doesn't matter either9")
art10 = acontr.create(entry_id="will be ignored",
feed_id=feed2.id,
title="garbage",
tags=['pattern5 garbage', 'garbage'],
content="doesn't matter10",
link="doesn't matter either10")
ClusterController(USER_ID).clusterize_pending_articles()
self.assertTrue(acontr.get(id=art1.id).cluster.read)
self.assertFalse(acontr.get(id=art1.id).cluster.liked)
self.assertFalse(acontr.get(id=art2.id).cluster.read)
self.assertFalse(acontr.get(id=art2.id).cluster.liked)
self.assertFalse(acontr.get(id=art3.id).cluster.read)
self.assertFalse(acontr.get(id=art3.id).cluster.liked)
self.assertFalse(acontr.get(id=art4.id).cluster.read)
self.assertFalse(acontr.get(id=art4.id).cluster.liked)
self.assertTrue(art5.cluster.read,
"should be read because it clustered")
self.assertTrue(art5.cluster.liked)
self.assertFalse(art6.cluster.read)
self.assertFalse(art6.cluster.liked)
self.assertTrue(art7.cluster.read)
self.assertTrue(art7.cluster.liked)
self.assertFalse(art8.cluster.read)
self.assertTrue(art8.cluster.liked)
self.assertIsNone(art9)
self.assertEqual(0, acontr.read(entry_id='unique9').count())
self.assertIsNone(art10)
self.assertEqual(0, acontr.read(entry_id='unique10').count())
def delete(feed_id):
"""Delete an existing feed."""
fctrl = FeedController(current_identity.id)
if not fctrl.update({'id': feed_id}, {'status': FeedStatus.to_delete}):
fctrl.assert_right_ok(feed_id)
return None, 204