def test_fetchable(self): fctrl = FeedController() total = fctrl.read().count() unix = datetime(1970, 1, 1).replace(tzinfo=timezone.utc) count = 0 for fd in fctrl.list_late(): count += 1 self.assertEqual(unix, fd.last_retrieved) self.assertEqual(unix, fd.expires) self.assertEqual(total, count) fetchables = fctrl.list_fetchable() now = utc_now() for fd in fetchables: self.assert_in_range(now - timedelta(seconds=1), fd.last_retrieved, now) self.assertEqual(unix, fd.expires) self.assert_late_count( 0, "no late feed to report because all just fetched") fctrl.update({}, {'expires': unix}) now = utc_now() for fd in fctrl.read(): # expires should be corrected self.assert_in_range( now + timedelta(seconds=conf.feed.min_expires - 1), fd.expires, now + timedelta(seconds=conf.feed.min_expires + 1)) lr_not_matter = timedelta(seconds=conf.feed.min_expires + 10) self.update_all_no_ctrl(expires=utc_now() - timedelta(seconds=1), last_retrieved=utc_now() - lr_not_matter) self.assert_late_count(total, "all feed just expired") self.update_all_no_ctrl(expires=utc_now() + timedelta(seconds=1)) self.assert_late_count( 0, "all feed will expire in a second, none are expired")
def test_adding_to_cluster_by_link(self): ccontr = ClusterController() cluster = ccontr.read().first() ccontr.update({'id': cluster.id}, { 'read': True, 'read_reason': 'marked' }) cluster = ccontr.get(id=cluster.id) self.assertTrue(cluster.read) article = cluster.articles[0] articles_count = len(cluster.articles) fcontr = FeedController(cluster.user_id) acontr = ArticleController(cluster.user_id) fcontr.update({'id': article.feed_id}, {'cluster_wake_up': True}) feed = fcontr.read(id__ne=article.feed_id).first() update_on_all_objs(articles=[article], feeds=[feed], cluster_enabled=True) self._clone_article(acontr, article, feed) ccontr.clusterize_pending_articles() cluster = ccontr.get(id=cluster.id) self.assertEqual(articles_count + 1, len(cluster.articles)) self.assertFalse(cluster.read)
def _test_unread_on_cluster(self, read_reason): ccontr = ClusterController() fcontr = FeedController() cluster = ccontr.read().first() clusterizer = Clusterizer() self.assertFalse(clusterizer.get_config(cluster, 'cluster_enabled')) self.assertTrue(clusterizer.get_config(cluster, 'cluster_wake_up')) ccontr.update({'id': cluster.id}, { 'read': True, 'read_reason': read_reason }) target_feed = fcontr.read(id__ne=cluster.main_article.feed_id, user_id=cluster.user_id).first() clusterizer = Clusterizer() self.assertFalse(clusterizer.get_config(target_feed, 'cluster_enabled')) fcontr.update( {'id__in': [f.id for f in cluster.feeds] + [target_feed.id]}, { 'cluster_wake_up': True, 'cluster_enabled': True }) clusterizer = Clusterizer() self.assertTrue(clusterizer.get_config(cluster, 'cluster_enabled')) target_feed = fcontr.read(id__ne=cluster.main_article.feed_id, user_id=cluster.user_id).first() article = self._clone_article(ArticleController(), cluster.main_article, target_feed) clusterizer = Clusterizer() self.assertTrue(clusterizer.get_config(article, 'cluster_wake_up')) ClusterController(cluster.user_id).clusterize_pending_articles() self.assertEqual(2, len(article.cluster.articles)) self.assertInCluster(article, cluster) return ccontr.get(id=cluster.id)
def put(feed_id): """Update an existing feed.""" fctrl = FeedController(current_identity.id) attrs = parse_meaningful_params(parser_edit) changed = fctrl.update({'id': feed_id}, attrs) if not changed: fctrl.assert_right_ok(feed_id) return None, 204
def reset_feeds(): """Will reschedule all active feeds to be fetched in the next two hours""" fcontr = FeedController(ignore_context=True) now = utc_now() feeds = [ feed[0] for feed in fcontr.get_active_feed().with_entities(fcontr._db_cls.id) ] step = timedelta(seconds=conf.feed.max_expires / len(feeds)) for i, feed_id in enumerate(feeds): fcontr.update( {'id': feed_id}, { 'etag': '', 'last_modified': '', 'last_retrieved': datetime(1970, 1, 1, tzinfo=timezone.utc), 'expires': now + i * step })
def feed_cleaner(feed_id): logger.warning("Feed cleaner - start => %s", feed_id) WORKER_BATCH.labels(worker_type='delete').observe(1) fctrl = FeedController() result = fctrl.update({'id': feed_id, 'status': FeedStatus.to_delete}, {'status': FeedStatus.deleting}) if not result: logger.error('feed %r seems locked, not doing anything', feed_id) return try: logger.warning("Deleting feed %r", feed_id) fctrl.delete(feed_id) except Exception: logger.exception('something went wrong when deleting feeds %r', feed_id) fctrl.update({'id': feed_id}, {'status': FeedStatus.to_delete}) raise finally: REDIS_CONN.delete(JARR_FEED_DEL_KEY)
def test_time(self): naive = dateutil.parser.parse('2016-11-17T16:18:02.727802') aware = dateutil.parser.parse('2016-11-17T16:18:02.727802+00:00') aware2 = dateutil.parser.parse('2016-11-17T16:18:02.727802+12:00') fctrl = FeedController() fctrl.update({'id': 1}, {'last_retrieved': naive}) self.assertEqual(fctrl.read(id=1).first().last_retrieved, aware) fctrl.update({'id': 1}, {'last_retrieved': aware}) self.assertEqual(fctrl.read(id=1).first().last_retrieved, aware) fctrl.update({'id': 1}, {'last_retrieved': aware2}) self.assertEqual(fctrl.read(id=1).first().last_retrieved, aware2) self.assertEqual( fctrl.read(id=1).first().last_retrieved, aware - timedelta(hours=12))
def _test_create_using_filters(self): # FIXME wait redo filters feed_ctr = FeedController(USER_ID) acontr = ArticleController(USER_ID) feed1, feed2, feed3 = [f for f in feed_ctr.read()][0:3] feed_ctr.update({'id': feed3.id}, { 'cluster_enabled': True, 'filters': [{ "type": "regex", "pattern": ".*(pattern1|pattern2).*", "action on": "no match", "action": "mark as favorite" }, { "type": "simple match", "pattern": "pattern3", "action on": "match", "action": "mark as read" }] }) feed_ctr.update({'id': feed1.id}, { 'filters': [{ "type": "simple match", "pattern": "pattern3", "action on": "match", "action": "mark as read" }] }) feed_ctr.update({'id': feed2.id}, { 'filters': [{ "type": "tag match", "pattern": "pattern4", "action on": "match", "action": "skipped" }, { "type": "tag contains", "pattern": "pattern5", "action on": "match", "action": "skipped" }] }) art1 = acontr.create(entry_id="will be read and faved 1", feed_id=feed1.id, title="garbage pattern1 pattern3 garbage", content="doesn't matter", link="cluster1") art2 = acontr.create(entry_id="will be ignored 2", feed_id=feed1.id, title="garbage see pattern garbage", content="doesn't matter2", link="is ignored 2") art3 = acontr.create(entry_id="will be read 3", user_id=2, feed_id=feed2.id, title="garbage pattern3 garbage", content="doesn't matter", link="doesn't matter either3") art4 = acontr.create(entry_id="will be ignored 4", user_id=2, feed_id=feed2.id, title="garbage see pattern garbage", content="doesn't matter2", link="doesn't matter either4") art5 = acontr.create(entry_id="will be faved 5", feed_id=feed3.id, title="garbage anti-attern3 garbage", content="doesn't matter", link="cluster1") art6 = acontr.create(entry_id="will be faved 6", feed_id=feed3.id, title="garbage pattern1 garbage", content="doesn't matter2", link="doesn't matter 6") art7 = acontr.create(entry_id="will be read 7", feed_id=feed3.id, title="garbage pattern3 garbage", content="doesn't matter3", link="doesn't matter either7") art8 = acontr.create(entry_id="will be ignored", feed_id=feed3.id, title="garbage pattern4 garbage", content="doesn't matter4-matter4_matter4", lang='fa_ke', link="doesn't matter either8") art9 = acontr.create(entry_id="unique9", feed_id=feed2.id, title="garbage", tags=['garbage', 'pattern4'], content="doesn't matterç", link="doesn't matter either9") art10 = acontr.create(entry_id="will be ignored", feed_id=feed2.id, title="garbage", tags=['pattern5 garbage', 'garbage'], content="doesn't matter10", link="doesn't matter either10") ClusterController(USER_ID).clusterize_pending_articles() self.assertTrue(acontr.get(id=art1.id).cluster.read) self.assertFalse(acontr.get(id=art1.id).cluster.liked) self.assertFalse(acontr.get(id=art2.id).cluster.read) self.assertFalse(acontr.get(id=art2.id).cluster.liked) self.assertFalse(acontr.get(id=art3.id).cluster.read) self.assertFalse(acontr.get(id=art3.id).cluster.liked) self.assertFalse(acontr.get(id=art4.id).cluster.read) self.assertFalse(acontr.get(id=art4.id).cluster.liked) self.assertTrue(art5.cluster.read, "should be read because it clustered") self.assertTrue(art5.cluster.liked) self.assertFalse(art6.cluster.read) self.assertFalse(art6.cluster.liked) self.assertTrue(art7.cluster.read) self.assertTrue(art7.cluster.liked) self.assertFalse(art8.cluster.read) self.assertTrue(art8.cluster.liked) self.assertIsNone(art9) self.assertEqual(0, acontr.read(entry_id='unique9').count()) self.assertIsNone(art10) self.assertEqual(0, acontr.read(entry_id='unique10').count())
def delete(feed_id): """Delete an existing feed.""" fctrl = FeedController(current_identity.id) if not fctrl.update({'id': feed_id}, {'status': FeedStatus.to_delete}): fctrl.assert_right_ok(feed_id) return None, 204