def test_delete_main_cluster_handling(self): suffix = 'suffix' clu = ClusterController().get(id=10) acontr = ArticleController(clu.user_id) fcontr = FeedController(clu.user_id) old_title = clu.main_title old_feed_title, old_art_id = clu.main_feed_title, clu.main_article_id for art_to_del in acontr.read(link=clu.main_article.link, id__ne=clu.main_article.id): acontr.delete(art_to_del.id) other_feed = fcontr.read(id__ne=clu.main_article.feed_id).first() update_on_all_objs(articles=[clu.main_article], feeds=[other_feed], cluster_enabled=True) acontr.create( feed_id=other_feed.id, entry_id=clu.main_article.entry_id + suffix, link=clu.main_article.link, title=clu.main_article.title + suffix, content=clu.main_article.content + suffix, date=clu.main_article.date + timedelta(1), retrieved_date=clu.main_article.retrieved_date + timedelta(1), ) ClusterController(clu.user_id).clusterize_pending_articles() clu = ClusterController().get(id=10) self.assertEqual(2, len(clu.articles)) fcontr.delete(clu.main_article.feed_id) new_cluster = ClusterController(clu.user_id).get(id=clu.id) self.assertEqual(1, len(new_cluster.articles)) self.assertNotEqual(old_title, new_cluster.main_title) self.assertNotEqual(old_feed_title, new_cluster.main_feed_title) self.assertNotEqual(old_art_id, new_cluster.main_article_id)
def test_no_mixup(self): acontr = ArticleController() ccontr = ClusterController() total_clusters = len(list(ccontr.read())) total_articles = len(list(acontr.read())) for cluster in ccontr.read(): self.assertEqual(1, len(cluster.articles)) for article in acontr.read(): acontr.create(entry_id=article.entry_id, feed_id=article.feed_id, title=article.title, content=article.content, link=article.link) for user_id in ArticleController.get_user_id_with_pending_articles(): ClusterController(user_id).clusterize_pending_articles() self.assertEqual(2 * total_articles, len(list(acontr.read()))) self.assertEqual(2 * total_clusters, len(list(ccontr.read()))) for cluster in ccontr.read(): self.assertEqual(1, len(cluster.articles)) self.assertEqual(1, len({a.user_id for a in cluster.articles})) main_article = acontr.read().first() for article in acontr.read(): acontr.create(user_id=main_article.user_id, feed_id=main_article.feed_id, entry_id=article.entry_id, title=article.title, content=article.content, link=article.link) for cluster in ccontr.read(): self.assertEqual(1, len({a.user_id for a in cluster.articles}))
def create_article_from(self, cluster, feed, link=None): self.assertEqual(cluster.user_id, feed.user_id) suffix = str(randint(0, 9999)) acontr = ArticleController(cluster.user_id) article = acontr.create( feed_id=feed.id, entry_id=cluster.main_article.entry_id + suffix, link=link or cluster.main_article.link, title=cluster.main_article.title + suffix, content=cluster.main_article.content + suffix, date=cluster.main_article.date + timedelta(1), retrieved_date=cluster.main_article.retrieved_date) ClusterController(cluster.user_id).clusterize_pending_articles() return acontr.read(id=article.id).first()
def create_missing_article(self, response): logger.info('%r: cache validation failed, challenging entries', self.feed) parsed = self.parse_feed_response(response) if parsed is None: return ids, entries, skipped_list = [], {}, [] for entry in parsed['entries']: if not entry: continue builder = self.article_builder(self.feed, entry) if builder.do_skip_creation: skipped_list.append(builder.entry_ids) logger.debug('%r: skipping article', self.feed) continue entry_ids = builder.entry_ids entries[tuple(sorted(entry_ids.items()))] = builder ids.append(entry_ids) if not ids and skipped_list: logger.debug('%r: nothing to add (skipped %r) %r', self.feed, skipped_list, parsed) return logger.debug("%r: found %d entries %r", self.feed, len(ids), ids) article_created = False actrl = ArticleController(self.feed.user_id) new_entries_ids = list(actrl.challenge(ids=ids)) logger.debug("%r: %d entries wern't matched and will be created", self.feed, len(new_entries_ids)) for id_to_create in new_entries_ids: article_created = True builder = entries[tuple(sorted(id_to_create.items()))] new_article = builder.enhance() article = actrl.create(**new_article) logger.info('%r: created %r', self.feed, article) if not article_created: logger.info('%r: all article matched in db, adding nothing', self.feed)
def _test_create_using_filters(self): # FIXME wait redo filters feed_ctr = FeedController(USER_ID) acontr = ArticleController(USER_ID) feed1, feed2, feed3 = [f for f in feed_ctr.read()][0:3] feed_ctr.update({'id': feed3.id}, { 'cluster_enabled': True, 'filters': [{ "type": "regex", "pattern": ".*(pattern1|pattern2).*", "action on": "no match", "action": "mark as favorite" }, { "type": "simple match", "pattern": "pattern3", "action on": "match", "action": "mark as read" }] }) feed_ctr.update({'id': feed1.id}, { 'filters': [{ "type": "simple match", "pattern": "pattern3", "action on": "match", "action": "mark as read" }] }) feed_ctr.update({'id': feed2.id}, { 'filters': [{ "type": "tag match", "pattern": "pattern4", "action on": "match", "action": "skipped" }, { "type": "tag contains", "pattern": "pattern5", "action on": "match", "action": "skipped" }] }) art1 = acontr.create(entry_id="will be read and faved 1", feed_id=feed1.id, title="garbage pattern1 pattern3 garbage", content="doesn't matter", link="cluster1") art2 = acontr.create(entry_id="will be ignored 2", feed_id=feed1.id, title="garbage see pattern garbage", content="doesn't matter2", link="is ignored 2") art3 = acontr.create(entry_id="will be read 3", user_id=2, feed_id=feed2.id, title="garbage pattern3 garbage", content="doesn't matter", link="doesn't matter either3") art4 = acontr.create(entry_id="will be ignored 4", user_id=2, feed_id=feed2.id, title="garbage see pattern garbage", content="doesn't matter2", link="doesn't matter either4") art5 = acontr.create(entry_id="will be faved 5", feed_id=feed3.id, title="garbage anti-attern3 garbage", content="doesn't matter", link="cluster1") art6 = acontr.create(entry_id="will be faved 6", feed_id=feed3.id, title="garbage pattern1 garbage", content="doesn't matter2", link="doesn't matter 6") art7 = acontr.create(entry_id="will be read 7", feed_id=feed3.id, title="garbage pattern3 garbage", content="doesn't matter3", link="doesn't matter either7") art8 = acontr.create(entry_id="will be ignored", feed_id=feed3.id, title="garbage pattern4 garbage", content="doesn't matter4-matter4_matter4", lang='fa_ke', link="doesn't matter either8") art9 = acontr.create(entry_id="unique9", feed_id=feed2.id, title="garbage", tags=['garbage', 'pattern4'], content="doesn't matterç", link="doesn't matter either9") art10 = acontr.create(entry_id="will be ignored", feed_id=feed2.id, title="garbage", tags=['pattern5 garbage', 'garbage'], content="doesn't matter10", link="doesn't matter either10") ClusterController(USER_ID).clusterize_pending_articles() self.assertTrue(acontr.get(id=art1.id).cluster.read) self.assertFalse(acontr.get(id=art1.id).cluster.liked) self.assertFalse(acontr.get(id=art2.id).cluster.read) self.assertFalse(acontr.get(id=art2.id).cluster.liked) self.assertFalse(acontr.get(id=art3.id).cluster.read) self.assertFalse(acontr.get(id=art3.id).cluster.liked) self.assertFalse(acontr.get(id=art4.id).cluster.read) self.assertFalse(acontr.get(id=art4.id).cluster.liked) self.assertTrue(art5.cluster.read, "should be read because it clustered") self.assertTrue(art5.cluster.liked) self.assertFalse(art6.cluster.read) self.assertFalse(art6.cluster.liked) self.assertTrue(art7.cluster.read) self.assertTrue(art7.cluster.liked) self.assertFalse(art8.cluster.read) self.assertTrue(art8.cluster.liked) self.assertIsNone(art9) self.assertEqual(0, acontr.read(entry_id='unique9').count()) self.assertIsNone(art10) self.assertEqual(0, acontr.read(entry_id='unique10').count())