def test_delete_main_cluster_handling(self):
suffix = 'suffix'
clu = ClusterController().get(id=10)
acontr = ArticleController(clu.user_id)
fcontr = FeedController(clu.user_id)
old_title = clu.main_title
old_feed_title, old_art_id = clu.main_feed_title, clu.main_article_id
for art_to_del in acontr.read(link=clu.main_article.link,
id__ne=clu.main_article.id):
acontr.delete(art_to_del.id)
other_feed = fcontr.read(id__ne=clu.main_article.feed_id).first()
update_on_all_objs(articles=[clu.main_article],
feeds=[other_feed],
cluster_enabled=True)
acontr.create(
feed_id=other_feed.id,
entry_id=clu.main_article.entry_id + suffix,
link=clu.main_article.link,
title=clu.main_article.title + suffix,
content=clu.main_article.content + suffix,
date=clu.main_article.date + timedelta(1),
retrieved_date=clu.main_article.retrieved_date + timedelta(1),
)
ClusterController(clu.user_id).clusterize_pending_articles()
clu = ClusterController().get(id=10)
self.assertEqual(2, len(clu.articles))
fcontr.delete(clu.main_article.feed_id)
new_cluster = ClusterController(clu.user_id).get(id=clu.id)
self.assertEqual(1, len(new_cluster.articles))
self.assertNotEqual(old_title, new_cluster.main_title)
self.assertNotEqual(old_feed_title, new_cluster.main_feed_title)
self.assertNotEqual(old_art_id, new_cluster.main_article_id)
def test_no_mixup(self):
acontr = ArticleController()
ccontr = ClusterController()
total_clusters = len(list(ccontr.read()))
total_articles = len(list(acontr.read()))
for cluster in ccontr.read():
self.assertEqual(1, len(cluster.articles))
for article in acontr.read():
acontr.create(entry_id=article.entry_id,
feed_id=article.feed_id,
title=article.title,
content=article.content,
link=article.link)
for user_id in ArticleController.get_user_id_with_pending_articles():
ClusterController(user_id).clusterize_pending_articles()
self.assertEqual(2 * total_articles, len(list(acontr.read())))
self.assertEqual(2 * total_clusters, len(list(ccontr.read())))
for cluster in ccontr.read():
self.assertEqual(1, len(cluster.articles))
self.assertEqual(1, len({a.user_id for a in cluster.articles}))
main_article = acontr.read().first()
for article in acontr.read():
acontr.create(user_id=main_article.user_id,
feed_id=main_article.feed_id,
entry_id=article.entry_id,
title=article.title,
content=article.content,
link=article.link)
for cluster in ccontr.read():
self.assertEqual(1, len({a.user_id for a in cluster.articles}))
def create_article_from(self, cluster, feed, link=None):
self.assertEqual(cluster.user_id, feed.user_id)
suffix = str(randint(0, 9999))
acontr = ArticleController(cluster.user_id)
article = acontr.create(
feed_id=feed.id,
entry_id=cluster.main_article.entry_id + suffix,
link=link or cluster.main_article.link,
title=cluster.main_article.title + suffix,
content=cluster.main_article.content + suffix,
date=cluster.main_article.date + timedelta(1),
retrieved_date=cluster.main_article.retrieved_date)
ClusterController(cluster.user_id).clusterize_pending_articles()
return acontr.read(id=article.id).first()
def create_missing_article(self, response):
logger.info('%r: cache validation failed, challenging entries',
self.feed)
parsed = self.parse_feed_response(response)
if parsed is None:
return
ids, entries, skipped_list = [], {}, []
for entry in parsed['entries']:
if not entry:
continue
builder = self.article_builder(self.feed, entry)
if builder.do_skip_creation:
skipped_list.append(builder.entry_ids)
logger.debug('%r: skipping article', self.feed)
continue
entry_ids = builder.entry_ids
entries[tuple(sorted(entry_ids.items()))] = builder
ids.append(entry_ids)
if not ids and skipped_list:
logger.debug('%r: nothing to add (skipped %r) %r', self.feed,
skipped_list, parsed)
return
logger.debug("%r: found %d entries %r", self.feed, len(ids), ids)
article_created = False
actrl = ArticleController(self.feed.user_id)
new_entries_ids = list(actrl.challenge(ids=ids))
logger.debug("%r: %d entries wern't matched and will be created",
self.feed, len(new_entries_ids))
for id_to_create in new_entries_ids:
article_created = True
builder = entries[tuple(sorted(id_to_create.items()))]
new_article = builder.enhance()
article = actrl.create(**new_article)
logger.info('%r: created %r', self.feed, article)
if not article_created:
logger.info('%r: all article matched in db, adding nothing',
self.feed)
def _test_create_using_filters(self):
# FIXME wait redo filters
feed_ctr = FeedController(USER_ID)
acontr = ArticleController(USER_ID)
feed1, feed2, feed3 = [f for f in feed_ctr.read()][0:3]
feed_ctr.update({'id': feed3.id}, {
'cluster_enabled':
True,
'filters': [{
"type": "regex",
"pattern": ".*(pattern1|pattern2).*",
"action on": "no match",
"action": "mark as favorite"
}, {
"type": "simple match",
"pattern": "pattern3",
"action on": "match",
"action": "mark as read"
}]
})
feed_ctr.update({'id': feed1.id}, {
'filters': [{
"type": "simple match",
"pattern": "pattern3",
"action on": "match",
"action": "mark as read"
}]
})
feed_ctr.update({'id': feed2.id}, {
'filters': [{
"type": "tag match",
"pattern": "pattern4",
"action on": "match",
"action": "skipped"
}, {
"type": "tag contains",
"pattern": "pattern5",
"action on": "match",
"action": "skipped"
}]
})
art1 = acontr.create(entry_id="will be read and faved 1",
feed_id=feed1.id,
title="garbage pattern1 pattern3 garbage",
content="doesn't matter",
link="cluster1")
art2 = acontr.create(entry_id="will be ignored 2",
feed_id=feed1.id,
title="garbage see pattern garbage",
content="doesn't matter2",
link="is ignored 2")
art3 = acontr.create(entry_id="will be read 3",
user_id=2,
feed_id=feed2.id,
title="garbage pattern3 garbage",
content="doesn't matter",
link="doesn't matter either3")
art4 = acontr.create(entry_id="will be ignored 4",
user_id=2,
feed_id=feed2.id,
title="garbage see pattern garbage",
content="doesn't matter2",
link="doesn't matter either4")
art5 = acontr.create(entry_id="will be faved 5",
feed_id=feed3.id,
title="garbage anti-attern3 garbage",
content="doesn't matter",
link="cluster1")
art6 = acontr.create(entry_id="will be faved 6",
feed_id=feed3.id,
title="garbage pattern1 garbage",
content="doesn't matter2",
link="doesn't matter 6")
art7 = acontr.create(entry_id="will be read 7",
feed_id=feed3.id,
title="garbage pattern3 garbage",
content="doesn't matter3",
link="doesn't matter either7")
art8 = acontr.create(entry_id="will be ignored",
feed_id=feed3.id,
title="garbage pattern4 garbage",
content="doesn't matter4-matter4_matter4",
lang='fa_ke',
link="doesn't matter either8")
art9 = acontr.create(entry_id="unique9",
feed_id=feed2.id,
title="garbage",
tags=['garbage', 'pattern4'],
content="doesn't matterç",
link="doesn't matter either9")
art10 = acontr.create(entry_id="will be ignored",
feed_id=feed2.id,
title="garbage",
tags=['pattern5 garbage', 'garbage'],
content="doesn't matter10",
link="doesn't matter either10")
ClusterController(USER_ID).clusterize_pending_articles()
self.assertTrue(acontr.get(id=art1.id).cluster.read)
self.assertFalse(acontr.get(id=art1.id).cluster.liked)
self.assertFalse(acontr.get(id=art2.id).cluster.read)
self.assertFalse(acontr.get(id=art2.id).cluster.liked)
self.assertFalse(acontr.get(id=art3.id).cluster.read)
self.assertFalse(acontr.get(id=art3.id).cluster.liked)
self.assertFalse(acontr.get(id=art4.id).cluster.read)
self.assertFalse(acontr.get(id=art4.id).cluster.liked)
self.assertTrue(art5.cluster.read,
"should be read because it clustered")
self.assertTrue(art5.cluster.liked)
self.assertFalse(art6.cluster.read)
self.assertFalse(art6.cluster.liked)
self.assertTrue(art7.cluster.read)
self.assertTrue(art7.cluster.liked)
self.assertFalse(art8.cluster.read)
self.assertTrue(art8.cluster.liked)
self.assertIsNone(art9)
self.assertEqual(0, acontr.read(entry_id='unique9').count())
self.assertIsNone(art10)
self.assertEqual(0, acontr.read(entry_id='unique10').count())
