def test_get_public_tutos(self): # Start with 0 self.assertEqual(len(self.user1.get_public_tutos()), 0) # Create Tuto ! publictuto = PublishableContentFactory(type="TUTORIAL") publictuto.authors.add(self.user1.user) publictuto.gallery = GalleryFactory() publictuto.sha_public = "whatever" publictuto.save() # Should be 0 because publication was not used publictutos = self.user1.get_public_tutos() self.assertEqual(len(publictutos), 0) PublishedContentFactory(author_list=[self.user1.user]) self.assertEqual(len(self.user1.get_public_tutos()), 1)
def test_get_public_articles(self): # Start with 0 self.assertEqual(len(self.user1.get_public_articles()), 0) # Create article ! article = PublishableContentFactory(type="ARTICLE") article.authors.add(self.user1.user) article.sha_public = "whatever" article.save() # Should be 0 articles = self.user1.get_public_articles() self.assertEqual(len(articles), 0) # Should be 1 PublishedContentFactory(author_list=[self.user1.user], type="ARTICLE") self.assertEqual(len(self.user1.get_public_articles()), 1) self.assertEqual(len(self.user1.get_public_tutos()), 0)
def test_char_count_after_publication(self): """Test the ``get_char_count()`` function. Special care should be taken with this function, since: - The username of the author is, by default "Firmxxx" where "xxx" depends on the tests before ; - The titles (!) also contains a number that also depends on the number of tests before ; - The date is ``datetime.now()`` and contains the months, which is never a fixed number of letters. """ author = ProfileFactory().user author.username = "******" author.save() len_date_now = len(date(datetime.now(), "d F Y")) article = PublishedContentFactory(type="ARTICLE", author_list=[author], title="Un titre") published = PublishedContent.objects.filter(content=article).first() self.assertEqual(published.get_char_count(), 160 + len_date_now) tuto = PublishableContentFactory(type="TUTORIAL", author_list=[author], title="Un titre") # add a chapter, so it becomes a middle tutorial tuto_draft = tuto.load_version() chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto, title="Un chapitre") ExtractFactory(container=chapter1, db_object=tuto, title="Un extrait") published = publish_content(tuto, tuto_draft, is_major_update=True) tuto.sha_public = tuto_draft.current_version tuto.sha_draft = tuto_draft.current_version tuto.public_version = published tuto.save() published = PublishedContent.objects.filter(content=tuto).first() self.assertEqual(published.get_char_count(), 335 + len_date_now)
def test_upercase_and_lowercase_search_give_same_results(self): """Pretty self-explanatory function name, isn't it ?""" if not self.manager.connected_to_es: return # 1. Index lowercase stuffs text_lc = "test" topic_1_lc = TopicFactory(forum=self.forum, author=self.user, title=text_lc) tag_lc = TagFactory(title=text_lc) topic_1_lc.tags.add(tag_lc) topic_1_lc.subtitle = text_lc topic_1_lc.save() post_1_lc = PostFactory(topic=topic_1_lc, author=self.user, position=1) post_1_lc.text = post_1_lc.text_html = text_lc post_1_lc.save() tuto_lc = PublishableContentFactory(type="TUTORIAL") tuto_draft_lc = tuto_lc.load_version() tuto_lc.title = text_lc tuto_lc.authors.add(self.user) subcategory_lc = SubCategoryFactory(title=text_lc) tuto_lc.subcategory.add(subcategory_lc) tuto_lc.tags.add(tag_lc) tuto_lc.save() tuto_draft_lc.description = text_lc tuto_draft_lc.repo_update_top_container(text_lc, tuto_lc.slug, text_lc, text_lc) chapter1_lc = ContainerFactory(parent=tuto_draft_lc, db_object=tuto_lc) extract_lc = ExtractFactory(container=chapter1_lc, db_object=tuto_lc) extract_lc.repo_update(text_lc, text_lc) published_lc = publish_content(tuto_lc, tuto_draft_lc, is_major_update=True) tuto_lc.sha_public = tuto_draft_lc.current_version tuto_lc.sha_draft = tuto_draft_lc.current_version tuto_lc.public_version = published_lc tuto_lc.save() # 2. Index uppercase stuffs text_uc = "TEST" topic_1_uc = TopicFactory(forum=self.forum, author=self.user, title=text_uc) topic_1_uc.tags.add( tag_lc) # Note: a constraint forces tags title to be unique topic_1_uc.subtitle = text_uc topic_1_uc.save() post_1_uc = PostFactory(topic=topic_1_uc, author=self.user, position=1) post_1_uc.text = post_1_uc.text_html = text_uc post_1_uc.save() tuto_uc = PublishableContentFactory(type="TUTORIAL") tuto_draft_uc = tuto_uc.load_version() tuto_uc.title = text_uc tuto_uc.authors.add(self.user) tuto_uc.subcategory.add(subcategory_lc) tuto_uc.tags.add(tag_lc) tuto_uc.save() tuto_draft_uc.description = text_uc tuto_draft_uc.repo_update_top_container(text_uc, tuto_uc.slug, text_uc, text_uc) chapter1_uc = ContainerFactory(parent=tuto_draft_uc, db_object=tuto_uc) extract_uc = ExtractFactory(container=chapter1_uc, db_object=tuto_uc) extract_uc.repo_update(text_uc, text_uc) published_uc = publish_content(tuto_uc, tuto_draft_uc, is_major_update=True) tuto_uc.sha_public = tuto_draft_uc.current_version tuto_uc.sha_draft = tuto_draft_uc.current_version tuto_uc.public_version = published_uc tuto_uc.save() # 3. Index and search: self.assertEqual( len( self.manager.setup_search(Search().query( MatchAll())).execute()), 0) # index for model in self.indexable: if model is FakeChapter: continue self.manager.es_bulk_indexing_of_model(model) self.manager.refresh_index() result = self.client.get(reverse("search:query") + "?q=" + text_lc, follow=False) self.assertEqual(result.status_code, 200) response_lc = result.context["object_list"].execute() self.assertEqual(response_lc.hits.total, 8) result = self.client.get(reverse("search:query") + "?q=" + text_uc, follow=False) self.assertEqual(result.status_code, 200) response_uc = result.context["object_list"].execute() self.assertEqual(response_uc.hits.total, 8) for responses in zip( response_lc, response_uc): # we should get results in the same order! self.assertEqual(responses[0].meta.id, responses[1].meta.id)
def test_change_publishedcontents_impacts_chapter(self): if not self.manager.connected_to_es: return # 1. Create middle-size content and index it text = "test" tuto = PublishableContentFactory(type="TUTORIAL") tuto_draft = tuto.load_version() tuto.title = text tuto.authors.add(self.user) tuto.save() tuto_draft.repo_update_top_container( text, tuto.slug, text, text) # change title to be sure it will match chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto) chapter1.repo_update(text, text, text) extract = ExtractFactory(container=chapter1, db_object=tuto) extract.repo_update(text, text) published = publish_content(tuto, tuto_draft, is_major_update=True) tuto.sha_public = tuto_draft.current_version tuto.sha_draft = tuto_draft.current_version tuto.public_version = published tuto.save() self.manager.es_bulk_indexing_of_model(PublishedContent) self.manager.refresh_index() self.assertEqual( len( self.manager.setup_search(Search().query( MatchAll())).execute()), 2) # indexing ok result = self.client.get(reverse("search:query") + "?q=" + text + "&models=content", follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 2) chapters = [r for r in response if r.meta.doc_type == "chapter"] self.assertEqual(chapters[0].meta.doc_type, FakeChapter.get_es_document_type()) self.assertEqual(chapters[0].meta.id, published.content_public_slug + "__" + chapter1.slug) # 2. Change tuto: delete chapter and insert new one ! tuto = PublishableContent.objects.get(pk=tuto.pk) tuto_draft = tuto.load_version() tuto_draft.children[0].repo_delete() # chapter 1 is gone ! another_text = "another thing" self.assertTrue( text not in another_text ) # to prevent a future modification from breaking this test chapter2 = ContainerFactory(parent=tuto_draft, db_object=tuto) chapter2.repo_update(another_text, another_text, another_text) extract2 = ExtractFactory(container=chapter2, db_object=tuto) extract2.repo_update(another_text, another_text) published = publish_content(tuto, tuto_draft, is_major_update=False) tuto.sha_public = tuto_draft.current_version tuto.sha_draft = tuto_draft.current_version tuto.public_version = published tuto.save() self.manager.es_bulk_indexing_of_model(PublishedContent) self.manager.refresh_index() self.assertEqual( len( self.manager.setup_search(Search().query( MatchAll())).execute()), 2) # 2 objects, not 3 ! result = self.client.get(reverse("search:query") + "?q=" + text + "&models=content", follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() contents = [r for r in response if r.meta.doc_type != "chapter"] self.assertEqual(response.hits.total, len(contents)) # no chapter found anymore result = self.client.get(reverse("search:query") + "?q=" + another_text + "&models=content", follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() chapters = [r for r in response if r.meta.doc_type == "chapter"] self.assertEqual(response.hits.total, 1) self.assertEqual(chapters[0].meta.doc_type, FakeChapter.get_es_document_type()) self.assertEqual(chapters[0].meta.id, published.content_public_slug + "__" + chapter2.slug) # got new chapter
def test_basic_search(self): """Basic search and filtering""" if not self.manager.connected_to_es: return # 1. Index and test search: text = "test" topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text) post_1 = PostFactory(topic=topic_1, author=self.user, position=1) post_1.text = post_1.text_html = text post_1.save() # create a middle-size content and publish it tuto = PublishableContentFactory(type="TUTORIAL") tuto_draft = tuto.load_version() tuto.title = text tuto.authors.add(self.user) tuto.save() tuto_draft.repo_update_top_container( text, tuto.slug, text, text) # change title to be sure it will match chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto) extract = ExtractFactory(container=chapter1, db_object=tuto) extract.repo_update(text, text) published = publish_content(tuto, tuto_draft, is_major_update=True) tuto.sha_public = tuto_draft.current_version tuto.sha_draft = tuto_draft.current_version tuto.public_version = published tuto.save() # nothing has been indexed yet: self.assertEqual( len( self.manager.setup_search(Search().query( MatchAll())).execute()), 0) # index for model in self.indexable: if model is FakeChapter: continue self.manager.es_bulk_indexing_of_model(model) self.manager.refresh_index() result = self.client.get(reverse("search:query") + "?q=" + text, follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 4) # get 4 results # 2. Test filtering: topic_1 = Topic.objects.get(pk=topic_1.pk) post_1 = Post.objects.get(pk=post_1.pk) published = PublishedContent.objects.get(pk=published.pk) ids = { "topic": [topic_1.es_id], "post": [post_1.es_id], "content": [ published.es_id, published.content_public_slug + "__" + chapter1.slug ], } search_groups = [ k for k, v in settings.ZDS_APP["search"]["search_groups"].items() ] group_to_model = { k: v[1] for k, v in settings.ZDS_APP["search"]["search_groups"].items() } for doc_type in search_groups: result = self.client.get(reverse("search:query") + "?q=" + text + "&models=" + doc_type, follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, len(ids[doc_type])) # get 1 result of each … for i, r in enumerate(response): self.assertIn( r.meta.doc_type, group_to_model[doc_type]) # … and only of the right type … self.assertEqual(r.meta.id, ids[doc_type][i]) # … with the right id !
def test_boosts(self): """Check if boosts are doing their job""" if not self.manager.connected_to_es: return # 1. Create topics (with identical titles), posts (with identical texts), an article and a tuto text = "test" topic_1_solved_sticky = TopicFactory(forum=self.forum, author=self.user) topic_1_solved_sticky.title = text topic_1_solved_sticky.subtitle = "" topic_1_solved_sticky.solved_by = self.user topic_1_solved_sticky.is_sticky = True topic_1_solved_sticky.save() post_1 = PostFactory(topic=topic_1_solved_sticky, author=self.user, position=1) post_1.text = post_1.text_html = text post_1.save() post_2_useful = PostFactory(topic=topic_1_solved_sticky, author=self.user, position=2) post_2_useful.text = post_2_useful.text_html = text post_2_useful.is_useful = True post_2_useful.like = 5 post_2_useful.dislike = 2 # l/d ratio above 1 post_2_useful.save() topic_2_locked = TopicFactory(forum=self.forum, author=self.user, title=text) topic_2_locked.title = text topic_2_locked.subtitle = "" topic_2_locked.is_locked = True topic_2_locked.save() post_3_ld_below_1 = PostFactory(topic=topic_2_locked, author=self.user, position=1) post_3_ld_below_1.text = post_3_ld_below_1.text_html = text post_3_ld_below_1.like = 2 post_3_ld_below_1.dislike = 5 # l/d ratio below 1 post_3_ld_below_1.save() tuto = PublishableContentFactory(type="TUTORIAL") tuto_draft = tuto.load_version() tuto.title = text tuto.authors.add(self.user) tuto.save() tuto_draft.repo_update_top_container(text, tuto.slug, text, text) chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto) chapter1.repo_update(text, "Who cares ?", "Same here") ExtractFactory(container=chapter1, db_object=tuto) published_tuto = publish_content(tuto, tuto_draft, is_major_update=True) tuto.sha_public = tuto_draft.current_version tuto.sha_draft = tuto_draft.current_version tuto.public_version = published_tuto tuto.save() article = PublishedContentFactory(type="ARTICLE", title=text) published_article = PublishedContent.objects.get(content_pk=article.pk) opinion_not_picked = PublishedContentFactory(type="OPINION", title=text) published_opinion_not_picked = PublishedContent.objects.get( content_pk=opinion_not_picked.pk) opinion_picked = PublishedContentFactory(type="OPINION", title=text) opinion_picked.sha_picked = opinion_picked.sha_draft opinion_picked.date_picked = datetime.datetime.now() opinion_picked.save() published_opinion_picked = PublishedContent.objects.get( content_pk=opinion_picked.pk) for model in self.indexable: if model is FakeChapter: continue self.manager.es_bulk_indexing_of_model(model) self.manager.refresh_index() self.assertEqual( len( self.manager.setup_search(Search().query( MatchAll())).execute()), 10) # 2. Reset all boosts to 1 for doc_type in settings.ZDS_APP["search"]["boosts"]: for key in settings.ZDS_APP["search"]["boosts"][doc_type]: settings.ZDS_APP["search"]["boosts"][doc_type][key] = 1.0 # 3. Test posts result = self.client.get(reverse("search:query") + "?q=" + text + "&models=" + Post.get_es_document_type(), follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 3) # score are equals without boost: self.assertTrue(response[0].meta.score == response[1].meta.score == response[2].meta.score) settings.ZDS_APP["search"]["boosts"]["post"]["if_first"] = 2.0 result = self.client.get(reverse("search:query") + "?q=" + text + "&models=" + Post.get_es_document_type(), follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 3) self.assertTrue(response[0].meta.score == response[1].meta.score > response[2].meta.score) self.assertEqual(response[2].meta.id, str( post_2_useful.pk)) # post 2 is the only one not first settings.ZDS_APP["search"]["boosts"]["post"]["if_first"] = 1.0 settings.ZDS_APP["search"]["boosts"]["post"]["if_useful"] = 2.0 result = self.client.get(reverse("search:query") + "?q=" + text + "&models=" + Post.get_es_document_type(), follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 3) self.assertTrue(response[0].meta.score > response[1].meta.score == response[2].meta.score) self.assertEqual(response[0].meta.id, str(post_2_useful.pk)) # post 2 is useful settings.ZDS_APP["search"]["boosts"]["post"]["if_useful"] = 1.0 settings.ZDS_APP["search"]["boosts"]["post"]["ld_ratio_above_1"] = 2.0 result = self.client.get(reverse("search:query") + "?q=" + text + "&models=" + Post.get_es_document_type(), follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 3) self.assertTrue(response[0].meta.score > response[1].meta.score == response[2].meta.score) self.assertEqual(response[0].meta.id, str( post_2_useful.pk)) # post 2 have a l/d ratio of 5/2 settings.ZDS_APP["search"]["boosts"]["post"]["ld_ratio_above_1"] = 1.0 settings.ZDS_APP["search"]["boosts"]["post"][ "ld_ratio_below_1"] = 2.0 # no one would do that in real life result = self.client.get(reverse("search:query") + "?q=" + text + "&models=" + Post.get_es_document_type(), follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 3) self.assertTrue(response[0].meta.score > response[1].meta.score == response[2].meta.score) self.assertEqual(response[0].meta.id, str( post_3_ld_below_1.pk)) # post 3 have a l/d ratio of 2/5 settings.ZDS_APP["search"]["boosts"]["post"]["ld_ratio_below_1"] = 1.0 # 4. Test topics result = self.client.get(reverse("search:query") + "?q=" + text + "&models=" + Topic.get_es_document_type(), follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 2) # score are equals without boost: self.assertTrue(response[0].meta.score == response[1].meta.score) settings.ZDS_APP["search"]["boosts"]["topic"]["if_sticky"] = 2.0 result = self.client.get(reverse("search:query") + "?q=" + text + "&models=" + Topic.get_es_document_type(), follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 2) self.assertTrue(response[0].meta.score > response[1].meta.score) self.assertEqual(response[0].meta.id, str(topic_1_solved_sticky.pk)) # topic 1 is sticky settings.ZDS_APP["search"]["boosts"]["topic"]["if_sticky"] = 1.0 settings.ZDS_APP["search"]["boosts"]["topic"]["if_solved"] = 2.0 result = self.client.get(reverse("search:query") + "?q=" + text + "&models=" + Topic.get_es_document_type(), follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 2) self.assertTrue(response[0].meta.score > response[1].meta.score) self.assertEqual(response[0].meta.id, str(topic_1_solved_sticky.pk)) # topic 1 is solved settings.ZDS_APP["search"]["boosts"]["topic"]["if_solved"] = 1.0 settings.ZDS_APP["search"]["boosts"]["topic"][ "if_locked"] = 2.0 # no one would do that in real life result = self.client.get(reverse("search:query") + "?q=" + text + "&models=" + Topic.get_es_document_type(), follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 2) self.assertTrue(response[0].meta.score > response[1].meta.score) self.assertEqual(response[0].meta.id, str(topic_2_locked.pk)) # topic 2 is locked settings.ZDS_APP["search"]["boosts"]["topic"][ "if_locked"] = 1.0 # no one would do that in real life # 5. Test published contents result = self.client.get(reverse("search:query") + "?q=" + text + "&models=content", follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 5) # score are equals without boost: self.assertTrue( response[0].meta.score == response[1].meta.score == response[2]. meta.score == response[3].meta.score == response[4].meta.score) settings.ZDS_APP["search"]["boosts"]["publishedcontent"][ "if_article"] = 2.0 result = self.client.get(reverse("search:query") + "?q=" + text + "&models=content", follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 5) self.assertTrue(response[0].meta.score > response[1].meta.score) self.assertEqual(response[0].meta.id, str(published_article.pk)) # obvious settings.ZDS_APP["search"]["boosts"]["publishedcontent"][ "if_article"] = 1.0 settings.ZDS_APP["search"]["boosts"]["publishedcontent"][ "if_tutorial"] = 2.0 result = self.client.get(reverse("search:query") + "?q=" + text + "&models=content", follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 5) self.assertTrue(response[0].meta.score > response[1].meta.score) self.assertEqual(response[0].meta.id, str(published_tuto.pk)) # obvious settings.ZDS_APP["search"]["boosts"]["publishedcontent"][ "if_tutorial"] = 1.0 settings.ZDS_APP["search"]["boosts"]["publishedcontent"][ "if_opinion"] = 2.0 settings.ZDS_APP["search"]["boosts"]["publishedcontent"][ "if_opinion_not_picked"] = 4.0 # Note: in "real life", unpicked opinion would get a boost < 1. result = self.client.get(reverse("search:query") + "?q=" + text + "&models=content", follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 5) self.assertTrue(response[0].meta.score > response[1].meta.score > response[2].meta.score) self.assertEqual( response[0].meta.id, str(published_opinion_not_picked.pk)) # unpicked opinion got first self.assertEqual(response[1].meta.id, str(published_opinion_picked.pk)) settings.ZDS_APP["search"]["boosts"]["publishedcontent"][ "if_opinion"] = 1.0 settings.ZDS_APP["search"]["boosts"]["publishedcontent"][ "if_opinion_not_picked"] = 1.0 settings.ZDS_APP["search"]["boosts"]["publishedcontent"][ "if_medium_or_big_tutorial"] = 2.0 result = self.client.get(reverse("search:query") + "?q=" + text + "&models=content", follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 5) self.assertTrue(response[0].meta.score > response[1].meta.score) self.assertEqual(response[0].meta.id, str(published_tuto.pk)) # obvious settings.ZDS_APP["search"]["boosts"]["publishedcontent"][ "if_medium_or_big_tutorial"] = 1.0 # 6. Test global boosts # NOTE: score are NOT the same for all documents, no matter how hard it tries to, small differences exists for model in self.indexable: # set a huge number to overcome the small differences: settings.ZDS_APP["search"]["boosts"][ model.get_es_document_type()]["global"] = 10.0 result = self.client.get(reverse("search:query") + "?q=" + text, follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 10) self.assertEqual(response[0].meta.doc_type, model.get_es_document_type()) # obvious settings.ZDS_APP["search"]["boosts"][ model.get_es_document_type()]["global"] = 1.0
def test_category_and_subcategory_impact_search(self): """If two contents do not belong to the same (sub)category""" if not self.manager.connected_to_es: return text = "Did you ever hear the tragedy of Darth Plagueis The Wise?" # 1. Create two contents with different subcategories category_1 = "category 1" subcategory_1 = SubCategoryFactory(title=category_1) category_2 = "category 2" subcategory_2 = SubCategoryFactory(title=category_2) tuto_1 = PublishableContentFactory(type="TUTORIAL") tuto_1_draft = tuto_1.load_version() tuto_1.title = text tuto_1.authors.add(self.user) tuto_1.subcategory.add(subcategory_1) tuto_1.save() tuto_1_draft.description = text tuto_1_draft.repo_update_top_container(text, tuto_1.slug, text, text) chapter_1 = ContainerFactory(parent=tuto_1_draft, db_object=tuto_1) extract_1 = ExtractFactory(container=chapter_1, db_object=tuto_1) extract_1.repo_update(text, text) published_1 = publish_content(tuto_1, tuto_1_draft, is_major_update=True) tuto_1.sha_public = tuto_1_draft.current_version tuto_1.sha_draft = tuto_1_draft.current_version tuto_1.public_version = published_1 tuto_1.save() tuto_2 = PublishableContentFactory(type="TUTORIAL") tuto_2_draft = tuto_2.load_version() tuto_2.title = text tuto_2.authors.add(self.user) tuto_2.subcategory.add(subcategory_2) tuto_2.save() tuto_2_draft.description = text tuto_2_draft.repo_update_top_container(text, tuto_2.slug, text, text) chapter_2 = ContainerFactory(parent=tuto_2_draft, db_object=tuto_2) extract_2 = ExtractFactory(container=chapter_2, db_object=tuto_2) extract_2.repo_update(text, text) published_2 = publish_content(tuto_2, tuto_2_draft, is_major_update=True) tuto_2.sha_public = tuto_2_draft.current_version tuto_2.sha_draft = tuto_2_draft.current_version tuto_2.public_version = published_2 tuto_2.save() # 2. Index: self.assertEqual( len( self.manager.setup_search(Search().query( MatchAll())).execute()), 0) # index for model in self.indexable: if model is FakeChapter: continue self.manager.es_bulk_indexing_of_model(model) self.manager.refresh_index() result = self.client.get(reverse("search:query") + "?q=" + text, follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 4) # Ok # 3. Test result = self.client.get(reverse("search:query") + "?q=" + text + "&model=content&subcategory=" + subcategory_1.slug, follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 2) self.assertEqual([ int(r.meta.id) for r in response if r.meta.doc_type == "publishedcontent" ][0], published_1.pk) self.assertEqual([ r.meta.id for r in response if r.meta.doc_type == "chapter" ][0], tuto_1.slug + "__" + chapter_1.slug) result = self.client.get(reverse("search:query") + "?q=" + text + "&model=content&subcategory=" + subcategory_2.slug, follow=False) self.assertEqual(result.status_code, 200) response = result.context["object_list"].execute() self.assertEqual(response.hits.total, 2) self.assertEqual([ int(r.meta.id) for r in response if r.meta.doc_type == "publishedcontent" ][0], published_2.pk) self.assertEqual([ r.meta.id for r in response if r.meta.doc_type == "chapter" ][0], tuto_2.slug + "__" + chapter_2.slug)
def test_es_manager(self): """Test the behavior of the ``es_manager`` command""" if not self.index_manager.connected_to_es: return # in the beginning: the void self.assertTrue(self.index_manager.index not in self.index_manager.es.cat.indices()) text = "Ceci est un texte de test" # create a topic with a post topic = TopicFactory(forum=self.forum, author=self.user, title=text) post = PostFactory(topic=topic, author=self.user, position=1) post.text = post.text_html = text post.save() topic = Topic.objects.get(pk=topic.pk) post = Post.objects.get(pk=post.pk) self.assertFalse(topic.es_already_indexed) self.assertTrue(topic.es_flagged) self.assertFalse(post.es_already_indexed) self.assertTrue(post.es_flagged) # create a middle-tutorial and publish it tuto = PublishableContentFactory(type="TUTORIAL") tuto.authors.add(self.user) tuto.save() tuto_draft = tuto.load_version() chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto) chapter1.repo_update(text, text, text) extract1 = ExtractFactory(container=chapter1, db_object=tuto) version = extract1.repo_update(text, text) published = publish_content(tuto, tuto_draft, is_major_update=True) tuto.sha_public = version tuto.sha_draft = version tuto.public_version = published tuto.save() published = PublishedContent.objects.get(content_pk=tuto.pk) self.assertFalse(published.es_already_indexed) self.assertTrue(published.es_flagged) # 1. test "index-all" call_command("es_manager", "index_all") self.assertTrue( self.index_manager.es.indices.exists(self.index_manager.index)) self.index_manager.index_exists = True topic = Topic.objects.get(pk=topic.pk) post = Post.objects.get(pk=post.pk) self.assertTrue(topic.es_already_indexed) self.assertFalse(topic.es_flagged) self.assertTrue(post.es_already_indexed) self.assertFalse(post.es_flagged) published = PublishedContent.objects.get(content_pk=tuto.pk) self.assertTrue(published.es_already_indexed) self.assertFalse(published.es_flagged) s = Search() s.query(MatchAll()) results = self.index_manager.setup_search(s).execute() self.assertEqual(len(results), 4) # get 4 results, one of each type must_contain = { "post": False, "topic": False, "publishedcontent": False, "chapter": False } id_must_be = { "post": str(post.pk), "topic": str(topic.pk), "publishedcontent": str(published.pk), "chapter": tuto.slug + "__" + chapter1.slug, } for hit in results: doc_type = hit.meta.doc_type must_contain[doc_type] = True self.assertEqual(hit.meta.id, id_must_be[doc_type]) self.assertTrue(all(must_contain)) # 2. test "clear" self.assertTrue(self.index_manager.index in self.index_manager.es.cat.indices()) # index in call_command("es_manager", "clear") self.assertFalse( self.index_manager.es.indices.exists(self.index_manager.index)) self.index_manager.index_exists = False # must reset every object topic = Topic.objects.get(pk=topic.pk) post = Post.objects.get(pk=post.pk) self.assertFalse(topic.es_already_indexed) self.assertTrue(topic.es_flagged) self.assertFalse(post.es_already_indexed) self.assertTrue(post.es_flagged) published = PublishedContent.objects.get(content_pk=tuto.pk) self.assertFalse(published.es_already_indexed) self.assertTrue(published.es_flagged) self.assertTrue( self.index_manager.index not in self.index_manager.es.cat.indices()) # index wiped out ! # 3. test "setup" call_command("es_manager", "setup") self.assertTrue( self.index_manager.es.indices.exists(self.index_manager.index)) self.index_manager.index_exists = True self.assertTrue( self.index_manager.index in self.index_manager.es.cat.indices()) # index back in ... s = Search() s.query(MatchAll()) results = self.index_manager.setup_search(s).execute() self.assertEqual(len(results), 0) # ... but with nothing in it result = self.index_manager.es.indices.get_settings( index=self.index_manager.index) settings_index = result[self.index_manager.index]["settings"]["index"] self.assertTrue("analysis" in settings_index) # custom analyzer was setup # 4. test "index-flagged" once ... call_command("es_manager", "index_flagged") topic = Topic.objects.get(pk=topic.pk) post = Post.objects.get(pk=post.pk) self.assertTrue(topic.es_already_indexed) self.assertFalse(topic.es_flagged) self.assertTrue(post.es_already_indexed) self.assertFalse(post.es_flagged) published = PublishedContent.objects.get(content_pk=tuto.pk) self.assertTrue(published.es_already_indexed) self.assertFalse(published.es_flagged) s = Search() s.query(MatchAll()) results = self.index_manager.setup_search(s).execute() self.assertEqual(len(results), 4) # get the 4 results back
def test_special_case_of_contents(self): """test that the old publishedcontent does not stay when a new one is created""" if not self.manager.connected_to_es: return # 1. Create a middle-tutorial, publish it, then index it tuto = PublishableContentFactory(type="TUTORIAL") tuto.authors.add(self.user) tuto.save() tuto_draft = tuto.load_version() chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto) ExtractFactory(container=chapter1, db_object=tuto) published = publish_content(tuto, tuto_draft, is_major_update=True) tuto.sha_public = tuto_draft.current_version tuto.sha_draft = tuto_draft.current_version tuto.public_version = published tuto.save() self.manager.es_bulk_indexing_of_model(PublishedContent, force_reindexing=True) # index self.manager.refresh_index() first_publication = PublishedContent.objects.get(content_pk=tuto.pk) self.assertTrue(first_publication.es_already_indexed) self.assertFalse(first_publication.es_flagged) s = Search() s.query(MatchAll()) results = self.manager.setup_search(s).execute() self.assertEqual( len(results), 2) # get 2 results, one for the content and one for the chapter self.assertEqual(PublishedContent.objects.count(), 1) # 2. Change thet title, which will trigger a change in the slug tuto = PublishableContent.objects.get(pk=tuto.pk) versioned = tuto.load_version(sha=tuto.sha_draft) tuto.title = "un titre complètement différent!" tuto.save(force_slug_update=True) versioned.repo_update_top_container(tuto.title, tuto.slug, "osef", "osef") second_publication = publish_content(tuto, versioned, True) tuto.sha_public = versioned.current_version tuto.sha_draft = versioned.current_version tuto.public_version = second_publication tuto.save() self.assertEqual(PublishedContent.objects.count(), 2) # now there is two objects ... first_publication = PublishedContent.objects.get( pk=first_publication.pk) self.assertTrue(first_publication.must_redirect ) # .. including the first one, for redirection self.manager.refresh_index() s = Search() s.query(MatchAll()) results = self.manager.setup_search(s).execute() self.assertEqual( len(results), 0 ) # the old one is gone (and we need to reindex to get the new one) # 3. Check if indexation brings the new one, and not the old one self.manager.es_bulk_indexing_of_model(PublishedContent, force_reindexing=True) # index self.manager.refresh_index() first_publication = PublishedContent.objects.get( pk=first_publication.pk) second_publication = PublishedContent.objects.get( pk=second_publication.pk) s = Search() s.query(MatchAll()) results = self.manager.setup_search(s).execute() self.assertEqual(len(results), 2) # Still 2, not 4 ! found_old = False found_new = False for hit in results: if hit.meta.doc_type == PublishedContent.get_es_document_type(): if hit.meta.id == first_publication.es_id: found_old = True if hit.meta.id == second_publication.es_id: found_new = True self.assertTrue(found_new) self.assertFalse(found_old)
def test_indexation(self): """test the indexation and deletion of the different documents""" if not self.manager.connected_to_es: return # create a topic with a post topic = TopicFactory(forum=self.forum, author=self.user) post = PostFactory(topic=topic, author=self.user, position=1) topic = Topic.objects.get(pk=topic.pk) post = Post.objects.get(pk=post.pk) self.assertFalse(topic.es_already_indexed) self.assertTrue(topic.es_flagged) self.assertFalse(post.es_already_indexed) self.assertTrue(post.es_flagged) # create a middle-tutorial and publish it tuto = PublishableContentFactory(type="TUTORIAL") tuto.authors.add(self.user) tuto.save() tuto_draft = tuto.load_version() chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto) ExtractFactory(container=chapter1, db_object=tuto) published = publish_content(tuto, tuto_draft, is_major_update=True) tuto.sha_public = tuto_draft.current_version tuto.sha_draft = tuto_draft.current_version tuto.public_version = published tuto.save() published = PublishedContent.objects.get(content_pk=tuto.pk) self.assertFalse(published.es_already_indexed) self.assertTrue(published.es_flagged) # 1. index all for model in self.indexable: if model is FakeChapter: continue self.manager.es_bulk_indexing_of_model(model, force_reindexing=False) self.manager.refresh_index() topic = Topic.objects.get(pk=topic.pk) post = Post.objects.get(pk=post.pk) self.assertTrue(topic.es_already_indexed) self.assertFalse(topic.es_flagged) self.assertTrue(post.es_already_indexed) self.assertFalse(post.es_flagged) published = PublishedContent.objects.get(content_pk=tuto.pk) self.assertTrue(published.es_already_indexed) self.assertFalse(published.es_flagged) s = Search() s.query(MatchAll()) results = self.manager.setup_search(s).execute() self.assertEqual(len(results), 4) # get 4 results, one of each type must_contain = { "post": False, "topic": False, "publishedcontent": False, "chapter": False } id_must_be = { "post": str(post.pk), "topic": str(topic.pk), "publishedcontent": str(published.pk), "chapter": tuto.slug + "__" + chapter1.slug, } for hit in results: doc_type = hit.meta.doc_type must_contain[doc_type] = True self.assertEqual(hit.meta.id, id_must_be[doc_type]) self.assertTrue(all(must_contain)) # 2. Test what reindexation will do: new_topic = TopicFactory(forum=self.forum, author=self.user) new_post = PostFactory(topic=new_topic, author=self.user, position=1) pk_of_topics_to_reindex = [] for item in Topic.get_es_indexable(force_reindexing=False): pk_of_topics_to_reindex.append(item.pk) pk_of_posts_to_reindex = [] for item in Post.get_es_indexable(force_reindexing=False): pk_of_posts_to_reindex.append(item.pk) self.assertTrue(topic.pk not in pk_of_topics_to_reindex) self.assertTrue(new_topic.pk in pk_of_topics_to_reindex) self.assertTrue(post.pk not in pk_of_posts_to_reindex) self.assertTrue(new_post.pk in pk_of_posts_to_reindex) for model in self.indexable: # ok, so let's index that if model is FakeChapter: continue self.manager.es_bulk_indexing_of_model(model, force_reindexing=False) self.manager.refresh_index() s = Search() s.query(MatchAll()) results = self.manager.setup_search(s).execute() self.assertEqual(len(results), 6) # good! # 3. Test single deletion: new_post = Post.objects.get(pk=new_post.pk) self.manager.delete_document(new_post) self.manager.refresh_index() s = Search() s.query(MatchAll()) results = self.manager.setup_search(s).execute() self.assertEqual(len(results), 5) # one is missing for hit in results: self.assertTrue(hit.meta.doc_type != Post.get_es_document_type() or hit.meta.id != new_post.es_id) # 4. Test "delete_by_query_deletion": topic = Topic.objects.get(pk=topic.pk) new_topic = Topic.objects.get(pk=new_topic.pk) self.manager.delete_by_query(Topic.get_es_document_type(), MatchAll()) # the two topic are deleted self.manager.refresh_index() s = Search() s.query(MatchAll()) results = self.manager.setup_search(s).execute() self.assertEqual(len(results), 3) for hit in results: self.assertTrue(hit.meta.doc_type != Topic.get_es_document_type() or hit.meta.id != new_topic.es_id) self.assertTrue(hit.meta.doc_type != Topic.get_es_document_type() or hit.meta.id != topic.es_id) # 5. Test that the deletion of an object also triggers its deletion in ES post = Post.objects.get(pk=post.pk) post.delete() self.manager.refresh_index() s = Search() s.query(MatchAll()) results = self.manager.setup_search(s).execute() self.assertEqual(len(results), 2) for hit in results: self.assertTrue(hit.meta.doc_type != Post.get_es_document_type() or hit.meta.id != post.es_id) # 6. Test full desindexation: for model in self.indexable: if model is FakeChapter: continue self.manager.clear_indexing_of_model(model) # note "topic" is gone since "post" is gone, due to relationships at the Django level new_topic = Topic.objects.get(pk=new_topic.pk) new_post = Post.objects.get(pk=new_post.pk) self.assertFalse(new_topic.es_already_indexed) self.assertTrue(new_topic.es_flagged) self.assertFalse(new_post.es_already_indexed) self.assertTrue(new_post.es_flagged) published = PublishedContent.objects.get(content_pk=tuto.pk) self.assertFalse(published.es_already_indexed) self.assertTrue(published.es_flagged)