def test_hidden_post_are_not_result(self):
"""Hidden posts should not show up in the search results"""
if not self.manager.connected_to_es:
return
# 1. Index and test search:
text = 'test'
topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text)
post_1 = PostFactory(topic=topic_1, author=self.user, position=1)
post_1.text = post_1.text_html = text
post_1.save()
self.manager.es_bulk_indexing_of_model(Topic)
self.manager.es_bulk_indexing_of_model(Post)
self.manager.refresh_index()
self.assertEqual(
len(
self.manager.setup_search(Search().query(
MatchAll())).execute()), 2) # indexing ok
post_1 = Post.objects.get(pk=post_1.pk)
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 1)
self.assertEqual(response[0].meta.id, post_1.es_id)
# 2. Hide, reindex and search again:
post_1.hide_comment_by_user(self.staff,
'Un abus de pouvoir comme un autre ;)')
self.manager.refresh_index()
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 0) # nothing in the results
def test_hidden_post_are_not_result(self):
"""Hidden posts should not show up in the search results"""
if not self.manager.connected_to_es:
return
# 1. Index and test search:
text = 'test'
topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text)
post_1 = PostFactory(topic=topic_1, author=self.user, position=1)
post_1.text = post_1.text_html = text
post_1.save()
self.manager.es_bulk_indexing_of_model(Topic)
self.manager.es_bulk_indexing_of_model(Post)
self.manager.refresh_index()
self.assertEqual(len(self.manager.setup_search(Search().query(MatchAll())).execute()), 2) # indexing ok
post_1 = Post.objects.get(pk=post_1.pk)
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 1)
self.assertEqual(response[0].meta.id, post_1.es_id)
# 2. Hide, reindex and search again:
post_1.hide_comment_by_user(self.staff, 'Un abus de pouvoir comme un autre ;)')
self.manager.refresh_index()
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 0) # nothing in the results
def test_change_topic_impacts_posts(self):
if not self.manager.connected_to_es:
return
# 1. Create a hidden forum belonging to a hidden group and add staff in it.
text = 'test'
group = Group.objects.create(name='Les illuminatis anonymes de ZdS')
_, hidden_forum = create_category(group)
self.staff.groups.add(group)
self.staff.save()
# 2. Create a normal topic and index it
topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text)
post_1 = PostFactory(topic=topic_1, author=self.user, position=1)
post_1.text = post_1.text_html = text
post_1.save()
self.manager.es_bulk_indexing_of_model(Topic)
self.manager.es_bulk_indexing_of_model(Post)
self.manager.refresh_index()
self.assertEqual(
len(
self.manager.setup_search(Search().query(
MatchAll())).execute()), 2) # indexing ok
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 1) # ok
self.assertEqual(response[0].meta.doc_type,
Post.get_es_document_type())
self.assertEqual(response[0].forum_pk, self.forum.pk)
self.assertEqual(response[0].topic_pk, topic_1.pk)
self.assertEqual(response[0].topic_title, topic_1.title)
# 3. Change topic title and reindex
topic_1.title = 'new title'
topic_1.save()
self.manager.es_bulk_indexing_of_model(Topic)
self.manager.es_bulk_indexing_of_model(Post)
self.manager.refresh_index()
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 1) # ok
self.assertEqual(response[0].topic_title,
topic_1.title) # title was changed
# 4. connect with staff and move topic
self.assertTrue(
self.client.login(username=self.staff.username,
password='******'))
data = {'move': '', 'forum': hidden_forum.pk, 'topic': topic_1.pk}
response = self.client.post(reverse('topic-edit'), data, follow=False)
self.assertEqual(302, response.status_code)
self.manager.es_bulk_indexing_of_model(Topic)
self.manager.es_bulk_indexing_of_model(Post)
self.manager.refresh_index()
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(
response.hits.total,
1) # Note: without staff, would not get any results (see below)
self.assertEqual(response[0].forum_pk,
hidden_forum.pk) # post was updated with new forum
# 5. Topic is now hidden
self.client.logout()
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 0) # ok
def test_boosts(self):
"""Check if boosts are doing their job"""
if not self.manager.connected_to_es:
return
# 1. Create topics (with identical titles), posts (with identical texts), an article and a tuto
text = 'test'
topic_1_solved_sticky = TopicFactory(forum=self.forum,
author=self.user)
topic_1_solved_sticky.title = text
topic_1_solved_sticky.subtitle = ''
topic_1_solved_sticky.is_solved = True
topic_1_solved_sticky.is_sticky = True
topic_1_solved_sticky.save()
post_1 = PostFactory(topic=topic_1_solved_sticky,
author=self.user,
position=1)
post_1.text = post_1.text_html = text
post_1.save()
post_2_useful = PostFactory(topic=topic_1_solved_sticky,
author=self.user,
position=2)
post_2_useful.text = post_2_useful.text_html = text
post_2_useful.is_useful = True
post_2_useful.like = 5
post_2_useful.dislike = 2 # l/d ratio above 1
post_2_useful.save()
topic_2_locked = TopicFactory(forum=self.forum,
author=self.user,
title=text)
topic_2_locked.title = text
topic_2_locked.subtitle = ''
topic_2_locked.is_locked = True
topic_2_locked.save()
post_3_ld_below_1 = PostFactory(topic=topic_2_locked,
author=self.user,
position=1)
post_3_ld_below_1.text = post_3_ld_below_1.text_html = text
post_3_ld_below_1.like = 2
post_3_ld_below_1.dislike = 5 # l/d ratio below 1
post_3_ld_below_1.save()
tuto = PublishableContentFactory(type='TUTORIAL')
tuto_draft = tuto.load_version()
tuto.title = text
tuto.authors.add(self.user)
tuto.save()
tuto_draft.repo_update_top_container(text, tuto.slug, text, text)
chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto)
chapter1.repo_update(text, 'Who cares ?', 'Same here')
ExtractFactory(container=chapter1, db_object=tuto)
published_tuto = publish_content(tuto,
tuto_draft,
is_major_update=True)
tuto.sha_public = tuto_draft.current_version
tuto.sha_draft = tuto_draft.current_version
tuto.public_version = published_tuto
tuto.save()
article = PublishedContentFactory(type='ARTICLE', title=text)
published_article = PublishedContent.objects.get(content_pk=article.pk)
opinion_not_picked = PublishedContentFactory(type='OPINION',
title=text)
published_opinion_not_picked = PublishedContent.objects.get(
content_pk=opinion_not_picked.pk)
opinion_picked = PublishedContentFactory(type='OPINION', title=text)
opinion_picked.sha_picked = opinion_picked.sha_draft
opinion_picked.date_picked = datetime.datetime.now()
opinion_picked.save()
published_opinion_picked = PublishedContent.objects.get(
content_pk=opinion_picked.pk)
for model in self.indexable:
if model is FakeChapter:
continue
self.manager.es_bulk_indexing_of_model(model)
self.manager.refresh_index()
self.assertEqual(
len(
self.manager.setup_search(Search().query(
MatchAll())).execute()), 10)
# 2. Reset all boosts to 1
for doc_type in settings.ZDS_APP['search']['boosts']:
for key in settings.ZDS_APP['search']['boosts'][doc_type]:
settings.ZDS_APP['search']['boosts'][doc_type][key] = 1.0
# 3. Test posts
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 3)
# score are equals without boost:
self.assertTrue(response[0].meta.score == response[1].meta.score ==
response[2].meta.score)
settings.ZDS_APP['search']['boosts']['post']['if_first'] = 2.0
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 3)
self.assertTrue(response[0].meta.score == response[1].meta.score >
response[2].meta.score)
self.assertEqual(response[2].meta.id, str(
post_2_useful.pk)) # post 2 is the only one not first
settings.ZDS_APP['search']['boosts']['post']['if_first'] = 1.0
settings.ZDS_APP['search']['boosts']['post']['if_useful'] = 2.0
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 3)
self.assertTrue(response[0].meta.score > response[1].meta.score ==
response[2].meta.score)
self.assertEqual(response[0].meta.id,
str(post_2_useful.pk)) # post 2 is useful
settings.ZDS_APP['search']['boosts']['post']['if_useful'] = 1.0
settings.ZDS_APP['search']['boosts']['post']['ld_ratio_above_1'] = 2.0
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 3)
self.assertTrue(response[0].meta.score > response[1].meta.score ==
response[2].meta.score)
self.assertEqual(response[0].meta.id, str(
post_2_useful.pk)) # post 2 have a l/d ratio of 5/2
settings.ZDS_APP['search']['boosts']['post']['ld_ratio_above_1'] = 1.0
settings.ZDS_APP['search']['boosts']['post'][
'ld_ratio_below_1'] = 2.0 # no one would do that in real life
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Post.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 3)
self.assertTrue(response[0].meta.score > response[1].meta.score ==
response[2].meta.score)
self.assertEqual(response[0].meta.id, str(
post_3_ld_below_1.pk)) # post 3 have a l/d ratio of 2/5
settings.ZDS_APP['search']['boosts']['post']['ld_ratio_below_1'] = 1.0
# 4. Test topics
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Topic.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 2)
# score are equals without boost:
self.assertTrue(response[0].meta.score == response[1].meta.score)
settings.ZDS_APP['search']['boosts']['topic']['if_sticky'] = 2.0
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Topic.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 2)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id,
str(topic_1_solved_sticky.pk)) # topic 1 is sticky
settings.ZDS_APP['search']['boosts']['topic']['if_sticky'] = 1.0
settings.ZDS_APP['search']['boosts']['topic']['if_solved'] = 2.0
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Topic.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 2)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id,
str(topic_1_solved_sticky.pk)) # topic 1 is solved
settings.ZDS_APP['search']['boosts']['topic']['if_solved'] = 1.0
settings.ZDS_APP['search']['boosts']['topic'][
'if_locked'] = 2.0 # no one would do that in real life
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=' + Topic.get_es_document_type(),
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 2)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id,
str(topic_2_locked.pk)) # topic 2 is locked
settings.ZDS_APP['search']['boosts']['topic'][
'if_locked'] = 1.0 # no one would do that in real life
# 5. Test published contents
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=content',
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 5)
# score are equals without boost:
self.assertTrue(
response[0].meta.score == response[1].meta.score == response[2].
meta.score == response[3].meta.score == response[4].meta.score)
settings.ZDS_APP['search']['boosts']['publishedcontent'][
'if_article'] = 2.0
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=content',
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 5)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id,
str(published_article.pk)) # obvious
settings.ZDS_APP['search']['boosts']['publishedcontent'][
'if_article'] = 1.0
settings.ZDS_APP['search']['boosts']['publishedcontent'][
'if_tutorial'] = 2.0
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=content',
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 5)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id,
str(published_tuto.pk)) # obvious
settings.ZDS_APP['search']['boosts']['publishedcontent'][
'if_tutorial'] = 1.0
settings.ZDS_APP['search']['boosts']['publishedcontent'][
'if_opinion'] = 2.0
settings.ZDS_APP['search']['boosts']['publishedcontent'][
'if_opinion_not_picked'] = 4.0
# Note: in "real life", unpicked opinion would get a boost < 1.
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=content',
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 5)
self.assertTrue(response[0].meta.score > response[1].meta.score >
response[2].meta.score)
self.assertEqual(
response[0].meta.id,
str(published_opinion_not_picked.pk)) # unpicked opinion got first
self.assertEqual(response[1].meta.id, str(published_opinion_picked.pk))
settings.ZDS_APP['search']['boosts']['publishedcontent'][
'if_opinion'] = 1.0
settings.ZDS_APP['search']['boosts']['publishedcontent'][
'if_opinion_not_picked'] = 1.0
settings.ZDS_APP['search']['boosts']['publishedcontent'][
'if_medium_or_big_tutorial'] = 2.0
result = self.client.get(reverse('search:query') + '?q=' + text +
'&models=content',
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 5)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id,
str(published_tuto.pk)) # obvious
settings.ZDS_APP['search']['boosts']['publishedcontent'][
'if_medium_or_big_tutorial'] = 1.0
# 6. Test global boosts
# NOTE: score are NOT the same for all documents, no matter how hard it tries to, small differences exists
for model in self.indexable:
# set a huge number to overcome the small differences:
settings.ZDS_APP['search']['boosts'][
model.get_es_document_type()]['global'] = 10.0
result = self.client.get(reverse('search:query') + '?q=' + text,
follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 10)
self.assertEqual(response[0].meta.doc_type,
model.get_es_document_type()) # obvious
settings.ZDS_APP['search']['boosts'][
model.get_es_document_type()]['global'] = 1.0
def test_indexation(self):
"""test the indexation and deletion of the different documents"""
if not self.manager.connected_to_es:
return
# create a topic with a post
topic = TopicFactory(forum=self.forum, author=self.user)
post = PostFactory(topic=topic, author=self.user, position=1)
topic = Topic.objects.get(pk=topic.pk)
post = Post.objects.get(pk=post.pk)
self.assertFalse(topic.es_already_indexed)
self.assertTrue(topic.es_flagged)
self.assertFalse(post.es_already_indexed)
self.assertTrue(post.es_flagged)
# create a middle-tutorial and publish it
tuto = PublishableContentFactory(type='TUTORIAL')
tuto.authors.add(self.user)
tuto.save()
tuto_draft = tuto.load_version()
chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto)
ExtractFactory(container=chapter1, db_object=tuto)
published = publish_content(tuto, tuto_draft, is_major_update=True)
tuto.sha_public = tuto_draft.current_version
tuto.sha_draft = tuto_draft.current_version
tuto.public_version = published
tuto.save()
published = PublishedContent.objects.get(content_pk=tuto.pk)
self.assertFalse(published.es_already_indexed)
self.assertTrue(published.es_flagged)
# 1. index all
for model in self.indexable:
if model is FakeChapter:
continue
self.manager.es_bulk_indexing_of_model(model, force_reindexing=False)
self.manager.refresh_index()
topic = Topic.objects.get(pk=topic.pk)
post = Post.objects.get(pk=post.pk)
self.assertTrue(topic.es_already_indexed)
self.assertFalse(topic.es_flagged)
self.assertTrue(post.es_already_indexed)
self.assertFalse(post.es_flagged)
published = PublishedContent.objects.get(content_pk=tuto.pk)
self.assertTrue(published.es_already_indexed)
self.assertFalse(published.es_flagged)
s = Search()
s.query(MatchAll())
results = self.manager.setup_search(s).execute()
self.assertEqual(len(results), 4) # get 4 results, one of each type
must_contain = {'post': False, 'topic': False, 'publishedcontent': False, 'chapter': False}
id_must_be = {
'post': str(post.pk),
'topic': str(topic.pk),
'publishedcontent': str(published.pk),
'chapter': tuto.slug + '__' + chapter1.slug
}
for hit in results:
doc_type = hit.meta.doc_type
must_contain[doc_type] = True
self.assertEqual(hit.meta.id, id_must_be[doc_type])
self.assertTrue(all(must_contain))
# 2. Test what reindexation will do:
new_topic = TopicFactory(forum=self.forum, author=self.user)
new_post = PostFactory(topic=new_topic, author=self.user, position=1)
pk_of_topics_to_reindex = []
for item in Topic.get_es_indexable(force_reindexing=False):
pk_of_topics_to_reindex.append(item.pk)
pk_of_posts_to_reindex = []
for item in Post.get_es_indexable(force_reindexing=False):
pk_of_posts_to_reindex.append(item.pk)
self.assertTrue(topic.pk not in pk_of_topics_to_reindex)
self.assertTrue(new_topic.pk in pk_of_topics_to_reindex)
self.assertTrue(post.pk not in pk_of_posts_to_reindex)
self.assertTrue(new_post.pk in pk_of_posts_to_reindex)
for model in self.indexable: # ok, so let's index that
if model is FakeChapter:
continue
self.manager.es_bulk_indexing_of_model(model, force_reindexing=False)
self.manager.refresh_index()
s = Search()
s.query(MatchAll())
results = self.manager.setup_search(s).execute()
self.assertEqual(len(results), 6) # good!
# 3. Test single deletion:
new_post = Post.objects.get(pk=new_post.pk)
self.manager.delete_document(new_post)
self.manager.refresh_index()
s = Search()
s.query(MatchAll())
results = self.manager.setup_search(s).execute()
self.assertEqual(len(results), 5) # one is missing
for hit in results:
self.assertTrue(hit.meta.doc_type != Post.get_es_document_type() or hit.meta.id != new_post.es_id)
# 4. Test "delete_by_query_deletion":
topic = Topic.objects.get(pk=topic.pk)
new_topic = Topic.objects.get(pk=new_topic.pk)
self.manager.delete_by_query(Topic.get_es_document_type(), MatchAll()) # the two topic are deleted
self.manager.refresh_index()
s = Search()
s.query(MatchAll())
results = self.manager.setup_search(s).execute()
self.assertEqual(len(results), 3)
for hit in results:
self.assertTrue(hit.meta.doc_type != Topic.get_es_document_type() or hit.meta.id != new_topic.es_id)
self.assertTrue(hit.meta.doc_type != Topic.get_es_document_type() or hit.meta.id != topic.es_id)
# 5. Test that the deletion of an object also triggers its deletion in ES
post = Post.objects.get(pk=post.pk)
post.delete()
self.manager.refresh_index()
s = Search()
s.query(MatchAll())
results = self.manager.setup_search(s).execute()
self.assertEqual(len(results), 2)
for hit in results:
self.assertTrue(hit.meta.doc_type != Post.get_es_document_type() or hit.meta.id != post.es_id)
# 6. Test full desindexation:
for model in self.indexable:
if model is FakeChapter:
continue
self.manager.clear_indexing_of_model(model)
# note "topic" is gone since "post" is gone, due to relationships at the Django level
new_topic = Topic.objects.get(pk=new_topic.pk)
new_post = Post.objects.get(pk=new_post.pk)
self.assertFalse(new_topic.es_already_indexed)
self.assertTrue(new_topic.es_flagged)
self.assertFalse(new_post.es_already_indexed)
self.assertTrue(new_post.es_flagged)
published = PublishedContent.objects.get(content_pk=tuto.pk)
self.assertFalse(published.es_already_indexed)
self.assertTrue(published.es_flagged)
def test_change_topic_impacts_posts(self):
if not self.manager.connected_to_es:
return
# 1. Create a hidden forum belonging to a hidden group and add staff in it.
text = 'test'
group = Group.objects.create(name='Les illuminatis anonymes de ZdS')
_, hidden_forum = create_category_and_forum(group)
self.staff.groups.add(group)
self.staff.save()
# 2. Create a normal topic and index it
topic_1 = TopicFactory(forum=self.forum, author=self.user, title=text)
post_1 = PostFactory(topic=topic_1, author=self.user, position=1)
post_1.text = post_1.text_html = text
post_1.save()
self.manager.es_bulk_indexing_of_model(Topic)
self.manager.es_bulk_indexing_of_model(Post)
self.manager.refresh_index()
self.assertEqual(len(self.manager.setup_search(Search().query(MatchAll())).execute()), 2) # indexing ok
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 1) # ok
self.assertEqual(response[0].meta.doc_type, Post.get_es_document_type())
self.assertEqual(response[0].forum_pk, self.forum.pk)
self.assertEqual(response[0].topic_pk, topic_1.pk)
self.assertEqual(response[0].topic_title, topic_1.title)
# 3. Change topic title and reindex
topic_1.title = 'new title'
topic_1.save()
self.manager.es_bulk_indexing_of_model(Topic)
self.manager.es_bulk_indexing_of_model(Post)
self.manager.refresh_index()
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 1) # ok
self.assertEqual(response[0].topic_title, topic_1.title) # title was changed
# 4. connect with staff and move topic
self.assertTrue(self.client.login(username=self.staff.username, password='******'))
data = {
'move': '',
'forum': hidden_forum.pk,
'topic': topic_1.pk
}
response = self.client.post(reverse('topic-edit'), data, follow=False)
self.assertEqual(302, response.status_code)
self.manager.es_bulk_indexing_of_model(Topic)
self.manager.es_bulk_indexing_of_model(Post)
self.manager.refresh_index()
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 1) # Note: without staff, would not get any results (see below)
self.assertEqual(response[0].forum_pk, hidden_forum.pk) # post was updated with new forum
# 5. Topic is now hidden
self.client.logout()
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 0) # ok
def test_boosts(self):
"""Check if boosts are doing their job"""
if not self.manager.connected_to_es:
return
# 1. Create topics (with identical titles), posts (with identical texts), an article and a tuto
text = 'test'
topic_1_solved_sticky = TopicFactory(forum=self.forum, author=self.user)
topic_1_solved_sticky.title = text
topic_1_solved_sticky.subtitle = ''
topic_1_solved_sticky.solved_by = self.user
topic_1_solved_sticky.is_sticky = True
topic_1_solved_sticky.save()
post_1 = PostFactory(topic=topic_1_solved_sticky, author=self.user, position=1)
post_1.text = post_1.text_html = text
post_1.save()
post_2_useful = PostFactory(topic=topic_1_solved_sticky, author=self.user, position=2)
post_2_useful.text = post_2_useful.text_html = text
post_2_useful.is_useful = True
post_2_useful.like = 5
post_2_useful.dislike = 2 # l/d ratio above 1
post_2_useful.save()
topic_2_locked = TopicFactory(forum=self.forum, author=self.user, title=text)
topic_2_locked.title = text
topic_2_locked.subtitle = ''
topic_2_locked.is_locked = True
topic_2_locked.save()
post_3_ld_below_1 = PostFactory(topic=topic_2_locked, author=self.user, position=1)
post_3_ld_below_1.text = post_3_ld_below_1.text_html = text
post_3_ld_below_1.like = 2
post_3_ld_below_1.dislike = 5 # l/d ratio below 1
post_3_ld_below_1.save()
tuto = PublishableContentFactory(type='TUTORIAL')
tuto_draft = tuto.load_version()
tuto.title = text
tuto.authors.add(self.user)
tuto.save()
tuto_draft.repo_update_top_container(text, tuto.slug, text, text)
chapter1 = ContainerFactory(parent=tuto_draft, db_object=tuto)
chapter1.repo_update(text, 'Who cares ?', 'Same here')
ExtractFactory(container=chapter1, db_object=tuto)
published_tuto = publish_content(tuto, tuto_draft, is_major_update=True)
tuto.sha_public = tuto_draft.current_version
tuto.sha_draft = tuto_draft.current_version
tuto.public_version = published_tuto
tuto.save()
article = PublishedContentFactory(type='ARTICLE', title=text)
published_article = PublishedContent.objects.get(content_pk=article.pk)
opinion_not_picked = PublishedContentFactory(type='OPINION', title=text)
published_opinion_not_picked = PublishedContent.objects.get(content_pk=opinion_not_picked.pk)
opinion_picked = PublishedContentFactory(type='OPINION', title=text)
opinion_picked.sha_picked = opinion_picked.sha_draft
opinion_picked.date_picked = datetime.datetime.now()
opinion_picked.save()
published_opinion_picked = PublishedContent.objects.get(content_pk=opinion_picked.pk)
for model in self.indexable:
if model is FakeChapter:
continue
self.manager.es_bulk_indexing_of_model(model)
self.manager.refresh_index()
self.assertEqual(len(self.manager.setup_search(Search().query(MatchAll())).execute()), 10)
# 2. Reset all boosts to 1
for doc_type in settings.ZDS_APP['search']['boosts']:
for key in settings.ZDS_APP['search']['boosts'][doc_type]:
settings.ZDS_APP['search']['boosts'][doc_type][key] = 1.0
# 3. Test posts
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 3)
# score are equals without boost:
self.assertTrue(response[0].meta.score == response[1].meta.score == response[2].meta.score)
settings.ZDS_APP['search']['boosts']['post']['if_first'] = 2.0
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 3)
self.assertTrue(response[0].meta.score == response[1].meta.score > response[2].meta.score)
self.assertEqual(response[2].meta.id, str(post_2_useful.pk)) # post 2 is the only one not first
settings.ZDS_APP['search']['boosts']['post']['if_first'] = 1.0
settings.ZDS_APP['search']['boosts']['post']['if_useful'] = 2.0
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 3)
self.assertTrue(response[0].meta.score > response[1].meta.score == response[2].meta.score)
self.assertEqual(response[0].meta.id, str(post_2_useful.pk)) # post 2 is useful
settings.ZDS_APP['search']['boosts']['post']['if_useful'] = 1.0
settings.ZDS_APP['search']['boosts']['post']['ld_ratio_above_1'] = 2.0
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 3)
self.assertTrue(response[0].meta.score > response[1].meta.score == response[2].meta.score)
self.assertEqual(response[0].meta.id, str(post_2_useful.pk)) # post 2 have a l/d ratio of 5/2
settings.ZDS_APP['search']['boosts']['post']['ld_ratio_above_1'] = 1.0
settings.ZDS_APP['search']['boosts']['post']['ld_ratio_below_1'] = 2.0 # no one would do that in real life
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Post.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 3)
self.assertTrue(response[0].meta.score > response[1].meta.score == response[2].meta.score)
self.assertEqual(response[0].meta.id, str(post_3_ld_below_1.pk)) # post 3 have a l/d ratio of 2/5
settings.ZDS_APP['search']['boosts']['post']['ld_ratio_below_1'] = 1.0
# 4. Test topics
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Topic.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 2)
# score are equals without boost:
self.assertTrue(response[0].meta.score == response[1].meta.score)
settings.ZDS_APP['search']['boosts']['topic']['if_sticky'] = 2.0
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Topic.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 2)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id, str(topic_1_solved_sticky.pk)) # topic 1 is sticky
settings.ZDS_APP['search']['boosts']['topic']['if_sticky'] = 1.0
settings.ZDS_APP['search']['boosts']['topic']['if_solved'] = 2.0
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Topic.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 2)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id, str(topic_1_solved_sticky.pk)) # topic 1 is solved
settings.ZDS_APP['search']['boosts']['topic']['if_solved'] = 1.0
settings.ZDS_APP['search']['boosts']['topic']['if_locked'] = 2.0 # no one would do that in real life
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=' + Topic.get_es_document_type(), follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 2)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id, str(topic_2_locked.pk)) # topic 2 is locked
settings.ZDS_APP['search']['boosts']['topic']['if_locked'] = 1.0 # no one would do that in real life
# 5. Test published contents
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=content', follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 5)
# score are equals without boost:
self.assertTrue(response[0].meta.score ==
response[1].meta.score ==
response[2].meta.score ==
response[3].meta.score ==
response[4].meta.score)
settings.ZDS_APP['search']['boosts']['publishedcontent']['if_article'] = 2.0
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=content', follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 5)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id, str(published_article.pk)) # obvious
settings.ZDS_APP['search']['boosts']['publishedcontent']['if_article'] = 1.0
settings.ZDS_APP['search']['boosts']['publishedcontent']['if_tutorial'] = 2.0
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=content', follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 5)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id, str(published_tuto.pk)) # obvious
settings.ZDS_APP['search']['boosts']['publishedcontent']['if_tutorial'] = 1.0
settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion'] = 2.0
settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion_not_picked'] = 4.0
# Note: in "real life", unpicked opinion would get a boost < 1.
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=content', follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 5)
self.assertTrue(response[0].meta.score > response[1].meta.score > response[2].meta.score)
self.assertEqual(response[0].meta.id, str(published_opinion_not_picked.pk)) # unpicked opinion got first
self.assertEqual(response[1].meta.id, str(published_opinion_picked.pk))
settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion'] = 1.0
settings.ZDS_APP['search']['boosts']['publishedcontent']['if_opinion_not_picked'] = 1.0
settings.ZDS_APP['search']['boosts']['publishedcontent']['if_medium_or_big_tutorial'] = 2.0
result = self.client.get(
reverse('search:query') + '?q=' + text + '&models=content', follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 5)
self.assertTrue(response[0].meta.score > response[1].meta.score)
self.assertEqual(response[0].meta.id, str(published_tuto.pk)) # obvious
settings.ZDS_APP['search']['boosts']['publishedcontent']['if_medium_or_big_tutorial'] = 1.0
# 6. Test global boosts
# NOTE: score are NOT the same for all documents, no matter how hard it tries to, small differences exists
for model in self.indexable:
# set a huge number to overcome the small differences:
settings.ZDS_APP['search']['boosts'][model.get_es_document_type()]['global'] = 10.0
result = self.client.get(
reverse('search:query') + '?q=' + text, follow=False)
self.assertEqual(result.status_code, 200)
response = result.context['object_list'].execute()
self.assertEqual(response.hits.total, 10)
self.assertEqual(response[0].meta.doc_type, model.get_es_document_type()) # obvious
settings.ZDS_APP['search']['boosts'][model.get_es_document_type()]['global'] = 1.0