def test_slice_string_for_completion(self):
"""
The slice_string_for_completion function slices a string into an array of strings suitable
for use in a completion field.
This means including all strings contained in the original string, that begin after a
space and end at the end of the original string.
"""
self.assertEqual(slice_string_for_completion(""), [])
self.assertEqual(slice_string_for_completion("Physics"), ["Physics"])
self.assertEqual(
slice_string_for_completion("Communication dans les organisations"),
[
"Communication dans les organisations",
"dans les organisations",
"les organisations",
"organisations",
],
)
# Trailing spaces are discarded and strings trimmed as those spaces are not useful &
# ES rejects empty strings from completion mappings
self.assertEqual(
slice_string_for_completion("Université Paris 18 "),
["Université Paris 18", "Paris 18", "18"],
)
def test_autocomplete_is_listed(self, *_):
"""
Courses that are not flagged for listing should not appear in results.
"""
all_courses = [
{
"complete": {
"en": slice_string_for_completion(
"Artificial intelligence for mushroom picking"
)
},
"course_runs": [],
"id": "24",
"path": "001000",
"title": {"en": "Artificial intelligence for mushroom picking"},
},
{
"complete": None,
"course_runs": [],
"id": "25",
"path": "001001",
"title": {"en": "Artificial intelligence for mushroom picking"},
},
]
results = self.execute_query(all_courses, querystring="query=intelligence")
self.assertEqual([all_courses[0]["id"]], [course["id"] for course in results])
def execute_query(self, querystring="", **extra):
"""
Not a test.
Prepare the ElasticSearch index and execute the query in it.
"""
licences = [
{"id": "1", "title": {"en": "CC-BY-SA"}},
{"id": "2", "title": {"en": "CC-BY-NC"}},
{"id": "3", "title": {"en": "All Rights Résërvés"}},
]
# Delete any existing indices so we get a clean slate
ES_INDICES_CLIENT.delete(index="_all")
# Create an index we'll use to test the ES features
ES_INDICES_CLIENT.create(index=LICENCES_INDEX)
# The index needs to be closed before we set an analyzer
ES_INDICES_CLIENT.close(index=LICENCES_INDEX)
ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS, index=LICENCES_INDEX)
ES_INDICES_CLIENT.open(index=LICENCES_INDEX)
# Use the default licences mapping from the Indexer
ES_INDICES_CLIENT.put_mapping(
body=LicencesIndexer.mapping, index=LICENCES_INDEX
)
# Actually insert our licences in the index
actions = [
{
"_id": licence["id"],
"_index": LICENCES_INDEX,
"_op_type": "create",
"complete": {"en": slice_string_for_completion(licence["title"]["en"])},
**licence,
}
for licence in licences
]
bulk_compat(actions=actions, chunk_size=500, client=ES_CLIENT)
ES_INDICES_CLIENT.refresh()
response = self.client.get(
f"/api/v1.0/licences/autocomplete/?{querystring:s}", **extra
)
self.assertEqual(response.status_code, 200)
return licences, json.loads(response.content)
from django.test import TestCase
from richie.apps.search import ES_CLIENT, ES_INDICES_CLIENT
from richie.apps.search.elasticsearch import bulk_compat
from richie.apps.search.indexers.courses import CoursesIndexer
from richie.apps.search.text_indexing import ANALYSIS_SETTINGS
from richie.apps.search.utils.indexers import slice_string_for_completion
COURSES_INDEX = "test_courses"
TEST_COURSES = [
{
"complete": {
"en": slice_string_for_completion(
"Artificial intelligence for mushroom picking"
),
"fr": slice_string_for_completion(
"Intelligence artificielle pour la cueillette de chàmpiñons"
),
},
"course_runs": [],
"id": "24",
"is_listed": True,
"path": "001000",
"title": {
"en": "Artificial intelligence for mushroom picking",
"fr": "Intelligence artificielle pour la cueillette de chàmpiñons",
},
},
{
def execute_query(self, querystring="", **extra):
"""
Not a test.
Prepare the ElasticSearch index and execute the query in it.
"""
categories = [
{
"complete": {
"en":
slice_string_for_completion("Electric Birdwatching"),
"fr":
slice_string_for_completion(
"Observation des oiseaux électriques"),
},
"id": "24",
"kind": "subjects",
"path": "001000",
"title": {
"en": "Electric Birdwatching",
"fr": "Observation des oiseaux électriques",
},
},
{
"complete": {
"en": slice_string_for_completion("Ocean biking"),
"fr": slice_string_for_completion("Cyclisme océanique"),
},
"id": "33",
"kind": "subjects",
"path": "001001",
"title": {
"en": "Ocean biking",
"fr": "Cyclisme océanique"
},
},
{
"complete": {
"en": slice_string_for_completion("Elegiac bikeshedding"),
"fr":
slice_string_for_completion("Élégie de l'abri à vélos"),
},
"id": "51",
"kind": "subjects",
"path": "001002",
"title": {
"en": "Elegiac bikeshedding",
"fr": "Élégie de l'abri à vélos",
},
},
{
"complete": {
"en": slice_string_for_completion("Electric Decoys"),
"fr": slice_string_for_completion("Leurres électriques"),
},
"id": "44",
"kind": "not_subjects",
"path": "001003",
"title": {
"en": "Electric Decoys",
"fr": "Leurres électriques"
},
},
]
indices_client = IndicesClient(client=ES_CLIENT)
# Delete any existing indexes so we get a clean slate
indices_client.delete(index="_all")
# Create an index we'll use to test the ES features
indices_client.create(index=CATEGORIES_INDEX)
# The index needs to be closed before we set an analyzer
indices_client.close(index=CATEGORIES_INDEX)
indices_client.put_settings(body=ANALYSIS_SETTINGS,
index=CATEGORIES_INDEX)
indices_client.open(index=CATEGORIES_INDEX)
# Use the default categories mapping from the Indexer
indices_client.put_mapping(body=CategoriesIndexer.mapping,
doc_type="category",
index=CATEGORIES_INDEX)
# Actually insert our categories in the index
actions = [{
"_id": category["id"],
"_index": CATEGORIES_INDEX,
"_op_type": "create",
"_type": "category",
"absolute_url": {
"en": "en/url",
"fr": "fr/url"
},
"cover_image": {
"en": "en/image",
"fr": "fr/image"
},
"is_meta": False,
"logo": {
"en": "en/some/img.png",
"fr": "fr/some/img.png"
},
"nb_children": 0,
**category,
} for category in categories]
bulk(actions=actions, chunk_size=500, client=ES_CLIENT)
indices_client.refresh()
response = self.client.get(
f"/api/v1.0/subjects/autocomplete/?{querystring:s}", **extra)
self.assertEqual(response.status_code, 200)
return categories, json.loads(response.content)
def execute_query(self, querystring="", **extra):
"""
Not a test.
Prepare the ElasticSearch index and execute the query in it.
"""
persons = [
{
"complete": {
"en": slice_string_for_completion("Éponine Thénardier")
},
"id": "25",
"title": {
"en": "Éponine Thénardier"
},
},
{
"complete": {
"en":
slice_string_for_completion("Monseigneur Bienvenu Myriel")
},
"id": "34",
"title": {
"en": "Monseigneur Bienvenu Myriel"
},
},
{
"complete": {
"en": slice_string_for_completion("Fantine")
},
"id": "52",
"title": {
"en": "Fantine"
},
},
]
# Delete any existing indices so we get a clean slate
ES_INDICES_CLIENT.delete(index="_all")
# Create an index we'll use to test the ES features
ES_INDICES_CLIENT.create(index=PERSONS_INDEX)
# The index needs to be closed before we set an analyzer
ES_INDICES_CLIENT.close(index=PERSONS_INDEX)
ES_INDICES_CLIENT.put_settings(body=ANALYSIS_SETTINGS,
index=PERSONS_INDEX)
ES_INDICES_CLIENT.open(index=PERSONS_INDEX)
# Use the default persons mapping from the Indexer
ES_INDICES_CLIENT.put_mapping(body=PersonsIndexer.mapping,
index=PERSONS_INDEX)
# Actually insert our persons in the index
actions = [{
"_id": person["id"],
"_index": PERSONS_INDEX,
"_op_type": "create",
"absolute_url": {
"en": "url"
},
"logo": {
"en": "/some/img.png"
},
**person,
} for person in persons]
bulk_compat(actions=actions, chunk_size=500, client=ES_CLIENT)
ES_INDICES_CLIENT.refresh()
response = self.client.get(
f"/api/v1.0/persons/autocomplete/?{querystring:s}", **extra)
self.assertEqual(response.status_code, 200)
return persons, json.loads(response.content)
ElasticsearchIndicesClientCompat7to6,
bulk_compat,
)
from richie.apps.search.indexers.categories import CategoriesIndexer
from richie.apps.search.text_indexing import ANALYSIS_SETTINGS
from richie.apps.search.utils.indexers import slice_string_for_completion
CATEGORIES = [
{
"id": "8312",
"kind": "subjects",
"title": {
"en": "Literature"
},
"complete": {
"en": slice_string_for_completion("Literature")
},
},
{
"id": "8399",
"kind": "subjects",
"title": {
"en": "Science for beginners"
},
"complete": {
"en": slice_string_for_completion("Science for beginners")
},
},
{
"id": "8421",
"kind": "levels",
def execute_query(self, querystring="", **extra):
"""
Not a test.
Prepare the ElasticSearch index and execute the query in it.
"""
organizations = [
{
"complete": {
"en":
slice_string_for_completion("University of Paris 18"),
"fr":
slice_string_for_completion("Université de Paris 18"),
},
"id": "25",
"path": "000000",
"title": {
"en": "University of Paris 18",
"fr": "Université de Paris 18",
},
},
{
"complete": {
"en":
slice_string_for_completion("School of bikeshedding"),
"fr": slice_string_for_completion("École d'abri-vélo"),
},
"id": "34",
"path": "000001",
"title": {
"en": "School of bikeshedding",
"fr": "École d'abri-vélo"
},
},
{
"complete": {
"en":
slice_string_for_completion("University of Paris 19"),
"fr":
slice_string_for_completion("Université de Paris 19"),
},
"id": "52",
"path": "000002",
"title": {
"en": "University of Paris 19",
"fr": "Université de Paris 19",
},
},
]
indices_client = IndicesClient(client=ES_CLIENT)
# Delete any existing indices so we get a clean slate
indices_client.delete(index="_all")
# Create an index we'll use to test the ES features
indices_client.create(index=ORGANIZATIONS_INDEX)
# The index needs to be closed before we set an analyzer
indices_client.close(index=ORGANIZATIONS_INDEX)
indices_client.put_settings(body=ANALYSIS_SETTINGS,
index=ORGANIZATIONS_INDEX)
indices_client.open(index=ORGANIZATIONS_INDEX)
# Use the default organizations mapping from the Indexer
indices_client.put_mapping(
body=OrganizationsIndexer.mapping,
doc_type="organization",
index=ORGANIZATIONS_INDEX,
)
# Actually insert our organizations in the index
actions = [{
"_id": organization["id"],
"_index": ORGANIZATIONS_INDEX,
"_op_type": "create",
"_type": "organization",
"absolute_url": {
"en": "url"
},
"cover_image": {
"en": "image"
},
"is_meta": False,
"logo": {
"en": "/some/img.png"
},
"nb_children": 0,
**organization,
} for organization in organizations]
bulk(actions=actions, chunk_size=500, client=ES_CLIENT)
indices_client.refresh()
response = self.client.get(
f"/api/v1.0/organizations/autocomplete/?{querystring:s}", **extra)
self.assertEqual(response.status_code, 200)
return organizations, json.loads(response.content)
def execute_query(self, querystring="", **extra):
"""
Not a test.
Prepare the ElasticSearch index and execute the query in it.
"""
courses = [
{
"complete": {
"en":
slice_string_for_completion(
"Artificial intelligence for mushroom picking"),
"fr":
slice_string_for_completion(
"Intelligence artificielle pour la cueillette de chàmpiñons"
),
},
"course_runs": [],
"id": "24",
"path": "001000",
"title": {
"en":
"Artificial intelligence for mushroom picking",
"fr":
"Intelligence artificielle pour la cueillette de chàmpiñons",
},
},
{
"complete": {
"en":
slice_string_for_completion(
"Kung-fu moves for cloud infrastructure security"),
"fr":
slice_string_for_completion(
"Protéger ses serveurs par la pratique des arts martiaux"
),
},
"course_runs": [],
"id": "33",
"path": "001001",
"title": {
"en":
"Kung-fu moves for cloud infrastructure security",
"fr":
"Prôtéger ses serveurs par la pratique des arts martiaux",
},
},
{
"complete": {
"en":
slice_string_for_completion(
"Securing funding through token sales"),
"fr":
slice_string_for_completion("Lever des fonds par des ICO"),
},
"course_runs": [],
"id": "51",
"path": "001002",
"title": {
"en": "Securing funding through token sales",
"fr": "Lever des fonds par des ICO",
},
},
]
indices_client = IndicesClient(client=ES_CLIENT)
# Delete any existing indices so we get a clean slate
indices_client.delete(index="_all")
# Create an index we'll use to test the ES features
indices_client.create(index=COURSES_INDEX)
# The index needs to be closed before we set an analyzer
indices_client.close(index=COURSES_INDEX)
indices_client.put_settings(body=ANALYSIS_SETTINGS,
index=COURSES_INDEX)
indices_client.open(index=COURSES_INDEX)
# Use the default courses mapping from the Indexer
indices_client.put_mapping(body=CoursesIndexer.mapping,
doc_type="course",
index=COURSES_INDEX)
# Add the sorting script
ES_CLIENT.put_script(id="state", body=CoursesIndexer.scripts["state"])
# Actually insert our courses in the index
actions = [{
"_id": course["id"],
"_index": COURSES_INDEX,
"_op_type": "create",
"_type": "course",
"absolute_url": {
"en": "en/url",
"fr": "fr/url"
},
"categories": ["1", "2", "3"],
"cover_image": {
"en": "en/image",
"fr": "fr/image"
},
"is_meta": False,
"logo": {
"en": "/en/some/img.png",
"fr": "/fr/some/img.png"
},
"nb_children": 0,
"organizations": ["11", "12", "13"],
**course,
} for course in courses]
bulk(actions=actions, chunk_size=500, client=ES_CLIENT)
indices_client.refresh()
response = self.client.get(
f"/api/v1.0/courses/autocomplete/?{querystring:s}", **extra)
self.assertEqual(response.status_code, 200)
return courses, json.loads(response.content)