def test_indexers_courses_build_es_query_search_with_invalid_params(self):
"""
Error case: the request contained invalid parameters
"""
with self.assertRaises(QueryFormatException):
CoursesIndexer.build_es_query(
SimpleNamespace(query_params=QueryDict(query_string="limit=-2"))
)
def test_indexers_courses_get_es_documents_unpublished_category(self):
"""
Unpublished categories and children of unpublished categories should not be indexed
"""
# Create a child category
meta = CategoryFactory(
page_parent=create_i18n_page("Categories", published=True),
page_reverse_id="subjects",
page_title="Subjects",
should_publish=True,
)
parent = CategoryFactory(page_parent=meta.extended_object, should_publish=True)
category = CategoryFactory(
page_parent=parent.extended_object,
page_title="my second subject",
should_publish=True,
)
CourseFactory(fill_categories=[category], should_publish=True)
# Unpublish the parent category
self.assertTrue(parent.extended_object.unpublish("en"))
course_document = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)[0]
# Neither the category not its parent should be linked to the course
self.assertEqual(course_document["categories"], [])
self.assertEqual(course_document["categories_names"], {})
def test_indexers_courses_get_data_for_es_with_unexpected_data_shape(self):
"""
Error case: the API returned an object that is not shape like an expected course
"""
responses.add(
method="GET",
url=settings.COURSE_API_ENDPOINT,
status=200,
json={
"count": 1,
"results": [
{
"end_date": "2018-02-28T06:00:00Z",
"enrollment_end_date": "2018-01-31T06:00:00Z",
"enrollment_start_date": "2018-01-01T06:00:00Z",
"id": 42,
# 'language': 'fr', missing language key will trigger the KeyError
"main_university": {"id": 21},
"session_number": 6,
"short_description": "Lorem ipsum dolor sit amet",
"start_date": "2018-02-01T06:00:00Z",
"subjects": [{"id": 168}, {"id": 336}],
"thumbnails": {"big": "whatever.png"},
"title": "A course in filler text",
"universities": [{"id": 21}, {"id": 84}],
}
],
},
)
with self.assertRaises(IndexerDataException):
list(
CoursesIndexer.get_data_for_es(index="some_index", action="some_action")
)
def test_indexers_courses_get_es_document_no_image_cover_picture(self):
"""
ES document is created without errors when a cover image for the course is
actually a Picture instance without an image on it.
"""
# Create the example course to index and get hold of its course_cover placeholder
course = CourseFactory(should_publish=True)
course_cover_placeholder = (
course.extended_object.get_public_object()
.placeholders.filter(slot="course_cover")
.first()
)
# Make sure we associate an image-less picture with the course through
# the cover placeholder
add_plugin(course_cover_placeholder, SimplePicturePlugin, "en", picture=None)
course.extended_object.publish("en")
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)
self.assertEqual(len(indexed_courses), 1)
self.assertEqual(
indexed_courses[0]["_id"],
str(course.extended_object.get_public_object().id),
)
self.assertEqual(indexed_courses[0]["cover_image"], {})
def test_indexers_courses_build_es_query_search_by_terms_organizations(self):
"""
Happy path: build a query that filters courses by more than 1 related organizations
"""
# Build a request stub
request = SimpleNamespace(
query_params=QueryDict(
query_string="organizations=13&organizations=15&limit=2"
)
)
terms_organizations = {"terms": {"organizations": [13, 15]}}
self.assertEqual(
CoursesIndexer.build_es_query(request),
(
2,
0,
{"bool": {"must": [terms_organizations]}},
{
"all_courses": {
"global": {},
"aggregations": {
"language@en": {
"filter": {
"bool": {
"must": [
{"term": {"language": "en"}},
terms_organizations,
]
}
}
},
"language@fr": {
"filter": {
"bool": {
"must": [
{"term": {"language": "fr"}},
terms_organizations,
]
}
}
},
"organizations": {
"filter": {"bool": {"must": []}},
"aggregations": {
"organizations": {
"terms": {"field": "organizations"}
}
},
},
"subjects": {
"filter": {"bool": {"must": [terms_organizations]}},
"aggregations": {
"subjects": {"terms": {"field": "subjects"}}
},
},
},
}
},
),
)
def test_indexers_courses_related_objects_consistency(self):
"""
The organization and category ids in the Elasticsearch course document should be
the same as the ids with which the corresponding organization and category objects
are indexed.
"""
# Create a course with a page in both english and french
organization = OrganizationFactory(should_publish=True)
category = CategoryFactory(should_publish=True)
course = CourseFactory(fill_organizations=[organization],
fill_categories=[category])
CourseRunFactory(direct_course=course)
course.extended_object.publish("en")
course_document = list(
CoursesIndexer.get_es_documents(index="some_index",
action="some_action"))[0]
self.assertEqual(
course_document["organizations"],
[
next(
OrganizationsIndexer.get_es_documents(
index="some_index", action="some_action"))["_id"]
],
)
self.assertEqual(
course_document["categories"],
[
next(
CategoriesIndexer.get_es_documents(
index="some_index", action="some_action"))["_id"]
],
)
def test_indexers_courses_get_es_documents_course_glimpse_organization_menu_title(
self, ):
"""
Linked organizations should display the indexed acronym if the menu_title
is filled.
"""
menu_title = "MTO"
organization_page = create_page(
"My Test Organization",
"richie/single_column.html",
"en",
menu_title=menu_title,
)
organization = OrganizationFactory(extended_object=organization_page,
should_publish=True)
course = CourseFactory(
fill_organizations=[organization],
should_publish=True,
)
course_document = CoursesIndexer.get_es_document_for_course(course)
# The organization should display the menu title and not the title itself
self.assertEqual(course_document["organization_highlighted"],
{"en": menu_title})
self.assertNotEqual(
course_document["organization_highlighted"],
{"en": course.extended_object.get_title()},
)
def test_indexers_courses_get_es_documents_catalog_visibility_one_each(
self):
"""
A course with 3 runs. A run with `hidden`, another with `course_and_search` and a last
one with `course_only` on the catalog visibility should have a single run on the index.
"""
course = CourseFactory()
CourseRunFactory(
direct_course=course,
catalog_visibility=CourseRunCatalogVisibility.HIDDEN,
)
CourseRunFactory(
direct_course=course,
catalog_visibility=CourseRunCatalogVisibility.COURSE_ONLY,
)
CourseRunFactory(
direct_course=course,
catalog_visibility=CourseRunCatalogVisibility.COURSE_AND_SEARCH,
)
self.assertTrue(course.extended_object.publish("en"))
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index",
action="some_action"))
self.assertEqual(len(indexed_courses), 1)
self.assertEqual(len(indexed_courses[0]["course_runs"]), 1)
def test_indexers_courses_format_es_document_for_autocomplete(self):
"""
Make sure format_es_document_for_autocomplete returns a properly
formatted course suggestion.
"""
es_course = {
"_id": 93,
"_source": {
"absolute_url": {"en": "/en/campo-qui-format-do"},
"categories": [43, 86],
"cover_image": {"en": "cover_image.jpg"},
"icon": {"en": "icon.jpg"},
"organizations": [42, 84],
"organizations_names": {"en": ["Org 42", "Org 84"]},
"title": {"en": "Duis eu arcu erat"},
},
"fields": {
"state": [
{"priority": 0, "date_time": "2019-03-17T21:25:52.179667+00:00"}
]
},
}
self.assertEqual(
CoursesIndexer.format_es_document_for_autocomplete(es_course, "en"),
{
"absolute_url": "/en/campo-qui-format-do",
"id": 93,
"kind": "courses",
"title": "Duis eu arcu erat",
},
)
def test_build_es_query_search_by_match_text(self):
"""
Happy path: build a query that filters courses by matching text
"""
# Build a request stub
request = SimpleNamespace(
query_params=QueryDict(
query_string="query=some%20phrase%20terms&limit=2&offset=20"
)
)
multi_match = {
"multi_match": {
"fields": ["short_description.*", "title.*"],
"query": "some phrase terms",
"type": "cross_fields",
}
}
self.assertEqual(
CoursesIndexer.build_es_query(request, self.facets),
(
2,
20,
{
"bool": {
"must": [
{
"multi_match": {
"fields": ["short_description.*", "title.*"],
"query": "some phrase terms",
"type": "cross_fields",
}
}
]
}
},
{
"all_courses": {
"global": {},
"aggregations": {
"organizations": {
"filter": {"bool": {"must": [multi_match]}},
"aggregations": {
"organizations": {
"terms": {"field": "organizations"}
}
},
},
"subjects": {
"filter": {"bool": {"must": [multi_match]}},
"aggregations": {
"subjects": {"terms": {"field": "subjects"}}
},
},
},
}
},
),
)
def test_indexers_courses_get_es_document_for_course_related_names_related_unpublished(
self,
):
"""
When related objects are unpublished in one language, that language should not be taken
into account to build related object names.
"""
# Create a course with related pages in both english and french but only
# published in one language
category = CategoryFactory(
page_title={
"en": "english category title",
"fr": "titre catégorie français",
},
should_publish=True,
)
category.extended_object.unpublish("fr")
organization = OrganizationFactory(
page_title={
"en": "english organization title",
"fr": "titre organisation français",
},
should_publish=True,
)
organization.extended_object.unpublish("fr")
person = PersonFactory(
page_title={"en": "Brian", "fr": "François"}, should_publish=True
)
person.extended_object.unpublish("fr")
course = CourseFactory(
fill_categories=[category],
fill_organizations=[organization],
fill_team=[person],
page_title={
"en": "an english course title",
"fr": "un titre cours français",
},
should_publish=True,
)
course_document = CoursesIndexer.get_es_document_for_course(course)
self.assertEqual(
course_document["organizations_names"],
{"en": ["english organization title"]},
)
self.assertEqual(
course_document["organization_highlighted"],
{"en": "english organization title"},
)
self.assertEqual(
course_document["categories_names"], {"en": ["english category title"]}
)
self.assertEqual(course_document["persons_names"], {"en": ["Brian"]})
def test_indexers_courses_get_es_document_no_organization(self):
"""
Courses with no linked organizations should get indexed without raising exceptions.
"""
course = CourseFactory(
duration=[12, WEEK],
effort=[5, MINUTE],
page_title="Enhanced incremental circuit",
should_publish=True,
)
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index",
action="some_action"))
self.assertEqual(
indexed_courses,
[{
"_id": str(course.extended_object.publisher_public_id),
"_index": "some_index",
"_op_type": "some_action",
"_type": "course",
"absolute_url": {
"en": "/en/enhanced-incremental-circuit/"
},
"categories": [],
"categories_names": {},
"complete": {
"en": [
"Enhanced incremental circuit",
"incremental circuit",
"circuit",
]
},
"course_runs": [],
"cover_image": {},
"description": {},
"duration": {
"en": "12 weeks",
"fr": "12 semaines"
},
"effort": {
"en": "5 minutes",
"fr": "5 minutes"
},
"icon": {},
"is_new": False,
"is_listed": True,
"organization_highlighted": None,
"organizations": [],
"organizations_names": {},
"persons": [],
"persons_names": {},
"title": {
"en": "Enhanced incremental circuit"
},
}],
)
def test_indexers_courses_get_es_documents_unpublished_course(self):
"""Unpublished courses should not be indexed"""
CourseFactory()
self.assertEqual(
list(
CoursesIndexer.get_es_documents(index="some_index",
action="some_action")),
[],
)
def test_indexers_courses_get_es_document_for_course_not_published(self):
"""
A course indexed with no published title shoud not be listed.
"""
course = CourseFactory(
page_title={"en": "a course", "fr": "un cours"}, should_publish=True
)
course_document = CoursesIndexer.get_es_document_for_course(course)
self.assertTrue(course_document["is_listed"])
# Only after unpublishing all languages, the course stops being listed
course.extended_object.unpublish("en")
course_document = CoursesIndexer.get_es_document_for_course(course)
self.assertTrue(course_document["is_listed"])
course.extended_object.unpublish("fr")
course_document = CoursesIndexer.get_es_document_for_course(course)
self.assertFalse(course_document["is_listed"])
def test_indexers_courses_get_es_documents_no_course_run(self):
"""
A course with no course run should still be indexed.
"""
CourseFactory(should_publish=True)
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)
self.assertEqual(len(indexed_courses), 1)
self.assertEqual(indexed_courses[0]["course_runs"], [])
def test_indexers_courses_get_es_documents_snapshots(self):
"""
Course snapshots should not get indexed.
"""
course = CourseFactory(should_publish=True)
CourseFactory(page_parent=course.extended_object, should_publish=True)
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)
self.assertEqual(len(indexed_courses), 1)
self.assertEqual(indexed_courses[0]["_id"], course.get_es_id())
def test_indexers_courses_get_es_documents_is_listed(self):
"""
Courses that are flagged to be hidden from the search page should be marked as such.
"""
CourseFactory(should_publish=True, is_listed=False)
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)
self.assertEqual(len(indexed_courses), 1)
self.assertFalse(indexed_courses[0]["is_listed"])
self.assertIsNone(indexed_courses[0]["complete"], None)
def test_indexers_courses_get_es_documents_no_enrollment_start(self):
"""
Course runs with no start of enrollment date should not get indexed.
"""
course = CourseFactory()
CourseRunFactory(direct_course=course, enrollment_start=None)
course.extended_object.publish("en")
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)
self.assertEqual(len(indexed_courses), 1)
self.assertEqual(indexed_courses[0]["course_runs"], [])
def test_indexers_courses_get_es_documents_no_start(self):
"""
Course runs with no start date should not get indexed.
"""
course = CourseFactory(should_publish=True)
CourseRunFactory(
page_parent=course.extended_object, start=None, should_publish=True
)
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)
self.assertEqual(len(indexed_courses), 1)
self.assertEqual(indexed_courses[0]["course_runs"], [])
def test_indexers_courses_get_es_documents_no_end(self):
"""
Course runs with no end date should be on-going for ever.
"""
course = CourseFactory()
CourseRunFactory(direct_course=course, end=None)
course.extended_object.publish("en")
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)
self.assertEqual(len(indexed_courses), 1)
self.assertEqual(len(indexed_courses[0]["course_runs"]), 1)
self.assertEqual(indexed_courses[0]["course_runs"][0]["end"].year, 9999)
def test_indexers_courses_format_es_object_for_api_no_organization(self):
"""
A course that has no organization and was indexed should not raise 500 errors (although
this should not happen if courses are correctly moderated).
"""
es_course = {
"_id": 93,
"_source": {
"absolute_url": {"en": "campo-qui-format-do"},
"categories": [43, 86],
"code": "abc123",
"cover_image": {"en": "cover_image.jpg"},
"duration": {"en": "3 weeks"},
"effort": {"en": "10 minutes"},
"icon": {"en": "icon.jpg"},
"introduction": {"en": "introductio est"},
"organization_highlighted": None,
"organization_highlighted_cover_image": {},
"organizations": [],
"organizations_names": {},
"title": {"en": "Duis eu arcu erat"},
},
"fields": {
"state": [
{"priority": 0, "date_time": "2019-03-17T21:25:52.179667+00:00"}
]
},
}
self.assertEqual(
CoursesIndexer.format_es_object_for_api(es_course, "en"),
{
"id": 93,
"absolute_url": "campo-qui-format-do",
"categories": [43, 86],
"code": "abc123",
"cover_image": "cover_image.jpg",
"duration": "3 weeks",
"effort": "10 minutes",
"icon": "icon.jpg",
"introduction": "introductio est",
"organization_highlighted": None,
"organization_highlighted_cover_image": None,
"organizations": [],
"title": "Duis eu arcu erat",
"state": CourseState(
0, datetime(2019, 3, 17, 21, 25, 52, 179667, pytz.utc)
),
},
)
def test_indexers_courses_get_es_documents_unpublished_person(self):
"""Unpublished persons should not be indexed."""
person = PersonFactory(should_publish=True)
CourseFactory(fill_team=[person], should_publish=True)
# Unpublish the person
self.assertTrue(person.extended_object.unpublish("en"))
course_document = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)[0]
# The unpublished person should not be linked to the course
self.assertEqual(course_document["persons"], [])
self.assertEqual(course_document["persons_names"], {})
def test_indexers_courses_get_es_documents_language_fallback(self):
"""Absolute urls should be computed as expected with language fallback."""
CourseFactory(should_publish=True, page_title={"fr": "un titre court français"})
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)
self.assertEqual(
indexed_courses[0]["absolute_url"],
{
"en": "/en/un-titre-court-francais/",
"fr": "/fr/un-titre-court-francais/",
},
)
def test_format_es_course_for_api(self):
"""
Make sure format_es_course_for_api returns a properly formatted course
"""
es_course = {
"_id": 93,
"_source": {
"end_date": "2018-02-28T06:00:00Z",
"enrollment_end_date": "2018-01-31T06:00:00Z",
"enrollment_start_date": "2018-01-01T06:00:00Z",
"language": "en",
"organization_main": 42,
"organizations": [42, 84],
"session_number": 1,
"short_description": {
"en": "Nam aliquet, arcu at sagittis sollicitudin."
},
"start_date": "2018-02-01T06:00:00Z",
"subjects": [43, 86],
"thumbnails": {
"big": "whatever_else.png"
},
"title": {
"en": "Duis eu arcu erat"
},
},
}
self.assertEqual(
CoursesIndexer.format_es_course_for_api(es_course, "en"),
{
"end_date": "2018-02-28T06:00:00Z",
"enrollment_end_date": "2018-01-31T06:00:00Z",
"enrollment_start_date": "2018-01-01T06:00:00Z",
"id": 93,
"language": "en",
"organization_main": 42,
"organizations": [42, 84],
"session_number": 1,
"short_description":
"Nam aliquet, arcu at sagittis sollicitudin.",
"start_date": "2018-02-01T06:00:00Z",
"subjects": [43, 86],
"thumbnails": {
"big": "whatever_else.png"
},
"title": "Duis eu arcu erat",
},
)
def test_indexers_courses_format_es_object_for_api(self):
"""
Make sure format_es_object_for_api returns a properly formatted course
"""
es_course = {
"_id": 93,
"_source": {
"absolute_url": {"en": "campo-qui-format-do"},
"categories": [43, 86],
"code": "abc123",
"cover_image": {"en": "cover_image.jpg"},
"duration": {"en": "6 months"},
"effort": {"en": "3 hours"},
"icon": {"en": "icon.jpg"},
"introduction": {"en": "introductio est"},
"organization_highlighted": {"en": "Org 84"},
"organization_highlighted_cover_image": {"en": "org_cover_image.jpg"},
"organizations": [42, 84],
"organizations_names": {"en": ["Org 42", "Org 84"]},
"title": {"en": "Duis eu arcu erat"},
},
"fields": {
"state": [
{"priority": 0, "date_time": "2019-03-17T21:25:52.179667+00:00"}
]
},
}
self.assertEqual(
CoursesIndexer.format_es_object_for_api(es_course, "en"),
{
"id": 93,
"absolute_url": "campo-qui-format-do",
"categories": [43, 86],
"code": "abc123",
"cover_image": "cover_image.jpg",
"duration": "6 months",
"effort": "3 hours",
"icon": "icon.jpg",
"introduction": "introductio est",
"organization_highlighted": "Org 84",
"organization_highlighted_cover_image": "org_cover_image.jpg",
"organizations": [42, 84],
"title": "Duis eu arcu erat",
"state": CourseState(
0, datetime(2019, 3, 17, 21, 25, 52, 179667, pytz.utc)
),
},
)
def test_indexers_courses_format_es_object_for_api_no_cover(self):
"""
A course that has no cover image and was indexed should not raise any errors.
"""
es_course = {
"_id": 93,
"_source": {
"absolute_url": {"en": "campo-qui-format-do"},
"categories": [43, 86],
"code": "abc123",
"cover_image": {},
"duration": {"en": "N/A"},
"effort": {"en": "N/A"},
"icon": {"en": "icon.jpg"},
"introduction": {"en": "introductio est"},
"organization_highlighted": {"en": "Org 42"},
"organization_highlighted_cover_image": {},
"organizations": [42, 84],
"organizations_names": {"en": ["Org 42", "Org 84"]},
"title": {"en": "Duis eu arcu erat"},
},
"fields": {
"state": [
{"priority": 0, "date_time": "2019-03-17T21:25:52.179667+00:00"}
]
},
}
self.assertEqual(
CoursesIndexer.format_es_object_for_api(es_course, "en"),
{
"id": 93,
"absolute_url": "campo-qui-format-do",
"categories": [43, 86],
"code": "abc123",
"cover_image": None,
"duration": "N/A",
"effort": "N/A",
"icon": "icon.jpg",
"introduction": "introductio est",
"organization_highlighted": "Org 42",
"organization_highlighted_cover_image": None,
"organizations": [42, 84],
"title": "Duis eu arcu erat",
"state": CourseState(
0, datetime(2019, 3, 17, 21, 25, 52, 179667, pytz.utc)
),
},
)
def test_indexers_courses_get_es_documents_catalog_visibility_hidden(
self, ):
"""
A course run with `hidden` on catalog visibility should not be indexed.
"""
course = CourseFactory()
CourseRunFactory(
direct_course=course,
catalog_visibility=CourseRunCatalogVisibility.HIDDEN,
)
self.assertTrue(course.extended_object.publish("en"))
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index",
action="some_action"))
self.assertEqual(len(indexed_courses), 1)
self.assertEqual(indexed_courses[0]["course_runs"], [])
def test_indexers_courses_get_es_documents_catalog_visibility_course_and_search(
self, ):
"""
A course run with `course_and_search` on catalog visibility should be indexed.
"""
course = CourseFactory()
CourseRunFactory(
direct_course=course,
catalog_visibility=CourseRunCatalogVisibility.COURSE_AND_SEARCH,
)
self.assertTrue(course.extended_object.publish("en"))
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index",
action="some_action"))
self.assertEqual(len(indexed_courses), 1)
self.assertEqual(len(indexed_courses[0]["course_runs"]), 1)
def test_indexers_courses_build_es_query_search_all_courses(self):
"""
Happy path: build a query that does not filter the courses at all
"""
# Build a request stub
request = SimpleNamespace(
query_params=QueryDict(query_string="limit=2&offset=10")
)
self.assertEqual(
CoursesIndexer.build_es_query(request),
(
2,
10,
{"match_all": {}},
{
"all_courses": {
"global": {},
"aggregations": {
"language@en": {
"filter": {
"bool": {"must": [{"term": {"language": "en"}}]}
}
},
"language@fr": {
"filter": {
"bool": {"must": [{"term": {"language": "fr"}}]}
}
},
"organizations": {
"filter": {"bool": {"must": []}},
"aggregations": {
"organizations": {
"terms": {"field": "organizations"}
}
},
},
"subjects": {
"filter": {"bool": {"must": []}},
"aggregations": {
"subjects": {"terms": {"field": "subjects"}}
},
},
},
}
},
),
)
def test_indexers_courses_get_es_documents_no_enrollment_end(self):
"""
Course runs with no end of enrollment date should get their end date as date of end
of enrollment.
"""
course = CourseFactory()
course_run = CourseRunFactory(direct_course=course, enrollment_end=None)
course.extended_object.publish("en")
indexed_courses = list(
CoursesIndexer.get_es_documents(index="some_index", action="some_action")
)
self.assertEqual(len(indexed_courses), 1)
self.assertEqual(len(indexed_courses[0]["course_runs"]), 1)
self.assertEqual(
indexed_courses[0]["course_runs"][0]["enrollment_end"], course_run.end
)