def get_resource_relations(self, instance):
""" Get resource_relations properties"""
course = Course.objects.get(id=instance.run.object_id)
return {
"name": "resourcefile",
"parent": gen_course_id(course.platform, course.course_id),
}
def upsert_course(course_id):
"""Upsert course based on stored database information"""
course_obj = Course.objects.get(id=course_id)
course_data = ESCourseSerializer(course_obj).data
api.upsert_document(
gen_course_id(course_obj.platform, course_obj.course_id),
course_data,
COURSE_TYPE,
retry_on_conflict=settings.INDEXING_ERROR_RETRIES,
)
def serialize_course_for_bulk(course_obj):
"""
Serialize a course for bulk API request
Args:
course_obj (Course): A course
"""
return {
"_id": gen_course_id(course_obj.platform, course_obj.course_id),
**ESCourseSerializer(course_obj).data,
}
def test_serialize_course_for_bulk():
"""
Test that serialize_course_for_bulk yields a valid ESCourseSerializer
"""
course = CourseFactory.create()
assert_json_equal(
serialize_course_for_bulk(course),
{
"_id": gen_course_id(course.platform, course.course_id),
**ESCourseSerializer(course).data,
},
)
def delete_content_file(content_file_obj):
"""
Runs a task to delete an ES CourseRunFile document
Args:
content_file_obj (course_catalog.models.ContentFile): A CourseRunFile object
"""
course = content_file_obj.run.content_object
delete_document.delay(
gen_content_file_id(content_file_obj.key),
COURSE_TYPE,
routing=gen_course_id(course.platform, course.course_id),
)
def delete_course(course_obj):
"""
Runs a task to delete an ES Course document
Args:
course_obj (course_catalog.models.Course): A Course object
"""
delete_document.delay(
gen_course_id(course_obj.platform, course_obj.course_id), COURSE_TYPE)
for content_file in ContentFile.objects.filter(
run__object_id=course_obj.id).filter(
run__content_type=ContentType.objects.get(model=COURSE_TYPE)):
delete_content_file(content_file)
def delete_run_content_files(run_id):
"""
Delete a list of content files by run from the index
Args:
run_id(int): Course run id
"""
run = LearningResourceRun.objects.get(id=run_id)
documents = (serialize_content_file_for_bulk_deletion(content_file)
for content_file in ContentFile.objects.filter(run=run))
course = run.content_object
index_items(documents,
COURSE_TYPE,
routing=gen_course_id(course.platform, course.course_id))
def test_delete_course(mocker):
"""
Tests that delete_course calls the delete tasks for the course and its content files
"""
patched_delete_task = mocker.patch("search.task_helpers.delete_document")
course = CourseFactory.create()
course_es_id = gen_course_id(course.platform, course.course_id)
content_files = [ContentFileFactory.create(run=run) for run in course.runs.all()]
delete_course(course)
patched_delete_task.delay.assert_any_call(course_es_id, COURSE_TYPE)
for content_file in content_files:
patched_delete_task.delay.assert_any_call(
gen_content_file_id(content_file.key), COURSE_TYPE, routing=course_es_id
)
def upsert_content_file(file_id):
"""Upsert content file based on stored database information"""
content_file_obj = ContentFile.objects.get(id=file_id)
content_file_data = ESContentFileSerializer(content_file_obj).data
api.upsert_document(
gen_content_file_id(content_file_obj.key),
content_file_data,
COURSE_TYPE,
retry_on_conflict=settings.INDEXING_ERROR_RETRIES,
routing=gen_course_id(
content_file_obj.run.content_object.platform,
content_file_obj.run.content_object.course_id,
),
)
def test_bulk_index_content_files(mocked_es, mocker, settings, errors,
indexing_func_name, doc): # pylint: disable=too-many-arguments
"""
index functions for content files should call bulk with correct arguments
"""
settings.ELASTICSEARCH_INDEXING_CHUNK_SIZE = 3
course = CourseFactory.create()
run = LearningResourceRunFactory.create(content_object=course)
content_files = ContentFileFactory.create_batch(5, run=run)
mock_get_aliases = mocker.patch("search.indexing_api.get_active_aliases",
autospec=True,
return_value=["a", "b"])
bulk_mock = mocker.patch("search.indexing_api.bulk",
autospec=True,
return_value=(0, errors))
mocker.patch(
f"search.indexing_api.serialize_content_file_for_bulk",
autospec=True,
return_value=doc,
)
mocker.patch(
f"search.indexing_api.serialize_content_file_for_bulk_deletion",
autospec=True,
return_value=doc,
)
index_func = getattr(indexing_api, indexing_func_name)
if errors:
with pytest.raises(ReindexException):
index_func(run.id)
else:
index_func(run.id)
for alias in mock_get_aliases.return_value:
for chunk in chunks(
[doc for _ in content_files],
chunk_size=settings.ELASTICSEARCH_INDEXING_CHUNK_SIZE,
):
bulk_mock.assert_any_call(
mocked_es.conn,
chunk,
index=alias,
doc_type=GLOBAL_DOC_TYPE,
chunk_size=settings.ELASTICSEARCH_INDEXING_CHUNK_SIZE,
routing=gen_course_id(course.platform, course.course_id),
)
def index_run_content_files(run_id):
"""
Index a list of content files by run id
Args:
run_id(int): Course run id
"""
run = LearningResourceRun.objects.get(id=run_id)
documents = (
serialize_content_file_for_bulk(content_file)
for content_file in run.content_files.select_related("run").
prefetch_related("run__content_object").defer("run__raw_json"))
index_items(
documents,
COURSE_TYPE,
routing=gen_course_id(run.content_object.platform,
run.content_object.course_id),
)
def test_es_content_file_serializer():
""" Verify that the ESContentFileSerializer has the correct data"""
content_kwargs = {
"content": "Some text",
"content_author": "MIT",
"content_language": "en",
"content_title": "test title",
}
content_file = ContentFileFactory.create(**content_kwargs)
serialized = ESContentFileSerializer(content_file).data
assert_json_equal(
serialized,
{
"object_type": RESOURCE_FILE_TYPE,
"run_id": content_file.run.run_id,
"run_title": content_file.run.title,
"semester": content_file.run.semester,
"year": int(content_file.run.year),
"topics": list(content_file.run.topics.values_list("name", flat=True)),
"key": content_file.key,
"uid": content_file.uid,
"resource_relations": {
"name": "resourcefile",
"parent": gen_course_id(
content_file.run.content_object.platform,
content_file.run.content_object.course_id,
),
},
"title": content_file.title,
"short_description": content_file.description,
"file_type": content_file.file_type,
"content_type": content_file.content_type,
"url": content_file.url,
"section": content_file.section,
"content": content_kwargs["content"],
"content_title": content_kwargs["content_title"],
"content_author": content_kwargs["content_author"],
"content_language": content_kwargs["content_language"],
},
)