def test_indexing_responses(self):
"""
Test do_course_reindex response with real data
"""
# results are indexed because they are published from ItemFactory
response = perform_search(
"unique",
user=self.user,
size=10,
from_=0,
course_id=unicode(self.course.id))
self.assertEqual(response['total'], 1)
# Start manual reindex
CoursewareSearchIndexer.do_course_reindex(modulestore(),
self.course.id)
# Check results are the same following reindex
response = perform_search(
"unique",
user=self.user,
size=10,
from_=0,
course_id=unicode(self.course.id))
self.assertEqual(response['total'], 1)
def _do_test_large_course_deletion(self, store, load_factor):
""" Test that deleting items from a course works even when present within a very large course """
def id_list(top_parent_object):
""" private function to get ids from object down the tree """
list_of_ids = [unicode(top_parent_object.location)]
for child in top_parent_object.get_children():
list_of_ids.extend(id_list(child))
return list_of_ids
course, course_size = create_large_course(store, load_factor)
self.course_id = unicode(course.id)
# index full course
CoursewareSearchIndexer.do_course_reindex(store, course.id)
self.assert_search_count(course_size)
# reload course to allow us to delete one single unit
course = store.get_course(course.id, depth=1)
# delete the first chapter
chapter_to_delete = course.get_children()[0]
self.delete_item(store, chapter_to_delete.location)
# index and check correctness
CoursewareSearchIndexer.do_course_reindex(store, course.id)
deleted_count = 1 + load_factor + (load_factor ** 2) + (load_factor ** 3)
self.assert_search_count(course_size - deleted_count)
def update_search_index(course_id, triggered_time_isoformat):
""" Updates course search index. """
try:
course_key = CourseKey.from_string(course_id)
CoursewareSearchIndexer.index(modulestore(), course_key, triggered_at=(_parse_time(triggered_time_isoformat)))
except SearchIndexingError as exc:
LOGGER.error(u'Search indexing error for complete course %s - %s', course_id, text_type(exc))
else:
LOGGER.debug(u'Search indexing successful for complete course %s', course_id)
def test_indexing_no_item(self, mock_get_course):
"""
Test system logs an error if no item found.
"""
# set mocked exception response
err = ItemNotFoundError
mock_get_course.return_value = err
# Start manual reindex and check error in response
with self.assertRaises(SearchIndexingError):
CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
def delete_course_task(user_id, course_key_string):
profile = UserProfile.objects.get(pk=user_id)
user = User.objects.get(pk=profile.user_id)
course_key = CourseKey.from_string(course_key_string)
delete_course_and_groups(course_key, user.id)
searcher = SearchEngine.get_search_engine(CoursewareSearchIndexer.INDEX_NAME)
if searcher != None:
CoursewareSearchIndexer.remove_deleted_items(searcher, CourseKey.from_string(course_key_string), [])
searcher.remove(CourseAboutSearchIndexer.DISCOVERY_DOCUMENT_TYPE, [course_key_string])
def handle(self, *args, **options):
"""
By convention set by Django developers, this method actually executes command's actions.
So, there could be no better docstring than emphasize this once again.
"""
course_ids = options['course_ids']
all_option = options['all']
setup_option = options['setup']
index_all_courses_option = all_option or setup_option
if (not len(course_ids) and not index_all_courses_option) or \
(len(course_ids) and index_all_courses_option):
raise CommandError("reindex_course requires one or more <course_id>s OR the --all or --setup flags.")
store = modulestore()
if index_all_courses_option:
index_name = CoursewareSearchIndexer.INDEX_NAME
doc_type = CoursewareSearchIndexer.DOCUMENT_TYPE
if setup_option:
try:
# try getting the ElasticSearch engine
searcher = SearchEngine.get_search_engine(index_name)
except exceptions.ElasticsearchException as exc:
logging.exception(u'Search Engine error - %s', exc)
return
index_exists = searcher._es.indices.exists(index=index_name) # pylint: disable=protected-access
doc_type_exists = searcher._es.indices.exists_type( # pylint: disable=protected-access
index=index_name,
doc_type=doc_type
)
index_mapping = searcher._es.indices.get_mapping( # pylint: disable=protected-access
index=index_name,
doc_type=doc_type
) if index_exists and doc_type_exists else {}
if index_exists and index_mapping:
return
# if reindexing is done during devstack setup step, don't prompt the user
if setup_option or query_yes_no(self.CONFIRMATION_PROMPT, default="no"):
# in case of --setup or --all, get the list of course keys from all courses
# that are stored in the modulestore
course_keys = [course.id for course in modulestore().get_courses()]
else:
return
else:
# in case course keys are provided as arguments
course_keys = map(self._parse_course_key, course_ids)
for course_key in course_keys:
CoursewareSearchIndexer.do_course_reindex(store, course_key)
def test_indexing_seq_error_responses(self, mock_index_dictionary):
"""
Test do_course_reindex response with mocked error data for sequence
"""
# results are indexed because they are published from ItemFactory
response = perform_search("unique", user=self.user, size=10, from_=0, course_id=unicode(self.course.id))
self.assertEqual(response["total"], 1)
# set mocked exception response
err = Exception
mock_index_dictionary.return_value = err
# Start manual reindex and check error in response
with self.assertRaises(SearchIndexingError):
CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
def update_search_index(course_id, triggered_time_isoformat):
""" Updates course search index. """
try:
course_key = CourseKey.from_string(course_id)
triggered_time = datetime.strptime(
# remove the +00:00 from the end of the formats generated within the system
triggered_time_isoformat.split('+')[0],
"%Y-%m-%dT%H:%M:%S.%f"
).replace(tzinfo=UTC)
CoursewareSearchIndexer.index_course(modulestore(), course_key, triggered_at=triggered_time)
except SearchIndexingError as exc:
LOGGER.error('Search indexing error for complete course %s - %s', course_id, unicode(exc))
else:
LOGGER.debug('Search indexing successful for complete course %s', course_id)
def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Receives publishing signal and performs publishing related workflows, such as
registering proctored exams, building up credit requirements, and performing
search indexing
"""
# first is to registered exams, the credit subsystem will assume that
# all proctored exams have already been registered, so we have to do that first
try:
register_special_exams(course_key)
# pylint: disable=broad-except
except Exception as exception:
log.exception(exception)
# then call into the credit subsystem (in /openedx/djangoapps/credit)
# to perform any 'on_publish' workflow
on_course_publish(course_key)
# Finally call into the course search subsystem
# to kick off an indexing action
if CoursewareSearchIndexer.indexing_is_enabled():
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
from contentstore.tasks import update_search_index
update_search_index.delay(unicode(course_key), datetime.now(UTC).isoformat())
def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Receives signal and kicks off celery task to update search index
"""
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
from .tasks import update_search_index
if CoursewareSearchIndexer.indexing_is_enabled():
update_search_index.delay(unicode(course_key), datetime.now(UTC).isoformat())
def index_recent_changes(self, store, since_time):
""" index course using recent changes """
trigger_time = datetime.now(UTC)
return CoursewareSearchIndexer.index(
store,
self.course.id,
triggered_at=trigger_time,
reindex_age=(trigger_time - since_time)
)
def delete_temp_user_task(request, user_id):
profile = UserProfile.objects.get(pk=user_id)
user = User.objects.get(pk=profile.user_id)
courses = [format_course_for_view(c) for c in get_courses_accessible_to_user(request, user)[0]]
libraries = [format_library_for_view(lib, user) for lib in accessible_libraries_list(user)]
for course in courses:
course_key = CourseKey.from_string(course["course_key"])
delete_course_and_groups(course_key, user.id)
searcher = SearchEngine.get_search_engine(CoursewareSearchIndexer.INDEX_NAME)
if searcher != None:
CoursewareSearchIndexer.remove_deleted_items(searcher, CourseKey.from_string(course_key_string), [])
searcher.remove(CourseAboutSearchIndexer.DISCOVERY_DOCUMENT_TYPE, [course_key_string])
for library in libraries:
library_key = CourseKey.from_string(library['library_key'])
delete_course_and_groups(library_key, user.id)
def test_indexing_responses(self):
"""
Test do_course_reindex response with real data
"""
# results are indexed because they are published from ItemFactory
response = perform_search(
"unique",
user=self.user,
size=10,
from_=0,
course_id=unicode(self.course.id))
self.assertEqual(response['total'], 1)
# Start manual reindex
CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
# Check results are the same following reindex
response = perform_search(
"unique",
user=self.user,
size=10,
from_=0,
course_id=unicode(self.course.id))
self.assertEqual(response['total'], 1)
def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=unused-argument
"""
Receives publishing signal and performs publishing related workflows, such as
registering proctored exams, building up credit requirements, and performing
search indexing
"""
# first is to registered exams, the credit subsystem will assume that
# all proctored exams have already been registered, so we have to do that first
register_proctored_exams(course_key)
# then call into the credit subsystem (in /openedx/djangoapps/credit)
# to perform any 'on_publish' workflow
on_course_publish(course_key)
# Finally call into the course search subsystem
# to kick off an indexing action
if CoursewareSearchIndexer.indexing_is_enabled():
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
from .tasks import update_search_index
update_search_index.delay(unicode(course_key), datetime.now(UTC).isoformat())
def reindex_course(self, store):
""" kick off complete reindex of the course """
return CoursewareSearchIndexer.do_course_reindex(store, self.course.id)
def reindex_course(self, store):
""" kick off complete reindex of the course """
return CoursewareSearchIndexer.do_course_reindex(store, self.course.id)
