def handle(self, *args, **options):
if options['clear-all']:
if options['force'] or query_yes_no(self.CONFIRMATION_PROMPT_CLEAR,
default="no"):
logging.info("Removing all libraries from the index")
ContentLibraryIndexer.remove_all_items()
LibraryBlockIndexer.remove_all_items()
return
if options['all']:
if options['force'] or query_yes_no(self.CONFIRMATION_PROMPT_ALL,
default="no"):
logging.info("Indexing all libraries")
library_keys = [
library.library_key
for library in ContentLibrary.objects.all()
]
else:
return
else:
logging.info("Indexing libraries: {}".format(
options['library_ids']))
library_keys = list(
map(LibraryLocatorV2.from_string, options['library_ids']))
ContentLibraryIndexer.index_items(library_keys)
for library_key in library_keys:
ref = ContentLibrary.objects.get_by_key(library_key)
lib_bundle = LibraryBundle(library_key,
ref.bundle_uuid,
draft_name=DRAFT_NAME)
LibraryBlockIndexer.index_items(lib_bundle.get_all_usages())
def test_update_libraries(self):
"""
Test if indexes are updated when libraries are updated
"""
lib = self._create_library(slug="test-lib-update",
title="Title",
description="Description")
library_key = LibraryLocatorV2.from_string(lib['id'])
self._update_library(lib['id'],
title="New Title",
description="New Title")
response = ContentLibraryIndexer.get_items([library_key])[0]
assert response['id'] == lib['id']
assert response['title'] == 'New Title'
assert response['description'] == 'New Title'
assert response['uuid'] == lib['bundle_uuid']
assert response['num_blocks'] == 0
assert response['version'] == lib['version']
assert response['last_published'] is None
assert response['has_unpublished_changes'] is False
assert response['has_unpublished_deletes'] is False
self._delete_library(lib['id'])
assert ContentLibraryIndexer.get_items([library_key]) == []
ContentLibraryIndexer.get_items([library_key])
def commit_library_and_verify(library_key):
"""
Commit library changes, and verify that there are no uncommited changes anymore
"""
last_published = ContentLibraryIndexer.get_libraries([library_key])[0]['last_published']
self._commit_library_changes(str(library_key))
response = ContentLibraryIndexer.get_libraries([library_key])[0]
self.assertEqual(response['has_unpublished_changes'], False)
self.assertEqual(response['has_unpublished_deletes'], False)
self.assertGreaterEqual(response['last_published'], last_published)
return response
def commit_library_and_verify(library_key):
"""
Commit library changes, and verify that there are no uncommited changes anymore
"""
last_published = ContentLibraryIndexer.get_items(
[library_key])[0]['last_published']
self._commit_library_changes(str(library_key))
response = ContentLibraryIndexer.get_items([library_key])[0]
assert response['has_unpublished_changes'] is False
assert response['has_unpublished_deletes'] is False
assert response['last_published'] >= last_published
return response
def test_remove_all_libraries(self):
"""
Test if remove_all_items() deletes all libraries
"""
lib1 = self._create_library(slug="test-lib-rm-all-1", title="Title 1", description="Description")
lib2 = self._create_library(slug="test-lib-rm-all-2", title="Title 2", description="Description")
library_key1 = LibraryLocatorV2.from_string(lib1['id'])
library_key2 = LibraryLocatorV2.from_string(lib2['id'])
self.assertEqual(len(ContentLibraryIndexer.get_items([library_key1, library_key2])), 2)
ContentLibraryIndexer.remove_all_items()
self.assertEqual(len(ContentLibraryIndexer.get_items()), 0)
def test_remove_all_libraries(self):
"""
Test if remove_all_libraries() deletes all libraries
"""
self._create_library(slug="test-lib-rm-all-1", title="Title 1", description="Description")
self._create_library(slug="test-lib-rm-all-2", title="Title 2", description="Description")
response = self.searcher.search(doc_type=ContentLibraryIndexer.LIBRARY_DOCUMENT_TYPE, filter_dictionary={})
self.assertEqual(response['total'], 2)
ContentLibraryIndexer.remove_all_libraries()
response = self.searcher.search(doc_type=ContentLibraryIndexer.LIBRARY_DOCUMENT_TYPE, filter_dictionary={})
self.assertEqual(response['total'], 0)
def verify_uncommitted_libraries(library_key, has_unpublished_changes, has_unpublished_deletes):
"""
Verify uncommitted changes and deletes in the index
"""
response = ContentLibraryIndexer.get_libraries([library_key])[0]
self.assertEqual(response['has_unpublished_changes'], has_unpublished_changes)
self.assertEqual(response['has_unpublished_deletes'], has_unpublished_deletes)
return response
def test_schema_updates(self):
"""
Test that outdated indexes aren't retrieved
"""
with patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SCHEMA_VERSION",
new=0):
result = self._create_library(slug="test-lib-schemaupdates-1", title="Title 1", description="Description")
library_key = LibraryLocatorV2.from_string(result['id'])
self.assertEqual(len(ContentLibraryIndexer.get_items([library_key])), 1)
with patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SCHEMA_VERSION",
new=1):
self.assertEqual(len(ContentLibraryIndexer.get_items([library_key])), 0)
call_command("reindex_content_library", all=True, force=True)
self.assertEqual(len(ContentLibraryIndexer.get_items([library_key])), 1)
def get_metadata_from_index(queryset):
"""
Take a list of ContentLibrary objects and return metadata stored in
ContentLibraryIndex.
"""
metadata = None
if ContentLibraryIndexer.indexing_is_enabled():
try:
library_keys = [lib.library_key for lib in queryset]
metadata = ContentLibraryIndexer.get_libraries(library_keys)
except (LibraryNotIndexedException, KeyError,
ElasticConnectionError) as e:
log.exception(e)
# If ContentLibraryIndex is not available, we query blockstore for a limited set of metadata
if metadata is None:
uuids = [lib.bundle_uuid for lib in queryset]
bundles = get_bundles(uuids)
bundle_dict = {
bundle.uuid: {
'uuid': bundle.uuid,
'title': bundle.title,
'description': bundle.description,
'version': bundle.latest_version,
}
for bundle in bundles
}
metadata = [bundle_dict[uuid] for uuid in uuids]
libraries = [
ContentLibraryMetadata(
key=lib.library_key,
bundle_uuid=metadata[i]['uuid'],
title=metadata[i]['title'],
description=metadata[i]['description'],
version=metadata[i]['version'],
allow_public_learning=queryset[i].allow_public_learning,
allow_public_read=queryset[i].allow_public_read,
num_blocks=metadata[i].get('num_blocks'),
last_published=metadata[i].get('last_published'),
has_unpublished_changes=metadata[i].get('has_unpublished_changes'),
has_unpublished_deletes=metadata[i].get('has_unpublished_deletes'),
) for i, lib in enumerate(queryset)
]
return libraries
def handle(self, *args, **options):
if options['clear-all']:
if options['force'] or query_yes_no(self.CONFIRMATION_PROMPT_CLEAR, default="no"):
logging.info("Removing all libraries from the index")
ContentLibraryIndexer.remove_all_libraries()
return
if options['all']:
if options['force'] or query_yes_no(self.CONFIRMATION_PROMPT_ALL, default="no"):
logging.info("Indexing all libraries")
library_keys = [library.library_key for library in ContentLibrary.objects.all()]
else:
return
else:
logging.info("Indexing libraries: {}".format(options['library_ids']))
library_keys = list(map(LibraryLocatorV2.from_string, options['library_ids']))
ContentLibraryIndexer.index_libraries(library_keys)
def verify_uncommitted_libraries(library_key, has_unpublished_changes,
has_unpublished_deletes):
"""
Verify uncommitted changes and deletes in the index
"""
response = ContentLibraryIndexer.get_items([library_key])[0]
assert response[
'has_unpublished_changes'] == has_unpublished_changes
assert response[
'has_unpublished_deletes'] == has_unpublished_deletes
return response
def test_update_libraries(self):
"""
Test if indexes are updated when libraries are updated
"""
lib = self._create_library(slug="test-lib-update", title="Title", description="Description")
library_key = LibraryLocatorV2.from_string(lib['id'])
self._update_library(lib['id'], title="New Title", description="New Title")
response = ContentLibraryIndexer.get_libraries([library_key])[0]
self.assertEqual(response['id'], lib['id'])
self.assertEqual(response['title'], "New Title")
self.assertEqual(response['description'], "New Title")
self.assertEqual(response['uuid'], lib['bundle_uuid'])
self.assertEqual(response['num_blocks'], 0)
self.assertEqual(response['version'], lib['version'])
self.assertEqual(response['last_published'], None)
self.assertEqual(response['has_unpublished_changes'], False)
self.assertEqual(response['has_unpublished_deletes'], False)
self._delete_library(lib['id'])
with self.assertRaises(LibraryNotIndexedException):
ContentLibraryIndexer.get_libraries([library_key])
def test_index_libraries(self):
"""
Test if libraries are being indexed correctly
"""
result1 = self._create_library(slug="test-lib-index-1", title="Title 1", description="Description")
result2 = self._create_library(slug="test-lib-index-2", title="Title 2", description="Description")
for result in [result1, result2]:
library_key = LibraryLocatorV2.from_string(result['id'])
response = ContentLibraryIndexer.get_items([library_key])[0]
self.assertEqual(response['id'], result['id'])
self.assertEqual(response['title'], result['title'])
self.assertEqual(response['description'], result['description'])
self.assertEqual(response['uuid'], result['bundle_uuid'])
self.assertEqual(response['num_blocks'], 0)
self.assertEqual(response['version'], result['version'])
self.assertEqual(response['last_published'], None)
self.assertEqual(response['has_unpublished_changes'], False)
self.assertEqual(response['has_unpublished_deletes'], False)
def test_schema_updates(self):
"""
Test that outdated indexes aren't retrieved
"""
result = self._create_library(slug="test-lib-schemaupdates-1", title="Title 1", description="Description")
library_key = LibraryLocatorV2.from_string(result['id'])
ContentLibraryIndexer.get_libraries([library_key])
with patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SCHEMA_VERSION",
new=1):
with self.assertRaises(LibraryNotIndexedException):
ContentLibraryIndexer.get_libraries([library_key])
call_command("reindex_content_library", all=True, quiet=True)
ContentLibraryIndexer.get_libraries([library_key])
def setUp(self):
super().setUp()
if settings.ENABLE_ELASTICSEARCH_FOR_TESTS:
ContentLibraryIndexer.remove_all_items()
LibraryBlockIndexer.remove_all_items()
def get_metadata_from_index(queryset, text_search=None):
"""
Take a list of ContentLibrary objects and return metadata stored in
ContentLibraryIndex.
"""
metadata = None
if ContentLibraryIndexer.indexing_is_enabled():
try:
library_keys = [str(lib.library_key) for lib in queryset]
metadata = ContentLibraryIndexer.get_items(library_keys,
text_search=text_search)
metadata_dict = {item["id"]: item for item in metadata}
metadata = [
metadata_dict[key] if key in metadata_dict else None
for key in library_keys
]
except ElasticConnectionError as e:
log.exception(e)
# If ContentLibraryIndex is not available, we query blockstore for a limited set of metadata
if metadata is None:
uuids = [lib.bundle_uuid for lib in queryset]
bundles = get_bundles(uuids=uuids, text_search=text_search)
if text_search:
# Bundle APIs can't apply text_search on a bundle's org, so including those results here
queryset_org_search = queryset.filter(
org__short_name__icontains=text_search)
if queryset_org_search.exists():
uuids_org_search = [
lib.bundle_uuid for lib in queryset_org_search
]
bundles += get_bundles(uuids=uuids_org_search)
bundle_dict = {
bundle.uuid: {
'uuid': bundle.uuid,
'title': bundle.title,
'description': bundle.description,
'version': bundle.latest_version,
}
for bundle in bundles
}
metadata = [
bundle_dict[uuid] if uuid in bundle_dict else None
for uuid in uuids
]
libraries = [
ContentLibraryMetadata(
key=lib.library_key,
bundle_uuid=metadata[i]['uuid'],
title=metadata[i]['title'],
description=metadata[i]['description'],
version=metadata[i]['version'],
allow_public_learning=queryset[i].allow_public_learning,
allow_public_read=queryset[i].allow_public_read,
num_blocks=metadata[i].get('num_blocks'),
last_published=metadata[i].get('last_published'),
has_unpublished_changes=metadata[i].get('has_unpublished_changes'),
has_unpublished_deletes=metadata[i].get('has_unpublished_deletes'),
) for i, lib in enumerate(queryset) if metadata[i] is not None
]
return libraries
def setUp(self):
super().setUp()
ContentLibraryIndexer.remove_all_items()
LibraryBlockIndexer.remove_all_items()
self.searcher = SearchEngine.get_search_engine(
LibraryBlockIndexer.INDEX_NAME)
def setUp(self):
super().setUp()
ContentLibraryIndexer.remove_all_libraries()
self.searcher = SearchEngine.get_search_engine(ContentLibraryIndexer.INDEX_NAME)
