def test_indexing_responses(self):
"""
Test add_to_search_index response with real data
"""
# Check results not indexed
response = perform_search("unique",
user=self.user,
size=10,
from_=0,
course_id=unicode(self.course.id))
self.assertEqual(response['results'], [])
# Start manual reindex
errors = CoursewareSearchIndexer.do_course_reindex(
modulestore(), self.course.id)
self.assertEqual(errors, None)
self.html.display_name = "My expanded HTML"
modulestore().update_item(self.html, ModuleStoreEnum.UserID.test)
# Start manual reindex
errors = CoursewareSearchIndexer.do_course_reindex(
modulestore(), self.course.id)
self.assertEqual(errors, None)
# Check results indexed now
response = perform_search("unique",
user=self.user,
size=10,
from_=0,
course_id=unicode(self.course.id))
self.assertEqual(response['total'], 1)
def publish(self, location, user_id, blacklist=None, **kwargs):
"""
Publishes the subtree under location from the draft branch to the published branch
Returns the newly published item.
"""
super(DraftVersioningModuleStore, self).copy(
user_id,
# Directly using the replace function rather than the for_branch function
# because for_branch obliterates the version_guid and will lead to missed version conflicts.
# TODO Instead, the for_branch implementation should be fixed in the Opaque Keys library.
location.course_key.replace(branch=ModuleStoreEnum.BranchName.draft
),
# We clear out the version_guid here because the location here is from the draft branch, and that
# won't have the same version guid
location.course_key.replace(
branch=ModuleStoreEnum.BranchName.published,
version_guid=None),
[location],
blacklist=blacklist)
# Now it's been published, add the object to the courseware search index so that it appears in search results
CoursewareSearchIndexer.add_to_search_index(self, location)
return self.get_item(
location.for_branch(ModuleStoreEnum.BranchName.published),
**kwargs)
def test_indexing_responses(self):
"""
Test add_to_search_index response with real data
"""
# Check results not indexed
response = perform_search(
"unique",
user=self.user,
size=10,
from_=0,
course_id=unicode(self.course.id))
self.assertEqual(response['results'], [])
# Start manual reindex
errors = CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
self.assertEqual(errors, None)
self.html.display_name = "My expanded HTML"
modulestore().update_item(self.html, ModuleStoreEnum.UserID.test)
# Start manual reindex
errors = CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
self.assertEqual(errors, None)
# Check results indexed now
response = perform_search(
"unique",
user=self.user,
size=10,
from_=0,
course_id=unicode(self.course.id))
self.assertEqual(response['total'], 1)
def test_indexing_no_item(self, mock_get_course):
"""
Test system logs an error if no item found.
"""
# set mocked exception response
err = ItemNotFoundError
mock_get_course.return_value = err
# Start manual reindex and check error in response
with self.assertRaises(SearchIndexingError):
CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
def test_indexing_no_item(self, mock_get_course):
"""
Test system logs an error if no item found.
"""
# set mocked exception response
err = ItemNotFoundError
mock_get_course.return_value = err
# Start manual reindex and check error in response
with self.assertRaises(SearchIndexingError):
CoursewareSearchIndexer.do_course_reindex(modulestore(),
self.course.id)
def delete_item(self, location, user_id, revision=None, **kwargs):
"""
Delete the given item from persistence. kwargs allow modulestore specific parameters.
Args:
location: UsageKey of the item to be deleted
user_id: id of the user deleting the item
revision:
None - deletes the item and its subtree, and updates the parents per description above
ModuleStoreEnum.RevisionOption.published_only - removes only Published versions
ModuleStoreEnum.RevisionOption.all - removes both Draft and Published parents
currently only provided by contentstore.views.item.orphan_handler
Otherwise, raises a ValueError.
"""
with self.bulk_operations(location.course_key):
if isinstance(location, LibraryUsageLocator):
branches_to_delete = [
ModuleStoreEnum.BranchName.library
] # Libraries don't yet have draft/publish support
elif revision == ModuleStoreEnum.RevisionOption.published_only:
branches_to_delete = [ModuleStoreEnum.BranchName.published]
elif revision == ModuleStoreEnum.RevisionOption.all:
branches_to_delete = [
ModuleStoreEnum.BranchName.published,
ModuleStoreEnum.BranchName.draft
]
elif revision is None:
branches_to_delete = [ModuleStoreEnum.BranchName.draft]
else:
raise UnsupportedRevisionError([
None, ModuleStoreEnum.RevisionOption.published_only,
ModuleStoreEnum.RevisionOption.all
])
for branch in branches_to_delete:
branched_location = location.for_branch(branch)
parent_loc = self.get_parent_location(branched_location)
SplitMongoModuleStore.delete_item(self, branched_location,
user_id)
# publish parent w/o child if deleted element is direct only (not based on type of parent)
if branch == ModuleStoreEnum.BranchName.draft and branched_location.block_type in DIRECT_ONLY_CATEGORIES:
self.publish(parent_loc.version_agnostic(),
user_id,
blacklist=EXCLUDE_ALL,
**kwargs)
# Remove this location from the courseware search index so that searches
# will refrain from showing it as a result
CoursewareSearchIndexer.add_to_search_index(self,
location,
delete=True)
def delete_item(self, location, user_id, revision=None, **kwargs):
"""
Delete the given item from persistence. kwargs allow modulestore specific parameters.
Args:
location: UsageKey of the item to be deleted
user_id: id of the user deleting the item
revision:
None - deletes the item and its subtree, and updates the parents per description above
ModuleStoreEnum.RevisionOption.published_only - removes only Published versions
ModuleStoreEnum.RevisionOption.all - removes both Draft and Published parents
currently only provided by contentstore.views.item.orphan_handler
Otherwise, raises a ValueError.
"""
with self.bulk_operations(location.course_key):
if isinstance(location, LibraryUsageLocator):
branches_to_delete = [ModuleStoreEnum.BranchName.library] # Libraries don't yet have draft/publish support
elif revision == ModuleStoreEnum.RevisionOption.published_only:
branches_to_delete = [ModuleStoreEnum.BranchName.published]
elif revision == ModuleStoreEnum.RevisionOption.all:
branches_to_delete = [ModuleStoreEnum.BranchName.published, ModuleStoreEnum.BranchName.draft]
elif revision is None:
branches_to_delete = [ModuleStoreEnum.BranchName.draft]
else:
raise UnsupportedRevisionError(
[
None,
ModuleStoreEnum.RevisionOption.published_only,
ModuleStoreEnum.RevisionOption.all
]
)
self._flag_publish_event(location.course_key)
for branch in branches_to_delete:
branched_location = location.for_branch(branch)
parent_loc = self.get_parent_location(branched_location)
SplitMongoModuleStore.delete_item(self, branched_location, user_id)
# publish parent w/o child if deleted element is direct only (not based on type of parent)
if branch == ModuleStoreEnum.BranchName.draft and branched_location.block_type in DIRECT_ONLY_CATEGORIES:
self.publish(parent_loc.version_agnostic(), user_id, blacklist=EXCLUDE_ALL, **kwargs)
# Remove this location from the courseware search index so that searches
# will refrain from showing it as a result
CoursewareSearchIndexer.add_to_search_index(self, location, delete=True)
def test_indexing_seq_error_responses(self, mock_index_dictionary):
"""
Test add_to_search_index response with mocked error data for sequence
"""
# Check results not indexed
response = perform_search(
"unique",
user=self.user,
size=10,
from_=0,
course_id=unicode(self.course.id))
self.assertEqual(response['results'], [])
# set mocked exception response
err = Exception
mock_index_dictionary.return_value = err
# Start manual reindex and check error in response
with self.assertRaises(SearchIndexingError):
CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
def test_indexing_seq_error_responses(self, mock_index_dictionary):
"""
Test add_to_search_index response with mocked error data for sequence
"""
# Check results not indexed
response = perform_search("unique",
user=self.user,
size=10,
from_=0,
course_id=unicode(self.course.id))
self.assertEqual(response['results'], [])
# set mocked exception response
err = Exception
mock_index_dictionary.return_value = err
# Start manual reindex and check error in response
with self.assertRaises(SearchIndexingError):
CoursewareSearchIndexer.do_course_reindex(modulestore(),
self.course.id)
def publish(self, location, user_id, blacklist=None, **kwargs):
"""
Publishes the subtree under location from the draft branch to the published branch
Returns the newly published item.
"""
super(DraftVersioningModuleStore, self).copy(
user_id,
# Directly using the replace function rather than the for_branch function
# because for_branch obliterates the version_guid and will lead to missed version conflicts.
# TODO Instead, the for_branch implementation should be fixed in the Opaque Keys library.
location.course_key.replace(branch=ModuleStoreEnum.BranchName.draft),
# We clear out the version_guid here because the location here is from the draft branch, and that
# won't have the same version guid
location.course_key.replace(branch=ModuleStoreEnum.BranchName.published, version_guid=None),
[location],
blacklist=blacklist
)
# Now it's been published, add the object to the courseware search index so that it appears in search results
CoursewareSearchIndexer.add_to_search_index(self, location)
return self.get_item(location.for_branch(ModuleStoreEnum.BranchName.published), **kwargs)
def publish(self, location, user_id, **kwargs):
"""
Publish the subtree rooted at location to the live course and remove the drafts.
Such publishing may cause the deletion of previously published but subsequently deleted
child trees. Overwrites any existing published xblocks from the subtree.
Treats the publishing of non-draftable items as merely a subtree selection from
which to descend.
Raises:
ItemNotFoundError: if any of the draft subtree nodes aren't found
Returns:
The newly published xblock
"""
# NOTE: cannot easily use self._breadth_first b/c need to get pub'd and draft as pairs
# (could do it by having 2 breadth first scans, the first to just get all published children
# and the second to do the publishing on the drafts looking for the published in the cached
# list of published ones.)
to_be_deleted = []
def _internal_depth_first(item_location, is_root):
"""
Depth first publishing from the given location
"""
try:
# handle child does not exist w/o killing publish
item = self.get_item(item_location)
except ItemNotFoundError:
log.warning('Cannot find: %s', item_location)
return
# publish the children first
if item.has_children:
for child_loc in item.children:
_internal_depth_first(child_loc, False)
if item_location.category in DIRECT_ONLY_CATEGORIES or not getattr(item, 'is_draft', False):
# ignore noop attempt to publish something that can't be or isn't currently draft
return
# try to find the originally PUBLISHED version, if it exists
try:
original_published = super(DraftModuleStore, self).get_item(item_location)
except ItemNotFoundError:
original_published = None
# if the category of this item allows having children
if item.has_children:
if original_published is not None:
# see if previously published children were deleted. 2 reasons for children lists to differ:
# Case 1: child deleted
# Case 2: child moved
for orig_child in original_published.children:
if orig_child not in item.children:
published_parent = self.get_parent_location(orig_child)
if published_parent == item_location:
# Case 1: child was deleted in draft parent item
# So, delete published version of the child now that we're publishing the draft parent
self._delete_subtree(orig_child, [as_published])
else:
# Case 2: child was moved to a new draft parent item
# So, do not delete the child. It will be published when the new parent is published.
pass
# update the published (not draft) item (ignoring that item is "draft"). The published
# may not exist; (if original_published is None); so, allow_not_found
super(DraftModuleStore, self).update_item(
item, user_id, isPublish=True, is_publish_root=is_root, allow_not_found=True
)
to_be_deleted.append(as_draft(item_location).to_deprecated_son())
# verify input conditions
self._verify_branch_setting(ModuleStoreEnum.Branch.draft_preferred)
_verify_revision_is_published(location)
_internal_depth_first(location, True)
course_key = location.course_key
bulk_record = self._get_bulk_ops_record(course_key)
if len(to_be_deleted) > 0:
bulk_record.dirty = True
self.collection.remove({'_id': {'$in': to_be_deleted}})
self._flag_publish_event(course_key)
# Now it's been published, add the object to the courseware search index so that it appears in search results
CoursewareSearchIndexer.do_publish_index(self, location)
return self.get_item(as_published(location))
def delete_item(self, location, user_id, revision=None, **kwargs):
"""
Delete an item from this modulestore.
The method determines which revisions to delete. It disconnects and deletes the subtree.
In general, it assumes deletes only occur on drafts except for direct_only. The only exceptions
are internal calls like deleting orphans (during publishing as well as from delete_orphan view).
To indicate that all versions should be deleted, pass the keyword revision=ModuleStoreEnum.RevisionOption.all.
* Deleting a DIRECT_ONLY_CATEGORIES block, deletes both draft and published children and removes from parent.
* Deleting a specific version of block whose parent is of DIRECT_ONLY_CATEGORIES, only removes it from parent if
the other version of the block does not exist. Deletes only children of same version.
* Other deletions remove from parent of same version and subtree of same version
Args:
location: UsageKey of the item to be deleted
user_id: id of the user deleting the item
revision:
None - deletes the item and its subtree, and updates the parents per description above
ModuleStoreEnum.RevisionOption.published_only - removes only Published versions
ModuleStoreEnum.RevisionOption.all - removes both Draft and Published parents
currently only provided by contentstore.views.item.orphan_handler
Otherwise, raises a ValueError.
"""
self._verify_branch_setting(ModuleStoreEnum.Branch.draft_preferred)
_verify_revision_is_published(location)
is_item_direct_only = location.category in DIRECT_ONLY_CATEGORIES
if is_item_direct_only or revision == ModuleStoreEnum.RevisionOption.published_only:
parent_revision = MongoRevisionKey.published
elif revision == ModuleStoreEnum.RevisionOption.all:
parent_revision = ModuleStoreEnum.RevisionOption.all
else:
parent_revision = MongoRevisionKey.draft
# remove subtree from its parent
parent_locations = self._get_raw_parent_locations(location, key_revision=parent_revision)
# if no parents, then we're trying to delete something which we should convert to draft
if not parent_locations:
# find the published parent, convert it to draft, then manipulate the draft
parent_locations = self._get_raw_parent_locations(location, key_revision=MongoRevisionKey.published)
# parent_locations will still be empty if the object was an orphan
if parent_locations:
draft_parent = self.convert_to_draft(parent_locations[0], user_id)
parent_locations = [draft_parent.location]
# there could be 2 parents if
# Case 1: the draft item moved from one parent to another
# Case 2: revision==ModuleStoreEnum.RevisionOption.all and the single
# parent has 2 versions: draft and published
for parent_location in parent_locations:
# don't remove from direct_only parent if other versions of this still exists (this code
# assumes that there's only one parent_location in this case)
if not is_item_direct_only and parent_location.category in DIRECT_ONLY_CATEGORIES:
# see if other version of to-be-deleted root exists
query = location.to_deprecated_son(prefix='_id.')
del query['_id.revision']
if self.collection.find(query).count() > 1:
continue
parent_block = super(DraftModuleStore, self).get_item(parent_location)
parent_block.children.remove(location)
parent_block.location = parent_location # ensure the location is with the correct revision
self.update_item(parent_block, user_id, child_update=True)
self._flag_publish_event(location.course_key)
if is_item_direct_only or revision == ModuleStoreEnum.RevisionOption.all:
as_functions = [as_draft, as_published]
elif revision == ModuleStoreEnum.RevisionOption.published_only:
as_functions = [as_published]
elif revision is None:
as_functions = [as_draft]
else:
raise UnsupportedRevisionError(
[
None,
ModuleStoreEnum.RevisionOption.published_only,
ModuleStoreEnum.RevisionOption.all
]
)
self._delete_subtree(location, as_functions)
# Remove this location from the courseware search index so that searches
# will refrain from showing it as a result
CoursewareSearchIndexer.add_to_search_index(self, location, delete=True)
def publish(self, location, user_id, **kwargs):
"""
Publish the subtree rooted at location to the live course and remove the drafts.
Such publishing may cause the deletion of previously published but subsequently deleted
child trees. Overwrites any existing published xblocks from the subtree.
Treats the publishing of non-draftable items as merely a subtree selection from
which to descend.
Raises:
ItemNotFoundError: if any of the draft subtree nodes aren't found
Returns:
The newly published xblock
"""
# NOTE: cannot easily use self._breadth_first b/c need to get pub'd and draft as pairs
# (could do it by having 2 breadth first scans, the first to just get all published children
# and the second to do the publishing on the drafts looking for the published in the cached
# list of published ones.)
to_be_deleted = []
def _internal_depth_first(item_location, is_root):
"""
Depth first publishing from the given location
"""
try:
# handle child does not exist w/o killing publish
item = self.get_item(item_location)
except ItemNotFoundError:
log.warning('Cannot find: %s', item_location)
return
# publish the children first
if item.has_children:
for child_loc in item.children:
_internal_depth_first(child_loc, False)
if item_location.category in DIRECT_ONLY_CATEGORIES or not getattr(
item, 'is_draft', False):
# ignore noop attempt to publish something that can't be or isn't currently draft
return
# try to find the originally PUBLISHED version, if it exists
try:
original_published = super(DraftModuleStore,
self).get_item(item_location)
except ItemNotFoundError:
original_published = None
# if the category of this item allows having children
if item.has_children:
if original_published is not None:
# see if previously published children were deleted. 2 reasons for children lists to differ:
# Case 1: child deleted
# Case 2: child moved
for orig_child in original_published.children:
if orig_child not in item.children:
published_parent = self.get_parent_location(
orig_child)
if published_parent == item_location:
# Case 1: child was deleted in draft parent item
# So, delete published version of the child now that we're publishing the draft parent
self._delete_subtree(orig_child,
[as_published])
else:
# Case 2: child was moved to a new draft parent item
# So, do not delete the child. It will be published when the new parent is published.
pass
# update the published (not draft) item (ignoring that item is "draft"). The published
# may not exist; (if original_published is None); so, allow_not_found
super(DraftModuleStore, self).update_item(item,
user_id,
isPublish=True,
is_publish_root=is_root,
allow_not_found=True)
to_be_deleted.append(as_draft(item_location).to_deprecated_son())
# verify input conditions
self._verify_branch_setting(ModuleStoreEnum.Branch.draft_preferred)
_verify_revision_is_published(location)
_internal_depth_first(location, True)
course_key = location.course_key
bulk_record = self._get_bulk_ops_record(course_key)
if len(to_be_deleted) > 0:
bulk_record.dirty = True
self.collection.remove({'_id': {'$in': to_be_deleted}})
self._flag_publish_event(course_key)
# Now it's been published, add the object to the courseware search index so that it appears in search results
CoursewareSearchIndexer.do_publish_index(self, location)
return self.get_item(as_published(location))
def delete_item(self, location, user_id, revision=None, **kwargs):
"""
Delete an item from this modulestore.
The method determines which revisions to delete. It disconnects and deletes the subtree.
In general, it assumes deletes only occur on drafts except for direct_only. The only exceptions
are internal calls like deleting orphans (during publishing as well as from delete_orphan view).
To indicate that all versions should be deleted, pass the keyword revision=ModuleStoreEnum.RevisionOption.all.
* Deleting a DIRECT_ONLY_CATEGORIES block, deletes both draft and published children and removes from parent.
* Deleting a specific version of block whose parent is of DIRECT_ONLY_CATEGORIES, only removes it from parent if
the other version of the block does not exist. Deletes only children of same version.
* Other deletions remove from parent of same version and subtree of same version
Args:
location: UsageKey of the item to be deleted
user_id: id of the user deleting the item
revision:
None - deletes the item and its subtree, and updates the parents per description above
ModuleStoreEnum.RevisionOption.published_only - removes only Published versions
ModuleStoreEnum.RevisionOption.all - removes both Draft and Published parents
currently only provided by contentstore.views.item.orphan_handler
Otherwise, raises a ValueError.
"""
self._verify_branch_setting(ModuleStoreEnum.Branch.draft_preferred)
_verify_revision_is_published(location)
is_item_direct_only = location.category in DIRECT_ONLY_CATEGORIES
if is_item_direct_only or revision == ModuleStoreEnum.RevisionOption.published_only:
parent_revision = MongoRevisionKey.published
elif revision == ModuleStoreEnum.RevisionOption.all:
parent_revision = ModuleStoreEnum.RevisionOption.all
else:
parent_revision = MongoRevisionKey.draft
# remove subtree from its parent
parent_locations = self._get_raw_parent_locations(
location, key_revision=parent_revision)
# if no parents, then we're trying to delete something which we should convert to draft
if not parent_locations:
# find the published parent, convert it to draft, then manipulate the draft
parent_locations = self._get_raw_parent_locations(
location, key_revision=MongoRevisionKey.published)
# parent_locations will still be empty if the object was an orphan
if parent_locations:
draft_parent = self.convert_to_draft(parent_locations[0],
user_id)
parent_locations = [draft_parent.location]
# there could be 2 parents if
# Case 1: the draft item moved from one parent to another
# Case 2: revision==ModuleStoreEnum.RevisionOption.all and the single
# parent has 2 versions: draft and published
for parent_location in parent_locations:
# don't remove from direct_only parent if other versions of this still exists (this code
# assumes that there's only one parent_location in this case)
if not is_item_direct_only and parent_location.category in DIRECT_ONLY_CATEGORIES:
# see if other version of to-be-deleted root exists
query = location.to_deprecated_son(prefix='_id.')
del query['_id.revision']
if self.collection.find(query).count() > 1:
continue
parent_block = super(DraftModuleStore,
self).get_item(parent_location)
parent_block.children.remove(location)
parent_block.location = parent_location # ensure the location is with the correct revision
self.update_item(parent_block, user_id, child_update=True)
self._flag_publish_event(location.course_key)
if is_item_direct_only or revision == ModuleStoreEnum.RevisionOption.all:
as_functions = [as_draft, as_published]
elif revision == ModuleStoreEnum.RevisionOption.published_only:
as_functions = [as_published]
elif revision is None:
as_functions = [as_draft]
else:
raise UnsupportedRevisionError([
None, ModuleStoreEnum.RevisionOption.published_only,
ModuleStoreEnum.RevisionOption.all
])
self._delete_subtree(location, as_functions)
# Remove this location from the courseware search index so that searches
# will refrain from showing it as a result
CoursewareSearchIndexer.add_to_search_index(self,
location,
delete=True)