def insert_course_index(self, course_index, course_context=None): # pylint: disable=arguments-differ
"""
Create the course_index in the db
"""
# clear the whole course_index request cache, required for sucessfully cloning a course.
# This is a relatively large hammer for the problem, but we mostly only use one course at a time.
RequestCache(namespace="course_index_cache").clear()
course_index['last_update'] = datetime.datetime.now(pytz.utc)
new_index = SplitModulestoreCourseIndex(
**SplitModulestoreCourseIndex.fields_from_v1_schema(course_index))
new_index.save()
# TEMP: Also write to MongoDB, so we can switch back to using it if this new MySQL version doesn't work well:
super().insert_course_index(course_index,
course_context,
last_update_already_set=True)
def test_course_id_case_sensitive(self):
"""
Make sure the course_id column is case sensitive.
Although the platform code generally tries to prevent having two courses whose IDs differ only by case
(e.g. https://git.io/J6voR , note `ignore_case=True`), we found at least one pair of courses on stage that
differs only by case in its `org` ID (`edx` vs `edX`). So for backwards compatibility with MongoDB and to avoid
issues for anyone else with similar course IDs that differ only by case, we've made the new version case
sensitive too. The system still tries to prevent creation of courses that differ only by course (that hasn't
changed), but now the MySQL version won't break if that has somehow happened.
"""
course_index_common = {
"course": "TL101",
"run": "2015",
"edited_by": ModuleStoreEnum.UserID.mgmt_command,
"edited_on": datetime.now(),
"last_update": datetime.now(),
"versions": {},
"schema_version": 1,
"search_targets": {
"wiki_slug": "TLslug"
},
}
course_index_1 = {
**course_index_common, "_id": ObjectId("553115a9d15a010b5c6f7228"),
"org": "edx"
}
course_index_2 = {
**course_index_common, "_id": ObjectId("550869e42d00970b5b082d2a"),
"org": "edX"
}
data1 = SplitModulestoreCourseIndex.fields_from_v1_schema(
course_index_1)
data2 = SplitModulestoreCourseIndex.fields_from_v1_schema(
course_index_2)
SplitModulestoreCourseIndex(**data1).save()
# This next line will fail if the course_id column is not case-sensitive:
SplitModulestoreCourseIndex(**data2).save()
# Also check deletion, to ensure the course_id historical record is not unique or case sensitive:
SplitModulestoreCourseIndex.objects.get(
course_id=CourseKey.from_string(
"course-v1:edx+TL101+2015")).delete()
SplitModulestoreCourseIndex.objects.get(
course_id=CourseKey.from_string(
"course-v1:edX+TL101+2015")).delete()
def find_matching_course_indexes( # pylint: disable=arguments-differ
self,
branch=None,
search_targets=None,
org_target=None,
course_context=None,
course_keys=None,
force_mongo=False,
):
"""
Find the course_index matching particular conditions.
Arguments:
branch: If specified, this branch must exist in the returned courses
search_targets: If specified, this must be a dictionary specifying field values
that must exist in the search_targets of the returned courses
org_target: If specified, this is an ORG filter so that only course_indexs are
returned for the specified ORG
"""
#######################
# TEMP: as we migrate, we are currently reading from MongoDB only, but writing to both MySQL + MongoDB
force_mongo = True
#######################
if force_mongo:
# For data migration purposes, this argument will read from MongoDB instead of MySQL
return super().find_matching_course_indexes(
branch=branch,
search_targets=search_targets,
org_target=org_target,
course_context=course_context,
course_keys=course_keys,
)
queryset = SplitModulestoreCourseIndex.objects.all()
if course_keys:
queryset = queryset.filter(course_id__in=course_keys)
if search_targets:
if "wiki_slug" in search_targets:
queryset = queryset.filter(
wiki_slug=search_targets.pop("wiki_slug"))
if search_targets: # If there are any search targets besides wiki_slug (which we've handled by this point):
raise ValueError(
f"Unsupported search_targets: {', '.join(search_targets.keys())}"
)
if org_target:
queryset = queryset.filter(org=org_target)
if branch is not None:
branch_field = SplitModulestoreCourseIndex.field_name_for_branch(
branch)
queryset = queryset.exclude(**{branch_field: ""})
return (course_index.as_v1_schema() for course_index in queryset)
def update_course_index(self,
course_index,
from_index=None,
course_context=None): # pylint: disable=arguments-differ
"""
Update the db record for course_index.
Arguments:
from_index: If set, only update an index if it matches the one specified in `from_index`.
Exceptions:
SplitModulestoreCourseIndex.DoesNotExist: If the given object_id is not valid
"""
# "last_update not only tells us when this course was last updated but also helps prevent collisions"
# This code is just copying the behavior of the existing MongoPersistenceBackend
# See https://github.com/edx/edx-platform/pull/5200 for context
RequestCache(namespace="course_index_cache").clear()
course_index['last_update'] = datetime.datetime.now(pytz.utc)
# Find the SplitModulestoreCourseIndex entry that we'll be updating:
try:
index_obj = SplitModulestoreCourseIndex.objects.get(
objectid=course_index["_id"])
except SplitModulestoreCourseIndex.DoesNotExist:
#######################
# TEMP: Maybe the data migration hasn't (completely) run yet?
data = SplitModulestoreCourseIndex.fields_from_v1_schema(
course_index)
if super().get_course_index(data["course_id"]) is None:
raise # This course doesn't exist in MySQL or in MongoDB
# This course record exists in MongoDB but not yet in MySQL
index_obj = SplitModulestoreCourseIndex(**data)
if from_index:
index_obj.last_update = from_index[
"last_update"] # Make sure this won't get marked as a collision
#######################
# Check for collisions:
# Except this collision logic doesn't work when using both MySQL and MongoDB together, one for writes and one
# for reads, so we're temporarily defering to Mongo's colision logic.
# if from_index and index_obj.last_update != from_index["last_update"]:
# # "last_update not only tells us when this course was last updated but also helps prevent collisions"
# log.warning(
# "Collision in Split Mongo when applying course index. This can happen in dev if django debug toolbar "
# "is enabled, as it slows down parallel queries. \nNew index was: %s\nFrom index was: %s",
# course_index, from_index,
# )
# return # Collision; skip this update
# Apply updates to the index entry. While doing so, track which branch versions were changed (if any).
changed_branches = []
for attr, value in SplitModulestoreCourseIndex.fields_from_v1_schema(
course_index).items():
if attr in ("objectid", "course_id"):
# Enforce these attributes as immutable.
if getattr(index_obj, attr) != value:
raise ValueError(
f"Attempted to change the {attr} key of a course index entry ({index_obj.course_id})"
)
else:
if attr.endswith("_version"):
# Model fields ending in _version are branches. If the branch version has changed, convert the field
# name to a branch name and report it in the history below.
if getattr(index_obj, attr) != value:
changed_branches.append(attr[:-8])
setattr(index_obj, attr, value)
if changed_branches:
# For the django simple history, indicate what was changed. Unfortunately at this point we only really know
# which branch(es) were changed, not anything more useful than that.
index_obj._change_reason = f'Updated {" and ".join(changed_branches)} branch' # pylint: disable=protected-access
# TEMP: Also write to MongoDB, so we can switch back to using it if this new MySQL version doesn't work well:
mongo_updated = super().update_course_index(
course_index,
from_index,
course_context,
last_update_already_set=True)
if mongo_updated:
# Save the course index entry and create a historical record:
index_obj.save()