def test_version_only_lib_key(self):
version_only_lib_key = LibraryLocator(
version_guid=ObjectId('519665f6223ebd6980884f2b'))
self.assertEqual(version_only_lib_key.org, None)
self.assertEqual(version_only_lib_key.library, None) # pylint: disable=no-member
with self.assertRaises(InvalidKeyError):
version_only_lib_key.for_branch("test")
def test_lib_key_constructor_bad_ids(self):
with self.assertRaises(InvalidKeyError):
LibraryLocator(org="!@#{$%^&*}", library="lib1")
with self.assertRaises(InvalidKeyError):
LibraryLocator(org="TestX", library="lib+1")
with self.assertRaises(InvalidKeyError):
LibraryLocator(org="TestX", library="lib1", branch="!+!")
def test_constructor_using_course(self):
org = 'TestX'
code = 'test-problem-bank'
lib_key = LibraryLocator(org=org, library=code)
with self.assertDeprecationWarning():
lib_key2 = LibraryLocator(org=org, course=code)
self.assertEqual(lib_key, lib_key2)
self.assertEqual(lib_key2.library, code) # pylint: disable=no-member
def test_for_branch(self):
lib_key = LibraryLocator(org='TestX', library='test', branch='initial')
branch2 = "br2"
branch2_key = lib_key.for_branch(branch2)
self.assertEqual(branch2_key.branch, branch2) # pylint: disable=no-member
normal_branch = lib_key.for_branch(None)
self.assertEqual(normal_branch.branch, None) # pylint: disable=no-member
def test_for_branch(self):
lib_key = LibraryLocator(org='TestX', library='test', branch='initial')
branch2 = "br2"
branch2_key = lib_key.for_branch(branch2)
self.assertEqual(branch2_key.branch, branch2) # pylint: disable=no-member
normal_branch = lib_key.for_branch(None)
self.assertEqual(normal_branch.branch, None) # pylint: disable=no-member
def test_replace(self):
# pylint: disable=no-member
org1, lib1, block_type1, block_id1 = "org1", "lib1", "type1", "id1"
lib_key1 = LibraryLocator(org=org1, library=lib1)
usage1 = LibraryUsageLocator(library_key=lib_key1,
block_type=block_type1,
block_id=block_id1)
self.assertEqual(usage1.org, org1)
self.assertEqual(usage1.library_key, lib_key1)
org2, lib2 = "org2", "lib2"
lib_key2 = LibraryLocator(org=org2, library=lib2)
usage2 = usage1.replace(library_key=lib_key2)
self.assertEqual(usage2.library_key, lib_key2)
self.assertEqual(usage2.course_key, lib_key2)
self.assertEqual(usage2.block_type, block_type1) # Unchanged
self.assertEqual(usage2.block_id, block_id1) # Unchanged
block_id3 = "id3"
lib3 = "lib3"
usage3 = usage1.replace(block_id=block_id3, library=lib3)
self.assertEqual(usage3.library_key.org, org1)
self.assertEqual(usage3.library_key.library, lib3)
self.assertEqual(usage2.block_type, block_type1) # Unchanged
self.assertEqual(usage3.block_id, block_id3)
lib_key4 = LibraryLocator(org="org4", library="lib4")
usage4 = usage1.replace(course_key=lib_key4)
self.assertEqual(usage4.library_key, lib_key4)
self.assertEqual(usage4.course_key, lib_key4)
self.assertEqual(usage4.block_type, block_type1) # Unchanged
self.assertEqual(usage4.block_id, block_id1) # Unchanged
usage5a = usage1.replace(version='aaaaaaaaaaaaaaaaaaaaaaaa')
usage5b = usage1.replace(
version_guid=ObjectId('bbbbbbbbbbbbbbbbbbbbbbbb'))
usage5c = usage1.for_version(ObjectId('cccccccccccccccccccccccc'))
self.assertEqual(usage5a.library_key.version_guid,
ObjectId('aaaaaaaaaaaaaaaaaaaaaaaa'))
self.assertEqual(usage5b.course_key.version_guid,
ObjectId('bbbbbbbbbbbbbbbbbbbbbbbb'))
self.assertEqual(usage5c.version_guid,
ObjectId('cccccccccccccccccccccccc'))
self.assertEqual(usage5a.block_type, block_type1) # Unchanged
self.assertEqual(usage5a.block_id, block_id1) # Unchanged
self.assertEqual(usage5b.block_type, block_type1) # Unchanged
self.assertEqual(usage5b.block_id, block_id1) # Unchanged
self.assertEqual(usage5c.block_type, block_type1) # Unchanged
self.assertEqual(usage5c.block_id, block_id1) # Unchanged
usage6 = usage5a.version_agnostic()
self.assertEqual(usage6, usage1)
usage7 = usage1.for_branch('tribble')
self.assertEqual(usage7.branch, 'tribble')
self.assertEqual(usage7.library_key.branch, 'tribble')
def test_lib_key_no_deprecated_support(self):
lib_key = CourseKey.from_string('library-v1:TestX+lib1')
with self.assertRaises(AttributeError):
lib_key.to_deprecated_string()
with self.assertRaises(NotImplementedError):
lib_key._to_deprecated_string() # pylint: disable=protected-access
with self.assertRaises(NotImplementedError):
LibraryLocator._from_deprecated_string('test/test/test') # pylint: disable=protected-access
with self.assertRaises(InvalidKeyError):
LibraryLocator(org='org', library='code', deprecated=True)
def test_lib_key_no_deprecated_support(self):
lib_key = CourseKey.from_string('library-v1:TestX+lib1')
with self.assertRaises(AttributeError):
lib_key.to_deprecated_string()
with self.assertRaises(NotImplementedError):
lib_key._to_deprecated_string() # pylint: disable=protected-access
with self.assertRaises(NotImplementedError):
LibraryLocator._from_deprecated_string('test/test/test') # pylint: disable=protected-access
with self.assertRaises(InvalidKeyError):
LibraryLocator(org='org', library='code', deprecated=True)
def test_from_json_studio_editor_style(self):
"""
Test that LibraryList can parse raw libraries list as passed by studio editor
"""
lib_list = LibraryList()
lib1_key, lib1_version = u'library-v1:Org1+Lib1', '5436ffec56c02c13806a4c1b'
lib2_key, lib2_version = u'library-v1:Org2+Lib2', '112dbaf312c0daa019ce9992'
raw = [lib1_key + ',' + lib1_version, lib2_key + ',' + lib2_version]
parsed = lib_list.from_json(raw)
self.assertEqual(len(parsed), 2)
self.assertEquals(parsed[0].library_id, LibraryLocator.from_string(lib1_key))
self.assertEquals(parsed[0].version, ObjectId(lib1_version))
self.assertEquals(parsed[1].library_id, LibraryLocator.from_string(lib2_key))
self.assertEquals(parsed[1].version, ObjectId(lib2_version))
def test_from_json_studio_editor_style(self):
"""
Test that LibraryList can parse raw libraries list as passed by studio editor
"""
lib_list = LibraryList()
lib1_key, lib1_version = u'library-v1:Org1+Lib1', '5436ffec56c02c13806a4c1b'
lib2_key, lib2_version = u'library-v1:Org2+Lib2', '112dbaf312c0daa019ce9992'
raw = [lib1_key + ',' + lib1_version, lib2_key + ',' + lib2_version]
parsed = lib_list.from_json(raw)
self.assertEqual(len(parsed), 2)
self.assertEquals(parsed[0].library_id, LibraryLocator.from_string(lib1_key))
self.assertEquals(parsed[0].version, ObjectId(lib1_version))
self.assertEquals(parsed[1].library_id, LibraryLocator.from_string(lib2_key))
self.assertEquals(parsed[1].version, ObjectId(lib2_version))
def test_lib_key_constructor_version_guid(self, version_id):
version_id_str = str(version_id)
version_id_obj = ObjectId(version_id)
lib_key = LibraryLocator(version_guid=version_id)
self.assertEqual(lib_key.version_guid, version_id_obj) # pylint: disable=no-member
self.assertEqual(lib_key.org, None)
self.assertEqual(lib_key.library, None) # pylint: disable=no-member
self.assertEqual(str(lib_key.version_guid), version_id_str) # pylint: disable=no-member
# Allow access to _to_string
# pylint: disable=protected-access
expected_str = u'@'.join((lib_key.VERSION_PREFIX, version_id_str))
self.assertEqual(lib_key._to_string(), expected_str)
self.assertEqual(str(lib_key), u'library-v1:' + expected_str)
self.assertEqual(lib_key.html_id(), u'library-v1:' + expected_str)
def test_lib_key_constructor_version_guid(self, version_id):
version_id_str = str(version_id)
version_id_obj = ObjectId(version_id)
lib_key = LibraryLocator(version_guid=version_id)
self.assertEqual(lib_key.version_guid, version_id_obj) # pylint: disable=no-member
self.assertEqual(lib_key.org, None)
self.assertEqual(lib_key.library, None) # pylint: disable=no-member
self.assertEqual(str(lib_key.version_guid), version_id_str) # pylint: disable=no-member
# Allow access to _to_string
# pylint: disable=protected-access
expected_str = '@'.join((lib_key.VERSION_PREFIX, version_id_str))
self.assertEqual(lib_key._to_string(), expected_str)
self.assertEqual(str(lib_key), 'library-v1:' + expected_str)
self.assertEqual(lib_key.html_id(), 'library-v1:' + expected_str)
def test_no_libraries(self):
"""
Verify that only Course IDs are returned, not anything else like libraries.
"""
# Make this user a course staff user for a course, AND a library.
course_staff_user = self.create_user(username='******',
is_staff=False)
add_users(self.global_admin, CourseStaffRole(self.course.id),
course_staff_user)
add_users(
self.global_admin,
CourseStaffRole(
LibraryLocator.from_string(
'library-v1:library_org+library_name')),
course_staff_user,
)
# Requesting the courses should return *only* courses and not libraries.
self.setup_user(self.staff_user)
filtered_response = self.verify_response(
params={
'username': course_staff_user.username,
'role': 'staff'
})
self.assertEqual(len(filtered_response.data['results']), 1)
self.assertTrue(filtered_response.data['results'][0].startswith(
self.course.org))
def test_sandbox_inclusion(self, context_key):
"""
Test to make sure that a match works across course runs
"""
self.validate_can_execute_unsafe_code(context_key, True)
self.validate_can_execute_unsafe_code(
LibraryLocator('edX', 'test_bank'), False)
def _all_branches(self):
"""Retrieve list of all ActiveVersionBranch objects in the database."""
branches = []
LOG.info("Fetching all Active Version Branches...")
for av_doc in self._active_versions.find():
for branch, obj_id in av_doc['versions'].items():
structure_id = str(obj_id)
if branch == 'library':
key = LibraryLocator(av_doc['org'], av_doc['course'])
else:
key = CourseLocator(av_doc['org'], av_doc['course'],
av_doc['run'])
branches.append(
ActiveVersionBranch(
str(av_doc['_id']),
branch,
structure_id,
key,
av_doc['edited_on'],
))
LOG.info("Fetched %s Active Version Branches", len(branches))
return sorted(branches)
def test_sandbox_inclusion(self):
"""
Test to make sure that a match works across course runs
"""
self.assertTrue(can_execute_unsafe_code(CourseKey.from_string('edX/full/2012_Fall')))
self.assertTrue(can_execute_unsafe_code(CourseKey.from_string('edX/full/2013_Spring')))
self.assertFalse(can_execute_unsafe_code(LibraryLocator('edX', 'test_bank')))
def test_courselikes_with_unsafe_code_default(self):
"""
Test that the default setting for COURSES_WITH_UNSAFE_CODE is an empty setting, e.g. we don't use @override_settings in these tests # lint-amnesty, pylint: disable=line-too-long
"""
self.assertFalse(can_execute_unsafe_code(CourseLocator('edX', 'full', '2012_Fall')))
self.assertFalse(can_execute_unsafe_code(CourseLocator('edX', 'full', '2013_Spring')))
self.assertFalse(can_execute_unsafe_code(LibraryLocator('edX', 'test_bank')))
def get_key(self):
"""
Get the library locator for the current library key.
"""
return LibraryLocator(
self.courselike_key.org, self.courselike_key.library
)
def test_with_library_locator(self, mock_olxcleaner_validate):
"""
Tests that olx is validation is skipped with library locator.
"""
library_key = LibraryLocator(org='TestOrg', library='TestProbs')
self.assertTrue(validate_course_olx(library_key, self.toy_course_path, self.status))
self.assertFalse(mock_olxcleaner_validate.called)
def test_structures_graph(self):
"""Test pulling a full graph out."""
graph = self.backend.structures_graph(0, 100)
self.assertEqual(graph.branches, [
ActiveVersionBranch(
id=str_id(100),
branch='draft-branch',
structure_id=str_id(4),
key=CourseLocator('edx', 'split_course', '2017'),
edited_on=datetime(2012, 5, 2),
),
ActiveVersionBranch(
id=str_id(100),
branch='published-branch',
structure_id=str_id(11),
key=CourseLocator('edx', 'split_course', '2017'),
edited_on=datetime(2012, 5, 2),
),
ActiveVersionBranch(
id=str_id(101),
branch='library',
structure_id=str_id(20),
key=LibraryLocator('edx', 'split_library'),
edited_on=datetime(2012, 5, 3),
),
])
self.assertEqual(list(graph.structures.keys()),
[str_id(i) for i in [1, 2, 3, 4, 10, 11, 20]])
def test_is_course(self):
course_key = CourseLocator("SchoolX", "course1", "2020")
self.assertEqual(course_key.is_course, True)
lib_key = LibraryLocator("SchoolX", "lib1")
self.assertEqual(lib_key.is_course, False)
lib2_key = LibraryLocatorV2("SchoolX", "lib-slug")
self.assertEqual(lib2_key.is_course, False)
def create_library(self, org, library, user_id, fields, **kwargs):
"""
Creates and returns a new library.
Args:
org (str): the organization that owns the course
library (str): the code/number/name of the library
user_id: id of the user creating the course
fields (dict): Fields to set on the course at initialization - e.g. display_name
kwargs: Any optional arguments understood by a subset of modulestores to customize instantiation
Returns: a LibraryRoot
"""
# first make sure an existing course/lib doesn't already exist in the mapping
lib_key = LibraryLocator(org=org, library=library)
if lib_key in self.mappings:
raise DuplicateCourseError(lib_key, lib_key)
# create the library
store = self._verify_modulestore_support(None, 'create_library')
library = store.create_library(org, library, user_id, fields, **kwargs)
# add new library to the mapping
self.mappings[lib_key] = store
return library
def fields_from_v1_schema(values):
""" Convert the MongoDB-style dict shape to a dict of fields that match this model """
if values[
"run"] == LibraryLocator.RUN and ModuleStoreEnum.BranchName.library in values[
"versions"]:
# This is a content library:
locator = LibraryLocator(org=values["org"],
library=values["course"])
else:
# This is a course:
locator = CourseLocator(org=values["org"],
course=values["course"],
run=values["run"])
result = {
"course_id": locator,
"org": values["org"],
"edited_by_id": values["edited_by"],
"edited_on": values["edited_on"],
"base_store": SplitModulestoreCourseIndex.BASE_STORE_MONGO,
}
if "_id" in values:
result["objectid"] = str(
values["_id"]) # Convert ObjectId to its hex representation
if "last_update" in values:
result["last_update"] = values["last_update"]
if "search_targets" in values and "wiki_slug" in values[
"search_targets"]:
result["wiki_slug"] = values["search_targets"]["wiki_slug"]
for branch in ("draft", "published", "library"):
version = values["versions"].get(
getattr(ModuleStoreEnum.BranchName, branch))
if version:
result[f"{branch}_version"] = str(
version) # Convert version from ObjectId to hex string
return result
def test_sandbox_exclusion(self):
"""
Test to make sure that a non-match returns false
"""
assert not can_execute_unsafe_code(
CourseLocator('edX', 'notful', 'empty'))
assert not can_execute_unsafe_code(LibraryLocator('edY', 'test_bank'))
def test_no_deprecated_support(self):
lib_key = LibraryLocator(org="TestX", library="problem-bank-15")
with self.assertRaises(InvalidKeyError):
LibraryUsageLocator(library_key=lib_key,
block_type="html",
block_id="html1",
deprecated=True)
def get_dest_id(self, courselike_key):
"""
Get the LibraryLocator that will be used in the target modulestore.
"""
if self.target_id is not None:
dest_id = self.target_id
else:
dest_id = LibraryLocator(self.target_id.org, self.target_id.library)
existing_lib = self.store.get_library(dest_id, ignore_case=True)
runtime = None
if existing_lib:
dest_id = existing_lib.location.library_key
runtime = existing_lib.runtime
if self.create_if_not_present and not existing_lib:
try:
library = self.store.create_library(
org=self.target_id.org,
library=self.target_id.library,
user_id=self.user_id,
)
runtime = library.runtime
except DuplicateCourseError:
log.debug(
"Skipping import of Library with id %s, "
"since it collides with an existing one", dest_id
)
raise
return dest_id, runtime
def test_library_import_branch_settings_again(self, branch_setting):
# Construct the contentstore for storing the import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the import (using the previously created contentstore)
with SPLIT_MODULESTORE_SETUP.build(
contentstore=source_content) as source_store:
# Use the test branch setting.
with source_store.branch_setting(branch_setting):
source_library_key = LibraryLocator(org='TestOrg',
library='TestProbs')
extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR))
# the extract_dir needs to be passed as a relative dir to
# import_library_from_xml
extract_dir_relative = path.relpath(
extract_dir, settings.DATA_DIR)
try:
with tarfile.open(
path(TEST_DATA_DIR) / 'imports' /
'library.HhJfPD.tar.gz') as tar:
safetar_extractall(tar, extract_dir)
import_library_from_xml(
source_store,
self.user.id,
settings.GITHUB_REPO_ROOT,
[extract_dir_relative / 'library'],
static_content_store=source_content,
target_id=source_library_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
finally:
shutil.rmtree(extract_dir)
def import_single_library(filename):
print >> sys.stderr, 'IMPORTING library:', filename
no_extension = path.basename(filename).replace('.tar.gz', '')
library_xml_dir = path.join(XML_EXTRACT_DIR, no_extension)
mkdir(library_xml_dir)
subprocess.call(['tar', '-xzf', filename, '-C', library_xml_dir])
with open(path.join(library_xml_dir, 'library/library.xml')) as lib_xml_file:
lib_xml = BeautifulSoup(lib_xml_file.read())
lib_element = lib_xml.find('library')
target_id = LibraryLocator(org=str(lib_element['org']), library=str(lib_element['library']))
print >> sys.stderr, 'IMPORTING library:', target_id
import_library_from_xml(
store=MOD_STORE,
user_id=ModuleStoreEnum.UserID.mgmt_command,
data_dir=DATA_DIR,
source_dirs=[path.join(library_xml_dir, 'library')], # Open edX needs `library` dir
load_error_modules=False,
static_content_store=contentstore(),
verbose=True,
do_import_static=True,
target_id=target_id,
create_if_not_present=True,
)
def test_create_library(self):
"""
From the home page:
Click "New Library"
Fill out the form
Submit the form
We should be redirected to the edit view for the library
Return to the home page
The newly created library should now appear in the list of libraries
"""
name = "New Library Name"
org = "TestOrgX"
number = "TESTLIB"
self.auth_page.visit()
self.dashboard_page.visit()
self.assertFalse(self.dashboard_page.has_library(name=name, org=org, number=number))
self.assertTrue(self.dashboard_page.has_new_library_button())
self.dashboard_page.click_new_library()
self.assertTrue(self.dashboard_page.is_new_library_form_visible())
self.dashboard_page.fill_new_library_form(name, org, number)
self.assertTrue(self.dashboard_page.is_new_library_form_valid())
self.dashboard_page.submit_new_library_form()
# The next page is the library edit view; make sure it loads:
lib_page = LibraryEditPage(self.browser, LibraryLocator(org, number))
lib_page.wait_for_page()
# Then go back to the home page and make sure the new library is listed there:
self.dashboard_page.visit()
self.assertTrue(self.dashboard_page.has_library(name=name, org=org, number=number))
def test_has_course_with_library(self):
"""
Test that has_course() returns False when called with a LibraryLocator.
This is required because MixedModuleStore will use has_course() to check
where a given library are stored.
"""
lib_key = LibraryLocator("TestOrg", "TestLib")
result = self.draft_store.has_course(lib_key)
assert_false(result)
def test_sandbox_exclusion(self):
"""
Test to make sure that a non-match returns false
"""
self.assertFalse(
can_execute_unsafe_code(
SlashSeparatedCourseKey('edX', 'notful', 'empty')))
self.assertFalse(
can_execute_unsafe_code(LibraryLocator('edY', 'test_bank')))
def _assert_cannot_create_library(self, org="org", library="libfail", expected_code=403):
""" Ensure the current user is not able to create a library. """
self.assertGreaterEqual(expected_code, 300)
response = self.client.ajax_post(
LIBRARY_REST_URL,
{'org': org, 'library': library, 'display_name': "Irrelevant"}
)
self.assertEqual(response.status_code, expected_code)
key = LibraryLocator(org=org, library=library)
self.assertEqual(modulestore().get_library(key), None)
def test_lib_key_branch_support(self):
org = 'TestX'
code = 'test-branch-support'
branch = 'future-purposes-perhaps'
lib_key = LibraryLocator(org=org, library=code, branch=branch)
self.assertEqual(lib_key.org, org)
self.assertEqual(lib_key.library, code) # pylint: disable=no-member
self.assertEqual(lib_key.branch, branch) # pylint: disable=no-member
lib_key2 = CourseKey.from_string(str(lib_key))
self.assertEqual(lib_key, lib_key2)
self.assertEqual(lib_key.branch, branch) # pylint: disable=no-member
def test_lib_key_constructor(self):
org = 'TestX'
code = 'test-problem-bank'
lib_key = LibraryLocator(org=org, library=code)
self.assertEqual(lib_key.org, org)
self.assertEqual(lib_key.library, code) # pylint: disable=no-member
with self.assertDeprecationWarning():
self.assertEqual(lib_key.course, code)
with self.assertDeprecationWarning():
self.assertEqual(lib_key.run, 'library')
self.assertEqual(lib_key.branch, None) # pylint: disable=no-member
def __new__(cls, library_id, version=None):
# pylint: disable=super-on-old-class
if not isinstance(library_id, LibraryLocator):
library_id = LibraryLocator.from_string(library_id)
if library_id.version_guid:
assert (version is None) or (version == library_id.version_guid)
if not version:
version = library_id.version_guid
library_id = library_id.for_version(None)
if version and not isinstance(version, ObjectId):
try:
version = ObjectId(version)
except InvalidId:
raise ValueError(version)
return super(LibraryVersionReference, cls).__new__(cls, library_id, version)
def _get_library(self, library_key):
"""
Given a library key like "library-v1:ProblemX+PR0B", return the
'library' XBlock with meta-information about the library.
Returns None on error.
"""
if not isinstance(library_key, LibraryLocator):
library_key = LibraryLocator.from_string(library_key)
assert library_key.version_guid is None
try:
return self.store.get_library(library_key, remove_version=False, remove_branch=False)
except ItemNotFoundError:
return None
def source_library_key(self):
"""
Convenience method to get the library ID as a LibraryLocator and not just a string
"""
return LibraryLocator.from_string(self.source_library_id)
def test_changing_course(self):
lib_key = LibraryLocator(org="TestX", library="test")
with self.assertRaises(AttributeError):
lib_key.course = "PHYS"
with self.assertRaises(KeyError):
lib_key.replace(course="PHYS")
def test_lib_key_from_invalid_string(self, lib_id_str):
with self.assertRaises(InvalidKeyError):
LibraryLocator.from_string(lib_id_str)
def test_lib_key_with_trailing_whitespace(self, lib_id_fmt, whitespace):
with self.assertRaises(InvalidKeyError):
LibraryLocator.from_string(lib_id_fmt.format(whitespace))
def test_version_only_lib_key(self):
version_only_lib_key = LibraryLocator(version_guid=ObjectId('519665f6223ebd6980884f2b'))
self.assertEqual(version_only_lib_key.org, None)
self.assertEqual(version_only_lib_key.library, None) # pylint: disable=no-member
with self.assertRaises(InvalidKeyError):
version_only_lib_key.for_branch("test")
