def test_branch_setting(self):
"""
Test the branch setting context manager
"""
store = XMLModuleStore(DATA_DIR, source_dirs=['toy'])
course = store.get_courses()[0]
# XML store allows published_only branch setting
with store.branch_setting(ModuleStoreEnum.Branch.published_only, course.id):
store.get_item(course.location)
# XML store does NOT allow draft_preferred branch setting
with self.assertRaises(ValueError):
with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, course.id):
# verify that the above context manager raises a ValueError
pass # pragma: no cover
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir", source_dirs=[],
load_error_modules=load_error_modules)
course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run')
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
)
def test_graphicslidertool_import(self):
'''
Check to see if definition_from_xml in gst_module.py
works properly. Pulls data from the graphic_slider_tool directory
in the test data directory.
'''
modulestore = XMLModuleStore(DATA_DIR,
course_dirs=['graphic_slider_tool'])
sa_id = SlashSeparatedCourseKey("edX", "gst_test", "2012_Fall")
location = sa_id.make_usage_key("graphical_slider_tool", "sample_gst")
gst_sample = modulestore.get_item(location)
render_string_from_sample_gst_xml = """
<slider var="a" style="width:400px;float:left;"/>\
<plot style="margin-top:15px;margin-bottom:15px;"/>""".strip()
self.assertIn(render_string_from_sample_gst_xml, gst_sample.data)
def test_definition_loading(self):
"""When two courses share the same org and course name and
both have a module with the same url_name, the definitions shouldn't clash.
TODO (vshnayder): once we have a CMS, this shouldn't
happen--locations should uniquely name definitions. But in
our imperfect XML world, it can (and likely will) happen."""
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'two_toys'])
location = Location("edX", "toy", "2012_Fall", "video", "Welcome", None)
toy_video = modulestore.get_item(location)
location_two = Location("edX", "toy", "TT_2012_Fall", "video", "Welcome", None)
two_toy_video = modulestore.get_item(location_two)
self.assertEqual(toy_video.youtube_id_1_0, "p2Q6BrNhdh8")
self.assertEqual(two_toy_video.youtube_id_1_0, "p2Q6BrNhdh9")
def test_static_tabs_import(self):
"""Make sure that the static tabs are imported correctly"""
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])
location_tab_syllabus = BlockUsageLocator(CourseLocator("edX", "toy", "2012_Fall", deprecated=True),
"static_tab", "syllabus", deprecated=True)
toy_tab_syllabus = modulestore.get_item(location_tab_syllabus)
assert toy_tab_syllabus.display_name == 'Syllabus'
assert toy_tab_syllabus.course_staff_only is False
location_tab_resources = BlockUsageLocator(CourseLocator("edX", "toy", "2012_Fall", deprecated=True),
"static_tab", "resources", deprecated=True)
toy_tab_resources = modulestore.get_item(location_tab_resources)
assert toy_tab_resources.display_name == 'Resources'
assert toy_tab_resources.course_staff_only is True
def test_static_tabs_import(self):
"""Make sure that the static tabs are imported correctly"""
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])
location_tab_syllabus = Location("edX", "toy", "2012_Fall",
"static_tab", "syllabus", None)
toy_tab_syllabus = modulestore.get_item(location_tab_syllabus)
self.assertEqual(toy_tab_syllabus.display_name, 'Syllabus')
self.assertEqual(toy_tab_syllabus.course_staff_only, False)
location_tab_resources = Location("edX", "toy", "2012_Fall",
"static_tab", "resources", None)
toy_tab_resources = modulestore.get_item(location_tab_resources)
self.assertEqual(toy_tab_resources.display_name, 'Resources')
self.assertEqual(toy_tab_resources.course_staff_only, True)
def build_with_contentstore(self, contentstore=None, course_ids=None, **kwargs):
"""
A contextmanager that returns an isolated xml modulestore
Args:
contentstore: The contentstore that this modulestore should use to store
all of its assets.
"""
modulestore = XMLModuleStore(
DATA_DIR,
course_ids=course_ids,
default_class='xmodule.hidden_module.HiddenDescriptor',
xblock_mixins=XBLOCK_MIXINS,
)
yield modulestore
def test_word_cloud_import(self):
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['word_cloud'])
course = modulestore.get_courses()[0]
chapters = course.get_children()
ch1 = chapters[0]
sections = ch1.get_children()
self.assertEqual(len(sections), 1)
location = course.location
location = Location(location.tag, location.org, location.course,
'word_cloud', 'cloud1')
module = modulestore.get_instance(course.id, location)
self.assertEqual(len(module.get_children()), 0)
self.assertEqual(module.num_inputs, 5)
self.assertEqual(module.num_top_words, 250)
def test_unicode_chars_in_xml_content(self):
# edX/full/6.002_Spring_2012 has non-ASCII chars, and during
# uniquification of names, would raise a UnicodeError. It no longer does.
# Ensure that there really is a non-ASCII character in the course.
with open(os.path.join(DATA_DIR, "toy/sequential/vertical_sequential.xml")) as xmlf:
xml = xmlf.read()
with self.assertRaises(UnicodeDecodeError):
xml.decode('ascii')
# Load the course, but don't make error modules. This will succeed,
# but will record the errors.
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'], load_error_modules=False)
# Look up the errors during load. There should be none.
errors = modulestore.get_course_errors(SlashSeparatedCourseKey("edX", "toy", "2012_Fall"))
assert errors == []
def test_graphicslidertool_import(self):
'''
Check to see if definition_from_xml in gst_module.py
works properly. Pulls data from the graphic_slider_tool directory
in the test data directory.
'''
modulestore = XMLModuleStore(DATA_DIR,
course_dirs=['graphic_slider_tool'])
sa_id = "edX/gst_test/2012_Fall"
location = Location(
["i4x", "edX", "gst_test", "graphical_slider_tool", "sample_gst"])
gst_sample = modulestore.get_instance(sa_id, location)
render_string_from_sample_gst_xml = """
<slider var="a" style="width:400px;float:left;"/>\
<plot style="margin-top:15px;margin-bottom:15px;"/>""".strip()
self.assertEqual(gst_sample.render, render_string_from_sample_gst_xml)
def __init__(self, load_error_modules, course_id=None):
xmlstore = XMLModuleStore("data_dir", source_dirs=[],
load_error_modules=load_error_modules)
if course_id is None:
course_id = CourseKey.from_string('/'.join([ORG, COURSE, 'test_run']))
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
)
def __init__(self, load_error_modules, course_id=None):
xmlstore = XMLModuleStore("data_dir", source_dirs=[],
load_error_modules=load_error_modules)
if course_id is None:
course_id = CourseKey.from_string('/'.join([ORG, COURSE, 'test_run']))
course_dir = "test_dir"
error_tracker = Mock()
super().__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
)
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir", course_dirs=[], load_error_modules=load_error_modules)
course_id = "/".join([ORG, COURSE, 'test_run'])
course_dir = "test_dir"
policy = {}
error_tracker = Mock()
parent_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore,
course_id,
course_dir,
policy,
error_tracker,
parent_tracker,
load_error_modules=load_error_modules,
)
def __init__(self, load_error_modules, library=False):
if library:
xmlstore = LibraryXMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules)
else:
xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules)
course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run')
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
mixins=(InheritanceMixin, XModuleMixin),
field_data=KvsFieldData(DictKeyValueStore()),
)
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir", course_dirs=[],
load_error_modules=load_error_modules)
course_id = "/".join([ORG, COURSE, 'test_run'])
course_dir = "test_dir"
error_tracker = Mock()
parent_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
parent_tracker=parent_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
id_reader=LocationReader(),
)
def __init__(self, load_error_modules, library=False):
if library:
xmlstore = LibraryXMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules)
else:
xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules)
course_id = CourseKey.from_string('/'.join([ORG, COURSE, RUN]))
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
mixins=(InheritanceMixin, XModuleMixin),
field_data=KvsFieldData(DictKeyValueStore()),
)
def test_definition_loading(self):
"""When two courses share the same org and course name and
both have a module with the same url_name, the definitions shouldn't clash.
TODO (vshnayder): once we have a CMS, this shouldn't
happen--locations should uniquely name definitions. But in
our imperfect XML world, it can (and likely will) happen."""
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'two_toys'])
toy_id = "edX/toy/2012_Fall"
two_toy_id = "edX/toy/TT_2012_Fall"
location = Location(["i4x", "edX", "toy", "video", "Welcome"])
toy_video = modulestore.get_instance(toy_id, location)
two_toy_video = modulestore.get_instance(two_toy_id, location)
self.assertEqual(
etree.fromstring(toy_video.data).get('youtube'), "1.0:p2Q6BrNhdh8")
self.assertEqual(
etree.fromstring(two_toy_video.data).get('youtube'),
"1.0:p2Q6BrNhdh9")
def test_url_name_mangling(self):
"""
Make sure that url_names are only mangled once.
"""
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])
toy_id = CourseKey.from_string('edX/toy/2012_Fall')
course = modulestore.get_course(toy_id)
chapters = course.get_children()
ch1 = chapters[0]
sections = ch1.get_children()
self.assertEqual(len(sections), 4)
for i in (2, 3):
video = sections[i]
# Name should be 'video_{hash}'
print("video {0} url_name: {1}".format(i, video.url_name))
self.assertEqual(len(video.url_name), len('video_') + 12)
def test_poll_and_conditional_import(self):
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['conditional_and_poll'])
course = modulestore.get_courses()[0]
chapters = course.get_children()
ch1 = chapters[0]
sections = ch1.get_children()
assert len(sections) == 1
conditional_location = course.id.make_usage_key('conditional', 'condone')
module = modulestore.get_item(conditional_location)
assert len(module.children) == 1
poll_location = course.id.make_usage_key('poll_question', 'first_poll')
module = modulestore.get_item(poll_location)
assert len(module.get_children()) == 0
assert module.voted is False
assert module.poll_answer == ''
assert module.poll_answers == {}
assert module.answers ==\
[{'text': u'Yes', 'id': 'Yes'}, {'text': u'No', 'id': 'No'}, {'text': u"Don't know", 'id': 'Dont_know'}]
def test_dag_course(self, mock_logging):
"""
Test a course whose structure is not a tree.
"""
store = XMLModuleStore(DATA_DIR, course_dirs=['xml_dag'])
course_key = store.get_courses()[0].id
mock_logging.warning.assert_called_with(
"%s has more than one definition", course_key.make_usage_key('discussion', 'duplicate_def')
)
shared_item_loc = course_key.make_usage_key('html', 'toyhtml')
shared_item = store.get_item(shared_item_loc)
parent = shared_item.get_parent()
self.assertIsNotNone(parent, "get_parent failed to return a value")
parent_loc = course_key.make_usage_key('vertical', 'vertical_test')
self.assertEqual(parent.location, parent_loc)
self.assertIn(shared_item, parent.get_children())
# ensure it's still a child of the other parent even tho it doesn't claim the other parent as its parent
other_parent_loc = course_key.make_usage_key('vertical', 'zeta')
other_parent = store.get_item(other_parent_loc)
# children rather than get_children b/c the instance returned by get_children != shared_item
self.assertIn(shared_item_loc, other_parent.children)
def test_unicode(self):
"""Check that courses with unicode characters in filenames and in
org/course/name import properly. Currently, this means: (a) Having
files with unicode names does not prevent import; (b) if files are not
loaded because of unicode filenames, there are appropriate
exceptions/errors to that effect."""
print("Starting import")
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['test_unicode'])
courses = modulestore.get_courses()
assert len(courses) == 1
course = courses[0]
print("course errors:")
# Expect to find an error/exception about characters in "®esources"
expect = "InvalidKeyError"
errors = modulestore.get_course_errors(course.id)
assert any(
((expect in msg) or (expect in err)) for (msg, err) in errors)
chapters = course.get_children()
assert len(chapters) == 4
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError(u'Must called with arguments: {}'.format(
self.args))
xml_module_store = XMLModuleStore(
data_dir=settings.DATA_DIR,
default_class='xmodule.hidden_module.HiddenDescriptor',
load_error_modules=True,
xblock_mixins=settings.XBLOCK_MIXINS,
xblock_select=settings.XBLOCK_SELECT_FUNCTION,
)
export_dir = path(args[0])
for course_id, course_modules in six.iteritems(
xml_module_store.modules):
course_path = course_id.replace('/', '_')
for location, descriptor in six.iteritems(course_modules):
location_path = text_type(location).replace('/', '_')
data = {}
for field_name, field in six.iteritems(descriptor.fields):
try:
data[field_name] = field.read_json(descriptor)
except Exception as exc: # pylint: disable=broad-except
data[field_name] = {
'$type':
str(type(exc)),
'$value':
descriptor._field_data.get(descriptor, field_name) # pylint: disable=protected-access
}
outdir = export_dir / course_path
outdir.makedirs_p()
with open(outdir / location_path + '.json', 'w') as outfile:
json.dump(data, outfile, sort_keys=True, indent=4)
print('', file=outfile)
def test_cohort_config(self):
"""
Check that cohort config parsing works right.
"""
modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy'])
toy_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
course = modulestore.get_course(toy_id)
# No config -> False
self.assertFalse(course.is_cohorted)
# empty config -> False
course.cohort_config = {}
self.assertFalse(course.is_cohorted)
# false config -> False
course.cohort_config = {'cohorted': False}
self.assertFalse(course.is_cohorted)
# and finally...
course.cohort_config = {'cohorted': True}
self.assertTrue(course.is_cohorted)
def test_cohort_config(self):
"""
Check that cohort config parsing works right.
"""
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'])
toy_id = "edX/toy/2012_Fall"
course = modulestore.get_course(toy_id)
# No config -> False
self.assertFalse(course.is_cohorted)
# empty config -> False
course.cohort_config = {}
self.assertFalse(course.is_cohorted)
# false config -> False
course.cohort_config = {'cohorted': False}
self.assertFalse(course.is_cohorted)
# and finally...
course.cohort_config = {'cohorted': True}
self.assertTrue(course.is_cohorted)
def test_get_courses_for_wiki(self):
"""
Test the get_courses_for_wiki method
"""
store = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'simple'])
for course in store.get_courses():
course_locations = store.get_courses_for_wiki(course.wiki_slug)
self.assertEqual(len(course_locations), 1)
self.assertIn(course.location, course_locations)
course_locations = store.get_courses_for_wiki('no_such_wiki')
self.assertEqual(len(course_locations), 0)
# now set toy course to share the wiki with simple course
toy_course = store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
toy_course.wiki_slug = 'simple'
course_locations = store.get_courses_for_wiki('toy')
self.assertEqual(len(course_locations), 0)
course_locations = store.get_courses_for_wiki('simple')
self.assertEqual(len(course_locations), 2)
for course_number in ['toy', 'simple']:
self.assertIn(Location('edX', course_number, '2012_Fall', 'course', '2012_Fall'), course_locations)
def test_get_courses_for_wiki(self):
"""
Test the get_courses_for_wiki method
"""
store = XMLModuleStore(DATA_DIR, source_dirs=['toy', 'simple'])
for course in store.get_courses():
course_locations = store.get_courses_for_wiki(course.wiki_slug)
self.assertEqual(len(course_locations), 1)
self.assertIn(course.location.course_key, course_locations)
course_locations = store.get_courses_for_wiki('no_such_wiki')
self.assertEqual(len(course_locations), 0)
# now set toy course to share the wiki with simple course
toy_course = store.get_course(CourseKey.from_string('edX/toy/2012_Fall'))
toy_course.wiki_slug = 'simple'
course_locations = store.get_courses_for_wiki('toy')
self.assertEqual(len(course_locations), 0)
course_locations = store.get_courses_for_wiki('simple')
self.assertEqual(len(course_locations), 2)
for course_number in ['toy', 'simple']:
self.assertIn(CourseKey.from_string('/'.join(['edX', course_number, '2012_Fall'])), course_locations)
def test_poll_and_conditional_import(self):
modulestore = XMLModuleStore(DATA_DIR,
course_dirs=['conditional_and_poll'])
course = modulestore.get_courses()[0]
chapters = course.get_children()
ch1 = chapters[0]
sections = ch1.get_children()
self.assertEqual(len(sections), 1)
location = course.location
conditional_location = Location(location.tag, location.org,
location.course, 'conditional',
'condone')
module = modulestore.get_instance(course.id, conditional_location)
self.assertEqual(len(module.children), 1)
poll_location = Location(location.tag, location.org, location.course,
'poll_question', 'first_poll')
module = modulestore.get_instance(course.id, poll_location)
self.assertEqual(len(module.get_children()), 0)
self.assertEqual(module.voted, False)
self.assertEqual(module.poll_answer, '')
self.assertEqual(module.poll_answers, {})
self.assertEqual(module.answers, [{
'text': u'Yes',
'id': 'Yes'
}, {
'text': u'No',
'id': 'No'
}, {
'text': u"Don't know",
'id': 'Dont_know'
}])
def test_unicode(self):
"""Check that courses with unicode characters in filenames and in
org/course/name import properly. Currently, this means: (a) Having
files with unicode names does not prevent import; (b) if files are not
loaded because of unicode filenames, there are appropriate
exceptions/errors to that effect."""
print("Starting import")
modulestore = XMLModuleStore(DATA_DIR, course_dirs=['test_unicode'])
courses = modulestore.get_courses()
self.assertEquals(len(courses), 1)
course = courses[0]
print("course errors:")
# Expect to find an error/exception about characters in "®esources"
expect = "Invalid characters"
errors = [(msg.encode("utf-8"), err.encode("utf-8"))
for msg, err in modulestore.get_course_errors(course.id)]
self.assertTrue(
any(expect in msg or expect in err for msg, err in errors))
chapters = course.get_children()
self.assertEqual(len(chapters), 4)
def test_export_roundtrip(self, course_dir, mock_get):
# Patch network calls to retrieve the textbook TOC
mock_get.return_value.text = dedent("""
<?xml version="1.0"?><table_of_contents>
<entry page="5" page_label="ii" name="Table of Contents"/>
</table_of_contents>
""").strip()
root_dir = path(self.temp_dir)
print "Copying test course to temp dir {0}".format(root_dir)
data_dir = path(DATA_DIR)
shutil.copytree(data_dir / course_dir, root_dir / course_dir)
print "Starting import"
initial_import = XMLModuleStore(root_dir,
source_dirs=[course_dir],
xblock_mixins=(XModuleMixin, ))
courses = initial_import.get_courses()
self.assertEquals(len(courses), 1)
initial_course = courses[0]
# export to the same directory--that way things like the custom_tags/ folder
# will still be there.
print "Starting export"
file_system = OSFS(root_dir)
initial_course.runtime.export_fs = file_system.makedir(course_dir,
recreate=True)
root = lxml.etree.Element('root')
initial_course.add_xml_to_node(root)
with initial_course.runtime.export_fs.open('course.xml',
'wb') as course_xml:
lxml.etree.ElementTree(root).write(course_xml, encoding='utf-8')
print "Starting second import"
second_import = XMLModuleStore(root_dir,
source_dirs=[course_dir],
xblock_mixins=(XModuleMixin, ))
courses2 = second_import.get_courses()
self.assertEquals(len(courses2), 1)
exported_course = courses2[0]
print "Checking course equality"
# HACK: filenames change when changing file formats
# during imports from old-style courses. Ignore them.
strip_filenames(initial_course)
strip_filenames(exported_course)
self.assertTrue(blocks_are_equivalent(initial_course, exported_course))
self.assertEquals(initial_course.id, exported_course.id)
course_id = initial_course.id
print "Checking key equality"
self.assertItemsEqual(initial_import.modules[course_id].keys(),
second_import.modules[course_id].keys())
print "Checking module equality"
for location in initial_import.modules[course_id].keys():
print("Checking", location)
self.assertTrue(
blocks_are_equivalent(
initial_import.modules[course_id][location],
second_import.modules[course_id][location]))
def setup_modulestore(self, name):
self.modulestore = XMLModuleStore(DATA_DIR, course_dirs=[name])
