def test_branch_setting(self): """ Test the branch setting context manager """ store = XMLModuleStore(DATA_DIR, source_dirs=['toy']) course = store.get_courses()[0] # XML store allows published_only branch setting with store.branch_setting(ModuleStoreEnum.Branch.published_only, course.id): store.get_item(course.location) # XML store does NOT allow draft_preferred branch setting with self.assertRaises(ValueError): with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, course.id): # verify that the above context manager raises a ValueError pass # pragma: no cover
def __init__(self, load_error_modules): xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules) course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run') course_dir = "test_dir" error_tracker = Mock() super(DummySystem, self).__init__( xmlstore=xmlstore, course_id=course_id, course_dir=course_dir, error_tracker=error_tracker, load_error_modules=load_error_modules, field_data=KvsFieldData(DictKeyValueStore()), )
def test_graphicslidertool_import(self): ''' Check to see if definition_from_xml in gst_module.py works properly. Pulls data from the graphic_slider_tool directory in the test data directory. ''' modulestore = XMLModuleStore(DATA_DIR, course_dirs=['graphic_slider_tool']) sa_id = SlashSeparatedCourseKey("edX", "gst_test", "2012_Fall") location = sa_id.make_usage_key("graphical_slider_tool", "sample_gst") gst_sample = modulestore.get_item(location) render_string_from_sample_gst_xml = """ <slider var="a" style="width:400px;float:left;"/>\ <plot style="margin-top:15px;margin-bottom:15px;"/>""".strip() self.assertIn(render_string_from_sample_gst_xml, gst_sample.data)
def test_definition_loading(self): """When two courses share the same org and course name and both have a module with the same url_name, the definitions shouldn't clash. TODO (vshnayder): once we have a CMS, this shouldn't happen--locations should uniquely name definitions. But in our imperfect XML world, it can (and likely will) happen.""" modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'two_toys']) location = Location("edX", "toy", "2012_Fall", "video", "Welcome", None) toy_video = modulestore.get_item(location) location_two = Location("edX", "toy", "TT_2012_Fall", "video", "Welcome", None) two_toy_video = modulestore.get_item(location_two) self.assertEqual(toy_video.youtube_id_1_0, "p2Q6BrNhdh8") self.assertEqual(two_toy_video.youtube_id_1_0, "p2Q6BrNhdh9")
def test_static_tabs_import(self): """Make sure that the static tabs are imported correctly""" modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy']) location_tab_syllabus = BlockUsageLocator(CourseLocator("edX", "toy", "2012_Fall", deprecated=True), "static_tab", "syllabus", deprecated=True) toy_tab_syllabus = modulestore.get_item(location_tab_syllabus) assert toy_tab_syllabus.display_name == 'Syllabus' assert toy_tab_syllabus.course_staff_only is False location_tab_resources = BlockUsageLocator(CourseLocator("edX", "toy", "2012_Fall", deprecated=True), "static_tab", "resources", deprecated=True) toy_tab_resources = modulestore.get_item(location_tab_resources) assert toy_tab_resources.display_name == 'Resources' assert toy_tab_resources.course_staff_only is True
def test_static_tabs_import(self): """Make sure that the static tabs are imported correctly""" modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy']) location_tab_syllabus = Location("edX", "toy", "2012_Fall", "static_tab", "syllabus", None) toy_tab_syllabus = modulestore.get_item(location_tab_syllabus) self.assertEqual(toy_tab_syllabus.display_name, 'Syllabus') self.assertEqual(toy_tab_syllabus.course_staff_only, False) location_tab_resources = Location("edX", "toy", "2012_Fall", "static_tab", "resources", None) toy_tab_resources = modulestore.get_item(location_tab_resources) self.assertEqual(toy_tab_resources.display_name, 'Resources') self.assertEqual(toy_tab_resources.course_staff_only, True)
def build_with_contentstore(self, contentstore=None, course_ids=None, **kwargs): """ A contextmanager that returns an isolated xml modulestore Args: contentstore: The contentstore that this modulestore should use to store all of its assets. """ modulestore = XMLModuleStore( DATA_DIR, course_ids=course_ids, default_class='xmodule.hidden_module.HiddenDescriptor', xblock_mixins=XBLOCK_MIXINS, ) yield modulestore
def test_word_cloud_import(self): modulestore = XMLModuleStore(DATA_DIR, course_dirs=['word_cloud']) course = modulestore.get_courses()[0] chapters = course.get_children() ch1 = chapters[0] sections = ch1.get_children() self.assertEqual(len(sections), 1) location = course.location location = Location(location.tag, location.org, location.course, 'word_cloud', 'cloud1') module = modulestore.get_instance(course.id, location) self.assertEqual(len(module.get_children()), 0) self.assertEqual(module.num_inputs, 5) self.assertEqual(module.num_top_words, 250)
def test_unicode_chars_in_xml_content(self): # edX/full/6.002_Spring_2012 has non-ASCII chars, and during # uniquification of names, would raise a UnicodeError. It no longer does. # Ensure that there really is a non-ASCII character in the course. with open(os.path.join(DATA_DIR, "toy/sequential/vertical_sequential.xml")) as xmlf: xml = xmlf.read() with self.assertRaises(UnicodeDecodeError): xml.decode('ascii') # Load the course, but don't make error modules. This will succeed, # but will record the errors. modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy'], load_error_modules=False) # Look up the errors during load. There should be none. errors = modulestore.get_course_errors(SlashSeparatedCourseKey("edX", "toy", "2012_Fall")) assert errors == []
def test_graphicslidertool_import(self): ''' Check to see if definition_from_xml in gst_module.py works properly. Pulls data from the graphic_slider_tool directory in the test data directory. ''' modulestore = XMLModuleStore(DATA_DIR, course_dirs=['graphic_slider_tool']) sa_id = "edX/gst_test/2012_Fall" location = Location( ["i4x", "edX", "gst_test", "graphical_slider_tool", "sample_gst"]) gst_sample = modulestore.get_instance(sa_id, location) render_string_from_sample_gst_xml = """ <slider var="a" style="width:400px;float:left;"/>\ <plot style="margin-top:15px;margin-bottom:15px;"/>""".strip() self.assertEqual(gst_sample.render, render_string_from_sample_gst_xml)
def __init__(self, load_error_modules, course_id=None): xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules) if course_id is None: course_id = CourseKey.from_string('/'.join([ORG, COURSE, 'test_run'])) course_dir = "test_dir" error_tracker = Mock() super(DummySystem, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments xmlstore=xmlstore, course_id=course_id, course_dir=course_dir, error_tracker=error_tracker, load_error_modules=load_error_modules, field_data=KvsFieldData(DictKeyValueStore()), )
def __init__(self, load_error_modules, course_id=None): xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules) if course_id is None: course_id = CourseKey.from_string('/'.join([ORG, COURSE, 'test_run'])) course_dir = "test_dir" error_tracker = Mock() super().__init__( xmlstore=xmlstore, course_id=course_id, course_dir=course_dir, error_tracker=error_tracker, load_error_modules=load_error_modules, field_data=KvsFieldData(DictKeyValueStore()), )
def __init__(self, load_error_modules): xmlstore = XMLModuleStore("data_dir", course_dirs=[], load_error_modules=load_error_modules) course_id = "/".join([ORG, COURSE, 'test_run']) course_dir = "test_dir" policy = {} error_tracker = Mock() parent_tracker = Mock() super(DummySystem, self).__init__( xmlstore, course_id, course_dir, policy, error_tracker, parent_tracker, load_error_modules=load_error_modules, )
def __init__(self, load_error_modules, library=False): if library: xmlstore = LibraryXMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules) else: xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules) course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run') course_dir = "test_dir" error_tracker = Mock() super(DummySystem, self).__init__( xmlstore=xmlstore, course_id=course_id, course_dir=course_dir, error_tracker=error_tracker, load_error_modules=load_error_modules, mixins=(InheritanceMixin, XModuleMixin), field_data=KvsFieldData(DictKeyValueStore()), )
def __init__(self, load_error_modules): xmlstore = XMLModuleStore("data_dir", course_dirs=[], load_error_modules=load_error_modules) course_id = "/".join([ORG, COURSE, 'test_run']) course_dir = "test_dir" error_tracker = Mock() parent_tracker = Mock() super(DummySystem, self).__init__( xmlstore=xmlstore, course_id=course_id, course_dir=course_dir, error_tracker=error_tracker, parent_tracker=parent_tracker, load_error_modules=load_error_modules, field_data=KvsFieldData(DictKeyValueStore()), id_reader=LocationReader(), )
def __init__(self, load_error_modules, library=False): if library: xmlstore = LibraryXMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules) else: xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules) course_id = CourseKey.from_string('/'.join([ORG, COURSE, RUN])) course_dir = "test_dir" error_tracker = Mock() super(DummySystem, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments xmlstore=xmlstore, course_id=course_id, course_dir=course_dir, error_tracker=error_tracker, load_error_modules=load_error_modules, mixins=(InheritanceMixin, XModuleMixin), field_data=KvsFieldData(DictKeyValueStore()), )
def test_definition_loading(self): """When two courses share the same org and course name and both have a module with the same url_name, the definitions shouldn't clash. TODO (vshnayder): once we have a CMS, this shouldn't happen--locations should uniquely name definitions. But in our imperfect XML world, it can (and likely will) happen.""" modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'two_toys']) toy_id = "edX/toy/2012_Fall" two_toy_id = "edX/toy/TT_2012_Fall" location = Location(["i4x", "edX", "toy", "video", "Welcome"]) toy_video = modulestore.get_instance(toy_id, location) two_toy_video = modulestore.get_instance(two_toy_id, location) self.assertEqual( etree.fromstring(toy_video.data).get('youtube'), "1.0:p2Q6BrNhdh8") self.assertEqual( etree.fromstring(two_toy_video.data).get('youtube'), "1.0:p2Q6BrNhdh9")
def test_url_name_mangling(self): """ Make sure that url_names are only mangled once. """ modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy']) toy_id = CourseKey.from_string('edX/toy/2012_Fall') course = modulestore.get_course(toy_id) chapters = course.get_children() ch1 = chapters[0] sections = ch1.get_children() self.assertEqual(len(sections), 4) for i in (2, 3): video = sections[i] # Name should be 'video_{hash}' print("video {0} url_name: {1}".format(i, video.url_name)) self.assertEqual(len(video.url_name), len('video_') + 12)
def test_poll_and_conditional_import(self): modulestore = XMLModuleStore(DATA_DIR, source_dirs=['conditional_and_poll']) course = modulestore.get_courses()[0] chapters = course.get_children() ch1 = chapters[0] sections = ch1.get_children() assert len(sections) == 1 conditional_location = course.id.make_usage_key('conditional', 'condone') module = modulestore.get_item(conditional_location) assert len(module.children) == 1 poll_location = course.id.make_usage_key('poll_question', 'first_poll') module = modulestore.get_item(poll_location) assert len(module.get_children()) == 0 assert module.voted is False assert module.poll_answer == '' assert module.poll_answers == {} assert module.answers ==\ [{'text': u'Yes', 'id': 'Yes'}, {'text': u'No', 'id': 'No'}, {'text': u"Don't know", 'id': 'Dont_know'}]
def test_dag_course(self, mock_logging): """ Test a course whose structure is not a tree. """ store = XMLModuleStore(DATA_DIR, course_dirs=['xml_dag']) course_key = store.get_courses()[0].id mock_logging.warning.assert_called_with( "%s has more than one definition", course_key.make_usage_key('discussion', 'duplicate_def') ) shared_item_loc = course_key.make_usage_key('html', 'toyhtml') shared_item = store.get_item(shared_item_loc) parent = shared_item.get_parent() self.assertIsNotNone(parent, "get_parent failed to return a value") parent_loc = course_key.make_usage_key('vertical', 'vertical_test') self.assertEqual(parent.location, parent_loc) self.assertIn(shared_item, parent.get_children()) # ensure it's still a child of the other parent even tho it doesn't claim the other parent as its parent other_parent_loc = course_key.make_usage_key('vertical', 'zeta') other_parent = store.get_item(other_parent_loc) # children rather than get_children b/c the instance returned by get_children != shared_item self.assertIn(shared_item_loc, other_parent.children)
def test_unicode(self): """Check that courses with unicode characters in filenames and in org/course/name import properly. Currently, this means: (a) Having files with unicode names does not prevent import; (b) if files are not loaded because of unicode filenames, there are appropriate exceptions/errors to that effect.""" print("Starting import") modulestore = XMLModuleStore(DATA_DIR, source_dirs=['test_unicode']) courses = modulestore.get_courses() assert len(courses) == 1 course = courses[0] print("course errors:") # Expect to find an error/exception about characters in "®esources" expect = "InvalidKeyError" errors = modulestore.get_course_errors(course.id) assert any( ((expect in msg) or (expect in err)) for (msg, err) in errors) chapters = course.get_children() assert len(chapters) == 4
def handle(self, *args, **options): if len(args) != 1: raise CommandError(u'Must called with arguments: {}'.format( self.args)) xml_module_store = XMLModuleStore( data_dir=settings.DATA_DIR, default_class='xmodule.hidden_module.HiddenDescriptor', load_error_modules=True, xblock_mixins=settings.XBLOCK_MIXINS, xblock_select=settings.XBLOCK_SELECT_FUNCTION, ) export_dir = path(args[0]) for course_id, course_modules in six.iteritems( xml_module_store.modules): course_path = course_id.replace('/', '_') for location, descriptor in six.iteritems(course_modules): location_path = text_type(location).replace('/', '_') data = {} for field_name, field in six.iteritems(descriptor.fields): try: data[field_name] = field.read_json(descriptor) except Exception as exc: # pylint: disable=broad-except data[field_name] = { '$type': str(type(exc)), '$value': descriptor._field_data.get(descriptor, field_name) # pylint: disable=protected-access } outdir = export_dir / course_path outdir.makedirs_p() with open(outdir / location_path + '.json', 'w') as outfile: json.dump(data, outfile, sort_keys=True, indent=4) print('', file=outfile)
def test_cohort_config(self): """ Check that cohort config parsing works right. """ modulestore = XMLModuleStore(DATA_DIR, source_dirs=['toy']) toy_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall') course = modulestore.get_course(toy_id) # No config -> False self.assertFalse(course.is_cohorted) # empty config -> False course.cohort_config = {} self.assertFalse(course.is_cohorted) # false config -> False course.cohort_config = {'cohorted': False} self.assertFalse(course.is_cohorted) # and finally... course.cohort_config = {'cohorted': True} self.assertTrue(course.is_cohorted)
def test_cohort_config(self): """ Check that cohort config parsing works right. """ modulestore = XMLModuleStore(DATA_DIR, course_dirs=['toy']) toy_id = "edX/toy/2012_Fall" course = modulestore.get_course(toy_id) # No config -> False self.assertFalse(course.is_cohorted) # empty config -> False course.cohort_config = {} self.assertFalse(course.is_cohorted) # false config -> False course.cohort_config = {'cohorted': False} self.assertFalse(course.is_cohorted) # and finally... course.cohort_config = {'cohorted': True} self.assertTrue(course.is_cohorted)
def test_get_courses_for_wiki(self): """ Test the get_courses_for_wiki method """ store = XMLModuleStore(DATA_DIR, course_dirs=['toy', 'simple']) for course in store.get_courses(): course_locations = store.get_courses_for_wiki(course.wiki_slug) self.assertEqual(len(course_locations), 1) self.assertIn(course.location, course_locations) course_locations = store.get_courses_for_wiki('no_such_wiki') self.assertEqual(len(course_locations), 0) # now set toy course to share the wiki with simple course toy_course = store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')) toy_course.wiki_slug = 'simple' course_locations = store.get_courses_for_wiki('toy') self.assertEqual(len(course_locations), 0) course_locations = store.get_courses_for_wiki('simple') self.assertEqual(len(course_locations), 2) for course_number in ['toy', 'simple']: self.assertIn(Location('edX', course_number, '2012_Fall', 'course', '2012_Fall'), course_locations)
def test_get_courses_for_wiki(self): """ Test the get_courses_for_wiki method """ store = XMLModuleStore(DATA_DIR, source_dirs=['toy', 'simple']) for course in store.get_courses(): course_locations = store.get_courses_for_wiki(course.wiki_slug) self.assertEqual(len(course_locations), 1) self.assertIn(course.location.course_key, course_locations) course_locations = store.get_courses_for_wiki('no_such_wiki') self.assertEqual(len(course_locations), 0) # now set toy course to share the wiki with simple course toy_course = store.get_course(CourseKey.from_string('edX/toy/2012_Fall')) toy_course.wiki_slug = 'simple' course_locations = store.get_courses_for_wiki('toy') self.assertEqual(len(course_locations), 0) course_locations = store.get_courses_for_wiki('simple') self.assertEqual(len(course_locations), 2) for course_number in ['toy', 'simple']: self.assertIn(CourseKey.from_string('/'.join(['edX', course_number, '2012_Fall'])), course_locations)
def test_poll_and_conditional_import(self): modulestore = XMLModuleStore(DATA_DIR, course_dirs=['conditional_and_poll']) course = modulestore.get_courses()[0] chapters = course.get_children() ch1 = chapters[0] sections = ch1.get_children() self.assertEqual(len(sections), 1) location = course.location conditional_location = Location(location.tag, location.org, location.course, 'conditional', 'condone') module = modulestore.get_instance(course.id, conditional_location) self.assertEqual(len(module.children), 1) poll_location = Location(location.tag, location.org, location.course, 'poll_question', 'first_poll') module = modulestore.get_instance(course.id, poll_location) self.assertEqual(len(module.get_children()), 0) self.assertEqual(module.voted, False) self.assertEqual(module.poll_answer, '') self.assertEqual(module.poll_answers, {}) self.assertEqual(module.answers, [{ 'text': u'Yes', 'id': 'Yes' }, { 'text': u'No', 'id': 'No' }, { 'text': u"Don't know", 'id': 'Dont_know' }])
def test_unicode(self): """Check that courses with unicode characters in filenames and in org/course/name import properly. Currently, this means: (a) Having files with unicode names does not prevent import; (b) if files are not loaded because of unicode filenames, there are appropriate exceptions/errors to that effect.""" print("Starting import") modulestore = XMLModuleStore(DATA_DIR, course_dirs=['test_unicode']) courses = modulestore.get_courses() self.assertEquals(len(courses), 1) course = courses[0] print("course errors:") # Expect to find an error/exception about characters in "®esources" expect = "Invalid characters" errors = [(msg.encode("utf-8"), err.encode("utf-8")) for msg, err in modulestore.get_course_errors(course.id)] self.assertTrue( any(expect in msg or expect in err for msg, err in errors)) chapters = course.get_children() self.assertEqual(len(chapters), 4)
def test_export_roundtrip(self, course_dir, mock_get): # Patch network calls to retrieve the textbook TOC mock_get.return_value.text = dedent(""" <?xml version="1.0"?><table_of_contents> <entry page="5" page_label="ii" name="Table of Contents"/> </table_of_contents> """).strip() root_dir = path(self.temp_dir) print "Copying test course to temp dir {0}".format(root_dir) data_dir = path(DATA_DIR) shutil.copytree(data_dir / course_dir, root_dir / course_dir) print "Starting import" initial_import = XMLModuleStore(root_dir, source_dirs=[course_dir], xblock_mixins=(XModuleMixin, )) courses = initial_import.get_courses() self.assertEquals(len(courses), 1) initial_course = courses[0] # export to the same directory--that way things like the custom_tags/ folder # will still be there. print "Starting export" file_system = OSFS(root_dir) initial_course.runtime.export_fs = file_system.makedir(course_dir, recreate=True) root = lxml.etree.Element('root') initial_course.add_xml_to_node(root) with initial_course.runtime.export_fs.open('course.xml', 'wb') as course_xml: lxml.etree.ElementTree(root).write(course_xml, encoding='utf-8') print "Starting second import" second_import = XMLModuleStore(root_dir, source_dirs=[course_dir], xblock_mixins=(XModuleMixin, )) courses2 = second_import.get_courses() self.assertEquals(len(courses2), 1) exported_course = courses2[0] print "Checking course equality" # HACK: filenames change when changing file formats # during imports from old-style courses. Ignore them. strip_filenames(initial_course) strip_filenames(exported_course) self.assertTrue(blocks_are_equivalent(initial_course, exported_course)) self.assertEquals(initial_course.id, exported_course.id) course_id = initial_course.id print "Checking key equality" self.assertItemsEqual(initial_import.modules[course_id].keys(), second_import.modules[course_id].keys()) print "Checking module equality" for location in initial_import.modules[course_id].keys(): print("Checking", location) self.assertTrue( blocks_are_equivalent( initial_import.modules[course_id][location], second_import.modules[course_id][location]))
def setup_modulestore(self, name): self.modulestore = XMLModuleStore(DATA_DIR, course_dirs=[name])