def setUpClass(cls): super(ContentStoreToyCourseTest, cls).setUpClass() cls.contentstore = contentstore() cls.modulestore = modulestore() cls.course_key = cls.modulestore.make_course_key('edX', 'toy', '2012_Fall') import_course_from_xml( cls.modulestore, 1, TEST_DATA_DIR, ['toy'], static_content_store=cls.contentstore, verbose=True ) # A locked asset cls.locked_asset = cls.course_key.make_asset_key('asset', 'sample_static.html') cls.url_locked = six.text_type(cls.locked_asset) cls.url_locked_versioned = get_versioned_asset_url(cls.url_locked) cls.url_locked_versioned_old_style = get_old_style_versioned_asset_url(cls.url_locked) cls.contentstore.set_attr(cls.locked_asset, 'locked', True) # An unlocked asset cls.unlocked_asset = cls.course_key.make_asset_key('asset', 'another_static.txt') cls.url_unlocked = six.text_type(cls.unlocked_asset) cls.url_unlocked_versioned = get_versioned_asset_url(cls.url_unlocked) cls.url_unlocked_versioned_old_style = get_old_style_versioned_asset_url(cls.url_unlocked) cls.length_unlocked = cls.contentstore.get_attr(cls.unlocked_asset, 'length')
def test_rewrite_reference_list(self): # This test fails with split modulestore (the HTML component is not in "different_course_id" namespace). # More investigation needs to be done. module_store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) target_id = module_store.make_course_key('testX', 'conditional_copy', 'copy_run') import_course_from_xml( module_store, self.user.id, TEST_DATA_DIR, ['conditional'], target_id=target_id ) conditional_module = module_store.get_item( target_id.make_usage_key('conditional', 'condone') ) self.assertIsNotNone(conditional_module) different_course_id = module_store.make_course_key('edX', 'different_course', None) self.assertListEqual( [ target_id.make_usage_key('problem', 'choiceprob'), different_course_id.make_usage_key('html', 'for_testing_import_rewrites') ], conditional_module.sources_list ) self.assertListEqual( [ target_id.make_usage_key('html', 'congrats'), target_id.make_usage_key('html', 'secret_page') ], conditional_module.show_tag_list )
def load_courses(cls): """ Load test courses and return list of ids """ store = modulestore() unique_org = factory.Sequence(lambda n: 'edX.%d' % n) cls.course = CourseFactory.create( emit_signals=True, org=unique_org, course='simple', run="run", display_name=u'2012_Fáĺĺ', modulestore=store ) cls.discussion = ItemFactory.create( category='discussion', parent_location=cls.course.location ) courses = store.get_courses() # NOTE: if xml store owns these, it won't import them into mongo if cls.test_course_key not in [c.id for c in courses]: import_course_from_xml( store, ModuleStoreEnum.UserID.mgmt_command, DATA_DIR, XML_COURSE_DIRS, create_if_not_present=True ) return [course.id for course in store.get_courses()]
def setUp(self): """ Set up the tests """ super(StaticTabDateTestCaseXML, self).setUp() # The following XML test course (which lives at common/test/data/2014) # is closed; we're testing that tabs still appear when # the course is already closed self.xml_course_key = self.store.make_course_key('edX', 'detached_pages', '2014') import_course_from_xml( self.store, 'test_user', TEST_DATA_DIR, source_dirs=['2014'], static_content_store=None, target_id=self.xml_course_key, raise_on_failure=True, create_if_not_present=True, ) # this text appears in the test course's tab # common/test/data/2014/tabs/8e4cce2b4aaf4ba28b1220804619e41f.html self.xml_data = "static 463139" self.xml_url = "8e4cce2b4aaf4ba28b1220804619e41f"
def import_test_course(self, solution_attribute=None, solution_element=None): """ Import the test course with the sga unit """ # adapted from edx-platform/cms/djangoapps/contentstore/management/commands/tests/test_cleanup_assets.py root = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) input_dir = os.path.join(root, "test_data") temp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(temp_dir)) xml_dir = os.path.join(temp_dir, "xml") shutil.copytree(input_dir, xml_dir) with open(os.path.join(xml_dir, "2017_SGA", "vertical", "vertical.xml"), "w") as f: f.write(self.make_test_vertical(solution_attribute, solution_element)) store = modulestore() import_course_from_xml( store, 'sga_user', xml_dir, ) return store.get_course(CourseLocator.from_string('SGAU/SGA101/course'))
def setUp(self): """ Create user and login. """ self.staff_pwd = super(ContentStoreToyCourseTest, self).setUp() self.staff_usr = self.user self.non_staff_usr, self.non_staff_pwd = self.create_non_staff_user() self.client = Client() self.contentstore = contentstore() store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) # pylint: disable=protected-access self.course_key = store.make_course_key('edX', 'toy', '2012_Fall') import_course_from_xml( store, self.user.id, TEST_DATA_DIR, ['toy'], static_content_store=self.contentstore, verbose=True ) # A locked asset self.locked_asset = self.course_key.make_asset_key('asset', 'sample_static.txt') self.url_locked = unicode(self.locked_asset) self.contentstore.set_attr(self.locked_asset, 'locked', True) # An unlocked asset self.unlocked_asset = self.course_key.make_asset_key('asset', 'another_static.txt') self.url_unlocked = unicode(self.unlocked_asset) self.length_unlocked = self.contentstore.get_attr(self.unlocked_asset, 'length')
def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ if CodeBlockTimer is None: raise SkipTest("CodeBlockTimer undefined.") desc = "XMLRoundTrip:{}->{}:{}".format( SHORT_NAME_MAP[source_ms], SHORT_NAME_MAP[dest_ms], num_assets ) with CodeBlockTimer(desc): with CodeBlockTimer("fake_assets"): # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) with source_ms.build() as (source_content, source_store): with dest_ms.build() as (dest_content, dest_store): source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') with CodeBlockTimer("initial_import"): import_course_from_xml( source_store, 'test_user', TEST_DATA_ROOT, source_dirs=TEST_COURSE, static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with CodeBlockTimer("export"): export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with CodeBlockTimer("second_import"): import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, create_if_not_present=True, raise_on_failure=True, )
def test_generate_find_timings(self, source_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ if CodeBlockTimer is None: raise SkipTest("CodeBlockTimer undefined.") desc = "FindAssetTest:{}:{}".format( SHORT_NAME_MAP[source_ms], num_assets, ) with CodeBlockTimer(desc): with CodeBlockTimer("fake_assets"): # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) with source_ms.build() as (source_content, source_store): source_course_key = source_store.make_course_key('a', 'course', 'course') asset_key = source_course_key.make_asset_key( AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif' ) with CodeBlockTimer("initial_import"): import_course_from_xml( source_store, 'test_user', TEST_DATA_ROOT, source_dirs=TEST_COURSE, static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with CodeBlockTimer("find_nonexistent_asset"): # More correct would be using the AssetManager.find() - but since the test # has created its own test modulestore, the AssetManager can't be used. __ = source_store.find_asset_metadata(asset_key) # Perform get_all_asset_metadata for each sort. for sort in ALL_SORTS: with CodeBlockTimer("get_asset_list:{}-{}".format( sort[0], 'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc' )): # Grab two ranges of 50 assets using different sorts. # Why 50? That's how many are displayed on the current Studio "Files & Uploads" page. start_middle = num_assets / 2 __ = source_store.get_all_asset_metadata( source_course_key, 'asset', start=0, sort=sort, maxresults=50 ) __ = source_store.get_all_asset_metadata( source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50 )
def test_split_course_export_import(self): # Construct the contentstore for storing the first import with MongoContentstoreBuilder().build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with SPLIT_MODULESTORE_SETUP.build(contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with MongoContentstoreBuilder().build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with SPLIT_MODULESTORE_SETUP.build(contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key('a', 'source', '2015_Fall') dest_course_key = dest_store.make_course_key('a', 'dest', '2015_Fall') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['split_course_with_static_tabs'], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME, ) source_course = source_store.get_course(source_course_key, depth=None, lazy=False) self.assertEqual(source_course.url_name, 'course') export_dir_path = path(self.export_dir) policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course_key.run policy_path = policy_dir / 'policy.json' self.assertTrue(os.path.exists(policy_path)) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=[EXPORTED_COURSE_DIR_NAME], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) dest_course = dest_store.get_course(dest_course_key, depth=None, lazy=False) self.assertEqual(dest_course.url_name, 'course')
def test_rewrite_reference(self): module_store = modulestore() target_id = module_store.make_course_key("testX", "peergrading_copy", "copy_run") import_course_from_xml( module_store, self.user.id, TEST_DATA_DIR, ["open_ended"], target_id=target_id, create_if_not_present=True ) peergrading_module = module_store.get_item(target_id.make_usage_key("peergrading", "PeerGradingLinked")) self.assertIsNotNone(peergrading_module) self.assertEqual( target_id.make_usage_key("combinedopenended", "SampleQuestion"), peergrading_module.link_to_location )
def test_asset_sizes(self, source_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) with source_ms.build() as (source_content, source_store): source_course_key = source_store.make_course_key('a', 'course', 'course') import_course_from_xml( source_store, 'test_user', TEST_DATA_ROOT, source_dirs=TEST_COURSE, static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) asset_collection = source_ms.asset_collection() # Ensure the asset collection exists. if asset_collection.name in asset_collection.database.collection_names(): # Map gets the size of each structure. mapper = Code(""" function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) } """) # Reduce finds the largest structure size and returns only it. reducer = Code(""" function(key, values) { var max_size = 0; for (var i=0; i < values.length; i++) { if (values[i] > max_size) { max_size = values[i]; } } return max_size; } """) results = asset_collection.map_reduce(mapper, reducer, "size_results") result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format( self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()] ) with open("bson_sizes.txt", "a") as f: f.write(result_str)
def setUpClass(cls): super(CommandExecutionTestCase, cls).setUpClass() cls.course_key = cls.store.make_course_key(u'edX', u'lti_provider', u'3000') import_course_from_xml( cls.store, u'test_user', TEST_DATA_DIR, source_dirs=[u'simple'], static_content_store=None, target_id=cls.course_key, raise_on_failure=True, create_if_not_present=True, ) cls.lti_block = u'block-v1:edX+lti_provider+3000+type@chapter+block@chapter_2'
def setUp(self): super(TestPeerGradingFound, self).setUp() self.user = factories.UserFactory() store = modulestore() course_items = import_course_from_xml(store, self.user.id, TEST_DATA_DIR, ['open_ended_nopath']) # pylint: disable=maybe-no-member self.course = course_items[0] self.course_key = self.course.id
def handle(self, *args, **options): "Execute the command" if len(args) == 0: raise CommandError("import requires at least one argument: <data directory> [--nostatic] [<course dir>...]") data_dir = args[0] do_import_static = not options.get('nostatic', False) if len(args) > 1: source_dirs = args[1:] else: source_dirs = None self.stdout.write("Importing. Data_dir={data}, source_dirs={courses}\n".format( data=data_dir, courses=source_dirs, )) mstore = modulestore() course_items = import_course_from_xml( mstore, ModuleStoreEnum.UserID.mgmt_command, data_dir, source_dirs, load_error_modules=False, static_content_store=contentstore(), verbose=True, do_import_static=do_import_static, create_if_not_present=True, ) for course in course_items: course_id = course.id if not are_permissions_roles_seeded(course_id): self.stdout.write('Seeding forum roles for course {0}\n'.format(course_id)) seed_permissions_roles(course_id)
def test_unicode_chars_in_course_name_import(self): """ # Test that importing course with unicode 'id' and 'display name' doesn't give UnicodeEncodeError """ # Test with the split modulestore because store.has_course fails in old mongo with unicode characters. with modulestore().default_store(ModuleStoreEnum.Type.split): module_store = modulestore() course_id = module_store.make_course_key(u"Юникода", u"unicode_course", u"échantillon") import_course_from_xml( module_store, self.user.id, TEST_DATA_DIR, ["2014_Uni"], target_id=course_id, create_if_not_present=True ) course = module_store.get_course(course_id) self.assertIsNotNone(course) # test that course 'display_name' same as imported course 'display_name' self.assertEqual(course.display_name, u"Φυσικά το όνομα Unicode")
def _import_course(self, content_store, modulestore): """ Imports a course for testing. Returns the course key. """ course_key = modulestore.make_course_key('a', 'course', 'course') import_course_from_xml( modulestore, 'test_user', TEST_DATA_DIR, source_dirs=['manual-testing-complete'], static_content_store=content_store, target_id=course_key, create_if_not_present=True, raise_on_failure=True, ) return course_key
def test_export_all_courses(self): """ This test validates that redundant Mac metadata files ('._example.txt', '.DS_Store') are cleaned up on import """ import_course_from_xml( self.module_store, '**replace_user**', TEST_DATA_DIR, ['course_ignore'], static_content_store=self.content_store, do_import_static=True, verbose=True ) course = self.module_store.get_course(CourseKey.from_string('/'.join(['edX', 'course_ignore', '2014_Fall']))) self.assertIsNotNone(course) # check that there are two assets ['example.txt', '.example.txt'] in contentstore for imported course all_assets, count = self.content_store.get_all_content_for_course(course.id) self.assertEqual(count, 2) self.assertEqual(set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt'])) # manually add redundant assets (file ".DS_Store" and filename starts with "._") course_filter = course.id.make_asset_key("asset", None) query = location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG) query['_id.name'] = all_assets[0]['_id']['name'] asset_doc = self.content_store.fs_files.find_one(query) asset_doc['_id']['name'] = u'._example_test.txt' self.content_store.fs_files.insert(asset_doc) asset_doc['_id']['name'] = u'.DS_Store' self.content_store.fs_files.insert(asset_doc) # check that now course has four assets all_assets, count = self.content_store.get_all_content_for_course(course.id) self.assertEqual(count, 4) self.assertEqual( set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt', u'._example_test.txt', u'.DS_Store']) ) # now call asset_cleanup command and check that there is only two proper assets in contentstore for the course call_command('cleanup_assets') all_assets, count = self.content_store.get_all_content_for_course(course.id) self.assertEqual(count, 2) self.assertEqual(set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt']))
def test_import_course_into_similar_namespace(self): # Checks to make sure that a course with an org/course like # edx/course can be imported into a namespace with an org/course # like edx/course_name module_store, __, course = self.load_test_import_course() course_items = import_course_from_xml( module_store, self.user.id, TEST_DATA_DIR, ["test_import_course_2"], target_id=course.id, verbose=True ) self.assertEqual(len(course_items), 1)
def add_mobile_available_toy_course(self): """ use toy course with handouts, and make it mobile_available """ course_items = import_course_from_xml( self.store, self.user.id, settings.COMMON_TEST_DATA_ROOT, ["toy"], create_if_not_present=True ) self.course = course_items[0] self.course.mobile_available = True self.store.update_item(self.course, self.user.id) self.login_and_enroll()
def _verify_split_test_import(self, target_course_name, source_course_name, split_test_name, groups_to_verticals): module_store = modulestore() target_id = module_store.make_course_key("testX", target_course_name, "copy_run") import_course_from_xml( module_store, self.user.id, TEST_DATA_DIR, [source_course_name], target_id=target_id, create_if_not_present=True, ) split_test_module = module_store.get_item(target_id.make_usage_key("split_test", split_test_name)) self.assertIsNotNone(split_test_module) remapped_verticals = { key: target_id.make_usage_key("vertical", value) for key, value in groups_to_verticals.iteritems() } self.assertEqual(remapped_verticals, split_test_module.group_id_to_child)
def test_asset_import_nostatic(self): ''' This test validates that an image asset is NOT imported when do_import_static=False ''' content_store = contentstore() module_store = modulestore() import_course_from_xml( module_store, self.user.id, TEST_DATA_DIR, ['toy'], static_content_store=content_store, do_import_static=False, create_if_not_present=True, verbose=True ) course = module_store.get_course(module_store.make_course_key('edX', 'toy', '2012_Fall')) # make sure we have NO assets in our contentstore all_assets, count = content_store.get_all_content_for_course(course.id) self.assertEqual(len(all_assets), 0) self.assertEqual(count, 0)
def test_rewrite_reference_list(self): # This test fails with split modulestore (the HTML component is not in "different_course_id" namespace). # More investigation needs to be done. module_store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) target_id = module_store.make_course_key("testX", "conditional_copy", "copy_run") import_course_from_xml(module_store, self.user.id, TEST_DATA_DIR, ["conditional"], target_id=target_id) conditional_module = module_store.get_item(target_id.make_usage_key("conditional", "condone")) self.assertIsNotNone(conditional_module) different_course_id = module_store.make_course_key("edX", "different_course", None) self.assertListEqual( [ target_id.make_usage_key("problem", "choiceprob"), different_course_id.make_usage_key("html", "for_testing_import_rewrites"), ], conditional_module.sources_list, ) self.assertListEqual( [target_id.make_usage_key("html", "congrats"), target_id.make_usage_key("html", "secret_page")], conditional_module.show_tag_list, )
def setUp(self): """ Set up the course and user context """ super(CoursesRenderTest, self).setUp() store = modulestore() course_items = import_course_from_xml(store, self.user.id, TEST_DATA_DIR, ['toy']) course_key = course_items[0].id self.course = get_course_by_id(course_key) self.request = get_request_for_user(UserFactory.create())
def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes): with store_builder.build() as (source_content, source_store): with store_builder.build() as (dest_content, dest_store): source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') # An extra import write occurs in the first Split import due to the mismatch between # the course id and the wiki_slug in the test XML course. The course must be updated # with the correct wiki_slug during import. with check_mongo_calls(import_reads, first_import_writes): import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['manual-testing-complete'], static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with check_mongo_calls(export_reads): export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with check_mongo_calls(import_reads, second_import_writes): import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, create_if_not_present=True, raise_on_failure=True, )
def load_courses(self): """Load test courses and return list of ids""" store = modulestore() # Add a course with a unicode name. unique_org = factory.Sequence(lambda n: u'ëḋẌ.%d' % n) CourseFactory.create( org=unique_org, course=u'śíḿṕĺé', display_name=u'2012_Fáĺĺ', modulestore=store ) courses = store.get_courses() # NOTE: if xml store owns these, it won't import them into mongo if self.test_course_key not in [c.id for c in courses]: import_course_from_xml( store, ModuleStoreEnum.UserID.mgmt_command, DATA_DIR, XML_COURSE_DIRS, create_if_not_present=True ) return [course.id for course in store.get_courses()]
def setUp(self): super(TestStudentProblemList, self).setUp() # Load an open ended course with several problems. self.user = factories.UserFactory() store = modulestore() course_items = import_course_from_xml(store, self.user.id, TEST_DATA_DIR, ['open_ended']) # pylint: disable=maybe-no-member self.course = course_items[0] self.course_key = self.course.id # Enroll our user in our course and make them an instructor. make_instructor(self.course, self.user.email)
def setUp(self): super(TestHandouts, self).setUp() # Deleting handouts fails with split modulestore because the handout has no parent. # This needs further investigation to determine if it is a bug in the split modulestore. # pylint: disable=protected-access self.store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) # use toy course with handouts, and make it mobile_available course_items = import_course_from_xml(self.store, self.user.id, settings.COMMON_TEST_DATA_ROOT, ['toy']) self.course = course_items[0] self.course.mobile_available = True self.store.update_item(self.course, self.user.id)
def test_number_mongo_calls(self, store, depth, lazy, access_all_block_fields, num_mongo_calls): with store.build() as (source_content, source_store): source_course_key = source_store.make_course_key('a', 'course', 'course') # First, import a course. import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['manual-testing-complete'], static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) # Course traversal modeled after the traversal done here: # lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline # Starting at the root course block, do a breadth-first traversal using # get_children() to retrieve each block's children. with check_mongo_calls(num_mongo_calls): with source_store.bulk_operations(source_course_key): start_block = source_store.get_course(source_course_key, depth=depth, lazy=lazy) all_blocks = [] stack = [start_block] while stack: curr_block = stack.pop() all_blocks.append(curr_block) if curr_block.has_children: for block in reversed(curr_block.get_children()): stack.append(block) if access_all_block_fields: # Read the fields on each block in order to ensure each block and its definition is loaded. for xblock in all_blocks: for __, field in xblock.fields.iteritems(): if field.is_set_on(xblock): __ = field.read_from(xblock)
def test_no_static_link_rewrites_on_import(self): module_store = modulestore() courses = import_course_from_xml( module_store, self.user.id, TEST_DATA_DIR, ['toy'], do_import_static=False, verbose=True, create_if_not_present=True ) course_key = courses[0].id handouts = module_store.get_item(course_key.make_usage_key('course_info', 'handouts')) self.assertIn('/static/', handouts.data) handouts = module_store.get_item(course_key.make_usage_key('html', 'toyhtml')) self.assertIn('/static/', handouts.data)
def initdb(cls): # connect to the db doc_store_config = { 'host': HOST, 'port': PORT, 'db': DB, 'collection': COLLECTION, } cls.add_asset_collection(doc_store_config) # since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class # as well content_store = MongoContentStore(HOST, DB, port=PORT) # # Also test draft store imports # draft_store = DraftModuleStore( content_store, doc_store_config, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS, branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred, xblock_mixins=(EditInfoMixin, InheritanceMixin, LocationMixin, XModuleMixin) ) with patch('xmodule.tabs.CourseTab.from_json', side_effect=mock_tab_from_json): import_course_from_xml( draft_store, 999, DATA_DIR, cls.courses, static_content_store=content_store ) # also test a course with no importing of static content import_course_from_xml( draft_store, 999, DATA_DIR, ['test_import_course'], static_content_store=content_store, do_import_static=False, verbose=True ) # also import a course under a different course_id (especially ORG) import_course_from_xml( draft_store, 999, DATA_DIR, ['test_import_course'], static_content_store=content_store, do_import_static=False, verbose=True, target_id=SlashSeparatedCourseKey('guestx', 'foo', 'bar') ) return content_store, draft_store
def initdb(cls): # connect to the db doc_store_config = { 'host': HOST, 'port': PORT, 'db': DB, 'collection': COLLECTION, } cls.add_asset_collection(doc_store_config) # since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class # as well content_store = MongoContentStore(HOST, DB, port=PORT) # # Also test draft store imports # draft_store = DraftModuleStore( content_store, doc_store_config, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS, branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred, xblock_mixins=(EditInfoMixin, InheritanceMixin, LocationMixin, XModuleMixin)) with patch('xmodule.tabs.CourseTab.from_json', side_effect=mock_tab_from_json): import_course_from_xml(draft_store, 999, DATA_DIR, cls.courses, static_content_store=content_store) # also test a course with no importing of static content import_course_from_xml(draft_store, 999, DATA_DIR, ['test_import_course'], static_content_store=content_store, do_import_static=False, verbose=True) # also import a course under a different course_id (especially ORG) import_course_from_xml(draft_store, 999, DATA_DIR, ['test_import_course'], static_content_store=content_store, do_import_static=False, verbose=True, target_id=SlashSeparatedCourseKey( 'guestx', 'foo', 'bar')) return content_store, draft_store
def setUp(self): super(OpenEndedStatsTest, self).setUp() self.user = UserFactory() store = modulestore() course_items = import_course_from_xml(store, self.user.id, TEST_DATA_DIR, ['open_ended']) # pylint: disable=maybe-no-member self.course = course_items[0] self.course_id = self.course.id self.problem_location = Location("edX", "open_ended", "2012_Fall", "combinedopenended", "SampleQuestion") self.task_number = 1 self.invalid_task_number = 3 self.student_on_initial = UserFactory() self.student_on_accessing = UserFactory() self.student_on_post_assessment = UserFactory() StudentModuleFactory.create(course_id=self.course_id, module_state_key=self.problem_location, student=self.student_on_initial, grade=0, max_grade=1, state=STATE_INITIAL) StudentModuleFactory.create(course_id=self.course_id, module_state_key=self.problem_location, student=self.student_on_accessing, grade=0, max_grade=1, state=STATE_ACCESSING) StudentModuleFactory.create(course_id=self.course_id, module_state_key=self.problem_location, student=self.student_on_post_assessment, grade=0, max_grade=1, state=STATE_POST_ASSESSMENT) self.students = [ self.student_on_initial, self.student_on_accessing, self.student_on_post_assessment ]
def handle(self, *args, **options): data_dir = options['data_directory'] source_dirs = options['course_dirs'] if len(source_dirs) == 0: source_dirs = None do_import_static = not options.get('nostatic', False) # If the static content is not skipped, the python lib should be imported regardless # of the 'nopythonlib' flag. do_import_python_lib = do_import_static or not options.get( 'nopythonlib', False) python_lib_filename = options.get('python_lib_filename') output = (u"Importing...\n" u" data_dir={data}, source_dirs={courses}\n" u" Importing static content? {import_static}\n" u" Importing python lib? {import_python_lib}").format( data=data_dir, courses=source_dirs, import_static=do_import_static, import_python_lib=do_import_python_lib) self.stdout.write(output) mstore = modulestore() course_items = import_course_from_xml( mstore, ModuleStoreEnum.UserID.mgmt_command, data_dir, source_dirs, load_error_modules=False, static_content_store=contentstore(), verbose=True, do_import_static=do_import_static, do_import_python_lib=do_import_python_lib, create_if_not_present=True, python_lib_filename=python_lib_filename, ) for course in course_items: course_id = course.id if not are_permissions_roles_seeded(course_id): self.stdout.write( u'Seeding forum roles for course {0}\n'.format(course_id)) seed_permissions_roles(course_id)
def load_scoreable_course(cls): """ This test course lives at `common/test/data/scoreable`. For details on the contents and structure of the file, see `common/test/data/scoreable/README`. """ course_items = import_course_from_xml( cls.store, 'test_user', TEST_DATA_DIR, source_dirs=['scoreable'], static_content_store=None, target_id=cls.store.make_course_key('edX', 'scoreable', '3000'), raise_on_failure=True, create_if_not_present=True, ) cls.course = course_items[0]
def import_test_course(self): """ Import the test course with the sga unit """ # adapted from edx-platform/cms/djangoapps/contentstore/ # management/commands/tests/test_cleanup_assets.py input_dir = os.path.join(BASE_DIR, "..", "test_data") temp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(temp_dir)) xml_dir = os.path.join(temp_dir, "xml") shutil.copytree(input_dir, xml_dir) store = modulestore() courses = import_course_from_xml( store, 'sga_user', xml_dir, ) return courses[0]
def test_pdf_asset(self): module_store = modulestore() course_items = import_course_from_xml( module_store, self.user.id, TEST_DATA_DIR, ['toy'], static_content_store=contentstore(), verbose=True) course = course_items[0] url = reverse_course_url('assets_handler', course.id) # Test valid contentType for pdf asset (textbook.pdf) resp = self.client.get(url, HTTP_ACCEPT='application/json') self.assertContains(resp, "/c4x/edX/toy/asset/textbook.pdf") asset_location = AssetKey.from_string( '/c4x/edX/toy/asset/textbook.pdf') content = contentstore().find(asset_location) # Check after import textbook.pdf has valid contentType ('application/pdf') # Note: Actual contentType for textbook.pdf in asset.json is 'text/pdf' self.assertEqual(content.content_type, 'application/pdf')
def _assert_import(self, course_dir, expected_field_val, has_draft=False): """ Import a course from XML, then verify that the XBlock was loaded with the correct field value. Args: course_dir (str): The name of the course directory (relative to the test data directory) expected_xblock_loc (str): The location of the XBlock in the course. expected_field_val (str): The expected value of the XBlock's test field. Kwargs: has_draft (bool): If true, check that a draft of the XBlock exists with the expected field value set. """ # It is necessary to use the "old mongo" modulestore because split doesn't work # with the "has_draft" logic below. store = modulestore()._get_modulestore_by_type( ModuleStoreEnum.Type.mongo) # pylint: disable=protected-access courses = import_course_from_xml(store, self.user.id, TEST_DATA_DIR, [course_dir], create_if_not_present=True) xblock_location = courses[0].id.make_usage_key('stubxblock', 'xblock_test') if has_draft: xblock_location = as_draft(xblock_location) xblock = store.get_item(xblock_location) self.assertTrue(isinstance(xblock, StubXBlock)) self.assertEqual(xblock.test_field, expected_field_val) if has_draft: draft_xblock = store.get_item(xblock_location) self.assertTrue(getattr(draft_xblock, 'is_draft', False)) self.assertTrue(isinstance(draft_xblock, StubXBlock)) self.assertEqual(draft_xblock.test_field, expected_field_val)
def handle(self, *args, **options): "Execute the command" if len(args) == 0: raise CommandError( "import requires at least one argument: <data directory> [--nostatic] [<course dir>...]" ) data_dir = args[0] do_import_static = not (options.get('nostatic', False)) if len(args) > 1: source_dirs = args[1:] else: source_dirs = None self.stdout.write( "Importing. Data_dir={data}, source_dirs={courses}\n".format( data=data_dir, courses=source_dirs, )) mstore = modulestore() course_items = import_course_from_xml( mstore, ModuleStoreEnum.UserID.mgmt_command, data_dir, source_dirs, load_error_modules=False, static_content_store=contentstore(), verbose=True, do_import_static=do_import_static, create_if_not_present=True, ) for course in course_items: course_id = course.id if not are_permissions_roles_seeded(course_id): self.stdout.write( 'Seeding forum roles for course {0}\n'.format(course_id)) seed_permissions_roles(course_id)
def import_single_course(filename): print >> sys.stderr, 'IMPORTING course:', filename org, course_id, course_run = _filename_to_org_id_and_run(filename) course_full_id = 'course-v1:{org}+{id}+{run}'.format(org=org, id=course_id, run=course_run) course_xml_dir = path.join( XML_EXTRACT_DIR, '{id}-{run}'.format(id=course_id, run=course_run)) mkdir(course_xml_dir) subprocess.call(['tar', '-xzf', filename, '-C', course_xml_dir]) _fix_library_source_bug(course_xml_dir) print >> sys.stderr, 'IMPORTING course:', course_full_id course_items = import_course_from_xml( store=MOD_STORE, user_id=ModuleStoreEnum.UserID.mgmt_command, data_dir=DATA_DIR, source_dirs=[path.join(course_xml_dir, 'course')], # Open edX needs `course` dir load_error_modules=False, static_content_store=contentstore(), verbose=True, do_import_static=True, target_id=CourseKey.from_string(course_full_id), create_if_not_present=True, ) for course in course_items: course_id = course.id if not are_permissions_roles_seeded(course_id): print >> sys.stderr, 'Seeding forum roles for course', course_id seed_permissions_roles(course_id) _set_course_dates(course_id)
def test_generate_find_timings(self, source_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ if CodeBlockTimer is None: raise SkipTest("CodeBlockTimer undefined.") desc = "FindAssetTest:{}:{}".format( SHORT_NAME_MAP[source_ms], num_assets, ) with CodeBlockTimer(desc): with CodeBlockTimer("fake_assets"): # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) with source_ms.build() as (source_content, source_store): source_course_key = source_store.make_course_key( 'a', 'course', 'course') asset_key = source_course_key.make_asset_key( AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif') with CodeBlockTimer("initial_import"): import_course_from_xml( source_store, 'test_user', TEST_DATA_ROOT, source_dirs=TEST_COURSE, static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with CodeBlockTimer("find_nonexistent_asset"): # More correct would be using the AssetManager.find() - but since the test # has created its own test modulestore, the AssetManager can't be used. __ = source_store.find_asset_metadata(asset_key) # Perform get_all_asset_metadata for each sort. for sort in ALL_SORTS: with CodeBlockTimer("get_asset_list:{}-{}".format( sort[0], 'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc')): # Grab two ranges of 50 assets using different sorts. # Why 50? That's how many are displayed on the current Studio "Files & Uploads" page. start_middle = num_assets / 2 __ = source_store.get_all_asset_metadata( source_course_key, 'asset', start=0, sort=sort, maxresults=50) __ = source_store.get_all_asset_metadata( source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50)
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build( contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build( contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=[course_data_name], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) # NOT CURRENTLY USED # export_course_to_xml( # dest_store, # dest_content, # dest_course_key, # self.export_dir, # 'exported_dest_course', # ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.exclude_field(None, 'parent') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, ) self.assertAssetsMetadataEqual( source_store, source_course_key, dest_store, dest_course_key, )
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name, _mock_tab_from_json): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build( contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build( contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=[course_data_name], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME, ) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=[EXPORTED_COURSE_DIR_NAME], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) # NOT CURRENTLY USED # export_course_to_xml( # dest_store, # dest_content, # dest_course_key, # self.export_dir, # 'exported_dest_course', # ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.exclude_field(None, 'parent') # discussion_ids are auto-generated based on usage_id, so they should change across # modulestores - see TNL-5001 self.exclude_field(None, 'discussion_id') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, ) self.assertAssetsMetadataEqual( source_store, source_course_key, dest_store, dest_course_key, )
def test_split_course_export_import(self): # Construct the contentstore for storing the first import with MongoContentstoreBuilder().build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with SPLIT_MODULESTORE_SETUP.build( contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with MongoContentstoreBuilder().build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with SPLIT_MODULESTORE_SETUP.build( contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key( 'a', 'source', '2015_Fall') dest_course_key = dest_store.make_course_key( 'a', 'dest', '2015_Fall') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['split_course_with_static_tabs'], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME, ) source_course = source_store.get_course( source_course_key, depth=None, lazy=False) self.assertEqual(source_course.url_name, 'course') export_dir_path = path(self.export_dir) policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course_key.run policy_path = policy_dir / 'policy.json' self.assertTrue(os.path.exists(policy_path)) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=[EXPORTED_COURSE_DIR_NAME], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) dest_course = dest_store.get_course(dest_course_key, depth=None, lazy=False) self.assertEqual(dest_course.url_name, 'course')
def import_and_populate_course(self): """ Imports the test toy course and populates it with additional test data """ content_store = contentstore() import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], static_content_store=content_store) course_id = CourseKey.from_string('/'.join(['edX', 'toy', '2012_Fall'])) # create an Orphan # We had a bug where orphaned draft nodes caused export to fail. This is here to cover that case. vertical = self.store.get_item(course_id.make_usage_key( 'vertical', self.TEST_VERTICAL), depth=1) vertical.location = vertical.location.replace(name='no_references') self.store.update_item(vertical, self.user.id, allow_not_found=True) orphan_vertical = self.store.get_item(vertical.location) self.assertEqual(orphan_vertical.location.name, 'no_references') self.assertEqual(len(orphan_vertical.children), len(vertical.children)) # create an orphan vertical and html; we already don't try to import # the orphaned vertical, but we should make sure we don't import # the orphaned vertical's child html, too orphan_draft_vertical = self.store.create_item( self.user.id, course_id, 'vertical', self.ORPHAN_DRAFT_VERTICAL) orphan_draft_html = self.store.create_item(self.user.id, course_id, 'html', self.ORPHAN_DRAFT_HTML) orphan_draft_vertical.children.append(orphan_draft_html.location) self.store.update_item(orphan_draft_vertical, self.user.id) # create a Draft vertical vertical = self.store.get_item(course_id.make_usage_key( 'vertical', self.TEST_VERTICAL), depth=1) draft_vertical = self.store.convert_to_draft(vertical.location, self.user.id) self.assertTrue(self.store.has_published_version(draft_vertical)) # create a Private (draft only) vertical private_vertical = self.store.create_item(self.user.id, course_id, 'vertical', self.PRIVATE_VERTICAL) self.assertFalse(self.store.has_published_version(private_vertical)) # create a Published (no draft) vertical public_vertical = self.store.create_item(self.user.id, course_id, 'vertical', self.PUBLISHED_VERTICAL) public_vertical = self.store.publish(public_vertical.location, self.user.id) self.assertTrue(self.store.has_published_version(public_vertical)) # add the new private and new public as children of the sequential sequential = self.store.get_item( course_id.make_usage_key('sequential', self.SEQUENTIAL)) sequential.children.append(private_vertical.location) sequential.children.append(public_vertical.location) self.store.update_item(sequential, self.user.id) # create an html and video component to make drafts: draft_html = self.store.create_item(self.user.id, course_id, 'html', self.DRAFT_HTML) draft_video = self.store.create_item(self.user.id, course_id, 'video', self.DRAFT_VIDEO) # add them as children to the public_vertical public_vertical.children.append(draft_html.location) public_vertical.children.append(draft_video.location) self.store.update_item(public_vertical, self.user.id) # publish changes to vertical self.store.publish(public_vertical.location, self.user.id) # convert html/video to draft self.store.convert_to_draft(draft_html.location, self.user.id) self.store.convert_to_draft(draft_video.location, self.user.id) # lock an asset content_store.set_attr(self.LOCKED_ASSET_KEY, 'locked', True) # create a non-portable link - should be rewritten in new courses html_module = self.store.get_item( course_id.make_usage_key('html', 'nonportable')) new_data = html_module.data = html_module.data.replace( '/static/', '/c4x/{0}/{1}/asset/'.format(course_id.org, course_id.course)) self.store.update_item(html_module, self.user.id) html_module = self.store.get_item(html_module.location) self.assertEqual(new_data, html_module.data) return course_id