def test_library_import_branch_settings_again(self, branch_setting): # Construct the contentstore for storing the import with MongoContentstoreBuilder().build() as source_content: # Construct the modulestore for storing the import (using the previously created contentstore) with SPLIT_MODULESTORE_SETUP.build( contentstore=source_content) as source_store: # Use the test branch setting. with source_store.branch_setting(branch_setting): source_library_key = LibraryLocator(org='TestOrg', library='TestProbs') extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR)) # the extract_dir needs to be passed as a relative dir to # import_library_from_xml extract_dir_relative = path.relpath( extract_dir, settings.DATA_DIR) try: with tarfile.open( path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar: safetar_extractall(tar, extract_dir) import_library_from_xml( source_store, self.user.id, settings.GITHUB_REPO_ROOT, [extract_dir_relative / 'library'], static_content_store=source_content, target_id=source_library_key, load_error_modules=False, raise_on_failure=True, create_if_not_present=True, ) finally: shutil.rmtree(extract_dir)
def import_single_library(filename): print >> sys.stderr, 'IMPORTING library:', filename no_extension = path.basename(filename).replace('.tar.gz', '') library_xml_dir = path.join(XML_EXTRACT_DIR, no_extension) mkdir(library_xml_dir) subprocess.call(['tar', '-xzf', filename, '-C', library_xml_dir]) with open(path.join(library_xml_dir, 'library/library.xml')) as lib_xml_file: lib_xml = BeautifulSoup(lib_xml_file.read()) lib_element = lib_xml.find('library') target_id = LibraryLocator(org=str(lib_element['org']), library=str(lib_element['library'])) print >> sys.stderr, 'IMPORTING library:', target_id import_library_from_xml( store=MOD_STORE, user_id=ModuleStoreEnum.UserID.mgmt_command, data_dir=DATA_DIR, source_dirs=[path.join(library_xml_dir, 'library')], # Open edX needs `library` dir load_error_modules=False, static_content_store=contentstore(), verbose=True, do_import_static=True, target_id=target_id, create_if_not_present=True, )
def test_library_import_branch_settings(self, branch_setting): """ Try importing a known good library archive under either branch setting. The branch setting should have no effect on library import. """ with self.store.branch_setting(branch_setting): library = LibraryFactory.create(modulestore=self.store) lib_key = library.location.library_key extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR)) # the extract_dir needs to be passed as a relative dir to # import_library_from_xml extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR) try: with tarfile.open( path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar: safetar_extractall(tar, extract_dir) import_library_from_xml(self.store, self.user.id, settings.GITHUB_REPO_ROOT, [extract_dir_relative / 'library'], load_error_modules=False, static_content_store=contentstore(), target_id=lib_key) finally: shutil.rmtree(extract_dir)
def test_library_import_branch_settings_again(self, branch_setting): # Construct the contentstore for storing the import with MongoContentstoreBuilder().build() as source_content: # Construct the modulestore for storing the import (using the previously created contentstore) with SPLIT_MODULESTORE_SETUP.build(contentstore=source_content) as source_store: # Use the test branch setting. with source_store.branch_setting(branch_setting): source_library_key = LibraryLocator(org='TestOrg', library='TestProbs') extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR)) # the extract_dir needs to be passed as a relative dir to # import_library_from_xml extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR) try: with tarfile.open(path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar: safetar_extractall(tar, extract_dir) import_library_from_xml( source_store, self.user.id, settings.GITHUB_REPO_ROOT, [extract_dir_relative / 'library'], static_content_store=source_content, target_id=source_library_key, load_error_modules=False, raise_on_failure=True, create_if_not_present=True, ) finally: shutil.rmtree(extract_dir)
def test_library_import_branch_settings(self, branch_setting): """ Try importing a known good library archive under either branch setting. The branch setting should have no effect on library import. """ with self.store.branch_setting(branch_setting): library = LibraryFactory.create(modulestore=self.store) lib_key = library.location.library_key extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR)) # the extract_dir needs to be passed as a relative dir to # import_library_from_xml extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR) try: with tarfile.open(path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar: safetar_extractall(tar, extract_dir) import_library_from_xml( self.store, self.user.id, settings.GITHUB_REPO_ROOT, [extract_dir_relative / 'library'], load_error_modules=False, static_content_store=contentstore(), target_id=lib_key ) finally: shutil.rmtree(extract_dir)
def test_content_library_export_import(self): library1 = LibraryFactory.create(modulestore=self.store) source_library1_key = library1.location.library_key library2 = LibraryFactory.create(modulestore=self.store) source_library2_key = library2.location.library_key import_library_from_xml( self.store, 'test_user', TEST_DATA_DIR, ['library_empty_problem'], static_content_store=contentstore(), target_id=source_library1_key, load_error_modules=False, raise_on_failure=True, create_if_not_present=True, ) export_library_to_xml( self.store, contentstore(), source_library1_key, self.export_dir, 'exported_source_library', ) source_library = self.store.get_library(source_library1_key) self.assertEqual(source_library.url_name, 'library') # Import the exported library into a different content library. import_library_from_xml( self.store, 'test_user', self.export_dir, ['exported_source_library'], static_content_store=contentstore(), target_id=source_library2_key, load_error_modules=False, raise_on_failure=True, create_if_not_present=True, ) # Compare the two content libraries for equality. self.assertCoursesEqual(source_library1_key, source_library2_key)
def test_library_import(self): """ Try importing a known good library archive, and verify that the contents of the library have completely replaced the old contents. """ # Create some blocks to overwrite library = LibraryFactory.create(modulestore=self.store) lib_key = library.location.library_key test_block = ItemFactory.create( category="vertical", parent_location=library.location, user_id=self.user.id, publish_item=False, ) test_block2 = ItemFactory.create(category="vertical", parent_location=library.location, user_id=self.user.id, publish_item=False) # Create a library and blocks that should remain unmolested. unchanged_lib = LibraryFactory.create() unchanged_key = unchanged_lib.location.library_key test_block3 = ItemFactory.create( category="vertical", parent_location=unchanged_lib.location, user_id=self.user.id, publish_item=False) test_block4 = ItemFactory.create( category="vertical", parent_location=unchanged_lib.location, user_id=self.user.id, publish_item=False) # Refresh library. library = self.store.get_library(lib_key) children = [ self.store.get_item(child).url_name for child in library.children ] self.assertEqual(len(children), 2) self.assertIn(test_block.url_name, children) self.assertIn(test_block2.url_name, children) unchanged_lib = self.store.get_library(unchanged_key) children = [ self.store.get_item(child).url_name for child in unchanged_lib.children ] self.assertEqual(len(children), 2) self.assertIn(test_block3.url_name, children) self.assertIn(test_block4.url_name, children) extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR)) # the extract_dir needs to be passed as a relative dir to # import_library_from_xml extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR) try: with tarfile.open( path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar: safetar_extractall(tar, extract_dir) library_items = import_library_from_xml( self.store, self.user.id, settings.GITHUB_REPO_ROOT, [extract_dir_relative / 'library'], load_error_modules=False, static_content_store=contentstore(), target_id=lib_key) finally: shutil.rmtree(extract_dir) self.assertEqual(lib_key, library_items[0].location.library_key) library = self.store.get_library(lib_key) children = [ self.store.get_item(child).url_name for child in library.children ] self.assertEqual(len(children), 3) self.assertNotIn(test_block.url_name, children) self.assertNotIn(test_block2.url_name, children) unchanged_lib = self.store.get_library(unchanged_key) children = [ self.store.get_item(child).url_name for child in unchanged_lib.children ] self.assertEqual(len(children), 2) self.assertIn(test_block3.url_name, children) self.assertIn(test_block4.url_name, children)
def handle(self, *args, **options): """ Given a content library archive path, import the corresponding course to mongo. """ archive_path = options['archive_path'] username = options['owner_username'] data_root = Path(settings.GITHUB_REPO_ROOT) subdir = base64.urlsafe_b64encode(os.path.basename(archive_path)) course_dir = data_root / subdir # Extract library archive tar_file = tarfile.open(archive_path) try: safetar_extractall(tar_file, course_dir.encode('utf-8')) except SuspiciousOperation as exc: raise CommandError( u'\n=== Course import {0}: Unsafe tar file - {1}\n'.format( archive_path, exc.args[0])) finally: tar_file.close() # Paths to the library.xml file abs_xml_path = os.path.join(course_dir, 'library') rel_xml_path = os.path.relpath(abs_xml_path, data_root) # Gather library metadata from XML file xml_root = etree.parse(abs_xml_path / 'library.xml').getroot() if xml_root.tag != 'library': raise CommandError( u'Failed to import {0}: Not a library archive'.format( archive_path)) metadata = xml_root.attrib org = metadata['org'] library = metadata['library'] display_name = metadata['display_name'] # Fetch user and library key user = User.objects.get(username=username) courselike_key, created = _get_or_create_library( org, library, display_name, user) # Check if data would be overwritten ans = '' while not created and ans not in ['y', 'yes', 'n', 'no']: inp = raw_input( u'Library "{0}" already exists, overwrite it? [y/n] '.format( courselike_key)) ans = inp.lower() if ans.startswith('n'): print(u'Aborting import of "{0}"'.format(courselike_key)) return # At last, import the library try: import_library_from_xml(modulestore(), user.id, settings.GITHUB_REPO_ROOT, [rel_xml_path], load_error_modules=False, static_content_store=contentstore(), target_id=courselike_key) except Exception: print(u'\n=== Failed to import library-v1:{0}+{1}'.format( org, library)) raise print(u'Library "{0}" imported to "{1}"'.format( archive_path, courselike_key))
def test_library_import(self): """ Try importing a known good library archive, and verify that the contents of the library have completely replaced the old contents. """ # Create some blocks to overwrite library = LibraryFactory.create(modulestore=self.store) lib_key = library.location.library_key test_block = ItemFactory.create( category="vertical", parent_location=library.location, user_id=self.user.id, publish_item=False, ) test_block2 = ItemFactory.create( category="vertical", parent_location=library.location, user_id=self.user.id, publish_item=False ) # Create a library and blocks that should remain unmolested. unchanged_lib = LibraryFactory.create() unchanged_key = unchanged_lib.location.library_key test_block3 = ItemFactory.create( category="vertical", parent_location=unchanged_lib.location, user_id=self.user.id, publish_item=False ) test_block4 = ItemFactory.create( category="vertical", parent_location=unchanged_lib.location, user_id=self.user.id, publish_item=False ) # Refresh library. library = self.store.get_library(lib_key) children = [self.store.get_item(child).url_name for child in library.children] self.assertEqual(len(children), 2) self.assertIn(test_block.url_name, children) self.assertIn(test_block2.url_name, children) unchanged_lib = self.store.get_library(unchanged_key) children = [self.store.get_item(child).url_name for child in unchanged_lib.children] self.assertEqual(len(children), 2) self.assertIn(test_block3.url_name, children) self.assertIn(test_block4.url_name, children) extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR)) # the extract_dir needs to be passed as a relative dir to # import_library_from_xml extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR) try: with tarfile.open(path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar: safetar_extractall(tar, extract_dir) library_items = import_library_from_xml( self.store, self.user.id, settings.GITHUB_REPO_ROOT, [extract_dir_relative / 'library'], load_error_modules=False, static_content_store=contentstore(), target_id=lib_key ) finally: shutil.rmtree(extract_dir) self.assertEqual(lib_key, library_items[0].location.library_key) library = self.store.get_library(lib_key) children = [self.store.get_item(child).url_name for child in library.children] self.assertEqual(len(children), 3) self.assertNotIn(test_block.url_name, children) self.assertNotIn(test_block2.url_name, children) unchanged_lib = self.store.get_library(unchanged_key) children = [self.store.get_item(child).url_name for child in unchanged_lib.children] self.assertEqual(len(children), 2) self.assertIn(test_block3.url_name, children) self.assertIn(test_block4.url_name, children)
def handle(self, *args, **options): """ Given a content library archive path, import the corresponding course to mongo. """ archive_path = options['archive_path'] username = options['owner_username'] data_root = Path(settings.GITHUB_REPO_ROOT) subdir = base64.urlsafe_b64encode(os.path.basename(archive_path)) course_dir = data_root / subdir # Extract library archive tar_file = tarfile.open(archive_path) try: safetar_extractall(tar_file, course_dir.encode('utf-8')) except SuspiciousOperation as exc: raise CommandError(u'\n=== Course import {0}: Unsafe tar file - {1}\n'.format(archive_path, exc.args[0])) finally: tar_file.close() # Paths to the library.xml file abs_xml_path = os.path.join(course_dir, 'library') rel_xml_path = os.path.relpath(abs_xml_path, data_root) # Gather library metadata from XML file xml_root = etree.parse(abs_xml_path / 'library.xml').getroot() if xml_root.tag != 'library': raise CommandError(u'Failed to import {0}: Not a library archive'.format(archive_path)) metadata = xml_root.attrib org = metadata['org'] library = metadata['library'] display_name = metadata['display_name'] # Fetch user and library key user = User.objects.get(username=username) courselike_key, created = _get_or_create_library(org, library, display_name, user) # Check if data would be overwritten ans = '' while not created and ans not in ['y', 'yes', 'n', 'no']: inp = raw_input(u'Library "{0}" already exists, overwrite it? [y/n] '.format(courselike_key)) ans = inp.lower() if ans.startswith('n'): print(u'Aborting import of "{0}"'.format(courselike_key)) return # At last, import the library try: import_library_from_xml( modulestore(), user.id, settings.GITHUB_REPO_ROOT, [rel_xml_path], load_error_modules=False, static_content_store=contentstore(), target_id=courselike_key ) except Exception: print(u'\n=== Failed to import library-v1:{0}+{1}'.format(org, library)) raise print(u'Library "{0}" imported to "{1}"'.format(archive_path, courselike_key))