def test_library_import_branch_settings_again(self, branch_setting):
# Construct the contentstore for storing the import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the import (using the previously created contentstore)
with SPLIT_MODULESTORE_SETUP.build(
contentstore=source_content) as source_store:
# Use the test branch setting.
with source_store.branch_setting(branch_setting):
source_library_key = LibraryLocator(org='TestOrg',
library='TestProbs')
extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR))
# the extract_dir needs to be passed as a relative dir to
# import_library_from_xml
extract_dir_relative = path.relpath(
extract_dir, settings.DATA_DIR)
try:
with tarfile.open(
path(TEST_DATA_DIR) / 'imports' /
'library.HhJfPD.tar.gz') as tar:
safetar_extractall(tar, extract_dir)
import_library_from_xml(
source_store,
self.user.id,
settings.GITHUB_REPO_ROOT,
[extract_dir_relative / 'library'],
static_content_store=source_content,
target_id=source_library_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
finally:
shutil.rmtree(extract_dir)
def import_single_library(filename):
print >> sys.stderr, 'IMPORTING library:', filename
no_extension = path.basename(filename).replace('.tar.gz', '')
library_xml_dir = path.join(XML_EXTRACT_DIR, no_extension)
mkdir(library_xml_dir)
subprocess.call(['tar', '-xzf', filename, '-C', library_xml_dir])
with open(path.join(library_xml_dir, 'library/library.xml')) as lib_xml_file:
lib_xml = BeautifulSoup(lib_xml_file.read())
lib_element = lib_xml.find('library')
target_id = LibraryLocator(org=str(lib_element['org']), library=str(lib_element['library']))
print >> sys.stderr, 'IMPORTING library:', target_id
import_library_from_xml(
store=MOD_STORE,
user_id=ModuleStoreEnum.UserID.mgmt_command,
data_dir=DATA_DIR,
source_dirs=[path.join(library_xml_dir, 'library')], # Open edX needs `library` dir
load_error_modules=False,
static_content_store=contentstore(),
verbose=True,
do_import_static=True,
target_id=target_id,
create_if_not_present=True,
)
def test_library_import_branch_settings(self, branch_setting):
"""
Try importing a known good library archive under either branch setting.
The branch setting should have no effect on library import.
"""
with self.store.branch_setting(branch_setting):
library = LibraryFactory.create(modulestore=self.store)
lib_key = library.location.library_key
extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR))
# the extract_dir needs to be passed as a relative dir to
# import_library_from_xml
extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR)
try:
with tarfile.open(
path(TEST_DATA_DIR) / 'imports' /
'library.HhJfPD.tar.gz') as tar:
safetar_extractall(tar, extract_dir)
import_library_from_xml(self.store,
self.user.id,
settings.GITHUB_REPO_ROOT,
[extract_dir_relative / 'library'],
load_error_modules=False,
static_content_store=contentstore(),
target_id=lib_key)
finally:
shutil.rmtree(extract_dir)
def test_library_import_branch_settings_again(self, branch_setting):
# Construct the contentstore for storing the import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the import (using the previously created contentstore)
with SPLIT_MODULESTORE_SETUP.build(contentstore=source_content) as source_store:
# Use the test branch setting.
with source_store.branch_setting(branch_setting):
source_library_key = LibraryLocator(org='TestOrg', library='TestProbs')
extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR))
# the extract_dir needs to be passed as a relative dir to
# import_library_from_xml
extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR)
try:
with tarfile.open(path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar:
safetar_extractall(tar, extract_dir)
import_library_from_xml(
source_store,
self.user.id,
settings.GITHUB_REPO_ROOT,
[extract_dir_relative / 'library'],
static_content_store=source_content,
target_id=source_library_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
finally:
shutil.rmtree(extract_dir)
def test_library_import_branch_settings(self, branch_setting):
"""
Try importing a known good library archive under either branch setting.
The branch setting should have no effect on library import.
"""
with self.store.branch_setting(branch_setting):
library = LibraryFactory.create(modulestore=self.store)
lib_key = library.location.library_key
extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR))
# the extract_dir needs to be passed as a relative dir to
# import_library_from_xml
extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR)
try:
with tarfile.open(path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar:
safetar_extractall(tar, extract_dir)
import_library_from_xml(
self.store,
self.user.id,
settings.GITHUB_REPO_ROOT,
[extract_dir_relative / 'library'],
load_error_modules=False,
static_content_store=contentstore(),
target_id=lib_key
)
finally:
shutil.rmtree(extract_dir)
def test_content_library_export_import(self):
library1 = LibraryFactory.create(modulestore=self.store)
source_library1_key = library1.location.library_key
library2 = LibraryFactory.create(modulestore=self.store)
source_library2_key = library2.location.library_key
import_library_from_xml(
self.store,
'test_user',
TEST_DATA_DIR,
['library_empty_problem'],
static_content_store=contentstore(),
target_id=source_library1_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
export_library_to_xml(
self.store,
contentstore(),
source_library1_key,
self.export_dir,
'exported_source_library',
)
source_library = self.store.get_library(source_library1_key)
self.assertEqual(source_library.url_name, 'library')
# Import the exported library into a different content library.
import_library_from_xml(
self.store,
'test_user',
self.export_dir,
['exported_source_library'],
static_content_store=contentstore(),
target_id=source_library2_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
# Compare the two content libraries for equality.
self.assertCoursesEqual(source_library1_key, source_library2_key)
def test_content_library_export_import(self):
library1 = LibraryFactory.create(modulestore=self.store)
source_library1_key = library1.location.library_key
library2 = LibraryFactory.create(modulestore=self.store)
source_library2_key = library2.location.library_key
import_library_from_xml(
self.store,
'test_user',
TEST_DATA_DIR,
['library_empty_problem'],
static_content_store=contentstore(),
target_id=source_library1_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
export_library_to_xml(
self.store,
contentstore(),
source_library1_key,
self.export_dir,
'exported_source_library',
)
source_library = self.store.get_library(source_library1_key)
self.assertEqual(source_library.url_name, 'library')
# Import the exported library into a different content library.
import_library_from_xml(
self.store,
'test_user',
self.export_dir,
['exported_source_library'],
static_content_store=contentstore(),
target_id=source_library2_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
# Compare the two content libraries for equality.
self.assertCoursesEqual(source_library1_key, source_library2_key)
def test_library_import(self):
"""
Try importing a known good library archive, and verify that the
contents of the library have completely replaced the old contents.
"""
# Create some blocks to overwrite
library = LibraryFactory.create(modulestore=self.store)
lib_key = library.location.library_key
test_block = ItemFactory.create(
category="vertical",
parent_location=library.location,
user_id=self.user.id,
publish_item=False,
)
test_block2 = ItemFactory.create(category="vertical",
parent_location=library.location,
user_id=self.user.id,
publish_item=False)
# Create a library and blocks that should remain unmolested.
unchanged_lib = LibraryFactory.create()
unchanged_key = unchanged_lib.location.library_key
test_block3 = ItemFactory.create(
category="vertical",
parent_location=unchanged_lib.location,
user_id=self.user.id,
publish_item=False)
test_block4 = ItemFactory.create(
category="vertical",
parent_location=unchanged_lib.location,
user_id=self.user.id,
publish_item=False)
# Refresh library.
library = self.store.get_library(lib_key)
children = [
self.store.get_item(child).url_name for child in library.children
]
self.assertEqual(len(children), 2)
self.assertIn(test_block.url_name, children)
self.assertIn(test_block2.url_name, children)
unchanged_lib = self.store.get_library(unchanged_key)
children = [
self.store.get_item(child).url_name
for child in unchanged_lib.children
]
self.assertEqual(len(children), 2)
self.assertIn(test_block3.url_name, children)
self.assertIn(test_block4.url_name, children)
extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR))
# the extract_dir needs to be passed as a relative dir to
# import_library_from_xml
extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR)
try:
with tarfile.open(
path(TEST_DATA_DIR) / 'imports' /
'library.HhJfPD.tar.gz') as tar:
safetar_extractall(tar, extract_dir)
library_items = import_library_from_xml(
self.store,
self.user.id,
settings.GITHUB_REPO_ROOT, [extract_dir_relative / 'library'],
load_error_modules=False,
static_content_store=contentstore(),
target_id=lib_key)
finally:
shutil.rmtree(extract_dir)
self.assertEqual(lib_key, library_items[0].location.library_key)
library = self.store.get_library(lib_key)
children = [
self.store.get_item(child).url_name for child in library.children
]
self.assertEqual(len(children), 3)
self.assertNotIn(test_block.url_name, children)
self.assertNotIn(test_block2.url_name, children)
unchanged_lib = self.store.get_library(unchanged_key)
children = [
self.store.get_item(child).url_name
for child in unchanged_lib.children
]
self.assertEqual(len(children), 2)
self.assertIn(test_block3.url_name, children)
self.assertIn(test_block4.url_name, children)
def handle(self, *args, **options):
"""
Given a content library archive path, import the corresponding course to mongo.
"""
archive_path = options['archive_path']
username = options['owner_username']
data_root = Path(settings.GITHUB_REPO_ROOT)
subdir = base64.urlsafe_b64encode(os.path.basename(archive_path))
course_dir = data_root / subdir
# Extract library archive
tar_file = tarfile.open(archive_path)
try:
safetar_extractall(tar_file, course_dir.encode('utf-8'))
except SuspiciousOperation as exc:
raise CommandError(
u'\n=== Course import {0}: Unsafe tar file - {1}\n'.format(
archive_path, exc.args[0]))
finally:
tar_file.close()
# Paths to the library.xml file
abs_xml_path = os.path.join(course_dir, 'library')
rel_xml_path = os.path.relpath(abs_xml_path, data_root)
# Gather library metadata from XML file
xml_root = etree.parse(abs_xml_path / 'library.xml').getroot()
if xml_root.tag != 'library':
raise CommandError(
u'Failed to import {0}: Not a library archive'.format(
archive_path))
metadata = xml_root.attrib
org = metadata['org']
library = metadata['library']
display_name = metadata['display_name']
# Fetch user and library key
user = User.objects.get(username=username)
courselike_key, created = _get_or_create_library(
org, library, display_name, user)
# Check if data would be overwritten
ans = ''
while not created and ans not in ['y', 'yes', 'n', 'no']:
inp = raw_input(
u'Library "{0}" already exists, overwrite it? [y/n] '.format(
courselike_key))
ans = inp.lower()
if ans.startswith('n'):
print(u'Aborting import of "{0}"'.format(courselike_key))
return
# At last, import the library
try:
import_library_from_xml(modulestore(),
user.id,
settings.GITHUB_REPO_ROOT, [rel_xml_path],
load_error_modules=False,
static_content_store=contentstore(),
target_id=courselike_key)
except Exception:
print(u'\n=== Failed to import library-v1:{0}+{1}'.format(
org, library))
raise
print(u'Library "{0}" imported to "{1}"'.format(
archive_path, courselike_key))
def test_library_import(self):
"""
Try importing a known good library archive, and verify that the
contents of the library have completely replaced the old contents.
"""
# Create some blocks to overwrite
library = LibraryFactory.create(modulestore=self.store)
lib_key = library.location.library_key
test_block = ItemFactory.create(
category="vertical",
parent_location=library.location,
user_id=self.user.id,
publish_item=False,
)
test_block2 = ItemFactory.create(
category="vertical",
parent_location=library.location,
user_id=self.user.id,
publish_item=False
)
# Create a library and blocks that should remain unmolested.
unchanged_lib = LibraryFactory.create()
unchanged_key = unchanged_lib.location.library_key
test_block3 = ItemFactory.create(
category="vertical",
parent_location=unchanged_lib.location,
user_id=self.user.id,
publish_item=False
)
test_block4 = ItemFactory.create(
category="vertical",
parent_location=unchanged_lib.location,
user_id=self.user.id,
publish_item=False
)
# Refresh library.
library = self.store.get_library(lib_key)
children = [self.store.get_item(child).url_name for child in library.children]
self.assertEqual(len(children), 2)
self.assertIn(test_block.url_name, children)
self.assertIn(test_block2.url_name, children)
unchanged_lib = self.store.get_library(unchanged_key)
children = [self.store.get_item(child).url_name for child in unchanged_lib.children]
self.assertEqual(len(children), 2)
self.assertIn(test_block3.url_name, children)
self.assertIn(test_block4.url_name, children)
extract_dir = path(tempfile.mkdtemp(dir=settings.DATA_DIR))
# the extract_dir needs to be passed as a relative dir to
# import_library_from_xml
extract_dir_relative = path.relpath(extract_dir, settings.DATA_DIR)
try:
with tarfile.open(path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz') as tar:
safetar_extractall(tar, extract_dir)
library_items = import_library_from_xml(
self.store,
self.user.id,
settings.GITHUB_REPO_ROOT,
[extract_dir_relative / 'library'],
load_error_modules=False,
static_content_store=contentstore(),
target_id=lib_key
)
finally:
shutil.rmtree(extract_dir)
self.assertEqual(lib_key, library_items[0].location.library_key)
library = self.store.get_library(lib_key)
children = [self.store.get_item(child).url_name for child in library.children]
self.assertEqual(len(children), 3)
self.assertNotIn(test_block.url_name, children)
self.assertNotIn(test_block2.url_name, children)
unchanged_lib = self.store.get_library(unchanged_key)
children = [self.store.get_item(child).url_name for child in unchanged_lib.children]
self.assertEqual(len(children), 2)
self.assertIn(test_block3.url_name, children)
self.assertIn(test_block4.url_name, children)
def handle(self, *args, **options):
"""
Given a content library archive path, import the corresponding course to mongo.
"""
archive_path = options['archive_path']
username = options['owner_username']
data_root = Path(settings.GITHUB_REPO_ROOT)
subdir = base64.urlsafe_b64encode(os.path.basename(archive_path))
course_dir = data_root / subdir
# Extract library archive
tar_file = tarfile.open(archive_path)
try:
safetar_extractall(tar_file, course_dir.encode('utf-8'))
except SuspiciousOperation as exc:
raise CommandError(u'\n=== Course import {0}: Unsafe tar file - {1}\n'.format(archive_path, exc.args[0]))
finally:
tar_file.close()
# Paths to the library.xml file
abs_xml_path = os.path.join(course_dir, 'library')
rel_xml_path = os.path.relpath(abs_xml_path, data_root)
# Gather library metadata from XML file
xml_root = etree.parse(abs_xml_path / 'library.xml').getroot()
if xml_root.tag != 'library':
raise CommandError(u'Failed to import {0}: Not a library archive'.format(archive_path))
metadata = xml_root.attrib
org = metadata['org']
library = metadata['library']
display_name = metadata['display_name']
# Fetch user and library key
user = User.objects.get(username=username)
courselike_key, created = _get_or_create_library(org, library, display_name, user)
# Check if data would be overwritten
ans = ''
while not created and ans not in ['y', 'yes', 'n', 'no']:
inp = raw_input(u'Library "{0}" already exists, overwrite it? [y/n] '.format(courselike_key))
ans = inp.lower()
if ans.startswith('n'):
print(u'Aborting import of "{0}"'.format(courselike_key))
return
# At last, import the library
try:
import_library_from_xml(
modulestore(), user.id,
settings.GITHUB_REPO_ROOT, [rel_xml_path],
load_error_modules=False,
static_content_store=contentstore(),
target_id=courselike_key
)
except Exception:
print(u'\n=== Failed to import library-v1:{0}+{1}'.format(org, library))
raise
print(u'Library "{0}" imported to "{1}"'.format(archive_path, courselike_key))
