def export_courses_to_output_path(output_path):
"""
Export all courses to target directory and return the list of courses which failed to export
"""
content_store = contentstore()
module_store = modulestore()
root_dir = output_path
courses = module_store.get_courses()
course_ids = [x.id for x in courses]
failed_export_courses = []
for course_id in course_ids:
print u"-" * 80
print u"Exporting course id = {0} to {1}".format(course_id, output_path)
try:
course_dir = course_id.to_deprecated_string().replace('/', '...')
export_course_to_xml(module_store, content_store, course_id, root_dir, course_dir)
except Exception as err: # pylint: disable=broad-except
failed_export_courses.append(unicode(course_id))
print u"=" * 30 + u"> Oops, failed to export {0}".format(course_id)
print u"Error:"
print err
return courses, failed_export_courses
def handle(self, *args, **options):
"Execute the command"
if len(args) != 2:
raise CommandError("export requires two arguments: <course id> <output path>")
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
try:
course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
except InvalidKeyError:
raise CommandError("Invalid course_key: '%s'. " % args[0])
if not modulestore().get_course(course_key):
raise CommandError("Course with %s key not found." % args[0])
output_path = args[1]
print "Exporting course id = {0} to {1}".format(course_key, output_path)
if not output_path.endswith('/'):
output_path += '/'
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_course_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def handle(self, *args, **options):
"Execute the command"
if len(args) != 2:
raise CommandError(
"export requires two arguments: <course id> <output path>")
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(
args[0])
output_path = args[1]
print("Exporting course id = {0} to {1}".format(
course_key, output_path))
if not output_path.endswith('/'):
output_path += '/'
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_course_to_xml(modulestore(), contentstore(), course_key,
root_dir, course_dir)
def handle(self, *args, **options):
"""
Given a course id(old or new style), and an output_path folder. Export the
corresponding course from mongo and put it directly in the folder.
"""
try:
course_key = CourseKey.from_string(options['course_id'])
except InvalidKeyError:
raise CommandError(u"Invalid course_key: '%s'." %
options['course_id'])
if not modulestore().get_course(course_key):
raise CommandError(u"Course with %s key not found." %
options['course_id'])
output_path = options['output_path']
print(u"Exporting course id = {0} to {1}".format(
course_key, output_path))
if not output_path.endswith('/'):
output_path += '/'
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_course_to_xml(modulestore(), contentstore(), course_key,
root_dir, course_dir)
def handle(self, *args, **options):
"""Execute the command"""
try:
course_key = CourseKey.from_string(options['course_id'])
except InvalidKeyError:
try:
course_key = SlashSeparatedCourseKey.from_deprecated_string(
options['course_id'])
except InvalidKeyError:
raise CommandError("Invalid course_key: '%s'." %
options['course_id'])
if not modulestore().get_course(course_key):
raise CommandError("Course with %s key not found." %
options['course_id'])
output_path = options['output_path']
print "Exporting course id = {0} to {1}".format(
course_key, output_path)
if not output_path.endswith('/'):
output_path += '/'
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_course_to_xml(modulestore(), contentstore(), course_key,
root_dir, course_dir)
def test_library_content_on_course_export_import(self, publish_item):
"""
Verify that library contents in destination and source courses are same after importing
the source course into destination course.
"""
self._setup_source_course_with_library_content(publish=publish_item)
# Create a course to import source course.
dest_course = CourseFactory.create(
default_store=ModuleStoreEnum.Type.split)
# Export the source course.
export_course_to_xml(
self.store,
contentstore(),
self.source_course.location.course_key,
self.export_dir,
'exported_source_course',
)
# Now, import it back to dest_course.
import_course_from_xml(
self.store,
self.user.id,
self.export_dir,
['exported_source_course'],
static_content_store=contentstore(),
target_id=dest_course.location.course_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
self.assert_problem_display_names(self.source_course.location,
dest_course.location, publish_item)
def test_problem_content_on_course_export_import(self, problem_data,
expected_problem_content):
"""
Verify that problem content in destination matches expected problem content,
specifically concerned with pre tag data with problem.
"""
self._setup_source_course_with_problem_content(problem_data)
dest_course = CourseFactory.create(
default_store=ModuleStoreEnum.Type.split)
export_course_to_xml(
self.store,
contentstore(),
self.source_course.location.course_key,
self.export_dir,
'exported_source_course',
)
import_course_from_xml(
self.store,
self.user.id,
self.export_dir,
['exported_source_course'],
static_content_store=contentstore(),
target_id=dest_course.location.course_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
self.assert_problem_definition(dest_course.location,
expected_problem_content)
def handle(self, *args, **options):
"""
Given a course id(old or new style), and an output_path folder. Export the
corresponding course from mongo and put it directly in the folder.
"""
try:
course_key = CourseKey.from_string(options['course_id'])
except InvalidKeyError:
try:
course_key = SlashSeparatedCourseKey.from_deprecated_string(options['course_id'])
except InvalidKeyError:
raise CommandError("Invalid course_key: '%s'." % options['course_id'])
if not modulestore().get_course(course_key):
raise CommandError("Course with %s key not found." % options['course_id'])
output_path = options['output_path']
print "Exporting course id = {0} to {1}".format(course_key, output_path)
if not output_path.endswith('/'):
output_path += '/'
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_course_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def export_courses_to_output_path(output_path):
"""
Export all courses to target directory and return the list of courses which failed to export
"""
content_store = contentstore()
module_store = modulestore()
root_dir = output_path
courses = module_store.get_courses()
course_ids = [x.id for x in courses]
failed_export_courses = []
for course_id in course_ids:
print("-" * 80)
print(u"Exporting course id = {0} to {1}".format(
course_id, output_path))
try:
course_dir = text_type(course_id).replace('/', '...')
export_course_to_xml(module_store, content_store, course_id,
root_dir, course_dir)
except Exception as err: # pylint: disable=broad-except
failed_export_courses.append(text_type(course_id))
print(u"=" * 30 +
u"> Oops, failed to export {0}".format(course_id))
print("Error:")
print(err)
return courses, failed_export_courses
def create_export_tarball(course_module, course_key, context, status=None):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
if status:
status.set_state(u'Compressing')
status.increment_completed_steps()
LOGGER.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
LOGGER.exception(u'There was an error exporting %s', course_key, exc_info=True)
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'],
'edit_unit_url': context['edit_unit_url']}))
raise
except Exception as exc:
LOGGER.exception('There was an error exporting %s', course_key, exc_info=True)
context.update({
'in_err': True,
'edit_unit_url': None,
'raw_err_msg': str(exc)})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']}))
raise
finally:
if os.path.exists(root_dir / name):
shutil.rmtree(root_dir / name)
return export_file
def create_export_tarball(course_module, course_key, context, status=None):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
if status:
status.set_state(u'Compressing')
status.increment_completed_steps()
LOGGER.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
LOGGER.exception(u'There was an error exporting %s', course_key, exc_info=True)
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'],
'edit_unit_url': context['edit_unit_url']}))
raise
except Exception as exc:
LOGGER.exception(u'There was an error exporting %s', course_key, exc_info=True)
context.update({
'in_err': True,
'edit_unit_url': None,
'raw_err_msg': str(exc)})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']}))
raise
finally:
if os.path.exists(root_dir / name):
shutil.rmtree(root_dir / name)
return export_file
def test_generate_import_export_timings(self, source_ms, dest_ms,
num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
if CodeBlockTimer is None:
pytest.skip("CodeBlockTimer undefined.")
desc = "XMLRoundTrip:{}->{}:{}".format(SHORT_NAME_MAP[source_ms],
SHORT_NAME_MAP[dest_ms],
num_assets)
with CodeBlockTimer(desc):
with CodeBlockTimer("fake_assets"):
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
with source_ms.build() as (source_content, source_store):
with dest_ms.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
with CodeBlockTimer("initial_import"):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
source_dirs=TEST_COURSE,
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with CodeBlockTimer("export"):
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with CodeBlockTimer("second_import"):
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
if CodeBlockTimer is None:
raise SkipTest("CodeBlockTimer undefined.")
desc = "XMLRoundTrip:{}->{}:{}".format(
SHORT_NAME_MAP[source_ms],
SHORT_NAME_MAP[dest_ms],
num_assets
)
with CodeBlockTimer(desc):
with CodeBlockTimer("fake_assets"):
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
with source_ms.build() as (source_content, source_store):
with dest_ms.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
with CodeBlockTimer("initial_import"):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
source_dirs=TEST_COURSE,
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with CodeBlockTimer("export"):
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with CodeBlockTimer("second_import"):
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
def create_export_tarball(course_module, course_key, context):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
logging.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(u'There was an error exporting %s', course_key)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
raise
except Exception as exc:
log.exception('There was an error exporting %s', course_key)
context.update({
'in_err': True,
'unit': None,
'raw_err_msg': str(exc)})
raise
finally:
shutil.rmtree(root_dir / name)
return export_file
def create_export_tarball(course_module, course_key, context):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
logging.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(u'There was an error exporting %s', course_key)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
raise
except Exception as exc:
log.exception('There was an error exporting %s', course_key)
context.update({
'in_err': True,
'unit': None,
'raw_err_msg': str(exc)})
raise
finally:
shutil.rmtree(root_dir / name)
return export_file
def test_course_without_image(self):
"""
Make sure we elegantly passover our code when there isn't a static
image
"""
course = self.draft_store.get_course(CourseKey.from_string('edX/simple_with_draft/2012_Fall'))
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, u'test_export')
self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
self.assertFalse(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
def test_split_course_export_import(self):
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with SPLIT_MODULESTORE_SETUP.build(contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with SPLIT_MODULESTORE_SETUP.build(contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'source', '2015_Fall') # lint-amnesty, pylint: disable=no-member
dest_course_key = dest_store.make_course_key('a', 'dest', '2015_Fall') # lint-amnesty, pylint: disable=no-member
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['split_course_with_static_tabs'],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
EXPORTED_COURSE_DIR_NAME,
)
source_course = source_store.get_course(source_course_key, depth=None, lazy=False) # lint-amnesty, pylint: disable=no-member
assert source_course.url_name == 'course'
export_dir_path = path(self.export_dir)
policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course_key.run
policy_path = policy_dir / 'policy.json'
assert os.path.exists(policy_path)
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=[EXPORTED_COURSE_DIR_NAME],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
dest_course = dest_store.get_course(dest_course_key, depth=None, lazy=False) # lint-amnesty, pylint: disable=no-member
assert dest_course.url_name == 'course'
def test_course_without_image(self):
"""
Make sure we elegantly passover our code when there isn't a static
image
"""
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'simple_with_draft', '2012_Fall'))
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
self.assertFalse(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
def test_split_course_export_import(self):
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with SPLIT_MODULESTORE_SETUP.build(contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with SPLIT_MODULESTORE_SETUP.build(contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'source', '2015_Fall')
dest_course_key = dest_store.make_course_key('a', 'dest', '2015_Fall')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['split_course_with_static_tabs'],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
EXPORTED_COURSE_DIR_NAME,
)
source_course = source_store.get_course(source_course_key, depth=None, lazy=False)
self.assertEqual(source_course.url_name, 'course')
export_dir_path = path(self.export_dir)
policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course_key.run
policy_path = policy_dir / 'policy.json'
self.assertTrue(os.path.exists(policy_path))
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=[EXPORTED_COURSE_DIR_NAME],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
dest_course = dest_store.get_course(dest_course_key, depth=None, lazy=False)
self.assertEqual(dest_course.url_name, 'course')
def test_export_course_with_peer_component(self):
"""
Test export course when link_to_location is given in peer grading interface settings.
"""
name = "export_peer_component"
locations = self._create_test_tree(name)
# Insert the test block directly into the module store
problem_location = Location('edX', 'tree{}'.format(name), name, 'combinedopenended', 'test_peer_problem')
self.draft_store.create_child(
self.dummy_user,
locations["child"],
problem_location.block_type,
block_id=problem_location.block_id
)
interface_location = Location('edX', 'tree{}'.format(name), name, 'peergrading', 'test_peer_interface')
self.draft_store.create_child(
self.dummy_user,
locations["child"],
interface_location.block_type,
block_id=interface_location.block_id
)
self.draft_store._update_single_item(
as_draft(interface_location),
{
'definition.data': {},
'metadata': {
'link_to_location': unicode(problem_location),
'use_for_single_location': True,
},
},
)
component = self.draft_store.get_item(interface_location)
self.assertEqual(unicode(component.link_to_location), unicode(problem_location))
root_dir = path(mkdtemp())
# export_course_to_xml should work.
try:
export_course_to_xml(
self.draft_store, self.content_store, interface_location.course_key,
root_dir, 'test_export'
)
finally:
shutil.rmtree(root_dir)
def test_export_course_with_peer_component(self):
"""
Test export course when link_to_location is given in peer grading interface settings.
"""
name = "export_peer_component"
locations = self._create_test_tree(name)
# Insert the test block directly into the module store
problem_location = Location('edX', 'tree{}'.format(name), name, 'combinedopenended', 'test_peer_problem')
self.draft_store.create_child(
self.dummy_user,
locations["child"],
problem_location.block_type,
block_id=problem_location.block_id
)
interface_location = Location('edX', 'tree{}'.format(name), name, 'peergrading', 'test_peer_interface')
self.draft_store.create_child(
self.dummy_user,
locations["child"],
interface_location.block_type,
block_id=interface_location.block_id
)
self.draft_store._update_single_item(
as_draft(interface_location),
{
'definition.data': {},
'metadata': {
'link_to_location': unicode(problem_location),
'use_for_single_location': True,
},
},
)
component = self.draft_store.get_item(interface_location)
self.assertEqual(unicode(component.link_to_location), unicode(problem_location))
root_dir = path(mkdtemp())
# export_course_to_xml should work.
try:
export_course_to_xml(
self.draft_store, self.content_store, interface_location.course_key,
root_dir, 'test_export'
)
finally:
shutil.rmtree(root_dir)
def test_course_without_image(self):
"""
Make sure we elegantly passover our code when there isn't a static
image
"""
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'simple_with_draft', '2012_Fall'))
root_dir = path(mkdtemp())
try:
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
assert_false(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
finally:
shutil.rmtree(root_dir)
def test_export_course_image_nondefault(self, _from_json):
"""
Make sure that if a non-default image path is specified that we
don't export it to the static default location
"""
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
assert_true(course.course_image, 'just_a_test.jpg')
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
self.assertTrue(path(root_dir / 'test_export/static/just_a_test.jpg').isfile())
self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
def test_course_without_image(self):
"""
Make sure we elegantly passover our code when there isn't a static
image
"""
course = self.draft_store.get_course(SlashSeparatedCourseKey("edX", "simple_with_draft", "2012_Fall"))
root_dir = path(mkdtemp())
try:
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, "test_export")
assert_false(path(root_dir / "test_export/static/images/course_image.jpg").isfile())
assert_false(path(root_dir / "test_export/static/images_course_image.jpg").isfile())
finally:
shutil.rmtree(root_dir)
def test_export_course_image_nondefault(self, _from_json):
"""
Make sure that if a non-default image path is specified that we
don't export it to the static default location
"""
course = self.draft_store.get_course(CourseKey.from_string('edX/toy/2012_Fall'))
assert_true(course.course_image, 'just_a_test.jpg')
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, u'test_export')
self.assertTrue(path(root_dir / 'test_export/static/just_a_test.jpg').isfile())
self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
def test_export_course_image_nondefault(self):
"""
Make sure that if a non-default image path is specified that we
don't export it to the static default location
"""
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
assert_true(course.course_image, 'just_a_test.jpg')
root_dir = path(mkdtemp())
try:
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
assert_true(path(root_dir / 'test_export/static/just_a_test.jpg').isfile())
assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
finally:
shutil.rmtree(root_dir)
def test_export_course_image_nondefault(self):
"""
Make sure that if a non-default image path is specified that we
don't export it to the static default location
"""
course = self.draft_store.get_course(SlashSeparatedCourseKey("edX", "toy", "2012_Fall"))
assert_true(course.course_image, "just_a_test.jpg")
root_dir = path(mkdtemp())
try:
export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, "test_export")
assert_true(path(root_dir / "test_export/static/just_a_test.jpg").isfile())
assert_false(path(root_dir / "test_export/static/images/course_image.jpg").isfile())
finally:
shutil.rmtree(root_dir)
def export_course_to_directory(course_key, root_dir):
"""Export course into a directory"""
store = modulestore()
course = store.get_course(course_key)
if course is None:
raise CommandError("Invalid course_id")
# The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>.
# We represent the first four with \w.
# TODO: Once we support courses with unicode characters, we will need to revisit this.
course_dir = course.url_name
export_course_to_xml(store, None, course.id, root_dir, course_dir)
export_dir = path(root_dir) / course_dir
return export_dir
def test_export_course_image(self, _from_json):
"""
Test to make sure that we have a course image in the contentstore,
then export it to ensure it gets copied to both file locations.
"""
course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
location = course_key.make_asset_key('asset', 'images_course_image.jpg')
# This will raise if the course image is missing
self.content_store.find(location)
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export')
self.assertTrue(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
self.assertTrue(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
def test_export_course_image(self, _from_json):
"""
Test to make sure that we have a course image in the contentstore,
then export it to ensure it gets copied to both file locations.
"""
course_key = CourseKey.from_string('edX/simple/2012_Fall')
location = course_key.make_asset_key('asset', 'images_course_image.jpg')
# This will raise if the course image is missing
self.content_store.find(location)
root_dir = path(mkdtemp())
self.addCleanup(shutil.rmtree, root_dir)
export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, u'test_export')
self.assertTrue(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
self.assertTrue(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
def test_export_course_image(self):
"""
Test to make sure that we have a course image in the contentstore,
then export it to ensure it gets copied to both file locations.
"""
course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
location = course_key.make_asset_key('asset', 'images_course_image.jpg')
# This will raise if the course image is missing
self.content_store.find(location)
root_dir = path(mkdtemp())
try:
export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export')
assert_true(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
assert_true(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
finally:
shutil.rmtree(root_dir)
def test_export_course_image(self):
"""
Test to make sure that we have a course image in the contentstore,
then export it to ensure it gets copied to both file locations.
"""
course_key = SlashSeparatedCourseKey("edX", "simple", "2012_Fall")
location = course_key.make_asset_key("asset", "images_course_image.jpg")
# This will raise if the course image is missing
self.content_store.find(location)
root_dir = path(mkdtemp())
try:
export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, "test_export")
assert_true(path(root_dir / "test_export/static/images/course_image.jpg").isfile())
assert_true(path(root_dir / "test_export/static/images_course_image.jpg").isfile())
finally:
shutil.rmtree(root_dir)
def handle(self, *args, **options):
"Execute the command"
if len(args) != 2:
raise CommandError("export requires two arguments: <course id> <output path>")
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
output_path = args[1]
print("Exporting course id = {0} to {1}".format(course_key, output_path))
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_course_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def export_course_to_directory(course_key, root_dir):
"""Export course into a directory"""
store = modulestore()
course = store.get_course(course_key)
if course is None:
raise CommandError("Invalid course_id")
# The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>.
# We represent the first four with \w.
# TODO: Once we support courses with unicode characters, we will need to revisit this.
replacement_char = u"-"
course_dir = replacement_char.join([course.id.org, course.id.course, course.id.run])
course_dir = re.sub(r"[^\w\.\-]", replacement_char, course_dir)
export_course_to_xml(store, None, course.id, root_dir, course_dir)
export_dir = path(root_dir) / course_dir
return export_dir
def test_import_export(self, store_builder, export_reads, import_reads,
first_import_writes, second_import_writes):
with store_builder.build() as (source_content, source_store):
with store_builder.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
# An extra import write occurs in the first Split import due to the mismatch between
# the course id and the wiki_slug in the test XML course. The course must be updated
# with the correct wiki_slug during import.
with check_mongo_calls(import_reads, first_import_writes):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['manual-testing-complete'],
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with check_mongo_calls(export_reads):
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with check_mongo_calls(import_reads, second_import_writes):
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
def test_export(
self, solution_attribute_value, solution_element_value,
expected_solution_attribute, expected_solution_element):
"""Export the test course with the SGA module"""
course = self.import_test_course(solution_attribute_value, solution_element_value)
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(temp_dir))
store = modulestore()
export_course_to_xml(store, None, course.id, temp_dir, "2017_SGA")
with open(os.path.join(temp_dir, "2017_SGA", "vertical", "vertical.xml")) as f:
content = f.read()
# If both are true the expected output should only have the attribute, since it took precedence
# and the attribute contents are broken XML
assert reformat_xml(content) == reformat_xml(
self.make_test_vertical(expected_solution_attribute, expected_solution_element)
)
def test_export(
self, solution_attribute_value, solution_element_value,
expected_solution_attribute, expected_solution_element):
"""Export the test course with the SGA module"""
course = self.import_test_course(solution_attribute_value, solution_element_value)
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(temp_dir))
store = modulestore()
export_course_to_xml(store, None, course.id, temp_dir, "2017_SGA")
with open(os.path.join(temp_dir, "2017_SGA", "vertical", "vertical.xml")) as f:
content = f.read()
# If both are true the expected output should only have the attribute, since it took precedence
# and the attribute contents are broken XML
assert reformat_xml(content) == reformat_xml(
self.make_test_vertical(expected_solution_attribute, expected_solution_element)
)
def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes):
with store_builder.build() as (source_content, source_store):
with store_builder.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
# An extra import write occurs in the first Split import due to the mismatch between
# the course id and the wiki_slug in the test XML course. The course must be updated
# with the correct wiki_slug during import.
with check_mongo_calls(import_reads, first_import_writes):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['manual-testing-complete'],
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with check_mongo_calls(export_reads):
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with check_mongo_calls(import_reads, second_import_writes):
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
def test_library_content_on_course_export_import(self, publish_item):
"""
Verify that library contents in destination and source courses are same after importing
the source course into destination course.
"""
self._setup_source_course_with_library_content(publish=publish_item)
# Create a course to import source course.
dest_course = CourseFactory.create(default_store=ModuleStoreEnum.Type.split)
# Export the source course.
export_course_to_xml(
self.store,
contentstore(),
self.source_course.location.course_key,
self.export_dir,
'exported_source_course',
)
# Now, import it back to dest_course.
import_course_from_xml(
self.store,
self.user.id,
self.export_dir,
['exported_source_course'],
static_content_store=contentstore(),
target_id=dest_course.location.course_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
self.assert_problem_display_names(
self.source_course.location,
dest_course.location,
publish_item
)
def test_round_trip(self, source_builder, dest_builder,
source_content_builder, dest_content_builder,
course_data_name, _mock_tab_from_json):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(
contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(
contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=[course_data_name],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
EXPORTED_COURSE_DIR_NAME,
)
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=[EXPORTED_COURSE_DIR_NAME],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
# NOT CURRENTLY USED
# export_course_to_xml(
# dest_store,
# dest_content,
# dest_course_key,
# self.export_dir,
# 'exported_dest_course',
# )
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.exclude_field(None, 'parent')
# discussion_ids are auto-generated based on usage_id, so they should change across
# modulestores - see TNL-5001
self.exclude_field(None, 'discussion_id')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
self.assertAssetsMetadataEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
def export_to_git(course_id, repo, user='', rdir=None):
"""Export a course to git."""
# pylint: disable=too-many-statements
if not GIT_REPO_EXPORT_DIR:
raise GitExportError(GitExportError.NO_EXPORT_DIR)
if not os.path.isdir(GIT_REPO_EXPORT_DIR):
raise GitExportError(GitExportError.NO_EXPORT_DIR)
# Check for valid writable git url
if not (repo.endswith('.git') or repo.startswith(
('http:', 'https:', 'file:'))):
raise GitExportError(GitExportError.URL_BAD)
# Check for username and password if using http[s]
if repo.startswith('http:') or repo.startswith('https:'):
parsed = urlparse(repo)
if parsed.username is None or parsed.password is None:
raise GitExportError(GitExportError.URL_NO_AUTH)
if rdir:
rdir = os.path.basename(rdir)
else:
rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
log.debug(u"rdir = %s", rdir)
# Pull or clone repo before exporting to xml
# and update url in case origin changed.
rdirp = '{0}/{1}'.format(GIT_REPO_EXPORT_DIR, rdir)
branch = None
if os.path.exists(rdirp):
log.info('Directory already exists, doing a git reset and pull '
'instead of git clone.')
cwd = rdirp
# Get current branch
cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
try:
branch = cmd_log(cmd, cwd).decode('utf-8').strip('\n')
except subprocess.CalledProcessError as ex:
log.exception(u'Failed to get branch: %r', ex.output)
raise GitExportError(GitExportError.DETACHED_HEAD)
cmds = [
['git', 'remote', 'set-url', 'origin', repo],
['git', 'fetch', 'origin'],
['git', 'reset', '--hard', 'origin/{0}'.format(branch)],
['git', 'pull'],
['git', 'clean', '-d', '-f'],
]
else:
cmds = [['git', 'clone', repo]]
cwd = GIT_REPO_EXPORT_DIR
cwd = os.path.abspath(cwd)
for cmd in cmds:
try:
cmd_log(cmd, cwd)
except subprocess.CalledProcessError as ex:
log.exception(u'Failed to pull git repository: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_PULL)
# export course as xml before commiting and pushing
root_dir = os.path.dirname(rdirp)
course_dir = os.path.basename(rdirp).rsplit('.git', 1)[0]
try:
export_course_to_xml(modulestore(), contentstore(), course_id,
root_dir, course_dir)
except (EnvironmentError, AttributeError):
log.exception('Failed export to xml')
raise GitExportError(GitExportError.XML_EXPORT_FAIL)
# Get current branch if not already set
if not branch:
cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
try:
branch = cmd_log(
cmd, os.path.abspath(rdirp)).decode('utf-8').strip('\n')
except subprocess.CalledProcessError as ex:
log.exception(u'Failed to get branch from freshly cloned repo: %r',
ex.output)
raise GitExportError(GitExportError.MISSING_BRANCH)
# Now that we have fresh xml exported, set identity, add
# everything to git, commit, and push to the right branch.
ident = {}
try:
user = User.objects.get(username=user)
ident['name'] = user.username
ident['email'] = user.email
except User.DoesNotExist:
# That's ok, just use default ident
ident = GIT_EXPORT_DEFAULT_IDENT
time_stamp = timezone.now()
cwd = os.path.abspath(rdirp)
commit_msg = u"Export from Studio at {time_stamp}".format(
time_stamp=time_stamp, )
try:
cmd_log(['git', 'config', 'user.email', ident['email']], cwd)
cmd_log(['git', 'config', 'user.name', ident['name']], cwd)
except subprocess.CalledProcessError as ex:
log.exception(u'Error running git configure commands: %r', ex.output)
raise GitExportError(GitExportError.CONFIG_ERROR)
try:
cmd_log(['git', 'add', '.'], cwd)
cmd_log(['git', 'commit', '-a', '-m', commit_msg], cwd)
except subprocess.CalledProcessError as ex:
log.exception(u'Unable to commit changes: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_COMMIT)
try:
cmd_log(['git', 'push', '-q', 'origin', branch], cwd)
except subprocess.CalledProcessError as ex:
log.exception(u'Error running git push command: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_PUSH)
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=[course_data_name],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
# NOT CURRENTLY USED
# export_course_to_xml(
# dest_store,
# dest_content,
# dest_course_key,
# self.export_dir,
# 'exported_dest_course',
# )
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.exclude_field(None, 'parent')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
self.assertAssetsMetadataEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
def get(self, request, course_key_string):
"""
The restful handler for exporting a full course or content library.
GET
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request
header can specify application/x-tgz via HTTP_ACCEPT, or a query
parameter can be used (?accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails,
a JSON string will be returned which describes the error
"""
redirect_url = request.QUERY_PARAMS.get('redirect', None)
courselike_key = CourseKey.from_string(course_key_string)
library = isinstance(courselike_key, LibraryLocator)
if library:
courselike_module = modulestore().get_library(courselike_key)
else:
courselike_module = modulestore().get_course(courselike_key)
name = courselike_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp_clean())
try:
if library:
export_library_to_xml(modulestore(), contentstore(),
courselike_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(),
courselike_module.id, root_dir, name)
logging.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(u'There was an error exporting course %s',
courselike_key)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(
failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except Exception: # pylint: disable=broad-except
# if we have a nested exception, then we'll show the more
# generic error message
pass
return self._export_error_response(
{
"context_course": str(courselike_module.location),
"error": True,
"error_message": str(exc),
"failed_module":
str(failed_item.location) if failed_item else None,
"unit": str(unit.location) if unit else None
},
redirect_url=redirect_url)
except Exception as exc: # pylint: disable=broad-except
log.exception('There was an error exporting course %s',
courselike_key)
return self._export_error_response(
{
"context_course": courselike_module.url_name,
"error": True,
"error_message": str(exc),
"unit": None
},
redirect_url=redirect_url)
# The course is all set; return the tar.gz
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response['Content-Disposition'] = 'attachment; filename={}'.format(
os.path.basename(export_file.name.encode('utf-8')))
response['Content-Length'] = os.path.getsize(export_file.name)
return response
def test_round_trip(self, source_builder, dest_builder,
source_content_builder, dest_content_builder,
course_data_name):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(
contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(
contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=[course_data_name],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
# NOT CURRENTLY USED
# export_course_to_xml(
# dest_store,
# dest_content,
# dest_course_key,
# self.export_dir,
# 'exported_dest_course',
# )
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.exclude_field(None, 'parent')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
self.assertAssetsMetadataEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
def test_round_trip(
self, source_builder, dest_builder, source_content_builder,
dest_content_builder, course_data_name, _mock_tab_from_json
):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=[course_data_name],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
EXPORTED_COURSE_DIR_NAME,
)
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=[EXPORTED_COURSE_DIR_NAME],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
# NOT CURRENTLY USED
# export_course_to_xml(
# dest_store,
# dest_content,
# dest_course_key,
# self.export_dir,
# 'exported_dest_course',
# )
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.exclude_field(None, 'parent')
# discussion_ids are auto-generated based on usage_id, so they should change across
# modulestores - see TNL-5001
self.exclude_field(None, 'discussion_id')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
self.assertAssetsMetadataEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
def export_to_git(course_id, repo, user='', rdir=None):
"""Export a course to git."""
# pylint: disable=too-many-statements
if not GIT_REPO_EXPORT_DIR:
raise GitExportError(GitExportError.NO_EXPORT_DIR)
if not os.path.isdir(GIT_REPO_EXPORT_DIR):
raise GitExportError(GitExportError.NO_EXPORT_DIR)
# Check for valid writable git url
if not (repo.endswith('.git') or
repo.startswith(('http:', 'https:', 'file:'))):
raise GitExportError(GitExportError.URL_BAD)
# Check for username and password if using http[s]
if repo.startswith('http:') or repo.startswith('https:'):
parsed = urlparse(repo)
if parsed.username is None or parsed.password is None:
raise GitExportError(GitExportError.URL_NO_AUTH)
if rdir:
rdir = os.path.basename(rdir)
else:
rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
log.debug(u"rdir = %s", rdir)
# Pull or clone repo before exporting to xml
# and update url in case origin changed.
rdirp = '{0}/{1}'.format(GIT_REPO_EXPORT_DIR, rdir)
branch = None
if os.path.exists(rdirp):
log.info('Directory already exists, doing a git reset and pull '
'instead of git clone.')
cwd = rdirp
# Get current branch
cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
try:
branch = cmd_log(cmd, cwd).strip('\n')
except subprocess.CalledProcessError as ex:
log.exception(u'Failed to get branch: %r', ex.output)
raise GitExportError(GitExportError.DETACHED_HEAD)
cmds = [
['git', 'remote', 'set-url', 'origin', repo],
['git', 'fetch', 'origin'],
['git', 'reset', '--hard', 'origin/{0}'.format(branch)],
['git', 'pull'],
['git', 'clean', '-d', '-f'],
]
else:
cmds = [['git', 'clone', repo]]
cwd = GIT_REPO_EXPORT_DIR
cwd = os.path.abspath(cwd)
for cmd in cmds:
try:
cmd_log(cmd, cwd)
except subprocess.CalledProcessError as ex:
log.exception(u'Failed to pull git repository: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_PULL)
# export course as xml before commiting and pushing
root_dir = os.path.dirname(rdirp)
course_dir = os.path.basename(rdirp).rsplit('.git', 1)[0]
try:
export_course_to_xml(modulestore(), contentstore(), course_id,
root_dir, course_dir)
except (EnvironmentError, AttributeError):
log.exception('Failed export to xml')
raise GitExportError(GitExportError.XML_EXPORT_FAIL)
# Get current branch if not already set
if not branch:
cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
try:
branch = cmd_log(cmd, os.path.abspath(rdirp)).strip('\n')
except subprocess.CalledProcessError as ex:
log.exception(u'Failed to get branch from freshly cloned repo: %r',
ex.output)
raise GitExportError(GitExportError.MISSING_BRANCH)
# Now that we have fresh xml exported, set identity, add
# everything to git, commit, and push to the right branch.
ident = {}
try:
user = User.objects.get(username=user)
ident['name'] = user.username
ident['email'] = user.email
except User.DoesNotExist:
# That's ok, just use default ident
ident = GIT_EXPORT_DEFAULT_IDENT
time_stamp = timezone.now()
cwd = os.path.abspath(rdirp)
commit_msg = u"Export from Studio at {time_stamp}".format(
time_stamp=time_stamp,
)
try:
cmd_log(['git', 'config', 'user.email', ident['email']], cwd)
cmd_log(['git', 'config', 'user.name', ident['name']], cwd)
except subprocess.CalledProcessError as ex:
log.exception(u'Error running git configure commands: %r', ex.output)
raise GitExportError(GitExportError.CONFIG_ERROR)
try:
cmd_log(['git', 'add', '.'], cwd)
cmd_log(['git', 'commit', '-a', '-m', commit_msg], cwd)
except subprocess.CalledProcessError as ex:
log.exception(u'Unable to commit changes: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_COMMIT)
try:
cmd_log(['git', 'push', '-q', 'origin', branch], cwd)
except subprocess.CalledProcessError as ex:
log.exception(u'Error running git push command: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_PUSH)
def test_round_trip(
self,
source_builder,
dest_builder,
source_content_builder,
dest_content_builder,
course_data_name,
_mock_tab_from_json,
):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key("a", "course", "course")
dest_course_key = dest_store.make_course_key("a", "course", "course")
import_course_from_xml(
source_store,
"test_user",
TEST_DATA_DIR,
source_dirs=[course_data_name],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME
)
import_course_from_xml(
dest_store,
"test_user",
self.export_dir,
source_dirs=[EXPORTED_COURSE_DIR_NAME],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
# NOT CURRENTLY USED
# export_course_to_xml(
# dest_store,
# dest_content,
# dest_course_key,
# self.export_dir,
# 'exported_dest_course',
# )
self.exclude_field(None, "wiki_slug")
self.exclude_field(None, "xml_attributes")
self.exclude_field(None, "parent")
self.ignore_asset_key("_id")
self.ignore_asset_key("uploadDate")
self.ignore_asset_key("content_son")
self.ignore_asset_key("thumbnail_location")
self.assertCoursesEqual(source_store, source_course_key, dest_store, dest_course_key)
self.assertAssetsEqual(source_content, source_course_key, dest_content, dest_course_key)
self.assertAssetsMetadataEqual(source_store, source_course_key, dest_store, dest_course_key)
def export_to_git(course_id, repo, user="", rdir=None):
"""Export a course to git."""
# pylint: disable=too-many-statements
if not GIT_REPO_EXPORT_DIR:
raise GitExportError(GitExportError.NO_EXPORT_DIR)
if not os.path.isdir(GIT_REPO_EXPORT_DIR):
raise GitExportError(GitExportError.NO_EXPORT_DIR)
# Check for valid writable git url
if not (repo.endswith(".git") or repo.startswith(("http:", "https:", "file:"))):
raise GitExportError(GitExportError.URL_BAD)
# Check for username and password if using http[s]
if repo.startswith("http:") or repo.startswith("https:"):
parsed = urlparse(repo)
if parsed.username is None or parsed.password is None:
raise GitExportError(GitExportError.URL_NO_AUTH)
if rdir:
rdir = os.path.basename(rdir)
else:
rdir = repo.rsplit("/", 1)[-1].rsplit(".git", 1)[0]
log.debug("rdir = %s", rdir)
# Pull or clone repo before exporting to xml
# and update url in case origin changed.
rdirp = "{0}/{1}".format(GIT_REPO_EXPORT_DIR, rdir)
branch = None
if os.path.exists(rdirp):
log.info("Directory already exists, doing a git reset and pull " "instead of git clone.")
cwd = rdirp
# Get current branch
cmd = ["git", "symbolic-ref", "--short", "HEAD"]
try:
branch = cmd_log(cmd, cwd).strip("\n")
except subprocess.CalledProcessError as ex:
log.exception("Failed to get branch: %r", ex.output)
raise GitExportError(GitExportError.DETACHED_HEAD)
cmds = [
["git", "remote", "set-url", "origin", repo],
["git", "fetch", "origin"],
["git", "reset", "--hard", "origin/{0}".format(branch)],
["git", "pull"],
["git", "clean", "-d", "-f"],
]
else:
cmds = [["git", "clone", repo]]
cwd = GIT_REPO_EXPORT_DIR
cwd = os.path.abspath(cwd)
for cmd in cmds:
try:
cmd_log(cmd, cwd)
except subprocess.CalledProcessError as ex:
log.exception("Failed to pull git repository: %r", ex.output)
raise GitExportError(GitExportError.CANNOT_PULL)
# export course as xml before commiting and pushing
root_dir = os.path.dirname(rdirp)
course_dir = os.path.basename(rdirp).rsplit(".git", 1)[0]
try:
export_course_to_xml(modulestore(), contentstore(), course_id, root_dir, course_dir)
except (EnvironmentError, AttributeError):
log.exception("Failed export to xml")
raise GitExportError(GitExportError.XML_EXPORT_FAIL)
# Get current branch if not already set
if not branch:
cmd = ["git", "symbolic-ref", "--short", "HEAD"]
try:
branch = cmd_log(cmd, os.path.abspath(rdirp)).strip("\n")
except subprocess.CalledProcessError as ex:
log.exception("Failed to get branch from freshly cloned repo: %r", ex.output)
raise GitExportError(GitExportError.MISSING_BRANCH)
# Now that we have fresh xml exported, set identity, add
# everything to git, commit, and push to the right branch.
ident = {}
try:
user = User.objects.get(username=user)
ident["name"] = user.username
ident["email"] = user.email
except User.DoesNotExist:
# That's ok, just use default ident
ident = GIT_EXPORT_DEFAULT_IDENT
time_stamp = timezone.now()
cwd = os.path.abspath(rdirp)
commit_msg = "Export from Studio at {time_stamp}".format(time_stamp=time_stamp)
try:
cmd_log(["git", "config", "user.email", ident["email"]], cwd)
cmd_log(["git", "config", "user.name", ident["name"]], cwd)
except subprocess.CalledProcessError as ex:
log.exception("Error running git configure commands: %r", ex.output)
raise GitExportError(GitExportError.CONFIG_ERROR)
try:
cmd_log(["git", "add", "."], cwd)
cmd_log(["git", "commit", "-a", "-m", commit_msg], cwd)
except subprocess.CalledProcessError as ex:
log.exception("Unable to commit changes: %r", ex.output)
raise GitExportError(GitExportError.CANNOT_COMMIT)
try:
cmd_log(["git", "push", "-q", "origin", branch], cwd)
except subprocess.CalledProcessError as ex:
log.exception("Error running git push command: %r", ex.output)
raise GitExportError(GitExportError.CANNOT_PUSH)
def get(self, request, course_key_string):
"""
The restful handler for exporting a full course or content library.
GET
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request
header can specify application/x-tgz via HTTP_ACCEPT, or a query
parameter can be used (?accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails,
a JSON string will be returned which describes the error
"""
redirect_url = request.QUERY_PARAMS.get('redirect', None)
courselike_key = CourseKey.from_string(course_key_string)
library = isinstance(courselike_key, LibraryLocator)
if library:
courselike_module = modulestore().get_library(courselike_key)
else:
courselike_module = modulestore().get_course(courselike_key)
name = courselike_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp_clean())
try:
if library:
export_library_to_xml(
modulestore(),
contentstore(),
courselike_key,
root_dir,
name
)
else:
export_course_to_xml(
modulestore(),
contentstore(),
courselike_module.id,
root_dir,
name
)
logging.debug(
u'tar file being generated at %s', export_file.name
)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(
u'There was an error exporting course %s',
courselike_key
)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(
failed_item.location
)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except Exception: # pylint: disable=broad-except
# if we have a nested exception, then we'll show the more
# generic error message
pass
return self._export_error_response(
{
"context_course": str(courselike_module.location),
"error": True,
"error_message": str(exc),
"failed_module":
str(failed_item.location) if failed_item else None,
"unit":
str(unit.location) if unit else None
},
redirect_url=redirect_url
)
except Exception as exc: # pylint: disable=broad-except
log.exception(
'There was an error exporting course %s',
courselike_key
)
return self._export_error_response(
{
"context_course": courselike_module.url_name,
"error": True,
"error_message": str(exc),
"unit": None
},
redirect_url=redirect_url
)
# The course is all set; return the tar.gz
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response['Content-Disposition'] = 'attachment; filename={}'.format(
os.path.basename(
export_file.name.encode('utf-8')
)
)
response['Content-Length'] = os.path.getsize(export_file.name)
return response
