def test_library_export(self):
"""
Verify that useable library data can be exported.
"""
youtube_id = "qS4NO9MNC6w"
library = LibraryFactory.create(modulestore=self.store)
video_block = ItemFactory.create(
category="video",
parent_location=library.location,
user_id=self.user.id,
publish_item=False,
youtube_id_1_0=youtube_id
)
name = library.url_name
lib_key = library.location.library_key
root_dir = path(tempfile.mkdtemp())
try:
export_library_to_xml(self.store, contentstore(), lib_key, root_dir, name)
lib_xml = lxml.etree.XML(open(root_dir / name / LIBRARY_ROOT).read())
self.assertEqual(lib_xml.get('org'), lib_key.org)
self.assertEqual(lib_xml.get('library'), lib_key.library)
block = lib_xml.find('video')
self.assertIsNotNone(block)
self.assertEqual(block.get('url_name'), video_block.url_name)
video_xml = lxml.etree.XML(open(root_dir / name / 'video' / video_block.url_name + '.xml').read())
self.assertEqual(video_xml.tag, 'video')
self.assertEqual(video_xml.get('youtube_id_1_0'), youtube_id)
finally:
shutil.rmtree(root_dir / name)
def test_library_export(self):
"""
Verify that useable library data can be exported.
"""
youtube_id = "qS4NO9MNC6w"
library = LibraryFactory.create(modulestore=self.store)
video_block = ItemFactory.create(category="video",
parent_location=library.location,
user_id=self.user.id,
publish_item=False,
youtube_id_1_0=youtube_id)
name = library.url_name
lib_key = library.location.library_key
root_dir = path(tempfile.mkdtemp())
try:
export_library_to_xml(self.store, contentstore(), lib_key,
root_dir, name)
lib_xml = lxml.etree.XML(
open(root_dir / name / LIBRARY_ROOT).read())
self.assertEqual(lib_xml.get('org'), lib_key.org)
self.assertEqual(lib_xml.get('library'), lib_key.library)
block = lib_xml.find('video')
self.assertIsNotNone(block)
self.assertEqual(block.get('url_name'), video_block.url_name)
video_xml = lxml.etree.XML(
open(root_dir / name / 'video' / video_block.url_name +
'.xml').read())
self.assertEqual(video_xml.tag, 'video')
self.assertEqual(video_xml.get('youtube_id_1_0'), youtube_id)
finally:
shutil.rmtree(root_dir / name)
def create_export_tarball(course_module, course_key, context, status=None):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
if status:
status.set_state(u'Compressing')
status.increment_completed_steps()
LOGGER.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
LOGGER.exception(u'There was an error exporting %s', course_key, exc_info=True)
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'],
'edit_unit_url': context['edit_unit_url']}))
raise
except Exception as exc:
LOGGER.exception('There was an error exporting %s', course_key, exc_info=True)
context.update({
'in_err': True,
'edit_unit_url': None,
'raw_err_msg': str(exc)})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']}))
raise
finally:
if os.path.exists(root_dir / name):
shutil.rmtree(root_dir / name)
return export_file
def create_export_tarball(course_module, course_key, context, status=None):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
if status:
status.set_state(u'Compressing')
status.increment_completed_steps()
LOGGER.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
LOGGER.exception(u'There was an error exporting %s', course_key, exc_info=True)
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'],
'edit_unit_url': context['edit_unit_url']}))
raise
except Exception as exc:
LOGGER.exception(u'There was an error exporting %s', course_key, exc_info=True)
context.update({
'in_err': True,
'edit_unit_url': None,
'raw_err_msg': str(exc)})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']}))
raise
finally:
if os.path.exists(root_dir / name):
shutil.rmtree(root_dir / name)
return export_file
def create_export_tarball(course_module, course_key, context):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
logging.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(u'There was an error exporting %s', course_key)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
raise
except Exception as exc:
log.exception('There was an error exporting %s', course_key)
context.update({
'in_err': True,
'unit': None,
'raw_err_msg': str(exc)})
raise
finally:
shutil.rmtree(root_dir / name)
return export_file
def create_export_tarball(course_module, course_key, context):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
logging.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(u'There was an error exporting %s', course_key)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
raise
except Exception as exc:
log.exception('There was an error exporting %s', course_key)
context.update({
'in_err': True,
'unit': None,
'raw_err_msg': str(exc)})
raise
finally:
shutil.rmtree(root_dir / name)
return export_file
def test_content_library_export_import(self):
library1 = LibraryFactory.create(modulestore=self.store)
source_library1_key = library1.location.library_key
library2 = LibraryFactory.create(modulestore=self.store)
source_library2_key = library2.location.library_key
import_library_from_xml(
self.store,
'test_user',
TEST_DATA_DIR,
['library_empty_problem'],
static_content_store=contentstore(),
target_id=source_library1_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
export_library_to_xml(
self.store,
contentstore(),
source_library1_key,
self.export_dir,
'exported_source_library',
)
source_library = self.store.get_library(source_library1_key)
self.assertEqual(source_library.url_name, 'library')
# Import the exported library into a different content library.
import_library_from_xml(
self.store,
'test_user',
self.export_dir,
['exported_source_library'],
static_content_store=contentstore(),
target_id=source_library2_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
# Compare the two content libraries for equality.
self.assertCoursesEqual(source_library1_key, source_library2_key)
def test_content_library_export_import(self):
library1 = LibraryFactory.create(modulestore=self.store)
source_library1_key = library1.location.library_key
library2 = LibraryFactory.create(modulestore=self.store)
source_library2_key = library2.location.library_key
import_library_from_xml(
self.store,
'test_user',
TEST_DATA_DIR,
['library_empty_problem'],
static_content_store=contentstore(),
target_id=source_library1_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
export_library_to_xml(
self.store,
contentstore(),
source_library1_key,
self.export_dir,
'exported_source_library',
)
source_library = self.store.get_library(source_library1_key)
self.assertEqual(source_library.url_name, 'library')
# Import the exported library into a different content library.
import_library_from_xml(
self.store,
'test_user',
self.export_dir,
['exported_source_library'],
static_content_store=contentstore(),
target_id=source_library2_key,
load_error_modules=False,
raise_on_failure=True,
create_if_not_present=True,
)
# Compare the two content libraries for equality.
self.assertCoursesEqual(source_library1_key, source_library2_key)
def get(self, request, course_key_string):
"""
The restful handler for exporting a full course or content library.
GET
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request
header can specify application/x-tgz via HTTP_ACCEPT, or a query
parameter can be used (?accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails,
a JSON string will be returned which describes the error
"""
redirect_url = request.QUERY_PARAMS.get('redirect', None)
courselike_key = CourseKey.from_string(course_key_string)
library = isinstance(courselike_key, LibraryLocator)
if library:
courselike_module = modulestore().get_library(courselike_key)
else:
courselike_module = modulestore().get_course(courselike_key)
name = courselike_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp_clean())
try:
if library:
export_library_to_xml(modulestore(), contentstore(),
courselike_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(),
courselike_module.id, root_dir, name)
logging.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(u'There was an error exporting course %s',
courselike_key)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(
failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except Exception: # pylint: disable=broad-except
# if we have a nested exception, then we'll show the more
# generic error message
pass
return self._export_error_response(
{
"context_course": str(courselike_module.location),
"error": True,
"error_message": str(exc),
"failed_module":
str(failed_item.location) if failed_item else None,
"unit": str(unit.location) if unit else None
},
redirect_url=redirect_url)
except Exception as exc: # pylint: disable=broad-except
log.exception('There was an error exporting course %s',
courselike_key)
return self._export_error_response(
{
"context_course": courselike_module.url_name,
"error": True,
"error_message": str(exc),
"unit": None
},
redirect_url=redirect_url)
# The course is all set; return the tar.gz
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response['Content-Disposition'] = 'attachment; filename={}'.format(
os.path.basename(export_file.name.encode('utf-8')))
response['Content-Length'] = os.path.getsize(export_file.name)
return response
def get(self, request, course_key_string):
"""
The restful handler for exporting a full course or content library.
GET
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request
header can specify application/x-tgz via HTTP_ACCEPT, or a query
parameter can be used (?accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails,
a JSON string will be returned which describes the error
"""
redirect_url = request.QUERY_PARAMS.get('redirect', None)
courselike_key = CourseKey.from_string(course_key_string)
library = isinstance(courselike_key, LibraryLocator)
if library:
courselike_module = modulestore().get_library(courselike_key)
else:
courselike_module = modulestore().get_course(courselike_key)
name = courselike_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp_clean())
try:
if library:
export_library_to_xml(
modulestore(),
contentstore(),
courselike_key,
root_dir,
name
)
else:
export_course_to_xml(
modulestore(),
contentstore(),
courselike_module.id,
root_dir,
name
)
logging.debug(
u'tar file being generated at %s', export_file.name
)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(
u'There was an error exporting course %s',
courselike_key
)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(
failed_item.location
)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except Exception: # pylint: disable=broad-except
# if we have a nested exception, then we'll show the more
# generic error message
pass
return self._export_error_response(
{
"context_course": str(courselike_module.location),
"error": True,
"error_message": str(exc),
"failed_module":
str(failed_item.location) if failed_item else None,
"unit":
str(unit.location) if unit else None
},
redirect_url=redirect_url
)
except Exception as exc: # pylint: disable=broad-except
log.exception(
'There was an error exporting course %s',
courselike_key
)
return self._export_error_response(
{
"context_course": courselike_module.url_name,
"error": True,
"error_message": str(exc),
"unit": None
},
redirect_url=redirect_url
)
# The course is all set; return the tar.gz
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response['Content-Disposition'] = 'attachment; filename={}'.format(
os.path.basename(
export_file.name.encode('utf-8')
)
)
response['Content-Length'] = os.path.getsize(export_file.name)
return response
