def test_generate_timings(self, source_ms, dest_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
desc = "XMLRoundTrip:{}->{}:{}".format(SHORT_NAME_MAP[source_ms],
SHORT_NAME_MAP[dest_ms],
num_assets)
with CodeBlockTimer(desc):
with CodeBlockTimer("fake_assets"):
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_ms.build(source_content) as source_store:
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_ms.build(dest_content) as dest_store:
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
with CodeBlockTimer("initial_import"):
import_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
course_dirs=TEST_COURSE,
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
with CodeBlockTimer("export"):
export_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with CodeBlockTimer("second_import"):
import_from_xml(
dest_store,
'test_user',
self.export_dir,
course_dirs=['exported_source_course'],
static_content_store=dest_content,
target_course_id=dest_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
def handle(self, *args, **options):
"Execute the command"
if len(args) != 1:
raise CommandError("export requires one argument: <output path>")
output_path = args[0]
cs = contentstore()
ms = modulestore('direct')
root_dir = output_path
courses = ms.get_courses()
print("%d courses to export:" % len(courses))
cids = [x.id for x in courses]
print(cids)
for course_id in cids:
print("-"*77)
print("Exporting course id = {0} to {1}".format(course_id, output_path))
if 1:
try:
course_dir = course_id.replace('/', '...')
export_to_xml(ms, cs, course_id, root_dir, course_dir, modulestore())
except Exception as err:
print("="*30 + "> Oops, failed to export %s" % course_id)
print("Error:")
print(err)
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("export requires one argument: <output path>")
output_path = args[0]
cs = contentstore()
ms = modulestore('direct')
root_dir = output_path
courses = ms.get_courses()
print "%d courses to export:" % len(courses)
cids = [x.id for x in courses]
print cids
for course_id in cids:
print "-" * 77
print "Exporting course id = {0} to {1}".format(
course_id, output_path)
if 1:
try:
location = CourseDescriptor.id_to_location(course_id)
course_dir = course_id.replace('/', '...')
export_to_xml(ms, cs, location, root_dir, course_dir,
modulestore())
except Exception as err:
print "=" * 30 + "> Oops, failed to export %s" % course_id
print "Error:"
print err
def generate_export_course(request, org, course, name):
"""
This method will serialize out a course to a .tar.gz file which contains a XML-based representation of
the course
"""
location = get_location_and_verify_access(request, org, course, name)
loc = Location(location)
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
# export out to a tempdir
logging.debug('root = {0}'.format(root_dir))
export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore())
logging.debug('tar file being generated at {0}'.format(export_file.name))
tar_file = tarfile.open(name=export_file.name, mode='w:gz')
tar_file.add(root_dir / name, arcname=name)
tar_file.close()
# remove temp dir
shutil.rmtree(root_dir / name)
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name)
response['Content-Length'] = os.path.getsize(export_file.name)
return response
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("export requires one argument: <output path>")
output_path = args[0]
cs = contentstore()
ms = modulestore("direct")
root_dir = output_path
courses = ms.get_courses()
print "%d courses to export:" % len(courses)
cids = [x.id for x in courses]
print cids
for course_id in cids:
print "-" * 77
print "Exporting course id = {0} to {1}".format(course_id, output_path)
if 1:
try:
location = CourseDescriptor.id_to_location(course_id)
course_dir = course_id.replace("/", "...")
export_to_xml(ms, cs, location, root_dir, course_dir, modulestore())
except Exception as err:
print "=" * 30 + "> Oops, failed to export %s" % course_id
print "Error:"
print err
def generate_export_course(request, org, course, name):
location = get_location_and_verify_access(request, org, course, name)
loc = Location(location)
export_file = NamedTemporaryFile(prefix=name + ".", suffix=".tar.gz")
root_dir = path(mkdtemp())
# export out to a tempdir
logging.debug("root = {0}".format(root_dir))
export_to_xml(modulestore("direct"), contentstore(), loc, root_dir, name, modulestore())
# filename = root_dir / name + '.tar.gz'
logging.debug("tar file being generated at {0}".format(export_file.name))
tar_file = tarfile.open(name=export_file.name, mode="w:gz")
tar_file.add(root_dir / name, arcname=name)
tar_file.close()
# remove temp dir
shutil.rmtree(root_dir / name)
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type="application/x-tgz")
response["Content-Disposition"] = "attachment; filename=%s" % os.path.basename(export_file.name)
response["Content-Length"] = os.path.getsize(export_file.name)
return response
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(dest_content) as dest_store:
source_course_key = source_store.make_course_key('source', 'course', 'key')
dest_course_key = dest_store.make_course_key('dest', 'course', 'key')
import_from_xml(
source_store,
'test_user',
'common/test/data',
course_dirs=[course_data_name],
static_content_store=source_content,
target_course_id=source_course_key,
create_new_course_if_not_present=True,
)
export_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_course',
)
import_from_xml(
dest_store,
'test_user',
self.export_dir,
static_content_store=dest_content,
target_course_id=dest_course_key,
create_new_course_if_not_present=True,
)
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
def export_courses_to_output_path(output_path):
"""
Export all courses to target directory and return the list of courses which failed to export
"""
content_store = contentstore()
module_store = modulestore()
root_dir = output_path
courses = module_store.get_courses()
course_ids = [x.id for x in courses]
failed_export_courses = []
for course_id in course_ids:
print(u"-" * 80)
print(u"Exporting course id = {0} to {1}".format(course_id, output_path))
try:
course_dir = course_id.to_deprecated_string().replace('/', '...')
export_to_xml(module_store, content_store, course_id, root_dir, course_dir)
except Exception as err: # pylint: disable=broad-except
failed_export_courses.append(unicode(course_id))
print(u"=" * 30 + u"> Oops, failed to export {0}".format(course_id))
print(u"Error:")
print(err)
return courses, failed_export_courses
def generate_export_course(request, org, course, name):
"""
This method will serialize out a course to a .tar.gz file which contains a
XML-based representation of the course
"""
location = get_location_and_verify_access(request, org, course, name)
course_module = modulestore().get_instance(location.course_id, location)
loc = Location(location)
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
new_location = loc_mapper().translate_location(
course_module.location.course_id, course_module.location, False, True)
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore('direct'), contentstore(), loc, root_dir,
name, modulestore())
except SerializationError, e:
logging.exception(
'There was an error exporting course {0}. {1}'.format(
course_module.location, unicode(e)))
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_instance(
course_module.location.course_id, e.location)
parent_locs = modulestore().get_parent_locations(
failed_item.location, course_module.location.course_id)
if len(parent_locs) > 0:
parent = modulestore().get_item(parent_locs[0])
if parent.location.category == 'vertical':
unit = parent
except:
# if we have a nested exception, then we'll show the more generic error message
pass
return render_to_response(
'export.html', {
'context_course':
course_module,
'successful_import_redirect_url':
'',
'in_err':
True,
'raw_err_msg':
str(e),
'failed_module':
failed_item,
'unit':
unit,
'edit_unit_url':
reverse('edit_unit', kwargs={'location': parent.location})
if parent else '',
'course_home_url':
new_location.url_reverse("course/", "")
})
def test_generate_import_export_timings(self, source_ms, dest_ms,
num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
if CodeBlockTimer is None:
raise SkipTest("CodeBlockTimer undefined.")
desc = "XMLRoundTrip:{}->{}:{}".format(SHORT_NAME_MAP[source_ms],
SHORT_NAME_MAP[dest_ms],
num_assets)
with CodeBlockTimer(desc):
with CodeBlockTimer("fake_assets"):
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
with source_ms.build() as (source_content, source_store):
with dest_ms.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
with CodeBlockTimer("initial_import"):
import_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
course_dirs=TEST_COURSE,
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
with CodeBlockTimer("export"):
export_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with CodeBlockTimer("second_import"):
import_from_xml(
dest_store,
'test_user',
self.export_dir,
course_dirs=['exported_source_course'],
static_content_store=dest_content,
target_course_id=dest_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
def export_course_to_directory(course_id, root_dir):
"""Export course into a directory"""
store = modulestore()
course = store.get_course(course_id)
if course is None:
raise CommandError("Invalid course_id")
course_name = course.location.course_id.replace('/', '-')
export_to_xml(store, None, course.location, root_dir, course_name)
course_dir = path(root_dir) / course_name
return course_dir
def export_course_to_directory(course_id, root_dir):
"""Export course into a directory"""
store = modulestore()
course = store.get_course(course_id)
if course is None:
raise CommandError("Invalid course_id")
course_name = course.id.to_deprecated_string().replace('/', '-')
export_to_xml(store, None, course.id, root_dir, course_name)
course_dir = path(root_dir) / course_name
return course_dir
def test_course_without_image(self):
"""
Make sure we elegantly passover our code when there isn't a static
image
"""
course = self.get_course_by_id('edX/simple_with_draft/2012_Fall')
root_dir = path(mkdtemp())
try:
export_to_xml(self.store, self.content_store, course.location, root_dir, 'test_export')
assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
assert_false(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
finally:
shutil.rmtree(root_dir)
def test_course_without_image(self):
"""
Make sure we elegantly passover our code when there isn't a static
image
"""
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'simple_with_draft', '2012_Fall'))
root_dir = path(mkdtemp())
try:
export_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
assert_false(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
finally:
shutil.rmtree(root_dir)
def test_course_without_image(self):
"""
Make sure we elegantly passover our code when there isn't a static
image
"""
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'simple_with_draft', '2012_Fall'))
root_dir = path(mkdtemp())
try:
export_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
assert_false(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
finally:
shutil.rmtree(root_dir)
def test_export_course_with_peer_component(self):
"""
Test export course when link_to_location is given in peer grading interface settings.
"""
name = "export_peer_component"
locations = self._create_test_tree(name)
# Insert the test block directly into the module store
problem_location = Location('edX', 'tree{}'.format(name), name,
'combinedopenended', 'test_peer_problem')
self.draft_store.create_child(self.dummy_user,
locations["child"],
problem_location.block_type,
block_id=problem_location.block_id)
interface_location = Location('edX', 'tree{}'.format(name), name,
'peergrading', 'test_peer_interface')
self.draft_store.create_child(self.dummy_user,
locations["child"],
interface_location.block_type,
block_id=interface_location.block_id)
self.draft_store._update_single_item(
as_draft(interface_location),
{
'definition.data': {},
'metadata': {
'link_to_location': unicode(problem_location),
'use_for_single_location': True,
},
},
)
component = self.draft_store.get_item(interface_location)
self.assertEqual(unicode(component.link_to_location),
unicode(problem_location))
root_dir = path(mkdtemp())
# export_to_xml should work.
try:
export_to_xml(self.draft_store, self.content_store,
interface_location.course_key, root_dir,
'test_export')
finally:
shutil.rmtree(root_dir)
def test_export_course_with_peer_component(self):
"""
Test export course when link_to_location is given in peer grading interface settings.
"""
name = "export_peer_component"
locations = self._create_test_tree(name)
# Insert the test block directly into the module store
problem_location = Location('edX', 'tree{}'.format(name), name, 'combinedopenended', 'test_peer_problem')
self.draft_store.create_child(
self.dummy_user,
locations["child"],
problem_location.block_type,
block_id=problem_location.block_id
)
interface_location = Location('edX', 'tree{}'.format(name), name, 'peergrading', 'test_peer_interface')
self.draft_store.create_child(
self.dummy_user,
locations["child"],
interface_location.block_type,
block_id=interface_location.block_id
)
self.draft_store._update_single_item(
as_draft(interface_location),
{
'definition.data': {},
'metadata': {
'link_to_location': unicode(problem_location),
'use_for_single_location': True,
},
},
)
component = self.draft_store.get_item(interface_location)
self.assertEqual(unicode(component.link_to_location), unicode(problem_location))
root_dir = path(mkdtemp())
# export_to_xml should work.
try:
export_to_xml(self.draft_store, self.content_store, interface_location.course_key, root_dir, 'test_export')
finally:
shutil.rmtree(root_dir)
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError("import requires two arguments: <course location> <output path>")
course_id = args[0]
output_path = args[1]
print "Exporting course id = {0} to {1}".format(course_id, output_path)
location = CourseDescriptor.id_to_location(course_id)
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_to_xml(modulestore('direct'), contentstore(), location, root_dir, course_dir)
def test_export_course_image_nondefault(self):
"""
Make sure that if a non-default image path is specified that we
don't export it to the static default location
"""
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
assert_true(course.course_image, 'just_a_test.jpg')
root_dir = path(mkdtemp())
try:
export_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
assert_true(path(root_dir / 'test_export/static/just_a_test.jpg').isfile())
assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
finally:
shutil.rmtree(root_dir)
def test_export_course_image_nondefault(self):
"""
Make sure that if a non-default image path is specified that we
don't export it to the static default location
"""
course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
assert_true(course.course_image, 'just_a_test.jpg')
root_dir = path(mkdtemp())
try:
export_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export')
assert_true(path(root_dir / 'test_export/static/just_a_test.jpg').isfile())
assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
finally:
shutil.rmtree(root_dir)
def generate_export_course(request, org, course, name):
"""
This method will serialize out a course to a .tar.gz file which contains a XML-based representation of
the course
"""
location = get_location_and_verify_access(request, org, course, name)
course_module = modulestore().get_instance(location.course_id, location)
loc = Location(location)
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore())
except SerializationError, e:
logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e)))
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_instance(course_module.location.course_id, e.location)
parent_locs = modulestore().get_parent_locations(failed_item.location, course_module.location.course_id)
if len(parent_locs) > 0:
parent = modulestore().get_item(parent_locs[0])
if parent.location.category == 'vertical':
unit = parent
except:
# if we have a nested exception, then we'll show the more generic error message
pass
return render_to_response('export.html', {
'context_course': course_module,
'successful_import_redirect_url': '',
'in_err': True,
'raw_err_msg': str(e),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': reverse('edit_unit', kwargs={
'location': parent.location
}) if parent else '',
'course_home_url': reverse('course_index', kwargs={
'org': org,
'course': course,
'name': name
})
})
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError(
"import requires two arguments: <course location> <output path>")
course_id = args[0]
output_path = args[1]
print "Exporting course id = {0} to {1}".format(course_id, output_path)
location = CourseDescriptor.id_to_location(course_id)
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_to_xml(modulestore(
'direct'), contentstore(), location, root_dir, course_dir)
def handle(self, *args, **options):
"Execute the command"
if len(args) != 2:
raise CommandError("export requires two arguments: <course id> <output path>")
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
output_path = args[1]
print("Exporting course id = {0} to {1}".format(course_key, output_path))
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_to_xml(modulestore('direct'), contentstore(), course_key, root_dir, course_dir, modulestore())
def test_export_course_image(self):
"""
Test to make sure that we have a course image in the contentstore,
then export it to ensure it gets copied to both file locations.
"""
course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
location = course_key.make_asset_key('asset', 'images_course_image.jpg')
# This will raise if the course image is missing
self.content_store.find(location)
root_dir = path(mkdtemp())
try:
export_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export')
assert_true(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
assert_true(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
finally:
shutil.rmtree(root_dir)
def export_course_to_directory(course_key, root_dir):
"""Export course into a directory"""
store = modulestore()
course = store.get_course(course_key)
if course is None:
raise CommandError("Invalid course_id")
# The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>.
# We represent the first four with \w.
# TODO: Once we support courses with unicode characters, we will need to revisit this.
replacement_char = u'-'
course_dir = replacement_char.join([course.id.org, course.id.course, course.id.run])
course_dir = re.sub(r'[^\w\.\-]', replacement_char, course_dir)
export_to_xml(store, None, course.id, root_dir, course_dir)
export_dir = path(root_dir) / course_dir
return export_dir
def test_export_course_image(self):
"""
Test to make sure that we have a course image in the contentstore,
then export it to ensure it gets copied to both file locations.
"""
course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
location = course_key.make_asset_key('asset', 'images_course_image.jpg')
# This will raise if the course image is missing
self.content_store.find(location)
root_dir = path(mkdtemp())
try:
export_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export')
assert_true(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
assert_true(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
finally:
shutil.rmtree(root_dir)
def handle(self, *args, **options):
"Execute the command"
if len(args) != 2:
raise CommandError("export requires two arguments: <course id> <output path>")
try:
course_key = CourseKey.from_string(args[0])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
output_path = args[1]
print("Exporting course id = {0} to {1}".format(course_key, output_path))
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def export_course_to_directory(course_key, root_dir):
"""Export course into a directory"""
store = modulestore()
course = store.get_course(course_key)
if course is None:
raise CommandError("Invalid course_id")
# The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>.
# We represent the first four with \w.
# TODO: Once we support courses with unicode characters, we will need to revisit this.
replacement_char = u'-'
course_dir = replacement_char.join([course.id.org, course.id.course, course.id.run])
course_dir = re.sub(r'[^\w\.\-]', replacement_char, course_dir)
export_to_xml(store, None, course.id, root_dir, course_dir)
export_dir = path(root_dir) / course_dir
return export_dir
def test_import_export(self, store_builder, export_reads, import_reads,
first_import_writes, second_import_writes):
with store_builder.build() as (source_content, source_store):
with store_builder.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
# An extra import write occurs in the first Split import due to the mismatch between
# the course id and the wiki_slug in the test XML course. The course must be updated
# with the correct wiki_slug during import.
with check_mongo_calls(import_reads, first_import_writes):
import_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
course_dirs=['manual-testing-complete'],
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
with check_mongo_calls(export_reads):
export_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with check_mongo_calls(import_reads, second_import_writes):
import_from_xml(
dest_store,
'test_user',
self.export_dir,
course_dirs=['exported_source_course'],
static_content_store=dest_content,
target_course_id=dest_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
def test_export_course_with_unknown_metadata(self):
module_store = modulestore('direct')
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['full'])
location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
root_dir = path(mkdtemp_clean())
course = module_store.get_item(location)
metadata = own_metadata(course)
# add a bool piece of unknown metadata so we can verify we don't throw an exception
metadata['new_metadata'] = True
module_store.update_metadata(location, metadata)
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_to_xml(module_store, content_store, location, root_dir, 'test_export')
def export_to_git(course_loc, repo, user='', rdir=None):
"""Export a course to git."""
# pylint: disable=R0915
if course_loc.startswith('i4x://'):
course_loc = course_loc[6:]
if not GIT_REPO_EXPORT_DIR:
raise GitExportError(GitExportError.NO_EXPORT_DIR)
if not os.path.isdir(GIT_REPO_EXPORT_DIR):
raise GitExportError(GitExportError.NO_EXPORT_DIR)
# Check for valid writable git url
if not (repo.endswith('.git') or repo.startswith(
('http:', 'https:', 'file:'))):
raise GitExportError(GitExportError.URL_BAD)
# Check for username and password if using http[s]
if repo.startswith('http:') or repo.startswith('https:'):
parsed = urlparse(repo)
if parsed.username is None or parsed.password is None:
raise GitExportError(GitExportError.URL_NO_AUTH)
if rdir:
rdir = os.path.basename(rdir)
else:
rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
log.debug("rdir = %s", rdir)
# Pull or clone repo before exporting to xml
# and update url in case origin changed.
rdirp = '{0}/{1}'.format(GIT_REPO_EXPORT_DIR, rdir)
branch = None
if os.path.exists(rdirp):
log.info(
_('Directory already exists, doing a git reset and pull '
'instead of git clone.'))
cwd = rdirp
# Get current branch
cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
try:
branch = cmd_log(cmd, cwd).strip('\n')
except subprocess.CalledProcessError as ex:
log.exception('Failed to get branch: %r', ex.output)
raise GitExportError(GitExportError.DETACHED_HEAD)
cmds = [
['git', 'remote', 'set-url', 'origin', repo],
['git', 'fetch', 'origin'],
['git', 'reset', '--hard', 'origin/{0}'.format(branch)],
['git', 'pull'],
]
else:
cmds = [['git', 'clone', repo]]
cwd = GIT_REPO_EXPORT_DIR
cwd = os.path.abspath(cwd)
for cmd in cmds:
try:
cmd_log(cmd, cwd)
except subprocess.CalledProcessError as ex:
log.exception('Failed to pull git repository: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_PULL)
# export course as xml before commiting and pushing
try:
location = CourseDescriptor.id_to_location(course_loc)
except ValueError:
raise GitExportError(GitExportError.BAD_COURSE)
root_dir = os.path.dirname(rdirp)
course_dir = os.path.splitext(os.path.basename(rdirp))[0]
try:
export_to_xml(modulestore('direct'), contentstore(), location,
root_dir, course_dir, modulestore())
except (EnvironmentError, AttributeError):
log.exception('Failed export to xml')
raise GitExportError(GitExportError.XML_EXPORT_FAIL)
# Get current branch if not already set
if not branch:
cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
try:
branch = cmd_log(cmd, os.path.abspath(rdirp)).strip('\n')
except subprocess.CalledProcessError as ex:
log.exception('Failed to get branch from freshly cloned repo: %r',
ex.output)
raise GitExportError(GitExportError.MISSING_BRANCH)
# Now that we have fresh xml exported, set identity, add
# everything to git, commit, and push to the right branch.
ident = {}
try:
user = User.objects.get(username=user)
ident['name'] = user.username
ident['email'] = user.email
except User.DoesNotExist:
# That's ok, just use default ident
ident = GIT_EXPORT_DEFAULT_IDENT
time_stamp = timezone.now()
cwd = os.path.abspath(rdirp)
commit_msg = 'Export from Studio at {1}'.format(user, time_stamp)
try:
cmd_log(['git', 'config', 'user.email', ident['email']], cwd)
cmd_log(['git', 'config', 'user.name', ident['name']], cwd)
except subprocess.CalledProcessError as ex:
log.exception('Error running git configure commands: %r', ex.output)
raise GitExportError(GitExportError.CONFIG_ERROR)
try:
cmd_log(['git', 'add', '.'], cwd)
cmd_log(['git', 'commit', '-a', '-m', commit_msg], cwd)
except subprocess.CalledProcessError as ex:
log.exception('Unable to commit changes: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_COMMIT)
try:
cmd_log(['git', 'push', '-q', 'origin', branch], cwd)
except subprocess.CalledProcessError as ex:
log.exception('Error running git push command: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_PUSH)
def export_to_git(course_id, repo, user='', rdir=None):
"""Export a course to git."""
# pylint: disable=R0915
if not GIT_REPO_EXPORT_DIR:
raise GitExportError(GitExportError.NO_EXPORT_DIR)
if not os.path.isdir(GIT_REPO_EXPORT_DIR):
raise GitExportError(GitExportError.NO_EXPORT_DIR)
# Check for valid writable git url
if not (repo.endswith('.git') or
repo.startswith(('http:', 'https:', 'file:'))):
raise GitExportError(GitExportError.URL_BAD)
# Check for username and password if using http[s]
if repo.startswith('http:') or repo.startswith('https:'):
parsed = urlparse(repo)
if parsed.username is None or parsed.password is None:
raise GitExportError(GitExportError.URL_NO_AUTH)
if rdir:
rdir = os.path.basename(rdir)
else:
rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
log.debug("rdir = %s", rdir)
# Pull or clone repo before exporting to xml
# and update url in case origin changed.
rdirp = '{0}/{1}'.format(GIT_REPO_EXPORT_DIR, rdir)
branch = None
if os.path.exists(rdirp):
log.info('Directory already exists, doing a git reset and pull '
'instead of git clone.')
cwd = rdirp
# Get current branch
cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
try:
branch = cmd_log(cmd, cwd).strip('\n')
except subprocess.CalledProcessError as ex:
log.exception('Failed to get branch: %r', ex.output)
raise GitExportError(GitExportError.DETACHED_HEAD)
cmds = [
['git', 'remote', 'set-url', 'origin', repo],
['git', 'fetch', 'origin'],
['git', 'reset', '--hard', 'origin/{0}'.format(branch)],
['git', 'pull'],
['git', 'clean', '-d', '-f'],
]
else:
cmds = [['git', 'clone', repo]]
cwd = GIT_REPO_EXPORT_DIR
cwd = os.path.abspath(cwd)
for cmd in cmds:
try:
cmd_log(cmd, cwd)
except subprocess.CalledProcessError as ex:
log.exception('Failed to pull git repository: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_PULL)
# export course as xml before commiting and pushing
root_dir = os.path.dirname(rdirp)
course_dir = os.path.basename(rdirp).rsplit('.git', 1)[0]
try:
export_to_xml(modulestore(), contentstore(), course_id,
root_dir, course_dir)
except (EnvironmentError, AttributeError):
log.exception('Failed export to xml')
raise GitExportError(GitExportError.XML_EXPORT_FAIL)
# Get current branch if not already set
if not branch:
cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
try:
branch = cmd_log(cmd, os.path.abspath(rdirp)).strip('\n')
except subprocess.CalledProcessError as ex:
log.exception('Failed to get branch from freshly cloned repo: %r',
ex.output)
raise GitExportError(GitExportError.MISSING_BRANCH)
# Now that we have fresh xml exported, set identity, add
# everything to git, commit, and push to the right branch.
ident = {}
try:
user = User.objects.get(username=user)
ident['name'] = user.username
ident['email'] = user.email
except User.DoesNotExist:
# That's ok, just use default ident
ident = GIT_EXPORT_DEFAULT_IDENT
time_stamp = timezone.now()
cwd = os.path.abspath(rdirp)
commit_msg = 'Export from Studio at {1}'.format(user, time_stamp)
try:
cmd_log(['git', 'config', 'user.email', ident['email']], cwd)
cmd_log(['git', 'config', 'user.name', ident['name']], cwd)
except subprocess.CalledProcessError as ex:
log.exception('Error running git configure commands: %r', ex.output)
raise GitExportError(GitExportError.CONFIG_ERROR)
try:
cmd_log(['git', 'add', '.'], cwd)
cmd_log(['git', 'commit', '-a', '-m', commit_msg], cwd)
except subprocess.CalledProcessError as ex:
log.exception('Unable to commit changes: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_COMMIT)
try:
cmd_log(['git', 'push', '-q', 'origin', branch], cwd)
except subprocess.CalledProcessError as ex:
log.exception('Error running git push command: %r', ex.output)
raise GitExportError(GitExportError.CANNOT_PUSH)
def export_handler(request, course_key_string):
"""
The restful handler for exporting a course.
GET
html: return html page for import page
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request header can specify
application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails, an HTML page will be returned
which describes the error.
"""
course_key = CourseKey.from_string(course_key_string)
if not has_course_access(request.user, course_key):
raise PermissionDenied()
course_module = modulestore().get_course(course_key)
# an _accept URL parameter will be preferred over HTTP_ACCEPT in the header.
requested_format = request.REQUEST.get(
'_accept', request.META.get('HTTP_ACCEPT', 'text/html'))
export_url = reverse_course_url('export_handler',
course_key) + '?_accept=application/x-tgz'
if 'application/x-tgz' in requested_format:
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore('direct'), contentstore(),
course_module.id, root_dir, name, modulestore())
logging.debug('tar file being generated at {0}'.format(
export_file.name))
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception('There was an error exporting course %s',
course_module.id)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_locs = modulestore().get_parent_locations(
failed_item.location)
if len(parent_locs) > 0:
parent = modulestore().get_item(parent_locs[0])
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
return render_to_response(
'export.html', {
'context_course':
course_module,
'in_err':
True,
'raw_err_msg':
str(exc),
'failed_module':
failed_item,
'unit':
unit,
'edit_unit_url':
reverse_usage_url("unit_handler", parent.location)
if parent else "",
'course_home_url':
reverse_course_url("course_handler", course_key),
'export_url':
export_url
})
except Exception as exc:
log.exception('There was an error exporting course %s',
course_module.id)
return render_to_response(
'export.html', {
'context_course':
course_module,
'in_err':
True,
'unit':
None,
'raw_err_msg':
str(exc),
'course_home_url':
reverse_course_url("course_handler", course_key),
'export_url':
export_url
})
finally:
shutil.rmtree(root_dir / name)
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response[
'Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(
export_file.name)
response['Content-Length'] = os.path.getsize(export_file.name)
return response
elif 'text/html' in requested_format:
return render_to_response('export.html', {
'context_course': course_module,
'export_url': export_url
})
else:
# Only HTML or x-tgz request formats are supported (no JSON).
return HttpResponse(status=406)
def export_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None):
"""
The restful handler for exporting a course.
GET
html: return html page for import page
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request header can specify
application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails, an HTML page will be returned
which describes the error.
"""
location = BlockUsageLocator(package_id=package_id, branch=branch, version_guid=version_guid, block_id=block)
if not has_course_access(request.user, location):
raise PermissionDenied()
old_location = loc_mapper().translate_locator_to_location(location)
course_module = modulestore().get_item(old_location)
# an _accept URL parameter will be preferred over HTTP_ACCEPT in the header.
requested_format = request.REQUEST.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html'))
export_url = location.url_reverse('export') + '?_accept=application/x-tgz'
if 'application/x-tgz' in requested_format:
name = old_location.name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore('direct'), contentstore(), old_location, root_dir, name, modulestore())
logging.debug('tar file being generated at {0}'.format(export_file.name))
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError, e:
logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e)))
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_instance(course_module.location.course_id, e.location)
parent_locs = modulestore().get_parent_locations(failed_item.location, course_module.location.course_id)
if len(parent_locs) > 0:
parent = modulestore().get_item(parent_locs[0])
if parent.location.category == 'vertical':
unit = parent
except:
# if we have a nested exception, then we'll show the more generic error message
pass
unit_locator = loc_mapper().translate_location(old_location.course_id, parent.location, False, True)
return render_to_response('export.html', {
'context_course': course_module,
'in_err': True,
'raw_err_msg': str(e),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': unit_locator.url_reverse("unit") if parent else "",
'course_home_url': location.url_reverse("course"),
'export_url': export_url
})
except Exception, e:
logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e)))
return render_to_response('export.html', {
'context_course': course_module,
'in_err': True,
'unit': None,
'raw_err_msg': str(e),
'course_home_url': location.url_reverse("course"),
'export_url': export_url
})
def test_round_trip(self, source_builder, dest_builder,
source_content_builder, dest_content_builder,
course_data_name):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(dest_content) as dest_store:
source_course_key = source_store.make_course_key(
'source', 'course', 'key')
dest_course_key = dest_store.make_course_key(
'dest', 'course', 'key')
import_from_xml(
source_store,
'test_user',
'common/test/data',
course_dirs=[course_data_name],
static_content_store=source_content,
target_course_id=source_course_key,
create_new_course_if_not_present=True,
)
export_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_course',
)
import_from_xml(
dest_store,
'test_user',
self.export_dir,
static_content_store=dest_content,
target_course_id=dest_course_key,
create_new_course_if_not_present=True,
)
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
def export_handler(request, tag=None, course_id=None, branch=None, version_guid=None, block=None):
"""
The restful handler for exporting a course.
GET
html: return html page for import page
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request header can specify
application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails, an HTML page will be returned
which describes the error.
"""
location = BlockUsageLocator(course_id=course_id, branch=branch, version_guid=version_guid, usage_id=block)
if not has_access(request.user, location):
raise PermissionDenied()
old_location = loc_mapper().translate_locator_to_location(location)
course_module = modulestore().get_item(old_location)
# an _accept URL parameter will be preferred over HTTP_ACCEPT in the header.
requested_format = request.REQUEST.get("_accept", request.META.get("HTTP_ACCEPT", "text/html"))
export_url = location.url_reverse("export") + "?_accept=application/x-tgz"
if "application/x-tgz" in requested_format:
name = old_location.name
export_file = NamedTemporaryFile(prefix=name + ".", suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore("direct"), contentstore(), old_location, root_dir, name, modulestore())
except SerializationError, e:
logging.exception("There was an error exporting course {0}. {1}".format(course_module.location, unicode(e)))
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_instance(course_module.location.course_id, e.location)
parent_locs = modulestore().get_parent_locations(failed_item.location, course_module.location.course_id)
if len(parent_locs) > 0:
parent = modulestore().get_item(parent_locs[0])
if parent.location.category == "vertical":
unit = parent
except:
# if we have a nested exception, then we'll show the more generic error message
pass
unit_locator = loc_mapper().translate_location(old_location.course_id, parent.location, False, True)
return render_to_response(
"export.html",
{
"context_course": course_module,
"in_err": True,
"raw_err_msg": str(e),
"failed_module": failed_item,
"unit": unit,
"edit_unit_url": unit_locator.url_reverse("unit") if parent else "",
"course_home_url": location.url_reverse("course"),
"export_url": export_url,
},
)
except Exception, e:
logging.exception("There was an error exporting course {0}. {1}".format(course_module.location, unicode(e)))
return render_to_response(
"export.html",
{
"context_course": course_module,
"in_err": True,
"unit": None,
"raw_err_msg": str(e),
"course_home_url": location.url_reverse("course"),
"export_url": export_url,
},
)
def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
if CodeBlockTimer is None:
raise SkipTest("CodeBlockTimer undefined.")
desc = "XMLRoundTrip:{}->{}:{}".format(
SHORT_NAME_MAP[source_ms],
SHORT_NAME_MAP[dest_ms],
num_assets
)
with CodeBlockTimer(desc):
with CodeBlockTimer("fake_assets"):
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_ms.build(source_content) as source_store:
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_ms.build(dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
with CodeBlockTimer("initial_import"):
import_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
course_dirs=TEST_COURSE,
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
with CodeBlockTimer("export"):
export_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with CodeBlockTimer("second_import"):
import_from_xml(
dest_store,
'test_user',
self.export_dir,
course_dirs=['exported_source_course'],
static_content_store=dest_content,
target_course_id=dest_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
def test_export_course(self):
module_store = modulestore('direct')
draft_store = modulestore('draft')
content_store = contentstore()
import_from_xml(module_store, 'common/test/data/', ['full'])
location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
# get a vertical (and components in it) to put into 'draft'
vertical = module_store.get_item(Location(['i4x', 'edX', 'full',
'vertical', 'vertical_66', None]), depth=1)
draft_store.clone_item(vertical.location, vertical.location)
# We had a bug where orphaned draft nodes caused export to fail. This is here to cover that case.
draft_store.clone_item(vertical.location, Location(['i4x', 'edX', 'full',
'vertical', 'no_references', 'draft']))
for child in vertical.get_children():
draft_store.clone_item(child.location, child.location)
root_dir = path(mkdtemp_clean())
# now create a private vertical
private_vertical = draft_store.clone_item(vertical.location,
Location(['i4x', 'edX', 'full', 'vertical', 'a_private_vertical', None]))
# add private to list of children
sequential = module_store.get_item(Location(['i4x', 'edX', 'full',
'sequential', 'Administrivia_and_Circuit_Elements', None]))
private_location_no_draft = private_vertical.location.replace(revision=None)
module_store.update_children(sequential.location, sequential.children +
[private_location_no_draft.url()])
# read back the sequential, to make sure we have a pointer to
sequential = module_store.get_item(Location(['i4x', 'edX', 'full',
'sequential', 'Administrivia_and_Circuit_Elements', None]))
self.assertIn(private_location_no_draft.url(), sequential.children)
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_to_xml(module_store, content_store, location, root_dir, 'test_export', draft_modulestore=draft_store)
# check for static tabs
self.verify_content_existence(module_store, root_dir, location, 'tabs', 'static_tab', '.html')
# check for custom_tags
self.verify_content_existence(module_store, root_dir, location, 'info', 'course_info', '.html')
# check for custom_tags
self.verify_content_existence(module_store, root_dir, location, 'custom_tags', 'custom_tag_template')
# check for about content
self.verify_content_existence(module_store, root_dir, location, 'about', 'about', '.html')
# check for graiding_policy.json
filesystem = OSFS(root_dir / 'test_export/policies/6.002_Spring_2012')
self.assertTrue(filesystem.exists('grading_policy.json'))
course = module_store.get_item(location)
# compare what's on disk compared to what we have in our course
with filesystem.open('grading_policy.json', 'r') as grading_policy:
on_disk = loads(grading_policy.read())
self.assertEqual(on_disk, course.grading_policy)
#check for policy.json
self.assertTrue(filesystem.exists('policy.json'))
# compare what's on disk to what we have in the course module
with filesystem.open('policy.json', 'r') as course_policy:
on_disk = loads(course_policy.read())
self.assertIn('course/6.002_Spring_2012', on_disk)
self.assertEqual(on_disk['course/6.002_Spring_2012'], own_metadata(course))
# remove old course
delete_course(module_store, content_store, location)
# reimport
import_from_xml(module_store, root_dir, ['test_export'], draft_store=draft_store)
items = module_store.get_items(Location(['i4x', 'edX', 'full', 'vertical', None]))
self.assertGreater(len(items), 0)
for descriptor in items:
# don't try to look at private verticals. Right now we're running
# the service in non-draft aware
if getattr(descriptor, 'is_draft', False):
print "Checking {0}....".format(descriptor.location.url())
resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()}))
self.assertEqual(resp.status_code, 200)
# verify that we have the content in the draft store as well
vertical = draft_store.get_item(Location(['i4x', 'edX', 'full',
'vertical', 'vertical_66', None]), depth=1)
self.assertTrue(getattr(vertical, 'is_draft', False))
for child in vertical.get_children():
self.assertTrue(getattr(child, 'is_draft', False))
# make sure that we don't have a sequential that is in draft mode
sequential = draft_store.get_item(Location(['i4x', 'edX', 'full',
'sequential', 'Administrivia_and_Circuit_Elements', None]))
self.assertFalse(getattr(sequential, 'is_draft', False))
# verify that we have the private vertical
test_private_vertical = draft_store.get_item(Location(['i4x', 'edX', 'full',
'vertical', 'vertical_66', None]))
self.assertTrue(getattr(test_private_vertical, 'is_draft', False))
# make sure the textbook survived the export/import
course = module_store.get_item(Location(['i4x', 'edX', 'full', 'course', '6.002_Spring_2012', None]))
self.assertGreater(len(course.textbooks), 0)
shutil.rmtree(root_dir)
def export_handler(request, course_key_string):
"""
The restful handler for exporting a course.
GET
html: return html page for import page
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request header can specify
application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails, an HTML page will be returned
which describes the error.
"""
course_key = CourseKey.from_string(course_key_string)
if not has_course_access(request.user, course_key):
raise PermissionDenied()
course_module = modulestore().get_course(course_key)
# an _accept URL parameter will be preferred over HTTP_ACCEPT in the header.
requested_format = request.REQUEST.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html'))
export_url = reverse_course_url('export_handler', course_key) + '?_accept=application/x-tgz'
if 'application/x-tgz' in requested_format:
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
logging.debug('tar file being generated at {0}'.format(export_file.name))
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception('There was an error exporting course %s', course_module.id)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
return render_to_response('export.html', {
'context_course': course_module,
'in_err': True,
'raw_err_msg': str(exc),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': reverse_usage_url("unit_handler", parent.location) if parent else "",
'course_home_url': reverse_course_url("course_handler", course_key),
'export_url': export_url
})
except Exception as exc:
log.exception('There was an error exporting course %s', course_module.id)
return render_to_response('export.html', {
'context_course': course_module,
'in_err': True,
'unit': None,
'raw_err_msg': str(exc),
'course_home_url': reverse_course_url("course_handler", course_key),
'export_url': export_url
})
finally:
shutil.rmtree(root_dir / name)
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name)
response['Content-Length'] = os.path.getsize(export_file.name)
return response
elif 'text/html' in requested_format:
return render_to_response('export.html', {
'context_course': course_module,
'export_url': export_url
})
else:
# Only HTML or x-tgz request formats are supported (no JSON).
return HttpResponse(status=406)
def test_round_trip(self, source_builder, dest_builder,
source_content_builder, dest_content_builder,
course_data_name):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(dest_content) as dest_store:
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
import_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
course_dirs=[course_data_name],
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
export_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
import_from_xml(
dest_store,
'test_user',
self.export_dir,
course_dirs=['exported_source_course'],
static_content_store=dest_content,
target_course_id=dest_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
# NOT CURRENTLY USED
# export_to_xml(
# dest_store,
# dest_content,
# dest_course_key,
# self.export_dir,
# 'exported_dest_course',
# )
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.exclude_field(None, 'parent')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
self.assertAssetsMetadataEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
import_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
course_dirs=[course_data_name],
static_content_store=source_content,
target_course_id=source_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
export_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
import_from_xml(
dest_store,
'test_user',
self.export_dir,
course_dirs=['exported_source_course'],
static_content_store=dest_content,
target_course_id=dest_course_key,
create_course_if_not_present=True,
raise_on_failure=True,
)
# NOT CURRENTLY USED
# export_to_xml(
# dest_store,
# dest_content,
# dest_course_key,
# self.export_dir,
# 'exported_dest_course',
# )
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.exclude_field(None, 'parent')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
self.assertAssetsMetadataEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
