def export_courses_to_output_path(output_path): """ Export all courses to target directory and return the list of courses which failed to export """ content_store = contentstore() module_store = modulestore() root_dir = output_path courses = module_store.get_courses() course_ids = [x.id for x in courses] failed_export_courses = [] for course_id in course_ids: print u"-" * 80 print u"Exporting course id = {0} to {1}".format(course_id, output_path) try: course_dir = course_id.to_deprecated_string().replace('/', '...') export_course_to_xml(module_store, content_store, course_id, root_dir, course_dir) except Exception as err: # pylint: disable=broad-except failed_export_courses.append(unicode(course_id)) print u"=" * 30 + u"> Oops, failed to export {0}".format(course_id) print u"Error:" print err return courses, failed_export_courses
def handle(self, *args, **options): "Execute the command" if len(args) != 2: raise CommandError("export requires two arguments: <course id> <output path>") try: course_key = CourseKey.from_string(args[0]) except InvalidKeyError: try: course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0]) except InvalidKeyError: raise CommandError("Invalid course_key: '%s'. " % args[0]) if not modulestore().get_course(course_key): raise CommandError("Course with %s key not found." % args[0]) output_path = args[1] print "Exporting course id = {0} to {1}".format(course_key, output_path) if not output_path.endswith('/'): output_path += '/' root_dir = os.path.dirname(output_path) course_dir = os.path.splitext(os.path.basename(output_path))[0] export_course_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def handle(self, *args, **options): "Execute the command" if len(args) != 2: raise CommandError( "export requires two arguments: <course id> <output path>") try: course_key = CourseKey.from_string(args[0]) except InvalidKeyError: course_key = SlashSeparatedCourseKey.from_deprecated_string( args[0]) output_path = args[1] print("Exporting course id = {0} to {1}".format( course_key, output_path)) if not output_path.endswith('/'): output_path += '/' root_dir = os.path.dirname(output_path) course_dir = os.path.splitext(os.path.basename(output_path))[0] export_course_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def handle(self, *args, **options): """ Given a course id(old or new style), and an output_path folder. Export the corresponding course from mongo and put it directly in the folder. """ try: course_key = CourseKey.from_string(options['course_id']) except InvalidKeyError: raise CommandError(u"Invalid course_key: '%s'." % options['course_id']) if not modulestore().get_course(course_key): raise CommandError(u"Course with %s key not found." % options['course_id']) output_path = options['output_path'] print(u"Exporting course id = {0} to {1}".format( course_key, output_path)) if not output_path.endswith('/'): output_path += '/' root_dir = os.path.dirname(output_path) course_dir = os.path.splitext(os.path.basename(output_path))[0] export_course_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def handle(self, *args, **options): """Execute the command""" try: course_key = CourseKey.from_string(options['course_id']) except InvalidKeyError: try: course_key = SlashSeparatedCourseKey.from_deprecated_string( options['course_id']) except InvalidKeyError: raise CommandError("Invalid course_key: '%s'." % options['course_id']) if not modulestore().get_course(course_key): raise CommandError("Course with %s key not found." % options['course_id']) output_path = options['output_path'] print "Exporting course id = {0} to {1}".format( course_key, output_path) if not output_path.endswith('/'): output_path += '/' root_dir = os.path.dirname(output_path) course_dir = os.path.splitext(os.path.basename(output_path))[0] export_course_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def test_library_content_on_course_export_import(self, publish_item): """ Verify that library contents in destination and source courses are same after importing the source course into destination course. """ self._setup_source_course_with_library_content(publish=publish_item) # Create a course to import source course. dest_course = CourseFactory.create( default_store=ModuleStoreEnum.Type.split) # Export the source course. export_course_to_xml( self.store, contentstore(), self.source_course.location.course_key, self.export_dir, 'exported_source_course', ) # Now, import it back to dest_course. import_course_from_xml( self.store, self.user.id, self.export_dir, ['exported_source_course'], static_content_store=contentstore(), target_id=dest_course.location.course_key, load_error_modules=False, raise_on_failure=True, create_if_not_present=True, ) self.assert_problem_display_names(self.source_course.location, dest_course.location, publish_item)
def test_problem_content_on_course_export_import(self, problem_data, expected_problem_content): """ Verify that problem content in destination matches expected problem content, specifically concerned with pre tag data with problem. """ self._setup_source_course_with_problem_content(problem_data) dest_course = CourseFactory.create( default_store=ModuleStoreEnum.Type.split) export_course_to_xml( self.store, contentstore(), self.source_course.location.course_key, self.export_dir, 'exported_source_course', ) import_course_from_xml( self.store, self.user.id, self.export_dir, ['exported_source_course'], static_content_store=contentstore(), target_id=dest_course.location.course_key, load_error_modules=False, raise_on_failure=True, create_if_not_present=True, ) self.assert_problem_definition(dest_course.location, expected_problem_content)
def handle(self, *args, **options): """ Given a course id(old or new style), and an output_path folder. Export the corresponding course from mongo and put it directly in the folder. """ try: course_key = CourseKey.from_string(options['course_id']) except InvalidKeyError: try: course_key = SlashSeparatedCourseKey.from_deprecated_string(options['course_id']) except InvalidKeyError: raise CommandError("Invalid course_key: '%s'." % options['course_id']) if not modulestore().get_course(course_key): raise CommandError("Course with %s key not found." % options['course_id']) output_path = options['output_path'] print "Exporting course id = {0} to {1}".format(course_key, output_path) if not output_path.endswith('/'): output_path += '/' root_dir = os.path.dirname(output_path) course_dir = os.path.splitext(os.path.basename(output_path))[0] export_course_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def export_courses_to_output_path(output_path): """ Export all courses to target directory and return the list of courses which failed to export """ content_store = contentstore() module_store = modulestore() root_dir = output_path courses = module_store.get_courses() course_ids = [x.id for x in courses] failed_export_courses = [] for course_id in course_ids: print("-" * 80) print(u"Exporting course id = {0} to {1}".format( course_id, output_path)) try: course_dir = text_type(course_id).replace('/', '...') export_course_to_xml(module_store, content_store, course_id, root_dir, course_dir) except Exception as err: # pylint: disable=broad-except failed_export_courses.append(text_type(course_id)) print(u"=" * 30 + u"> Oops, failed to export {0}".format(course_id)) print("Error:") print(err) return courses, failed_export_courses
def create_export_tarball(course_module, course_key, context, status=None): """ Generates the export tarball, or returns None if there was an error. Updates the context with any error information if applicable. """ name = course_module.url_name export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp()) try: if isinstance(course_key, LibraryLocator): export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name) else: export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name) if status: status.set_state(u'Compressing') status.increment_completed_steps() LOGGER.debug(u'tar file being generated at %s', export_file.name) with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: tar_file.add(root_dir / name, arcname=name) except SerializationError as exc: LOGGER.exception(u'There was an error exporting %s', course_key, exc_info=True) parent = None try: failed_item = modulestore().get_item(exc.location) parent_loc = modulestore().get_parent_location(failed_item.location) if parent_loc is not None: parent = modulestore().get_item(parent_loc) except: # pylint: disable=bare-except # if we have a nested exception, then we'll show the more generic error message pass context.update({ 'in_err': True, 'raw_err_msg': str(exc), 'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "", }) if status: status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'], 'edit_unit_url': context['edit_unit_url']})) raise except Exception as exc: LOGGER.exception('There was an error exporting %s', course_key, exc_info=True) context.update({ 'in_err': True, 'edit_unit_url': None, 'raw_err_msg': str(exc)}) if status: status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']})) raise finally: if os.path.exists(root_dir / name): shutil.rmtree(root_dir / name) return export_file
def create_export_tarball(course_module, course_key, context, status=None): """ Generates the export tarball, or returns None if there was an error. Updates the context with any error information if applicable. """ name = course_module.url_name export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp()) try: if isinstance(course_key, LibraryLocator): export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name) else: export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name) if status: status.set_state(u'Compressing') status.increment_completed_steps() LOGGER.debug(u'tar file being generated at %s', export_file.name) with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: tar_file.add(root_dir / name, arcname=name) except SerializationError as exc: LOGGER.exception(u'There was an error exporting %s', course_key, exc_info=True) parent = None try: failed_item = modulestore().get_item(exc.location) parent_loc = modulestore().get_parent_location(failed_item.location) if parent_loc is not None: parent = modulestore().get_item(parent_loc) except: # pylint: disable=bare-except # if we have a nested exception, then we'll show the more generic error message pass context.update({ 'in_err': True, 'raw_err_msg': str(exc), 'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "", }) if status: status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'], 'edit_unit_url': context['edit_unit_url']})) raise except Exception as exc: LOGGER.exception(u'There was an error exporting %s', course_key, exc_info=True) context.update({ 'in_err': True, 'edit_unit_url': None, 'raw_err_msg': str(exc)}) if status: status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']})) raise finally: if os.path.exists(root_dir / name): shutil.rmtree(root_dir / name) return export_file
def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ if CodeBlockTimer is None: pytest.skip("CodeBlockTimer undefined.") desc = "XMLRoundTrip:{}->{}:{}".format(SHORT_NAME_MAP[source_ms], SHORT_NAME_MAP[dest_ms], num_assets) with CodeBlockTimer(desc): with CodeBlockTimer("fake_assets"): # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) with source_ms.build() as (source_content, source_store): with dest_ms.build() as (dest_content, dest_store): source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') with CodeBlockTimer("initial_import"): import_course_from_xml( source_store, 'test_user', TEST_DATA_ROOT, source_dirs=TEST_COURSE, static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with CodeBlockTimer("export"): export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with CodeBlockTimer("second_import"): import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, create_if_not_present=True, raise_on_failure=True, )
def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ if CodeBlockTimer is None: raise SkipTest("CodeBlockTimer undefined.") desc = "XMLRoundTrip:{}->{}:{}".format( SHORT_NAME_MAP[source_ms], SHORT_NAME_MAP[dest_ms], num_assets ) with CodeBlockTimer(desc): with CodeBlockTimer("fake_assets"): # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) with source_ms.build() as (source_content, source_store): with dest_ms.build() as (dest_content, dest_store): source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') with CodeBlockTimer("initial_import"): import_course_from_xml( source_store, 'test_user', TEST_DATA_ROOT, source_dirs=TEST_COURSE, static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with CodeBlockTimer("export"): export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with CodeBlockTimer("second_import"): import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, create_if_not_present=True, raise_on_failure=True, )
def create_export_tarball(course_module, course_key, context): """ Generates the export tarball, or returns None if there was an error. Updates the context with any error information if applicable. """ name = course_module.url_name export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp()) try: if isinstance(course_key, LibraryLocator): export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name) else: export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name) logging.debug(u'tar file being generated at %s', export_file.name) with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: tar_file.add(root_dir / name, arcname=name) except SerializationError as exc: log.exception(u'There was an error exporting %s', course_key) unit = None failed_item = None parent = None try: failed_item = modulestore().get_item(exc.location) parent_loc = modulestore().get_parent_location(failed_item.location) if parent_loc is not None: parent = modulestore().get_item(parent_loc) if parent.location.category == 'vertical': unit = parent except: # pylint: disable=bare-except # if we have a nested exception, then we'll show the more generic error message pass context.update({ 'in_err': True, 'raw_err_msg': str(exc), 'failed_module': failed_item, 'unit': unit, 'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "", }) raise except Exception as exc: log.exception('There was an error exporting %s', course_key) context.update({ 'in_err': True, 'unit': None, 'raw_err_msg': str(exc)}) raise finally: shutil.rmtree(root_dir / name) return export_file
def create_export_tarball(course_module, course_key, context): """ Generates the export tarball, or returns None if there was an error. Updates the context with any error information if applicable. """ name = course_module.url_name export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp()) try: if isinstance(course_key, LibraryLocator): export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name) else: export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name) logging.debug(u'tar file being generated at %s', export_file.name) with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: tar_file.add(root_dir / name, arcname=name) except SerializationError as exc: log.exception(u'There was an error exporting %s', course_key) unit = None failed_item = None parent = None try: failed_item = modulestore().get_item(exc.location) parent_loc = modulestore().get_parent_location(failed_item.location) if parent_loc is not None: parent = modulestore().get_item(parent_loc) if parent.location.category == 'vertical': unit = parent except: # pylint: disable=bare-except # if we have a nested exception, then we'll show the more generic error message pass context.update({ 'in_err': True, 'raw_err_msg': str(exc), 'failed_module': failed_item, 'unit': unit, 'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "", }) raise except Exception as exc: log.exception('There was an error exporting %s', course_key) context.update({ 'in_err': True, 'unit': None, 'raw_err_msg': str(exc)}) raise finally: shutil.rmtree(root_dir / name) return export_file
def test_course_without_image(self): """ Make sure we elegantly passover our code when there isn't a static image """ course = self.draft_store.get_course(CourseKey.from_string('edX/simple_with_draft/2012_Fall')) root_dir = path(mkdtemp()) self.addCleanup(shutil.rmtree, root_dir) export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, u'test_export') self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) self.assertFalse(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
def test_split_course_export_import(self): # Construct the contentstore for storing the first import with MongoContentstoreBuilder().build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with SPLIT_MODULESTORE_SETUP.build(contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with MongoContentstoreBuilder().build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with SPLIT_MODULESTORE_SETUP.build(contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key('a', 'source', '2015_Fall') # lint-amnesty, pylint: disable=no-member dest_course_key = dest_store.make_course_key('a', 'dest', '2015_Fall') # lint-amnesty, pylint: disable=no-member import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['split_course_with_static_tabs'], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME, ) source_course = source_store.get_course(source_course_key, depth=None, lazy=False) # lint-amnesty, pylint: disable=no-member assert source_course.url_name == 'course' export_dir_path = path(self.export_dir) policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course_key.run policy_path = policy_dir / 'policy.json' assert os.path.exists(policy_path) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=[EXPORTED_COURSE_DIR_NAME], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) dest_course = dest_store.get_course(dest_course_key, depth=None, lazy=False) # lint-amnesty, pylint: disable=no-member assert dest_course.url_name == 'course'
def test_course_without_image(self): """ Make sure we elegantly passover our code when there isn't a static image """ course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'simple_with_draft', '2012_Fall')) root_dir = path(mkdtemp()) self.addCleanup(shutil.rmtree, root_dir) export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export') self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) self.assertFalse(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
def test_split_course_export_import(self): # Construct the contentstore for storing the first import with MongoContentstoreBuilder().build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with SPLIT_MODULESTORE_SETUP.build(contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with MongoContentstoreBuilder().build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with SPLIT_MODULESTORE_SETUP.build(contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key('a', 'source', '2015_Fall') dest_course_key = dest_store.make_course_key('a', 'dest', '2015_Fall') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['split_course_with_static_tabs'], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME, ) source_course = source_store.get_course(source_course_key, depth=None, lazy=False) self.assertEqual(source_course.url_name, 'course') export_dir_path = path(self.export_dir) policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course_key.run policy_path = policy_dir / 'policy.json' self.assertTrue(os.path.exists(policy_path)) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=[EXPORTED_COURSE_DIR_NAME], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) dest_course = dest_store.get_course(dest_course_key, depth=None, lazy=False) self.assertEqual(dest_course.url_name, 'course')
def test_export_course_with_peer_component(self): """ Test export course when link_to_location is given in peer grading interface settings. """ name = "export_peer_component" locations = self._create_test_tree(name) # Insert the test block directly into the module store problem_location = Location('edX', 'tree{}'.format(name), name, 'combinedopenended', 'test_peer_problem') self.draft_store.create_child( self.dummy_user, locations["child"], problem_location.block_type, block_id=problem_location.block_id ) interface_location = Location('edX', 'tree{}'.format(name), name, 'peergrading', 'test_peer_interface') self.draft_store.create_child( self.dummy_user, locations["child"], interface_location.block_type, block_id=interface_location.block_id ) self.draft_store._update_single_item( as_draft(interface_location), { 'definition.data': {}, 'metadata': { 'link_to_location': unicode(problem_location), 'use_for_single_location': True, }, }, ) component = self.draft_store.get_item(interface_location) self.assertEqual(unicode(component.link_to_location), unicode(problem_location)) root_dir = path(mkdtemp()) # export_course_to_xml should work. try: export_course_to_xml( self.draft_store, self.content_store, interface_location.course_key, root_dir, 'test_export' ) finally: shutil.rmtree(root_dir)
def test_export_course_with_peer_component(self): """ Test export course when link_to_location is given in peer grading interface settings. """ name = "export_peer_component" locations = self._create_test_tree(name) # Insert the test block directly into the module store problem_location = Location('edX', 'tree{}'.format(name), name, 'combinedopenended', 'test_peer_problem') self.draft_store.create_child( self.dummy_user, locations["child"], problem_location.block_type, block_id=problem_location.block_id ) interface_location = Location('edX', 'tree{}'.format(name), name, 'peergrading', 'test_peer_interface') self.draft_store.create_child( self.dummy_user, locations["child"], interface_location.block_type, block_id=interface_location.block_id ) self.draft_store._update_single_item( as_draft(interface_location), { 'definition.data': {}, 'metadata': { 'link_to_location': unicode(problem_location), 'use_for_single_location': True, }, }, ) component = self.draft_store.get_item(interface_location) self.assertEqual(unicode(component.link_to_location), unicode(problem_location)) root_dir = path(mkdtemp()) # export_course_to_xml should work. try: export_course_to_xml( self.draft_store, self.content_store, interface_location.course_key, root_dir, 'test_export' ) finally: shutil.rmtree(root_dir)
def test_course_without_image(self): """ Make sure we elegantly passover our code when there isn't a static image """ course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'simple_with_draft', '2012_Fall')) root_dir = path(mkdtemp()) try: export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export') assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) assert_false(path(root_dir / 'test_export/static/images_course_image.jpg').isfile()) finally: shutil.rmtree(root_dir)
def test_export_course_image_nondefault(self, _from_json): """ Make sure that if a non-default image path is specified that we don't export it to the static default location """ course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')) assert_true(course.course_image, 'just_a_test.jpg') root_dir = path(mkdtemp()) self.addCleanup(shutil.rmtree, root_dir) export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export') self.assertTrue(path(root_dir / 'test_export/static/just_a_test.jpg').isfile()) self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
def test_course_without_image(self): """ Make sure we elegantly passover our code when there isn't a static image """ course = self.draft_store.get_course(SlashSeparatedCourseKey("edX", "simple_with_draft", "2012_Fall")) root_dir = path(mkdtemp()) try: export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, "test_export") assert_false(path(root_dir / "test_export/static/images/course_image.jpg").isfile()) assert_false(path(root_dir / "test_export/static/images_course_image.jpg").isfile()) finally: shutil.rmtree(root_dir)
def test_export_course_image_nondefault(self, _from_json): """ Make sure that if a non-default image path is specified that we don't export it to the static default location """ course = self.draft_store.get_course(CourseKey.from_string('edX/toy/2012_Fall')) assert_true(course.course_image, 'just_a_test.jpg') root_dir = path(mkdtemp()) self.addCleanup(shutil.rmtree, root_dir) export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, u'test_export') self.assertTrue(path(root_dir / 'test_export/static/just_a_test.jpg').isfile()) self.assertFalse(path(root_dir / 'test_export/static/images/course_image.jpg').isfile())
def test_export_course_image_nondefault(self): """ Make sure that if a non-default image path is specified that we don't export it to the static default location """ course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')) assert_true(course.course_image, 'just_a_test.jpg') root_dir = path(mkdtemp()) try: export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export') assert_true(path(root_dir / 'test_export/static/just_a_test.jpg').isfile()) assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) finally: shutil.rmtree(root_dir)
def test_export_course_image_nondefault(self): """ Make sure that if a non-default image path is specified that we don't export it to the static default location """ course = self.draft_store.get_course(SlashSeparatedCourseKey("edX", "toy", "2012_Fall")) assert_true(course.course_image, "just_a_test.jpg") root_dir = path(mkdtemp()) try: export_course_to_xml(self.draft_store, self.content_store, course.id, root_dir, "test_export") assert_true(path(root_dir / "test_export/static/just_a_test.jpg").isfile()) assert_false(path(root_dir / "test_export/static/images/course_image.jpg").isfile()) finally: shutil.rmtree(root_dir)
def export_course_to_directory(course_key, root_dir): """Export course into a directory""" store = modulestore() course = store.get_course(course_key) if course is None: raise CommandError("Invalid course_id") # The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>. # We represent the first four with \w. # TODO: Once we support courses with unicode characters, we will need to revisit this. course_dir = course.url_name export_course_to_xml(store, None, course.id, root_dir, course_dir) export_dir = path(root_dir) / course_dir return export_dir
def test_export_course_image(self, _from_json): """ Test to make sure that we have a course image in the contentstore, then export it to ensure it gets copied to both file locations. """ course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall') location = course_key.make_asset_key('asset', 'images_course_image.jpg') # This will raise if the course image is missing self.content_store.find(location) root_dir = path(mkdtemp()) self.addCleanup(shutil.rmtree, root_dir) export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export') self.assertTrue(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) self.assertTrue(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
def test_export_course_image(self, _from_json): """ Test to make sure that we have a course image in the contentstore, then export it to ensure it gets copied to both file locations. """ course_key = CourseKey.from_string('edX/simple/2012_Fall') location = course_key.make_asset_key('asset', 'images_course_image.jpg') # This will raise if the course image is missing self.content_store.find(location) root_dir = path(mkdtemp()) self.addCleanup(shutil.rmtree, root_dir) export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, u'test_export') self.assertTrue(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) self.assertTrue(path(root_dir / 'test_export/static/images_course_image.jpg').isfile())
def test_export_course_image(self): """ Test to make sure that we have a course image in the contentstore, then export it to ensure it gets copied to both file locations. """ course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall') location = course_key.make_asset_key('asset', 'images_course_image.jpg') # This will raise if the course image is missing self.content_store.find(location) root_dir = path(mkdtemp()) try: export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export') assert_true(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) assert_true(path(root_dir / 'test_export/static/images_course_image.jpg').isfile()) finally: shutil.rmtree(root_dir)
def test_export_course_image(self): """ Test to make sure that we have a course image in the contentstore, then export it to ensure it gets copied to both file locations. """ course_key = SlashSeparatedCourseKey("edX", "simple", "2012_Fall") location = course_key.make_asset_key("asset", "images_course_image.jpg") # This will raise if the course image is missing self.content_store.find(location) root_dir = path(mkdtemp()) try: export_course_to_xml(self.draft_store, self.content_store, course_key, root_dir, "test_export") assert_true(path(root_dir / "test_export/static/images/course_image.jpg").isfile()) assert_true(path(root_dir / "test_export/static/images_course_image.jpg").isfile()) finally: shutil.rmtree(root_dir)
def handle(self, *args, **options): "Execute the command" if len(args) != 2: raise CommandError("export requires two arguments: <course id> <output path>") try: course_key = CourseKey.from_string(args[0]) except InvalidKeyError: course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0]) output_path = args[1] print("Exporting course id = {0} to {1}".format(course_key, output_path)) root_dir = os.path.dirname(output_path) course_dir = os.path.splitext(os.path.basename(output_path))[0] export_course_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def export_course_to_directory(course_key, root_dir): """Export course into a directory""" store = modulestore() course = store.get_course(course_key) if course is None: raise CommandError("Invalid course_id") # The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>. # We represent the first four with \w. # TODO: Once we support courses with unicode characters, we will need to revisit this. replacement_char = u"-" course_dir = replacement_char.join([course.id.org, course.id.course, course.id.run]) course_dir = re.sub(r"[^\w\.\-]", replacement_char, course_dir) export_course_to_xml(store, None, course.id, root_dir, course_dir) export_dir = path(root_dir) / course_dir return export_dir
def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes): with store_builder.build() as (source_content, source_store): with store_builder.build() as (dest_content, dest_store): source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') # An extra import write occurs in the first Split import due to the mismatch between # the course id and the wiki_slug in the test XML course. The course must be updated # with the correct wiki_slug during import. with check_mongo_calls(import_reads, first_import_writes): import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['manual-testing-complete'], static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with check_mongo_calls(export_reads): export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with check_mongo_calls(import_reads, second_import_writes): import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, create_if_not_present=True, raise_on_failure=True, )
def test_export( self, solution_attribute_value, solution_element_value, expected_solution_attribute, expected_solution_element): """Export the test course with the SGA module""" course = self.import_test_course(solution_attribute_value, solution_element_value) temp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(temp_dir)) store = modulestore() export_course_to_xml(store, None, course.id, temp_dir, "2017_SGA") with open(os.path.join(temp_dir, "2017_SGA", "vertical", "vertical.xml")) as f: content = f.read() # If both are true the expected output should only have the attribute, since it took precedence # and the attribute contents are broken XML assert reformat_xml(content) == reformat_xml( self.make_test_vertical(expected_solution_attribute, expected_solution_element) )
def test_export( self, solution_attribute_value, solution_element_value, expected_solution_attribute, expected_solution_element): """Export the test course with the SGA module""" course = self.import_test_course(solution_attribute_value, solution_element_value) temp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(temp_dir)) store = modulestore() export_course_to_xml(store, None, course.id, temp_dir, "2017_SGA") with open(os.path.join(temp_dir, "2017_SGA", "vertical", "vertical.xml")) as f: content = f.read() # If both are true the expected output should only have the attribute, since it took precedence # and the attribute contents are broken XML assert reformat_xml(content) == reformat_xml( self.make_test_vertical(expected_solution_attribute, expected_solution_element) )
def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes): with store_builder.build() as (source_content, source_store): with store_builder.build() as (dest_content, dest_store): source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') # An extra import write occurs in the first Split import due to the mismatch between # the course id and the wiki_slug in the test XML course. The course must be updated # with the correct wiki_slug during import. with check_mongo_calls(import_reads, first_import_writes): import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['manual-testing-complete'], static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with check_mongo_calls(export_reads): export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with check_mongo_calls(import_reads, second_import_writes): import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, create_if_not_present=True, raise_on_failure=True, )
def test_library_content_on_course_export_import(self, publish_item): """ Verify that library contents in destination and source courses are same after importing the source course into destination course. """ self._setup_source_course_with_library_content(publish=publish_item) # Create a course to import source course. dest_course = CourseFactory.create(default_store=ModuleStoreEnum.Type.split) # Export the source course. export_course_to_xml( self.store, contentstore(), self.source_course.location.course_key, self.export_dir, 'exported_source_course', ) # Now, import it back to dest_course. import_course_from_xml( self.store, self.user.id, self.export_dir, ['exported_source_course'], static_content_store=contentstore(), target_id=dest_course.location.course_key, load_error_modules=False, raise_on_failure=True, create_if_not_present=True, ) self.assert_problem_display_names( self.source_course.location, dest_course.location, publish_item )
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name, _mock_tab_from_json): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build( contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build( contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=[course_data_name], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME, ) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=[EXPORTED_COURSE_DIR_NAME], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) # NOT CURRENTLY USED # export_course_to_xml( # dest_store, # dest_content, # dest_course_key, # self.export_dir, # 'exported_dest_course', # ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.exclude_field(None, 'parent') # discussion_ids are auto-generated based on usage_id, so they should change across # modulestores - see TNL-5001 self.exclude_field(None, 'discussion_id') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, ) self.assertAssetsMetadataEqual( source_store, source_course_key, dest_store, dest_course_key, )
def export_to_git(course_id, repo, user='', rdir=None): """Export a course to git.""" # pylint: disable=too-many-statements if not GIT_REPO_EXPORT_DIR: raise GitExportError(GitExportError.NO_EXPORT_DIR) if not os.path.isdir(GIT_REPO_EXPORT_DIR): raise GitExportError(GitExportError.NO_EXPORT_DIR) # Check for valid writable git url if not (repo.endswith('.git') or repo.startswith( ('http:', 'https:', 'file:'))): raise GitExportError(GitExportError.URL_BAD) # Check for username and password if using http[s] if repo.startswith('http:') or repo.startswith('https:'): parsed = urlparse(repo) if parsed.username is None or parsed.password is None: raise GitExportError(GitExportError.URL_NO_AUTH) if rdir: rdir = os.path.basename(rdir) else: rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0] log.debug(u"rdir = %s", rdir) # Pull or clone repo before exporting to xml # and update url in case origin changed. rdirp = '{0}/{1}'.format(GIT_REPO_EXPORT_DIR, rdir) branch = None if os.path.exists(rdirp): log.info('Directory already exists, doing a git reset and pull ' 'instead of git clone.') cwd = rdirp # Get current branch cmd = ['git', 'symbolic-ref', '--short', 'HEAD'] try: branch = cmd_log(cmd, cwd).decode('utf-8').strip('\n') except subprocess.CalledProcessError as ex: log.exception(u'Failed to get branch: %r', ex.output) raise GitExportError(GitExportError.DETACHED_HEAD) cmds = [ ['git', 'remote', 'set-url', 'origin', repo], ['git', 'fetch', 'origin'], ['git', 'reset', '--hard', 'origin/{0}'.format(branch)], ['git', 'pull'], ['git', 'clean', '-d', '-f'], ] else: cmds = [['git', 'clone', repo]] cwd = GIT_REPO_EXPORT_DIR cwd = os.path.abspath(cwd) for cmd in cmds: try: cmd_log(cmd, cwd) except subprocess.CalledProcessError as ex: log.exception(u'Failed to pull git repository: %r', ex.output) raise GitExportError(GitExportError.CANNOT_PULL) # export course as xml before commiting and pushing root_dir = os.path.dirname(rdirp) course_dir = os.path.basename(rdirp).rsplit('.git', 1)[0] try: export_course_to_xml(modulestore(), contentstore(), course_id, root_dir, course_dir) except (EnvironmentError, AttributeError): log.exception('Failed export to xml') raise GitExportError(GitExportError.XML_EXPORT_FAIL) # Get current branch if not already set if not branch: cmd = ['git', 'symbolic-ref', '--short', 'HEAD'] try: branch = cmd_log( cmd, os.path.abspath(rdirp)).decode('utf-8').strip('\n') except subprocess.CalledProcessError as ex: log.exception(u'Failed to get branch from freshly cloned repo: %r', ex.output) raise GitExportError(GitExportError.MISSING_BRANCH) # Now that we have fresh xml exported, set identity, add # everything to git, commit, and push to the right branch. ident = {} try: user = User.objects.get(username=user) ident['name'] = user.username ident['email'] = user.email except User.DoesNotExist: # That's ok, just use default ident ident = GIT_EXPORT_DEFAULT_IDENT time_stamp = timezone.now() cwd = os.path.abspath(rdirp) commit_msg = u"Export from Studio at {time_stamp}".format( time_stamp=time_stamp, ) try: cmd_log(['git', 'config', 'user.email', ident['email']], cwd) cmd_log(['git', 'config', 'user.name', ident['name']], cwd) except subprocess.CalledProcessError as ex: log.exception(u'Error running git configure commands: %r', ex.output) raise GitExportError(GitExportError.CONFIG_ERROR) try: cmd_log(['git', 'add', '.'], cwd) cmd_log(['git', 'commit', '-a', '-m', commit_msg], cwd) except subprocess.CalledProcessError as ex: log.exception(u'Unable to commit changes: %r', ex.output) raise GitExportError(GitExportError.CANNOT_COMMIT) try: cmd_log(['git', 'push', '-q', 'origin', branch], cwd) except subprocess.CalledProcessError as ex: log.exception(u'Error running git push command: %r', ex.output) raise GitExportError(GitExportError.CANNOT_PUSH)
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build(contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build(contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=[course_data_name], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) # NOT CURRENTLY USED # export_course_to_xml( # dest_store, # dest_content, # dest_course_key, # self.export_dir, # 'exported_dest_course', # ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.exclude_field(None, 'parent') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, ) self.assertAssetsMetadataEqual( source_store, source_course_key, dest_store, dest_course_key, )
def get(self, request, course_key_string): """ The restful handler for exporting a full course or content library. GET application/x-tgz: return tar.gz file containing exported course json: not supported Note that there are 2 ways to request the tar.gz file. The request header can specify application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?accept=application/x-tgz). If the tar.gz file has been requested but the export operation fails, a JSON string will be returned which describes the error """ redirect_url = request.QUERY_PARAMS.get('redirect', None) courselike_key = CourseKey.from_string(course_key_string) library = isinstance(courselike_key, LibraryLocator) if library: courselike_module = modulestore().get_library(courselike_key) else: courselike_module = modulestore().get_course(courselike_key) name = courselike_module.url_name export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp_clean()) try: if library: export_library_to_xml(modulestore(), contentstore(), courselike_key, root_dir, name) else: export_course_to_xml(modulestore(), contentstore(), courselike_module.id, root_dir, name) logging.debug(u'tar file being generated at %s', export_file.name) with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: tar_file.add(root_dir / name, arcname=name) except SerializationError as exc: log.exception(u'There was an error exporting course %s', courselike_key) unit = None failed_item = None parent = None try: failed_item = modulestore().get_item(exc.location) parent_loc = modulestore().get_parent_location( failed_item.location) if parent_loc is not None: parent = modulestore().get_item(parent_loc) if parent.location.category == 'vertical': unit = parent except Exception: # pylint: disable=broad-except # if we have a nested exception, then we'll show the more # generic error message pass return self._export_error_response( { "context_course": str(courselike_module.location), "error": True, "error_message": str(exc), "failed_module": str(failed_item.location) if failed_item else None, "unit": str(unit.location) if unit else None }, redirect_url=redirect_url) except Exception as exc: # pylint: disable=broad-except log.exception('There was an error exporting course %s', courselike_key) return self._export_error_response( { "context_course": courselike_module.url_name, "error": True, "error_message": str(exc), "unit": None }, redirect_url=redirect_url) # The course is all set; return the tar.gz wrapper = FileWrapper(export_file) response = HttpResponse(wrapper, content_type='application/x-tgz') response['Content-Disposition'] = 'attachment; filename={}'.format( os.path.basename(export_file.name.encode('utf-8'))) response['Content-Length'] = os.path.getsize(export_file.name) return response
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build( contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build( contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=[course_data_name], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) # NOT CURRENTLY USED # export_course_to_xml( # dest_store, # dest_content, # dest_course_key, # self.export_dir, # 'exported_dest_course', # ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.exclude_field(None, 'parent') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, ) self.assertAssetsMetadataEqual( source_store, source_course_key, dest_store, dest_course_key, )
def test_round_trip( self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name, _mock_tab_from_json ): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build(contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build(contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=[course_data_name], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME, ) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=[EXPORTED_COURSE_DIR_NAME], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) # NOT CURRENTLY USED # export_course_to_xml( # dest_store, # dest_content, # dest_course_key, # self.export_dir, # 'exported_dest_course', # ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.exclude_field(None, 'parent') # discussion_ids are auto-generated based on usage_id, so they should change across # modulestores - see TNL-5001 self.exclude_field(None, 'discussion_id') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, ) self.assertAssetsMetadataEqual( source_store, source_course_key, dest_store, dest_course_key, )
def export_to_git(course_id, repo, user='', rdir=None): """Export a course to git.""" # pylint: disable=too-many-statements if not GIT_REPO_EXPORT_DIR: raise GitExportError(GitExportError.NO_EXPORT_DIR) if not os.path.isdir(GIT_REPO_EXPORT_DIR): raise GitExportError(GitExportError.NO_EXPORT_DIR) # Check for valid writable git url if not (repo.endswith('.git') or repo.startswith(('http:', 'https:', 'file:'))): raise GitExportError(GitExportError.URL_BAD) # Check for username and password if using http[s] if repo.startswith('http:') or repo.startswith('https:'): parsed = urlparse(repo) if parsed.username is None or parsed.password is None: raise GitExportError(GitExportError.URL_NO_AUTH) if rdir: rdir = os.path.basename(rdir) else: rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0] log.debug(u"rdir = %s", rdir) # Pull or clone repo before exporting to xml # and update url in case origin changed. rdirp = '{0}/{1}'.format(GIT_REPO_EXPORT_DIR, rdir) branch = None if os.path.exists(rdirp): log.info('Directory already exists, doing a git reset and pull ' 'instead of git clone.') cwd = rdirp # Get current branch cmd = ['git', 'symbolic-ref', '--short', 'HEAD'] try: branch = cmd_log(cmd, cwd).strip('\n') except subprocess.CalledProcessError as ex: log.exception(u'Failed to get branch: %r', ex.output) raise GitExportError(GitExportError.DETACHED_HEAD) cmds = [ ['git', 'remote', 'set-url', 'origin', repo], ['git', 'fetch', 'origin'], ['git', 'reset', '--hard', 'origin/{0}'.format(branch)], ['git', 'pull'], ['git', 'clean', '-d', '-f'], ] else: cmds = [['git', 'clone', repo]] cwd = GIT_REPO_EXPORT_DIR cwd = os.path.abspath(cwd) for cmd in cmds: try: cmd_log(cmd, cwd) except subprocess.CalledProcessError as ex: log.exception(u'Failed to pull git repository: %r', ex.output) raise GitExportError(GitExportError.CANNOT_PULL) # export course as xml before commiting and pushing root_dir = os.path.dirname(rdirp) course_dir = os.path.basename(rdirp).rsplit('.git', 1)[0] try: export_course_to_xml(modulestore(), contentstore(), course_id, root_dir, course_dir) except (EnvironmentError, AttributeError): log.exception('Failed export to xml') raise GitExportError(GitExportError.XML_EXPORT_FAIL) # Get current branch if not already set if not branch: cmd = ['git', 'symbolic-ref', '--short', 'HEAD'] try: branch = cmd_log(cmd, os.path.abspath(rdirp)).strip('\n') except subprocess.CalledProcessError as ex: log.exception(u'Failed to get branch from freshly cloned repo: %r', ex.output) raise GitExportError(GitExportError.MISSING_BRANCH) # Now that we have fresh xml exported, set identity, add # everything to git, commit, and push to the right branch. ident = {} try: user = User.objects.get(username=user) ident['name'] = user.username ident['email'] = user.email except User.DoesNotExist: # That's ok, just use default ident ident = GIT_EXPORT_DEFAULT_IDENT time_stamp = timezone.now() cwd = os.path.abspath(rdirp) commit_msg = u"Export from Studio at {time_stamp}".format( time_stamp=time_stamp, ) try: cmd_log(['git', 'config', 'user.email', ident['email']], cwd) cmd_log(['git', 'config', 'user.name', ident['name']], cwd) except subprocess.CalledProcessError as ex: log.exception(u'Error running git configure commands: %r', ex.output) raise GitExportError(GitExportError.CONFIG_ERROR) try: cmd_log(['git', 'add', '.'], cwd) cmd_log(['git', 'commit', '-a', '-m', commit_msg], cwd) except subprocess.CalledProcessError as ex: log.exception(u'Unable to commit changes: %r', ex.output) raise GitExportError(GitExportError.CANNOT_COMMIT) try: cmd_log(['git', 'push', '-q', 'origin', branch], cwd) except subprocess.CalledProcessError as ex: log.exception(u'Error running git push command: %r', ex.output) raise GitExportError(GitExportError.CANNOT_PUSH)
def test_round_trip( self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name, _mock_tab_from_json, ): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build(contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build(contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key("a", "course", "course") dest_course_key = dest_store.make_course_key("a", "course", "course") import_course_from_xml( source_store, "test_user", TEST_DATA_DIR, source_dirs=[course_data_name], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME ) import_course_from_xml( dest_store, "test_user", self.export_dir, source_dirs=[EXPORTED_COURSE_DIR_NAME], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) # NOT CURRENTLY USED # export_course_to_xml( # dest_store, # dest_content, # dest_course_key, # self.export_dir, # 'exported_dest_course', # ) self.exclude_field(None, "wiki_slug") self.exclude_field(None, "xml_attributes") self.exclude_field(None, "parent") self.ignore_asset_key("_id") self.ignore_asset_key("uploadDate") self.ignore_asset_key("content_son") self.ignore_asset_key("thumbnail_location") self.assertCoursesEqual(source_store, source_course_key, dest_store, dest_course_key) self.assertAssetsEqual(source_content, source_course_key, dest_content, dest_course_key) self.assertAssetsMetadataEqual(source_store, source_course_key, dest_store, dest_course_key)
def export_to_git(course_id, repo, user="", rdir=None): """Export a course to git.""" # pylint: disable=too-many-statements if not GIT_REPO_EXPORT_DIR: raise GitExportError(GitExportError.NO_EXPORT_DIR) if not os.path.isdir(GIT_REPO_EXPORT_DIR): raise GitExportError(GitExportError.NO_EXPORT_DIR) # Check for valid writable git url if not (repo.endswith(".git") or repo.startswith(("http:", "https:", "file:"))): raise GitExportError(GitExportError.URL_BAD) # Check for username and password if using http[s] if repo.startswith("http:") or repo.startswith("https:"): parsed = urlparse(repo) if parsed.username is None or parsed.password is None: raise GitExportError(GitExportError.URL_NO_AUTH) if rdir: rdir = os.path.basename(rdir) else: rdir = repo.rsplit("/", 1)[-1].rsplit(".git", 1)[0] log.debug("rdir = %s", rdir) # Pull or clone repo before exporting to xml # and update url in case origin changed. rdirp = "{0}/{1}".format(GIT_REPO_EXPORT_DIR, rdir) branch = None if os.path.exists(rdirp): log.info("Directory already exists, doing a git reset and pull " "instead of git clone.") cwd = rdirp # Get current branch cmd = ["git", "symbolic-ref", "--short", "HEAD"] try: branch = cmd_log(cmd, cwd).strip("\n") except subprocess.CalledProcessError as ex: log.exception("Failed to get branch: %r", ex.output) raise GitExportError(GitExportError.DETACHED_HEAD) cmds = [ ["git", "remote", "set-url", "origin", repo], ["git", "fetch", "origin"], ["git", "reset", "--hard", "origin/{0}".format(branch)], ["git", "pull"], ["git", "clean", "-d", "-f"], ] else: cmds = [["git", "clone", repo]] cwd = GIT_REPO_EXPORT_DIR cwd = os.path.abspath(cwd) for cmd in cmds: try: cmd_log(cmd, cwd) except subprocess.CalledProcessError as ex: log.exception("Failed to pull git repository: %r", ex.output) raise GitExportError(GitExportError.CANNOT_PULL) # export course as xml before commiting and pushing root_dir = os.path.dirname(rdirp) course_dir = os.path.basename(rdirp).rsplit(".git", 1)[0] try: export_course_to_xml(modulestore(), contentstore(), course_id, root_dir, course_dir) except (EnvironmentError, AttributeError): log.exception("Failed export to xml") raise GitExportError(GitExportError.XML_EXPORT_FAIL) # Get current branch if not already set if not branch: cmd = ["git", "symbolic-ref", "--short", "HEAD"] try: branch = cmd_log(cmd, os.path.abspath(rdirp)).strip("\n") except subprocess.CalledProcessError as ex: log.exception("Failed to get branch from freshly cloned repo: %r", ex.output) raise GitExportError(GitExportError.MISSING_BRANCH) # Now that we have fresh xml exported, set identity, add # everything to git, commit, and push to the right branch. ident = {} try: user = User.objects.get(username=user) ident["name"] = user.username ident["email"] = user.email except User.DoesNotExist: # That's ok, just use default ident ident = GIT_EXPORT_DEFAULT_IDENT time_stamp = timezone.now() cwd = os.path.abspath(rdirp) commit_msg = "Export from Studio at {time_stamp}".format(time_stamp=time_stamp) try: cmd_log(["git", "config", "user.email", ident["email"]], cwd) cmd_log(["git", "config", "user.name", ident["name"]], cwd) except subprocess.CalledProcessError as ex: log.exception("Error running git configure commands: %r", ex.output) raise GitExportError(GitExportError.CONFIG_ERROR) try: cmd_log(["git", "add", "."], cwd) cmd_log(["git", "commit", "-a", "-m", commit_msg], cwd) except subprocess.CalledProcessError as ex: log.exception("Unable to commit changes: %r", ex.output) raise GitExportError(GitExportError.CANNOT_COMMIT) try: cmd_log(["git", "push", "-q", "origin", branch], cwd) except subprocess.CalledProcessError as ex: log.exception("Error running git push command: %r", ex.output) raise GitExportError(GitExportError.CANNOT_PUSH)
def get(self, request, course_key_string): """ The restful handler for exporting a full course or content library. GET application/x-tgz: return tar.gz file containing exported course json: not supported Note that there are 2 ways to request the tar.gz file. The request header can specify application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?accept=application/x-tgz). If the tar.gz file has been requested but the export operation fails, a JSON string will be returned which describes the error """ redirect_url = request.QUERY_PARAMS.get('redirect', None) courselike_key = CourseKey.from_string(course_key_string) library = isinstance(courselike_key, LibraryLocator) if library: courselike_module = modulestore().get_library(courselike_key) else: courselike_module = modulestore().get_course(courselike_key) name = courselike_module.url_name export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp_clean()) try: if library: export_library_to_xml( modulestore(), contentstore(), courselike_key, root_dir, name ) else: export_course_to_xml( modulestore(), contentstore(), courselike_module.id, root_dir, name ) logging.debug( u'tar file being generated at %s', export_file.name ) with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: tar_file.add(root_dir / name, arcname=name) except SerializationError as exc: log.exception( u'There was an error exporting course %s', courselike_key ) unit = None failed_item = None parent = None try: failed_item = modulestore().get_item(exc.location) parent_loc = modulestore().get_parent_location( failed_item.location ) if parent_loc is not None: parent = modulestore().get_item(parent_loc) if parent.location.category == 'vertical': unit = parent except Exception: # pylint: disable=broad-except # if we have a nested exception, then we'll show the more # generic error message pass return self._export_error_response( { "context_course": str(courselike_module.location), "error": True, "error_message": str(exc), "failed_module": str(failed_item.location) if failed_item else None, "unit": str(unit.location) if unit else None }, redirect_url=redirect_url ) except Exception as exc: # pylint: disable=broad-except log.exception( 'There was an error exporting course %s', courselike_key ) return self._export_error_response( { "context_course": courselike_module.url_name, "error": True, "error_message": str(exc), "unit": None }, redirect_url=redirect_url ) # The course is all set; return the tar.gz wrapper = FileWrapper(export_file) response = HttpResponse(wrapper, content_type='application/x-tgz') response['Content-Disposition'] = 'attachment; filename={}'.format( os.path.basename( export_file.name.encode('utf-8') ) ) response['Content-Length'] = os.path.getsize(export_file.name) return response