def test_generate_timings(self, source_ms, dest_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ desc = "XMLRoundTrip:{}->{}:{}".format(SHORT_NAME_MAP[source_ms], SHORT_NAME_MAP[dest_ms], num_assets) with CodeBlockTimer(desc): with CodeBlockTimer("fake_assets"): # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) # Construct the contentstore for storing the first import with MongoContentstoreBuilder().build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_ms.build(source_content) as source_store: # Construct the contentstore for storing the second import with MongoContentstoreBuilder().build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_ms.build(dest_content) as dest_store: source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') with CodeBlockTimer("initial_import"): import_from_xml( source_store, 'test_user', TEST_DATA_ROOT, course_dirs=TEST_COURSE, static_content_store=source_content, target_course_id=source_course_key, create_course_if_not_present=True, raise_on_failure=True, ) with CodeBlockTimer("export"): export_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with CodeBlockTimer("second_import"): import_from_xml( dest_store, 'test_user', self.export_dir, course_dirs=['exported_source_course'], static_content_store=dest_content, target_course_id=dest_course_key, create_course_if_not_present=True, raise_on_failure=True, )
def handle(self, *args, **options): "Execute the command" if len(args) != 1: raise CommandError("export requires one argument: <output path>") output_path = args[0] cs = contentstore() ms = modulestore('direct') root_dir = output_path courses = ms.get_courses() print("%d courses to export:" % len(courses)) cids = [x.id for x in courses] print(cids) for course_id in cids: print("-"*77) print("Exporting course id = {0} to {1}".format(course_id, output_path)) if 1: try: course_dir = course_id.replace('/', '...') export_to_xml(ms, cs, course_id, root_dir, course_dir, modulestore()) except Exception as err: print("="*30 + "> Oops, failed to export %s" % course_id) print("Error:") print(err)
def handle(self, *args, **options): if len(args) != 1: raise CommandError("export requires one argument: <output path>") output_path = args[0] cs = contentstore() ms = modulestore('direct') root_dir = output_path courses = ms.get_courses() print "%d courses to export:" % len(courses) cids = [x.id for x in courses] print cids for course_id in cids: print "-" * 77 print "Exporting course id = {0} to {1}".format( course_id, output_path) if 1: try: location = CourseDescriptor.id_to_location(course_id) course_dir = course_id.replace('/', '...') export_to_xml(ms, cs, location, root_dir, course_dir, modulestore()) except Exception as err: print "=" * 30 + "> Oops, failed to export %s" % course_id print "Error:" print err
def generate_export_course(request, org, course, name): """ This method will serialize out a course to a .tar.gz file which contains a XML-based representation of the course """ location = get_location_and_verify_access(request, org, course, name) loc = Location(location) export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp()) # export out to a tempdir logging.debug('root = {0}'.format(root_dir)) export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore()) logging.debug('tar file being generated at {0}'.format(export_file.name)) tar_file = tarfile.open(name=export_file.name, mode='w:gz') tar_file.add(root_dir / name, arcname=name) tar_file.close() # remove temp dir shutil.rmtree(root_dir / name) wrapper = FileWrapper(export_file) response = HttpResponse(wrapper, content_type='application/x-tgz') response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name) response['Content-Length'] = os.path.getsize(export_file.name) return response
def handle(self, *args, **options): if len(args) != 1: raise CommandError("export requires one argument: <output path>") output_path = args[0] cs = contentstore() ms = modulestore("direct") root_dir = output_path courses = ms.get_courses() print "%d courses to export:" % len(courses) cids = [x.id for x in courses] print cids for course_id in cids: print "-" * 77 print "Exporting course id = {0} to {1}".format(course_id, output_path) if 1: try: location = CourseDescriptor.id_to_location(course_id) course_dir = course_id.replace("/", "...") export_to_xml(ms, cs, location, root_dir, course_dir, modulestore()) except Exception as err: print "=" * 30 + "> Oops, failed to export %s" % course_id print "Error:" print err
def generate_export_course(request, org, course, name): location = get_location_and_verify_access(request, org, course, name) loc = Location(location) export_file = NamedTemporaryFile(prefix=name + ".", suffix=".tar.gz") root_dir = path(mkdtemp()) # export out to a tempdir logging.debug("root = {0}".format(root_dir)) export_to_xml(modulestore("direct"), contentstore(), loc, root_dir, name, modulestore()) # filename = root_dir / name + '.tar.gz' logging.debug("tar file being generated at {0}".format(export_file.name)) tar_file = tarfile.open(name=export_file.name, mode="w:gz") tar_file.add(root_dir / name, arcname=name) tar_file.close() # remove temp dir shutil.rmtree(root_dir / name) wrapper = FileWrapper(export_file) response = HttpResponse(wrapper, content_type="application/x-tgz") response["Content-Disposition"] = "attachment; filename=%s" % os.path.basename(export_file.name) response["Content-Length"] = os.path.getsize(export_file.name) return response
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build(source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build(dest_content) as dest_store: source_course_key = source_store.make_course_key('source', 'course', 'key') dest_course_key = dest_store.make_course_key('dest', 'course', 'key') import_from_xml( source_store, 'test_user', 'common/test/data', course_dirs=[course_data_name], static_content_store=source_content, target_course_id=source_course_key, create_new_course_if_not_present=True, ) export_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_course', ) import_from_xml( dest_store, 'test_user', self.export_dir, static_content_store=dest_content, target_course_id=dest_course_key, create_new_course_if_not_present=True, ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, )
def export_courses_to_output_path(output_path): """ Export all courses to target directory and return the list of courses which failed to export """ content_store = contentstore() module_store = modulestore() root_dir = output_path courses = module_store.get_courses() course_ids = [x.id for x in courses] failed_export_courses = [] for course_id in course_ids: print(u"-" * 80) print(u"Exporting course id = {0} to {1}".format(course_id, output_path)) try: course_dir = course_id.to_deprecated_string().replace('/', '...') export_to_xml(module_store, content_store, course_id, root_dir, course_dir) except Exception as err: # pylint: disable=broad-except failed_export_courses.append(unicode(course_id)) print(u"=" * 30 + u"> Oops, failed to export {0}".format(course_id)) print(u"Error:") print(err) return courses, failed_export_courses
def generate_export_course(request, org, course, name): """ This method will serialize out a course to a .tar.gz file which contains a XML-based representation of the course """ location = get_location_and_verify_access(request, org, course, name) course_module = modulestore().get_instance(location.course_id, location) loc = Location(location) export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") new_location = loc_mapper().translate_location( course_module.location.course_id, course_module.location, False, True) root_dir = path(mkdtemp()) try: export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore()) except SerializationError, e: logging.exception( 'There was an error exporting course {0}. {1}'.format( course_module.location, unicode(e))) unit = None failed_item = None parent = None try: failed_item = modulestore().get_instance( course_module.location.course_id, e.location) parent_locs = modulestore().get_parent_locations( failed_item.location, course_module.location.course_id) if len(parent_locs) > 0: parent = modulestore().get_item(parent_locs[0]) if parent.location.category == 'vertical': unit = parent except: # if we have a nested exception, then we'll show the more generic error message pass return render_to_response( 'export.html', { 'context_course': course_module, 'successful_import_redirect_url': '', 'in_err': True, 'raw_err_msg': str(e), 'failed_module': failed_item, 'unit': unit, 'edit_unit_url': reverse('edit_unit', kwargs={'location': parent.location}) if parent else '', 'course_home_url': new_location.url_reverse("course/", "") })
def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ if CodeBlockTimer is None: raise SkipTest("CodeBlockTimer undefined.") desc = "XMLRoundTrip:{}->{}:{}".format(SHORT_NAME_MAP[source_ms], SHORT_NAME_MAP[dest_ms], num_assets) with CodeBlockTimer(desc): with CodeBlockTimer("fake_assets"): # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) with source_ms.build() as (source_content, source_store): with dest_ms.build() as (dest_content, dest_store): source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') with CodeBlockTimer("initial_import"): import_from_xml( source_store, 'test_user', TEST_DATA_ROOT, course_dirs=TEST_COURSE, static_content_store=source_content, target_course_id=source_course_key, create_course_if_not_present=True, raise_on_failure=True, ) with CodeBlockTimer("export"): export_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with CodeBlockTimer("second_import"): import_from_xml( dest_store, 'test_user', self.export_dir, course_dirs=['exported_source_course'], static_content_store=dest_content, target_course_id=dest_course_key, create_course_if_not_present=True, raise_on_failure=True, )
def export_course_to_directory(course_id, root_dir): """Export course into a directory""" store = modulestore() course = store.get_course(course_id) if course is None: raise CommandError("Invalid course_id") course_name = course.location.course_id.replace('/', '-') export_to_xml(store, None, course.location, root_dir, course_name) course_dir = path(root_dir) / course_name return course_dir
def export_course_to_directory(course_id, root_dir): """Export course into a directory""" store = modulestore() course = store.get_course(course_id) if course is None: raise CommandError("Invalid course_id") course_name = course.id.to_deprecated_string().replace('/', '-') export_to_xml(store, None, course.id, root_dir, course_name) course_dir = path(root_dir) / course_name return course_dir
def test_course_without_image(self): """ Make sure we elegantly passover our code when there isn't a static image """ course = self.get_course_by_id('edX/simple_with_draft/2012_Fall') root_dir = path(mkdtemp()) try: export_to_xml(self.store, self.content_store, course.location, root_dir, 'test_export') assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) assert_false(path(root_dir / 'test_export/static/images_course_image.jpg').isfile()) finally: shutil.rmtree(root_dir)
def test_course_without_image(self): """ Make sure we elegantly passover our code when there isn't a static image """ course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'simple_with_draft', '2012_Fall')) root_dir = path(mkdtemp()) try: export_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export') assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) assert_false(path(root_dir / 'test_export/static/images_course_image.jpg').isfile()) finally: shutil.rmtree(root_dir)
def test_export_course_with_peer_component(self): """ Test export course when link_to_location is given in peer grading interface settings. """ name = "export_peer_component" locations = self._create_test_tree(name) # Insert the test block directly into the module store problem_location = Location('edX', 'tree{}'.format(name), name, 'combinedopenended', 'test_peer_problem') self.draft_store.create_child(self.dummy_user, locations["child"], problem_location.block_type, block_id=problem_location.block_id) interface_location = Location('edX', 'tree{}'.format(name), name, 'peergrading', 'test_peer_interface') self.draft_store.create_child(self.dummy_user, locations["child"], interface_location.block_type, block_id=interface_location.block_id) self.draft_store._update_single_item( as_draft(interface_location), { 'definition.data': {}, 'metadata': { 'link_to_location': unicode(problem_location), 'use_for_single_location': True, }, }, ) component = self.draft_store.get_item(interface_location) self.assertEqual(unicode(component.link_to_location), unicode(problem_location)) root_dir = path(mkdtemp()) # export_to_xml should work. try: export_to_xml(self.draft_store, self.content_store, interface_location.course_key, root_dir, 'test_export') finally: shutil.rmtree(root_dir)
def test_export_course_with_peer_component(self): """ Test export course when link_to_location is given in peer grading interface settings. """ name = "export_peer_component" locations = self._create_test_tree(name) # Insert the test block directly into the module store problem_location = Location('edX', 'tree{}'.format(name), name, 'combinedopenended', 'test_peer_problem') self.draft_store.create_child( self.dummy_user, locations["child"], problem_location.block_type, block_id=problem_location.block_id ) interface_location = Location('edX', 'tree{}'.format(name), name, 'peergrading', 'test_peer_interface') self.draft_store.create_child( self.dummy_user, locations["child"], interface_location.block_type, block_id=interface_location.block_id ) self.draft_store._update_single_item( as_draft(interface_location), { 'definition.data': {}, 'metadata': { 'link_to_location': unicode(problem_location), 'use_for_single_location': True, }, }, ) component = self.draft_store.get_item(interface_location) self.assertEqual(unicode(component.link_to_location), unicode(problem_location)) root_dir = path(mkdtemp()) # export_to_xml should work. try: export_to_xml(self.draft_store, self.content_store, interface_location.course_key, root_dir, 'test_export') finally: shutil.rmtree(root_dir)
def handle(self, *args, **options): if len(args) != 2: raise CommandError("import requires two arguments: <course location> <output path>") course_id = args[0] output_path = args[1] print "Exporting course id = {0} to {1}".format(course_id, output_path) location = CourseDescriptor.id_to_location(course_id) root_dir = os.path.dirname(output_path) course_dir = os.path.splitext(os.path.basename(output_path))[0] export_to_xml(modulestore('direct'), contentstore(), location, root_dir, course_dir)
def test_export_course_image_nondefault(self): """ Make sure that if a non-default image path is specified that we don't export it to the static default location """ course = self.draft_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')) assert_true(course.course_image, 'just_a_test.jpg') root_dir = path(mkdtemp()) try: export_to_xml(self.draft_store, self.content_store, course.id, root_dir, 'test_export') assert_true(path(root_dir / 'test_export/static/just_a_test.jpg').isfile()) assert_false(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) finally: shutil.rmtree(root_dir)
def generate_export_course(request, org, course, name): """ This method will serialize out a course to a .tar.gz file which contains a XML-based representation of the course """ location = get_location_and_verify_access(request, org, course, name) course_module = modulestore().get_instance(location.course_id, location) loc = Location(location) export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp()) try: export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore()) except SerializationError, e: logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e))) unit = None failed_item = None parent = None try: failed_item = modulestore().get_instance(course_module.location.course_id, e.location) parent_locs = modulestore().get_parent_locations(failed_item.location, course_module.location.course_id) if len(parent_locs) > 0: parent = modulestore().get_item(parent_locs[0]) if parent.location.category == 'vertical': unit = parent except: # if we have a nested exception, then we'll show the more generic error message pass return render_to_response('export.html', { 'context_course': course_module, 'successful_import_redirect_url': '', 'in_err': True, 'raw_err_msg': str(e), 'failed_module': failed_item, 'unit': unit, 'edit_unit_url': reverse('edit_unit', kwargs={ 'location': parent.location }) if parent else '', 'course_home_url': reverse('course_index', kwargs={ 'org': org, 'course': course, 'name': name }) })
def handle(self, *args, **options): if len(args) != 2: raise CommandError( "import requires two arguments: <course location> <output path>") course_id = args[0] output_path = args[1] print "Exporting course id = {0} to {1}".format(course_id, output_path) location = CourseDescriptor.id_to_location(course_id) root_dir = os.path.dirname(output_path) course_dir = os.path.splitext(os.path.basename(output_path))[0] export_to_xml(modulestore( 'direct'), contentstore(), location, root_dir, course_dir)
def handle(self, *args, **options): "Execute the command" if len(args) != 2: raise CommandError("export requires two arguments: <course id> <output path>") try: course_key = CourseKey.from_string(args[0]) except InvalidKeyError: course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0]) output_path = args[1] print("Exporting course id = {0} to {1}".format(course_key, output_path)) root_dir = os.path.dirname(output_path) course_dir = os.path.splitext(os.path.basename(output_path))[0] export_to_xml(modulestore('direct'), contentstore(), course_key, root_dir, course_dir, modulestore())
def test_export_course_image(self): """ Test to make sure that we have a course image in the contentstore, then export it to ensure it gets copied to both file locations. """ course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall') location = course_key.make_asset_key('asset', 'images_course_image.jpg') # This will raise if the course image is missing self.content_store.find(location) root_dir = path(mkdtemp()) try: export_to_xml(self.draft_store, self.content_store, course_key, root_dir, 'test_export') assert_true(path(root_dir / 'test_export/static/images/course_image.jpg').isfile()) assert_true(path(root_dir / 'test_export/static/images_course_image.jpg').isfile()) finally: shutil.rmtree(root_dir)
def export_course_to_directory(course_key, root_dir): """Export course into a directory""" store = modulestore() course = store.get_course(course_key) if course is None: raise CommandError("Invalid course_id") # The safest characters are A-Z, a-z, 0-9, <underscore>, <period> and <hyphen>. # We represent the first four with \w. # TODO: Once we support courses with unicode characters, we will need to revisit this. replacement_char = u'-' course_dir = replacement_char.join([course.id.org, course.id.course, course.id.run]) course_dir = re.sub(r'[^\w\.\-]', replacement_char, course_dir) export_to_xml(store, None, course.id, root_dir, course_dir) export_dir = path(root_dir) / course_dir return export_dir
def handle(self, *args, **options): "Execute the command" if len(args) != 2: raise CommandError("export requires two arguments: <course id> <output path>") try: course_key = CourseKey.from_string(args[0]) except InvalidKeyError: course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0]) output_path = args[1] print("Exporting course id = {0} to {1}".format(course_key, output_path)) root_dir = os.path.dirname(output_path) course_dir = os.path.splitext(os.path.basename(output_path))[0] export_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes): with store_builder.build() as (source_content, source_store): with store_builder.build() as (dest_content, dest_store): source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') # An extra import write occurs in the first Split import due to the mismatch between # the course id and the wiki_slug in the test XML course. The course must be updated # with the correct wiki_slug during import. with check_mongo_calls(import_reads, first_import_writes): import_from_xml( source_store, 'test_user', TEST_DATA_DIR, course_dirs=['manual-testing-complete'], static_content_store=source_content, target_course_id=source_course_key, create_course_if_not_present=True, raise_on_failure=True, ) with check_mongo_calls(export_reads): export_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with check_mongo_calls(import_reads, second_import_writes): import_from_xml( dest_store, 'test_user', self.export_dir, course_dirs=['exported_source_course'], static_content_store=dest_content, target_course_id=dest_course_key, create_course_if_not_present=True, raise_on_failure=True, )
def test_export_course_with_unknown_metadata(self): module_store = modulestore('direct') content_store = contentstore() import_from_xml(module_store, 'common/test/data/', ['full']) location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012') root_dir = path(mkdtemp_clean()) course = module_store.get_item(location) metadata = own_metadata(course) # add a bool piece of unknown metadata so we can verify we don't throw an exception metadata['new_metadata'] = True module_store.update_metadata(location, metadata) print 'Exporting to tempdir = {0}'.format(root_dir) # export out to a tempdir export_to_xml(module_store, content_store, location, root_dir, 'test_export')
def export_to_git(course_loc, repo, user='', rdir=None): """Export a course to git.""" # pylint: disable=R0915 if course_loc.startswith('i4x://'): course_loc = course_loc[6:] if not GIT_REPO_EXPORT_DIR: raise GitExportError(GitExportError.NO_EXPORT_DIR) if not os.path.isdir(GIT_REPO_EXPORT_DIR): raise GitExportError(GitExportError.NO_EXPORT_DIR) # Check for valid writable git url if not (repo.endswith('.git') or repo.startswith( ('http:', 'https:', 'file:'))): raise GitExportError(GitExportError.URL_BAD) # Check for username and password if using http[s] if repo.startswith('http:') or repo.startswith('https:'): parsed = urlparse(repo) if parsed.username is None or parsed.password is None: raise GitExportError(GitExportError.URL_NO_AUTH) if rdir: rdir = os.path.basename(rdir) else: rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0] log.debug("rdir = %s", rdir) # Pull or clone repo before exporting to xml # and update url in case origin changed. rdirp = '{0}/{1}'.format(GIT_REPO_EXPORT_DIR, rdir) branch = None if os.path.exists(rdirp): log.info( _('Directory already exists, doing a git reset and pull ' 'instead of git clone.')) cwd = rdirp # Get current branch cmd = ['git', 'symbolic-ref', '--short', 'HEAD'] try: branch = cmd_log(cmd, cwd).strip('\n') except subprocess.CalledProcessError as ex: log.exception('Failed to get branch: %r', ex.output) raise GitExportError(GitExportError.DETACHED_HEAD) cmds = [ ['git', 'remote', 'set-url', 'origin', repo], ['git', 'fetch', 'origin'], ['git', 'reset', '--hard', 'origin/{0}'.format(branch)], ['git', 'pull'], ] else: cmds = [['git', 'clone', repo]] cwd = GIT_REPO_EXPORT_DIR cwd = os.path.abspath(cwd) for cmd in cmds: try: cmd_log(cmd, cwd) except subprocess.CalledProcessError as ex: log.exception('Failed to pull git repository: %r', ex.output) raise GitExportError(GitExportError.CANNOT_PULL) # export course as xml before commiting and pushing try: location = CourseDescriptor.id_to_location(course_loc) except ValueError: raise GitExportError(GitExportError.BAD_COURSE) root_dir = os.path.dirname(rdirp) course_dir = os.path.splitext(os.path.basename(rdirp))[0] try: export_to_xml(modulestore('direct'), contentstore(), location, root_dir, course_dir, modulestore()) except (EnvironmentError, AttributeError): log.exception('Failed export to xml') raise GitExportError(GitExportError.XML_EXPORT_FAIL) # Get current branch if not already set if not branch: cmd = ['git', 'symbolic-ref', '--short', 'HEAD'] try: branch = cmd_log(cmd, os.path.abspath(rdirp)).strip('\n') except subprocess.CalledProcessError as ex: log.exception('Failed to get branch from freshly cloned repo: %r', ex.output) raise GitExportError(GitExportError.MISSING_BRANCH) # Now that we have fresh xml exported, set identity, add # everything to git, commit, and push to the right branch. ident = {} try: user = User.objects.get(username=user) ident['name'] = user.username ident['email'] = user.email except User.DoesNotExist: # That's ok, just use default ident ident = GIT_EXPORT_DEFAULT_IDENT time_stamp = timezone.now() cwd = os.path.abspath(rdirp) commit_msg = 'Export from Studio at {1}'.format(user, time_stamp) try: cmd_log(['git', 'config', 'user.email', ident['email']], cwd) cmd_log(['git', 'config', 'user.name', ident['name']], cwd) except subprocess.CalledProcessError as ex: log.exception('Error running git configure commands: %r', ex.output) raise GitExportError(GitExportError.CONFIG_ERROR) try: cmd_log(['git', 'add', '.'], cwd) cmd_log(['git', 'commit', '-a', '-m', commit_msg], cwd) except subprocess.CalledProcessError as ex: log.exception('Unable to commit changes: %r', ex.output) raise GitExportError(GitExportError.CANNOT_COMMIT) try: cmd_log(['git', 'push', '-q', 'origin', branch], cwd) except subprocess.CalledProcessError as ex: log.exception('Error running git push command: %r', ex.output) raise GitExportError(GitExportError.CANNOT_PUSH)
def export_to_git(course_id, repo, user='', rdir=None): """Export a course to git.""" # pylint: disable=R0915 if not GIT_REPO_EXPORT_DIR: raise GitExportError(GitExportError.NO_EXPORT_DIR) if not os.path.isdir(GIT_REPO_EXPORT_DIR): raise GitExportError(GitExportError.NO_EXPORT_DIR) # Check for valid writable git url if not (repo.endswith('.git') or repo.startswith(('http:', 'https:', 'file:'))): raise GitExportError(GitExportError.URL_BAD) # Check for username and password if using http[s] if repo.startswith('http:') or repo.startswith('https:'): parsed = urlparse(repo) if parsed.username is None or parsed.password is None: raise GitExportError(GitExportError.URL_NO_AUTH) if rdir: rdir = os.path.basename(rdir) else: rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0] log.debug("rdir = %s", rdir) # Pull or clone repo before exporting to xml # and update url in case origin changed. rdirp = '{0}/{1}'.format(GIT_REPO_EXPORT_DIR, rdir) branch = None if os.path.exists(rdirp): log.info('Directory already exists, doing a git reset and pull ' 'instead of git clone.') cwd = rdirp # Get current branch cmd = ['git', 'symbolic-ref', '--short', 'HEAD'] try: branch = cmd_log(cmd, cwd).strip('\n') except subprocess.CalledProcessError as ex: log.exception('Failed to get branch: %r', ex.output) raise GitExportError(GitExportError.DETACHED_HEAD) cmds = [ ['git', 'remote', 'set-url', 'origin', repo], ['git', 'fetch', 'origin'], ['git', 'reset', '--hard', 'origin/{0}'.format(branch)], ['git', 'pull'], ['git', 'clean', '-d', '-f'], ] else: cmds = [['git', 'clone', repo]] cwd = GIT_REPO_EXPORT_DIR cwd = os.path.abspath(cwd) for cmd in cmds: try: cmd_log(cmd, cwd) except subprocess.CalledProcessError as ex: log.exception('Failed to pull git repository: %r', ex.output) raise GitExportError(GitExportError.CANNOT_PULL) # export course as xml before commiting and pushing root_dir = os.path.dirname(rdirp) course_dir = os.path.basename(rdirp).rsplit('.git', 1)[0] try: export_to_xml(modulestore(), contentstore(), course_id, root_dir, course_dir) except (EnvironmentError, AttributeError): log.exception('Failed export to xml') raise GitExportError(GitExportError.XML_EXPORT_FAIL) # Get current branch if not already set if not branch: cmd = ['git', 'symbolic-ref', '--short', 'HEAD'] try: branch = cmd_log(cmd, os.path.abspath(rdirp)).strip('\n') except subprocess.CalledProcessError as ex: log.exception('Failed to get branch from freshly cloned repo: %r', ex.output) raise GitExportError(GitExportError.MISSING_BRANCH) # Now that we have fresh xml exported, set identity, add # everything to git, commit, and push to the right branch. ident = {} try: user = User.objects.get(username=user) ident['name'] = user.username ident['email'] = user.email except User.DoesNotExist: # That's ok, just use default ident ident = GIT_EXPORT_DEFAULT_IDENT time_stamp = timezone.now() cwd = os.path.abspath(rdirp) commit_msg = 'Export from Studio at {1}'.format(user, time_stamp) try: cmd_log(['git', 'config', 'user.email', ident['email']], cwd) cmd_log(['git', 'config', 'user.name', ident['name']], cwd) except subprocess.CalledProcessError as ex: log.exception('Error running git configure commands: %r', ex.output) raise GitExportError(GitExportError.CONFIG_ERROR) try: cmd_log(['git', 'add', '.'], cwd) cmd_log(['git', 'commit', '-a', '-m', commit_msg], cwd) except subprocess.CalledProcessError as ex: log.exception('Unable to commit changes: %r', ex.output) raise GitExportError(GitExportError.CANNOT_COMMIT) try: cmd_log(['git', 'push', '-q', 'origin', branch], cwd) except subprocess.CalledProcessError as ex: log.exception('Error running git push command: %r', ex.output) raise GitExportError(GitExportError.CANNOT_PUSH)
def export_handler(request, course_key_string): """ The restful handler for exporting a course. GET html: return html page for import page application/x-tgz: return tar.gz file containing exported course json: not supported Note that there are 2 ways to request the tar.gz file. The request header can specify application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz). If the tar.gz file has been requested but the export operation fails, an HTML page will be returned which describes the error. """ course_key = CourseKey.from_string(course_key_string) if not has_course_access(request.user, course_key): raise PermissionDenied() course_module = modulestore().get_course(course_key) # an _accept URL parameter will be preferred over HTTP_ACCEPT in the header. requested_format = request.REQUEST.get( '_accept', request.META.get('HTTP_ACCEPT', 'text/html')) export_url = reverse_course_url('export_handler', course_key) + '?_accept=application/x-tgz' if 'application/x-tgz' in requested_format: name = course_module.url_name export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp()) try: export_to_xml(modulestore('direct'), contentstore(), course_module.id, root_dir, name, modulestore()) logging.debug('tar file being generated at {0}'.format( export_file.name)) with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: tar_file.add(root_dir / name, arcname=name) except SerializationError as exc: log.exception('There was an error exporting course %s', course_module.id) unit = None failed_item = None parent = None try: failed_item = modulestore().get_item(exc.location) parent_locs = modulestore().get_parent_locations( failed_item.location) if len(parent_locs) > 0: parent = modulestore().get_item(parent_locs[0]) if parent.location.category == 'vertical': unit = parent except: # pylint: disable=bare-except # if we have a nested exception, then we'll show the more generic error message pass return render_to_response( 'export.html', { 'context_course': course_module, 'in_err': True, 'raw_err_msg': str(exc), 'failed_module': failed_item, 'unit': unit, 'edit_unit_url': reverse_usage_url("unit_handler", parent.location) if parent else "", 'course_home_url': reverse_course_url("course_handler", course_key), 'export_url': export_url }) except Exception as exc: log.exception('There was an error exporting course %s', course_module.id) return render_to_response( 'export.html', { 'context_course': course_module, 'in_err': True, 'unit': None, 'raw_err_msg': str(exc), 'course_home_url': reverse_course_url("course_handler", course_key), 'export_url': export_url }) finally: shutil.rmtree(root_dir / name) wrapper = FileWrapper(export_file) response = HttpResponse(wrapper, content_type='application/x-tgz') response[ 'Content-Disposition'] = 'attachment; filename=%s' % os.path.basename( export_file.name) response['Content-Length'] = os.path.getsize(export_file.name) return response elif 'text/html' in requested_format: return render_to_response('export.html', { 'context_course': course_module, 'export_url': export_url }) else: # Only HTML or x-tgz request formats are supported (no JSON). return HttpResponse(status=406)
def export_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None): """ The restful handler for exporting a course. GET html: return html page for import page application/x-tgz: return tar.gz file containing exported course json: not supported Note that there are 2 ways to request the tar.gz file. The request header can specify application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz). If the tar.gz file has been requested but the export operation fails, an HTML page will be returned which describes the error. """ location = BlockUsageLocator(package_id=package_id, branch=branch, version_guid=version_guid, block_id=block) if not has_course_access(request.user, location): raise PermissionDenied() old_location = loc_mapper().translate_locator_to_location(location) course_module = modulestore().get_item(old_location) # an _accept URL parameter will be preferred over HTTP_ACCEPT in the header. requested_format = request.REQUEST.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html')) export_url = location.url_reverse('export') + '?_accept=application/x-tgz' if 'application/x-tgz' in requested_format: name = old_location.name export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp()) try: export_to_xml(modulestore('direct'), contentstore(), old_location, root_dir, name, modulestore()) logging.debug('tar file being generated at {0}'.format(export_file.name)) with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: tar_file.add(root_dir / name, arcname=name) except SerializationError, e: logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e))) unit = None failed_item = None parent = None try: failed_item = modulestore().get_instance(course_module.location.course_id, e.location) parent_locs = modulestore().get_parent_locations(failed_item.location, course_module.location.course_id) if len(parent_locs) > 0: parent = modulestore().get_item(parent_locs[0]) if parent.location.category == 'vertical': unit = parent except: # if we have a nested exception, then we'll show the more generic error message pass unit_locator = loc_mapper().translate_location(old_location.course_id, parent.location, False, True) return render_to_response('export.html', { 'context_course': course_module, 'in_err': True, 'raw_err_msg': str(e), 'failed_module': failed_item, 'unit': unit, 'edit_unit_url': unit_locator.url_reverse("unit") if parent else "", 'course_home_url': location.url_reverse("course"), 'export_url': export_url }) except Exception, e: logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e))) return render_to_response('export.html', { 'context_course': course_module, 'in_err': True, 'unit': None, 'raw_err_msg': str(e), 'course_home_url': location.url_reverse("course"), 'export_url': export_url })
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build(source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build(dest_content) as dest_store: source_course_key = source_store.make_course_key( 'source', 'course', 'key') dest_course_key = dest_store.make_course_key( 'dest', 'course', 'key') import_from_xml( source_store, 'test_user', 'common/test/data', course_dirs=[course_data_name], static_content_store=source_content, target_course_id=source_course_key, create_new_course_if_not_present=True, ) export_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_course', ) import_from_xml( dest_store, 'test_user', self.export_dir, static_content_store=dest_content, target_course_id=dest_course_key, create_new_course_if_not_present=True, ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, )
def export_handler(request, tag=None, course_id=None, branch=None, version_guid=None, block=None): """ The restful handler for exporting a course. GET html: return html page for import page application/x-tgz: return tar.gz file containing exported course json: not supported Note that there are 2 ways to request the tar.gz file. The request header can specify application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz). If the tar.gz file has been requested but the export operation fails, an HTML page will be returned which describes the error. """ location = BlockUsageLocator(course_id=course_id, branch=branch, version_guid=version_guid, usage_id=block) if not has_access(request.user, location): raise PermissionDenied() old_location = loc_mapper().translate_locator_to_location(location) course_module = modulestore().get_item(old_location) # an _accept URL parameter will be preferred over HTTP_ACCEPT in the header. requested_format = request.REQUEST.get("_accept", request.META.get("HTTP_ACCEPT", "text/html")) export_url = location.url_reverse("export") + "?_accept=application/x-tgz" if "application/x-tgz" in requested_format: name = old_location.name export_file = NamedTemporaryFile(prefix=name + ".", suffix=".tar.gz") root_dir = path(mkdtemp()) try: export_to_xml(modulestore("direct"), contentstore(), old_location, root_dir, name, modulestore()) except SerializationError, e: logging.exception("There was an error exporting course {0}. {1}".format(course_module.location, unicode(e))) unit = None failed_item = None parent = None try: failed_item = modulestore().get_instance(course_module.location.course_id, e.location) parent_locs = modulestore().get_parent_locations(failed_item.location, course_module.location.course_id) if len(parent_locs) > 0: parent = modulestore().get_item(parent_locs[0]) if parent.location.category == "vertical": unit = parent except: # if we have a nested exception, then we'll show the more generic error message pass unit_locator = loc_mapper().translate_location(old_location.course_id, parent.location, False, True) return render_to_response( "export.html", { "context_course": course_module, "in_err": True, "raw_err_msg": str(e), "failed_module": failed_item, "unit": unit, "edit_unit_url": unit_locator.url_reverse("unit") if parent else "", "course_home_url": location.url_reverse("course"), "export_url": export_url, }, ) except Exception, e: logging.exception("There was an error exporting course {0}. {1}".format(course_module.location, unicode(e))) return render_to_response( "export.html", { "context_course": course_module, "in_err": True, "unit": None, "raw_err_msg": str(e), "course_home_url": location.url_reverse("course"), "export_url": export_url, }, )
def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ if CodeBlockTimer is None: raise SkipTest("CodeBlockTimer undefined.") desc = "XMLRoundTrip:{}->{}:{}".format( SHORT_NAME_MAP[source_ms], SHORT_NAME_MAP[dest_ms], num_assets ) with CodeBlockTimer(desc): with CodeBlockTimer("fake_assets"): # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) # Construct the contentstore for storing the first import with MongoContentstoreBuilder().build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_ms.build(source_content) as source_store: # Construct the contentstore for storing the second import with MongoContentstoreBuilder().build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_ms.build(dest_content) as dest_store: source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') with CodeBlockTimer("initial_import"): import_from_xml( source_store, 'test_user', TEST_DATA_ROOT, course_dirs=TEST_COURSE, static_content_store=source_content, target_course_id=source_course_key, create_course_if_not_present=True, raise_on_failure=True, ) with CodeBlockTimer("export"): export_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with CodeBlockTimer("second_import"): import_from_xml( dest_store, 'test_user', self.export_dir, course_dirs=['exported_source_course'], static_content_store=dest_content, target_course_id=dest_course_key, create_course_if_not_present=True, raise_on_failure=True, )
def test_export_course(self): module_store = modulestore('direct') draft_store = modulestore('draft') content_store = contentstore() import_from_xml(module_store, 'common/test/data/', ['full']) location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012') # get a vertical (and components in it) to put into 'draft' vertical = module_store.get_item(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_66', None]), depth=1) draft_store.clone_item(vertical.location, vertical.location) # We had a bug where orphaned draft nodes caused export to fail. This is here to cover that case. draft_store.clone_item(vertical.location, Location(['i4x', 'edX', 'full', 'vertical', 'no_references', 'draft'])) for child in vertical.get_children(): draft_store.clone_item(child.location, child.location) root_dir = path(mkdtemp_clean()) # now create a private vertical private_vertical = draft_store.clone_item(vertical.location, Location(['i4x', 'edX', 'full', 'vertical', 'a_private_vertical', None])) # add private to list of children sequential = module_store.get_item(Location(['i4x', 'edX', 'full', 'sequential', 'Administrivia_and_Circuit_Elements', None])) private_location_no_draft = private_vertical.location.replace(revision=None) module_store.update_children(sequential.location, sequential.children + [private_location_no_draft.url()]) # read back the sequential, to make sure we have a pointer to sequential = module_store.get_item(Location(['i4x', 'edX', 'full', 'sequential', 'Administrivia_and_Circuit_Elements', None])) self.assertIn(private_location_no_draft.url(), sequential.children) print 'Exporting to tempdir = {0}'.format(root_dir) # export out to a tempdir export_to_xml(module_store, content_store, location, root_dir, 'test_export', draft_modulestore=draft_store) # check for static tabs self.verify_content_existence(module_store, root_dir, location, 'tabs', 'static_tab', '.html') # check for custom_tags self.verify_content_existence(module_store, root_dir, location, 'info', 'course_info', '.html') # check for custom_tags self.verify_content_existence(module_store, root_dir, location, 'custom_tags', 'custom_tag_template') # check for about content self.verify_content_existence(module_store, root_dir, location, 'about', 'about', '.html') # check for graiding_policy.json filesystem = OSFS(root_dir / 'test_export/policies/6.002_Spring_2012') self.assertTrue(filesystem.exists('grading_policy.json')) course = module_store.get_item(location) # compare what's on disk compared to what we have in our course with filesystem.open('grading_policy.json', 'r') as grading_policy: on_disk = loads(grading_policy.read()) self.assertEqual(on_disk, course.grading_policy) #check for policy.json self.assertTrue(filesystem.exists('policy.json')) # compare what's on disk to what we have in the course module with filesystem.open('policy.json', 'r') as course_policy: on_disk = loads(course_policy.read()) self.assertIn('course/6.002_Spring_2012', on_disk) self.assertEqual(on_disk['course/6.002_Spring_2012'], own_metadata(course)) # remove old course delete_course(module_store, content_store, location) # reimport import_from_xml(module_store, root_dir, ['test_export'], draft_store=draft_store) items = module_store.get_items(Location(['i4x', 'edX', 'full', 'vertical', None])) self.assertGreater(len(items), 0) for descriptor in items: # don't try to look at private verticals. Right now we're running # the service in non-draft aware if getattr(descriptor, 'is_draft', False): print "Checking {0}....".format(descriptor.location.url()) resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()})) self.assertEqual(resp.status_code, 200) # verify that we have the content in the draft store as well vertical = draft_store.get_item(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_66', None]), depth=1) self.assertTrue(getattr(vertical, 'is_draft', False)) for child in vertical.get_children(): self.assertTrue(getattr(child, 'is_draft', False)) # make sure that we don't have a sequential that is in draft mode sequential = draft_store.get_item(Location(['i4x', 'edX', 'full', 'sequential', 'Administrivia_and_Circuit_Elements', None])) self.assertFalse(getattr(sequential, 'is_draft', False)) # verify that we have the private vertical test_private_vertical = draft_store.get_item(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_66', None])) self.assertTrue(getattr(test_private_vertical, 'is_draft', False)) # make sure the textbook survived the export/import course = module_store.get_item(Location(['i4x', 'edX', 'full', 'course', '6.002_Spring_2012', None])) self.assertGreater(len(course.textbooks), 0) shutil.rmtree(root_dir)
def export_handler(request, course_key_string): """ The restful handler for exporting a course. GET html: return html page for import page application/x-tgz: return tar.gz file containing exported course json: not supported Note that there are 2 ways to request the tar.gz file. The request header can specify application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz). If the tar.gz file has been requested but the export operation fails, an HTML page will be returned which describes the error. """ course_key = CourseKey.from_string(course_key_string) if not has_course_access(request.user, course_key): raise PermissionDenied() course_module = modulestore().get_course(course_key) # an _accept URL parameter will be preferred over HTTP_ACCEPT in the header. requested_format = request.REQUEST.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html')) export_url = reverse_course_url('export_handler', course_key) + '?_accept=application/x-tgz' if 'application/x-tgz' in requested_format: name = course_module.url_name export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") root_dir = path(mkdtemp()) try: export_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name) logging.debug('tar file being generated at {0}'.format(export_file.name)) with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: tar_file.add(root_dir / name, arcname=name) except SerializationError as exc: log.exception('There was an error exporting course %s', course_module.id) unit = None failed_item = None parent = None try: failed_item = modulestore().get_item(exc.location) parent_loc = modulestore().get_parent_location(failed_item.location) if parent_loc is not None: parent = modulestore().get_item(parent_loc) if parent.location.category == 'vertical': unit = parent except: # pylint: disable=bare-except # if we have a nested exception, then we'll show the more generic error message pass return render_to_response('export.html', { 'context_course': course_module, 'in_err': True, 'raw_err_msg': str(exc), 'failed_module': failed_item, 'unit': unit, 'edit_unit_url': reverse_usage_url("unit_handler", parent.location) if parent else "", 'course_home_url': reverse_course_url("course_handler", course_key), 'export_url': export_url }) except Exception as exc: log.exception('There was an error exporting course %s', course_module.id) return render_to_response('export.html', { 'context_course': course_module, 'in_err': True, 'unit': None, 'raw_err_msg': str(exc), 'course_home_url': reverse_course_url("course_handler", course_key), 'export_url': export_url }) finally: shutil.rmtree(root_dir / name) wrapper = FileWrapper(export_file) response = HttpResponse(wrapper, content_type='application/x-tgz') response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name) response['Content-Length'] = os.path.getsize(export_file.name) return response elif 'text/html' in requested_format: return render_to_response('export.html', { 'context_course': course_module, 'export_url': export_url }) else: # Only HTML or x-tgz request formats are supported (no JSON). return HttpResponse(status=406)
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build(source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build(dest_content) as dest_store: source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') import_from_xml( source_store, 'test_user', TEST_DATA_DIR, course_dirs=[course_data_name], static_content_store=source_content, target_course_id=source_course_key, create_course_if_not_present=True, raise_on_failure=True, ) export_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) import_from_xml( dest_store, 'test_user', self.export_dir, course_dirs=['exported_source_course'], static_content_store=dest_content, target_course_id=dest_course_key, create_course_if_not_present=True, raise_on_failure=True, ) # NOT CURRENTLY USED # export_to_xml( # dest_store, # dest_content, # dest_course_key, # self.export_dir, # 'exported_dest_course', # ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.exclude_field(None, 'parent') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, ) self.assertAssetsMetadataEqual( source_store, source_course_key, dest_store, dest_course_key, )
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build(source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build(dest_content) as dest_store: source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') import_from_xml( source_store, 'test_user', TEST_DATA_DIR, course_dirs=[course_data_name], static_content_store=source_content, target_course_id=source_course_key, create_course_if_not_present=True, raise_on_failure=True, ) export_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) import_from_xml( dest_store, 'test_user', self.export_dir, course_dirs=['exported_source_course'], static_content_store=dest_content, target_course_id=dest_course_key, create_course_if_not_present=True, raise_on_failure=True, ) # NOT CURRENTLY USED # export_to_xml( # dest_store, # dest_content, # dest_course_key, # self.export_dir, # 'exported_dest_course', # ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.exclude_field(None, 'parent') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, ) self.assertAssetsMetadataEqual( source_store, source_course_key, dest_store, dest_course_key, )