def export(self):
"""
Perform the export given the parameters handed to this class at init.
"""
with self.modulestore.bulk_operations(self.courselike_key):
fsm = OSFS(self.root_dir)
root = lxml.etree.Element("unknown")
# export only the published content
with self.modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, self.courselike_key):
courselike = self.get_courselike()
export_fs = courselike.runtime.export_fs = fsm.makeopendir(self.target_dir)
# change all of the references inside the course to use the xml expected key type w/o version & branch
xml_centric_courselike_key = self.get_key()
adapt_references(courselike, xml_centric_courselike_key, export_fs)
courselike.add_xml_to_node(root)
# Make any needed adjustments to the root node.
self.process_root(root, export_fs)
# Process extra items-- drafts, assets, etc
root_courselike_dir = self.root_dir + "/" + self.target_dir
self.process_extra(root, courselike, root_courselike_dir, xml_centric_courselike_key, export_fs)
# Any last pass adjustments
self.post_process(root, export_fs)
def export(self):
"""
Perform the export given the parameters handed to this class at init.
"""
with self.modulestore.bulk_operations(self.courselike_key):
# depth = None: Traverses down the entire course structure.
# lazy = False: Loads and caches all block definitions during traversal for fast access later
# -and- to eliminate many round-trips to read individual definitions.
# Why these parameters? Because a course export needs to access all the course block information
# eventually. Accessing it all now at the beginning increases performance of the export.
fsm = OSFS(self.root_dir)
courselike = self.get_courselike()
export_fs = courselike.runtime.export_fs = fsm.makeopendir(self.target_dir)
root_courselike_dir = self.root_dir + '/' + self.target_dir
root = lxml.etree.Element('unknown') # pylint: disable=no-member
# export only the published content
with self.modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, self.courselike_key):
# change all of the references inside the course to use the xml expected key type w/o version & branch
xml_centric_courselike_key = self.get_key()
adapt_references(courselike, xml_centric_courselike_key, export_fs)
courselike.add_xml_to_node(root)
# Make any needed adjustments to the root node.
self.process_root(root, export_fs)
# Process extra items-- drafts, assets, etc
self.process_extra(root, courselike, root_courselike_dir, xml_centric_courselike_key, export_fs)
# Any last pass adjustments
self.post_process(root, export_fs)
def export(self):
"""
Perform the export given the parameters handed to this class at init.
"""
with self.modulestore.bulk_operations(self.courselike_key):
fsm = OSFS(self.root_dir)
root = lxml.etree.Element('unknown')
# export only the published content
with self.modulestore.branch_setting(
ModuleStoreEnum.Branch.published_only,
self.courselike_key):
courselike = self.get_courselike()
export_fs = courselike.runtime.export_fs = fsm.makeopendir(
self.target_dir)
# change all of the references inside the course to use the xml expected key type w/o version & branch
xml_centric_courselike_key = self.get_key()
adapt_references(courselike, xml_centric_courselike_key,
export_fs)
courselike.add_xml_to_node(root)
# Make any needed adjustments to the root node.
self.process_root(root, export_fs)
# Process extra items-- drafts, assets, etc
root_courselike_dir = self.root_dir + '/' + self.target_dir
self.process_extra(root, courselike, root_courselike_dir,
xml_centric_courselike_key, export_fs)
# Any last pass adjustments
self.post_process(root, export_fs)
def check_export_roundtrip(self, data_dir, course_dir, mock_get):
# Patch network calls to retrieve the textbook TOC
mock_get.return_value.text = dedent("""
<?xml version="1.0"?><table_of_contents>
<entry page="5" page_label="ii" name="Table of Contents"/>
</table_of_contents>
""").strip()
root_dir = path(self.temp_dir)
print("Copying test course to temp dir {0}".format(root_dir))
data_dir = path(data_dir)
shutil.copytree(data_dir / course_dir, root_dir / course_dir)
print("Starting import")
initial_import = XMLModuleStore(root_dir, course_dirs=[course_dir])
courses = initial_import.get_courses()
self.assertEquals(len(courses), 1)
initial_course = courses[0]
# export to the same directory--that way things like the custom_tags/ folder
# will still be there.
print("Starting export")
fs = OSFS(root_dir)
export_fs = fs.makeopendir(course_dir)
xml = initial_course.export_to_xml(export_fs)
with export_fs.open('course.xml', 'w') as course_xml:
course_xml.write(xml)
print("Starting second import")
second_import = XMLModuleStore(root_dir, course_dirs=[course_dir])
courses2 = second_import.get_courses()
self.assertEquals(len(courses2), 1)
exported_course = courses2[0]
print("Checking course equality")
# HACK: filenames change when changing file formats
# during imports from old-style courses. Ignore them.
strip_filenames(initial_course)
strip_filenames(exported_course)
self.assertEquals(initial_course, exported_course)
self.assertEquals(initial_course.id, exported_course.id)
course_id = initial_course.id
print("Checking key equality")
self.assertEquals(sorted(initial_import.modules[course_id].keys()),
sorted(second_import.modules[course_id].keys()))
print("Checking module equality")
for location in initial_import.modules[course_id].keys():
print("Checking", location)
self.assertEquals(initial_import.modules[course_id][location],
second_import.modules[course_id][location])
def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir, draft_modulestore=None):
course = modulestore.get_item(course_location)
fs = OSFS(root_dir)
export_fs = fs.makeopendir(course_dir)
xml = course.export_to_xml(export_fs)
with export_fs.open("course.xml", "w") as course_xml:
course_xml.write(xml)
# export the static assets
contentstore.export_all_for_course(course_location, root_dir + "/" + course_dir + "/static/")
# export the static tabs
export_extra_content(export_fs, modulestore, course_location, "static_tab", "tabs", ".html")
# export the custom tags
export_extra_content(export_fs, modulestore, course_location, "custom_tag_template", "custom_tags")
# export the course updates
export_extra_content(export_fs, modulestore, course_location, "course_info", "info", ".html")
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_location, "about", "about", ".html")
# export the grading policy
policies_dir = export_fs.makeopendir("policies")
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open("grading_policy.json", "w") as grading_policy:
grading_policy.write(dumps(course.grading_policy))
# export all of the course metadata in policy.json
with course_run_policy_dir.open("policy.json", "w") as course_policy:
policy = {"course/" + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy))
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items(
[None, course_location.org, course_location.course, "vertical", None, "draft"]
)
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir("drafts")
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location, course.location.course_id)
# Don't try to export orphaned items.
if len(parent_locs) > 0:
logging.debug("parent_locs = {0}".format(parent_locs))
draft_vertical.xml_attributes["parent_sequential_url"] = Location(parent_locs[0]).url()
sequential = modulestore.get_item(Location(parent_locs[0]))
index = sequential.children.index(draft_vertical.location.url())
draft_vertical.xml_attributes["index_in_children_list"] = str(index)
draft_vertical.export_to_xml(draft_course_dir)
def check_export_roundtrip(self, data_dir, course_dir):
root_dir = path(self.temp_dir)
print("Copying test course to temp dir {0}".format(root_dir))
data_dir = path(data_dir)
shutil.copytree(data_dir / course_dir, root_dir / course_dir)
print("Starting import")
initial_import = XMLModuleStore(root_dir, course_dirs=[course_dir])
courses = initial_import.get_courses()
self.assertEquals(len(courses), 1)
initial_course = courses[0]
# export to the same directory--that way things like the custom_tags/ folder
# will still be there.
print("Starting export")
fs = OSFS(root_dir)
export_fs = fs.makeopendir(course_dir)
xml = initial_course.export_to_xml(export_fs)
with export_fs.open('course.xml', 'w') as course_xml:
course_xml.write(xml)
print("Starting second import")
second_import = XMLModuleStore(root_dir, course_dirs=[course_dir])
courses2 = second_import.get_courses()
self.assertEquals(len(courses2), 1)
exported_course = courses2[0]
print("Checking course equality")
# HACK: filenames change when changing file formats
# during imports from old-style courses. Ignore them.
strip_filenames(initial_course)
strip_filenames(exported_course)
self.assertEquals(initial_course, exported_course)
self.assertEquals(initial_course.id, exported_course.id)
course_id = initial_course.id
print("Checking key equality")
self.assertEquals(sorted(initial_import.modules[course_id].keys()),
sorted(second_import.modules[course_id].keys()))
print("Checking module equality")
for location in initial_import.modules[course_id].keys():
print("Checking", location)
if location.category == 'html':
print(
"Skipping html modules--they can't import in"
" final form without writing files..."
)
continue
self.assertEquals(initial_import.modules[course_id][location],
second_import.modules[course_id][location])
def check_export_roundtrip(self, data_dir, course_dir):
root_dir = path(self.temp_dir)
print("Copying test course to temp dir {0}".format(root_dir))
data_dir = path(data_dir)
shutil.copytree(data_dir / course_dir, root_dir / course_dir)
print("Starting import")
initial_import = XMLModuleStore(root_dir, course_dirs=[course_dir])
courses = initial_import.get_courses()
self.assertEquals(len(courses), 1)
initial_course = courses[0]
# export to the same directory--that way things like the custom_tags/ folder
# will still be there.
print("Starting export")
fs = OSFS(root_dir)
export_fs = fs.makeopendir(course_dir)
xml = initial_course.export_to_xml(export_fs)
with export_fs.open('course.xml', 'w') as course_xml:
course_xml.write(xml)
print("Starting second import")
second_import = XMLModuleStore(root_dir, course_dirs=[course_dir])
courses2 = second_import.get_courses()
self.assertEquals(len(courses2), 1)
exported_course = courses2[0]
print("Checking course equality")
# HACK: filenames change when changing file formats
# during imports from old-style courses. Ignore them.
strip_filenames(initial_course)
strip_filenames(exported_course)
self.assertEquals(initial_course, exported_course)
self.assertEquals(initial_course.id, exported_course.id)
course_id = initial_course.id
print("Checking key equality")
self.assertEquals(sorted(initial_import.modules[course_id].keys()),
sorted(second_import.modules[course_id].keys()))
print("Checking module equality")
for location in initial_import.modules[course_id].keys():
print("Checking", location)
if location.category == 'html':
print(
"Skipping html modules--they can't import in"
" final form without writing files..."
)
continue
self.assertEquals(initial_import.modules[course_id][location],
second_import.modules[course_id][location])
def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir, draft_modulestore=None):
course = modulestore.get_item(course_location)
fs = OSFS(root_dir)
export_fs = fs.makeopendir(course_dir)
xml = course.export_to_xml(export_fs)
with export_fs.open('course.xml', 'w') as course_xml:
course_xml.write(xml)
# export the static assets
contentstore.export_all_for_course(course_location, root_dir + '/' + course_dir + '/static/')
# export the static tabs
export_extra_content(export_fs, modulestore, course_location, 'static_tab', 'tabs', '.html')
# export the custom tags
export_extra_content(export_fs, modulestore, course_location, 'custom_tag_template', 'custom_tags')
# export the course updates
export_extra_content(export_fs, modulestore, course_location, 'course_info', 'info', '.html')
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_location, 'about', 'about', '.html')
# export the grading policy
policies_dir = export_fs.makeopendir('policies')
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open('grading_policy.json', 'w') as grading_policy:
grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder))
# export all of the course metadata in policy.json
with course_run_policy_dir.open('policy.json', 'w') as course_policy:
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder))
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items([None, course_location.org, course_location.course,
'vertical', None, 'draft'])
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir('drafts')
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location, course.location.course_id)
# Don't try to export orphaned items.
if len(parent_locs) > 0:
logging.debug('parent_locs = {0}'.format(parent_locs))
draft_vertical.xml_attributes['parent_sequential_url'] = Location(parent_locs[0]).url()
sequential = modulestore.get_item(Location(parent_locs[0]))
index = sequential.children.index(draft_vertical.location.url())
draft_vertical.xml_attributes['index_in_children_list'] = str(index)
draft_vertical.export_to_xml(draft_course_dir)
def export(self):
"""
Perform the export given the parameters handed to this class at init.
"""
with self.modulestore.bulk_operations(self.courselike_key):
# depth = None: Traverses down the entire course structure.
# lazy = False: Loads and caches all block definitions during traversal for fast access later
# -and- to eliminate many round-trips to read individual definitions.
# Why these parameters? Because a course export needs to access all the course block information
# eventually. Accessing it all now at the beginning increases performance of the export.
fsm = OSFS(self.root_dir)
courselike = self.get_courselike()
export_fs = courselike.runtime.export_fs = fsm.makeopendir(
self.target_dir)
root_courselike_dir = self.root_dir + '/' + self.target_dir
root = lxml.etree.Element('unknown') # pylint: disable=no-member
# export only the published content
with self.modulestore.branch_setting(
ModuleStoreEnum.Branch.published_only,
self.courselike_key):
# change all of the references inside the course to use the xml expected key type w/o version & branch
xml_centric_courselike_key = self.get_key()
adapt_references(courselike, xml_centric_courselike_key,
export_fs)
courselike.add_xml_to_node(root)
# Make any needed adjustments to the root node.
self.process_root(root, export_fs)
# Process extra items-- drafts, assets, etc
self.process_extra(root, courselike, root_courselike_dir,
xml_centric_courselike_key, export_fs)
# Any last pass adjustments
self.post_process(root, export_fs)
def cmd_get(self, *params, **options):
try:
url = params[0]
except IndexError:
print "get <url>"
return
media_fs = OSFS(settings.MEDIA_ROOT)
media_fs.makedir('favicons', allow_recreate=True)
favicon_path = url_to_path(url)
favicon_fs = media_fs.makeopendir(favicon_path, recursive=True)
orig_favicon_fs = favicon_fs.makeopendir('originals')
output_path = orig_favicon_fs.getsyspath('orig')
convert_path = orig_favicon_fs.getsyspath('icon.png')
try:
icon_info = download_favicon(url, output_path, convert_path)
except URLError, e:
print "ERROR: %s (%s)" % (e, url)
return
def export_to_xml(modulestore,
contentstore,
course_location,
root_dir,
course_dir,
draft_modulestore=None):
course = modulestore.get_item(course_location)
fs = OSFS(root_dir)
export_fs = fs.makeopendir(course_dir)
xml = course.export_to_xml(export_fs)
with export_fs.open('course.xml', 'w') as course_xml:
course_xml.write(xml)
# export the static assets
contentstore.export_all_for_course(
course_location, root_dir + '/' + course_dir + '/static/')
# export the static tabs
export_extra_content(export_fs, modulestore, course_location, 'static_tab',
'tabs', '.html')
# export the custom tags
export_extra_content(export_fs, modulestore, course_location,
'custom_tag_template', 'custom_tags')
# export the course updates
export_extra_content(export_fs, modulestore, course_location,
'course_info', 'info', '.html')
# export the grading policy
policies_dir = export_fs.makeopendir('policies')
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open('grading_policy.json',
'w') as grading_policy:
grading_policy.write(dumps(course.grading_policy))
# export all of the course metadata in policy.json
with course_run_policy_dir.open('policy.json', 'w') as course_policy:
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy))
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items([
None, course_location.org, course_location.course, 'vertical',
None, 'draft'
])
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir('drafts')
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(
draft_vertical.location, course.location.course_id)
# Don't try to export orphaned items.
if len(parent_locs) > 0:
logging.debug('parent_locs = {0}'.format(parent_locs))
draft_vertical.xml_attributes[
'parent_sequential_url'] = Location(
parent_locs[0]).url()
sequential = modulestore.get_item(Location(parent_locs[0]))
index = sequential.children.index(
draft_vertical.location.url())
draft_vertical.xml_attributes[
'index_in_children_list'] = str(index)
draft_vertical.export_to_xml(draft_course_dir)
def test_export_roundtrip(self, course_dir, mock_get):
# Patch network calls to retrieve the textbook TOC
mock_get.return_value.text = dedent("""
<?xml version="1.0"?><table_of_contents>
<entry page="5" page_label="ii" name="Table of Contents"/>
</table_of_contents>
""").strip()
root_dir = path(self.temp_dir)
print("Copying test course to temp dir {0}".format(root_dir))
data_dir = path(DATA_DIR)
shutil.copytree(data_dir / course_dir, root_dir / course_dir)
print("Starting import")
initial_import = XMLModuleStore(root_dir,
course_dirs=[course_dir],
xblock_mixins=(XModuleMixin, ))
courses = initial_import.get_courses()
self.assertEquals(len(courses), 1)
initial_course = courses[0]
# export to the same directory--that way things like the custom_tags/ folder
# will still be there.
print("Starting export")
file_system = OSFS(root_dir)
initial_course.runtime.export_fs = file_system.makeopendir(course_dir)
root = lxml.etree.Element('root')
initial_course.add_xml_to_node(root)
with initial_course.runtime.export_fs.open('course.xml',
'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
print("Starting second import")
second_import = XMLModuleStore(root_dir,
course_dirs=[course_dir],
xblock_mixins=(XModuleMixin, ))
courses2 = second_import.get_courses()
self.assertEquals(len(courses2), 1)
exported_course = courses2[0]
print("Checking course equality")
# HACK: filenames change when changing file formats
# during imports from old-style courses. Ignore them.
strip_filenames(initial_course)
strip_filenames(exported_course)
self.assertTrue(blocks_are_equivalent(initial_course, exported_course))
self.assertEquals(initial_course.id, exported_course.id)
course_id = initial_course.id
print("Checking key equality")
self.assertItemsEqual(initial_import.modules[course_id].keys(),
second_import.modules[course_id].keys())
print("Checking module equality")
for location in initial_import.modules[course_id].keys():
print("Checking", location)
self.assertTrue(
blocks_are_equivalent(
initial_import.modules[course_id][location],
second_import.modules[course_id][location]))
def test_export_roundtrip(self, course_dir, mock_get):
# Patch network calls to retrieve the textbook TOC
mock_get.return_value.text = dedent("""
<?xml version="1.0"?><table_of_contents>
<entry page="5" page_label="ii" name="Table of Contents"/>
</table_of_contents>
""").strip()
root_dir = path(self.temp_dir)
print "Copying test course to temp dir {0}".format(root_dir)
data_dir = path(DATA_DIR)
shutil.copytree(data_dir / course_dir, root_dir / course_dir)
print "Starting import"
initial_import = XMLModuleStore(root_dir, source_dirs=[course_dir], xblock_mixins=(XModuleMixin,))
courses = initial_import.get_courses()
self.assertEquals(len(courses), 1)
initial_course = courses[0]
# export to the same directory--that way things like the custom_tags/ folder
# will still be there.
print "Starting export"
file_system = OSFS(root_dir)
initial_course.runtime.export_fs = file_system.makeopendir(course_dir)
root = lxml.etree.Element('root')
initial_course.add_xml_to_node(root)
with initial_course.runtime.export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
print "Starting second import"
second_import = XMLModuleStore(root_dir, source_dirs=[course_dir], xblock_mixins=(XModuleMixin,))
courses2 = second_import.get_courses()
self.assertEquals(len(courses2), 1)
exported_course = courses2[0]
print "Checking course equality"
# HACK: filenames change when changing file formats
# during imports from old-style courses. Ignore them.
strip_filenames(initial_course)
strip_filenames(exported_course)
self.assertTrue(blocks_are_equivalent(initial_course, exported_course))
self.assertEquals(initial_course.id, exported_course.id)
course_id = initial_course.id
print "Checking key equality"
self.assertItemsEqual(
initial_import.modules[course_id].keys(),
second_import.modules[course_id].keys()
)
print "Checking module equality"
for location in initial_import.modules[course_id].keys():
print("Checking", location)
self.assertTrue(blocks_are_equivalent(
initial_import.modules[course_id][location],
second_import.modules[course_id][location]
))
def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir, draft_modulestore=None):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
`modulestore`: A `ModuleStore` object that is the source of the modules to export
`contentstore`: A `ContentStore` object that is the source of the content to export, can be None
`course_location`: The `Location` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
`draft_modulestore`: An optional `DraftModuleStore` that contains draft content, which will be exported
alongside the public content in the course.
"""
course_id = course_location.course_id
course = modulestore.get_course(course_id)
fs = OSFS(root_dir)
export_fs = fs.makeopendir(course_dir)
xml = course.export_to_xml(export_fs)
with export_fs.open('course.xml', 'w') as course_xml:
course_xml.write(xml)
# export the static assets
policies_dir = export_fs.makeopendir('policies')
if contentstore:
contentstore.export_all_for_course(
course_location,
root_dir + '/' + course_dir + '/static/',
root_dir + '/' + course_dir + '/policies/assets.json',
)
# export the static tabs
export_extra_content(export_fs, modulestore, course_id, course_location, 'static_tab', 'tabs', '.html')
# export the custom tags
export_extra_content(export_fs, modulestore, course_id, course_location, 'custom_tag_template', 'custom_tags')
# export the course updates
export_extra_content(export_fs, modulestore, course_id, course_location, 'course_info', 'info', '.html')
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_id, course_location, 'about', 'about', '.html')
# export the grading policy
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open('grading_policy.json', 'w') as grading_policy:
grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder))
# export all of the course metadata in policy.json
with course_run_policy_dir.open('policy.json', 'w') as course_policy:
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder))
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items([None, course_location.org, course_location.course,
'vertical', None, 'draft'])
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location, course.location.course_id)
# Don't try to export orphaned items.
if len(parent_locs) > 0:
logging.debug('parent_locs = {0}'.format(parent_locs))
draft_vertical.xml_attributes['parent_sequential_url'] = Location(parent_locs[0]).url()
sequential = modulestore.get_item(Location(parent_locs[0]))
index = sequential.children.index(draft_vertical.location.url())
draft_vertical.xml_attributes['index_in_children_list'] = str(index)
draft_vertical.export_to_xml(draft_course_dir)
def export_to_xml(modulestore,
contentstore,
course_location,
root_dir,
course_dir,
draft_modulestore=None):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
`modulestore`: A `ModuleStore` object that is the source of the modules to export
`contentstore`: A `ContentStore` object that is the source of the content to export, can be None
`course_location`: The `Location` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
`draft_modulestore`: An optional `DraftModuleStore` that contains draft content, which will be exported
alongside the public content in the course.
"""
course_id = course_location.course_id
course = modulestore.get_course(course_id)
fs = OSFS(root_dir)
export_fs = course.runtime.export_fs = fs.makeopendir(course_dir)
root = lxml.etree.Element('unknown')
course.add_xml_to_node(root)
with export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
# export the static assets
policies_dir = export_fs.makeopendir('policies')
if contentstore:
contentstore.export_all_for_course(
course_location,
root_dir + '/' + course_dir + '/static/',
root_dir + '/' + course_dir + '/policies/assets.json',
)
# If we are using the default course image, export it to the
# legacy location to support backwards compatibility.
if course.course_image == course.fields['course_image'].default:
try:
course_image = contentstore.find(
StaticContent.compute_location(course.location.org,
course.location.course,
course.course_image), )
except NotFoundError:
pass
else:
output_dir = root_dir + '/' + course_dir + '/static/images/'
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with OSFS(output_dir).open('course_image.jpg',
'wb') as course_image_file:
course_image_file.write(course_image.data)
# export the static tabs
export_extra_content(export_fs, modulestore, course_id, course_location,
'static_tab', 'tabs', '.html')
# export the custom tags
export_extra_content(export_fs, modulestore, course_id, course_location,
'custom_tag_template', 'custom_tags')
# export the course updates
export_extra_content(export_fs, modulestore, course_id, course_location,
'course_info', 'info', '.html')
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_id, course_location,
'about', 'about', '.html')
# export the grading policy
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open('grading_policy.json',
'w') as grading_policy:
grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder))
# export all of the course metadata in policy.json
with course_run_policy_dir.open('policy.json', 'w') as course_policy:
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder))
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items([
None, course_location.org, course_location.course, 'vertical',
None, 'draft'
])
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(
draft_vertical.location, course.location.course_id)
# Don't try to export orphaned items.
if len(parent_locs) > 0:
logging.debug('parent_locs = {0}'.format(parent_locs))
draft_vertical.xml_attributes[
'parent_sequential_url'] = Location(
parent_locs[0]).url()
sequential = modulestore.get_item(Location(parent_locs[0]))
index = sequential.children.index(
draft_vertical.location.url())
draft_vertical.xml_attributes[
'index_in_children_list'] = str(index)
draft_vertical.runtime.export_fs = draft_course_dir
node = lxml.etree.Element('unknown')
draft_vertical.add_xml_to_node(node)
# Turn on captcha for registration
if int(myconf.take('recaptcha.use')):
auth.settings.captcha = Recaptcha2(request,
myconf.take('recaptcha.site_key'),
myconf.take('recaptcha.secret_key'))
# -----------------------------------------------------------------------------
# IMPORT the CKEDITOR PLUGIN TO GIVE A WYSIWYG EDITOR FOR BLOGS AND NEWS
# -- OK, so this editor is neat but one issue is that it dumps files into the
# root of uploads, which is messy
# -- Ordinarily, this would be controlled by the upload_folder setting but
# this is hardcoded in the module. Could edit it there but you can also use
# a fs object to provide a folder
# -- You'd think it might be possible to have multiple upload folders but
# it turns out to be quite hard to switch the settings
# -----------------------------------------------------------------------------
ckeditor = CKEditor(db)
app_root = request.folder
app_root_fs = OSFS(app_root)
if not app_root_fs.exists('uploads/news_and_blogs/'):
blog_fs = app_root_fs.makeopendir('uploads/news_and_blogs/')
else:
blog_fs = app_root_fs.opendir('uploads/news_and_blogs/')
ckeditor.settings.uploadfs = blog_fs
ckeditor.settings.table_upload_name = 'ckeditor_uploads'
ckeditor.define_tables(fake_migrate=True)
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
`modulestore`: A `ModuleStore` object that is the source of the modules to export
`contentstore`: A `ContentStore` object that is the source of the content to export, can be None
`course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
"""
course = modulestore.get_course(course_key)
fsm = OSFS(root_dir)
export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
root = lxml.etree.Element('unknown')
# export only the published content
with modulestore.branch_setting(ModuleStoreEnum.Branch.published_only,
course_key):
course.add_xml_to_node(root)
with export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
# export the static assets
policies_dir = export_fs.makeopendir('policies')
if contentstore:
contentstore.export_all_for_course(
course_key,
root_dir + '/' + course_dir + '/static/',
root_dir + '/' + course_dir + '/policies/assets.json',
)
# If we are using the default course image, export it to the
# legacy location to support backwards compatibility.
if course.course_image == course.fields['course_image'].default:
try:
course_image = contentstore.find(
StaticContent.compute_location(course.id,
course.course_image), )
except NotFoundError:
pass
else:
output_dir = root_dir + '/' + course_dir + '/static/images/'
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with OSFS(output_dir).open('course_image.jpg',
'wb') as course_image_file:
course_image_file.write(course_image.data)
# export the static tabs
export_extra_content(export_fs, modulestore, course_key, 'static_tab',
'tabs', '.html')
# export the custom tags
export_extra_content(export_fs, modulestore, course_key,
'custom_tag_template', 'custom_tags')
# export the course updates
export_extra_content(export_fs, modulestore, course_key, 'course_info',
'info', '.html')
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_key, 'about', 'about',
'.html')
# export the grading policy
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open('grading_policy.json',
'w') as grading_policy:
grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder))
# export all of the course metadata in policy.json
with course_run_policy_dir.open('policy.json', 'w') as course_policy:
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder))
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer be valid
# NOTE: we need to explicitly implement the logic for setting the vertical's parent
# and index here since the XML modulestore cannot load draft modules
draft_verticals = modulestore.get_items(
course_key,
category='vertical',
revision=ModuleStoreEnum.RevisionOption.draft_only)
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
parent_loc = modulestore.get_parent_location(
draft_vertical.location,
revision=ModuleStoreEnum.RevisionOption.draft_preferred)
# Don't try to export orphaned items.
if parent_loc is not None:
logging.debug('parent_loc = {0}'.format(parent_loc))
if parent_loc.category in DIRECT_ONLY_CATEGORIES:
draft_vertical.xml_attributes[
'parent_sequential_url'] = parent_loc.to_deprecated_string(
)
sequential = modulestore.get_item(parent_loc)
index = sequential.children.index(draft_vertical.location)
draft_vertical.xml_attributes[
'index_in_children_list'] = str(index)
draft_vertical.runtime.export_fs = draft_course_dir
node = lxml.etree.Element('unknown')
draft_vertical.add_xml_to_node(node)
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
`modulestore`: A `ModuleStore` object that is the source of the modules to export
`contentstore`: A `ContentStore` object that is the source of the content to export, can be None
`course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
"""
with modulestore.bulk_operations(course_key):
course = modulestore.get_course(course_key, depth=None) # None means infinite
fsm = OSFS(root_dir)
export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
root_course_dir = root_dir + '/' + course_dir
root = lxml.etree.Element('unknown')
# export only the published content
with modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, course_key):
# change all of the references inside the course to use the xml expected key type w/o version & branch
xml_centric_course_key = CourseLocator(course_key.org, course_key.course, course_key.run, deprecated=True)
adapt_references(course, xml_centric_course_key, export_fs)
course.add_xml_to_node(root)
with export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
# Export the modulestore's asset metadata.
asset_dir = root_course_dir + '/' + AssetMetadata.EXPORTED_ASSET_DIR + '/'
if not os.path.isdir(asset_dir):
os.makedirs(asset_dir)
asset_root = lxml.etree.Element(AssetMetadata.ALL_ASSETS_XML_TAG)
course_assets = modulestore.get_all_asset_metadata(course_key, None)
for asset_md in course_assets:
# All asset types are exported using the "asset" tag - but their asset type is specified in each asset key.
asset = lxml.etree.SubElement(asset_root, AssetMetadata.ASSET_XML_TAG)
asset_md.to_xml(asset)
with OSFS(asset_dir).open(AssetMetadata.EXPORTED_ASSET_FILENAME, 'w') as asset_xml_file:
lxml.etree.ElementTree(asset_root).write(asset_xml_file)
# export the static assets
policies_dir = export_fs.makeopendir('policies')
if contentstore:
contentstore.export_all_for_course(
course_key,
root_course_dir + '/static/',
root_course_dir + '/policies/assets.json',
)
# If we are using the default course image, export it to the
# legacy location to support backwards compatibility.
if course.course_image == course.fields['course_image'].default:
try:
course_image = contentstore.find(
StaticContent.compute_location(
course.id,
course.course_image
),
)
except NotFoundError:
pass
else:
output_dir = root_course_dir + '/static/images/'
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with OSFS(output_dir).open('course_image.jpg', 'wb') as course_image_file:
course_image_file.write(course_image.data)
# export the static tabs
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'static_tab', 'tabs', '.html')
# export the custom tags
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'custom_tag_template', 'custom_tags')
# export the course updates
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'course_info', 'info', '.html')
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'about', 'about', '.html')
# export the grading policy
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open('grading_policy.json', 'w') as grading_policy:
grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
# export all of the course metadata in policy.json
with course_run_policy_dir.open('policy.json', 'w') as course_policy:
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
#### DRAFTS ####
# xml backed courses don't support drafts!
if course.runtime.modulestore.get_modulestore_type() != ModuleStoreEnum.Type.xml:
# NOTE: we need to explicitly implement the logic for setting the vertical's parent
# and index here since the XML modulestore cannot load draft modules
with modulestore.branch_setting(ModuleStoreEnum.Branch.draft_preferred, course_key):
draft_modules = modulestore.get_items(
course_key,
qualifiers={'category': {'$nin': DIRECT_ONLY_CATEGORIES}},
revision=ModuleStoreEnum.RevisionOption.draft_only
)
if draft_modules:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
# accumulate tuples of draft_modules and their parents in
# this list:
draft_node_list = []
for draft_module in draft_modules:
parent_loc = modulestore.get_parent_location(
draft_module.location,
revision=ModuleStoreEnum.RevisionOption.draft_preferred
)
# if module has no parent, set its parent_url to `None`
parent_url = None
if parent_loc is not None:
parent_url = parent_loc.to_deprecated_string()
draft_node = draft_node_constructor(
draft_module,
location=draft_module.location,
url=draft_module.location.to_deprecated_string(),
parent_location=parent_loc,
parent_url=parent_url,
)
draft_node_list.append(draft_node)
for draft_node in get_draft_subtree_roots(draft_node_list):
# only export the roots of the draft subtrees
# since export_from_xml (called by `add_xml_to_node`)
# exports a whole tree
# ensure module has "xml_attributes" attr
if not hasattr(draft_node.module, 'xml_attributes'):
draft_node.module.xml_attributes = {}
# Don't try to export orphaned items
# and their descendents
if draft_node.parent_location is None:
continue
logging.debug('parent_loc = {0}'.format(draft_node.parent_location))
draft_node.module.xml_attributes['parent_url'] = draft_node.parent_url
parent = modulestore.get_item(draft_node.parent_location)
index = parent.children.index(draft_node.module.location)
draft_node.module.xml_attributes['index_in_children_list'] = str(index)
draft_node.module.runtime.export_fs = draft_course_dir
adapt_references(draft_node.module, xml_centric_course_key, draft_course_dir)
node = lxml.etree.Element('unknown')
draft_node.module.add_xml_to_node(node)
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
`modulestore`: A `ModuleStore` object that is the source of the modules to export
`contentstore`: A `ContentStore` object that is the source of the content to export, can be None
`course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
"""
with modulestore.bulk_operations(course_key):
course = modulestore.get_course(course_key, depth=None) # None means infinite
fsm = OSFS(root_dir)
export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
root = lxml.etree.Element('unknown')
# export only the published content
with modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, course_key):
# change all of the references inside the course to use the xml expected key type w/o version & branch
xml_centric_course_key = CourseLocator(course_key.org, course_key.course, course_key.run, deprecated=True)
adapt_references(course, xml_centric_course_key, export_fs)
course.add_xml_to_node(root)
with export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
# export the static assets
policies_dir = export_fs.makeopendir('policies')
if contentstore:
contentstore.export_all_for_course(
course_key,
root_dir + '/' + course_dir + '/static/',
root_dir + '/' + course_dir + '/policies/assets.json',
)
# If we are using the default course image, export it to the
# legacy location to support backwards compatibility.
if course.course_image == course.fields['course_image'].default:
try:
course_image = contentstore.find(
StaticContent.compute_location(
course.id,
course.course_image
),
)
except NotFoundError:
pass
else:
output_dir = root_dir + '/' + course_dir + '/static/images/'
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with OSFS(output_dir).open('course_image.jpg', 'wb') as course_image_file:
course_image_file.write(course_image.data)
# export the static tabs
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'static_tab', 'tabs', '.html')
# export the custom tags
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'custom_tag_template', 'custom_tags')
# export the course updates
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'course_info', 'info', '.html')
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, 'about', 'about', '.html')
# export the grading policy
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open('grading_policy.json', 'w') as grading_policy:
grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
# export all of the course metadata in policy.json
with course_run_policy_dir.open('policy.json', 'w') as course_policy:
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
#### DRAFTS ####
# xml backed courses don't support drafts!
if course.runtime.modulestore.get_modulestore_type() != ModuleStoreEnum.Type.xml:
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer be valid
# NOTE: we need to explicitly implement the logic for setting the vertical's parent
# and index here since the XML modulestore cannot load draft modules
with modulestore.branch_setting(ModuleStoreEnum.Branch.draft_preferred, course_key):
draft_verticals = modulestore.get_items(
course_key,
qualifiers={'category': 'vertical'},
revision=ModuleStoreEnum.RevisionOption.draft_only
)
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
parent_loc = modulestore.get_parent_location(
draft_vertical.location,
revision=ModuleStoreEnum.RevisionOption.draft_preferred
)
# Don't try to export orphaned items.
if parent_loc is not None:
logging.debug('parent_loc = {0}'.format(parent_loc))
if parent_loc.category in DIRECT_ONLY_CATEGORIES:
draft_vertical.xml_attributes['parent_sequential_url'] = parent_loc.to_deprecated_string()
sequential = modulestore.get_item(parent_loc)
index = sequential.children.index(draft_vertical.location)
draft_vertical.xml_attributes['index_in_children_list'] = str(index)
draft_vertical.runtime.export_fs = draft_course_dir
adapt_references(draft_vertical, xml_centric_course_key, draft_course_dir)
node = lxml.etree.Element('unknown')
draft_vertical.add_xml_to_node(node)
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
`modulestore`: A `ModuleStore` object that is the source of the modules to export
`contentstore`: A `ContentStore` object that is the source of the content to export, can be None
`course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
"""
with modulestore.bulk_operations(course_key):
course = modulestore.get_course(course_key, depth=None) # None means infinite
fsm = OSFS(root_dir)
export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
root = lxml.etree.Element("unknown")
# export only the published content
with modulestore.branch_setting(ModuleStoreEnum.Branch.published_only, course_key):
# change all of the references inside the course to use the xml expected key type w/o version & branch
xml_centric_course_key = CourseLocator(course_key.org, course_key.course, course_key.run, deprecated=True)
adapt_references(course, xml_centric_course_key, export_fs)
course.add_xml_to_node(root)
with export_fs.open("course.xml", "w") as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
# export the static assets
policies_dir = export_fs.makeopendir("policies")
if contentstore:
contentstore.export_all_for_course(
course_key,
root_dir + "/" + course_dir + "/static/",
root_dir + "/" + course_dir + "/policies/assets.json",
)
# If we are using the default course image, export it to the
# legacy location to support backwards compatibility.
if course.course_image == course.fields["course_image"].default:
try:
course_image = contentstore.find(StaticContent.compute_location(course.id, course.course_image))
except NotFoundError:
pass
else:
output_dir = root_dir + "/" + course_dir + "/static/images/"
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with OSFS(output_dir).open("course_image.jpg", "wb") as course_image_file:
course_image_file.write(course_image.data)
# export the static tabs
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, "static_tab", "tabs", ".html")
# export the custom tags
export_extra_content(
export_fs, modulestore, course_key, xml_centric_course_key, "custom_tag_template", "custom_tags"
)
# export the course updates
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, "course_info", "info", ".html")
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_key, xml_centric_course_key, "about", "about", ".html")
# export the grading policy
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open("grading_policy.json", "w") as grading_policy:
grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
# export all of the course metadata in policy.json
with course_run_policy_dir.open("policy.json", "w") as course_policy:
policy = {"course/" + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
#### DRAFTS ####
# xml backed courses don't support drafts!
if course.runtime.modulestore.get_modulestore_type() != ModuleStoreEnum.Type.xml:
# NOTE: we need to explicitly implement the logic for setting the vertical's parent
# and index here since the XML modulestore cannot load draft modules
with modulestore.branch_setting(ModuleStoreEnum.Branch.draft_preferred, course_key):
draft_modules = modulestore.get_items(
course_key,
qualifiers={"category": {"$nin": DIRECT_ONLY_CATEGORIES}},
revision=ModuleStoreEnum.RevisionOption.draft_only,
)
if draft_modules:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
# accumulate tuples of draft_modules and their parents in
# this list:
draft_node_list = []
for draft_module in draft_modules:
parent_loc = modulestore.get_parent_location(
draft_module.location, revision=ModuleStoreEnum.RevisionOption.draft_preferred
)
# Don't try to export orphaned items.
if parent_loc is not None:
logging.debug("parent_loc = {0}".format(parent_loc))
draft_node = draft_node_constructor(
draft_module,
location=draft_module.location,
url=draft_module.location.to_deprecated_string(),
parent_location=parent_loc,
parent_url=parent_loc.to_deprecated_string(),
)
draft_node_list.append(draft_node)
for draft_node in get_draft_subtree_roots(draft_node_list):
# only export the roots of the draft subtrees
# since export_from_xml (called by `add_xml_to_node`)
# exports a whole tree
draft_node.module.xml_attributes["parent_url"] = draft_node.parent_url
parent = modulestore.get_item(draft_node.parent_location)
index = parent.children.index(draft_node.module.location)
draft_node.module.xml_attributes["index_in_children_list"] = str(index)
draft_node.module.runtime.export_fs = draft_course_dir
adapt_references(draft_node.module, xml_centric_course_key, draft_course_dir)
node = lxml.etree.Element("unknown")
draft_node.module.add_xml_to_node(node)
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
`modulestore`: A `ModuleStore` object that is the source of the modules to export
`contentstore`: A `ContentStore` object that is the source of the content to export, can be None
`course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
"""
with modulestore.bulk_operations(course_key):
course = modulestore.get_course(course_key,
depth=None) # None means infinite
fsm = OSFS(root_dir)
export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
root = lxml.etree.Element('unknown')
# export only the published content
with modulestore.branch_setting(ModuleStoreEnum.Branch.published_only,
course_key):
# change all of the references inside the course to use the xml expected key type w/o version & branch
xml_centric_course_key = CourseLocator(course_key.org,
course_key.course,
course_key.run,
deprecated=True)
adapt_references(course, xml_centric_course_key, export_fs)
course.add_xml_to_node(root)
with export_fs.open('course.xml', 'w') as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
# export the static assets
policies_dir = export_fs.makeopendir('policies')
if contentstore:
contentstore.export_all_for_course(
course_key,
root_dir + '/' + course_dir + '/static/',
root_dir + '/' + course_dir + '/policies/assets.json',
)
# If we are using the default course image, export it to the
# legacy location to support backwards compatibility.
if course.course_image == course.fields['course_image'].default:
try:
course_image = contentstore.find(
StaticContent.compute_location(course.id,
course.course_image), )
except NotFoundError:
pass
else:
output_dir = root_dir + '/' + course_dir + '/static/images/'
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with OSFS(output_dir).open('course_image.jpg',
'wb') as course_image_file:
course_image_file.write(course_image.data)
# export the static tabs
export_extra_content(export_fs, modulestore, course_key,
xml_centric_course_key, 'static_tab', 'tabs',
'.html')
# export the custom tags
export_extra_content(export_fs, modulestore, course_key,
xml_centric_course_key, 'custom_tag_template',
'custom_tags')
# export the course updates
export_extra_content(export_fs, modulestore, course_key,
xml_centric_course_key, 'course_info', 'info',
'.html')
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_key,
xml_centric_course_key, 'about', 'about', '.html')
# export the grading policy
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open('grading_policy.json',
'w') as grading_policy:
grading_policy.write(
dumps(course.grading_policy,
cls=EdxJSONEncoder,
sort_keys=True,
indent=4))
# export all of the course metadata in policy.json
with course_run_policy_dir.open('policy.json', 'w') as course_policy:
policy = {'course/' + course.location.name: own_metadata(course)}
course_policy.write(
dumps(policy, cls=EdxJSONEncoder, sort_keys=True, indent=4))
#### DRAFTS ####
# xml backed courses don't support drafts!
if course.runtime.modulestore.get_modulestore_type(
) != ModuleStoreEnum.Type.xml:
# NOTE: we need to explicitly implement the logic for setting the vertical's parent
# and index here since the XML modulestore cannot load draft modules
with modulestore.branch_setting(
ModuleStoreEnum.Branch.draft_preferred, course_key):
draft_modules = modulestore.get_items(
course_key,
qualifiers={'category': {
'$nin': DIRECT_ONLY_CATEGORIES
}},
revision=ModuleStoreEnum.RevisionOption.draft_only)
if draft_modules:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
# accumulate tuples of draft_modules and their parents in
# this list:
draft_node_list = []
for draft_module in draft_modules:
parent_loc = modulestore.get_parent_location(
draft_module.location,
revision=ModuleStoreEnum.RevisionOption.
draft_preferred)
# if module has no parent, set its parent_url to `None`
parent_url = None
if parent_loc is not None:
parent_url = parent_loc.to_deprecated_string()
draft_node = draft_node_constructor(
draft_module,
location=draft_module.location,
url=draft_module.location.to_deprecated_string(),
parent_location=parent_loc,
parent_url=parent_url,
)
draft_node_list.append(draft_node)
for draft_node in get_draft_subtree_roots(draft_node_list):
# only export the roots of the draft subtrees
# since export_from_xml (called by `add_xml_to_node`)
# exports a whole tree
# ensure module has "xml_attributes" attr
if not hasattr(draft_node.module, 'xml_attributes'):
draft_node.module.xml_attributes = {}
# Don't try to export orphaned items
# and their descendents
if draft_node.parent_location is None:
continue
logging.debug('parent_loc = {0}'.format(
draft_node.parent_location))
draft_node.module.xml_attributes[
'parent_url'] = draft_node.parent_url
parent = modulestore.get_item(
draft_node.parent_location)
index = parent.children.index(
draft_node.module.location)
draft_node.module.xml_attributes[
'index_in_children_list'] = str(index)
draft_node.module.runtime.export_fs = draft_course_dir
adapt_references(draft_node.module,
xml_centric_course_key,
draft_course_dir)
node = lxml.etree.Element('unknown')
draft_node.module.add_xml_to_node(node)
except IndexError:
src = '~/projects/linkstop/webapp/linkstop/media/faviconsx'
dst = '~/projects/linkstop/webapp/linkstop/media/favicons'
src_fs = OSFS(src)
dst_fs = OSFS(dst)
count = 0
max_count = 2
for path in src_fs.listdir(dirs_only=True):
icon_fs = src_fs.opendir(path)
if icon_fs.isfile('scan.pik'):
try:
icon = pickle.load(icon_fs.open('scan.pik'))
except Exception, e:
print "%s (%s)" % (str(e), path)
continue
normalized_url = icon['normalized_url']
out_dir = hash_path(normalized_url) + '/' + url_to_filename(normalized_url)
print out_dir
dest_dir_fs = dst_fs.makeopendir(out_dir, recursive=True)
movedir(icon_fs, dest_dir_fs, overwrite=True, ignore_errors=True)
#count += 1
if count >= max_count:
break
def export_to_xml(modulestore, contentstore, course_key, root_dir, course_dir, draft_modulestore=None):
"""
Export all modules from `modulestore` and content from `contentstore` as xml to `root_dir`.
`modulestore`: A `ModuleStore` object that is the source of the modules to export
`contentstore`: A `ContentStore` object that is the source of the content to export, can be None
`course_key`: The `CourseKey` of the `CourseModuleDescriptor` to export
`root_dir`: The directory to write the exported xml to
`course_dir`: The name of the directory inside `root_dir` to write the course content to
`draft_modulestore`: An optional `DraftModuleStore` that contains draft content, which will be exported
alongside the public content in the course.
"""
course = modulestore.get_course(course_key)
fsm = OSFS(root_dir)
export_fs = course.runtime.export_fs = fsm.makeopendir(course_dir)
root = lxml.etree.Element("unknown")
course.add_xml_to_node(root)
with export_fs.open("course.xml", "w") as course_xml:
lxml.etree.ElementTree(root).write(course_xml)
# export the static assets
policies_dir = export_fs.makeopendir("policies")
if contentstore:
contentstore.export_all_for_course(
course_key, root_dir + "/" + course_dir + "/static/", root_dir + "/" + course_dir + "/policies/assets.json"
)
# If we are using the default course image, export it to the
# legacy location to support backwards compatibility.
if course.course_image == course.fields["course_image"].default:
try:
course_image = contentstore.find(StaticContent.compute_location(course.id, course.course_image))
except NotFoundError:
pass
else:
output_dir = root_dir + "/" + course_dir + "/static/images/"
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
with OSFS(output_dir).open("course_image.jpg", "wb") as course_image_file:
course_image_file.write(course_image.data)
# export the static tabs
export_extra_content(export_fs, modulestore, course_key, "static_tab", "tabs", ".html")
# export the custom tags
export_extra_content(export_fs, modulestore, course_key, "custom_tag_template", "custom_tags")
# export the course updates
export_extra_content(export_fs, modulestore, course_key, "course_info", "info", ".html")
# export the 'about' data (e.g. overview, etc.)
export_extra_content(export_fs, modulestore, course_key, "about", "about", ".html")
# export the grading policy
course_run_policy_dir = policies_dir.makeopendir(course.location.name)
with course_run_policy_dir.open("grading_policy.json", "w") as grading_policy:
grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder))
# export all of the course metadata in policy.json
with course_run_policy_dir.open("policy.json", "w") as course_policy:
policy = {"course/" + course.location.name: own_metadata(course)}
course_policy.write(dumps(policy, cls=EdxJSONEncoder))
# export draft content
# NOTE: this code assumes that verticals are the top most draftable container
# should we change the application, then this assumption will no longer
# be valid
if draft_modulestore is not None:
draft_verticals = draft_modulestore.get_items(course_key, category="vertical", revision="draft")
if len(draft_verticals) > 0:
draft_course_dir = export_fs.makeopendir(DRAFT_DIR)
for draft_vertical in draft_verticals:
parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location)
# Don't try to export orphaned items.
if len(parent_locs) > 0:
logging.debug("parent_locs = {0}".format(parent_locs))
draft_vertical.xml_attributes["parent_sequential_url"] = parent_locs[0].to_deprecated_string()
sequential = modulestore.get_item(parent_locs[0])
index = sequential.children.index(draft_vertical.location)
draft_vertical.xml_attributes["index_in_children_list"] = str(index)
draft_vertical.runtime.export_fs = draft_course_dir
node = lxml.etree.Element("unknown")
draft_vertical.add_xml_to_node(node)
from fs.osfs import OSFS
from fs.errors import ResourceNotFoundError
import settings as s
from backend import fetch_repo, create_production, build
from utils import load_repos
root_fs = OSFS(s.root_path)
build_fs = root_fs.makeopendir(s.build_path)
build_fs.makedir(s.sources_path)
build_fs.makedir(s.staging_path)
build_fs.makedir(s.register_path)
# create production directory if needed
try:
production_fs = OSFS(s.production_path)
except ResourceNotFoundError:
# grap production dir's parent dir
path = s.production_path.split('/')[-2]
print path
production_fs = OSFS(s.production_path[:len(s.production_path) - (len(path) + 1)]).makeopendir(path)
print production_fs
if production_fs.exists(s.backups_path) == False:
production_fs.makedir(s.backups_path)
# fetch repos from GitHub
for repo in load_repos(s.repos_path)[0].values():
print 'Fetching %s for the first time' % repo['id']
fetch_repo(root_fs, s.sources_path, repo['id'], repo['url'], s.build_path)
build(s.sources_path, s.register_path, root_fs, repo['id'])
