def test_export_import(self):
"""
Test export then import.
"""
bundle = XBundle()
cxmls = input_data.COURSE
pxmls = input_data.POLICIES
bundle.set_course(etree.XML(cxmls))
bundle.add_policies(etree.XML(pxmls))
bundle.add_about_file("overview.html", "hello overview")
xbin = str(bundle)
tdir = mkdtemp()
try:
bundle.export_to_directory(tdir)
# Test round- trip.
xb2 = XBundle()
xb2.import_from_directory(os.path.join(tdir, 'mitx.01'))
xbreloaded = str(xb2)
self.assertEqual(clean_xml(xbin), clean_xml(xbreloaded))
finally:
rmtree(tdir)
def test_export_import(self):
"""
Test export then import.
"""
bundle = XBundle()
cxmls = input_data.COURSE
pxmls = input_data.POLICIES
bundle.set_course(etree.XML(cxmls))
bundle.add_policies(etree.XML(pxmls))
bundle.add_about_file("overview.html", "hello overview")
xbin = str(bundle)
tdir = mkdtemp()
try:
bundle.export_to_directory(tdir)
# Test round- trip.
xb2 = XBundle()
xb2.import_from_directory(os.path.join(tdir, 'mitx.01'))
xbreloaded = str(xb2)
self.assertEqual(clean_xml(xbin), clean_xml(xbreloaded))
finally:
rmtree(tdir)
def test_nested_leaves(self):
"""
Test that nested leaves are not imported.
"""
template = """
<course org="DevOps" course="0.001" url_name="2015_Summer"
semester="2015_Summer">
<chapter>
<sequential>
<vertical>
<{tag}><{tag}></{tag}></{tag}>
</vertical>
</sequential>
</chapter>
</course>
"""
for tag in ("html", "problem", "discussion", "video"):
repo = create_repo(
"{tag}_repo".format(tag=tag), "...", self.user.id)
xml = etree.fromstring(template.format(tag=tag))
bundle = XBundle(
keep_urls=True, keep_studio_urls=True, preserve_url_name=True
)
bundle.set_course(xml)
import_course(bundle, repo.id, self.user.id, "")
self.assertEqual(
LearningResource.objects.filter(
learning_resource_type__name=tag
).count(),
1
)
def test_parent_preview_link(self):
"""
Test that if url_name is blank we import the parent's url_name when
viewing the preview link.
"""
xml = """
<course org="DevOps" course="0.001" url_name="2015_Summer"
semester="2015_Summer">
<chapter>
<sequential>
<vertical>
<html></html>
</vertical>
</sequential>
</chapter>
</course>
"""
repo = create_repo("html_repo", "...", self.user.id)
xml = etree.fromstring(xml)
bundle = XBundle(
keep_urls=True, keep_studio_urls=True, preserve_url_name=True
)
bundle.set_course(xml)
import_course(bundle, repo.id, self.user.id, "")
html_resources = LearningResource.objects.filter(
learning_resource_type__name="html"
)
self.assertEqual(html_resources.count(), 1)
html_resource = html_resources.first()
self.assertEqual(
get_preview_url(html_resource),
"{base}courses/{org}/{course}/{run}/jump_to_id/{url_path}".format(
base=settings.LORE_PREVIEW_BASE_URL,
org=html_resource.course.org,
course=html_resource.course.course_number,
run=html_resource.course.run,
url_path="2015_Summer"
)
)
def test_set_course(self):
"""
Test functionality of set_course.
"""
input_xml = input_data.EMPTY_COURSE
bundle = XBundle(keep_urls=True)
bundle.load(file_from_string(input_xml))
# No org or semester is specified in XML above.
self.assertEqual(bundle.course.get("org"), None)
self.assertEqual(bundle.course.get("semester"), None)
self.assertEqual(bundle.semester, "")
# Note lack of org attribute and url_name for course element.
course_str = input_data.NO_COURSE
with self.assertRaises(Exception) as ex:
bundle.set_course(etree.XML("<x>" + course_str + "</x>"))
self.assertTrue(
"set_course should be called with a <course> element"
in ex.exception.args)
with self.assertRaises(Exception) as ex:
bundle.set_course(etree.XML("<course />"))
self.assertTrue("No semester found." in ex.exception.args)
bundle.set_course(etree.XML("<course url_name='x' />"))
self.assertEqual(bundle.semester, "x")
bundle.set_course(etree.XML(course_str))
# MITx is not present in data, it is automatically set.
self.assertEqual(bundle.course.get("org"), "MITx")
self.assertEqual(bundle.course.get("semester"), "2013_Spring")
self.assertEqual(bundle.semester, "2013_Spring")
bundle_string = str(bundle)
expected = expected_data.SET_COURSE
self.assertEqual(clean_xml(bundle_string), clean_xml(expected))
def test_set_course(self):
"""
Test functionality of set_course.
"""
input_xml = input_data.EMPTY_COURSE
bundle = XBundle(keep_urls=True)
bundle.load(file_from_string(input_xml))
# No org or semester is specified in XML above.
self.assertEqual(bundle.course.get("org"), None)
self.assertEqual(bundle.course.get("semester"), None)
self.assertEqual(bundle.semester, "")
# Note lack of org attribute and url_name for course element.
course_str = input_data.NO_COURSE
with self.assertRaises(Exception) as ex:
bundle.set_course(etree.XML("<x>" + course_str + "</x>"))
self.assertTrue("set_course should be called with a <course> element"
in ex.exception.args)
with self.assertRaises(Exception) as ex:
bundle.set_course(etree.XML("<course />"))
self.assertTrue("No semester found." in ex.exception.args)
bundle.set_course(etree.XML("<course url_name='x' />"))
self.assertEqual(bundle.semester, "x")
bundle.set_course(etree.XML(course_str))
# MITx is not present in data, it is automatically set.
self.assertEqual(bundle.course.get("org"), "MITx")
self.assertEqual(bundle.course.get("semester"), "2013_Spring")
self.assertEqual(bundle.semester, "2013_Spring")
bundle_string = str(bundle)
expected = expected_data.SET_COURSE
self.assertEqual(clean_xml(bundle_string), clean_xml(expected))
def export(self):
meta = self.meta
sys.stderr.write("metadata = %s\n" % meta)
fn = self.dir / 'contents/Syllabus/index.htm'
sxml = self.parse_broken_html(fn=fn)
edxxml = etree.Element('course')
edxxml.set('dirname', os.path.basename(os.getcwd()))
edxxml.set('semester', self.DefaultSemester)
for k, v in meta.items():
edxxml.set(k, v)
self.processed_files = [
fn
] # track which content files have been ingested, to avoid duplication
self.files_to_copy = {
} # dict of files (key=OCW source, val=edX static dest) to copy to "/static"
self.processed_pdf_files = []
self.element_counts = defaultdict(int)
self.do_chapters(sxml, edxxml)
policies = self.policies
# grab course image via index.htm
self.get_course_image()
# make xbundle
xb = XBundle(force_studio_format=True)
xb.DefaultOrg = self.DefaultOrg
xb.set_course(edxxml)
xb.add_policies(policies)
self.add_about_files(xb)
def c(x):
return len(xb.course.findall(".//%s" % x))
elist = [
"chapter", "sequential", "vertical", "problem", "html", "video"
]
xbundle_counts = {x: c(x) for x in elist}
self.element_counts['n_static_files'] = len(self.files_to_copy)
self.element_counts['n_ocw_files_processed'] = len(
self.processed_files)
# save it
outfn = self.output_fn or ('%s_xbundle.xml' % self.cid)
if outfn.endswith(".xml"):
xb.save(outfn)
self.copy_static_files(".")
elif outfn.endswith(".tar.gz") or outfn.endswith(".tgz"):
tempd = tempfile.mkdtemp(prefix="tmp_ocw2xbundle")
cdir = path(tempd) / "course"
os.mkdir(cdir)
self.copy_static_files(cdir)
xb.export_to_directory(cdir, dir_include_course_id=False)
curdir = os.path.abspath(os.curdir)
cmd = "cd %s; tar czf '%s/%s' course" % (tempd, curdir, outfn)
print cmd
os.system(cmd)
shutil.rmtree(tempd)
else:
if not os.path.exists(outfn):
print "Making directory for output: %s" % outfn
os.mkdir(outfn)
self.copy_static_files(outfn)
xb.export_to_directory(outfn, dir_include_course_id=False)
print "OCW element counts: %s" % json.dumps(self.element_counts,
indent=4)
print "edX XML element counts: %s" % json.dumps(xbundle_counts,
indent=4)
print "Done, wrote to %s" % outfn
def export(self):
meta = self.meta
sys.stderr.write("metadata = %s\n" % meta)
fn = self.dir / 'contents/Syllabus/index.htm'
sxml = self.parse_broken_html(fn=fn)
edxxml = etree.Element('course')
edxxml.set('dirname',os.path.basename(os.getcwd()))
edxxml.set('semester', self.DefaultSemester)
for k, v in meta.items():
edxxml.set(k,v)
self.processed_files = [fn] # track which content files have been ingested, to avoid duplication
self.files_to_copy = {} # dict of files (key=OCW source, val=edX static dest) to copy to "/static"
self.processed_pdf_files = []
self.element_counts = defaultdict(int)
self.do_chapters(sxml, edxxml)
policies = self.policies
# grab course image via index.htm
self.get_course_image()
# make xbundle
xb = XBundle(force_studio_format=True)
xb.DefaultOrg = self.DefaultOrg
xb.set_course(edxxml)
xb.add_policies(policies)
self.add_about_files(xb)
def c(x):
return len(xb.course.findall(".//%s" % x))
elist = ["chapter", "sequential", "vertical", "problem", "html", "video"]
xbundle_counts = {x:c(x) for x in elist}
self.element_counts['n_static_files'] = len(self.files_to_copy)
self.element_counts['n_ocw_files_processed'] = len(self.processed_files)
# save it
outfn = self.output_fn or ('%s_xbundle.xml' % self.cid)
if outfn.endswith(".xml"):
xb.save(outfn)
self.copy_static_files(".")
elif outfn.endswith(".tar.gz") or outfn.endswith(".tgz"):
tempd = tempfile.mkdtemp(prefix="tmp_ocw2xbundle")
cdir = path(tempd) / "course"
os.mkdir(cdir)
self.copy_static_files(cdir)
xb.export_to_directory(cdir, dir_include_course_id=False)
curdir = os.path.abspath(os.curdir)
cmd = "cd %s; tar czf '%s/%s' course" % (tempd, curdir, outfn)
print cmd
os.system(cmd)
shutil.rmtree(tempd)
else:
if not os.path.exists(outfn):
print "Making directory for output: %s" % outfn
os.mkdir(outfn)
self.copy_static_files(outfn)
xb.export_to_directory(outfn, dir_include_course_id=False)
print "OCW element counts: %s" % json.dumps(self.element_counts, indent=4)
print "edX XML element counts: %s" % json.dumps(xbundle_counts, indent=4)
print "Done, wrote to %s" % outfn
