def test_new_resource(self):
"""
When a new LearningResource is created, it will have no
terms assigned to it.
"""
resource = create_resource(
course=self.course,
parent=None,
resource_type="problem",
title="newly created",
content_xml="<xml/>",
mpath="",
url_name="",
dpath="",
)
terms = resource.terms.all()
self.assertEqual(terms.count(), 0)
def create_resource(self, **kwargs):
"""Creates a learning resource with extra fields"""
learn_res = create_resource(
course=self.course,
parent=kwargs.get('parent'),
resource_type=kwargs.get('resource_type', "example"),
title=kwargs.get('title', "other silly example"),
content_xml=kwargs.get('content_xml', "<blah>other blah</blah>"),
mpath=kwargs.get('mpath', "/otherblah"),
url_name=kwargs.get('url_name'),
dpath='')
learn_res.xa_nr_views = kwargs.get('xa_nr_views', 0)
learn_res.xa_nr_attempts = kwargs.get('xa_nr_attempts', 0)
learn_res.xa_avg_grade = kwargs.get('xa_avg_grade', 0)
learn_res.save()
update_description_path(learn_res)
return learn_res
def create_resource(self, **kwargs):
"""Creates a learning resource with extra fields"""
learn_res = create_resource(
course=self.course,
parent=kwargs.get('parent'),
resource_type=kwargs.get('resource_type', "example"),
title=kwargs.get('title', "other silly example"),
content_xml=kwargs.get('content_xml', "<blah>other blah</blah>"),
mpath=kwargs.get('mpath', "/otherblah"),
url_name=kwargs.get('url_name'),
dpath=''
)
learn_res.xa_nr_views = kwargs.get('xa_nr_views', 0)
learn_res.xa_nr_attempts = kwargs.get('xa_nr_attempts', 0)
learn_res.xa_avg_grade = kwargs.get('xa_avg_grade', 0)
learn_res.save()
update_description_path(learn_res)
return learn_res
def import_children(course, element, parent):
"""
Create LearningResource instances for each element
of an XML tree.
Args:
course (learningresources.Course): Course
element (lxml.etree): XML element within xbundle
parent (learningresources.LearningResource): parent LearningResource
"""
mpath = etree.ElementTree(element).getpath(element)
resource = create_resource(
course=course, parent=parent, resource_type=element.tag,
title=element.attrib.get("display_name", "MISSING"),
content_xml=etree.tostring(element),
mpath=mpath,
)
for child in element.getchildren():
if child.tag in DESCRIPTOR_TAGS:
import_children(course, child, resource)
def setUp(self):
super(TestExport, self).setUp()
# Add some LearningResources on top of the default to make things
# interesting.
tarball_file = self.get_course_single_tarball()
import_file(tarball_file, self.repo.id, self.user.id)
# Add a resource with a '/' in the title and too many characters.
course = self.repo.course_set.first()
resource = create_resource(
course=course,
resource_type=LearningResourceType.objects.first().name,
title="//x"*300,
content_xml="",
mpath="",
url_name=None,
parent=None,
dpath=''
)
update_description_path(resource)
# Add static assets.
with TemporaryFile() as temp1:
with TemporaryFile() as temp2:
temp1.write(b"file1")
temp2.write(b"file2")
file1 = File(temp1, name="iamafile1.txt")
file2 = File(temp2, name="iamafile2.txt")
asset1 = create_static_asset(course.id, file1)
resource.static_assets.add(asset1)
# If url_name is missing we should just use id in filename.
resource.url_name = None
resource.save()
asset2 = create_static_asset(course.id, file2)
self.resource.static_assets.add(asset2)
def setUp(self):
super(TestExport, self).setUp()
# Add some LearningResources on top of the default to make things
# interesting.
tarball_file = self.get_course_single_tarball()
import_file(tarball_file, self.repo.id, self.user.id)
# Add a resource with a '/' in the title and too many characters.
course = self.repo.course_set.first()
resource = create_resource(
course=course,
resource_type=LearningResourceType.objects.first().name,
title="//x" * 300,
content_xml="",
mpath="",
url_name=None,
parent=None,
dpath='')
update_description_path(resource)
# Add static assets.
with TemporaryFile() as temp1:
with TemporaryFile() as temp2:
temp1.write(b"file1")
temp2.write(b"file2")
file1 = File(temp1, name="iamafile1.txt")
file2 = File(temp2, name="iamafile2.txt")
asset1 = create_static_asset(course.id, file1)
resource.static_assets.add(asset1)
# If url_name is missing we should just use id in filename.
resource.url_name = None
resource.save()
asset2 = create_static_asset(course.id, file2)
self.resource.static_assets.add(asset2)
def test_duplicate(self):
"""
Test that exporting resources with same file path
is reported and handled correctly.
"""
with TemporaryFile() as temp1:
with TemporaryFile() as temp2:
temp1.write(b"first")
temp2.write(b"second")
# Create two static assets with the same name but different
# paths so no renaming is done on storage side.
static_filename = "iamafile.txt"
static_path1 = os.path.join(
STATIC_ASSET_BASEPATH.format(
org=self.resource.course.org,
course_number=self.resource.course.course_number,
run=self.resource.course.run,
),
static_filename
)
file1 = File(temp1, name=static_filename)
file2 = File(temp2, name=static_filename)
default_storage.delete(static_path1)
asset1 = create_static_asset(self.resource.course.id, file1)
self.resource.static_assets.add(asset1)
course2 = create_course(
"org2", self.repo.id, self.resource.course.course_number,
self.resource.course.run, self.user.id
)
resource2 = create_resource(
course=course2,
parent=None,
resource_type=self.resource.learning_resource_type.name,
title=self.resource.title,
dpath="",
mpath="",
content_xml="",
url_name=self.resource.url_name
)
static_path2 = os.path.join(
STATIC_ASSET_BASEPATH.format(
org=course2.org,
course_number=course2.course_number,
run=course2.run,
),
static_filename
)
default_storage.delete(static_path2)
asset2 = create_static_asset(resource2.course.id, file2)
resource2.static_assets.add(asset2)
# Export the resources. The second static asset should have
# the number _1 attached to it.
try:
resources = [self.resource, resource2]
tempdir, collision = export_resources_to_directory(
resources)
try:
self.assertTrue(collision)
assert_resource_directory(self, resources, tempdir)
finally:
rmtree(tempdir)
finally:
default_storage.delete(asset1.asset.name)
default_storage.delete(asset2.asset.name)
def import_children(course, element, parent, parent_dpath):
"""
Create LearningResource instances for each element
of an XML tree.
Args:
course (learningresources.models.Course): Course
element (lxml.etree): XML element within xbundle
parent (learningresources.models.LearningResource):
Parent LearningResource
parent_dpath (unicode): parent description path
Returns:
None
"""
# pylint: disable=too-many-locals
title = element.attrib.get("display_name", "MISSING")
mpath = etree.ElementTree(element).getpath(element)
dpath = join_description_paths(parent_dpath, title)
resource = create_resource(
course=course, parent=parent, resource_type=element.tag,
title=title,
content_xml=etree.tostring(element),
mpath=mpath,
url_name=element.attrib.get("url_name", None),
dpath=dpath,
)
target = "/static/"
if element.tag == "video":
subname = get_video_sub(element)
if subname != "":
assets = StaticAsset.objects.filter(
course__id=resource.course_id,
asset=course_asset_basepath(course, subname),
)
for asset in assets:
resource.static_assets.add(asset)
else:
# Recursively find all sub-elements, looking for anything which
# refers to /static/. Then make the association between the
# LearningResource and StaticAsset if the StaticAsset exists.
# This is like doing soup.findAll("a") and checking for whether
# "/static/" is in the href, which would work but also requires
# more code to check for link, img, iframe, script, and others,
# and within those, check for href or src existing.
soup = BeautifulSoup(etree.tostring(element), 'lxml')
for child in soup.findAll():
for _, val in child.attrs.items():
try:
if val.startswith(target):
path = val[len(target):]
try:
asset = StaticAsset.objects.get(
course__id=resource.course_id,
asset=course_asset_basepath(course, path),
)
resource.static_assets.add(asset)
except StaticAsset.DoesNotExist:
continue
except AttributeError:
continue # not a string
for child in element.getchildren():
if child.tag in DESCRIPTOR_TAGS:
import_children(course, child, resource, dpath)
def import_children(course, element, parent, parent_dpath):
"""
Create LearningResource instances for each element
of an XML tree.
Args:
course (learningresources.models.Course): Course
element (lxml.etree): XML element within xbundle
parent (learningresources.models.LearningResource):
Parent LearningResource
parent_dpath (unicode): parent description path
Returns:
None
"""
# pylint: disable=too-many-locals
title = element.attrib.get("display_name", MissingTitle.for_title_field)
desc_path = title
if desc_path == MissingTitle.for_title_field:
desc_path = MissingTitle.for_desc_path_field
mpath = etree.ElementTree(element).getpath(element)
dpath = join_description_paths(parent_dpath, desc_path)
resource = create_resource(
course=course,
parent=parent,
resource_type=element.tag,
title=title,
content_xml=etree.tostring(element),
mpath=mpath,
url_name=element.attrib.get("url_name", None),
dpath=dpath,
)
# temp variable to store static assets for bulk insert
static_assets_to_save = set()
target = "/static/"
if element.tag == "video":
subname = get_video_sub(element)
if subname != "":
assets = StaticAsset.objects.filter(
course__id=resource.course_id,
asset=course_asset_basepath(course, subname),
)
for asset in assets:
static_assets_to_save.add((resource, asset))
else:
# Recursively find all sub-elements, looking for anything which
# refers to /static/. Then make the association between the
# LearningResource and StaticAsset if the StaticAsset exists.
# This is like doing soup.findAll("a") and checking for whether
# "/static/" is in the href, which would work but also requires
# more code to check for link, img, iframe, script, and others,
# and within those, check for href or src existing.
soup = BeautifulSoup(etree.tostring(element), 'lxml')
for child in soup.findAll():
for _, val in child.attrs.items():
try:
if val.startswith(target):
path = val[len(target):]
try:
asset = StaticAsset.objects.get(
course__id=resource.course_id,
asset=course_asset_basepath(course, path),
)
static_assets_to_save.add((resource, asset))
except StaticAsset.DoesNotExist:
continue
except AttributeError:
continue # not a string
# Bulk insert of static assets
# Using this approach to avoid signals during the learning resource .save()
# Each signal triggers a reindex of the learning resource that is useless
# during import because all the learning resources are indexed in bulk at
# the end of the import anyway
ThroughModel = LearningResource.static_assets.through
ThroughModel.objects.bulk_create([
ThroughModel(learningresource_id=resource.id, staticasset_id=asset.id)
for resource, asset in static_assets_to_save
])
for child in element.getchildren():
if child.tag in DESCRIPTOR_TAGS:
import_children(course, child, resource, dpath)
def import_children(course, element, parent, parent_dpath):
"""
Create LearningResource instances for each element
of an XML tree.
Args:
course (learningresources.models.Course): Course
element (lxml.etree): XML element within xbundle
parent (learningresources.models.LearningResource):
Parent LearningResource
parent_dpath (unicode): parent description path
Returns:
None
"""
# pylint: disable=too-many-locals
title = element.attrib.get(
"display_name", MissingTitle.for_title_field)
desc_path = title
if desc_path == MissingTitle.for_title_field:
desc_path = MissingTitle.for_desc_path_field
mpath = etree.ElementTree(element).getpath(element)
dpath = join_description_paths(parent_dpath, desc_path)
resource = create_resource(
course=course, parent=parent, resource_type=element.tag,
title=title,
content_xml=etree.tostring(element),
mpath=mpath,
url_name=element.attrib.get("url_name", None),
dpath=dpath,
)
# temp variable to store static assets for bulk insert
static_assets_to_save = set()
target = "/static/"
if element.tag == "video":
subname = get_video_sub(element)
if subname != "":
assets = StaticAsset.objects.filter(
course__id=resource.course_id,
asset=course_asset_basepath(course, subname),
)
for asset in assets:
static_assets_to_save.add((resource, asset))
else:
# Recursively find all sub-elements, looking for anything which
# refers to /static/. Then make the association between the
# LearningResource and StaticAsset if the StaticAsset exists.
# This is like doing soup.findAll("a") and checking for whether
# "/static/" is in the href, which would work but also requires
# more code to check for link, img, iframe, script, and others,
# and within those, check for href or src existing.
soup = BeautifulSoup(etree.tostring(element), 'lxml')
for child in soup.findAll():
for _, val in child.attrs.items():
try:
if val.startswith(target):
path = val[len(target):]
try:
asset = StaticAsset.objects.get(
course__id=resource.course_id,
asset=course_asset_basepath(course, path),
)
static_assets_to_save.add((resource, asset))
except StaticAsset.DoesNotExist:
continue
except AttributeError:
continue # not a string
# Bulk insert of static assets
# Using this approach to avoid signals during the learning resource .save()
# Each signal triggers a reindex of the learning resource that is useless
# during import because all the learning resources are indexed in bulk at
# the end of the import anyway
ThroughModel = LearningResource.static_assets.through
ThroughModel.objects.bulk_create(
[
ThroughModel(
learningresource_id=resource.id,
staticasset_id=asset.id
)
for resource, asset in static_assets_to_save
]
)
for child in element.getchildren():
if child.tag in DESCRIPTOR_TAGS:
import_children(course, child, resource, dpath)
def test_duplicate(self):
"""
Test that exporting resources with same file path
is reported and handled correctly.
"""
with TemporaryFile() as temp1:
with TemporaryFile() as temp2:
temp1.write(b"first")
temp2.write(b"second")
# Create two static assets with the same name but different
# paths so no renaming is done on storage side.
static_filename = "iamafile.txt"
static_path1 = os.path.join(
STATIC_ASSET_BASEPATH.format(
org=self.resource.course.org,
course_number=self.resource.course.course_number,
run=self.resource.course.run,
), static_filename)
file1 = File(temp1, name=static_filename)
file2 = File(temp2, name=static_filename)
default_storage.delete(static_path1)
asset1 = create_static_asset(self.resource.course.id, file1)
self.resource.static_assets.add(asset1)
course2 = create_course("org2", self.repo.id,
self.resource.course.course_number,
self.resource.course.run, self.user.id)
resource2 = create_resource(
course=course2,
parent=None,
resource_type=self.resource.learning_resource_type.name,
title=self.resource.title,
dpath="",
mpath="",
content_xml="",
url_name=self.resource.url_name)
static_path2 = os.path.join(
STATIC_ASSET_BASEPATH.format(
org=course2.org,
course_number=course2.course_number,
run=course2.run,
), static_filename)
default_storage.delete(static_path2)
asset2 = create_static_asset(resource2.course.id, file2)
resource2.static_assets.add(asset2)
# Export the resources. The second static asset should have
# the number _1 attached to it.
try:
resources = [self.resource, resource2]
tempdir, collision = export_resources_to_directory(
resources)
try:
self.assertTrue(collision)
assert_resource_directory(self, resources, tempdir)
finally:
rmtree(tempdir)
finally:
default_storage.delete(asset1.asset.name)
default_storage.delete(asset2.asset.name)
