def get_name(static_asset_obj):
"""Method to get the name of the asset."""
basepath = STATIC_ASSET_BASEPATH.format(
org=static_asset_obj.course.org,
course_number=static_asset_obj.course.course_number,
run=static_asset_obj.course.run,
)
return static_asset_obj.asset.name.replace(basepath, '')
def get_name(static_asset_obj):
"""Method to get the name of the asset."""
basepath = STATIC_ASSET_BASEPATH.format(
org=static_asset_obj.course.org,
course_number=static_asset_obj.course.course_number,
run=static_asset_obj.course.run,
)
return static_asset_obj.asset.name.replace(basepath, '')
def assert_resource_directory(test_case, resources, tempdir):
"""Assert that files are present with correct content."""
def make_name(resource):
"""Format expected filename."""
if resource.url_name is not None:
return "{id}_{url_name}.xml".format(
id=resource.id,
url_name=slugify(resource.url_name)[:200],
)
else:
return "{id}.xml".format(id=resource.id)
# Key is filename, value is count of filenames encountered.
asset_map = {}
# Verify that content XML exists in right place.
for resource in resources:
type_name = resource.learning_resource_type.name
abs_path = os.path.join(tempdir, type_name, make_name(resource))
test_case.assertTrue(os.path.exists(abs_path))
with open(abs_path) as f:
test_case.assertEqual(f.read(), resource.content_xml)
# Verify static asset placement.
for static_asset in resource.static_assets.all():
prefix = STATIC_ASSET_BASEPATH.format(
org=static_asset.course.org,
course_number=static_asset.course.course_number,
run=static_asset.course.run
)
static_filename = static_asset.asset.name[len(prefix):]
if static_filename in asset_map:
count = asset_map[static_filename]
asset_map[static_filename] += 1
base, ext = os.path.splitext(static_filename)
static_filename = "{base}_{count}{ext}".format(
base=base,
ext=ext,
count=count
)
else:
asset_map[static_filename] = 1
relative_path = os.path.join("static", static_filename)
abs_path = os.path.join(tempdir, relative_path)
test_case.assertTrue(os.path.exists(abs_path))
with open(abs_path, 'rb') as f:
test_case.assertEqual(f.read(), static_asset.asset.read())
def assert_resource_directory(test_case, resources, tempdir):
"""Assert that files are present with correct content."""
def make_name(resource):
"""Format expected filename."""
if resource.url_name is not None:
return "{id}_{url_name}.xml".format(
id=resource.id,
url_name=slugify(resource.url_name)[:200],
)
else:
return "{id}.xml".format(id=resource.id)
# Key is filename, value is count of filenames encountered.
asset_map = {}
# Verify that content XML exists in right place.
for resource in resources:
type_name = resource.learning_resource_type.name
abs_path = os.path.join(tempdir, type_name, make_name(resource))
test_case.assertTrue(os.path.exists(abs_path))
with open(abs_path) as f:
test_case.assertEqual(f.read(), resource.content_xml)
# Verify static asset placement.
for static_asset in resource.static_assets.all():
prefix = STATIC_ASSET_BASEPATH.format(
org=static_asset.course.org,
course_number=static_asset.course.course_number,
run=static_asset.course.run)
static_filename = static_asset.asset.name[len(prefix):]
if static_filename in asset_map:
count = asset_map[static_filename]
asset_map[static_filename] += 1
base, ext = os.path.splitext(static_filename)
static_filename = "{base}_{count}{ext}".format(base=base,
ext=ext,
count=count)
else:
asset_map[static_filename] = 1
relative_path = os.path.join("static", static_filename)
abs_path = os.path.join(tempdir, relative_path)
test_case.assertTrue(os.path.exists(abs_path))
with open(abs_path, 'rb') as f:
test_case.assertEqual(f.read(), static_asset.asset.read())
def test_duplicate(self):
"""
Test that exporting resources with same file path
is reported and handled correctly.
"""
with TemporaryFile() as temp1:
with TemporaryFile() as temp2:
temp1.write(b"first")
temp2.write(b"second")
# Create two static assets with the same name but different
# paths so no renaming is done on storage side.
static_filename = "iamafile.txt"
static_path1 = os.path.join(
STATIC_ASSET_BASEPATH.format(
org=self.resource.course.org,
course_number=self.resource.course.course_number,
run=self.resource.course.run,
),
static_filename
)
file1 = File(temp1, name=static_filename)
file2 = File(temp2, name=static_filename)
default_storage.delete(static_path1)
asset1 = create_static_asset(self.resource.course.id, file1)
self.resource.static_assets.add(asset1)
course2 = create_course(
"org2", self.repo.id, self.resource.course.course_number,
self.resource.course.run, self.user.id
)
resource2 = create_resource(
course=course2,
parent=None,
resource_type=self.resource.learning_resource_type.name,
title=self.resource.title,
dpath="",
mpath="",
content_xml="",
url_name=self.resource.url_name
)
static_path2 = os.path.join(
STATIC_ASSET_BASEPATH.format(
org=course2.org,
course_number=course2.course_number,
run=course2.run,
),
static_filename
)
default_storage.delete(static_path2)
asset2 = create_static_asset(resource2.course.id, file2)
resource2.static_assets.add(asset2)
# Export the resources. The second static asset should have
# the number _1 attached to it.
try:
resources = [self.resource, resource2]
tempdir, collision = export_resources_to_directory(
resources)
try:
self.assertTrue(collision)
assert_resource_directory(self, resources, tempdir)
finally:
rmtree(tempdir)
finally:
default_storage.delete(asset1.asset.name)
default_storage.delete(asset2.asset.name)
def export_resources_to_directory(learning_resources):
"""
Create files of LearningResource and StaticAsset contents inside directory.
Args:
learning_resources (list of learningresources.models.LearningResource):
LearningResources to export in tarball
Returns:
(unicode, bool):
First item is newly created temp directory with files inside of it.
Second item is True if a static asset collision was detected.
"""
def sanitize(url_name):
"""Sanitize title for use in filename."""
# Limit filename to 200 characters since limit is 256
# and we have a number and extension too.
return slugify(url_name)[:200]
tempdir = mkdtemp()
collision = False
type_dirs_created = set()
try:
os.mkdir(os.path.join(tempdir, "static"))
for learning_resource in learning_resources:
if learning_resource.url_name is not None:
filename = "{id}_{url_name}.xml".format(
id=learning_resource.id,
url_name=sanitize(learning_resource.url_name),
)
else:
filename = "{id}.xml".format(id=learning_resource.id)
type_name = learning_resource.learning_resource_type.name
if type_name not in type_dirs_created:
os.mkdir(os.path.join(tempdir, type_name))
type_dirs_created.add(type_name)
with open(os.path.join(tempdir, type_name, filename), 'w') as f:
f.write(learning_resource.content_xml)
# Output to static directory.
for static_asset in learning_resource.static_assets.all():
prefix = STATIC_ASSET_BASEPATH.format(
org=static_asset.course.org,
course_number=static_asset.course.course_number,
run=static_asset.course.run,
)
asset_path = static_asset.asset.name
if asset_path.startswith(prefix):
asset_path = asset_path[len(prefix):]
static_path = os.path.join(tempdir, "static", asset_path)
static_path, found_collision = _find_unused_path(static_path)
if found_collision:
collision = found_collision
path, _ = os.path.split(static_path)
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
with open(static_path, 'wb') as outfile:
outfile.write(static_asset.asset.read())
# If we never put anything in static, remove it.
if not os.listdir(os.path.join(tempdir, "static")):
rmtree(os.path.join(tempdir, "static"))
except:
# Clean up temporary directory since we can't return it anymore.
rmtree(tempdir)
raise
return tempdir, collision
def export_resources_to_directory(learning_resources):
"""
Create files of LearningResource and StaticAsset contents inside directory.
Args:
learning_resources (list of learningresources.models.LearningResource):
LearningResources to export in tarball
Returns:
(unicode, bool):
First item is newly created temp directory with files inside of it.
Second item is True if a static asset collision was detected.
"""
def sanitize(url_name):
"""Sanitize title for use in filename."""
# Limit filename to 200 characters since limit is 256
# and we have a number and extension too.
return slugify(url_name)[:200]
tempdir = mkdtemp()
collision = False
type_dirs_created = set()
try:
os.mkdir(os.path.join(tempdir, "static"))
for learning_resource in learning_resources:
if learning_resource.url_name is not None:
filename = "{id}_{url_name}.xml".format(
id=learning_resource.id,
url_name=sanitize(learning_resource.url_name),
)
else:
filename = "{id}.xml".format(id=learning_resource.id)
type_name = learning_resource.learning_resource_type.name
if type_name not in type_dirs_created:
os.mkdir(os.path.join(tempdir, type_name))
type_dirs_created.add(type_name)
with open(os.path.join(tempdir, type_name, filename), 'w') as f:
f.write(learning_resource.content_xml)
# Output to static directory.
for static_asset in learning_resource.static_assets.all():
prefix = STATIC_ASSET_BASEPATH.format(
org=static_asset.course.org,
course_number=static_asset.course.course_number,
run=static_asset.course.run,
)
asset_path = static_asset.asset.name
if asset_path.startswith(prefix):
asset_path = asset_path[len(prefix):]
static_path = os.path.join(tempdir, "static", asset_path)
static_path, found_collision = _find_unused_path(static_path)
if found_collision:
collision = found_collision
path, _ = os.path.split(static_path)
try:
os.makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
with open(static_path, 'wb') as outfile:
outfile.write(static_asset.asset.read())
# If we never put anything in static, remove it.
if not os.listdir(os.path.join(tempdir, "static")):
rmtree(os.path.join(tempdir, "static"))
except:
# Clean up temporary directory since we can't return it anymore.
rmtree(tempdir)
raise
return tempdir, collision
def test_duplicate(self):
"""
Test that exporting resources with same file path
is reported and handled correctly.
"""
with TemporaryFile() as temp1:
with TemporaryFile() as temp2:
temp1.write(b"first")
temp2.write(b"second")
# Create two static assets with the same name but different
# paths so no renaming is done on storage side.
static_filename = "iamafile.txt"
static_path1 = os.path.join(
STATIC_ASSET_BASEPATH.format(
org=self.resource.course.org,
course_number=self.resource.course.course_number,
run=self.resource.course.run,
), static_filename)
file1 = File(temp1, name=static_filename)
file2 = File(temp2, name=static_filename)
default_storage.delete(static_path1)
asset1 = create_static_asset(self.resource.course.id, file1)
self.resource.static_assets.add(asset1)
course2 = create_course("org2", self.repo.id,
self.resource.course.course_number,
self.resource.course.run, self.user.id)
resource2 = create_resource(
course=course2,
parent=None,
resource_type=self.resource.learning_resource_type.name,
title=self.resource.title,
dpath="",
mpath="",
content_xml="",
url_name=self.resource.url_name)
static_path2 = os.path.join(
STATIC_ASSET_BASEPATH.format(
org=course2.org,
course_number=course2.course_number,
run=course2.run,
), static_filename)
default_storage.delete(static_path2)
asset2 = create_static_asset(resource2.course.id, file2)
resource2.static_assets.add(asset2)
# Export the resources. The second static asset should have
# the number _1 attached to it.
try:
resources = [self.resource, resource2]
tempdir, collision = export_resources_to_directory(
resources)
try:
self.assertTrue(collision)
assert_resource_directory(self, resources, tempdir)
finally:
rmtree(tempdir)
finally:
default_storage.delete(asset1.asset.name)
default_storage.delete(asset2.asset.name)
