def test_get_location_from_path(self):
asset_location = StaticContent.get_location_from_path(u'/c4x/a/b/asset/images_course_image.jpg')
self.assertEqual(
AssetLocator(CourseLocator(u'a', u'b', None, deprecated=True),
u'asset', u'images_course_image.jpg', deprecated=True),
asset_location
)
def get_asset_location(asset_id):
""" Helper method to get the location (and verify it is valid). """
try:
return StaticContent.get_location_from_path(asset_id)
except InvalidLocationError as err:
# return a 'Bad Request' to browser as we have a malformed Location
return JsonResponse({"error": err.message}, status=400)
def test_delete_image_type_asset(self):
""" Tests deletion of image type asset """
image_asset = self.get_sample_asset(self.asset_name, asset_type="image")
thumbnail_image_asset = self.get_sample_asset('delete_test_thumbnail', asset_type="image")
# upload image
response = self.client.post(self.url, {"name": "delete_image_test", "file": image_asset})
self.assertEquals(response.status_code, 200)
uploaded_image_url = json.loads(response.content)['asset']['url']
# upload image thumbnail
response = self.client.post(self.url, {"name": "delete_image_thumb_test", "file": thumbnail_image_asset})
self.assertEquals(response.status_code, 200)
thumbnail_url = json.loads(response.content)['asset']['url']
thumbnail_location = StaticContent.get_location_from_path(thumbnail_url)
image_asset_location = AssetLocation.from_deprecated_string(uploaded_image_url)
content = contentstore().find(image_asset_location)
content.thumbnail_location = thumbnail_location
contentstore().save(content)
with mock.patch('opaque_keys.edx.locator.CourseLocator.make_asset_key') as mock_asset_key:
mock_asset_key.return_value = thumbnail_location
test_url = reverse_course_url(
'assets_handler', self.course.id, kwargs={'asset_key_string': unicode(uploaded_image_url)})
resp = self.client.delete(test_url, HTTP_ACCEPT="application/json")
self.assertEquals(resp.status_code, 204)
def test_happy_path(self, modulestore_type, create_after_overview):
"""
What happens when everything works like we expect it to.
If `create_after_overview` is True, we will temporarily disable
thumbnail creation so that the initial CourseOverview is created without
an image_set, and the CourseOverviewImageSet is created afterwards. If
`create_after_overview` is False, we'll create the CourseOverviewImageSet
at the same time as the CourseOverview.
"""
# Create a real (oversized) image...
image = Image.new("RGB", (800, 400), "blue")
image_buff = StringIO()
image.save(image_buff, format="JPEG")
image_buff.seek(0)
image_name = "big_course_image.jpeg"
with self.store.default_store(modulestore_type):
course = CourseFactory.create(
default_store=modulestore_type, course_image=image_name
)
# Save a real image here...
course_image_asset_key = StaticContent.compute_location(course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key, image_name, 'image/jpeg', image_buff)
contentstore().save(course_image_content)
# If create_after_overview is True, disable thumbnail generation so
# that the CourseOverview object is created and saved without an
# image_set at first (it will be lazily created later).
if create_after_overview:
self.set_config(enabled=False)
# Now generate the CourseOverview...
course_overview = CourseOverview.get_from_id(course.id)
# If create_after_overview is True, no image_set exists yet. Verify
# that, then switch config back over to True and it should lazily
# create the image_set on the next get_from_id() call.
if create_after_overview:
self.assertFalse(hasattr(course_overview, 'image_set'))
self.set_config(enabled=True)
course_overview = CourseOverview.get_from_id(course.id)
self.assertTrue(hasattr(course_overview, 'image_set'))
image_urls = course_overview.image_urls
config = CourseOverviewImageConfig.current()
# Make sure the thumbnail names come out as expected...
self.assertTrue(image_urls['raw'].endswith('big_course_image.jpeg'))
self.assertTrue(image_urls['small'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.small)))
self.assertTrue(image_urls['large'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.large)))
# Now make sure our thumbnails are of the sizes we expect...
for image_url, expected_size in [(image_urls['small'], config.small), (image_urls['large'], config.large)]:
image_key = StaticContent.get_location_from_path(image_url)
image_content = AssetManager.find(image_key)
image = Image.open(StringIO(image_content.data))
self.assertEqual(image.size, expected_size)
def test_different_resolutions(self, src_dimensions):
"""
Test various resolutions of images to make thumbnails of.
Note that our test sizes are small=(200, 100) and large=(400, 200).
1. Images should won't be blown up if it's too small, so a (100, 50)
resolution image will remain (100, 50).
2. However, images *will* be converted using our format and quality
settings (JPEG, 75% -- the PIL default). This is because images with
relatively small dimensions not compressed properly.
3. Image thumbnail naming will maintain the naming convention of the
target resolution, even if the image was not actually scaled to that
size (i.e. it was already smaller). This is mostly because it's
simpler to be consistent, but it also lets us more easily tell which
configuration a thumbnail was created under.
"""
# Create a source image...
image = Image.new("RGB", src_dimensions, "blue")
image_buff = StringIO()
image.save(image_buff, format="PNG")
image_buff.seek(0)
image_name = "src_course_image.png"
course = CourseFactory.create(course_image=image_name)
# Save the image to the contentstore...
course_image_asset_key = StaticContent.compute_location(course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key, image_name, 'image/png', image_buff)
contentstore().save(course_image_content)
# Now generate the CourseOverview...
config = CourseOverviewImageConfig.current()
course_overview = CourseOverview.get_from_id(course.id)
image_urls = course_overview.image_urls
for image_url, target in [(image_urls['small'], config.small), (image_urls['large'], config.large)]:
image_key = StaticContent.get_location_from_path(image_url)
image_content = AssetManager.find(image_key)
image = Image.open(StringIO(image_content.data))
# Naming convention for thumbnail
self.assertTrue(image_url.endswith('src_course_image-png-{}x{}.jpg'.format(*target)))
# Actual thumbnail data
src_x, src_y = src_dimensions
target_x, target_y = target
image_x, image_y = image.size
# I'm basically going to assume the image library knows how to do
# the right thing in terms of handling aspect ratio. We're just
# going to make sure that small images aren't blown up, and that
# we never exceed our target sizes
self.assertLessEqual(image_x, target_x)
self.assertLessEqual(image_y, target_y)
if src_x < target_x and src_y < target_y:
self.assertEqual(src_x, image_x)
self.assertEqual(src_y, image_y)
def test_delete_asset_with_invalid_thumbnail(self):
""" Tests the sad path :( """
test_url = reverse_course_url(
'assets_handler', self.course.id, kwargs={'asset_key_string': unicode(self.uploaded_url)})
self.content.thumbnail_location = StaticContent.get_location_from_path('/c4x/edX/toy/asset/invalid')
contentstore().save(self.content)
resp = self.client.delete(test_url, HTTP_ACCEPT="application/json")
self.assertEquals(resp.status_code, 204)
def memory_image_file(self):
try:
asset_location = StaticContent.get_location_from_path(self.image_url)
content = contentstore().find(asset_location, as_stream=True)
except (NotFoundError, InvalidKeyError):
return None
image_file = StringIO.StringIO(content.copy_to_in_mem().data)
return image_file
def process_request(self, request):
# look to see if the request is prefixed with 'c4x' tag
if request.path.startswith("/" + XASSET_LOCATION_TAG + "/"):
try:
loc = StaticContent.get_location_from_path(request.path)
except InvalidLocationError:
# return a 'Bad Request' to browser as we have a malformed Location
response = HttpResponse()
response.status_code = 400
return response
# first look in our cache so we don't have to round-trip to the DB
content = get_cached_content(loc)
if content is None:
# nope, not in cache, let's fetch from DB
try:
content = contentstore().find(loc, as_stream=True)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
return response
# since we fetched it from DB, let's cache it going forward, but only if it's < 1MB
# this is because I haven't been able to find a means to stream data out of memcached
if content.length is not None:
if content.length < 1048576:
# since we've queried as a stream, let's read in the stream into memory to set in cache
content = content.copy_to_in_mem()
set_cached_content(content)
else:
# NOP here, but we may wish to add a "cache-hit" counter in the future
pass
# Check that user has access to content
if getattr(content, "locked", False):
if not hasattr(request, "user") or not request.user.is_authenticated():
return HttpResponseForbidden("Unauthorized")
course_partial_id = "/".join([loc.org, loc.course])
if not request.user.is_staff and not CourseEnrollment.is_enrolled_by_partial(
request.user, course_partial_id
):
return HttpResponseForbidden("Unauthorized")
# convert over the DB persistent last modified timestamp to a HTTP compatible
# timestamp, so we can simply compare the strings
last_modified_at_str = content.last_modified_at.strftime("%a, %d-%b-%Y %H:%M:%S GMT")
# see if the client has cached this content, if so then compare the
# timestamps, if they are the same then just return a 304 (Not Modified)
if "HTTP_IF_MODIFIED_SINCE" in request.META:
if_modified_since = request.META["HTTP_IF_MODIFIED_SINCE"]
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
response = HttpResponse(content.stream_data(), content_type=content.content_type)
response["Last-Modified"] = last_modified_at_str
return response
def process_request(self, request):
if request.path.startswith('/' + XASSET_LOCATION_TAG + '/'):
if request.GET.get('width') or request.GET.get('height'):
# generate the requested thumbnail location
width = int(request.GET['width'])
path = request.path.split('/')
category = 'thumbnail'
name = os.path.splitext(path[5])
path[5] = "%s-w%d%s" % (name[0], width, name[1])
path[4] = category
path = '/'.join(path)
thumbnail_location = StaticContent.get_location_from_path(path) # calculate thumbnail 'location' in gridfs
try: # is it already created ?
thumbnail_content = contentstore().find(thumbnail_location, as_stream=True)
except NotFoundError:
# get original asset
asset_location = StaticContent.get_location_from_path(request.path)
try:
content = contentstore().find(asset_location, as_stream=True)
except NotFoundError:
return # if original asset do not exists, let the request pass by
# generate thumbnail
im = Image.open(StringIO.StringIO(content.copy_to_in_mem().data))
im = im.convert('RGB')
size = (width, width) # PIL is dumb, we shall use a better tool to generate smartly framed thumbnails
im.thumbnail(size, Image.ANTIALIAS)
thumbnail_file = StringIO.StringIO()
im.save(thumbnail_file, 'JPEG')
thumbnail_file.seek(0)
# store thumbnail in contentstore
thumbnail_name = StaticContent.generate_thumbnail_name(thumbnail_location.name)
thumbnail_content = StaticContentStream(thumbnail_location, thumbnail_name,
'image/jpeg', thumbnail_file)
contentstore().save(thumbnail_content.copy_to_in_mem())
# return found or generated tumbnail
response = HttpResponse(thumbnail_content.copy_to_in_mem().stream_data(), content_type=thumbnail_content.content_type)
return response
return super(ThumbnailStaticContentServer, self).process_request(request)
def test_delete_asset_with_invalid_thumbnail(self):
""" Tests the sad path :( """
test_url = reverse_course_url(
'assets_handler',
self.course.id,
kwargs={'asset_key_string': unicode(self.uploaded_url)})
self.content.thumbnail_location = StaticContent.get_location_from_path(
'/c4x/edX/toy/asset/invalid')
contentstore().save(self.content)
resp = self.client.delete(test_url, HTTP_ACCEPT="application/json")
self.assertEquals(resp.status_code, 204)
def get_versioned_asset_url(asset_path):
"""
Creates a versioned asset URL.
"""
try:
locator = StaticContent.get_location_from_path(asset_path)
content = AssetManager.find(locator, as_stream=True)
return StaticContent.add_version_to_asset_path(asset_path, content.content_digest)
except (InvalidKeyError, ItemNotFoundError):
pass
return asset_path
def get_old_style_versioned_asset_url(asset_path):
"""
Creates an old-style versioned asset URL.
"""
try:
locator = StaticContent.get_location_from_path(asset_path)
content = AssetManager.find(locator, as_stream=True)
return u'{}/{}{}'.format(VERSIONED_ASSETS_PREFIX, content.content_digest, asset_path)
except (InvalidKeyError, ItemNotFoundError):
pass
return asset_path
def get_versioned_asset_url(asset_path):
"""
Creates a versioned asset URL.
"""
try:
locator = StaticContent.get_location_from_path(asset_path)
content = AssetManager.find(locator, as_stream=True)
return StaticContent.add_version_to_asset_path(asset_path, content.content_digest)
except (InvalidKeyError, ItemNotFoundError):
pass
return asset_path
def get_old_style_versioned_asset_url(asset_path):
"""
Creates an old-style versioned asset URL.
"""
try:
locator = StaticContent.get_location_from_path(asset_path)
content = AssetManager.find(locator, as_stream=True)
return f'{VERSIONED_ASSETS_PREFIX}/{content.content_digest}{asset_path}'
except (InvalidKeyError, ItemNotFoundError):
pass
return asset_path
def process_request(self, request):
# look to see if the request is prefixed with 'c4x' tag
if request.path.startswith('/' + XASSET_LOCATION_TAG + '/'):
try:
loc = StaticContent.get_location_from_path(request.path)
except InvalidLocationError:
# return a 'Bad Request' to browser as we have a malformed
# Location
response = HttpResponse()
response.status_code = 400
return response
# first look in our cache so we don't have to round-trip to the DB
content = get_cached_content(loc)
if content is None:
# nope, not in cache, let's fetch from DB
try:
content = contentstore().find(loc)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
return response
# since we fetched it from DB, let's cache it going forward
set_cached_content(content)
else:
# @todo: we probably want to have 'cache hit' counters so we can
# measure the efficacy of our caches
pass
# see if the last-modified at hasn't changed, if not return a 302
# (Not Modified)
# convert over the DB persistent last modified timestamp to a HTTP compatible
# timestamp, so we can simply compare the strings
last_modified_at_str = content.last_modified_at.strftime(
"%a, %d-%b-%Y %H:%M:%S GMT")
# see if the client has cached this content, if so then compare the
# timestamps, if they are the same then just return a 304 (Not
# Modified)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
response = HttpResponse(content.data,
content_type=content.content_type)
response['Last-Modified'] = last_modified_at_str
return response
def process_request(self, request):
# look to see if the request is prefixed with 'c4x' tag
if request.path.startswith('/' + XASSET_LOCATION_TAG + '/'):
try:
loc = StaticContent.get_location_from_path(request.path)
except InvalidLocationError:
# return a 'Bad Request' to browser as we have a malformed
# Location
response = HttpResponse()
response.status_code = 400
return response
# first look in our cache so we don't have to round-trip to the DB
content = get_cached_content(loc)
if content is None:
# nope, not in cache, let's fetch from DB
try:
content = contentstore().find(loc)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
return response
# since we fetched it from DB, let's cache it going forward
set_cached_content(content)
else:
# @todo: we probably want to have 'cache hit' counters so we can
# measure the efficacy of our caches
pass
# see if the last-modified at hasn't changed, if not return a 302
# (Not Modified)
# convert over the DB persistent last modified timestamp to a HTTP compatible
# timestamp, so we can simply compare the strings
last_modified_at_str = content.last_modified_at.strftime(
"%a, %d-%b-%Y %H:%M:%S GMT")
# see if the client has cached this content, if so then compare the
# timestamps, if they are the same then just return a 304 (Not
# Modified)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
response = HttpResponse(
content.data, content_type=content.content_type)
response['Last-Modified'] = last_modified_at_str
return response
def process_request(self, request):
# look to see if the request is prefixed with 'c4x' tag
if request.path.startswith('/' + XASSET_LOCATION_TAG + '/'):
try:
loc = StaticContent.get_location_from_path(request.path)
except InvalidLocationError:
# return a 'Bad Request' to browser as we have a malformed Location
response = HttpResponse()
response.status_code = 400
return response
# first look in our cache so we don't have to round-trip to the DB
content = get_cached_content(loc)
if content is None:
# nope, not in cache, let's fetch from DB
try:
content = contentstore().find(loc, as_stream=True)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
return response
# since we fetched it from DB, let's cache it going forward, but only if it's < 1MB
# this is because I haven't been able to find a means to stream data out of memcached
if content.length is not None:
if content.length < 1048576:
# since we've queried as a stream, let's read in the stream into memory to set in cache
content = content.copy_to_in_mem()
set_cached_content(content)
else:
# NOP here, but we may wish to add a "cache-hit" counter in the future
pass
# see if the last-modified at hasn't changed, if not return a 302 (Not Modified)
# convert over the DB persistent last modified timestamp to a HTTP compatible
# timestamp, so we can simply compare the strings
last_modified_at_str = content.last_modified_at.strftime("%a, %d-%b-%Y %H:%M:%S GMT")
# see if the client has cached this content, if so then compare the
# timestamps, if they are the same then just return a 304 (Not Modified)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
response = HttpResponse(content.stream_data(), content_type=content.content_type)
response['Last-Modified'] = last_modified_at_str
return response
def remove_asset(request, org, course, name):
'''
This method will perform a 'soft-delete' of an asset, which is basically to
copy the asset from the main GridFS collection and into a Trashcan
'''
get_location_and_verify_access(request, org, course, name)
location = request.POST['location']
# make sure the location is valid
try:
loc = StaticContent.get_location_from_path(location)
except InvalidLocationError:
# return a 'Bad Request' to browser as we have a malformed Location
response = HttpResponse()
response.status_code = 400
return response
# also make sure the item to delete actually exists
try:
content = contentstore().find(loc)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
return response
# ok, save the content into the trashcan
contentstore('trashcan').save(content)
# see if there is a thumbnail as well, if so move that as well
if content.thumbnail_location is not None:
try:
thumbnail_content = contentstore().find(content.thumbnail_location)
contentstore('trashcan').save(thumbnail_content)
# hard delete thumbnail from origin
contentstore().delete(thumbnail_content.get_id())
# remove from any caching
del_cached_content(thumbnail_content.location)
except:
pass # OK if this is left dangling
# delete the original
contentstore().delete(content.get_id())
# remove from cache
del_cached_content(content.location)
return HttpResponse()
def remove_asset(request, org, course, name):
'''
This method will perform a 'soft-delete' of an asset, which is basically to
copy the asset from the main GridFS collection and into a Trashcan
'''
get_location_and_verify_access(request, org, course, name)
location = request.POST['location']
# make sure the location is valid
try:
loc = StaticContent.get_location_from_path(location)
except InvalidLocationError:
# return a 'Bad Request' to browser as we have a malformed Location
response = HttpResponse()
response.status_code = 400
return response
# also make sure the item to delete actually exists
try:
content = contentstore().find(loc)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
return response
# ok, save the content into the trashcan
contentstore('trashcan').save(content)
# see if there is a thumbnail as well, if so move that as well
if content.thumbnail_location is not None:
try:
thumbnail_content = contentstore().find(content.thumbnail_location)
contentstore('trashcan').save(thumbnail_content)
# hard delete thumbnail from origin
contentstore().delete(thumbnail_content.get_id())
# remove from any caching
del_cached_content(thumbnail_content.location)
except:
pass # OK if this is left dangling
# delete the original
contentstore().delete(content.get_id())
# remove from cache
del_cached_content(content.location)
return HttpResponse()
def update_asset(request, org, course, name, asset_id):
"""
restful CRUD operations for a course asset.
Currently only the DELETE method is implemented.
org, course, name: Attributes of the Location for the item to edit
asset_id: the URL of the asset (used by Backbone as the id)
"""
get_location_and_verify_access(request, org, course, name)
# make sure the location is valid
try:
loc = StaticContent.get_location_from_path(asset_id)
except InvalidLocationError as err:
# return a 'Bad Request' to browser as we have a malformed Location
return JsonResponse({"error": err.message}, status=400)
# also make sure the item to delete actually exists
try:
content = contentstore().find(loc)
except NotFoundError:
return JsonResponse(status=404)
# ok, save the content into the trashcan
contentstore('trashcan').save(content)
# see if there is a thumbnail as well, if so move that as well
if content.thumbnail_location is not None:
try:
thumbnail_content = contentstore().find(content.thumbnail_location)
contentstore('trashcan').save(thumbnail_content)
# hard delete thumbnail from origin
contentstore().delete(thumbnail_content.get_id())
# remove from any caching
del_cached_content(thumbnail_content.location)
except:
logging.warning('Could not delete thumbnail: ' +
content.thumbnail_location)
# delete the original
contentstore().delete(content.get_id())
# remove from cache
del_cached_content(content.location)
return JsonResponse()
def test_delete_image_type_asset(self):
""" Tests deletion of image type asset """
image_asset = self.get_sample_asset(self.asset_name,
asset_type="image")
thumbnail_image_asset = self.get_sample_asset('delete_test_thumbnail',
asset_type="image")
# upload image
response = self.client.post(self.url, {
"name": "delete_image_test",
"file": image_asset
})
self.assertEqual(response.status_code, 200)
uploaded_image_url = json.loads(
response.content.decode('utf-8'))['asset']['url']
# upload image thumbnail
response = self.client.post(self.url, {
"name": "delete_image_thumb_test",
"file": thumbnail_image_asset
})
self.assertEqual(response.status_code, 200)
thumbnail_url = json.loads(
response.content.decode('utf-8'))['asset']['url']
thumbnail_location = StaticContent.get_location_from_path(
thumbnail_url)
image_asset_location = AssetKey.from_string(uploaded_image_url)
content = contentstore().find(image_asset_location)
content.thumbnail_location = thumbnail_location
contentstore().save(content)
with mock.patch('opaque_keys.edx.locator.CourseLocator.make_asset_key'
) as mock_asset_key:
mock_asset_key.return_value = thumbnail_location
test_url = reverse_course_url(
'assets_handler',
self.course.id,
kwargs={'asset_key_string': six.text_type(uploaded_image_url)})
resp = self.client.delete(test_url, HTTP_ACCEPT="application/json")
self.assertEqual(resp.status_code, 204)
def restore_asset_from_trashcan(location):
'''
This method will restore an asset which got soft deleted and put back in the original course
'''
trash = contentstore('trashcan')
store = contentstore()
loc = StaticContent.get_location_from_path(location)
content = trash.find(loc)
# ok, save the content into the courseware
store.save(content)
# see if there is a thumbnail as well, if so move that as well
if content.thumbnail_location is not None:
try:
thumbnail_content = trash.find(content.thumbnail_location)
store.save(thumbnail_content)
except:
pass # OK if this is left dangling
def restore_asset_from_trashcan(location):
'''
This method will restore an asset which got soft deleted and put back in the original course
'''
trash = contentstore('trashcan')
store = contentstore()
loc = StaticContent.get_location_from_path(location)
content = trash.find(loc)
# ok, save the content into the courseware
store.save(content)
# see if there is a thumbnail as well, if so move that as well
if content.thumbnail_location is not None:
try:
thumbnail_content = trash.find(content.thumbnail_location)
store.save(thumbnail_content)
except Exception: # lint-amnesty, pylint: disable=broad-except
pass # OK if this is left dangling
def static_updater(self, course, source_courselike, courselike_key, dest_id, runtime):
"""
Update special static assets, such as PDF textbooks and wiki resources.
"""
for entry in course.pdf_textbooks:
for chapter in entry.get('chapters', []):
if StaticContent.is_c4x_path(chapter.get('url', '')):
asset_key = StaticContent.get_location_from_path(chapter['url'])
chapter['url'] = StaticContent.get_static_path_from_location(asset_key)
# Original wiki_slugs had value location.course. To make them unique this was changed to 'org.course.name'.
# If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
# values then remap it so that the wiki does not point to the old wiki.
if courselike_key != course.id:
if self.xml_module_store.course_run is None:
original_course_run = courselike_key.run
else:
original_course_run = self.xml_module_store.course_run
original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
courselike_key.org,
courselike_key.course,
original_course_run,
)
if course.wiki_slug == original_unique_wiki_slug or course.wiki_slug == courselike_key.course:
course.wiki_slug = u'{0}.{1}.{2}'.format(
course.id.org,
course.id.course,
course.id.run,
)
# cdodge: more hacks (what else). Seems like we have a
# problem when importing a course (like 6.002) which
# does not have any tabs defined in the policy file.
# The import goes fine and then displays fine in LMS,
# but if someone tries to add a new tab in the CMS, then
# the LMS barfs because it expects that -- if there are
# *any* tabs -- then there at least needs to be
# some predefined ones
if course.tabs is None or len(course.tabs) == 0:
CourseTabList.initialize_default(course)
def test_pdf_asset(self):
module_store = modulestore('direct')
_, course_items = import_from_xml(
module_store,
'common/test/data/',
['toy'],
static_content_store=contentstore(),
verbose=True
)
course = course_items[0]
location = loc_mapper().translate_location(course.location.course_id, course.location, False, True)
url = location.url_reverse('assets/', '')
# Test valid contentType for pdf asset (textbook.pdf)
resp = self.client.get(url, HTTP_ACCEPT='application/json')
self.assertContains(resp, "/c4x/edX/toy/asset/textbook.pdf")
asset_location = StaticContent.get_location_from_path('/c4x/edX/toy/asset/textbook.pdf')
content = contentstore().find(asset_location)
# Check after import textbook.pdf has valid contentType ('application/pdf')
# Note: Actual contentType for textbook.pdf in asset.json is 'text/pdf'
self.assertEqual(content.content_type, 'application/pdf')
def test_pdf_asset(self):
module_store = modulestore('direct')
_, course_items = import_from_xml(module_store,
'common/test/data/', ['toy'],
static_content_store=contentstore(),
verbose=True)
course = course_items[0]
location = loc_mapper().translate_location(course.location.course_id,
course.location, False,
True)
url = location.url_reverse('assets/', '')
# Test valid contentType for pdf asset (textbook.pdf)
resp = self.client.get(url, HTTP_ACCEPT='application/json')
self.assertContains(resp, "/c4x/edX/toy/asset/textbook.pdf")
asset_location = StaticContent.get_location_from_path(
'/c4x/edX/toy/asset/textbook.pdf')
content = contentstore().find(asset_location)
# Check after import textbook.pdf has valid contentType ('application/pdf')
# Note: Actual contentType for textbook.pdf in asset.json is 'text/pdf'
self.assertEqual(content.content_type, 'application/pdf')
def test_static_import(self):
'''
Stuff in static_import should always be imported into contentstore
'''
_, content_store, course, course_location = self.load_test_import_course()
# make sure we have ONE asset in our contentstore ("should_be_imported.html")
all_assets = content_store.get_all_content_for_course(course_location)
print "len(all_assets)=%d" % len(all_assets)
self.assertEqual(len(all_assets), 1)
content = None
try:
location = StaticContent.get_location_from_path('/c4x/edX/test_import_course/asset/should_be_imported.html')
content = content_store.find(location)
except NotFoundError:
pass
self.assertIsNotNone(content)
# make sure course.static_asset_path is correct
print "static_asset_path = {0}".format(course.static_asset_path)
self.assertEqual(course.static_asset_path, 'test_import_course')
def static_updater(self, course, source_courselike, courselike_key,
dest_id, runtime):
"""
Update special static assets, such as PDF textbooks and wiki resources.
"""
for entry in course.pdf_textbooks:
for chapter in entry.get('chapters', []):
if StaticContent.is_c4x_path(chapter.get('url', '')):
asset_key = StaticContent.get_location_from_path(
chapter['url'])
chapter[
'url'] = StaticContent.get_static_path_from_location(
asset_key)
# Original wiki_slugs had value location.course. To make them unique this was changed to 'org.course.name'.
# If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
# values then remap it so that the wiki does not point to the old wiki.
if courselike_key != course.id:
original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
courselike_key.org, courselike_key.course, courselike_key.run)
if course.wiki_slug == original_unique_wiki_slug or course.wiki_slug == courselike_key.course:
course.wiki_slug = u'{0}.{1}.{2}'.format(
course.id.org,
course.id.course,
course.id.run,
)
# cdodge: more hacks (what else). Seems like we have a
# problem when importing a course (like 6.002) which
# does not have any tabs defined in the policy file.
# The import goes fine and then displays fine in LMS,
# but if someone tries to add a new tab in the CMS, then
# the LMS barfs because it expects that -- if there are
# *any* tabs -- then there at least needs to be
# some predefined ones
if course.tabs is None or len(course.tabs) == 0:
CourseTabList.initialize_default(course)
def test_static_import(self):
'''
Stuff in static_import should always be imported into contentstore
'''
_, content_store, course, course_location = self.load_test_import_course()
# make sure we have ONE asset in our contentstore ("should_be_imported.html")
all_assets, count = content_store.get_all_content_for_course(course_location)
print "len(all_assets)=%d" % len(all_assets)
self.assertEqual(len(all_assets), 1)
self.assertEqual(count, 1)
content = None
try:
location = StaticContent.get_location_from_path('/c4x/edX/test_import_course/asset/should_be_imported.html')
content = content_store.find(location)
except NotFoundError:
pass
self.assertIsNotNone(content)
# make sure course.static_asset_path is correct
print "static_asset_path = {0}".format(course.static_asset_path)
self.assertEqual(course.static_asset_path, 'test_import_course')
def _import_course_module(
store, runtime, user_id, data_dir, course_key, dest_course_id, source_course, do_import_static,
verbose,
):
if verbose:
log.debug("Scanning {0} for course module...".format(course_key))
# Quick scan to get course module as we need some info from there.
# Also we need to make sure that the course module is committed
# first into the store
course_data_path = path(data_dir) / source_course.data_dir
log.debug(u'======> IMPORTING course {course_key}'.format(
course_key=course_key,
))
if not do_import_static:
# for old-style xblock where this was actually linked to kvs
source_course.static_asset_path = source_course.data_dir
source_course.save()
log.debug('course static_asset_path={path}'.format(
path=source_course.static_asset_path
))
log.debug('course data_dir={0}'.format(source_course.data_dir))
with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, dest_course_id):
course = _import_module_and_update_references(
source_course, store, user_id,
course_key,
dest_course_id,
do_import_static=do_import_static,
runtime=runtime,
)
for entry in course.pdf_textbooks:
for chapter in entry.get('chapters', []):
if StaticContent.is_c4x_path(chapter.get('url', '')):
asset_key = StaticContent.get_location_from_path(chapter['url'])
chapter['url'] = StaticContent.get_static_path_from_location(asset_key)
# Original wiki_slugs had value location.course. To make them unique this was changed to 'org.course.name'.
# If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
# values then remap it so that the wiki does not point to the old wiki.
if course_key != course.id:
original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
course_key.org,
course_key.course,
course_key.run
)
if course.wiki_slug == original_unique_wiki_slug or course.wiki_slug == course_key.course:
course.wiki_slug = u'{0}.{1}.{2}'.format(
course.id.org,
course.id.course,
course.id.run,
)
# cdodge: more hacks (what else). Seems like we have a
# problem when importing a course (like 6.002) which
# does not have any tabs defined in the policy file.
# The import goes fine and then displays fine in LMS,
# but if someone tries to add a new tab in the CMS, then
# the LMS barfs because it expects that -- if there are
# *any* tabs -- then there at least needs to be
# some predefined ones
if course.tabs is None or len(course.tabs) == 0:
CourseTabList.initialize_default(course)
store.update_item(course, user_id)
return course, course_data_path
def _find_asset_urls_in_block(
task_id,
value,
block_loc,
block_assets,
course_key,
environment,
staff_user_id,
update,
dictionary=None,
value_key=None,
):
if type(value) == dict:
for key, val in value.items():
_find_asset_urls_in_block(task_id,
val,
block_loc,
block_assets,
course_key,
environment,
staff_user_id,
update,
dictionary=value,
value_key=key)
elif type(value) == list:
for item in value:
_find_asset_urls_in_block(task_id,
item,
block_loc,
block_assets,
course_key,
environment,
staff_user_id,
update,
dictionary=dictionary,
value_key=value_key)
elif type(value) in (str, unicode):
save_updated = False
urls = re.findall(URL_RE, value)
for url in urls:
url = strip_tags(url)
parsed_url = urlparse(url)
asset_url = StaticContent.ASSET_URL_RE.match(parsed_url.path)
if asset_url is not None:
# check if asset URL belongs to some other server or course
if parsed_url.hostname != environment or \
asset_url.groupdict().get('course') != course_key.course or \
asset_url.groupdict().get('org') != course_key.org:
asset_info = {
'name': asset_url.groupdict().get('name'),
'module': block_loc,
'available': False
}
asset_path = '{}{}'.format(
StaticContent.get_base_url_path_for_course_assets(
course_key),
asset_url.groupdict().get('name'))
# check if asset exists in this course
try:
loc = StaticContent.get_location_from_path(asset_path)
except (InvalidLocationError, InvalidKeyError):
pass
else:
try:
AssetManager.find(loc, as_stream=True)
except (ItemNotFoundError, NotFoundError):
pass
else:
asset_info['available'] = True
if update:
# replace url with the `asset_path`
full_asset_path = urljoin(
'https://{}'.format(environment),
asset_path)
value = value.replace(url, full_asset_path, 1)
save_updated = True
log.info(
'[{}] Replacing `{}` with new path `{}` in module `{}`'
.format(task_id, url, full_asset_path,
block_loc))
block_assets.append(asset_info)
if urls and save_updated and update:
dictionary[value_key] = value
def process_request(self, request):
"""Process the given request"""
if self.is_asset_request(request):
# Make sure we can convert this request into a location.
if AssetLocator.CANONICAL_NAMESPACE in request.path:
request.path = request.path.replace('block/', 'block@', 1)
try:
loc = StaticContent.get_location_from_path(request.path)
except (InvalidLocationError, InvalidKeyError):
return HttpResponseBadRequest()
# Try and load the asset.
content = None
try:
content = self.load_asset_from_location(loc)
except (ItemNotFoundError, NotFoundError):
return HttpResponseNotFound()
# Set the basics for this request.
newrelic.agent.add_custom_parameter('course_id', loc.course_key)
newrelic.agent.add_custom_parameter('org', loc.org)
newrelic.agent.add_custom_parameter('contentserver.path', loc.path)
# Figure out if this is a CDN using us as the origin.
is_from_cdn = StaticContentServer.is_cdn_request(request)
newrelic.agent.add_custom_parameter('contentserver.from_cdn',
True if is_from_cdn else False)
# Check if this content is locked or not.
locked = self.is_content_locked(content)
newrelic.agent.add_custom_parameter('contentserver.locked',
True if locked else False)
# Check that user has access to the content.
if not self.is_user_authorized(request, content, loc):
return HttpResponseForbidden('Unauthorized')
# Figure out if the client sent us a conditional request, and let them know
# if this asset has changed since then.
last_modified_at_str = content.last_modified_at.strftime(
HTTP_DATE_FORMAT)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
# *** File streaming within a byte range ***
# If a Range is provided, parse Range attribute of the request
# Add Content-Range in the response if Range is structurally correct
# Request -> Range attribute structure: "Range: bytes=first-[last]"
# Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength"
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
response = None
if request.META.get('HTTP_RANGE'):
# If we have a StaticContent, get a StaticContentStream. Can't manipulate the bytes otherwise.
if type(content) == StaticContent:
content = AssetManager.find(loc, as_stream=True)
header_value = request.META['HTTP_RANGE']
try:
unit, ranges = parse_range_header(header_value,
content.length)
except ValueError as exception:
# If the header field is syntactically invalid it should be ignored.
log.exception(u"%s in Range header: %s for content: %s",
exception.message, header_value,
unicode(loc))
else:
if unit != 'bytes':
# Only accept ranges in bytes
log.warning(
u"Unknown unit in Range header: %s for content: %s",
header_value, unicode(loc))
elif len(ranges) > 1:
# According to Http/1.1 spec content for multiple ranges should be sent as a multipart message.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.16
# But we send back the full content.
log.warning(
u"More than 1 ranges in Range header: %s for content: %s",
header_value, unicode(loc))
else:
first, last = ranges[0]
if 0 <= first <= last < content.length:
# If the byte range is satisfiable
response = HttpResponse(
content.stream_data_in_range(first, last))
response[
'Content-Range'] = 'bytes {first}-{last}/{length}'.format(
first=first,
last=last,
length=content.length)
range_len = last - first + 1
response['Content-Length'] = str(range_len)
response.status_code = 206 # Partial Content
newrelic.agent.add_custom_parameter(
'contentserver.range_len', range_len)
else:
log.warning(
u"Cannot satisfy ranges in Range header: %s for content: %s",
header_value, unicode(loc))
return HttpResponse(
status=416) # Requested Range Not Satisfiable
# If Range header is absent or syntactically invalid return a full content response.
if response is None:
response = HttpResponse(content.stream_data())
response['Content-Length'] = content.length
newrelic.agent.add_custom_parameter(
'contentserver.content_len', content.length)
newrelic.agent.add_custom_parameter(
'contentserver.content_type', content.content_type)
# "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed
response['Accept-Ranges'] = 'bytes'
response['Content-Type'] = content.content_type
# Set any caching headers, and do any response cleanup needed. Based on how much
# middleware we have in place, there's no easy way to use the built-in Django
# utilities and properly sanitize and modify a response to ensure that it is as
# cacheable as possible, which is why we do it ourselves.
self.set_caching_headers(content, response)
return response
def test_different_resolutions(self, src_dimensions):
"""
Test various resolutions of images to make thumbnails of.
Note that our test sizes are small=(200, 100) and large=(400, 200).
1. Images should won't be blown up if it's too small, so a (100, 50)
resolution image will remain (100, 50).
2. However, images *will* be converted using our format and quality
settings (JPEG, 75% -- the PIL default). This is because images with
relatively small dimensions not compressed properly.
3. Image thumbnail naming will maintain the naming convention of the
target resolution, even if the image was not actually scaled to that
size (i.e. it was already smaller). This is mostly because it's
simpler to be consistent, but it also lets us more easily tell which
configuration a thumbnail was created under.
"""
# Create a source image...
image = Image.new("RGB", src_dimensions, "blue")
image_buff = StringIO()
image.save(image_buff, format="PNG")
image_buff.seek(0)
image_name = "src_course_image.png"
course = CourseFactory.create(course_image=image_name)
# Save the image to the contentstore...
course_image_asset_key = StaticContent.compute_location(
course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key,
image_name, 'image/png',
image_buff)
contentstore().save(course_image_content)
# Now generate the CourseOverview...
config = CourseOverviewImageConfig.current()
course_overview = CourseOverview.get_from_id(course.id)
image_urls = course_overview.image_urls
for image_url, target in [(image_urls['small'], config.small),
(image_urls['large'], config.large)]:
image_key = StaticContent.get_location_from_path(image_url)
image_content = AssetManager.find(image_key)
image = Image.open(StringIO(image_content.data))
# Naming convention for thumbnail
self.assertTrue(
image_url.endswith(
'src_course_image-png-{}x{}.jpg'.format(*target)))
# Actual thumbnail data
src_x, src_y = src_dimensions
target_x, target_y = target
image_x, image_y = image.size
# I'm basically going to assume the image library knows how to do
# the right thing in terms of handling aspect ratio. We're just
# going to make sure that small images aren't blown up, and that
# we never exceed our target sizes
self.assertLessEqual(image_x, target_x)
self.assertLessEqual(image_y, target_y)
if src_x < target_x and src_y < target_y:
self.assertEqual(src_x, image_x)
self.assertEqual(src_y, image_y)
def _parse_video_xml(cls, xml, id_generator=None):
"""
Parse video fields out of xml_data. The fields are set if they are
present in the XML.
Arguments:
id_generator is used to generate course-specific urls and identifiers
"""
field_data = {}
# Convert between key types for certain attributes --
# necessary for backwards compatibility.
conversions = {
# example: 'start_time': cls._example_convert_start_time
}
# Convert between key names for certain attributes --
# necessary for backwards compatibility.
compat_keys = {
'from': 'start_time',
'to': 'end_time'
}
sources = xml.findall('source')
if sources:
field_data['html5_sources'] = [ele.get('src') for ele in sources]
track = xml.find('track')
if track is not None:
field_data['track'] = track.get('src')
handout = xml.find('handout')
if handout is not None:
field_data['handout'] = handout.get('src')
transcripts = xml.findall('transcript')
if transcripts:
field_data['transcripts'] = {tr.get('language'): tr.get('src') for tr in transcripts}
for attr, value in xml.items():
if attr in compat_keys:
attr = compat_keys[attr]
if attr in cls.metadata_to_strip + ('url_name', 'name'):
continue
if attr == 'youtube':
speeds = cls._parse_youtube(value)
for speed, youtube_id in speeds.items():
# should have made these youtube_id_1_00 for
# cleanliness, but hindsight doesn't need glasses
normalized_speed = speed[:-1] if speed.endswith('0') else speed
# If the user has specified html5 sources, make sure we don't use the default video
if youtube_id != '' or 'html5_sources' in field_data:
field_data['youtube_id_{0}'.format(normalized_speed.replace('.', '_'))] = youtube_id
elif attr in conversions:
field_data[attr] = conversions[attr](value)
elif attr not in cls.fields:
field_data.setdefault('xml_attributes', {})[attr] = value
else:
# We export values with json.dumps (well, except for Strings, but
# for about a month we did it for Strings also).
field_data[attr] = deserialize_field(cls.fields[attr], value)
course_id = getattr(id_generator, 'target_course_id', None)
# Update the handout location with current course_id
if 'handout' in field_data.keys() and course_id:
handout_location = StaticContent.get_location_from_path(field_data['handout'])
if isinstance(handout_location, AssetLocator):
handout_new_location = StaticContent.compute_location(course_id, handout_location.path)
field_data['handout'] = StaticContent.serialize_asset_key_with_slash(handout_new_location)
# For backwards compatibility: Add `source` if XML doesn't have `download_video`
# attribute.
if 'download_video' not in field_data and sources:
field_data['source'] = field_data['html5_sources'][0]
# For backwards compatibility: if XML doesn't have `download_track` attribute,
# it means that it is an old format. So, if `track` has some value,
# `download_track` needs to have value `True`.
if 'download_track' not in field_data and track is not None:
field_data['download_track'] = True
video_asset_elem = xml.find('video_asset')
if (
edxval_api and
video_asset_elem is not None and
'edx_video_id' in field_data
):
# Allow ValCannotCreateError to escape
edxval_api.import_from_xml(
video_asset_elem,
field_data['edx_video_id'],
course_id=course_id
)
# load license if it exists
field_data = LicenseMixin.parse_license_from_xml(field_data, xml)
return field_data
def process_request(self, request):
# look to see if the request is prefixed with an asset prefix tag
if (request.path.startswith('/' + XASSET_LOCATION_TAG + '/')
or request.path.startswith('/' +
AssetLocator.CANONICAL_NAMESPACE)):
if AssetLocator.CANONICAL_NAMESPACE in request.path:
request.path = request.path.replace('block/', 'block@', 1)
try:
loc = StaticContent.get_location_from_path(request.path)
except (InvalidLocationError, InvalidKeyError):
# return a 'Bad Request' to browser as we have a malformed Location
response = HttpResponse()
response.status_code = 400
return response
# first look in our cache so we don't have to round-trip to the DB
content = get_cached_content(loc)
if content is None:
# nope, not in cache, let's fetch from DB
try:
content = AssetManager.find(loc, as_stream=True)
except (ItemNotFoundError, NotFoundError):
response = HttpResponse()
response.status_code = 404
return response
# since we fetched it from DB, let's cache it going forward, but only if it's < 1MB
# this is because I haven't been able to find a means to stream data out of memcached
if content.length is not None:
if content.length < 1048576:
# since we've queried as a stream, let's read in the stream into memory to set in cache
content = content.copy_to_in_mem()
set_cached_content(content)
else:
# NOP here, but we may wish to add a "cache-hit" counter in the future
pass
# Check that user has access to content
if getattr(content, "locked", False):
if not hasattr(request,
"user") or not request.user.is_authenticated():
return HttpResponseForbidden('Unauthorized')
if not request.user.is_staff:
if getattr(
loc, 'deprecated', False
) and not CourseEnrollment.is_enrolled_by_partial(
request.user, loc.course_key):
return HttpResponseForbidden('Unauthorized')
if not getattr(loc, 'deprecated',
False) and not CourseEnrollment.is_enrolled(
request.user, loc.course_key):
return HttpResponseForbidden('Unauthorized')
# convert over the DB persistent last modified timestamp to a HTTP compatible
# timestamp, so we can simply compare the strings
last_modified_at_str = content.last_modified_at.strftime(
"%a, %d-%b-%Y %H:%M:%S GMT")
# see if the client has cached this content, if so then compare the
# timestamps, if they are the same then just return a 304 (Not Modified)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
# *** File streaming within a byte range ***
# If a Range is provided, parse Range attribute of the request
# Add Content-Range in the response if Range is structurally correct
# Request -> Range attribute structure: "Range: bytes=first-[last]"
# Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength"
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
response = None
if request.META.get('HTTP_RANGE'):
# Data from cache (StaticContent) has no easy byte management, so we use the DB instead (StaticContentStream)
if type(content) == StaticContent:
content = AssetManager.find(loc, as_stream=True)
header_value = request.META['HTTP_RANGE']
try:
unit, ranges = parse_range_header(header_value,
content.length)
except ValueError as exception:
# If the header field is syntactically invalid it should be ignored.
log.exception(u"%s in Range header: %s for content: %s",
exception.message, header_value,
unicode(loc))
else:
if unit != 'bytes':
# Only accept ranges in bytes
log.warning(
u"Unknown unit in Range header: %s for content: %s",
header_value, unicode(loc))
elif len(ranges) > 1:
# According to Http/1.1 spec content for multiple ranges should be sent as a multipart message.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.16
# But we send back the full content.
log.warning(
u"More than 1 ranges in Range header: %s for content: %s",
header_value, unicode(loc))
else:
first, last = ranges[0]
if 0 <= first <= last < content.length:
# If the byte range is satisfiable
response = HttpResponse(
content.stream_data_in_range(first, last))
response[
'Content-Range'] = 'bytes {first}-{last}/{length}'.format(
first=first,
last=last,
length=content.length)
response['Content-Length'] = str(last - first + 1)
response.status_code = 206 # Partial Content
else:
log.warning(
u"Cannot satisfy ranges in Range header: %s for content: %s",
header_value, unicode(loc))
return HttpResponse(
status=416) # Requested Range Not Satisfiable
# If Range header is absent or syntactically invalid return a full content response.
if response is None:
response = HttpResponse(content.stream_data())
response['Content-Length'] = content.length
# "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed
response['Accept-Ranges'] = 'bytes'
response['Content-Type'] = content.content_type
response['Last-Modified'] = last_modified_at_str
return response
def process_request(self, request):
# look to see if the request is prefixed with an asset prefix tag
if (request.path.startswith('/' + XASSET_LOCATION_TAG + '/')
or request.path.startswith('/' +
AssetLocator.CANONICAL_NAMESPACE)):
try:
loc = StaticContent.get_location_from_path(request.path)
except (InvalidLocationError, InvalidKeyError):
# return a 'Bad Request' to browser as we have a malformed Location
response = HttpResponse()
response.status_code = 400
return response
# first look in our cache so we don't have to round-trip to the DB
content = get_cached_content(loc)
if content is None:
# nope, not in cache, let's fetch from DB
try:
content = contentstore().find(loc, as_stream=True)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
return response
# since we fetched it from DB, let's cache it going forward, but only if it's < 1MB
# this is because I haven't been able to find a means to stream data out of memcached
if content.length is not None:
if content.length < 1048576:
# since we've queried as a stream, let's read in the stream into memory to set in cache
content = content.copy_to_in_mem()
set_cached_content(content)
else:
# NOP here, but we may wish to add a "cache-hit" counter in the future
pass
# Check that user has access to content
if getattr(content, "locked", False):
if not hasattr(request,
"user") or not request.user.is_authenticated():
return HttpResponseForbidden('Unauthorized')
if not request.user.is_staff:
if getattr(
loc, 'deprecated', False
) and not CourseEnrollment.is_enrolled_by_partial(
request.user, loc.course_key):
return HttpResponseForbidden('Unauthorized')
if not getattr(loc, 'deprecated',
False) and not CourseEnrollment.is_enrolled(
request.user, loc.course_key):
return HttpResponseForbidden('Unauthorized')
# convert over the DB persistent last modified timestamp to a HTTP compatible
# timestamp, so we can simply compare the strings
last_modified_at_str = content.last_modified_at.strftime(
"%a, %d-%b-%Y %H:%M:%S GMT")
# see if the client has cached this content, if so then compare the
# timestamps, if they are the same then just return a 304 (Not Modified)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
response = HttpResponse(content.stream_data(),
content_type=content.content_type)
response['Last-Modified'] = last_modified_at_str
return response
def process_request(self, request):
# look to see if the request is prefixed with an asset prefix tag
if (request.path.startswith('/' + XASSET_LOCATION_TAG + '/')
or request.path.startswith('/' +
AssetLocator.CANONICAL_NAMESPACE)):
try:
loc = StaticContent.get_location_from_path(request.path)
except (InvalidLocationError, InvalidKeyError):
# return a 'Bad Request' to browser as we have a malformed Location
response = HttpResponse()
response.status_code = 400
return response
# first look in our cache so we don't have to round-trip to the DB
content = get_cached_content(loc)
if content is None:
# nope, not in cache, let's fetch from DB
try:
content = contentstore().find(loc, as_stream=True)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
return response
# since we fetched it from DB, let's cache it going forward, but only if it's < 1MB
# this is because I haven't been able to find a means to stream data out of memcached
if content.length is not None:
if content.length < 1048576:
# since we've queried as a stream, let's read in the stream into memory to set in cache
content = content.copy_to_in_mem()
set_cached_content(content)
else:
# NOP here, but we may wish to add a "cache-hit" counter in the future
pass
# Check that user has access to content
if getattr(content, "locked", False):
if not hasattr(request,
"user") or not request.user.is_authenticated():
return HttpResponseForbidden('Unauthorized')
if not request.user.is_staff:
if getattr(
loc, 'deprecated', False
) and not CourseEnrollment.is_enrolled_by_partial(
request.user, loc.course_key):
return HttpResponseForbidden('Unauthorized')
if not getattr(loc, 'deprecated',
False) and not CourseEnrollment.is_enrolled(
request.user, loc.course_key):
return HttpResponseForbidden('Unauthorized')
# convert over the DB persistent last modified timestamp to a HTTP compatible
# timestamp, so we can simply compare the strings
last_modified_at_str = content.last_modified_at.strftime(
"%a, %d-%b-%Y %H:%M:%S GMT")
# see if the client has cached this content, if so then compare the
# timestamps, if they are the same then just return a 304 (Not Modified)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
# *** File streaming within a byte range ***
# If a Range is provided, parse Range attribute of the request
# Add Content-Range in the response if Range is structurally correct
# Request -> Range attribute structure: "Range: bytes=first-[last]"
# Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength"
response = None
if request.META.get('HTTP_RANGE'):
# Data from cache (StaticContent) has no easy byte management, so we use the DB instead (StaticContentStream)
if type(content) == StaticContent:
content = contentstore().find(loc, as_stream=True)
# Let's parse the Range header, bytes=first-[last]
range_header = request.META['HTTP_RANGE']
if '=' in range_header:
unit, byte_range = range_header.split('=')
# "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed
if unit == 'bytes' and '-' in byte_range:
first, last = byte_range.split('-')
# "first" must be a valid integer
try:
first = int(first)
except ValueError:
pass
if type(first) is int:
# "last" default value is the last byte of the file
# Users can ask "bytes=0-" to request the whole file when they don't know the length
try:
last = int(last)
except ValueError:
last = content.length - 1
if 0 <= first <= last < content.length:
# Valid Range attribute
response = HttpResponse(
content.stream_data_in_range(first, last))
response[
'Content-Range'] = 'bytes {first}-{last}/{length}'.format(
first=first,
last=last,
length=content.length)
response['Content-Length'] = str(last - first +
1)
response.status_code = 206 # HTTP_206_PARTIAL_CONTENT
if not response:
# Malformed Range attribute
response = HttpResponse()
response.status_code = 400 # HTTP_400_BAD_REQUEST
return response
else:
# No Range attribute
response = HttpResponse(content.stream_data())
response['Content-Length'] = content.length
response['Accept-Ranges'] = 'bytes'
response['Content-Type'] = content.content_type
response['Last-Modified'] = last_modified_at_str
return response
def process_request(self, request):
# look to see if the request is prefixed with an asset prefix tag
if (
request.path.startswith('/' + XASSET_LOCATION_TAG + '/') or
request.path.startswith('/' + AssetLocator.CANONICAL_NAMESPACE)
):
try:
loc = StaticContent.get_location_from_path(request.path)
except (InvalidLocationError, InvalidKeyError):
# return a 'Bad Request' to browser as we have a malformed Location
response = HttpResponse()
response.status_code = 400
return response
# first look in our cache so we don't have to round-trip to the DB
content = get_cached_content(loc)
if content is None:
# nope, not in cache, let's fetch from DB
try:
content = contentstore().find(loc, as_stream=True)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
return response
# since we fetched it from DB, let's cache it going forward, but only if it's < 1MB
# this is because I haven't been able to find a means to stream data out of memcached
if content.length is not None:
if content.length < 1048576:
# since we've queried as a stream, let's read in the stream into memory to set in cache
content = content.copy_to_in_mem()
set_cached_content(content)
else:
# NOP here, but we may wish to add a "cache-hit" counter in the future
pass
# Check that user has access to content
if getattr(content, "locked", False):
if not hasattr(request, "user") or not request.user.is_authenticated():
return HttpResponseForbidden('Unauthorized')
if not request.user.is_staff:
if getattr(loc, 'deprecated', False) and not CourseEnrollment.is_enrolled_by_partial(
request.user, loc.course_key
):
return HttpResponseForbidden('Unauthorized')
if not getattr(loc, 'deprecated', False) and not CourseEnrollment.is_enrolled(
request.user, loc.course_key
):
return HttpResponseForbidden('Unauthorized')
# convert over the DB persistent last modified timestamp to a HTTP compatible
# timestamp, so we can simply compare the strings
last_modified_at_str = content.last_modified_at.strftime("%a, %d-%b-%Y %H:%M:%S GMT")
# see if the client has cached this content, if so then compare the
# timestamps, if they are the same then just return a 304 (Not Modified)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
# *** File streaming within a byte range ***
# If a Range is provided, parse Range attribute of the request
# Add Content-Range in the response if Range is structurally correct
# Request -> Range attribute structure: "Range: bytes=first-[last]"
# Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength"
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
response = None
if request.META.get('HTTP_RANGE'):
# Data from cache (StaticContent) has no easy byte management, so we use the DB instead (StaticContentStream)
if type(content) == StaticContent:
content = contentstore().find(loc, as_stream=True)
header_value = request.META['HTTP_RANGE']
try:
unit, ranges = parse_range_header(header_value, content.length)
except ValueError as exception:
# If the header field is syntactically invalid it should be ignored.
log.exception(
u"%s in Range header: %s for content: %s", exception.message, header_value, unicode(loc)
)
else:
if unit != 'bytes':
# Only accept ranges in bytes
log.warning(u"Unknown unit in Range header: %s for content: %s", header_value, unicode(loc))
elif len(ranges) > 1:
# According to Http/1.1 spec content for multiple ranges should be sent as a multipart message.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.16
# But we send back the full content.
log.warning(
u"More than 1 ranges in Range header: %s for content: %s", header_value, unicode(loc)
)
else:
first, last = ranges[0]
if 0 <= first <= last < content.length:
# If the byte range is satisfiable
response = HttpResponse(content.stream_data_in_range(first, last))
response['Content-Range'] = 'bytes {first}-{last}/{length}'.format(
first=first, last=last, length=content.length
)
response['Content-Length'] = str(last - first + 1)
response.status_code = 206 # Partial Content
else:
log.warning(
u"Cannot satisfy ranges in Range header: %s for content: %s", header_value, unicode(loc)
)
return HttpResponse(status=416) # Requested Range Not Satisfiable
# If Range header is absent or syntactically invalid return a full content response.
if response is None:
response = HttpResponse(content.stream_data())
response['Content-Length'] = content.length
# "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed
response['Accept-Ranges'] = 'bytes'
response['Content-Type'] = content.content_type
response['Last-Modified'] = last_modified_at_str
return response
def process_request(self, request):
# look to see if the request is prefixed with an asset prefix tag
if request.path.startswith("/" + XASSET_LOCATION_TAG + "/") or request.path.startswith(
"/" + AssetLocator.CANONICAL_NAMESPACE
):
try:
loc = StaticContent.get_location_from_path(request.path)
except (InvalidLocationError, InvalidKeyError):
# return a 'Bad Request' to browser as we have a malformed Location
response = HttpResponse()
response.status_code = 400
return response
# first look in our cache so we don't have to round-trip to the DB
content = get_cached_content(loc)
if content is None:
# nope, not in cache, let's fetch from DB
try:
content = contentstore().find(loc, as_stream=True)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
return response
# since we fetched it from DB, let's cache it going forward, but only if it's < 1MB
# this is because I haven't been able to find a means to stream data out of memcached
if content.length is not None:
if content.length < 1048576:
# since we've queried as a stream, let's read in the stream into memory to set in cache
content = content.copy_to_in_mem()
set_cached_content(content)
else:
# NOP here, but we may wish to add a "cache-hit" counter in the future
pass
# Check that user has access to content
if getattr(content, "locked", False):
if not hasattr(request, "user") or not request.user.is_authenticated():
return HttpResponseForbidden("Unauthorized")
if not request.user.is_staff:
if getattr(loc, "deprecated", False) and not CourseEnrollment.is_enrolled_by_partial(
request.user, loc.course_key
):
return HttpResponseForbidden("Unauthorized")
if not getattr(loc, "deprecated", False) and not CourseEnrollment.is_enrolled(
request.user, loc.course_key
):
return HttpResponseForbidden("Unauthorized")
# convert over the DB persistent last modified timestamp to a HTTP compatible
# timestamp, so we can simply compare the strings
last_modified_at_str = content.last_modified_at.strftime("%a, %d-%b-%Y %H:%M:%S GMT")
# see if the client has cached this content, if so then compare the
# timestamps, if they are the same then just return a 304 (Not Modified)
if "HTTP_IF_MODIFIED_SINCE" in request.META:
if_modified_since = request.META["HTTP_IF_MODIFIED_SINCE"]
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
# *** File streaming within a byte range ***
# If a Range is provided, parse Range attribute of the request
# Add Content-Range in the response if Range is structurally correct
# Request -> Range attribute structure: "Range: bytes=first-[last]"
# Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength"
response = None
if request.META.get("HTTP_RANGE"):
# Data from cache (StaticContent) has no easy byte management, so we use the DB instead (StaticContentStream)
if type(content) == StaticContent:
content = contentstore().find(loc, as_stream=True)
# Let's parse the Range header, bytes=first-[last]
range_header = request.META["HTTP_RANGE"]
if "=" in range_header:
unit, byte_range = range_header.split("=")
# "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed
if unit == "bytes" and "-" in byte_range:
first, last = byte_range.split("-")
# "first" must be a valid integer
try:
first = int(first)
except ValueError:
pass
if type(first) is int:
# "last" default value is the last byte of the file
# Users can ask "bytes=0-" to request the whole file when they don't know the length
try:
last = int(last)
except ValueError:
last = content.length - 1
if 0 <= first <= last < content.length:
# Valid Range attribute
response = HttpResponse(content.stream_data_in_range(first, last))
response["Content-Range"] = "bytes {first}-{last}/{length}".format(
first=first, last=last, length=content.length
)
response["Content-Length"] = str(last - first + 1)
response.status_code = 206 # HTTP_206_PARTIAL_CONTENT
if not response:
# Malformed Range attribute
response = HttpResponse()
response.status_code = 400 # HTTP_400_BAD_REQUEST
return response
else:
# No Range attribute
response = HttpResponse(content.stream_data())
response["Content-Length"] = content.length
response["Accept-Ranges"] = "bytes"
response["Content-Type"] = content.content_type
response["Last-Modified"] = last_modified_at_str
return response
def verify_asset_locked_state(locked):
""" Helper method to verify lock state in the contentstore """
asset_location = StaticContent.get_location_from_path(
'/c4x/edX/toy/asset/sample_static.txt')
content = contentstore().find(asset_location)
self.assertEqual(content.locked, locked)
def process_request(self, request):
"""Process the given request"""
if self.is_asset_request(request):
# Make sure we can convert this request into a location.
if AssetLocator.CANONICAL_NAMESPACE in request.path:
request.path = request.path.replace('block/', 'block@', 1)
try:
loc = StaticContent.get_location_from_path(request.path)
except (InvalidLocationError, InvalidKeyError):
return HttpResponseBadRequest()
# Try and load the asset.
content = None
try:
content = self.load_asset_from_location(loc)
except (ItemNotFoundError, NotFoundError):
return HttpResponseNotFound()
# Check that user has access to the content.
if not self.is_user_authorized(request, content, loc):
return HttpResponseForbidden('Unauthorized')
# Figure out if the client sent us a conditional request, and let them know
# if this asset has changed since then.
last_modified_at_str = content.last_modified_at.strftime(HTTP_DATE_FORMAT)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
# *** File streaming within a byte range ***
# If a Range is provided, parse Range attribute of the request
# Add Content-Range in the response if Range is structurally correct
# Request -> Range attribute structure: "Range: bytes=first-[last]"
# Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength"
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
response = None
if request.META.get('HTTP_RANGE'):
# If we have a StaticContent, get a StaticContentStream. Can't manipulate the bytes otherwise.
if type(content) == StaticContent:
content = AssetManager.find(loc, as_stream=True)
header_value = request.META['HTTP_RANGE']
try:
unit, ranges = parse_range_header(header_value, content.length)
except ValueError as exception:
# If the header field is syntactically invalid it should be ignored.
log.exception(
u"%s in Range header: %s for content: %s", exception.message, header_value, unicode(loc)
)
else:
if unit != 'bytes':
# Only accept ranges in bytes
log.warning(u"Unknown unit in Range header: %s for content: %s", header_value, unicode(loc))
elif len(ranges) > 1:
# According to Http/1.1 spec content for multiple ranges should be sent as a multipart message.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.16
# But we send back the full content.
log.warning(
u"More than 1 ranges in Range header: %s for content: %s", header_value, unicode(loc)
)
else:
first, last = ranges[0]
if 0 <= first <= last < content.length:
# If the byte range is satisfiable
response = HttpResponse(content.stream_data_in_range(first, last))
response['Content-Range'] = 'bytes {first}-{last}/{length}'.format(
first=first, last=last, length=content.length
)
response['Content-Length'] = str(last - first + 1)
response.status_code = 206 # Partial Content
else:
log.warning(
u"Cannot satisfy ranges in Range header: %s for content: %s", header_value, unicode(loc)
)
return HttpResponse(status=416) # Requested Range Not Satisfiable
# If Range header is absent or syntactically invalid return a full content response.
if response is None:
response = HttpResponse(content.stream_data())
response['Content-Length'] = content.length
# "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed
response['Accept-Ranges'] = 'bytes'
response['Content-Type'] = content.content_type
# Set any caching headers, and do any response cleanup needed. Based on how much
# middleware we have in place, there's no easy way to use the built-in Django
# utilities and properly sanitize and modify a response to ensure that it is as
# cacheable as possible, which is why we do it ourselves.
self.set_caching_headers(content, response)
return response
def remap_namespace(module, target_location_namespace):
if target_location_namespace is None:
return module
original_location = module.location
# This looks a bit wonky as we need to also change the 'name' of the
# imported course to be what the caller passed in
if module.location.category != 'course':
_update_module_location(
module,
module.location.replace(
tag=target_location_namespace.tag,
org=target_location_namespace.org,
course=target_location_namespace.course
)
)
else:
#
# module is a course module
#
module.location = module.location.replace(
tag=target_location_namespace.tag,
org=target_location_namespace.org,
course=target_location_namespace.course,
name=target_location_namespace.name
)
# There is more re-namespacing work we have to do when
# importing course modules
# remap pdf_textbook urls to portable static URLs
for entry in module.pdf_textbooks:
for chapter in entry.get('chapters', []):
if StaticContent.is_c4x_path(chapter.get('url', '')):
chapter_loc = StaticContent.get_location_from_path(chapter['url'])
chapter['url'] = StaticContent.get_static_path_from_location(
chapter_loc
)
# Original wiki_slugs had value location.course. To make them unique this was changed to 'org.course.name'.
# If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
# values then remap it so that the wiki does not point to the old wiki.
if original_location.course_id != target_location_namespace.course_id:
original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
original_location.org,
original_location.course,
original_location.name
)
if module.wiki_slug == original_unique_wiki_slug or module.wiki_slug == original_location.course:
module.wiki_slug = u'{0}.{1}.{2}'.format(
target_location_namespace.org,
target_location_namespace.course,
target_location_namespace.name,
)
module.save()
all_fields = module.get_explicitly_set_fields_by_scope(Scope.content)
all_fields.update(module.get_explicitly_set_fields_by_scope(Scope.settings))
if hasattr(module, 'children'):
all_fields['children'] = module.children
def convert_ref(reference):
"""
Convert a reference to the new namespace, but only
if the original namespace matched the original course.
Otherwise, returns the input value.
"""
new_ref = reference
ref = Location(reference)
in_original_namespace = (original_location.tag == ref.tag and
original_location.org == ref.org and
original_location.course == ref.course)
if in_original_namespace:
new_ref = ref.replace(
tag=target_location_namespace.tag,
org=target_location_namespace.org,
course=target_location_namespace.course
).url()
return new_ref
for field_name in all_fields:
field_object = module.fields.get(field_name)
if isinstance(field_object, Reference):
new_ref = convert_ref(getattr(module, field_name))
setattr(module, field_name, new_ref)
module.save()
elif isinstance(field_object, ReferenceList):
references = getattr(module, field_name)
new_references = [convert_ref(reference) for reference in references]
setattr(module, field_name, new_references)
module.save()
elif isinstance(field_object, ReferenceValueDict):
reference_dict = getattr(module, field_name)
new_reference_dict = {
key: convert_ref(reference)
for key, reference
in reference_dict.items()
}
setattr(module, field_name, new_reference_dict)
module.save()
return module
def test_get_location_from_path(self):
asset_location = StaticContent.get_location_from_path(u'/c4x/foo/bar/asset/images_course_image.jpg')
self.assertEqual(
AssetLocation(u'foo', u'bar', None, u'asset', u'images_course_image.jpg', None),
asset_location
)
def process_request(self, request):
"""Process the given request"""
asset_path = request.path
if self.is_asset_request(request): # lint-amnesty, pylint: disable=too-many-nested-blocks
# Make sure we can convert this request into a location.
if AssetLocator.CANONICAL_NAMESPACE in asset_path:
asset_path = asset_path.replace('block/', 'block@', 1)
# If this is a versioned request, pull out the digest and chop off the prefix.
requested_digest = None
if StaticContent.is_versioned_asset_path(asset_path):
requested_digest, asset_path = StaticContent.parse_versioned_asset_path(
asset_path)
# Make sure we have a valid location value for this asset.
try:
loc = StaticContent.get_location_from_path(asset_path)
except (InvalidLocationError, InvalidKeyError):
return HttpResponseBadRequest()
# Attempt to load the asset to make sure it exists, and grab the asset digest
# if we're able to load it.
actual_digest = None
try:
content = self.load_asset_from_location(loc)
actual_digest = getattr(content, "content_digest", None)
except (ItemNotFoundError, NotFoundError):
return HttpResponseNotFound()
# If this was a versioned asset, and the digest doesn't match, redirect
# them to the actual version.
if requested_digest is not None and actual_digest is not None and (
actual_digest != requested_digest):
actual_asset_path = StaticContent.add_version_to_asset_path(
asset_path, actual_digest)
return HttpResponsePermanentRedirect(actual_asset_path)
# Set the basics for this request. Make sure that the course key for this
# asset has a run, which old-style courses do not. Otherwise, this will
# explode when the key is serialized to be sent to NR.
safe_course_key = loc.course_key
if safe_course_key.run is None:
safe_course_key = safe_course_key.replace(run='only')
if newrelic:
newrelic.agent.add_custom_parameter('course_id',
safe_course_key)
newrelic.agent.add_custom_parameter('org', loc.org)
newrelic.agent.add_custom_parameter('contentserver.path',
loc.path)
# Figure out if this is a CDN using us as the origin.
is_from_cdn = StaticContentServer.is_cdn_request(request)
newrelic.agent.add_custom_parameter('contentserver.from_cdn',
is_from_cdn)
# Check if this content is locked or not.
locked = self.is_content_locked(content)
newrelic.agent.add_custom_parameter('contentserver.locked',
locked)
# Check that user has access to the content.
if not self.is_user_authorized(request, content, loc):
return HttpResponseForbidden('Unauthorized')
# Figure out if the client sent us a conditional request, and let them know
# if this asset has changed since then.
last_modified_at_str = content.last_modified_at.strftime(
HTTP_DATE_FORMAT)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
# *** File streaming within a byte range ***
# If a Range is provided, parse Range attribute of the request
# Add Content-Range in the response if Range is structurally correct
# Request -> Range attribute structure: "Range: bytes=first-[last]"
# Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength"
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
response = None
if request.META.get('HTTP_RANGE'):
# If we have a StaticContent, get a StaticContentStream. Can't manipulate the bytes otherwise.
if isinstance(content, StaticContent):
content = AssetManager.find(loc, as_stream=True)
header_value = request.META['HTTP_RANGE']
try:
unit, ranges = parse_range_header(header_value,
content.length)
except ValueError as exception:
# If the header field is syntactically invalid it should be ignored.
log.exception("%s in Range header: %s for content: %s",
str(exception), header_value, str(loc))
else:
if unit != 'bytes':
# Only accept ranges in bytes
log.warning(
"Unknown unit in Range header: %s for content: %s",
header_value, str(loc))
elif len(ranges) > 1:
# According to Http/1.1 spec content for multiple ranges should be sent as a multipart message.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.16
# But we send back the full content.
log.warning(
"More than 1 ranges in Range header: %s for content: %s",
header_value, str(loc))
else:
first, last = ranges[0]
if 0 <= first <= last < content.length:
# If the byte range is satisfiable
response = HttpResponse(
content.stream_data_in_range(first, last))
response[
'Content-Range'] = 'bytes {first}-{last}/{length}'.format(
first=first,
last=last,
length=content.length)
response['Content-Length'] = str(last - first + 1)
response.status_code = 206 # Partial Content
if newrelic:
newrelic.agent.add_custom_parameter(
'contentserver.ranged', True)
else:
log.warning(
"Cannot satisfy ranges in Range header: %s for content: %s",
header_value, str(loc))
return HttpResponse(
status=416) # Requested Range Not Satisfiable
# If Range header is absent or syntactically invalid return a full content response.
if response is None:
response = HttpResponse(content.stream_data())
response['Content-Length'] = content.length
if newrelic:
newrelic.agent.add_custom_parameter(
'contentserver.content_len', content.length)
newrelic.agent.add_custom_parameter(
'contentserver.content_type', content.content_type)
# "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed
response['Accept-Ranges'] = 'bytes'
response['Content-Type'] = content.content_type
response['X-Frame-Options'] = 'ALLOW'
# Set any caching headers, and do any response cleanup needed. Based on how much
# middleware we have in place, there's no easy way to use the built-in Django
# utilities and properly sanitize and modify a response to ensure that it is as
# cacheable as possible, which is why we do it ourselves.
self.set_caching_headers(content, response)
return response
def verify_asset_locked_state(locked):
""" Helper method to verify lock state in the contentstore """
asset_location = StaticContent.get_location_from_path('/c4x/edX/toy/asset/sample_static.txt')
content = contentstore().find(asset_location)
self.assertEqual(content.locked, locked)
def import_from_xml(store,
data_dir,
course_dirs=None,
default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True,
static_content_store=None,
target_course_id=None,
verbose=False,
draft_store=None,
do_import_static=True,
create_new_course=False):
"""
Import the specified xml data_dir into the "store" modulestore,
using org and course as the location org and course.
course_dirs: If specified, the list of course_dirs to load. Otherwise, load
all course dirs
target_course_id is the CourseKey that all modules should be remapped to
after import off disk. We do this remapping as a post-processing step
because there's logic in the importing which expects a 'url_name' as an
identifier to where things are on disk
e.g. ../policies/<url_name>/policy.json as well as metadata keys in
the policy.json. so we need to keep the original url_name during import
:param do_import_static:
if False, then static files are not imported into the static content
store. This can be employed for courses which have substantial
unchanging static content, which is to inefficient to import every
time the course is loaded. Static content for some courses may also be
served directly by nginx, instead of going through django.
: create_new_course:
If True, then courses whose ids already exist in the store are not imported.
The check for existing courses is case-insensitive.
"""
xml_module_store = XMLModuleStore(
data_dir,
default_class=default_class,
course_dirs=course_dirs,
load_error_modules=load_error_modules,
xblock_mixins=store.xblock_mixins,
xblock_select=store.xblock_select,
)
# If we're going to remap the course_id, then we can only do that with
# a single course
if target_course_id:
assert (len(xml_module_store.modules) == 1)
# NOTE: the XmlModuleStore does not implement get_items()
# which would be a preferable means to enumerate the entire collection
# of course modules. It will be left as a TBD to implement that
# method on XmlModuleStore.
course_items = []
for course_key in xml_module_store.modules.keys():
if target_course_id is not None:
dest_course_id = target_course_id
else:
dest_course_id = course_key
if create_new_course:
# this tests if exactly this course (ignoring case) exists; so, it checks the run
if store.has_course(dest_course_id, ignore_case=True):
log.debug("Skipping import of course with id, {0},"
"since it collides with an existing one".format(
dest_course_id))
continue
else:
try:
store.create_course(dest_course_id.org,
dest_course_id.offering)
except InvalidLocationError:
# course w/ same org and course exists and store is old mongo
log.debug("Skipping import of course with id, {0},"
"since it collides with an existing one".format(
dest_course_id))
continue
try:
# turn off all write signalling while importing as this
# is a high volume operation on stores that need it
if hasattr(store, 'ignore_write_events_on_courses'):
store.ignore_write_events_on_courses.add(dest_course_id)
course_data_path = None
if verbose:
log.debug(
"Scanning {0} for course module...".format(course_key))
# Quick scan to get course module as we need some info from there.
# Also we need to make sure that the course module is committed
# first into the store
for module in xml_module_store.modules[course_key].itervalues():
if module.scope_ids.block_type == 'course':
course_data_path = path(data_dir) / module.data_dir
log.debug(u'======> IMPORTING course {course_key}'.format(
course_key=course_key, ))
if not do_import_static:
# for old-style xblock where this was actually linked to kvs
module.static_asset_path = module.data_dir
module.save()
log.debug('course static_asset_path={path}'.format(
path=module.static_asset_path))
log.debug('course data_dir={0}'.format(module.data_dir))
course = import_module(module,
store,
course_key,
dest_course_id,
do_import_static=do_import_static)
for entry in course.pdf_textbooks:
for chapter in entry.get('chapters', []):
if StaticContent.is_c4x_path(chapter.get(
'url', '')):
asset_key = StaticContent.get_location_from_path(
chapter['url'])
chapter[
'url'] = StaticContent.get_static_path_from_location(
asset_key)
# Original wiki_slugs had value location.course. To make them unique this was changed to 'org.course.name'.
# If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
# values then remap it so that the wiki does not point to the old wiki.
if course_key != course.id:
original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
course_key.org, course_key.course, course_key.run)
if course.wiki_slug == original_unique_wiki_slug or course.wiki_slug == course_key.course:
course.wiki_slug = u'{0}.{1}.{2}'.format(
course.id.org,
course.id.course,
course.id.run,
)
# cdodge: more hacks (what else). Seems like we have a
# problem when importing a course (like 6.002) which
# does not have any tabs defined in the policy file.
# The import goes fine and then displays fine in LMS,
# but if someone tries to add a new tab in the CMS, then
# the LMS barfs because it expects that -- if there are
# *any* tabs -- then there at least needs to be
# some predefined ones
if course.tabs is None or len(course.tabs) == 0:
CourseTabList.initialize_default(course)
store.update_item(course)
course_items.append(course)
# then import all the static content
if static_content_store is not None and do_import_static:
# first pass to find everything in /static/
import_static_content(course_data_path,
static_content_store,
dest_course_id,
subpath='static',
verbose=verbose)
elif verbose and not do_import_static:
log.debug(
"Skipping import of static content, "
"since do_import_static={0}".format(do_import_static))
# no matter what do_import_static is, import "static_import" directory
# This is needed because the "about" pages (eg "overview") are
# loaded via load_extra_content, and do not inherit the lms
# metadata from the course module, and thus do not get
# "static_content_store" properly defined. Static content
# referenced in those extra pages thus need to come through the
# c4x:// contentstore, unfortunately. Tell users to copy that
# content into the "static_import" subdir.
simport = 'static_import'
if os.path.exists(course_data_path / simport):
import_static_content(course_data_path,
static_content_store,
dest_course_id,
subpath=simport,
verbose=verbose)
# finally loop through all the modules
for module in xml_module_store.modules[course_key].itervalues():
if module.scope_ids.block_type == 'course':
# we've already saved the course module up at the top
# of the loop so just skip over it in the inner loop
continue
if verbose:
log.debug('importing module location {loc}'.format(
loc=module.location))
import_module(module,
store,
course_key,
dest_course_id,
do_import_static=do_import_static,
system=course.runtime)
# now import any 'draft' items
if draft_store is not None:
import_course_draft(xml_module_store, store, draft_store,
course_data_path, static_content_store,
course_key, dest_course_id, course.runtime)
finally:
# turn back on all write signalling on stores that need it
if (hasattr(store, 'ignore_write_events_on_courses') and
dest_course_id in store.ignore_write_events_on_courses):
store.ignore_write_events_on_courses.remove(dest_course_id)
store.refresh_cached_metadata_inheritance_tree(dest_course_id)
return xml_module_store, course_items
def import_from_xml(
store, user_id, data_dir, course_dirs=None,
default_class='xmodule.raw_module.RawDescriptor',
load_error_modules=True, static_content_store=None,
target_course_id=None, verbose=False,
do_import_static=True, create_new_course_if_not_present=False):
"""
Import the specified xml data_dir into the "store" modulestore,
using org and course as the location org and course.
course_dirs: If specified, the list of course_dirs to load. Otherwise, load
all course dirs
target_course_id is the CourseKey that all modules should be remapped to
after import off disk. We do this remapping as a post-processing step
because there's logic in the importing which expects a 'url_name' as an
identifier to where things are on disk
e.g. ../policies/<url_name>/policy.json as well as metadata keys in
the policy.json. so we need to keep the original url_name during import
:param do_import_static:
if False, then static files are not imported into the static content
store. This can be employed for courses which have substantial
unchanging static content, which is to inefficient to import every
time the course is loaded. Static content for some courses may also be
served directly by nginx, instead of going through django.
: create_new_course_if_not_present:
If True, then a new course is created if it doesn't already exist.
The check for existing courses is case-insensitive.
"""
xml_module_store = XMLModuleStore(
data_dir,
default_class=default_class,
course_dirs=course_dirs,
load_error_modules=load_error_modules,
xblock_mixins=store.xblock_mixins,
xblock_select=store.xblock_select,
)
# If we're going to remap the course_id, then we can only do that with
# a single course
if target_course_id:
assert(len(xml_module_store.modules) == 1)
# NOTE: the XmlModuleStore does not implement get_items()
# which would be a preferable means to enumerate the entire collection
# of course modules. It will be left as a TBD to implement that
# method on XmlModuleStore.
course_items = []
with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred):
for course_key in xml_module_store.modules.keys():
if target_course_id is not None:
dest_course_id = target_course_id
else:
dest_course_id = course_key
# Creates a new course if it doesn't already exist
if create_new_course_if_not_present and not store.has_course(dest_course_id, ignore_case=True):
try:
store.create_course(dest_course_id.org, dest_course_id.course, dest_course_id.run, user_id)
except InvalidLocationError:
# course w/ same org and course exists
log.debug(
"Skipping import of course with id, {0},"
"since it collides with an existing one".format(dest_course_id)
)
continue
with store.bulk_write_operations(dest_course_id):
course_data_path = None
if verbose:
log.debug("Scanning {0} for course module...".format(course_key))
# Quick scan to get course module as we need some info from there.
# Also we need to make sure that the course module is committed
# first into the store
for module in xml_module_store.modules[course_key].itervalues():
if module.scope_ids.block_type == 'course':
course_data_path = path(data_dir) / module.data_dir
log.debug(u'======> IMPORTING course {course_key}'.format(
course_key=course_key,
))
if not do_import_static:
# for old-style xblock where this was actually linked to kvs
module.static_asset_path = module.data_dir
module.save()
log.debug('course static_asset_path={path}'.format(
path=module.static_asset_path
))
log.debug('course data_dir={0}'.format(module.data_dir))
course = _import_module_and_update_references(
module, store, user_id,
course_key,
dest_course_id,
do_import_static=do_import_static
)
for entry in course.pdf_textbooks:
for chapter in entry.get('chapters', []):
if StaticContent.is_c4x_path(chapter.get('url', '')):
asset_key = StaticContent.get_location_from_path(chapter['url'])
chapter['url'] = StaticContent.get_static_path_from_location(asset_key)
# Original wiki_slugs had value location.course. To make them unique this was changed to 'org.course.name'.
# If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
# values then remap it so that the wiki does not point to the old wiki.
if course_key != course.id:
original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
course_key.org,
course_key.course,
course_key.run
)
if course.wiki_slug == original_unique_wiki_slug or course.wiki_slug == course_key.course:
course.wiki_slug = u'{0}.{1}.{2}'.format(
course.id.org,
course.id.course,
course.id.run,
)
# cdodge: more hacks (what else). Seems like we have a
# problem when importing a course (like 6.002) which
# does not have any tabs defined in the policy file.
# The import goes fine and then displays fine in LMS,
# but if someone tries to add a new tab in the CMS, then
# the LMS barfs because it expects that -- if there are
# *any* tabs -- then there at least needs to be
# some predefined ones
if course.tabs is None or len(course.tabs) == 0:
CourseTabList.initialize_default(course)
store.update_item(course, user_id)
course_items.append(course)
break
# TODO: shouldn't this raise an exception if course wasn't found?
# then import all the static content
if static_content_store is not None and do_import_static:
# first pass to find everything in /static/
import_static_content(
course_data_path, static_content_store,
dest_course_id, subpath='static', verbose=verbose
)
elif verbose and not do_import_static:
log.debug(
"Skipping import of static content, "
"since do_import_static={0}".format(do_import_static)
)
# no matter what do_import_static is, import "static_import" directory
# This is needed because the "about" pages (eg "overview") are
# loaded via load_extra_content, and do not inherit the lms
# metadata from the course module, and thus do not get
# "static_content_store" properly defined. Static content
# referenced in those extra pages thus need to come through the
# c4x:// contentstore, unfortunately. Tell users to copy that
# content into the "static_import" subdir.
simport = 'static_import'
if os.path.exists(course_data_path / simport):
import_static_content(
course_data_path, static_content_store,
dest_course_id, subpath=simport, verbose=verbose
)
# now loop through all the modules
for module in xml_module_store.modules[course_key].itervalues():
if module.scope_ids.block_type == 'course':
# we've already saved the course module up at the top
# of the loop so just skip over it in the inner loop
continue
if verbose:
log.debug('importing module location {loc}'.format(
loc=module.location
))
_import_module_and_update_references(
module, store,
user_id,
course_key,
dest_course_id,
do_import_static=do_import_static,
runtime=course.runtime
)
# finally, publish the course
store.publish(course.location, user_id)
# now import any DRAFT items
_import_course_draft(
xml_module_store,
store,
user_id,
course_data_path,
course_key,
dest_course_id,
course.runtime
)
return xml_module_store, course_items
def process_request(self, request):
"""Process the given request"""
asset_path = request.path
if self.is_asset_request(request):
# Make sure we can convert this request into a location.
if AssetLocator.CANONICAL_NAMESPACE in asset_path:
asset_path = asset_path.replace('block/', 'block@', 1)
# If this is a versioned request, pull out the digest and chop off the prefix.
requested_digest = None
if StaticContent.is_versioned_asset_path(asset_path):
requested_digest, asset_path = StaticContent.parse_versioned_asset_path(asset_path)
# Make sure we have a valid location value for this asset.
try:
loc = StaticContent.get_location_from_path(asset_path)
except (InvalidLocationError, InvalidKeyError):
return HttpResponseBadRequest()
# Attempt to load the asset to make sure it exists, and grab the asset digest
# if we're able to load it.
actual_digest = None
try:
content = self.load_asset_from_location(loc)
actual_digest = content.content_digest
except (ItemNotFoundError, NotFoundError):
return HttpResponseNotFound()
# If this was a versioned asset, and the digest doesn't match, redirect
# them to the actual version.
if requested_digest is not None and (actual_digest != requested_digest):
actual_asset_path = StaticContent.add_version_to_asset_path(asset_path, actual_digest)
return HttpResponsePermanentRedirect(actual_asset_path)
# Set the basics for this request. Make sure that the course key for this
# asset has a run, which old-style courses do not. Otherwise, this will
# explode when the key is serialized to be sent to NR.
safe_course_key = loc.course_key
if safe_course_key.run is None:
safe_course_key = safe_course_key.replace(run='only')
newrelic.agent.add_custom_parameter('course_id', safe_course_key)
newrelic.agent.add_custom_parameter('org', loc.org)
newrelic.agent.add_custom_parameter('contentserver.path', loc.path)
# Figure out if this is a CDN using us as the origin.
is_from_cdn = StaticContentServer.is_cdn_request(request)
newrelic.agent.add_custom_parameter('contentserver.from_cdn', is_from_cdn)
# Check if this content is locked or not.
locked = self.is_content_locked(content)
newrelic.agent.add_custom_parameter('contentserver.locked', locked)
# Check that user has access to the content.
if not self.is_user_authorized(request, content, loc):
return HttpResponseForbidden('Unauthorized')
# Figure out if the client sent us a conditional request, and let them know
# if this asset has changed since then.
last_modified_at_str = content.last_modified_at.strftime(HTTP_DATE_FORMAT)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
# *** File streaming within a byte range ***
# If a Range is provided, parse Range attribute of the request
# Add Content-Range in the response if Range is structurally correct
# Request -> Range attribute structure: "Range: bytes=first-[last]"
# Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength"
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
response = None
if request.META.get('HTTP_RANGE'):
# If we have a StaticContent, get a StaticContentStream. Can't manipulate the bytes otherwise.
if type(content) == StaticContent:
content = AssetManager.find(loc, as_stream=True)
header_value = request.META['HTTP_RANGE']
try:
unit, ranges = parse_range_header(header_value, content.length)
except ValueError as exception:
# If the header field is syntactically invalid it should be ignored.
log.exception(
u"%s in Range header: %s for content: %s", exception.message, header_value, unicode(loc)
)
else:
if unit != 'bytes':
# Only accept ranges in bytes
log.warning(u"Unknown unit in Range header: %s for content: %s", header_value, unicode(loc))
elif len(ranges) > 1:
# According to Http/1.1 spec content for multiple ranges should be sent as a multipart message.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.16
# But we send back the full content.
log.warning(
u"More than 1 ranges in Range header: %s for content: %s", header_value, unicode(loc)
)
else:
first, last = ranges[0]
if 0 <= first <= last < content.length:
# If the byte range is satisfiable
response = HttpResponse(content.stream_data_in_range(first, last))
response['Content-Range'] = 'bytes {first}-{last}/{length}'.format(
first=first, last=last, length=content.length
)
response['Content-Length'] = str(last - first + 1)
response.status_code = 206 # Partial Content
newrelic.agent.add_custom_parameter('contentserver.ranged', True)
else:
log.warning(
u"Cannot satisfy ranges in Range header: %s for content: %s", header_value, unicode(loc)
)
return HttpResponse(status=416) # Requested Range Not Satisfiable
# If Range header is absent or syntactically invalid return a full content response.
if response is None:
response = HttpResponse(content.stream_data())
response['Content-Length'] = content.length
newrelic.agent.add_custom_parameter('contentserver.content_len', content.length)
newrelic.agent.add_custom_parameter('contentserver.content_type', content.content_type)
# "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed
response['Accept-Ranges'] = 'bytes'
response['Content-Type'] = content.content_type
# Set any caching headers, and do any response cleanup needed. Based on how much
# middleware we have in place, there's no easy way to use the built-in Django
# utilities and properly sanitize and modify a response to ensure that it is as
# cacheable as possible, which is why we do it ourselves.
self.set_caching_headers(content, response)
return response
def test_happy_path(self, modulestore_type, create_after_overview):
"""
What happens when everything works like we expect it to.
If `create_after_overview` is True, we will temporarily disable
thumbnail creation so that the initial CourseOverview is created without
an image_set, and the CourseOverviewImageSet is created afterwards. If
`create_after_overview` is False, we'll create the CourseOverviewImageSet
at the same time as the CourseOverview.
"""
# Create a real (oversized) image...
image = Image.new("RGB", (800, 400), "blue")
image_buff = StringIO()
image.save(image_buff, format="JPEG")
image_buff.seek(0)
image_name = "big_course_image.jpeg"
with self.store.default_store(modulestore_type):
course = CourseFactory.create(default_store=modulestore_type,
course_image=image_name)
# Save a real image here...
course_image_asset_key = StaticContent.compute_location(
course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key,
image_name, 'image/jpeg',
image_buff)
contentstore().save(course_image_content)
# If create_after_overview is True, disable thumbnail generation so
# that the CourseOverview object is created and saved without an
# image_set at first (it will be lazily created later).
if create_after_overview:
self.set_config(enabled=False)
# Now generate the CourseOverview...
course_overview = CourseOverview.get_from_id(course.id)
# If create_after_overview is True, no image_set exists yet. Verify
# that, then switch config back over to True and it should lazily
# create the image_set on the next get_from_id() call.
if create_after_overview:
self.assertFalse(hasattr(course_overview, 'image_set'))
self.set_config(enabled=True)
course_overview = CourseOverview.get_from_id(course.id)
self.assertTrue(hasattr(course_overview, 'image_set'))
image_urls = course_overview.image_urls
config = CourseOverviewImageConfig.current()
# Make sure the thumbnail names come out as expected...
self.assertTrue(
image_urls['raw'].endswith('big_course_image.jpeg'))
self.assertTrue(image_urls['small'].endswith(
'big_course_image-jpeg-{}x{}.jpg'.format(*config.small)))
self.assertTrue(image_urls['large'].endswith(
'big_course_image-jpeg-{}x{}.jpg'.format(*config.large)))
# Now make sure our thumbnails are of the sizes we expect...
for image_url, expected_size in [
(image_urls['small'], config.small),
(image_urls['large'], config.large)
]:
image_key = StaticContent.get_location_from_path(image_url)
image_content = AssetManager.find(image_key)
image = Image.open(StringIO(image_content.data))
self.assertEqual(image.size, expected_size)
