def setUp(self):
super(TestSaveSubsToStore, self).setUp()
self.course = CourseFactory.create(
org=self.org, number=self.number, display_name=self.display_name)
self.subs = {
'start': [100, 200, 240, 390, 1000],
'end': [200, 240, 380, 1000, 1500],
'text': [
'subs #1',
'subs #2',
'subs #3',
'subs #4',
'subs #5'
]
}
self.subs_id = str(uuid4())
filename = 'subs_{0}.srt.sjson'.format(self.subs_id)
self.content_location = StaticContent.compute_location(self.course.id, filename)
self.addCleanup(self.clear_subs_content)
# incorrect subs
self.unjsonable_subs = set([1]) # set can't be serialized
self.unjsonable_subs_id = str(uuid4())
filename_unjsonable = 'subs_{0}.srt.sjson'.format(self.unjsonable_subs_id)
self.content_location_unjsonable = StaticContent.compute_location(self.course.id, filename_unjsonable)
self.clear_subs_content()
def setUpClass(cls):
super(TestSaveSubsToStore, cls).setUpClass()
cls.course = CourseFactory.create(
org=cls.org, number=cls.number, display_name=cls.display_name)
cls.subs = {
'start': [100, 200, 240, 390, 1000],
'end': [200, 240, 380, 1000, 1500],
'text': [
'subs #1',
'subs #2',
'subs #3',
'subs #4',
'subs #5'
]
}
cls.subs_id = str(uuid4())
filename = 'subs_{0}.srt.sjson'.format(cls.subs_id)
cls.content_location = StaticContent.compute_location(cls.course.id, filename)
# incorrect subs
cls.unjsonable_subs = {1} # set can't be serialized
cls.unjsonable_subs_id = str(uuid4())
filename_unjsonable = 'subs_{0}.srt.sjson'.format(cls.unjsonable_subs_id)
cls.content_location_unjsonable = StaticContent.compute_location(cls.course.id, filename_unjsonable)
def verify_content_links(module, base_dir, static_content_store, link, remap_dict=None):
if link.startswith('/static/'):
# yes, then parse out the name
path = link[len('/static/'):]
static_pathname = base_dir / path
if os.path.exists(static_pathname):
try:
content_loc = StaticContent.compute_location(module.location.org, module.location.course, path)
filename = os.path.basename(path)
mime_type = mimetypes.guess_type(filename)[0]
with open(static_pathname, 'rb') as f:
data = f.read()
content = StaticContent(content_loc, filename, mime_type, data, import_path=path)
# first let's save a thumbnail so we can get back a thumbnail location
(thumbnail_content, thumbnail_location) = static_content_store.generate_thumbnail(content)
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
#then commit the content
static_content_store.save(content)
new_link = StaticContent.get_url_path_from_location(content_loc)
if remap_dict is not None:
remap_dict[link] = new_link
return new_link
except Exception, e:
logging.exception('Skipping failed content load from {0}. Exception: {1}'.format(path, e))
def remap_namespace(module, target_location_namespace):
if target_location_namespace is None:
return module
# This looks a bit wonky as we need to also change the 'name' of the
# imported course to be what the caller passed in
if module.location.category != 'course':
module.location = module.location._replace(
tag=target_location_namespace.tag,
org=target_location_namespace.org,
course=target_location_namespace.course
)
else:
original_location = module.location
#
# module is a course module
#
module.location = module.location._replace(
tag=target_location_namespace.tag,
org=target_location_namespace.org,
course=target_location_namespace.course,
name=target_location_namespace.name
)
# There is more re-namespacing work we have to do when
# importing course modules
# remap pdf_textbook urls
for entry in module.pdf_textbooks:
for chapter in entry.get('chapters', []):
if StaticContent.is_c4x_path(chapter.get('url', '')):
chapter['url'] = StaticContent.renamespace_c4x_path(
chapter['url'], target_location_namespace
)
# if there is a wiki_slug which is the same as the original location
# (aka default value), then remap that so the wiki doesn't point to
# the old Wiki.
if module.wiki_slug == original_location.course:
module.wiki_slug = target_location_namespace.course
module.save()
# then remap children pointers since they too will be re-namespaced
if hasattr(module, 'children'):
children_locs = module.children
if children_locs is not None and children_locs != []:
new_locs = []
for child in children_locs:
child_loc = Location(child)
new_child_loc = child_loc._replace(
tag=target_location_namespace.tag,
org=target_location_namespace.org,
course=target_location_namespace.course
)
new_locs.append(new_child_loc.url())
module.children = new_locs
return module
def test_happy_path(self, modulestore_type, create_after_overview):
"""
What happens when everything works like we expect it to.
If `create_after_overview` is True, we will temporarily disable
thumbnail creation so that the initial CourseOverview is created without
an image_set, and the CourseOverviewImageSet is created afterwards. If
`create_after_overview` is False, we'll create the CourseOverviewImageSet
at the same time as the CourseOverview.
"""
# Create a real (oversized) image...
image = Image.new("RGB", (800, 400), "blue")
image_buff = StringIO()
image.save(image_buff, format="JPEG")
image_buff.seek(0)
image_name = "big_course_image.jpeg"
with self.store.default_store(modulestore_type):
course = CourseFactory.create(
default_store=modulestore_type, course_image=image_name
)
# Save a real image here...
course_image_asset_key = StaticContent.compute_location(course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key, image_name, 'image/jpeg', image_buff)
contentstore().save(course_image_content)
# If create_after_overview is True, disable thumbnail generation so
# that the CourseOverview object is created and saved without an
# image_set at first (it will be lazily created later).
if create_after_overview:
self.set_config(enabled=False)
# Now generate the CourseOverview...
course_overview = CourseOverview.get_from_id(course.id)
# If create_after_overview is True, no image_set exists yet. Verify
# that, then switch config back over to True and it should lazily
# create the image_set on the next get_from_id() call.
if create_after_overview:
self.assertFalse(hasattr(course_overview, 'image_set'))
self.set_config(enabled=True)
course_overview = CourseOverview.get_from_id(course.id)
self.assertTrue(hasattr(course_overview, 'image_set'))
image_urls = course_overview.image_urls
config = CourseOverviewImageConfig.current()
# Make sure the thumbnail names come out as expected...
self.assertTrue(image_urls['raw'].endswith('big_course_image.jpeg'))
self.assertTrue(image_urls['small'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.small)))
self.assertTrue(image_urls['large'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.large)))
# Now make sure our thumbnails are of the sizes we expect...
for image_url, expected_size in [(image_urls['small'], config.small), (image_urls['large'], config.large)]:
image_key = StaticContent.get_location_from_path(image_url)
image_content = AssetManager.find(image_key)
image = Image.open(StringIO(image_content.data))
self.assertEqual(image.size, expected_size)
def test_different_resolutions(self, src_dimensions):
"""
Test various resolutions of images to make thumbnails of.
Note that our test sizes are small=(200, 100) and large=(400, 200).
1. Images should won't be blown up if it's too small, so a (100, 50)
resolution image will remain (100, 50).
2. However, images *will* be converted using our format and quality
settings (JPEG, 75% -- the PIL default). This is because images with
relatively small dimensions not compressed properly.
3. Image thumbnail naming will maintain the naming convention of the
target resolution, even if the image was not actually scaled to that
size (i.e. it was already smaller). This is mostly because it's
simpler to be consistent, but it also lets us more easily tell which
configuration a thumbnail was created under.
"""
# Create a source image...
image = Image.new("RGB", src_dimensions, "blue")
image_buff = StringIO()
image.save(image_buff, format="PNG")
image_buff.seek(0)
image_name = "src_course_image.png"
course = CourseFactory.create(course_image=image_name)
# Save the image to the contentstore...
course_image_asset_key = StaticContent.compute_location(course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key, image_name, 'image/png', image_buff)
contentstore().save(course_image_content)
# Now generate the CourseOverview...
config = CourseOverviewImageConfig.current()
course_overview = CourseOverview.get_from_id(course.id)
image_urls = course_overview.image_urls
for image_url, target in [(image_urls['small'], config.small), (image_urls['large'], config.large)]:
image_key = StaticContent.get_location_from_path(image_url)
image_content = AssetManager.find(image_key)
image = Image.open(StringIO(image_content.data))
# Naming convention for thumbnail
self.assertTrue(image_url.endswith('src_course_image-png-{}x{}.jpg'.format(*target)))
# Actual thumbnail data
src_x, src_y = src_dimensions
target_x, target_y = target
image_x, image_y = image.size
# I'm basically going to assume the image library knows how to do
# the right thing in terms of handling aspect ratio. We're just
# going to make sure that small images aren't blown up, and that
# we never exceed our target sizes
self.assertLessEqual(image_x, target_x)
self.assertLessEqual(image_y, target_y)
if src_x < target_x and src_y < target_y:
self.assertEqual(src_x, image_x)
self.assertEqual(src_y, image_y)
def course_image_url(course):
"""Returns the image url for the course."""
try:
loc = StaticContent.compute_location(course.location.course_key, course.course_image)
except InvalidKeyError:
return ''
path = StaticContent.serialize_asset_key_with_slash(loc)
return path
def course_image_url(course):
"""Try to look up the image url for the course. If it's not found,
log an error and return the dead link"""
if course.static_asset_path or modulestore().get_modulestore_type(course.location.course_id) == XML_MODULESTORE_TYPE:
return '/static/' + (course.static_asset_path or getattr(course, 'data_dir', '')) + "/images/course_image.jpg"
else:
loc = StaticContent.compute_location(course.location.org, course.location.course, course.course_image)
_path = StaticContent.get_url_path_from_location(loc)
return _path
def course_image_url(course):
"""
Return url of course image.
Args:
course(CourseDescriptor) : The course id to retrieve course image url.
Returns:
Absolute url of course image.
"""
loc = StaticContent.compute_location(course.id, course.course_image)
url = StaticContent.serialize_asset_key_with_slash(loc)
return url
def create_course_image_thumbnail(course, dimensions):
"""Create a course image thumbnail and return the URL.
- dimensions is a tuple of (width, height)
"""
course_image_asset_key = StaticContent.compute_location(course.id, course.course_image)
course_image = AssetManager.find(course_image_asset_key) # a StaticContent obj
_content, thumb_loc = contentstore().generate_thumbnail(course_image, dimensions=dimensions)
return StaticContent.serialize_asset_key_with_slash(thumb_loc)
def asset_index(request, org, course, name, start=None, maxresults=None):
"""
Display an editable asset library
org, course, name: Attributes of the Location for the item to edit
:param start: which index of the result list to start w/, used for paging results
:param maxresults: maximum results
"""
location = get_location_and_verify_access(request, org, course, name)
upload_asset_callback_url = reverse('upload_asset', kwargs={
'org': org,
'course': course,
'coursename': name
})
course_module = modulestore().get_item(location)
course_reference = StaticContent.compute_location(org, course, name)
if maxresults is not None:
maxresults = int(maxresults)
start = int(start) if start else 0
assets = contentstore().get_all_content_for_course(
course_reference, start=start, maxresults=maxresults,
sort=[('uploadDate', DESCENDING)]
)
else:
assets = contentstore().get_all_content_for_course(
course_reference, sort=[('uploadDate', DESCENDING)]
)
asset_json = []
for asset in assets:
asset_id = asset['_id']
asset_location = StaticContent.compute_location(asset_id['org'], asset_id['course'], asset_id['name'])
# note, due to the schema change we may not have a 'thumbnail_location' in the result set
_thumbnail_location = asset.get('thumbnail_location', None)
thumbnail_location = Location(_thumbnail_location) if _thumbnail_location is not None else None
asset_locked = asset.get('locked', False)
asset_json.append(_get_asset_json(asset['displayname'], asset['uploadDate'], asset_location, thumbnail_location, asset_locked))
return render_to_response('asset_index.html', {
'context_course': course_module,
'asset_list': json.dumps(asset_json),
'upload_asset_callback_url': upload_asset_callback_url,
'update_asset_callback_url': reverse('update_asset', kwargs={
'org': org,
'course': course,
'name': name
})
})
def _clear_assets(location):
store = contentstore()
content_location = StaticContent.compute_location(
location.org, location.course, location.name
)
assets, __ = store.get_all_content_for_course(content_location)
for asset in assets:
asset_location = Location(asset["_id"])
id = StaticContent.get_id_from_location(asset_location)
store.delete(id)
def asset_index(request, org, course, name):
"""
Display an editable asset library
org, course, name: Attributes of the Location for the item to edit
"""
location = get_location_and_verify_access(request, org, course, name)
upload_asset_callback_url = reverse('upload_asset', kwargs={
'org': org,
'course': course,
'coursename': name
})
course_module = modulestore().get_item(location)
course_reference = StaticContent.compute_location(org, course, name)
assets = contentstore().get_all_content_for_course(course_reference)
# sort in reverse upload date order
assets = sorted(assets, key=lambda asset: asset['uploadDate'], reverse=True)
if request.META.get('HTTP_ACCEPT', "").startswith("application/json"):
return JsonResponse(assets_to_json_dict(assets))
asset_display = []
for asset in assets:
asset_id = asset['_id']
display_info = {}
display_info['displayname'] = asset['displayname']
display_info['uploadDate'] = get_default_time_display(asset['uploadDate'])
asset_location = StaticContent.compute_location(asset_id['org'], asset_id['course'], asset_id['name'])
display_info['url'] = StaticContent.get_url_path_from_location(asset_location)
display_info['portable_url'] = StaticContent.get_static_path_from_location(asset_location)
# note, due to the schema change we may not have a 'thumbnail_location' in the result set
_thumbnail_location = asset.get('thumbnail_location', None)
thumbnail_location = Location(_thumbnail_location) if _thumbnail_location is not None else None
display_info['thumb_url'] = StaticContent.get_url_path_from_location(thumbnail_location) if thumbnail_location is not None else None
asset_display.append(display_info)
return render_to_response('asset_index.html', {
'context_course': course_module,
'assets': asset_display,
'upload_asset_callback_url': upload_asset_callback_url,
'remove_asset_callback_url': reverse('remove_asset', kwargs={
'org': org,
'course': course,
'name': name
})
})
def import_static_content(
modules,
course_loc,
course_data_path,
static_content_store,
target_location_namespace,
subpath="static",
verbose=False,
):
remap_dict = {}
# now import all static assets
static_dir = course_data_path / subpath
verbose = True
for dirname, dirnames, filenames in os.walk(static_dir):
for filename in filenames:
try:
content_path = os.path.join(dirname, filename)
if verbose:
log.debug("importing static content {0}...".format(content_path))
fullname_with_subpath = content_path.replace(static_dir, "") # strip away leading path from the name
if fullname_with_subpath.startswith("/"):
fullname_with_subpath = fullname_with_subpath[1:]
content_loc = StaticContent.compute_location(
target_location_namespace.org, target_location_namespace.course, fullname_with_subpath
)
mime_type = mimetypes.guess_type(filename)[0]
with open(content_path, "rb") as f:
data = f.read()
content = StaticContent(content_loc, filename, mime_type, data, import_path=fullname_with_subpath)
# first let's save a thumbnail so we can get back a thumbnail location
(thumbnail_content, thumbnail_location) = static_content_store.generate_thumbnail(content)
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
# then commit the content
static_content_store.save(content)
# store the remapping information which will be needed to subsitute in the module data
remap_dict[fullname_with_subpath] = content_loc.name
except:
raise
return remap_dict
def get_versioned_asset_url(asset_path):
"""
Creates a versioned asset URL.
"""
try:
locator = StaticContent.get_location_from_path(asset_path)
content = AssetManager.find(locator, as_stream=True)
return StaticContent.add_version_to_asset_path(asset_path, content.content_digest)
except (InvalidKeyError, ItemNotFoundError):
pass
return asset_path
def import_static_content(modules, course_loc, course_data_path, static_content_store, target_location_namespace,
subpath='static', verbose=False):
remap_dict = {}
# now import all static assets
static_dir = course_data_path / subpath
verbose = True
for dirname, _, filenames in os.walk(static_dir):
for filename in filenames:
content_path = os.path.join(dirname, filename)
if verbose:
log.debug('importing static content %s...', content_path)
try:
with open(content_path, 'rb') as f:
data = f.read()
except IOError:
if filename.startswith('._'):
# OS X "companion files". See http://www.diigo.com/annotated/0c936fda5da4aa1159c189cea227e174
continue
# Not a 'hidden file', then re-raise exception
raise
fullname_with_subpath = content_path.replace(static_dir, '') # strip away leading path from the name
if fullname_with_subpath.startswith('/'):
fullname_with_subpath = fullname_with_subpath[1:]
content_loc = StaticContent.compute_location(target_location_namespace.org, target_location_namespace.course, fullname_with_subpath)
mime_type = mimetypes.guess_type(filename)[0]
content = StaticContent(content_loc, filename, mime_type, data, import_path=fullname_with_subpath)
# first let's save a thumbnail so we can get back a thumbnail location
(thumbnail_content, thumbnail_location) = static_content_store.generate_thumbnail(content)
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
#then commit the content
try:
static_content_store.save(content)
except Exception as err:
log.exception('Error importing {0}, error={1}'.format(fullname_with_subpath, err))
#store the remapping information which will be needed to subsitute in the module data
remap_dict[fullname_with_subpath] = content_loc.name
return remap_dict
def course_image_url(course, image_name=None):
"""Return url for image_name or default course image in given course assets.
It allows us to override default course image in templates when this function is
used whith image_name parameter, if the image is available. (see course_about.html)
"""
image = image_name or course.course_image
try:
loc = StaticContent.compute_location(course.location.course_key, image)
_ = contentstore().find(loc)
except NotFoundError:
loc = StaticContent.compute_location(course.location.course_key, course.course_image)
return loc.to_deprecated_string()
def _get_asset_json(display_name, date, location, thumbnail_location, locked):
"""
Helper method for formatting the asset information to send to client.
"""
asset_url = StaticContent.get_url_path_from_location(location)
return {
'display_name': display_name,
'date_added': get_default_time_display(date),
'url': asset_url,
'portable_url': StaticContent.get_static_path_from_location(location),
'thumbnail': StaticContent.get_url_path_from_location(thumbnail_location) if thumbnail_location is not None else None,
'locked': locked,
# Needed for Backbone delete/update.
'id': asset_url
}
def delete_course(modulestore, contentstore, source_location, commit=False):
# first check to see if the modulestore is Mongo backed
if not isinstance(modulestore, MongoModuleStore):
raise Exception("Expected a MongoModuleStore in the runtime. Aborting....")
# check to see if the source course is actually there
if not modulestore.has_item(source_location):
raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
# first delete all of the thumbnails
thumbs = contentstore.get_all_content_thumbnails_for_course(source_location)
for thumb in thumbs:
thumb_loc = Location(thumb["_id"])
id = StaticContent.get_id_from_location(thumb_loc)
print "Deleting {0}...".format(id)
if commit:
contentstore.delete(id)
# then delete all of the assets
assets = contentstore.get_all_content_for_course(source_location)
for asset in assets:
asset_loc = Location(asset["_id"])
id = StaticContent.get_id_from_location(asset_loc)
print "Deleting {0}...".format(id)
if commit:
contentstore.delete(id)
# then delete all course modules
modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None])
course_module = None
for module in modules:
if module.category != 'course': # save deleting the course module for last
print "Deleting {0}...".format(module.location)
if commit:
modulestore.delete_item(module.location)
else:
course_module = module
if course_module:
modulestore.delete_item(course_module.location)
# finally delete the top-level course module itself
print "Deleting {0}...".format(source_location)
if commit:
modulestore.delete_item(source_location)
return True
def test_success_generating_subs(self):
youtube_subs = {
0.5: 'JMD_ifUUfsU',
1.0: 'hI10vDNYz4M',
2.0: 'AKqURZnYqpk'
}
srt_filedata = textwrap.dedent("""
1
00:00:10,500 --> 00:00:13,000
Elephant's Dream
2
00:00:15,000 --> 00:00:18,000
At the left we can see...
""")
self.clear_subs_content(youtube_subs)
# Check transcripts_utils.TranscriptsGenerationException not thrown.
# Also checks that uppercase file extensions are supported.
transcripts_utils.generate_subs_from_source(youtube_subs, 'SRT', srt_filedata, self.course)
# Check assets status after importing subtitles.
for subs_id in youtube_subs.values():
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(
self.course.id, filename
)
self.assertTrue(contentstore().find(content_location))
self.clear_subs_content(youtube_subs)
def course_info_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None):
"""
GET
html: return html for editing the course info handouts and updates.
"""
__, course_module = _get_locator_and_course(
package_id, branch, version_guid, block, request.user
)
if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):
handouts_old_location = course_module.location.replace(category='course_info', name='handouts')
handouts_locator = loc_mapper().translate_location(
course_module.location.course_id, handouts_old_location, False, True
)
update_location = course_module.location.replace(category='course_info', name='updates')
update_locator = loc_mapper().translate_location(
course_module.location.course_id, update_location, False, True
)
return render_to_response(
'course_info.html',
{
'context_course': course_module,
'updates_url': update_locator.url_reverse('course_info_update/'),
'handouts_locator': handouts_locator,
'base_asset_url': StaticContent.get_base_url_path_for_course_assets(course_module.location) + '/'
}
)
else:
return HttpResponseBadRequest("Only supports html requests")
def test_success_video_module_source_subs_uploading(self):
self.item.data = textwrap.dedent("""
<video youtube="">
<source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
<source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
<source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
</video>
""")
modulestore().update_item(self.item, self.user.id)
link = reverse('upload_transcripts')
filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
resp = self.client.post(link, {
'locator': self.video_usage_key,
'transcript-file': self.good_srt_file,
'video_list': json.dumps([{
'type': 'html5',
'video': filename,
'mode': 'mp4',
}])
})
self.assertEqual(resp.status_code, 200)
self.assertEqual(json.loads(resp.content).get('status'), 'Success')
item = modulestore().get_item(self.video_usage_key)
self.assertEqual(item.sub, filename)
content_location = StaticContent.compute_location(
self.course.id, 'subs_{0}.srt.sjson'.format(filename))
self.assertTrue(contentstore().find(content_location))
def _upload_asset(request, course_key):
'''
This method allows for POST uploading of files into the course asset
library, which will be supported by GridFS in MongoDB.
'''
# Does the course actually exist?!? Get anything from it to prove its
# existence
try:
modulestore().get_course(course_key)
except ItemNotFoundError:
# no return it as a Bad Request response
logging.error("Could not find course: %s", course_key)
return HttpResponseBadRequest()
# compute a 'filename' which is similar to the location formatting, we're
# using the 'filename' nomenclature since we're using a FileSystem paradigm
# here. We're just imposing the Location string formatting expectations to
# keep things a bit more consistent
upload_file = request.FILES['file']
filename = upload_file.name
mime_type = upload_file.content_type
content_loc = StaticContent.compute_location(course_key, filename)
chunked = upload_file.multiple_chunks()
sc_partial = partial(StaticContent, content_loc, filename, mime_type)
if chunked:
content = sc_partial(upload_file.chunks())
tempfile_path = upload_file.temporary_file_path()
else:
content = sc_partial(upload_file.read())
tempfile_path = None
# first let's see if a thumbnail can be created
(thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(
content,
tempfile_path=tempfile_path
)
# delete cached thumbnail even if one couldn't be created this time (else
# the old thumbnail will continue to show)
del_cached_content(thumbnail_location)
# now store thumbnail location only if we could create it
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
# then commit the content
contentstore().save(content)
del_cached_content(content.location)
# readback the saved content - we need the database timestamp
readback = contentstore().find(content.location)
locked = getattr(content, 'locked', False)
response_payload = {
'asset': _get_asset_json(content.name, readback.last_modified_at, content.location, content.thumbnail_location, locked),
'msg': _('Upload completed')
}
return JsonResponse(response_payload)
def get_asset_location(asset_id):
""" Helper method to get the location (and verify it is valid). """
try:
return StaticContent.get_location_from_path(asset_id)
except InvalidLocationError as err:
# return a 'Bad Request' to browser as we have a malformed Location
return JsonResponse({"error": err.message}, status=400)
def test_compute_location(self):
# We had a bug that __ got converted into a single _. Make sure that substitution of INVALID_CHARS (like space)
# still happen.
asset_location = StaticContent.compute_location(
SlashSeparatedCourseKey('mitX', '400', 'ignore'), 'subs__1eo_jXvZnE .srt.sjson'
)
self.assertEqual(AssetLocation(u'mitX', u'400', u'ignore', u'asset', u'subs__1eo_jXvZnE_.srt.sjson', None), asset_location)
def get_html(self):
if isinstance(modulestore(), MongoModuleStore):
caption_asset_path = StaticContent.get_base_url_path_for_course_assets(self.location) + '/subs_'
else:
# VS[compat]
# cdodge: filesystem static content support.
caption_asset_path = "/static/subs/"
get_ext = lambda filename: filename.rpartition('.')[-1]
sources = {get_ext(src): src for src in self.html5_sources}
sources['main'] = self.source
return self.system.render_template('videoalpha.html', {
'youtube_streams': _create_youtube_string(self),
'id': self.location.html_id(),
'sub': self.sub,
'sources': sources,
'track': self.track,
'display_name': self.display_name_with_default,
# This won't work when we move to data that
# isn't on the filesystem
'data_dir': getattr(self, 'data_dir', None),
'caption_asset_path': caption_asset_path,
'show_captions': json.dumps(self.show_captions),
'start': self.start_time,
'end': self.end_time,
'autoplay': settings.MITX_FEATURES.get('AUTOPLAY_VIDEOS', True)
})
def test_delete_image_type_asset(self):
""" Tests deletion of image type asset """
image_asset = self.get_sample_asset(self.asset_name, asset_type="image")
thumbnail_image_asset = self.get_sample_asset('delete_test_thumbnail', asset_type="image")
# upload image
response = self.client.post(self.url, {"name": "delete_image_test", "file": image_asset})
self.assertEquals(response.status_code, 200)
uploaded_image_url = json.loads(response.content)['asset']['url']
# upload image thumbnail
response = self.client.post(self.url, {"name": "delete_image_thumb_test", "file": thumbnail_image_asset})
self.assertEquals(response.status_code, 200)
thumbnail_url = json.loads(response.content)['asset']['url']
thumbnail_location = StaticContent.get_location_from_path(thumbnail_url)
image_asset_location = AssetLocation.from_deprecated_string(uploaded_image_url)
content = contentstore().find(image_asset_location)
content.thumbnail_location = thumbnail_location
contentstore().save(content)
with mock.patch('opaque_keys.edx.locator.CourseLocator.make_asset_key') as mock_asset_key:
mock_asset_key.return_value = thumbnail_location
test_url = reverse_course_url(
'assets_handler', self.course.id, kwargs={'asset_key_string': unicode(uploaded_image_url)})
resp = self.client.delete(test_url, HTTP_ACCEPT="application/json")
self.assertEquals(resp.status_code, 204)
def course_info_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None):
"""
GET
html: return html for editing the course info handouts and updates.
"""
__, course_module = _get_locator_and_course(package_id, branch, version_guid, block, request.user)
if "text/html" in request.META.get("HTTP_ACCEPT", "text/html"):
handouts_old_location = course_module.location.replace(category="course_info", name="handouts")
handouts_locator = loc_mapper().translate_location(
course_module.location.course_id, handouts_old_location, False, True
)
update_location = course_module.location.replace(category="course_info", name="updates")
update_locator = loc_mapper().translate_location(course_module.location.course_id, update_location, False, True)
return render_to_response(
"course_info.html",
{
"context_course": course_module,
"updates_url": update_locator.url_reverse("course_info_update/"),
"handouts_locator": handouts_locator,
"base_asset_url": StaticContent.get_base_url_path_for_course_assets(course_module.location) + "/",
},
)
else:
return HttpResponseBadRequest("Only supports html requests")
def test_get_location_from_path(self):
asset_location = StaticContent.get_location_from_path(u'/c4x/a/b/asset/images_course_image.jpg')
self.assertEqual(
AssetLocator(CourseLocator(u'a', u'b', None, deprecated=True),
u'asset', u'images_course_image.jpg', deprecated=True),
asset_location
)
def test_images_upload(self):
# http://www.django-rest-framework.org/api-guide/parsers/#fileuploadparser
course_run = CourseFactory()
expected_filename = 'course_image.png'
content_key = StaticContent.compute_location(course_run.id, expected_filename)
assert course_run.course_image != expected_filename
try:
contentstore().find(content_key)
self.fail('No image should be associated with a new course run.')
except NotFoundError:
pass
url = reverse('api:v1:course_run-images', kwargs={'pk': str(course_run.id)})
# PNG. Single black pixel
content = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90wS' \
b'\xde\x00\x00\x00\x0cIDATx\x9cc```\x00\x00\x00\x04\x00\x01\xf6\x178U\x00\x00\x00\x00IEND\xaeB`\x82'
# We are intentionally passing the incorrect JPEG extension here
upload = SimpleUploadedFile('card_image.jpg', content, content_type='image/png')
response = self.client.post(url, {'card_image': upload}, format='multipart')
assert response.status_code == 200
course_run = modulestore().get_course(course_run.id)
assert course_run.course_image == expected_filename
expected = {'card_image': RequestFactory().get('').build_absolute_uri(course_image_url(course_run))}
assert response.data == expected
# There should now be an image stored
contentstore().find(content_key)
def test_fail_downloading_subs(self, mock_get):
mock_get.return_value = Mock(status_code=404, text='Error 404')
bad_youtube_sub = 'BAD_YOUTUBE_ID2'
self.clear_sub_content(bad_youtube_sub)
with self.assertRaises(transcripts_utils.GetTranscriptsFromYouTubeException):
transcripts_utils.download_youtube_subs(bad_youtube_sub, self.course, settings)
# Check asset status after import of transcript.
filename = 'subs_{0}.srt.sjson'.format(bad_youtube_sub)
content_location = StaticContent.compute_location(
self.course.id, filename
)
with self.assertRaises(NotFoundError):
contentstore().find(content_location)
self.clear_sub_content(bad_youtube_sub)
def test_downloading_subs_using_transcript_name(self, mock_get):
"""
Download transcript using transcript name in url
"""
good_youtube_sub = 'good_id_2'
self.clear_sub_content(good_youtube_sub)
transcripts_utils.download_youtube_subs(good_youtube_sub, self.course, settings)
mock_get.assert_any_call(
'http://video.google.com/timedtext',
params={'lang': 'en', 'v': 'good_id_2', 'name': 'Custom'}
)
# Check asset status after import of transcript.
filename = 'subs_{0}.srt.sjson'.format(good_youtube_sub)
content_location = StaticContent.compute_location(self.course.id, filename)
self.assertTrue(contentstore().find(content_location))
self.clear_sub_content(good_youtube_sub)
def _get_file_content_and_path(file_metadata, course_key):
"""returns contents of the uploaded file and path for temporary uploaded file"""
content_location = StaticContent.compute_location(
course_key, file_metadata['filename'])
upload_file = file_metadata['upload_file']
file_can_be_chunked = upload_file.multiple_chunks()
static_content_partial = partial(StaticContent, content_location,
file_metadata['filename'],
file_metadata['mime_type'])
if file_can_be_chunked:
content = static_content_partial(upload_file.chunks())
temporary_file_path = upload_file.temporary_file_path()
else:
content = static_content_partial(upload_file.read())
temporary_file_path = None
return content, temporary_file_path
def test_canonical_asset_path_with_c4x_style_assets(
self, base_url, start, expected, mongo_calls):
exts = ['.html', '.tm']
prefix = 'old'
base_c4x_block = 'c4x/a/b/asset'
adjusted_c4x_block = base_c4x_block
encoded_c4x_block = quote('/' + base_c4x_block + '/')
encoded_base_url = quote('//' + base_url)
encoded_base_c4x_block = encoded_c4x_block
start = start.format(prfx=prefix,
encoded_base_url=encoded_base_url,
c4x=base_c4x_block,
encoded_c4x=encoded_c4x_block)
# Adjust for content digest. This gets dicey quickly and we have to order our steps:
# - replace format markets because they have curly braces
# - encode Unicode characters to percent-encoded
# - finally shove back in our regex patterns
digest = CanonicalContentTest.get_content_digest_for_asset_path(
prefix, start)
if digest:
adjusted_c4x_block = 'assets/courseware/VMARK/HMARK/c4x/a/b/asset'
encoded_c4x_block = quote('/' + adjusted_c4x_block + '/')
expected = expected.format(
prfx=prefix,
encoded_base_url=encoded_base_url,
base_c4x=base_c4x_block,
c4x=adjusted_c4x_block,
encoded_c4x=encoded_c4x_block,
encoded_base_c4x=encoded_base_c4x_block,
)
expected = encode_unicode_characters_in_url(expected)
expected = expected.replace('VMARK', r'v[\d]')
expected = expected.replace('HMARK', '[a-f0-9]{32}')
expected = expected.replace('+', r'\+').replace('?', r'\?')
with check_mongo_calls(mongo_calls):
asset_path = StaticContent.get_canonicalized_asset_path(
self.courses[prefix].id, start, base_url, exts)
assert re.match(expected, asset_path) is not None
def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
"""
Does a regex replace on non-portable links:
/c4x/<org>/<course>/asset/<name> -> /static/<name>
/jump_to/i4x://<org>/<course>/<category>/<name> -> /jump_to_id/<id>
"""
course_id_dict = Location.parse_course_id(source_course_id)
course_id_dict['tag'] = 'i4x'
course_id_dict['category'] = 'course'
def portable_asset_link_subtitution(match):
quote = match.group('quote')
rest = match.group('rest')
return quote + '/static/' + rest + quote
def portable_jump_to_link_substitution(match):
quote = match.group('quote')
rest = match.group('rest')
return quote + '/jump_to_id/' + rest + quote
def generic_courseware_link_substitution(match):
parts = Location.parse_course_id(dest_course_id)
parts['quote'] = match.group('quote')
parts['rest'] = match.group('rest')
return u'{quote}/courses/{org}/{course}/{name}/{rest}{quote}'.format(
**parts)
course_location = Location(course_id_dict)
# NOTE: ultimately link updating is not a hard requirement, so if something blows up with
# the regex subsitution, log the error and continue
try:
c4x_link_base = u'{0}/'.format(
StaticContent.get_base_url_path_for_course_assets(course_location))
text = re.sub(_prefix_only_url_replace_regex(c4x_link_base),
portable_asset_link_subtitution, text)
except Exception, e:
logging.warning(
"Error going regex subtituion %r on text = %r.\n\nError msg = %s",
c4x_link_base, text, str(e))
def replace_static_url(match):
original = match.group(0)
prefix = match.group('prefix')
quote = match.group('quote')
rest = match.group('rest')
# Don't mess with things that end in '?raw'
if rest.endswith('?raw'):
return original
# In debug mode, if we can find the url as is,
if settings.DEBUG and finders.find(rest, True):
return original
# if we're running with a MongoBacked store course_namespace is not None, then use studio style urls
elif course_namespace is not None and not isinstance(
modulestore(), XMLModuleStore):
# first look in the static file pipeline and see if we are trying to reference
# a piece of static content which is in the mitx repo (e.g. JS associated with an xmodule)
if staticfiles_storage.exists(rest):
url = staticfiles_storage.url(rest)
else:
# if not, then assume it's courseware specific content and then look in the
# Mongo-backed database
url = StaticContent.convert_legacy_static_url(
rest, course_namespace)
# Otherwise, look the file up in staticfiles_storage, and append the data directory if needed
else:
course_path = "/".join((data_directory, rest))
try:
if staticfiles_storage.exists(rest):
url = staticfiles_storage.url(rest)
else:
url = staticfiles_storage.url(course_path)
# And if that fails, assume that it's course content, and add manually data directory
except Exception as err:
log.warning(
"staticfiles_storage couldn't find path {0}: {1}".format(
rest, str(err)))
url = "".join([prefix, course_path])
return "".join([quote, url, quote])
def test_delete_image_type_asset(self):
""" Tests deletion of image type asset """
image_asset = self.get_sample_asset(self.asset_name,
asset_type="image")
thumbnail_image_asset = self.get_sample_asset('delete_test_thumbnail',
asset_type="image")
# upload image
response = self.client.post(self.url, {
"name": "delete_image_test",
"file": image_asset
})
self.assertEqual(response.status_code, 200)
uploaded_image_url = json.loads(
response.content.decode('utf-8'))['asset']['url']
# upload image thumbnail
response = self.client.post(self.url, {
"name": "delete_image_thumb_test",
"file": thumbnail_image_asset
})
self.assertEqual(response.status_code, 200)
thumbnail_url = json.loads(
response.content.decode('utf-8'))['asset']['url']
thumbnail_location = StaticContent.get_location_from_path(
thumbnail_url)
image_asset_location = AssetKey.from_string(uploaded_image_url)
content = contentstore().find(image_asset_location)
content.thumbnail_location = thumbnail_location
contentstore().save(content)
with mock.patch('opaque_keys.edx.locator.CourseLocator.make_asset_key'
) as mock_asset_key:
mock_asset_key.return_value = thumbnail_location
test_url = reverse_course_url(
'assets_handler',
self.course.id,
kwargs={'asset_key_string': six.text_type(uploaded_image_url)})
resp = self.client.delete(test_url, HTTP_ACCEPT="application/json")
self.assertEqual(resp.status_code, 204)
def test_asset_and_course_deletion(self):
course_run = CourseFactory()
self.assertIsNotNone(modulestore().get_course(course_run.id))
store = contentstore()
asset_key = course_run.id.make_asset_key('asset', 'test.txt')
content = StaticContent(asset_key, 'test.txt', 'plain/text',
b'test data')
store.save(content)
__, asset_count = store.get_all_content_for_course(course_run.id)
assert asset_count == 1
with mock.patch(self.YESNO_PATCH_LOCATION) as patched_yes_no:
patched_yes_no.return_value = True
call_command('delete_course', str(course_run.id))
assert modulestore().get_course(course_run.id) is None
__, asset_count = store.get_all_content_for_course(course_run.id)
assert asset_count == 1
def course_info_handler(request, course_key_string):
"""
GET
html: return html for editing the course info handouts and updates.
"""
course_key = CourseKey.from_string(course_key_string)
course_module = _get_course_module(course_key, request.user)
if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):
return render_to_response(
'course_info.html',
{
'context_course': course_module,
'updates_url': reverse_course_url('course_info_update_handler', course_key),
'handouts_locator': course_key.make_usage_key('course_info', 'handouts'),
'base_asset_url': StaticContent.get_base_url_path_for_course_assets(course_module.id)
}
)
else:
return HttpResponseBadRequest("Only supports html requests")
def test_canonical_asset_path_with_c4x_style_assets(
self, base_url, start, expected, mongo_calls):
prefix = 'old'
c4x_block = 'c4x/a/b/asset'
encoded_c4x_block = quote_plus('/' + c4x_block + '/')
encoded_base_url = quote_plus('//' + base_url)
start = start.format(prefix=prefix,
encoded_base_url=encoded_base_url,
c4x=c4x_block,
encoded_c4x=encoded_c4x_block)
expected = expected.format(prefix=prefix,
encoded_base_url=encoded_base_url,
c4x=c4x_block,
encoded_c4x=encoded_c4x_block)
with check_mongo_calls(mongo_calls):
asset_path = StaticContent.get_canonicalized_asset_path(
self.courses[prefix].id, start, base_url)
self.assertEqual(asset_path, expected)
def restore_asset_from_trashcan(location):
'''
This method will restore an asset which got soft deleted and put back in the original course
'''
trash = contentstore('trashcan')
store = contentstore()
loc = StaticContent.get_location_from_path(location)
content = trash.find(loc)
# ok, save the content into the courseware
store.save(content)
# see if there is a thumbnail as well, if so move that as well
if content.thumbnail_location is not None:
try:
thumbnail_content = trash.find(content.thumbnail_location)
store.save(thumbnail_content)
except Exception: # lint-amnesty, pylint: disable=broad-except
pass # OK if this is left dangling
def copy_or_rename_transcript(new_name,
old_name,
item,
delete_old=False,
user=None):
"""
Renames `old_name` transcript file in storage to `new_name`.
If `old_name` is not found in storage, raises `NotFoundError`.
If `delete_old` is True, removes `old_name` files from storage.
"""
filename = u'subs_{0}.srt.sjson'.format(old_name)
content_location = StaticContent.compute_location(item.location.course_key,
filename)
transcripts = contentstore().find(content_location).data
save_subs_to_store(json.loads(transcripts), new_name, item)
item.sub = new_name
item.save_with_metadata(user)
if delete_old:
remove_subs_from_store(old_name, item)
def get_asset_content_from_path(course_key, asset_path):
"""
Locate the given asset content, load it into memory, and return it.
Returns None if the asset is not found.
"""
try:
from xmodule.contentstore.content import StaticContent
from xmodule.assetstore.assetmgr import AssetManager
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.exceptions import NotFoundError
except ImportError as exc:
raise EdXPlatformImportError(exc)
try:
asset_key = StaticContent.get_asset_key_from_path(
course_key, asset_path)
return AssetManager.find(asset_key)
except (ItemNotFoundError, NotFoundError) as exc:
return None
def rewrite_nonportable_content_links(source_course_id, dest_course_id, text):
"""
Does a regex replace on non-portable links:
/c4x/<org>/<course>/asset/<name> -> /static/<name>
/jump_to/i4x://<org>/<course>/<category>/<name> -> /jump_to_id/<id>
"""
org, course, run = source_course_id.split("/")
dest_org, dest_course, dest_run = dest_course_id.split("/")
def portable_asset_link_subtitution(match):
quote = match.group('quote')
rest = match.group('rest')
return quote + '/static/' + rest + quote
def portable_jump_to_link_substitution(match):
quote = match.group('quote')
rest = match.group('rest')
return quote + '/jump_to_id/' + rest + quote
def generic_courseware_link_substitution(match):
quote = match.group('quote')
rest = match.group('rest')
dest_generic_courseware_lik_base = '/courses/{org}/{course}/{run}/'.format(
org=dest_org, course=dest_course, run=dest_run)
return quote + dest_generic_courseware_lik_base + rest + quote
course_location = Location(['i4x', org, course, 'course', run])
# NOTE: ultimately link updating is not a hard requirement, so if something blows up with
# the regex subsitution, log the error and continue
try:
c4x_link_base = '{0}/'.format(
StaticContent.get_base_url_path_for_course_assets(course_location))
text = re.sub(_prefix_only_url_replace_regex(c4x_link_base),
portable_asset_link_subtitution, text)
except Exception, e:
logging.warning(
"Error going regex subtituion %r on text = %r.\n\nError msg = %s",
c4x_link_base, text, str(e))
def test_images_upload(self):
# http://www.django-rest-framework.org/api-guide/parsers/#fileuploadparser
course_run = CourseFactory()
expected_filename = 'course_image.png'
content_key = StaticContent.compute_location(course_run.id,
expected_filename)
assert course_run.course_image != expected_filename
try:
contentstore().find(content_key)
self.fail('No image should be associated with a new course run.')
except NotFoundError:
pass
url = reverse('api:v1:course_run-images',
kwargs={'pk': str(course_run.id)})
# PNG. Single black pixel
content = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90wS' \
b'\xde\x00\x00\x00\x0cIDATx\x9cc```\x00\x00\x00\x04\x00\x01\xf6\x178U\x00\x00\x00\x00IEND\xaeB`\x82'
# We are intentionally passing the incorrect JPEG extension here
upload = SimpleUploadedFile('card_image.jpg',
content,
content_type='image/png')
response = self.client.post(url, {'card_image': upload},
format='multipart')
assert response.status_code == 200
course_run = modulestore().get_course(course_run.id)
assert course_run.course_image == expected_filename
expected = {
'card_image':
RequestFactory().get('').build_absolute_uri(
course_image_url(course_run))
}
assert response.data == expected
# There should now be an image stored
contentstore().find(content_key)
def student_state(self):
"""
Returns a JSON serializable representation of student's state for
rendering in client view.
"""
submission = self.get_submission()
uploaded = None
if submission:
if 'user_response' in submission['answer'].keys():
uploaded = {"user_response": submission['answer']['user_response']}
elif 'filename' in submission['answer'].keys():
uploaded = {"filename": submission['answer']['filename']}
if self.annotated_sha1:
annotated = {"filename": force_text(self.annotated_filename)}
else:
annotated = None
score = self.score
if score is not None:
graded = {'score': score, 'comment': force_text(self.comment)}
else:
graded = None
if self.answer_available():
solution = self.runtime.replace_urls(force_text(self.solution))
else:
solution = ''
# pylint: disable=no-member
return {
"submission_type": force_text(self.submission_type),
"display_name": force_text(self.display_name),
"uploaded": uploaded,
"annotated": annotated,
"graded": graded,
"max_score": self.max_score(),
"upload_allowed": self.upload_allowed(submission_data=submission),
"solution": solution,
"base_asset_url": StaticContent.get_base_url_path_for_course_assets(self.location.course_key),
}
def download_transcripts(request):
"""
Passes to user requested transcripts file.
Raises Http404 if unsuccessful.
"""
locator = request.GET.get('locator')
if not locator:
log.debug('GET data without "locator" property.')
raise Http404
try:
item = _get_item(request, request.GET)
except (InvalidKeyError, ItemNotFoundError):
log.debug("Can't find item by locator.")
raise Http404
subs_id = request.GET.get('subs_id')
if not subs_id:
log.debug('GET data without "subs_id" property.')
raise Http404
if item.category != 'video':
log.debug('transcripts are supported only for video" modules.')
raise Http404
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(item.location.course_key, filename)
try:
sjson_transcripts = contentstore().find(content_location)
log.debug("Downloading subs for %s id", subs_id)
str_subs = generate_srt_from_sjson(json.loads(sjson_transcripts.data), speed=1.0)
if not str_subs:
log.debug('generate_srt_from_sjson produces no subtitles')
raise Http404
response = HttpResponse(str_subs, content_type='application/x-subrip')
response['Content-Disposition'] = 'attachment; filename="{0}.srt"'.format(subs_id)
return response
except NotFoundError:
log.debug("Can't find content in storage for %s subs", subs_id)
raise Http404
def read_from_contentstore(course_key, path):
"""
Loads a file directly from the course's content store.
"""
contents = None
try:
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from opaque_keys.edx.locator import CourseLocator
except ImportError:
# We're not running under edx-platform, so ignore.
pass
else:
if isinstance(course_key, six.text_type):
course_key = CourseLocator.from_string(course_key)
loc = StaticContent.compute_location(course_key, path)
asset = contentstore().find(loc)
contents = asset.data
return contents
def _upload_file(file, location):
filename = 'subs_{}.srt.sjson'.format(_get_subs_id(file.name))
mime_type = file.content_type
content_location = StaticContent.compute_location(location.org,
location.course,
filename)
sc_partial = partial(StaticContent, content_location, filename, mime_type)
content = sc_partial(file.read())
(thumbnail_content,
thumbnail_location) = contentstore().generate_thumbnail(
content, tempfile_path=None)
del_cached_content(thumbnail_location)
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
contentstore().save(content)
del_cached_content(content.location)
def course_video_url(course):
"""Try to look up the video url for the course. If it'is not found,
log an error and return the dead link"""
if course.static_asset_path or modulestore().get_modulestore_type(
course.id) == ModuleStoreEnum.Type.xml:
# If we are a static course with the course_image attribute
# set different than the default, return that path so that
# courses can use custom course image paths, otherwise just
# return the default static path.
url = '/static/' + (course.static_asset_path
or getattr(course, 'data_dir', ''))
if hasattr(
course, 'course_video'
) and course.course_video != course.fields['course_video'].default:
url += '/' + course.course_video
else:
url += '/videos/course_video.mp4'
else:
loc = StaticContent.compute_location(course.id, course.course_video)
url = loc.to_deprecated_string()
return url
def course_image_url(course):
"""Try to look up the image url for the course. If it's not found,
log an error and return the dead link"""
if course.static_asset_path or modulestore().get_modulestore_type(
course.id) == XML_MODULESTORE_TYPE:
# If we are a static course with the course_image attribute
# set different than the default, return that path so that
# courses can use custom course image paths, otherwise just
# return the default static path.
url = '/static/' + (course.static_asset_path
or getattr(course, 'data_dir', ''))
if hasattr(
course, 'course_image'
) and course.course_image != course.fields['course_image'].default:
url += '/' + course.course_image
else:
url += '/images/course_image.jpg'
else:
loc = StaticContent.compute_location(course.id, course.course_image)
url = loc.to_deprecated_string()
return url
def update_course_run_asset(course_key, upload_file):
filename = upload_file.name
mime_type = upload_file.content_type
size = get_file_size(upload_file)
max_size_in_mb = settings.MAX_ASSET_UPLOAD_FILE_SIZE_IN_MB
max_file_size_in_bytes = max_size_in_mb * 1000**2
if size > max_file_size_in_bytes:
msg = 'File {filename} exceeds the maximum size of {max_size_in_mb} MB.'.format(
filename=filename, max_size_in_mb=max_size_in_mb)
raise AssetSizeTooLargeException(msg)
content_loc = StaticContent.compute_location(course_key, filename)
chunked = upload_file.multiple_chunks()
sc_partial = partial(StaticContent, content_loc, filename, mime_type)
if chunked:
content = sc_partial(upload_file.chunks())
tempfile_path = upload_file.temporary_file_path()
else:
content = sc_partial(upload_file.read())
tempfile_path = None
# Verify a thumbnail can be created
(thumbnail_content,
thumbnail_location) = contentstore().generate_thumbnail(
content, tempfile_path=tempfile_path)
# delete cached thumbnail even if one couldn't be created this time (else the old thumbnail will continue to show)
del_cached_content(thumbnail_location)
# now store thumbnail location only if we could create it
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
# then commit the content
contentstore().save(content)
del_cached_content(content.location)
return content
def get_transcript(self, subs_id):
'''
Returns transcript without timecodes.
Args:
`subs_id`: str, subtitles id
Raises:
- NotFoundError if cannot find transcript file in storage.
- ValueError if transcript file is incorrect JSON.
- KeyError if transcript file has incorrect format.
'''
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(
self.location.org, self.location.course, filename
)
data = contentstore().find(content_location).data
text = json.loads(data)['text']
return HTMLParser().unescape("\n".join(text))
def test_pdf_asset(self):
module_store = modulestore('direct')
_, course_items = import_from_xml(module_store,
'common/test/data/', ['toy'],
static_content_store=contentstore(),
verbose=True)
course = course_items[0]
location = loc_mapper().translate_location(course.location.course_id,
course.location, False,
True)
url = location.url_reverse('assets/', '')
# Test valid contentType for pdf asset (textbook.pdf)
resp = self.client.get(url, HTTP_ACCEPT='application/json')
self.assertContains(resp, "/c4x/edX/toy/asset/textbook.pdf")
asset_location = StaticContent.get_location_from_path(
'/c4x/edX/toy/asset/textbook.pdf')
content = contentstore().find(asset_location)
# Check after import textbook.pdf has valid contentType ('application/pdf')
# Note: Actual contentType for textbook.pdf in asset.json is 'text/pdf'
self.assertEqual(content.content_type, 'application/pdf')
def test_fail_downloading_subs(self):
bad_youtube_subs = {
0.5: 'BAD_YOUTUBE_ID1',
1.0: 'BAD_YOUTUBE_ID2',
2.0: 'BAD_YOUTUBE_ID3'
}
self.clear_subs_content(bad_youtube_subs)
with self.assertRaises(
transcripts_utils.GetTranscriptsFromYouTubeException):
transcripts_utils.download_youtube_subs(bad_youtube_subs,
self.course)
# Check assets status after importing subtitles.
for subs_id in bad_youtube_subs.values():
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(
self.org, self.number, filename)
with self.assertRaises(NotFoundError):
contentstore().find(content_location)
self.clear_subs_content(bad_youtube_subs)
def course_info_handler(request,
tag=None,
package_id=None,
branch=None,
version_guid=None,
block=None):
"""
GET
html: return html for editing the course info handouts and updates.
"""
__, course_module = _get_locator_and_course(package_id, branch,
version_guid, block,
request.user)
if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):
handouts_old_location = course_module.location.replace(
category='course_info', name='handouts')
handouts_locator = loc_mapper().translate_location(
course_module.location.course_id, handouts_old_location, False,
True)
update_location = course_module.location.replace(
category='course_info', name='updates')
update_locator = loc_mapper().translate_location(
course_module.location.course_id, update_location, False, True)
return render_to_response(
'course_info.html', {
'context_course':
course_module,
'updates_url':
update_locator.url_reverse('course_info_update/'),
'handouts_locator':
handouts_locator,
'base_asset_url':
StaticContent.get_base_url_path_for_course_assets(
course_module.location) + '/'
})
else:
return HttpResponseBadRequest("Only supports html requests")
def test_success_downloading_chinese_transcripts(self):
# Disabled 11/14/13
# This test is flakey because it performs an HTTP request on an external service
# Re-enable when `requests.get` is patched using `mock.patch`
raise SkipTest
good_youtube_sub = 'j_jEn79vS3g' # Chinese, utf-8
self.clear_sub_content(good_youtube_sub)
# Check transcripts_utils.GetTranscriptsFromYouTubeException not thrown
transcripts_utils.download_youtube_subs(good_youtube_sub, self.course,
settings)
# Check assets status after importing subtitles.
for subs_id in good_youtube_subs.values():
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(
self.course.id, filename)
self.assertTrue(contentstore().find(content_location))
self.clear_sub_content(good_youtube_sub)
def student_view(self, context=None):
"""
The student view of the MarkdownXBlock.
"""
if self.filename:
# These can only be imported when the XBlock is running on the LMS
# or CMS. Do it at runtime so that the workbench is usable for
# regular XML content.
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import NotFoundError
from opaque_keys import InvalidKeyError
try:
course_id = self.xmodule_runtime.course_id
loc = StaticContent.compute_location(course_id, self.filename)
asset = contentstore().find(loc)
content = asset.data
except (NotFoundError, InvalidKeyError):
pass
else:
content = self.content
html_content = ""
if content:
html_content = markdown2.markdown(content, extras=self.extras)
# Render the HTML template
context = {'content': html_content}
html = loader.render_template('templates/main.html', context)
frag = Fragment(html)
if "fenced-code-blocks" in self.extras:
frag.add_css_url(
self.runtime.local_resource_url(self,
'public/css/pygments.css'))
return frag
def test_subs_uploading_with_byte_order_mark(self):
"""
Test uploading subs containing BOM(Byte Order Mark), e.g. U+FEFF
"""
filedata = textwrap.dedent("""
1
00:00:10,500 --> 00:00:13,000
Test ufeff characters
2
00:00:15,000 --> 00:00:18,000
At the left we can see...
""").encode('utf-8-sig')
# Verify that ufeff character is in filedata.
self.assertIn("ufeff", filedata)
self.ufeff_srt_file.write(filedata)
self.ufeff_srt_file.seek(0)
link = reverse('upload_transcripts')
filename = os.path.splitext(os.path.basename(self.ufeff_srt_file.name))[0]
resp = self.client.post(link, {
'locator': self.video_usage_key,
'transcript-file': self.ufeff_srt_file,
'video_list': json.dumps([{
'type': 'html5',
'video': filename,
'mode': 'mp4',
}])
})
self.assertEqual(resp.status_code, 200)
content_location = StaticContent.compute_location(
self.course.id, 'subs_{0}.srt.sjson'.format(filename))
self.assertTrue(contentstore().find(content_location))
subs_text = json.loads(contentstore().find(content_location).data).get('text')
self.assertIn("Test ufeff characters", subs_text)
def get_pgreport_csv(course_id):
"""Get progress of students."""
course_key = get_coursekey(course_id)
location = StaticContent.compute_location(course_key,
"progress_students.csv.gz")
store = contentstore()
try:
gzipfile = StringIO.StringIO()
content = store.find(location, throw_on_not_found=True, as_stream=True)
for gzipdata in content.stream_data():
gzipfile.write(gzipdata)
gzipfile.seek(0)
gzipcsv = gzip.GzipFile(fileobj=gzipfile, mode='rb')
for csvrow in gzipcsv.readlines():
print csvrow,
gzipcsv.close()
except NotFoundError as e:
log.warn(" * Csv does not exists: {}".format(e))
finally:
gzipfile.close()