def setUp(self): super(TestSaveSubsToStore, self).setUp() self.course = CourseFactory.create( org=self.org, number=self.number, display_name=self.display_name) self.subs = { 'start': [100, 200, 240, 390, 1000], 'end': [200, 240, 380, 1000, 1500], 'text': [ 'subs #1', 'subs #2', 'subs #3', 'subs #4', 'subs #5' ] } self.subs_id = str(uuid4()) filename = 'subs_{0}.srt.sjson'.format(self.subs_id) self.content_location = StaticContent.compute_location(self.course.id, filename) self.addCleanup(self.clear_subs_content) # incorrect subs self.unjsonable_subs = set([1]) # set can't be serialized self.unjsonable_subs_id = str(uuid4()) filename_unjsonable = 'subs_{0}.srt.sjson'.format(self.unjsonable_subs_id) self.content_location_unjsonable = StaticContent.compute_location(self.course.id, filename_unjsonable) self.clear_subs_content()
def setUpClass(cls): super(TestSaveSubsToStore, cls).setUpClass() cls.course = CourseFactory.create( org=cls.org, number=cls.number, display_name=cls.display_name) cls.subs = { 'start': [100, 200, 240, 390, 1000], 'end': [200, 240, 380, 1000, 1500], 'text': [ 'subs #1', 'subs #2', 'subs #3', 'subs #4', 'subs #5' ] } cls.subs_id = str(uuid4()) filename = 'subs_{0}.srt.sjson'.format(cls.subs_id) cls.content_location = StaticContent.compute_location(cls.course.id, filename) # incorrect subs cls.unjsonable_subs = {1} # set can't be serialized cls.unjsonable_subs_id = str(uuid4()) filename_unjsonable = 'subs_{0}.srt.sjson'.format(cls.unjsonable_subs_id) cls.content_location_unjsonable = StaticContent.compute_location(cls.course.id, filename_unjsonable)
def verify_content_links(module, base_dir, static_content_store, link, remap_dict=None): if link.startswith('/static/'): # yes, then parse out the name path = link[len('/static/'):] static_pathname = base_dir / path if os.path.exists(static_pathname): try: content_loc = StaticContent.compute_location(module.location.org, module.location.course, path) filename = os.path.basename(path) mime_type = mimetypes.guess_type(filename)[0] with open(static_pathname, 'rb') as f: data = f.read() content = StaticContent(content_loc, filename, mime_type, data, import_path=path) # first let's save a thumbnail so we can get back a thumbnail location (thumbnail_content, thumbnail_location) = static_content_store.generate_thumbnail(content) if thumbnail_content is not None: content.thumbnail_location = thumbnail_location #then commit the content static_content_store.save(content) new_link = StaticContent.get_url_path_from_location(content_loc) if remap_dict is not None: remap_dict[link] = new_link return new_link except Exception, e: logging.exception('Skipping failed content load from {0}. Exception: {1}'.format(path, e))
def remap_namespace(module, target_location_namespace): if target_location_namespace is None: return module # This looks a bit wonky as we need to also change the 'name' of the # imported course to be what the caller passed in if module.location.category != 'course': module.location = module.location._replace( tag=target_location_namespace.tag, org=target_location_namespace.org, course=target_location_namespace.course ) else: original_location = module.location # # module is a course module # module.location = module.location._replace( tag=target_location_namespace.tag, org=target_location_namespace.org, course=target_location_namespace.course, name=target_location_namespace.name ) # There is more re-namespacing work we have to do when # importing course modules # remap pdf_textbook urls for entry in module.pdf_textbooks: for chapter in entry.get('chapters', []): if StaticContent.is_c4x_path(chapter.get('url', '')): chapter['url'] = StaticContent.renamespace_c4x_path( chapter['url'], target_location_namespace ) # if there is a wiki_slug which is the same as the original location # (aka default value), then remap that so the wiki doesn't point to # the old Wiki. if module.wiki_slug == original_location.course: module.wiki_slug = target_location_namespace.course module.save() # then remap children pointers since they too will be re-namespaced if hasattr(module, 'children'): children_locs = module.children if children_locs is not None and children_locs != []: new_locs = [] for child in children_locs: child_loc = Location(child) new_child_loc = child_loc._replace( tag=target_location_namespace.tag, org=target_location_namespace.org, course=target_location_namespace.course ) new_locs.append(new_child_loc.url()) module.children = new_locs return module
def test_happy_path(self, modulestore_type, create_after_overview): """ What happens when everything works like we expect it to. If `create_after_overview` is True, we will temporarily disable thumbnail creation so that the initial CourseOverview is created without an image_set, and the CourseOverviewImageSet is created afterwards. If `create_after_overview` is False, we'll create the CourseOverviewImageSet at the same time as the CourseOverview. """ # Create a real (oversized) image... image = Image.new("RGB", (800, 400), "blue") image_buff = StringIO() image.save(image_buff, format="JPEG") image_buff.seek(0) image_name = "big_course_image.jpeg" with self.store.default_store(modulestore_type): course = CourseFactory.create( default_store=modulestore_type, course_image=image_name ) # Save a real image here... course_image_asset_key = StaticContent.compute_location(course.id, course.course_image) course_image_content = StaticContent(course_image_asset_key, image_name, 'image/jpeg', image_buff) contentstore().save(course_image_content) # If create_after_overview is True, disable thumbnail generation so # that the CourseOverview object is created and saved without an # image_set at first (it will be lazily created later). if create_after_overview: self.set_config(enabled=False) # Now generate the CourseOverview... course_overview = CourseOverview.get_from_id(course.id) # If create_after_overview is True, no image_set exists yet. Verify # that, then switch config back over to True and it should lazily # create the image_set on the next get_from_id() call. if create_after_overview: self.assertFalse(hasattr(course_overview, 'image_set')) self.set_config(enabled=True) course_overview = CourseOverview.get_from_id(course.id) self.assertTrue(hasattr(course_overview, 'image_set')) image_urls = course_overview.image_urls config = CourseOverviewImageConfig.current() # Make sure the thumbnail names come out as expected... self.assertTrue(image_urls['raw'].endswith('big_course_image.jpeg')) self.assertTrue(image_urls['small'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.small))) self.assertTrue(image_urls['large'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.large))) # Now make sure our thumbnails are of the sizes we expect... for image_url, expected_size in [(image_urls['small'], config.small), (image_urls['large'], config.large)]: image_key = StaticContent.get_location_from_path(image_url) image_content = AssetManager.find(image_key) image = Image.open(StringIO(image_content.data)) self.assertEqual(image.size, expected_size)
def test_different_resolutions(self, src_dimensions): """ Test various resolutions of images to make thumbnails of. Note that our test sizes are small=(200, 100) and large=(400, 200). 1. Images should won't be blown up if it's too small, so a (100, 50) resolution image will remain (100, 50). 2. However, images *will* be converted using our format and quality settings (JPEG, 75% -- the PIL default). This is because images with relatively small dimensions not compressed properly. 3. Image thumbnail naming will maintain the naming convention of the target resolution, even if the image was not actually scaled to that size (i.e. it was already smaller). This is mostly because it's simpler to be consistent, but it also lets us more easily tell which configuration a thumbnail was created under. """ # Create a source image... image = Image.new("RGB", src_dimensions, "blue") image_buff = StringIO() image.save(image_buff, format="PNG") image_buff.seek(0) image_name = "src_course_image.png" course = CourseFactory.create(course_image=image_name) # Save the image to the contentstore... course_image_asset_key = StaticContent.compute_location(course.id, course.course_image) course_image_content = StaticContent(course_image_asset_key, image_name, 'image/png', image_buff) contentstore().save(course_image_content) # Now generate the CourseOverview... config = CourseOverviewImageConfig.current() course_overview = CourseOverview.get_from_id(course.id) image_urls = course_overview.image_urls for image_url, target in [(image_urls['small'], config.small), (image_urls['large'], config.large)]: image_key = StaticContent.get_location_from_path(image_url) image_content = AssetManager.find(image_key) image = Image.open(StringIO(image_content.data)) # Naming convention for thumbnail self.assertTrue(image_url.endswith('src_course_image-png-{}x{}.jpg'.format(*target))) # Actual thumbnail data src_x, src_y = src_dimensions target_x, target_y = target image_x, image_y = image.size # I'm basically going to assume the image library knows how to do # the right thing in terms of handling aspect ratio. We're just # going to make sure that small images aren't blown up, and that # we never exceed our target sizes self.assertLessEqual(image_x, target_x) self.assertLessEqual(image_y, target_y) if src_x < target_x and src_y < target_y: self.assertEqual(src_x, image_x) self.assertEqual(src_y, image_y)
def course_image_url(course): """Returns the image url for the course.""" try: loc = StaticContent.compute_location(course.location.course_key, course.course_image) except InvalidKeyError: return '' path = StaticContent.serialize_asset_key_with_slash(loc) return path
def course_image_url(course): """Try to look up the image url for the course. If it's not found, log an error and return the dead link""" if course.static_asset_path or modulestore().get_modulestore_type(course.location.course_id) == XML_MODULESTORE_TYPE: return '/static/' + (course.static_asset_path or getattr(course, 'data_dir', '')) + "/images/course_image.jpg" else: loc = StaticContent.compute_location(course.location.org, course.location.course, course.course_image) _path = StaticContent.get_url_path_from_location(loc) return _path
def course_image_url(course): """ Return url of course image. Args: course(CourseDescriptor) : The course id to retrieve course image url. Returns: Absolute url of course image. """ loc = StaticContent.compute_location(course.id, course.course_image) url = StaticContent.serialize_asset_key_with_slash(loc) return url
def create_course_image_thumbnail(course, dimensions): """Create a course image thumbnail and return the URL. - dimensions is a tuple of (width, height) """ course_image_asset_key = StaticContent.compute_location(course.id, course.course_image) course_image = AssetManager.find(course_image_asset_key) # a StaticContent obj _content, thumb_loc = contentstore().generate_thumbnail(course_image, dimensions=dimensions) return StaticContent.serialize_asset_key_with_slash(thumb_loc)
def asset_index(request, org, course, name, start=None, maxresults=None): """ Display an editable asset library org, course, name: Attributes of the Location for the item to edit :param start: which index of the result list to start w/, used for paging results :param maxresults: maximum results """ location = get_location_and_verify_access(request, org, course, name) upload_asset_callback_url = reverse('upload_asset', kwargs={ 'org': org, 'course': course, 'coursename': name }) course_module = modulestore().get_item(location) course_reference = StaticContent.compute_location(org, course, name) if maxresults is not None: maxresults = int(maxresults) start = int(start) if start else 0 assets = contentstore().get_all_content_for_course( course_reference, start=start, maxresults=maxresults, sort=[('uploadDate', DESCENDING)] ) else: assets = contentstore().get_all_content_for_course( course_reference, sort=[('uploadDate', DESCENDING)] ) asset_json = [] for asset in assets: asset_id = asset['_id'] asset_location = StaticContent.compute_location(asset_id['org'], asset_id['course'], asset_id['name']) # note, due to the schema change we may not have a 'thumbnail_location' in the result set _thumbnail_location = asset.get('thumbnail_location', None) thumbnail_location = Location(_thumbnail_location) if _thumbnail_location is not None else None asset_locked = asset.get('locked', False) asset_json.append(_get_asset_json(asset['displayname'], asset['uploadDate'], asset_location, thumbnail_location, asset_locked)) return render_to_response('asset_index.html', { 'context_course': course_module, 'asset_list': json.dumps(asset_json), 'upload_asset_callback_url': upload_asset_callback_url, 'update_asset_callback_url': reverse('update_asset', kwargs={ 'org': org, 'course': course, 'name': name }) })
def _clear_assets(location): store = contentstore() content_location = StaticContent.compute_location( location.org, location.course, location.name ) assets, __ = store.get_all_content_for_course(content_location) for asset in assets: asset_location = Location(asset["_id"]) id = StaticContent.get_id_from_location(asset_location) store.delete(id)
def asset_index(request, org, course, name): """ Display an editable asset library org, course, name: Attributes of the Location for the item to edit """ location = get_location_and_verify_access(request, org, course, name) upload_asset_callback_url = reverse('upload_asset', kwargs={ 'org': org, 'course': course, 'coursename': name }) course_module = modulestore().get_item(location) course_reference = StaticContent.compute_location(org, course, name) assets = contentstore().get_all_content_for_course(course_reference) # sort in reverse upload date order assets = sorted(assets, key=lambda asset: asset['uploadDate'], reverse=True) if request.META.get('HTTP_ACCEPT', "").startswith("application/json"): return JsonResponse(assets_to_json_dict(assets)) asset_display = [] for asset in assets: asset_id = asset['_id'] display_info = {} display_info['displayname'] = asset['displayname'] display_info['uploadDate'] = get_default_time_display(asset['uploadDate']) asset_location = StaticContent.compute_location(asset_id['org'], asset_id['course'], asset_id['name']) display_info['url'] = StaticContent.get_url_path_from_location(asset_location) display_info['portable_url'] = StaticContent.get_static_path_from_location(asset_location) # note, due to the schema change we may not have a 'thumbnail_location' in the result set _thumbnail_location = asset.get('thumbnail_location', None) thumbnail_location = Location(_thumbnail_location) if _thumbnail_location is not None else None display_info['thumb_url'] = StaticContent.get_url_path_from_location(thumbnail_location) if thumbnail_location is not None else None asset_display.append(display_info) return render_to_response('asset_index.html', { 'context_course': course_module, 'assets': asset_display, 'upload_asset_callback_url': upload_asset_callback_url, 'remove_asset_callback_url': reverse('remove_asset', kwargs={ 'org': org, 'course': course, 'name': name }) })
def import_static_content( modules, course_loc, course_data_path, static_content_store, target_location_namespace, subpath="static", verbose=False, ): remap_dict = {} # now import all static assets static_dir = course_data_path / subpath verbose = True for dirname, dirnames, filenames in os.walk(static_dir): for filename in filenames: try: content_path = os.path.join(dirname, filename) if verbose: log.debug("importing static content {0}...".format(content_path)) fullname_with_subpath = content_path.replace(static_dir, "") # strip away leading path from the name if fullname_with_subpath.startswith("/"): fullname_with_subpath = fullname_with_subpath[1:] content_loc = StaticContent.compute_location( target_location_namespace.org, target_location_namespace.course, fullname_with_subpath ) mime_type = mimetypes.guess_type(filename)[0] with open(content_path, "rb") as f: data = f.read() content = StaticContent(content_loc, filename, mime_type, data, import_path=fullname_with_subpath) # first let's save a thumbnail so we can get back a thumbnail location (thumbnail_content, thumbnail_location) = static_content_store.generate_thumbnail(content) if thumbnail_content is not None: content.thumbnail_location = thumbnail_location # then commit the content static_content_store.save(content) # store the remapping information which will be needed to subsitute in the module data remap_dict[fullname_with_subpath] = content_loc.name except: raise return remap_dict
def get_versioned_asset_url(asset_path): """ Creates a versioned asset URL. """ try: locator = StaticContent.get_location_from_path(asset_path) content = AssetManager.find(locator, as_stream=True) return StaticContent.add_version_to_asset_path(asset_path, content.content_digest) except (InvalidKeyError, ItemNotFoundError): pass return asset_path
def import_static_content(modules, course_loc, course_data_path, static_content_store, target_location_namespace, subpath='static', verbose=False): remap_dict = {} # now import all static assets static_dir = course_data_path / subpath verbose = True for dirname, _, filenames in os.walk(static_dir): for filename in filenames: content_path = os.path.join(dirname, filename) if verbose: log.debug('importing static content %s...', content_path) try: with open(content_path, 'rb') as f: data = f.read() except IOError: if filename.startswith('._'): # OS X "companion files". See http://www.diigo.com/annotated/0c936fda5da4aa1159c189cea227e174 continue # Not a 'hidden file', then re-raise exception raise fullname_with_subpath = content_path.replace(static_dir, '') # strip away leading path from the name if fullname_with_subpath.startswith('/'): fullname_with_subpath = fullname_with_subpath[1:] content_loc = StaticContent.compute_location(target_location_namespace.org, target_location_namespace.course, fullname_with_subpath) mime_type = mimetypes.guess_type(filename)[0] content = StaticContent(content_loc, filename, mime_type, data, import_path=fullname_with_subpath) # first let's save a thumbnail so we can get back a thumbnail location (thumbnail_content, thumbnail_location) = static_content_store.generate_thumbnail(content) if thumbnail_content is not None: content.thumbnail_location = thumbnail_location #then commit the content try: static_content_store.save(content) except Exception as err: log.exception('Error importing {0}, error={1}'.format(fullname_with_subpath, err)) #store the remapping information which will be needed to subsitute in the module data remap_dict[fullname_with_subpath] = content_loc.name return remap_dict
def course_image_url(course, image_name=None): """Return url for image_name or default course image in given course assets. It allows us to override default course image in templates when this function is used whith image_name parameter, if the image is available. (see course_about.html) """ image = image_name or course.course_image try: loc = StaticContent.compute_location(course.location.course_key, image) _ = contentstore().find(loc) except NotFoundError: loc = StaticContent.compute_location(course.location.course_key, course.course_image) return loc.to_deprecated_string()
def _get_asset_json(display_name, date, location, thumbnail_location, locked): """ Helper method for formatting the asset information to send to client. """ asset_url = StaticContent.get_url_path_from_location(location) return { 'display_name': display_name, 'date_added': get_default_time_display(date), 'url': asset_url, 'portable_url': StaticContent.get_static_path_from_location(location), 'thumbnail': StaticContent.get_url_path_from_location(thumbnail_location) if thumbnail_location is not None else None, 'locked': locked, # Needed for Backbone delete/update. 'id': asset_url }
def delete_course(modulestore, contentstore, source_location, commit=False): # first check to see if the modulestore is Mongo backed if not isinstance(modulestore, MongoModuleStore): raise Exception("Expected a MongoModuleStore in the runtime. Aborting....") # check to see if the source course is actually there if not modulestore.has_item(source_location): raise Exception("Cannot find a course at {0}. Aborting".format(source_location)) # first delete all of the thumbnails thumbs = contentstore.get_all_content_thumbnails_for_course(source_location) for thumb in thumbs: thumb_loc = Location(thumb["_id"]) id = StaticContent.get_id_from_location(thumb_loc) print "Deleting {0}...".format(id) if commit: contentstore.delete(id) # then delete all of the assets assets = contentstore.get_all_content_for_course(source_location) for asset in assets: asset_loc = Location(asset["_id"]) id = StaticContent.get_id_from_location(asset_loc) print "Deleting {0}...".format(id) if commit: contentstore.delete(id) # then delete all course modules modules = modulestore.get_items([source_location.tag, source_location.org, source_location.course, None, None, None]) course_module = None for module in modules: if module.category != 'course': # save deleting the course module for last print "Deleting {0}...".format(module.location) if commit: modulestore.delete_item(module.location) else: course_module = module if course_module: modulestore.delete_item(course_module.location) # finally delete the top-level course module itself print "Deleting {0}...".format(source_location) if commit: modulestore.delete_item(source_location) return True
def test_success_generating_subs(self): youtube_subs = { 0.5: 'JMD_ifUUfsU', 1.0: 'hI10vDNYz4M', 2.0: 'AKqURZnYqpk' } srt_filedata = textwrap.dedent(""" 1 00:00:10,500 --> 00:00:13,000 Elephant's Dream 2 00:00:15,000 --> 00:00:18,000 At the left we can see... """) self.clear_subs_content(youtube_subs) # Check transcripts_utils.TranscriptsGenerationException not thrown. # Also checks that uppercase file extensions are supported. transcripts_utils.generate_subs_from_source(youtube_subs, 'SRT', srt_filedata, self.course) # Check assets status after importing subtitles. for subs_id in youtube_subs.values(): filename = 'subs_{0}.srt.sjson'.format(subs_id) content_location = StaticContent.compute_location( self.course.id, filename ) self.assertTrue(contentstore().find(content_location)) self.clear_subs_content(youtube_subs)
def course_info_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None): """ GET html: return html for editing the course info handouts and updates. """ __, course_module = _get_locator_and_course( package_id, branch, version_guid, block, request.user ) if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'): handouts_old_location = course_module.location.replace(category='course_info', name='handouts') handouts_locator = loc_mapper().translate_location( course_module.location.course_id, handouts_old_location, False, True ) update_location = course_module.location.replace(category='course_info', name='updates') update_locator = loc_mapper().translate_location( course_module.location.course_id, update_location, False, True ) return render_to_response( 'course_info.html', { 'context_course': course_module, 'updates_url': update_locator.url_reverse('course_info_update/'), 'handouts_locator': handouts_locator, 'base_asset_url': StaticContent.get_base_url_path_for_course_assets(course_module.location) + '/' } ) else: return HttpResponseBadRequest("Only supports html requests")
def test_success_video_module_source_subs_uploading(self): self.item.data = textwrap.dedent(""" <video youtube=""> <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/> <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/> <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/> </video> """) modulestore().update_item(self.item, self.user.id) link = reverse('upload_transcripts') filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0] resp = self.client.post(link, { 'locator': self.video_usage_key, 'transcript-file': self.good_srt_file, 'video_list': json.dumps([{ 'type': 'html5', 'video': filename, 'mode': 'mp4', }]) }) self.assertEqual(resp.status_code, 200) self.assertEqual(json.loads(resp.content).get('status'), 'Success') item = modulestore().get_item(self.video_usage_key) self.assertEqual(item.sub, filename) content_location = StaticContent.compute_location( self.course.id, 'subs_{0}.srt.sjson'.format(filename)) self.assertTrue(contentstore().find(content_location))
def _upload_asset(request, course_key): ''' This method allows for POST uploading of files into the course asset library, which will be supported by GridFS in MongoDB. ''' # Does the course actually exist?!? Get anything from it to prove its # existence try: modulestore().get_course(course_key) except ItemNotFoundError: # no return it as a Bad Request response logging.error("Could not find course: %s", course_key) return HttpResponseBadRequest() # compute a 'filename' which is similar to the location formatting, we're # using the 'filename' nomenclature since we're using a FileSystem paradigm # here. We're just imposing the Location string formatting expectations to # keep things a bit more consistent upload_file = request.FILES['file'] filename = upload_file.name mime_type = upload_file.content_type content_loc = StaticContent.compute_location(course_key, filename) chunked = upload_file.multiple_chunks() sc_partial = partial(StaticContent, content_loc, filename, mime_type) if chunked: content = sc_partial(upload_file.chunks()) tempfile_path = upload_file.temporary_file_path() else: content = sc_partial(upload_file.read()) tempfile_path = None # first let's see if a thumbnail can be created (thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail( content, tempfile_path=tempfile_path ) # delete cached thumbnail even if one couldn't be created this time (else # the old thumbnail will continue to show) del_cached_content(thumbnail_location) # now store thumbnail location only if we could create it if thumbnail_content is not None: content.thumbnail_location = thumbnail_location # then commit the content contentstore().save(content) del_cached_content(content.location) # readback the saved content - we need the database timestamp readback = contentstore().find(content.location) locked = getattr(content, 'locked', False) response_payload = { 'asset': _get_asset_json(content.name, readback.last_modified_at, content.location, content.thumbnail_location, locked), 'msg': _('Upload completed') } return JsonResponse(response_payload)
def get_asset_location(asset_id): """ Helper method to get the location (and verify it is valid). """ try: return StaticContent.get_location_from_path(asset_id) except InvalidLocationError as err: # return a 'Bad Request' to browser as we have a malformed Location return JsonResponse({"error": err.message}, status=400)
def test_compute_location(self): # We had a bug that __ got converted into a single _. Make sure that substitution of INVALID_CHARS (like space) # still happen. asset_location = StaticContent.compute_location( SlashSeparatedCourseKey('mitX', '400', 'ignore'), 'subs__1eo_jXvZnE .srt.sjson' ) self.assertEqual(AssetLocation(u'mitX', u'400', u'ignore', u'asset', u'subs__1eo_jXvZnE_.srt.sjson', None), asset_location)
def get_html(self): if isinstance(modulestore(), MongoModuleStore): caption_asset_path = StaticContent.get_base_url_path_for_course_assets(self.location) + '/subs_' else: # VS[compat] # cdodge: filesystem static content support. caption_asset_path = "/static/subs/" get_ext = lambda filename: filename.rpartition('.')[-1] sources = {get_ext(src): src for src in self.html5_sources} sources['main'] = self.source return self.system.render_template('videoalpha.html', { 'youtube_streams': _create_youtube_string(self), 'id': self.location.html_id(), 'sub': self.sub, 'sources': sources, 'track': self.track, 'display_name': self.display_name_with_default, # This won't work when we move to data that # isn't on the filesystem 'data_dir': getattr(self, 'data_dir', None), 'caption_asset_path': caption_asset_path, 'show_captions': json.dumps(self.show_captions), 'start': self.start_time, 'end': self.end_time, 'autoplay': settings.MITX_FEATURES.get('AUTOPLAY_VIDEOS', True) })
def test_delete_image_type_asset(self): """ Tests deletion of image type asset """ image_asset = self.get_sample_asset(self.asset_name, asset_type="image") thumbnail_image_asset = self.get_sample_asset('delete_test_thumbnail', asset_type="image") # upload image response = self.client.post(self.url, {"name": "delete_image_test", "file": image_asset}) self.assertEquals(response.status_code, 200) uploaded_image_url = json.loads(response.content)['asset']['url'] # upload image thumbnail response = self.client.post(self.url, {"name": "delete_image_thumb_test", "file": thumbnail_image_asset}) self.assertEquals(response.status_code, 200) thumbnail_url = json.loads(response.content)['asset']['url'] thumbnail_location = StaticContent.get_location_from_path(thumbnail_url) image_asset_location = AssetLocation.from_deprecated_string(uploaded_image_url) content = contentstore().find(image_asset_location) content.thumbnail_location = thumbnail_location contentstore().save(content) with mock.patch('opaque_keys.edx.locator.CourseLocator.make_asset_key') as mock_asset_key: mock_asset_key.return_value = thumbnail_location test_url = reverse_course_url( 'assets_handler', self.course.id, kwargs={'asset_key_string': unicode(uploaded_image_url)}) resp = self.client.delete(test_url, HTTP_ACCEPT="application/json") self.assertEquals(resp.status_code, 204)
def course_info_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None): """ GET html: return html for editing the course info handouts and updates. """ __, course_module = _get_locator_and_course(package_id, branch, version_guid, block, request.user) if "text/html" in request.META.get("HTTP_ACCEPT", "text/html"): handouts_old_location = course_module.location.replace(category="course_info", name="handouts") handouts_locator = loc_mapper().translate_location( course_module.location.course_id, handouts_old_location, False, True ) update_location = course_module.location.replace(category="course_info", name="updates") update_locator = loc_mapper().translate_location(course_module.location.course_id, update_location, False, True) return render_to_response( "course_info.html", { "context_course": course_module, "updates_url": update_locator.url_reverse("course_info_update/"), "handouts_locator": handouts_locator, "base_asset_url": StaticContent.get_base_url_path_for_course_assets(course_module.location) + "/", }, ) else: return HttpResponseBadRequest("Only supports html requests")
def test_get_location_from_path(self): asset_location = StaticContent.get_location_from_path(u'/c4x/a/b/asset/images_course_image.jpg') self.assertEqual( AssetLocator(CourseLocator(u'a', u'b', None, deprecated=True), u'asset', u'images_course_image.jpg', deprecated=True), asset_location )
def test_images_upload(self): # http://www.django-rest-framework.org/api-guide/parsers/#fileuploadparser course_run = CourseFactory() expected_filename = 'course_image.png' content_key = StaticContent.compute_location(course_run.id, expected_filename) assert course_run.course_image != expected_filename try: contentstore().find(content_key) self.fail('No image should be associated with a new course run.') except NotFoundError: pass url = reverse('api:v1:course_run-images', kwargs={'pk': str(course_run.id)}) # PNG. Single black pixel content = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90wS' \ b'\xde\x00\x00\x00\x0cIDATx\x9cc```\x00\x00\x00\x04\x00\x01\xf6\x178U\x00\x00\x00\x00IEND\xaeB`\x82' # We are intentionally passing the incorrect JPEG extension here upload = SimpleUploadedFile('card_image.jpg', content, content_type='image/png') response = self.client.post(url, {'card_image': upload}, format='multipart') assert response.status_code == 200 course_run = modulestore().get_course(course_run.id) assert course_run.course_image == expected_filename expected = {'card_image': RequestFactory().get('').build_absolute_uri(course_image_url(course_run))} assert response.data == expected # There should now be an image stored contentstore().find(content_key)
def test_fail_downloading_subs(self, mock_get): mock_get.return_value = Mock(status_code=404, text='Error 404') bad_youtube_sub = 'BAD_YOUTUBE_ID2' self.clear_sub_content(bad_youtube_sub) with self.assertRaises(transcripts_utils.GetTranscriptsFromYouTubeException): transcripts_utils.download_youtube_subs(bad_youtube_sub, self.course, settings) # Check asset status after import of transcript. filename = 'subs_{0}.srt.sjson'.format(bad_youtube_sub) content_location = StaticContent.compute_location( self.course.id, filename ) with self.assertRaises(NotFoundError): contentstore().find(content_location) self.clear_sub_content(bad_youtube_sub)
def test_downloading_subs_using_transcript_name(self, mock_get): """ Download transcript using transcript name in url """ good_youtube_sub = 'good_id_2' self.clear_sub_content(good_youtube_sub) transcripts_utils.download_youtube_subs(good_youtube_sub, self.course, settings) mock_get.assert_any_call( 'http://video.google.com/timedtext', params={'lang': 'en', 'v': 'good_id_2', 'name': 'Custom'} ) # Check asset status after import of transcript. filename = 'subs_{0}.srt.sjson'.format(good_youtube_sub) content_location = StaticContent.compute_location(self.course.id, filename) self.assertTrue(contentstore().find(content_location)) self.clear_sub_content(good_youtube_sub)
def _get_file_content_and_path(file_metadata, course_key): """returns contents of the uploaded file and path for temporary uploaded file""" content_location = StaticContent.compute_location( course_key, file_metadata['filename']) upload_file = file_metadata['upload_file'] file_can_be_chunked = upload_file.multiple_chunks() static_content_partial = partial(StaticContent, content_location, file_metadata['filename'], file_metadata['mime_type']) if file_can_be_chunked: content = static_content_partial(upload_file.chunks()) temporary_file_path = upload_file.temporary_file_path() else: content = static_content_partial(upload_file.read()) temporary_file_path = None return content, temporary_file_path
def test_canonical_asset_path_with_c4x_style_assets( self, base_url, start, expected, mongo_calls): exts = ['.html', '.tm'] prefix = 'old' base_c4x_block = 'c4x/a/b/asset' adjusted_c4x_block = base_c4x_block encoded_c4x_block = quote('/' + base_c4x_block + '/') encoded_base_url = quote('//' + base_url) encoded_base_c4x_block = encoded_c4x_block start = start.format(prfx=prefix, encoded_base_url=encoded_base_url, c4x=base_c4x_block, encoded_c4x=encoded_c4x_block) # Adjust for content digest. This gets dicey quickly and we have to order our steps: # - replace format markets because they have curly braces # - encode Unicode characters to percent-encoded # - finally shove back in our regex patterns digest = CanonicalContentTest.get_content_digest_for_asset_path( prefix, start) if digest: adjusted_c4x_block = 'assets/courseware/VMARK/HMARK/c4x/a/b/asset' encoded_c4x_block = quote('/' + adjusted_c4x_block + '/') expected = expected.format( prfx=prefix, encoded_base_url=encoded_base_url, base_c4x=base_c4x_block, c4x=adjusted_c4x_block, encoded_c4x=encoded_c4x_block, encoded_base_c4x=encoded_base_c4x_block, ) expected = encode_unicode_characters_in_url(expected) expected = expected.replace('VMARK', r'v[\d]') expected = expected.replace('HMARK', '[a-f0-9]{32}') expected = expected.replace('+', r'\+').replace('?', r'\?') with check_mongo_calls(mongo_calls): asset_path = StaticContent.get_canonicalized_asset_path( self.courses[prefix].id, start, base_url, exts) assert re.match(expected, asset_path) is not None
def rewrite_nonportable_content_links(source_course_id, dest_course_id, text): """ Does a regex replace on non-portable links: /c4x/<org>/<course>/asset/<name> -> /static/<name> /jump_to/i4x://<org>/<course>/<category>/<name> -> /jump_to_id/<id> """ course_id_dict = Location.parse_course_id(source_course_id) course_id_dict['tag'] = 'i4x' course_id_dict['category'] = 'course' def portable_asset_link_subtitution(match): quote = match.group('quote') rest = match.group('rest') return quote + '/static/' + rest + quote def portable_jump_to_link_substitution(match): quote = match.group('quote') rest = match.group('rest') return quote + '/jump_to_id/' + rest + quote def generic_courseware_link_substitution(match): parts = Location.parse_course_id(dest_course_id) parts['quote'] = match.group('quote') parts['rest'] = match.group('rest') return u'{quote}/courses/{org}/{course}/{name}/{rest}{quote}'.format( **parts) course_location = Location(course_id_dict) # NOTE: ultimately link updating is not a hard requirement, so if something blows up with # the regex subsitution, log the error and continue try: c4x_link_base = u'{0}/'.format( StaticContent.get_base_url_path_for_course_assets(course_location)) text = re.sub(_prefix_only_url_replace_regex(c4x_link_base), portable_asset_link_subtitution, text) except Exception, e: logging.warning( "Error going regex subtituion %r on text = %r.\n\nError msg = %s", c4x_link_base, text, str(e))
def replace_static_url(match): original = match.group(0) prefix = match.group('prefix') quote = match.group('quote') rest = match.group('rest') # Don't mess with things that end in '?raw' if rest.endswith('?raw'): return original # In debug mode, if we can find the url as is, if settings.DEBUG and finders.find(rest, True): return original # if we're running with a MongoBacked store course_namespace is not None, then use studio style urls elif course_namespace is not None and not isinstance( modulestore(), XMLModuleStore): # first look in the static file pipeline and see if we are trying to reference # a piece of static content which is in the mitx repo (e.g. JS associated with an xmodule) if staticfiles_storage.exists(rest): url = staticfiles_storage.url(rest) else: # if not, then assume it's courseware specific content and then look in the # Mongo-backed database url = StaticContent.convert_legacy_static_url( rest, course_namespace) # Otherwise, look the file up in staticfiles_storage, and append the data directory if needed else: course_path = "/".join((data_directory, rest)) try: if staticfiles_storage.exists(rest): url = staticfiles_storage.url(rest) else: url = staticfiles_storage.url(course_path) # And if that fails, assume that it's course content, and add manually data directory except Exception as err: log.warning( "staticfiles_storage couldn't find path {0}: {1}".format( rest, str(err))) url = "".join([prefix, course_path]) return "".join([quote, url, quote])
def test_delete_image_type_asset(self): """ Tests deletion of image type asset """ image_asset = self.get_sample_asset(self.asset_name, asset_type="image") thumbnail_image_asset = self.get_sample_asset('delete_test_thumbnail', asset_type="image") # upload image response = self.client.post(self.url, { "name": "delete_image_test", "file": image_asset }) self.assertEqual(response.status_code, 200) uploaded_image_url = json.loads( response.content.decode('utf-8'))['asset']['url'] # upload image thumbnail response = self.client.post(self.url, { "name": "delete_image_thumb_test", "file": thumbnail_image_asset }) self.assertEqual(response.status_code, 200) thumbnail_url = json.loads( response.content.decode('utf-8'))['asset']['url'] thumbnail_location = StaticContent.get_location_from_path( thumbnail_url) image_asset_location = AssetKey.from_string(uploaded_image_url) content = contentstore().find(image_asset_location) content.thumbnail_location = thumbnail_location contentstore().save(content) with mock.patch('opaque_keys.edx.locator.CourseLocator.make_asset_key' ) as mock_asset_key: mock_asset_key.return_value = thumbnail_location test_url = reverse_course_url( 'assets_handler', self.course.id, kwargs={'asset_key_string': six.text_type(uploaded_image_url)}) resp = self.client.delete(test_url, HTTP_ACCEPT="application/json") self.assertEqual(resp.status_code, 204)
def test_asset_and_course_deletion(self): course_run = CourseFactory() self.assertIsNotNone(modulestore().get_course(course_run.id)) store = contentstore() asset_key = course_run.id.make_asset_key('asset', 'test.txt') content = StaticContent(asset_key, 'test.txt', 'plain/text', b'test data') store.save(content) __, asset_count = store.get_all_content_for_course(course_run.id) assert asset_count == 1 with mock.patch(self.YESNO_PATCH_LOCATION) as patched_yes_no: patched_yes_no.return_value = True call_command('delete_course', str(course_run.id)) assert modulestore().get_course(course_run.id) is None __, asset_count = store.get_all_content_for_course(course_run.id) assert asset_count == 1
def course_info_handler(request, course_key_string): """ GET html: return html for editing the course info handouts and updates. """ course_key = CourseKey.from_string(course_key_string) course_module = _get_course_module(course_key, request.user) if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'): return render_to_response( 'course_info.html', { 'context_course': course_module, 'updates_url': reverse_course_url('course_info_update_handler', course_key), 'handouts_locator': course_key.make_usage_key('course_info', 'handouts'), 'base_asset_url': StaticContent.get_base_url_path_for_course_assets(course_module.id) } ) else: return HttpResponseBadRequest("Only supports html requests")
def test_canonical_asset_path_with_c4x_style_assets( self, base_url, start, expected, mongo_calls): prefix = 'old' c4x_block = 'c4x/a/b/asset' encoded_c4x_block = quote_plus('/' + c4x_block + '/') encoded_base_url = quote_plus('//' + base_url) start = start.format(prefix=prefix, encoded_base_url=encoded_base_url, c4x=c4x_block, encoded_c4x=encoded_c4x_block) expected = expected.format(prefix=prefix, encoded_base_url=encoded_base_url, c4x=c4x_block, encoded_c4x=encoded_c4x_block) with check_mongo_calls(mongo_calls): asset_path = StaticContent.get_canonicalized_asset_path( self.courses[prefix].id, start, base_url) self.assertEqual(asset_path, expected)
def restore_asset_from_trashcan(location): ''' This method will restore an asset which got soft deleted and put back in the original course ''' trash = contentstore('trashcan') store = contentstore() loc = StaticContent.get_location_from_path(location) content = trash.find(loc) # ok, save the content into the courseware store.save(content) # see if there is a thumbnail as well, if so move that as well if content.thumbnail_location is not None: try: thumbnail_content = trash.find(content.thumbnail_location) store.save(thumbnail_content) except Exception: # lint-amnesty, pylint: disable=broad-except pass # OK if this is left dangling
def copy_or_rename_transcript(new_name, old_name, item, delete_old=False, user=None): """ Renames `old_name` transcript file in storage to `new_name`. If `old_name` is not found in storage, raises `NotFoundError`. If `delete_old` is True, removes `old_name` files from storage. """ filename = u'subs_{0}.srt.sjson'.format(old_name) content_location = StaticContent.compute_location(item.location.course_key, filename) transcripts = contentstore().find(content_location).data save_subs_to_store(json.loads(transcripts), new_name, item) item.sub = new_name item.save_with_metadata(user) if delete_old: remove_subs_from_store(old_name, item)
def get_asset_content_from_path(course_key, asset_path): """ Locate the given asset content, load it into memory, and return it. Returns None if the asset is not found. """ try: from xmodule.contentstore.content import StaticContent from xmodule.assetstore.assetmgr import AssetManager from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.exceptions import NotFoundError except ImportError as exc: raise EdXPlatformImportError(exc) try: asset_key = StaticContent.get_asset_key_from_path( course_key, asset_path) return AssetManager.find(asset_key) except (ItemNotFoundError, NotFoundError) as exc: return None
def rewrite_nonportable_content_links(source_course_id, dest_course_id, text): """ Does a regex replace on non-portable links: /c4x/<org>/<course>/asset/<name> -> /static/<name> /jump_to/i4x://<org>/<course>/<category>/<name> -> /jump_to_id/<id> """ org, course, run = source_course_id.split("/") dest_org, dest_course, dest_run = dest_course_id.split("/") def portable_asset_link_subtitution(match): quote = match.group('quote') rest = match.group('rest') return quote + '/static/' + rest + quote def portable_jump_to_link_substitution(match): quote = match.group('quote') rest = match.group('rest') return quote + '/jump_to_id/' + rest + quote def generic_courseware_link_substitution(match): quote = match.group('quote') rest = match.group('rest') dest_generic_courseware_lik_base = '/courses/{org}/{course}/{run}/'.format( org=dest_org, course=dest_course, run=dest_run) return quote + dest_generic_courseware_lik_base + rest + quote course_location = Location(['i4x', org, course, 'course', run]) # NOTE: ultimately link updating is not a hard requirement, so if something blows up with # the regex subsitution, log the error and continue try: c4x_link_base = '{0}/'.format( StaticContent.get_base_url_path_for_course_assets(course_location)) text = re.sub(_prefix_only_url_replace_regex(c4x_link_base), portable_asset_link_subtitution, text) except Exception, e: logging.warning( "Error going regex subtituion %r on text = %r.\n\nError msg = %s", c4x_link_base, text, str(e))
def test_images_upload(self): # http://www.django-rest-framework.org/api-guide/parsers/#fileuploadparser course_run = CourseFactory() expected_filename = 'course_image.png' content_key = StaticContent.compute_location(course_run.id, expected_filename) assert course_run.course_image != expected_filename try: contentstore().find(content_key) self.fail('No image should be associated with a new course run.') except NotFoundError: pass url = reverse('api:v1:course_run-images', kwargs={'pk': str(course_run.id)}) # PNG. Single black pixel content = b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90wS' \ b'\xde\x00\x00\x00\x0cIDATx\x9cc```\x00\x00\x00\x04\x00\x01\xf6\x178U\x00\x00\x00\x00IEND\xaeB`\x82' # We are intentionally passing the incorrect JPEG extension here upload = SimpleUploadedFile('card_image.jpg', content, content_type='image/png') response = self.client.post(url, {'card_image': upload}, format='multipart') assert response.status_code == 200 course_run = modulestore().get_course(course_run.id) assert course_run.course_image == expected_filename expected = { 'card_image': RequestFactory().get('').build_absolute_uri( course_image_url(course_run)) } assert response.data == expected # There should now be an image stored contentstore().find(content_key)
def student_state(self): """ Returns a JSON serializable representation of student's state for rendering in client view. """ submission = self.get_submission() uploaded = None if submission: if 'user_response' in submission['answer'].keys(): uploaded = {"user_response": submission['answer']['user_response']} elif 'filename' in submission['answer'].keys(): uploaded = {"filename": submission['answer']['filename']} if self.annotated_sha1: annotated = {"filename": force_text(self.annotated_filename)} else: annotated = None score = self.score if score is not None: graded = {'score': score, 'comment': force_text(self.comment)} else: graded = None if self.answer_available(): solution = self.runtime.replace_urls(force_text(self.solution)) else: solution = '' # pylint: disable=no-member return { "submission_type": force_text(self.submission_type), "display_name": force_text(self.display_name), "uploaded": uploaded, "annotated": annotated, "graded": graded, "max_score": self.max_score(), "upload_allowed": self.upload_allowed(submission_data=submission), "solution": solution, "base_asset_url": StaticContent.get_base_url_path_for_course_assets(self.location.course_key), }
def download_transcripts(request): """ Passes to user requested transcripts file. Raises Http404 if unsuccessful. """ locator = request.GET.get('locator') if not locator: log.debug('GET data without "locator" property.') raise Http404 try: item = _get_item(request, request.GET) except (InvalidKeyError, ItemNotFoundError): log.debug("Can't find item by locator.") raise Http404 subs_id = request.GET.get('subs_id') if not subs_id: log.debug('GET data without "subs_id" property.') raise Http404 if item.category != 'video': log.debug('transcripts are supported only for video" modules.') raise Http404 filename = 'subs_{0}.srt.sjson'.format(subs_id) content_location = StaticContent.compute_location(item.location.course_key, filename) try: sjson_transcripts = contentstore().find(content_location) log.debug("Downloading subs for %s id", subs_id) str_subs = generate_srt_from_sjson(json.loads(sjson_transcripts.data), speed=1.0) if not str_subs: log.debug('generate_srt_from_sjson produces no subtitles') raise Http404 response = HttpResponse(str_subs, content_type='application/x-subrip') response['Content-Disposition'] = 'attachment; filename="{0}.srt"'.format(subs_id) return response except NotFoundError: log.debug("Can't find content in storage for %s subs", subs_id) raise Http404
def read_from_contentstore(course_key, path): """ Loads a file directly from the course's content store. """ contents = None try: from xmodule.contentstore.content import StaticContent from xmodule.contentstore.django import contentstore from opaque_keys.edx.locator import CourseLocator except ImportError: # We're not running under edx-platform, so ignore. pass else: if isinstance(course_key, six.text_type): course_key = CourseLocator.from_string(course_key) loc = StaticContent.compute_location(course_key, path) asset = contentstore().find(loc) contents = asset.data return contents
def _upload_file(file, location): filename = 'subs_{}.srt.sjson'.format(_get_subs_id(file.name)) mime_type = file.content_type content_location = StaticContent.compute_location(location.org, location.course, filename) sc_partial = partial(StaticContent, content_location, filename, mime_type) content = sc_partial(file.read()) (thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail( content, tempfile_path=None) del_cached_content(thumbnail_location) if thumbnail_content is not None: content.thumbnail_location = thumbnail_location contentstore().save(content) del_cached_content(content.location)
def course_video_url(course): """Try to look up the video url for the course. If it'is not found, log an error and return the dead link""" if course.static_asset_path or modulestore().get_modulestore_type( course.id) == ModuleStoreEnum.Type.xml: # If we are a static course with the course_image attribute # set different than the default, return that path so that # courses can use custom course image paths, otherwise just # return the default static path. url = '/static/' + (course.static_asset_path or getattr(course, 'data_dir', '')) if hasattr( course, 'course_video' ) and course.course_video != course.fields['course_video'].default: url += '/' + course.course_video else: url += '/videos/course_video.mp4' else: loc = StaticContent.compute_location(course.id, course.course_video) url = loc.to_deprecated_string() return url
def course_image_url(course): """Try to look up the image url for the course. If it's not found, log an error and return the dead link""" if course.static_asset_path or modulestore().get_modulestore_type( course.id) == XML_MODULESTORE_TYPE: # If we are a static course with the course_image attribute # set different than the default, return that path so that # courses can use custom course image paths, otherwise just # return the default static path. url = '/static/' + (course.static_asset_path or getattr(course, 'data_dir', '')) if hasattr( course, 'course_image' ) and course.course_image != course.fields['course_image'].default: url += '/' + course.course_image else: url += '/images/course_image.jpg' else: loc = StaticContent.compute_location(course.id, course.course_image) url = loc.to_deprecated_string() return url
def update_course_run_asset(course_key, upload_file): filename = upload_file.name mime_type = upload_file.content_type size = get_file_size(upload_file) max_size_in_mb = settings.MAX_ASSET_UPLOAD_FILE_SIZE_IN_MB max_file_size_in_bytes = max_size_in_mb * 1000**2 if size > max_file_size_in_bytes: msg = 'File {filename} exceeds the maximum size of {max_size_in_mb} MB.'.format( filename=filename, max_size_in_mb=max_size_in_mb) raise AssetSizeTooLargeException(msg) content_loc = StaticContent.compute_location(course_key, filename) chunked = upload_file.multiple_chunks() sc_partial = partial(StaticContent, content_loc, filename, mime_type) if chunked: content = sc_partial(upload_file.chunks()) tempfile_path = upload_file.temporary_file_path() else: content = sc_partial(upload_file.read()) tempfile_path = None # Verify a thumbnail can be created (thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail( content, tempfile_path=tempfile_path) # delete cached thumbnail even if one couldn't be created this time (else the old thumbnail will continue to show) del_cached_content(thumbnail_location) # now store thumbnail location only if we could create it if thumbnail_content is not None: content.thumbnail_location = thumbnail_location # then commit the content contentstore().save(content) del_cached_content(content.location) return content
def get_transcript(self, subs_id): ''' Returns transcript without timecodes. Args: `subs_id`: str, subtitles id Raises: - NotFoundError if cannot find transcript file in storage. - ValueError if transcript file is incorrect JSON. - KeyError if transcript file has incorrect format. ''' filename = 'subs_{0}.srt.sjson'.format(subs_id) content_location = StaticContent.compute_location( self.location.org, self.location.course, filename ) data = contentstore().find(content_location).data text = json.loads(data)['text'] return HTMLParser().unescape("\n".join(text))
def test_pdf_asset(self): module_store = modulestore('direct') _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=contentstore(), verbose=True) course = course_items[0] location = loc_mapper().translate_location(course.location.course_id, course.location, False, True) url = location.url_reverse('assets/', '') # Test valid contentType for pdf asset (textbook.pdf) resp = self.client.get(url, HTTP_ACCEPT='application/json') self.assertContains(resp, "/c4x/edX/toy/asset/textbook.pdf") asset_location = StaticContent.get_location_from_path( '/c4x/edX/toy/asset/textbook.pdf') content = contentstore().find(asset_location) # Check after import textbook.pdf has valid contentType ('application/pdf') # Note: Actual contentType for textbook.pdf in asset.json is 'text/pdf' self.assertEqual(content.content_type, 'application/pdf')
def test_fail_downloading_subs(self): bad_youtube_subs = { 0.5: 'BAD_YOUTUBE_ID1', 1.0: 'BAD_YOUTUBE_ID2', 2.0: 'BAD_YOUTUBE_ID3' } self.clear_subs_content(bad_youtube_subs) with self.assertRaises( transcripts_utils.GetTranscriptsFromYouTubeException): transcripts_utils.download_youtube_subs(bad_youtube_subs, self.course) # Check assets status after importing subtitles. for subs_id in bad_youtube_subs.values(): filename = 'subs_{0}.srt.sjson'.format(subs_id) content_location = StaticContent.compute_location( self.org, self.number, filename) with self.assertRaises(NotFoundError): contentstore().find(content_location) self.clear_subs_content(bad_youtube_subs)
def course_info_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None): """ GET html: return html for editing the course info handouts and updates. """ __, course_module = _get_locator_and_course(package_id, branch, version_guid, block, request.user) if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'): handouts_old_location = course_module.location.replace( category='course_info', name='handouts') handouts_locator = loc_mapper().translate_location( course_module.location.course_id, handouts_old_location, False, True) update_location = course_module.location.replace( category='course_info', name='updates') update_locator = loc_mapper().translate_location( course_module.location.course_id, update_location, False, True) return render_to_response( 'course_info.html', { 'context_course': course_module, 'updates_url': update_locator.url_reverse('course_info_update/'), 'handouts_locator': handouts_locator, 'base_asset_url': StaticContent.get_base_url_path_for_course_assets( course_module.location) + '/' }) else: return HttpResponseBadRequest("Only supports html requests")
def test_success_downloading_chinese_transcripts(self): # Disabled 11/14/13 # This test is flakey because it performs an HTTP request on an external service # Re-enable when `requests.get` is patched using `mock.patch` raise SkipTest good_youtube_sub = 'j_jEn79vS3g' # Chinese, utf-8 self.clear_sub_content(good_youtube_sub) # Check transcripts_utils.GetTranscriptsFromYouTubeException not thrown transcripts_utils.download_youtube_subs(good_youtube_sub, self.course, settings) # Check assets status after importing subtitles. for subs_id in good_youtube_subs.values(): filename = 'subs_{0}.srt.sjson'.format(subs_id) content_location = StaticContent.compute_location( self.course.id, filename) self.assertTrue(contentstore().find(content_location)) self.clear_sub_content(good_youtube_sub)
def student_view(self, context=None): """ The student view of the MarkdownXBlock. """ if self.filename: # These can only be imported when the XBlock is running on the LMS # or CMS. Do it at runtime so that the workbench is usable for # regular XML content. from xmodule.contentstore.content import StaticContent from xmodule.contentstore.django import contentstore from xmodule.exceptions import NotFoundError from opaque_keys import InvalidKeyError try: course_id = self.xmodule_runtime.course_id loc = StaticContent.compute_location(course_id, self.filename) asset = contentstore().find(loc) content = asset.data except (NotFoundError, InvalidKeyError): pass else: content = self.content html_content = "" if content: html_content = markdown2.markdown(content, extras=self.extras) # Render the HTML template context = {'content': html_content} html = loader.render_template('templates/main.html', context) frag = Fragment(html) if "fenced-code-blocks" in self.extras: frag.add_css_url( self.runtime.local_resource_url(self, 'public/css/pygments.css')) return frag
def test_subs_uploading_with_byte_order_mark(self): """ Test uploading subs containing BOM(Byte Order Mark), e.g. U+FEFF """ filedata = textwrap.dedent(""" 1 00:00:10,500 --> 00:00:13,000 Test ufeff characters 2 00:00:15,000 --> 00:00:18,000 At the left we can see... """).encode('utf-8-sig') # Verify that ufeff character is in filedata. self.assertIn("ufeff", filedata) self.ufeff_srt_file.write(filedata) self.ufeff_srt_file.seek(0) link = reverse('upload_transcripts') filename = os.path.splitext(os.path.basename(self.ufeff_srt_file.name))[0] resp = self.client.post(link, { 'locator': self.video_usage_key, 'transcript-file': self.ufeff_srt_file, 'video_list': json.dumps([{ 'type': 'html5', 'video': filename, 'mode': 'mp4', }]) }) self.assertEqual(resp.status_code, 200) content_location = StaticContent.compute_location( self.course.id, 'subs_{0}.srt.sjson'.format(filename)) self.assertTrue(contentstore().find(content_location)) subs_text = json.loads(contentstore().find(content_location).data).get('text') self.assertIn("Test ufeff characters", subs_text)
def get_pgreport_csv(course_id): """Get progress of students.""" course_key = get_coursekey(course_id) location = StaticContent.compute_location(course_key, "progress_students.csv.gz") store = contentstore() try: gzipfile = StringIO.StringIO() content = store.find(location, throw_on_not_found=True, as_stream=True) for gzipdata in content.stream_data(): gzipfile.write(gzipdata) gzipfile.seek(0) gzipcsv = gzip.GzipFile(fileobj=gzipfile, mode='rb') for csvrow in gzipcsv.readlines(): print csvrow, gzipcsv.close() except NotFoundError as e: log.warn(" * Csv does not exists: {}".format(e)) finally: gzipfile.close()