def test_thumbnail_none(self): # We had a bug where a thumbnail location of None was getting transformed into a Location tuple, with # all elements being None. It is important that the location be just None for rendering. content = StaticContent('loc', 'name', 'content_type', 'data', None, None, None) self.assertIsNone(content.thumbnail_location) content = StaticContent('loc', 'name', 'content_type', 'data') self.assertIsNone(content.thumbnail_location)
def create_image(cls, prefix, dimensions, color, name, locked=False): """ Creates an image. Args: prefix: the prefix to use e.g. split vs mongo dimensions: tuple of (width, height) color: the background color of the image name: the name of the image; can be a format string locked: whether or not the asset should be locked Returns: StaticContent: the StaticContent object for the created image """ new_image = Image.new('RGB', dimensions, color) new_buf = BytesIO() new_image.save(new_buf, format='png') new_buf.seek(0) new_name = name.format(prefix) new_key = StaticContent.compute_location(cls.courses[prefix].id, new_name) new_content = StaticContent(new_key, new_name, 'image/png', new_buf.getvalue(), locked=locked) contentstore().save(new_content) return new_content
def create_arbitrary_content(cls, prefix, name, locked=False): """ Creates an arbitrary piece of content with a fixed body, for when content doesn't matter. Args: prefix: the prefix to use e.g. split vs mongo name: the name of the content; can be a format string locked: whether or not the asset should be locked Returns: StaticContent: the StaticContent object for the created content """ new_buf = BytesIO(b'testingggggggggggg') new_name = name.format(prefix) new_key = StaticContent.compute_location(cls.courses[prefix].id, new_name) new_content = StaticContent(new_key, new_name, 'application/octet-stream', new_buf.getvalue(), locked=locked) contentstore().save(new_content) return new_content
def import_static_content(modules, course_loc, course_data_path, static_content_store, target_location_namespace, subpath='static', verbose=False): remap_dict = {} # now import all static assets static_dir = course_data_path / subpath verbose = True for dirname, dirnames, filenames in os.walk(static_dir): for filename in filenames: try: content_path = os.path.join(dirname, filename) if verbose: log.debug( 'importing static content {0}...'.format(content_path)) fullname_with_subpath = content_path.replace( static_dir, '') # strip away leading path from the name if fullname_with_subpath.startswith('/'): fullname_with_subpath = fullname_with_subpath[1:] content_loc = StaticContent.compute_location( target_location_namespace.org, target_location_namespace.course, fullname_with_subpath) mime_type = mimetypes.guess_type(filename)[0] with open(content_path, 'rb') as f: data = f.read() content = StaticContent(content_loc, filename, mime_type, data, import_path=fullname_with_subpath) # first let's save a thumbnail so we can get back a thumbnail # location (thumbnail_content, thumbnail_location ) = static_content_store.generate_thumbnail(content) if thumbnail_content is not None: content.thumbnail_location = thumbnail_location # then commit the content static_content_store.save(content) # store the remapping information which will be needed to # subsitute in the module data remap_dict[fullname_with_subpath] = content_loc.name except: raise return remap_dict
def create_pgreport_csv(course_id, update_state=None): """Create CSV of progress to MongoDB.""" course_key = get_coursekey(course_id) try: gzipfile = StringIO.StringIO() gzipcsv = gzip.GzipFile(filename="progress_students.csv.gz", mode='wb', fileobj=gzipfile) writer = csv.writer(gzipcsv, encoding='utf-8') progress = ProgressReport(course_key, update_state) for row in progress.yield_students_progress(): writer.writerow(row) finally: gzipcsv.close() try: content_loc = StaticContent.compute_location(course_key, gzipcsv.name) content = StaticContent(loc=content_loc, name=gzipcsv.name, content_type="application/x-gzip", data=gzipfile.getvalue()) contentstore().save(content) del_cached_content(content_loc) except GridFSError as e: store.delete(content_id) log.error(" * GridFS Error: {}".format(e)) raise finally: gzipfile.close()
def test_happy_path(self, modulestore_type, create_after_overview): """ What happens when everything works like we expect it to. If `create_after_overview` is True, we will temporarily disable thumbnail creation so that the initial CourseOverview is created without an image_set, and the CourseOverviewImageSet is created afterwards. If `create_after_overview` is False, we'll create the CourseOverviewImageSet at the same time as the CourseOverview. """ # Create a real (oversized) image... image = Image.new("RGB", (800, 400), "blue") image_buff = StringIO() image.save(image_buff, format="JPEG") image_buff.seek(0) image_name = "big_course_image.jpeg" with self.store.default_store(modulestore_type): course = CourseFactory.create( default_store=modulestore_type, course_image=image_name ) # Save a real image here... course_image_asset_key = StaticContent.compute_location(course.id, course.course_image) course_image_content = StaticContent(course_image_asset_key, image_name, 'image/jpeg', image_buff) contentstore().save(course_image_content) # If create_after_overview is True, disable thumbnail generation so # that the CourseOverview object is created and saved without an # image_set at first (it will be lazily created later). if create_after_overview: self.set_config(enabled=False) # Now generate the CourseOverview... course_overview = CourseOverview.get_from_id(course.id) # If create_after_overview is True, no image_set exists yet. Verify # that, then switch config back over to True and it should lazily # create the image_set on the next get_from_id() call. if create_after_overview: self.assertFalse(hasattr(course_overview, 'image_set')) self.set_config(enabled=True) course_overview = CourseOverview.get_from_id(course.id) self.assertTrue(hasattr(course_overview, 'image_set')) image_urls = course_overview.image_urls config = CourseOverviewImageConfig.current() # Make sure the thumbnail names come out as expected... self.assertTrue(image_urls['raw'].endswith('big_course_image.jpeg')) self.assertTrue(image_urls['small'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.small))) self.assertTrue(image_urls['large'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.large))) # Now make sure our thumbnails are of the sizes we expect... for image_url, expected_size in [(image_urls['small'], config.small), (image_urls['large'], config.large)]: image_key = StaticContent.get_location_from_path(image_url) image_content = AssetManager.find(image_key) image = Image.open(StringIO(image_content.data)) self.assertEqual(image.size, expected_size)
def test_different_resolutions(self, src_dimensions): """ Test various resolutions of images to make thumbnails of. Note that our test sizes are small=(200, 100) and large=(400, 200). 1. Images should won't be blown up if it's too small, so a (100, 50) resolution image will remain (100, 50). 2. However, images *will* be converted using our format and quality settings (JPEG, 75% -- the PIL default). This is because images with relatively small dimensions not compressed properly. 3. Image thumbnail naming will maintain the naming convention of the target resolution, even if the image was not actually scaled to that size (i.e. it was already smaller). This is mostly because it's simpler to be consistent, but it also lets us more easily tell which configuration a thumbnail was created under. """ # Create a source image... image = Image.new("RGB", src_dimensions, "blue") image_buff = StringIO() image.save(image_buff, format="PNG") image_buff.seek(0) image_name = "src_course_image.png" course = CourseFactory.create(course_image=image_name) # Save the image to the contentstore... course_image_asset_key = StaticContent.compute_location(course.id, course.course_image) course_image_content = StaticContent(course_image_asset_key, image_name, 'image/png', image_buff) contentstore().save(course_image_content) # Now generate the CourseOverview... config = CourseOverviewImageConfig.current() course_overview = CourseOverview.get_from_id(course.id) image_urls = course_overview.image_urls for image_url, target in [(image_urls['small'], config.small), (image_urls['large'], config.large)]: image_key = StaticContent.get_location_from_path(image_url) image_content = AssetManager.find(image_key) image = Image.open(StringIO(image_content.data)) # Naming convention for thumbnail self.assertTrue(image_url.endswith('src_course_image-png-{}x{}.jpg'.format(*target))) # Actual thumbnail data src_x, src_y = src_dimensions target_x, target_y = target image_x, image_y = image.size # I'm basically going to assume the image library knows how to do # the right thing in terms of handling aspect ratio. We're just # going to make sure that small images aren't blown up, and that # we never exceed our target sizes self.assertLessEqual(image_x, target_x) self.assertLessEqual(image_y, target_y) if src_x < target_x and src_y < target_y: self.assertEqual(src_x, image_x) self.assertEqual(src_y, image_y)
def _upload_file(subs_file, location, filename): mime_type = subs_file.content_type content_location = StaticContent.compute_location( location.course_key, filename ) content = StaticContent(content_location, filename, mime_type, subs_file.read()) contentstore().save(content) del_cached_content(content.location)
def upload_file(filename, location): path = os.path.join(TEST_ROOT, 'uploads/', filename) f = open(os.path.abspath(path)) mime_type = "application/json" content_location = StaticContent.compute_location(location.course_key, filename) content = StaticContent(content_location, filename, mime_type, f.read()) contentstore().save(content) del_cached_content(content.location)
def save_asset(self, filename, asset_key, displayname, locked): """ Load and save the given file. """ with open("{}/static/{}".format(DATA_DIR, filename), "rb") as f: content = StaticContent( asset_key, displayname, mimetypes.guess_type(filename)[0], f.read(), locked=locked ) self.contentstore.save(content)
def _create_fake_images(self, asset_keys): """ Creates fake image files for a list of asset_keys. """ for asset_key_string in asset_keys: asset_key = AssetKey.from_string(asset_key_string) content = StaticContent( asset_key, "Fake asset", "image/png", "data", ) contentstore().save(content)
def save_to_store(content, name, mime_type, location): """ Save named content to store by location. Returns location of saved content. """ content_location = Transcript.asset_location(location, name) content = StaticContent(content_location, name, mime_type, content) contentstore().save(content) return content_location
def setUp(self): super(TestCourseMixin, self).setUp() with modulestore().default_store(ModuleStoreEnum.Type.split): self.course = SampleCourseFactory.create( block_info_tree=TEST_COURSE) # And upload the course static asssets: asset_key = StaticContent.compute_location(self.course.id, 'sample_handout.txt') content = StaticContent(asset_key, "Fake asset", "application/text", "test".encode('utf8')) contentstore().save(content) asset_key = StaticContent.compute_location(self.course.id, 'edx.svg') content = StaticContent( asset_key, "Fake image", "image/svg+xml", """ <svg viewBox="0 0 403 403" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" > <title>edX</title> <desc> the edX logo is comprised of a red letter e, grey d, and blue uppercase X, all slightly overlapping each other. The d is slightly transparent. </desc> <g transform="translate(0,100)"> <path id="e-path" stroke-width="1" stroke="none" fill="rgb(185, 0, 88)" fill-rule="evenodd" d="M32.1,127 H141.9 A71,71.5 0 1,0 137.3,143 H103 A42,42 0 0,1 32.1,127 M32.1,102.5 H112 A42,42 0 0,0 32.1,102.5"/> <path id="x-path" stroke-width="1" stroke="none" fill="rgb(0, 162, 228)" fill-rule="evenodd" d="M228,1 H302 V31 H286 L315,67 L344,31 H328 V1 H401.5 V31 H385 L335.2,92.4 L387.5,156.8 H401.5 V187 H328 V156.8 H346.5 L315,117.4 L283,156.8 H302.0 V187 H228.5 V156.8 H243 L294.3,92.4 L244,30.5 H228 V1"/> <path id="d-path" stroke-width="1" stroke="none" fill="rgb(55, 55, 60)" fill-rule="evenodd" opacity="0.55" d="M198.5,1 L248,1 V156.5 H269.8 V187 H217.5 V174 A71.7,71.7 0 1,1 218,55.5 V30.5 H198.5 V1 M218,114 A41,41.5 0 1,1 136.1,114 A40.5,40.5 0 1,1 218,114"/> </g> </svg> """.strip().encode('utf8')) contentstore().save(content) # And the video data + transcript must also be stored in edx-val for the video export to work: edxval_api.create_video(VIDEO_B_VAL_DATA) edxval_api.create_video_transcript(**VIDEO_B_SRT_TRANSCRIPT_DATA)
def test_space_in_asset_name_for_rerun_course(self): """ Tests check the scenario where one course which has an asset with percentage(%) in its name, it should re-run successfully. """ org = 'edX' course_number = 'CS101' course_run = '2015_Q1' display_name = 'rerun' fields = {'display_name': display_name} course_assets = set([u'subs_Introduction%20To%20New.srt.sjson'], ) # Create a course using split modulestore course = CourseFactory.create(org=org, number=course_number, run=course_run, display_name=display_name, default_store=ModuleStoreEnum.Type.split) # add an asset asset_key = course.id.make_asset_key( 'asset', 'subs_Introduction%20To%20New.srt.sjson') content = StaticContent( asset_key, 'Dummy assert', 'application/json', 'dummy data', ) contentstore().save(content) # Get & verify all assets of the course assets, count = contentstore().get_all_content_for_course(course.id) self.assertEqual(count, 1) self.assertEqual( set([asset['asset_key'].block_id for asset in assets]), course_assets) # rerun from split into split split_rerun_id = CourseLocator(org=org, course=course_number, run="2012_Q2") CourseRerunState.objects.initiated(course.id, split_rerun_id, self.user, fields['display_name']) result = rerun_course.delay(six.text_type(course.id), six.text_type(split_rerun_id), self.user.id, json.dumps(fields, cls=EdxJSONEncoder)) # Check if re-run was successful self.assertEqual(result.get(), "succeeded") rerun_state = CourseRerunState.objects.find_first( course_key=split_rerun_id) self.assertEqual(rerun_state.state, CourseRerunUIStateManager.State.SUCCEEDED)
def save_subs_to_store(self, subs, subs_id): """Save transcripts into `StaticContent`.""" filedata = json.dumps(subs, indent=2) mime_type = 'application/json' filename = 'subs_{0}.srt.sjson'.format(subs_id) content_location = StaticContent.compute_location(self.course.id, filename) content = StaticContent(content_location, filename, mime_type, filedata) contentstore().save(content) del_cached_content(content_location) return content_location
def _create_course_image(self, course, image_name): """ Creates a course image in contentstore. """ # Create a source image... image = Image.new('RGB', (800, 400), 'blue') image_buff = StringIO() image.save(image_buff, format='PNG') image_buff.seek(0) # Save the image to the contentstore... course_image_asset_key = StaticContent.compute_location(course.id, course.course_image) course_image_content = StaticContent(course_image_asset_key, image_name, 'image/png', image_buff) contentstore().save(course_image_content)
def verify_content_links(module, base_dir, static_content_store, link, remap_dict=None): if link.startswith('/static/'): # yes, then parse out the name path = link[len('/static/'):] static_pathname = base_dir / path if os.path.exists(static_pathname): try: content_loc = StaticContent.compute_location( module.location.org, module.location.course, path) filename = os.path.basename(path) mime_type = mimetypes.guess_type(filename)[0] with open(static_pathname, 'rb') as f: data = f.read() content = StaticContent(content_loc, filename, mime_type, data, import_path=path) # first let's save a thumbnail so we can get back a thumbnail # location (thumbnail_content, thumbnail_location ) = static_content_store.generate_thumbnail(content) if thumbnail_content is not None: content.thumbnail_location = thumbnail_location # then commit the content static_content_store.save(content) new_link = StaticContent.get_url_path_from_location( content_loc) if remap_dict is not None: remap_dict[link] = new_link return new_link except Exception, e: logging.exception( 'Skipping failed content load from {0}. Exception: {1}'. format(path, e))
def test_remove_assets(self): course_run = CourseFactory() store = contentstore() asset_key = course_run.id.make_asset_key('asset', 'test.txt') content = StaticContent(asset_key, 'test.txt', 'plain/text', b'test data') store.save(content) __, asset_count = store.get_all_content_for_course(course_run.id) assert asset_count == 1 with mock.patch(self.YESNO_PATCH_LOCATION, return_value=True): call_command('delete_course', str(course_run.id), '--remove-assets') __, asset_count = store.get_all_content_for_course(course_run.id) assert asset_count == 0
def upload_file(self, subs_file, location, filename): """ Upload a file in content store. Arguments: subs_file (File): pointer to file to be uploaded location (Locator): Item location filename (unicode): Name of file to be uploaded """ mime_type = subs_file.content_type content_location = StaticContent.compute_location( location.course_key, filename) content = StaticContent(content_location, filename, mime_type, subs_file.read()) contentstore().save(content)
def upload_file_to_course(course_key, contentstore, source_file, target_filename): ''' Uploads the given source file to the given course, and returns the content of the file. ''' asset_key = course_key.make_asset_key('asset', target_filename) with open(source_file, "rb") as f: file_contents = f.read() mimetype = guess_type(source_file)[0] content = StaticContent(asset_key, target_filename, mimetype, file_contents, locked=False) contentstore.save(content) return file_contents
def save_subs_to_store(subs, subs_id, item, language='en'): """ Save transcripts into `StaticContent`. Args: `subs_id`: str, subtitles id `item`: video module instance `language`: two chars str ('uk'), language of translation of transcripts Returns: location of saved subtitles. """ filedata = json.dumps(subs, indent=2) mime_type = 'application/json' filename = subs_filename(subs_id, language) content_location = Transcript.asset_location(item.location, filename) content = StaticContent(content_location, filename, mime_type, filedata) contentstore().save(content) return content_location
def test_asset_and_course_deletion(self): course_run = CourseFactory() self.assertIsNotNone(modulestore().get_course(course_run.id)) store = contentstore() asset_key = course_run.id.make_asset_key('asset', 'test.txt') content = StaticContent(asset_key, 'test.txt', 'plain/text', b'test data') store.save(content) __, asset_count = store.get_all_content_for_course(course_run.id) assert asset_count == 1 with mock.patch(self.YESNO_PATCH_LOCATION) as patched_yes_no: patched_yes_no.return_value = True call_command('delete_course', str(course_run.id)) assert modulestore().get_course(course_run.id) is None __, asset_count = store.get_all_content_for_course(course_run.id) assert asset_count == 1
def save_subs_to_store(subs, subs_id, item): """ Save transcripts into `StaticContent`. Args: `subs_id`: str, subtitles id `item`: video module instance Returns: location of saved subtitles. """ filedata = json.dumps(subs, indent=2) mime_type = 'application/json' filename = 'subs_{0}.srt.sjson'.format(subs_id) content_location = StaticContent.compute_location(item.location.org, item.location.course, filename) content = StaticContent(content_location, filename, mime_type, filedata) contentstore().save(content) del_cached_content(content_location) return content_location
def upload_asset(request, org, course, coursename): ''' This method allows for POST uploading of files into the course asset library, which will be supported by GridFS in MongoDB. ''' # construct a location from the passed in path location = get_location_and_verify_access(request, org, course, coursename) # Does the course actually exist?!? Get anything from it to prove its existance try: modulestore().get_item(location) except: # no return it as a Bad Request response logging.error('Could not find course' + location) return HttpResponseBadRequest() if 'file' not in request.FILES: return HttpResponseBadRequest() # compute a 'filename' which is similar to the location formatting, we're using the 'filename' # nomenclature since we're using a FileSystem paradigm here. We're just imposing # the Location string formatting expectations to keep things a bit more consistent upload_file = request.FILES['file'] filename = upload_file.name mime_type = upload_file.content_type content_loc = StaticContent.compute_location(org, course, filename) chunked = upload_file.multiple_chunks() if chunked: content = StaticContent(content_loc, filename, mime_type, upload_file.chunks()) else: content = StaticContent(content_loc, filename, mime_type, upload_file.read()) thumbnail_content = None thumbnail_location = None # first let's see if a thumbnail can be created (thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(content, tempfile_path=None if not chunked else upload_file.temporary_file_path()) # delete cached thumbnail even if one couldn't be created this time (else the old thumbnail will continue to show) del_cached_content(thumbnail_location) # now store thumbnail location only if we could create it if thumbnail_content is not None: content.thumbnail_location = thumbnail_location # then commit the content contentstore().save(content) del_cached_content(content.location) # readback the saved content - we need the database timestamp readback = contentstore().find(content.location) response_payload = {'displayname': content.name, 'uploadDate': get_default_time_display(readback.last_modified_at), 'url': StaticContent.get_url_path_from_location(content.location), 'thumb_url': StaticContent.get_url_path_from_location(thumbnail_location) if thumbnail_content is not None else None, 'msg': 'Upload completed' } response = JsonResponse(response_payload) response['asset_url'] = StaticContent.get_url_path_from_location(content.location) return response
def import_static_content( modules, course_loc, course_data_path, static_content_store, target_location_namespace, subpath='static', verbose=False): remap_dict = {} # now import all static assets static_dir = course_data_path / subpath try: with open(course_data_path / 'policies/assets.json') as f: policy = json.load(f) except (IOError, ValueError) as err: # xml backed courses won't have this file, only exported courses; # so, its absence is not really an exception. policy = {} verbose = True mimetypes_list = mimetypes.types_map.values() for dirname, _, filenames in os.walk(static_dir): for filename in filenames: content_path = os.path.join(dirname, filename) if filename.endswith('~'): if verbose: log.debug('skipping static content %s...', content_path) continue if verbose: log.debug('importing static content %s...', content_path) try: with open(content_path, 'rb') as f: data = f.read() except IOError: if filename.startswith('._'): # OS X "companion files". See # http://www.diigo.com/annotated/0c936fda5da4aa1159c189cea227e174 continue # Not a 'hidden file', then re-raise exception raise # strip away leading path from the name fullname_with_subpath = content_path.replace(static_dir, '') if fullname_with_subpath.startswith('/'): fullname_with_subpath = fullname_with_subpath[1:] content_loc = StaticContent.compute_location( target_location_namespace.org, target_location_namespace.course, fullname_with_subpath ) policy_ele = policy.get(content_loc.name, {}) displayname = policy_ele.get('displayname', filename) locked = policy_ele.get('locked', False) mime_type = policy_ele.get('contentType') # Check extracted contentType in list of all valid mimetypes if not mime_type or mime_type not in mimetypes_list: mime_type = mimetypes.guess_type(filename)[0] # Assign guessed mimetype content = StaticContent( content_loc, displayname, mime_type, data, import_path=fullname_with_subpath, locked=locked ) # first let's save a thumbnail so we can get back a thumbnail location thumbnail_content, thumbnail_location = static_content_store.generate_thumbnail(content) if thumbnail_content is not None: content.thumbnail_location = thumbnail_location # then commit the content try: static_content_store.save(content) except Exception as err: log.exception('Error importing {0}, error={1}'.format( fullname_with_subpath, err )) # store the remapping information which will be needed # to subsitute in the module data remap_dict[fullname_with_subpath] = content_loc.name return remap_dict