def test_thumbnail_none(self):
# We had a bug where a thumbnail location of None was getting transformed into a Location tuple, with
# all elements being None. It is important that the location be just None for rendering.
content = StaticContent('loc', 'name', 'content_type', 'data', None, None, None)
self.assertIsNone(content.thumbnail_location)
content = StaticContent('loc', 'name', 'content_type', 'data')
self.assertIsNone(content.thumbnail_location)
def create_image(cls, prefix, dimensions, color, name, locked=False):
"""
Creates an image.
Args:
prefix: the prefix to use e.g. split vs mongo
dimensions: tuple of (width, height)
color: the background color of the image
name: the name of the image; can be a format string
locked: whether or not the asset should be locked
Returns:
StaticContent: the StaticContent object for the created image
"""
new_image = Image.new('RGB', dimensions, color)
new_buf = BytesIO()
new_image.save(new_buf, format='png')
new_buf.seek(0)
new_name = name.format(prefix)
new_key = StaticContent.compute_location(cls.courses[prefix].id,
new_name)
new_content = StaticContent(new_key,
new_name,
'image/png',
new_buf.getvalue(),
locked=locked)
contentstore().save(new_content)
return new_content
def create_arbitrary_content(cls, prefix, name, locked=False):
"""
Creates an arbitrary piece of content with a fixed body, for when content doesn't matter.
Args:
prefix: the prefix to use e.g. split vs mongo
name: the name of the content; can be a format string
locked: whether or not the asset should be locked
Returns:
StaticContent: the StaticContent object for the created content
"""
new_buf = BytesIO(b'testingggggggggggg')
new_name = name.format(prefix)
new_key = StaticContent.compute_location(cls.courses[prefix].id,
new_name)
new_content = StaticContent(new_key,
new_name,
'application/octet-stream',
new_buf.getvalue(),
locked=locked)
contentstore().save(new_content)
return new_content
def import_static_content(modules,
course_loc,
course_data_path,
static_content_store,
target_location_namespace,
subpath='static',
verbose=False):
remap_dict = {}
# now import all static assets
static_dir = course_data_path / subpath
verbose = True
for dirname, dirnames, filenames in os.walk(static_dir):
for filename in filenames:
try:
content_path = os.path.join(dirname, filename)
if verbose:
log.debug(
'importing static content {0}...'.format(content_path))
fullname_with_subpath = content_path.replace(
static_dir, '') # strip away leading path from the name
if fullname_with_subpath.startswith('/'):
fullname_with_subpath = fullname_with_subpath[1:]
content_loc = StaticContent.compute_location(
target_location_namespace.org,
target_location_namespace.course, fullname_with_subpath)
mime_type = mimetypes.guess_type(filename)[0]
with open(content_path, 'rb') as f:
data = f.read()
content = StaticContent(content_loc,
filename,
mime_type,
data,
import_path=fullname_with_subpath)
# first let's save a thumbnail so we can get back a thumbnail
# location
(thumbnail_content, thumbnail_location
) = static_content_store.generate_thumbnail(content)
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
# then commit the content
static_content_store.save(content)
# store the remapping information which will be needed to
# subsitute in the module data
remap_dict[fullname_with_subpath] = content_loc.name
except:
raise
return remap_dict
def create_pgreport_csv(course_id, update_state=None):
"""Create CSV of progress to MongoDB."""
course_key = get_coursekey(course_id)
try:
gzipfile = StringIO.StringIO()
gzipcsv = gzip.GzipFile(filename="progress_students.csv.gz",
mode='wb',
fileobj=gzipfile)
writer = csv.writer(gzipcsv, encoding='utf-8')
progress = ProgressReport(course_key, update_state)
for row in progress.yield_students_progress():
writer.writerow(row)
finally:
gzipcsv.close()
try:
content_loc = StaticContent.compute_location(course_key, gzipcsv.name)
content = StaticContent(loc=content_loc,
name=gzipcsv.name,
content_type="application/x-gzip",
data=gzipfile.getvalue())
contentstore().save(content)
del_cached_content(content_loc)
except GridFSError as e:
store.delete(content_id)
log.error(" * GridFS Error: {}".format(e))
raise
finally:
gzipfile.close()
def test_happy_path(self, modulestore_type, create_after_overview):
"""
What happens when everything works like we expect it to.
If `create_after_overview` is True, we will temporarily disable
thumbnail creation so that the initial CourseOverview is created without
an image_set, and the CourseOverviewImageSet is created afterwards. If
`create_after_overview` is False, we'll create the CourseOverviewImageSet
at the same time as the CourseOverview.
"""
# Create a real (oversized) image...
image = Image.new("RGB", (800, 400), "blue")
image_buff = StringIO()
image.save(image_buff, format="JPEG")
image_buff.seek(0)
image_name = "big_course_image.jpeg"
with self.store.default_store(modulestore_type):
course = CourseFactory.create(
default_store=modulestore_type, course_image=image_name
)
# Save a real image here...
course_image_asset_key = StaticContent.compute_location(course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key, image_name, 'image/jpeg', image_buff)
contentstore().save(course_image_content)
# If create_after_overview is True, disable thumbnail generation so
# that the CourseOverview object is created and saved without an
# image_set at first (it will be lazily created later).
if create_after_overview:
self.set_config(enabled=False)
# Now generate the CourseOverview...
course_overview = CourseOverview.get_from_id(course.id)
# If create_after_overview is True, no image_set exists yet. Verify
# that, then switch config back over to True and it should lazily
# create the image_set on the next get_from_id() call.
if create_after_overview:
self.assertFalse(hasattr(course_overview, 'image_set'))
self.set_config(enabled=True)
course_overview = CourseOverview.get_from_id(course.id)
self.assertTrue(hasattr(course_overview, 'image_set'))
image_urls = course_overview.image_urls
config = CourseOverviewImageConfig.current()
# Make sure the thumbnail names come out as expected...
self.assertTrue(image_urls['raw'].endswith('big_course_image.jpeg'))
self.assertTrue(image_urls['small'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.small)))
self.assertTrue(image_urls['large'].endswith('big_course_image-jpeg-{}x{}.jpg'.format(*config.large)))
# Now make sure our thumbnails are of the sizes we expect...
for image_url, expected_size in [(image_urls['small'], config.small), (image_urls['large'], config.large)]:
image_key = StaticContent.get_location_from_path(image_url)
image_content = AssetManager.find(image_key)
image = Image.open(StringIO(image_content.data))
self.assertEqual(image.size, expected_size)
def test_different_resolutions(self, src_dimensions):
"""
Test various resolutions of images to make thumbnails of.
Note that our test sizes are small=(200, 100) and large=(400, 200).
1. Images should won't be blown up if it's too small, so a (100, 50)
resolution image will remain (100, 50).
2. However, images *will* be converted using our format and quality
settings (JPEG, 75% -- the PIL default). This is because images with
relatively small dimensions not compressed properly.
3. Image thumbnail naming will maintain the naming convention of the
target resolution, even if the image was not actually scaled to that
size (i.e. it was already smaller). This is mostly because it's
simpler to be consistent, but it also lets us more easily tell which
configuration a thumbnail was created under.
"""
# Create a source image...
image = Image.new("RGB", src_dimensions, "blue")
image_buff = StringIO()
image.save(image_buff, format="PNG")
image_buff.seek(0)
image_name = "src_course_image.png"
course = CourseFactory.create(course_image=image_name)
# Save the image to the contentstore...
course_image_asset_key = StaticContent.compute_location(course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key, image_name, 'image/png', image_buff)
contentstore().save(course_image_content)
# Now generate the CourseOverview...
config = CourseOverviewImageConfig.current()
course_overview = CourseOverview.get_from_id(course.id)
image_urls = course_overview.image_urls
for image_url, target in [(image_urls['small'], config.small), (image_urls['large'], config.large)]:
image_key = StaticContent.get_location_from_path(image_url)
image_content = AssetManager.find(image_key)
image = Image.open(StringIO(image_content.data))
# Naming convention for thumbnail
self.assertTrue(image_url.endswith('src_course_image-png-{}x{}.jpg'.format(*target)))
# Actual thumbnail data
src_x, src_y = src_dimensions
target_x, target_y = target
image_x, image_y = image.size
# I'm basically going to assume the image library knows how to do
# the right thing in terms of handling aspect ratio. We're just
# going to make sure that small images aren't blown up, and that
# we never exceed our target sizes
self.assertLessEqual(image_x, target_x)
self.assertLessEqual(image_y, target_y)
if src_x < target_x and src_y < target_y:
self.assertEqual(src_x, image_x)
self.assertEqual(src_y, image_y)
def _upload_file(subs_file, location, filename):
mime_type = subs_file.content_type
content_location = StaticContent.compute_location(
location.course_key, filename
)
content = StaticContent(content_location, filename, mime_type, subs_file.read())
contentstore().save(content)
del_cached_content(content.location)
def upload_file(filename, location):
path = os.path.join(TEST_ROOT, 'uploads/', filename)
f = open(os.path.abspath(path))
mime_type = "application/json"
content_location = StaticContent.compute_location(location.course_key,
filename)
content = StaticContent(content_location, filename, mime_type, f.read())
contentstore().save(content)
del_cached_content(content.location)
def save_asset(self, filename, asset_key, displayname, locked):
"""
Load and save the given file.
"""
with open("{}/static/{}".format(DATA_DIR, filename), "rb") as f:
content = StaticContent(
asset_key, displayname, mimetypes.guess_type(filename)[0], f.read(),
locked=locked
)
self.contentstore.save(content)
def _create_fake_images(self, asset_keys):
"""
Creates fake image files for a list of asset_keys.
"""
for asset_key_string in asset_keys:
asset_key = AssetKey.from_string(asset_key_string)
content = StaticContent(
asset_key, "Fake asset", "image/png", "data",
)
contentstore().save(content)
def save_to_store(content, name, mime_type, location):
"""
Save named content to store by location.
Returns location of saved content.
"""
content_location = Transcript.asset_location(location, name)
content = StaticContent(content_location, name, mime_type, content)
contentstore().save(content)
return content_location
def setUp(self):
super(TestCourseMixin, self).setUp()
with modulestore().default_store(ModuleStoreEnum.Type.split):
self.course = SampleCourseFactory.create(
block_info_tree=TEST_COURSE)
# And upload the course static asssets:
asset_key = StaticContent.compute_location(self.course.id,
'sample_handout.txt')
content = StaticContent(asset_key, "Fake asset", "application/text",
"test".encode('utf8'))
contentstore().save(content)
asset_key = StaticContent.compute_location(self.course.id, 'edx.svg')
content = StaticContent(
asset_key, "Fake image", "image/svg+xml", """
<svg viewBox="0 0 403 403" version="1.1"
xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"
>
<title>edX</title>
<desc>
the edX logo is comprised of a red letter e, grey d, and blue uppercase X, all slightly overlapping
each other. The d is slightly transparent.
</desc>
<g transform="translate(0,100)">
<path id="e-path" stroke-width="1" stroke="none" fill="rgb(185, 0, 88)" fill-rule="evenodd"
d="M32.1,127 H141.9 A71,71.5 0 1,0 137.3,143 H103 A42,42 0 0,1 32.1,127 M32.1,102.5 H112 A42,42
0 0,0 32.1,102.5"/>
<path id="x-path" stroke-width="1" stroke="none" fill="rgb(0, 162, 228)" fill-rule="evenodd"
d="M228,1 H302 V31 H286 L315,67 L344,31 H328 V1 H401.5 V31 H385 L335.2,92.4 L387.5,156.8 H401.5
V187 H328 V156.8 H346.5 L315,117.4 L283,156.8 H302.0 V187 H228.5 V156.8 H243 L294.3,92.4
L244,30.5 H228 V1"/>
<path id="d-path" stroke-width="1" stroke="none" fill="rgb(55, 55, 60)" fill-rule="evenodd"
opacity="0.55" d="M198.5,1 L248,1 V156.5 H269.8 V187 H217.5 V174 A71.7,71.7 0 1,1 218,55.5
V30.5 H198.5 V1 M218,114 A41,41.5 0 1,1 136.1,114 A40.5,40.5 0 1,1 218,114"/>
</g>
</svg>
""".strip().encode('utf8'))
contentstore().save(content)
# And the video data + transcript must also be stored in edx-val for the video export to work:
edxval_api.create_video(VIDEO_B_VAL_DATA)
edxval_api.create_video_transcript(**VIDEO_B_SRT_TRANSCRIPT_DATA)
def test_space_in_asset_name_for_rerun_course(self):
"""
Tests check the scenario where one course which has an asset with percentage(%) in its
name, it should re-run successfully.
"""
org = 'edX'
course_number = 'CS101'
course_run = '2015_Q1'
display_name = 'rerun'
fields = {'display_name': display_name}
course_assets = set([u'subs_Introduction%20To%20New.srt.sjson'], )
# Create a course using split modulestore
course = CourseFactory.create(org=org,
number=course_number,
run=course_run,
display_name=display_name,
default_store=ModuleStoreEnum.Type.split)
# add an asset
asset_key = course.id.make_asset_key(
'asset', 'subs_Introduction%20To%20New.srt.sjson')
content = StaticContent(
asset_key,
'Dummy assert',
'application/json',
'dummy data',
)
contentstore().save(content)
# Get & verify all assets of the course
assets, count = contentstore().get_all_content_for_course(course.id)
self.assertEqual(count, 1)
self.assertEqual(
set([asset['asset_key'].block_id for asset in assets]),
course_assets)
# rerun from split into split
split_rerun_id = CourseLocator(org=org,
course=course_number,
run="2012_Q2")
CourseRerunState.objects.initiated(course.id, split_rerun_id,
self.user, fields['display_name'])
result = rerun_course.delay(six.text_type(course.id),
six.text_type(split_rerun_id),
self.user.id,
json.dumps(fields, cls=EdxJSONEncoder))
# Check if re-run was successful
self.assertEqual(result.get(), "succeeded")
rerun_state = CourseRerunState.objects.find_first(
course_key=split_rerun_id)
self.assertEqual(rerun_state.state,
CourseRerunUIStateManager.State.SUCCEEDED)
def save_subs_to_store(self, subs, subs_id):
"""Save transcripts into `StaticContent`."""
filedata = json.dumps(subs, indent=2)
mime_type = 'application/json'
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(self.course.id, filename)
content = StaticContent(content_location, filename, mime_type, filedata)
contentstore().save(content)
del_cached_content(content_location)
return content_location
def _create_course_image(self, course, image_name):
"""
Creates a course image in contentstore.
"""
# Create a source image...
image = Image.new('RGB', (800, 400), 'blue')
image_buff = StringIO()
image.save(image_buff, format='PNG')
image_buff.seek(0)
# Save the image to the contentstore...
course_image_asset_key = StaticContent.compute_location(course.id, course.course_image)
course_image_content = StaticContent(course_image_asset_key, image_name, 'image/png', image_buff)
contentstore().save(course_image_content)
def verify_content_links(module,
base_dir,
static_content_store,
link,
remap_dict=None):
if link.startswith('/static/'):
# yes, then parse out the name
path = link[len('/static/'):]
static_pathname = base_dir / path
if os.path.exists(static_pathname):
try:
content_loc = StaticContent.compute_location(
module.location.org, module.location.course, path)
filename = os.path.basename(path)
mime_type = mimetypes.guess_type(filename)[0]
with open(static_pathname, 'rb') as f:
data = f.read()
content = StaticContent(content_loc,
filename,
mime_type,
data,
import_path=path)
# first let's save a thumbnail so we can get back a thumbnail
# location
(thumbnail_content, thumbnail_location
) = static_content_store.generate_thumbnail(content)
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
# then commit the content
static_content_store.save(content)
new_link = StaticContent.get_url_path_from_location(
content_loc)
if remap_dict is not None:
remap_dict[link] = new_link
return new_link
except Exception, e:
logging.exception(
'Skipping failed content load from {0}. Exception: {1}'.
format(path, e))
def test_remove_assets(self):
course_run = CourseFactory()
store = contentstore()
asset_key = course_run.id.make_asset_key('asset', 'test.txt')
content = StaticContent(asset_key, 'test.txt', 'plain/text', b'test data')
store.save(content)
__, asset_count = store.get_all_content_for_course(course_run.id)
assert asset_count == 1
with mock.patch(self.YESNO_PATCH_LOCATION, return_value=True):
call_command('delete_course', str(course_run.id), '--remove-assets')
__, asset_count = store.get_all_content_for_course(course_run.id)
assert asset_count == 0
def upload_file(self, subs_file, location, filename):
"""
Upload a file in content store.
Arguments:
subs_file (File): pointer to file to be uploaded
location (Locator): Item location
filename (unicode): Name of file to be uploaded
"""
mime_type = subs_file.content_type
content_location = StaticContent.compute_location(
location.course_key, filename)
content = StaticContent(content_location, filename, mime_type,
subs_file.read())
contentstore().save(content)
def upload_file_to_course(course_key, contentstore, source_file,
target_filename):
'''
Uploads the given source file to the given course, and returns the content of the file.
'''
asset_key = course_key.make_asset_key('asset', target_filename)
with open(source_file, "rb") as f:
file_contents = f.read()
mimetype = guess_type(source_file)[0]
content = StaticContent(asset_key,
target_filename,
mimetype,
file_contents,
locked=False)
contentstore.save(content)
return file_contents
def save_subs_to_store(subs, subs_id, item, language='en'):
"""
Save transcripts into `StaticContent`.
Args:
`subs_id`: str, subtitles id
`item`: video module instance
`language`: two chars str ('uk'), language of translation of transcripts
Returns: location of saved subtitles.
"""
filedata = json.dumps(subs, indent=2)
mime_type = 'application/json'
filename = subs_filename(subs_id, language)
content_location = Transcript.asset_location(item.location, filename)
content = StaticContent(content_location, filename, mime_type, filedata)
contentstore().save(content)
return content_location
def test_asset_and_course_deletion(self):
course_run = CourseFactory()
self.assertIsNotNone(modulestore().get_course(course_run.id))
store = contentstore()
asset_key = course_run.id.make_asset_key('asset', 'test.txt')
content = StaticContent(asset_key, 'test.txt', 'plain/text', b'test data')
store.save(content)
__, asset_count = store.get_all_content_for_course(course_run.id)
assert asset_count == 1
with mock.patch(self.YESNO_PATCH_LOCATION) as patched_yes_no:
patched_yes_no.return_value = True
call_command('delete_course', str(course_run.id))
assert modulestore().get_course(course_run.id) is None
__, asset_count = store.get_all_content_for_course(course_run.id)
assert asset_count == 1
def save_subs_to_store(subs, subs_id, item):
"""
Save transcripts into `StaticContent`.
Args:
`subs_id`: str, subtitles id
`item`: video module instance
Returns: location of saved subtitles.
"""
filedata = json.dumps(subs, indent=2)
mime_type = 'application/json'
filename = 'subs_{0}.srt.sjson'.format(subs_id)
content_location = StaticContent.compute_location(item.location.org,
item.location.course,
filename)
content = StaticContent(content_location, filename, mime_type, filedata)
contentstore().save(content)
del_cached_content(content_location)
return content_location
def upload_asset(request, org, course, coursename):
'''
This method allows for POST uploading of files into the course asset library, which will
be supported by GridFS in MongoDB.
'''
# construct a location from the passed in path
location = get_location_and_verify_access(request, org, course, coursename)
# Does the course actually exist?!? Get anything from it to prove its existance
try:
modulestore().get_item(location)
except:
# no return it as a Bad Request response
logging.error('Could not find course' + location)
return HttpResponseBadRequest()
if 'file' not in request.FILES:
return HttpResponseBadRequest()
# compute a 'filename' which is similar to the location formatting, we're using the 'filename'
# nomenclature since we're using a FileSystem paradigm here. We're just imposing
# the Location string formatting expectations to keep things a bit more consistent
upload_file = request.FILES['file']
filename = upload_file.name
mime_type = upload_file.content_type
content_loc = StaticContent.compute_location(org, course, filename)
chunked = upload_file.multiple_chunks()
if chunked:
content = StaticContent(content_loc, filename, mime_type, upload_file.chunks())
else:
content = StaticContent(content_loc, filename, mime_type, upload_file.read())
thumbnail_content = None
thumbnail_location = None
# first let's see if a thumbnail can be created
(thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(content,
tempfile_path=None if not chunked else
upload_file.temporary_file_path())
# delete cached thumbnail even if one couldn't be created this time (else the old thumbnail will continue to show)
del_cached_content(thumbnail_location)
# now store thumbnail location only if we could create it
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
# then commit the content
contentstore().save(content)
del_cached_content(content.location)
# readback the saved content - we need the database timestamp
readback = contentstore().find(content.location)
response_payload = {'displayname': content.name,
'uploadDate': get_default_time_display(readback.last_modified_at),
'url': StaticContent.get_url_path_from_location(content.location),
'thumb_url': StaticContent.get_url_path_from_location(thumbnail_location) if thumbnail_content is not None else None,
'msg': 'Upload completed'
}
response = JsonResponse(response_payload)
response['asset_url'] = StaticContent.get_url_path_from_location(content.location)
return response
def import_static_content(
modules, course_loc, course_data_path, static_content_store,
target_location_namespace, subpath='static', verbose=False):
remap_dict = {}
# now import all static assets
static_dir = course_data_path / subpath
try:
with open(course_data_path / 'policies/assets.json') as f:
policy = json.load(f)
except (IOError, ValueError) as err:
# xml backed courses won't have this file, only exported courses;
# so, its absence is not really an exception.
policy = {}
verbose = True
mimetypes_list = mimetypes.types_map.values()
for dirname, _, filenames in os.walk(static_dir):
for filename in filenames:
content_path = os.path.join(dirname, filename)
if filename.endswith('~'):
if verbose:
log.debug('skipping static content %s...', content_path)
continue
if verbose:
log.debug('importing static content %s...', content_path)
try:
with open(content_path, 'rb') as f:
data = f.read()
except IOError:
if filename.startswith('._'):
# OS X "companion files". See
# http://www.diigo.com/annotated/0c936fda5da4aa1159c189cea227e174
continue
# Not a 'hidden file', then re-raise exception
raise
# strip away leading path from the name
fullname_with_subpath = content_path.replace(static_dir, '')
if fullname_with_subpath.startswith('/'):
fullname_with_subpath = fullname_with_subpath[1:]
content_loc = StaticContent.compute_location(
target_location_namespace.org, target_location_namespace.course,
fullname_with_subpath
)
policy_ele = policy.get(content_loc.name, {})
displayname = policy_ele.get('displayname', filename)
locked = policy_ele.get('locked', False)
mime_type = policy_ele.get('contentType')
# Check extracted contentType in list of all valid mimetypes
if not mime_type or mime_type not in mimetypes_list:
mime_type = mimetypes.guess_type(filename)[0] # Assign guessed mimetype
content = StaticContent(
content_loc, displayname, mime_type, data,
import_path=fullname_with_subpath, locked=locked
)
# first let's save a thumbnail so we can get back a thumbnail location
thumbnail_content, thumbnail_location = static_content_store.generate_thumbnail(content)
if thumbnail_content is not None:
content.thumbnail_location = thumbnail_location
# then commit the content
try:
static_content_store.save(content)
except Exception as err:
log.exception('Error importing {0}, error={1}'.format(
fullname_with_subpath, err
))
# store the remapping information which will be needed
# to subsitute in the module data
remap_dict[fullname_with_subpath] = content_loc.name
return remap_dict
