def setup_assets(self, course1_key, course2_key, store=None):
"""
Setup assets. Save in store if given
"""
asset_fields = ('filename', 'internal_name', 'basename', 'locked', 'edited_by', 'edited_on', 'curr_version', 'prev_version')
all_asset_data = (
('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '14', '13'),
('shout.ogg', 'KFMDONSKF39K', 'sounds', True, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '1', None),
('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, ModuleStoreEnum.UserID.test * 2, datetime.now(pytz.utc), 'AB', 'AA'),
('dog.png', 'PUPY4242X', 'pictures/animals', True, ModuleStoreEnum.UserID.test * 3, datetime.now(pytz.utc), '5', '4'),
('not_here.txt', 'JJJCCC747', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49'),
('asset.txt', 'JJJCCC747858', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49'),
('roman_history.pdf', 'JASDUNSADK', 'texts/italy', True, ModuleStoreEnum.UserID.test * 7, datetime.now(pytz.utc), '1.1', '1.01'),
('weather_patterns.bmp', '928SJXX2EB', 'science', False, ModuleStoreEnum.UserID.test * 8, datetime.now(pytz.utc), '52', '51'),
('demo.swf', 'DFDFGGGG14', 'demos/easy', False, ModuleStoreEnum.UserID.test * 9, datetime.now(pytz.utc), '5', '4'),
)
for i, asset in enumerate(all_asset_data):
asset_dict = dict(zip(asset_fields[1:], asset[1:]))
if i in (0, 1) and course1_key:
asset_key = course1_key.make_asset_key('asset', asset[0])
asset_md = AssetMetadata(asset_key, **asset_dict)
if store is not None:
store.save_asset_metadata(asset_md, asset[4])
elif course2_key:
asset_key = course2_key.make_asset_key('asset', asset[0])
asset_md = AssetMetadata(asset_key, **asset_dict)
# Don't save assets 5 and 6.
if store is not None and i not in (4, 5):
store.save_asset_metadata(asset_md, asset[4])
def setUp(self):
super(VideoUploadTestCase, self).setUp()
self.url = VideoUploadTestCase.get_url_for_course_key(self.course.id)
self.test_token = "test_token"
self.course.video_upload_pipeline = {
"course_video_upload_token": self.test_token,
}
self.save_course()
self.previous_uploads = [{
"edx_video_id": "test1",
"client_video_id": "test1.mp4",
"duration": 42.0,
"status": "transcode_active",
"encoded_videos": [],
}, {
"edx_video_id": "test2",
"client_video_id": "test2.mp4",
"duration": 128.0,
"status": "file_complete",
"encoded_videos": [],
}]
for video in self.previous_uploads:
create_video(video)
modulestore().save_asset_metadata(
AssetMetadata(
self.course.id.make_asset_key(VIDEO_ASSET_TYPE,
video["edx_video_id"])),
self.user.id)
def copy_all_asset_metadata(self, source_course_key, dest_course_key,
user_id):
"""
Copy all the course assets from source_course_key to dest_course_key.
Arguments:
source_course_key (CourseKey): identifier of course to copy from
dest_course_key (CourseKey): identifier of course to copy to
user_id (int|long): user copying the asset metadata
"""
source_store = self._get_modulestore_for_courselike(source_course_key)
dest_store = self._get_modulestore_for_courselike(dest_course_key)
if source_store != dest_store:
with self.bulk_operations(dest_course_key):
# Get all the asset metadata in the source course.
all_assets = source_store.get_all_asset_metadata(
source_course_key, 'asset')
# Store it all in the dest course.
for asset in all_assets:
new_asset_key = dest_course_key.make_asset_key(
'asset', asset.asset_id.path)
copied_asset = AssetMetadata(new_asset_key)
copied_asset.from_storable(asset.to_storable())
dest_store.save_asset_metadata(copied_asset, user_id)
else:
# Courses in the same modulestore can be handled by the modulestore itself.
source_store.copy_all_asset_metadata(source_course_key,
dest_course_key, user_id)
def setup_assets(self, course1_key, course2_key, store=None):
"""
Setup assets. Save in store if given
"""
for i, asset in enumerate(AssetStoreTestData.all_asset_data):
asset_dict = dict(list(zip(AssetStoreTestData.asset_fields[1:], asset[1:])))
if i in (0, 1) and course1_key:
asset_key = course1_key.make_asset_key('asset', asset[0])
asset_md = AssetMetadata(asset_key, **asset_dict)
if store is not None:
store.save_asset_metadata(asset_md, asset[4])
elif course2_key:
asset_key = course2_key.make_asset_key('asset', asset[0])
asset_md = AssetMetadata(asset_key, **asset_dict)
# Don't save assets 5 and 6.
if store is not None and i not in (4, 5):
store.save_asset_metadata(asset_md, asset[4])
def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
"""
Returns a list of asset metadata for all assets of the given asset_type in the course.
Args:
course_key (CourseKey): course identifier
asset_type (str): the block_type of the assets to return
start (int): optional - start at this asset number. Zero-based!
maxresults (int): optional - return at most this many, -1 means no limit
sort (array): optional - None means no sort
(sort_by (str), sort_order (str))
sort_by - one of 'uploadDate' or 'displayname'
sort_order - one of SortOrder.ascending or SortOrder.descending
Returns:
List of AssetMetadata objects.
"""
course_assets = self._find_course_assets(course_key)
if course_assets is None:
# If no course assets are found, return None instead of empty list
# to distinguish zero assets from "not able to retrieve assets".
return None
# Determine the proper sort - with defaults of ('displayname', SortOrder.ascending).
key_func = itemgetter('filename')
sort_order = ModuleStoreEnum.SortOrder.ascending
if sort:
if sort[0] == 'uploadDate':
key_func = lambda x: x['edit_info']['edited_on']
if sort[1] == ModuleStoreEnum.SortOrder.descending:
sort_order = ModuleStoreEnum.SortOrder.descending
all_assets = SortedListWithKey(course_assets.get(asset_type, []), key=key_func)
num_assets = len(all_assets)
start_idx = start
end_idx = min(num_assets, start + maxresults)
if maxresults < 0:
# No limit on the results.
end_idx = num_assets
step_incr = 1
if sort_order == ModuleStoreEnum.SortOrder.descending:
# Flip the indices and iterate backwards.
step_incr = -1
start_idx = (num_assets - 1) - start_idx
end_idx = (num_assets - 1) - end_idx
ret_assets = []
for idx in xrange(start_idx, end_idx, step_incr):
raw_asset = all_assets[idx]
new_asset = AssetMetadata(course_key.make_asset_key(asset_type, raw_asset['filename']))
new_asset.from_storable(raw_asset)
ret_assets.append(new_asset)
return ret_assets
def _make_asset_metadata(self, asset_loc):
"""
Make a single test asset metadata.
"""
return AssetMetadata(
asset_loc, internal_name='EKMND332DDBK',
basename='pictures/historical', contenttype='image/jpeg',
locked=False, fields={'md5': '77631ca4f0e08419b70726a447333ab6'},
edited_by=ModuleStoreEnum.UserID.test, edited_on=datetime.now(pytz.utc),
curr_version='v1.0', prev_version='v0.95'
)
def test_export_with_None_value(self):
"""
Export and import a single AssetMetadata to XML with a None created_by field, without causing an exception.
"""
asset_md = AssetMetadata(
self.course_id.make_asset_key('asset', 'none_value'),
created_by=None,
)
asset = etree.Element("asset")
asset_md.to_xml(asset)
asset_md.from_xml(asset)
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
"""
Returns a list of static asset (or thumbnail) metadata for a course.
Args:
course_key (CourseKey): course identifier
start (int): optional - start at this asset number
maxresults (int): optional - return at most this many, -1 means no limit
sort (array): optional - None means no sort
(sort_by (str), sort_order (str))
sort_by - one of 'uploadDate' or 'displayname'
sort_order - one of 'ascending' or 'descending'
get_thumbnails (bool): True if getting thumbnail metadata, else getting asset metadata
Returns:
List of AssetMetadata or AssetThumbnailMetadata objects.
"""
course_assets = self._find_course_assets(course_key)
if course_assets is None:
# If no course assets are found, return None instead of empty list
# to distinguish zero assets from "not able to retrieve assets".
return None
if get_thumbnails:
all_assets = course_assets.get('thumbnails', [])
else:
all_assets = course_assets.get('assets', [])
# DO_NEXT: Add start/maxresults/sort functionality as part of https://openedx.atlassian.net/browse/PLAT-74
if start and maxresults and sort:
pass
ret_assets = []
for asset in all_assets:
if get_thumbnails:
thumb = AssetThumbnailMetadata(
course_key.make_asset_key('thumbnail', asset['filename']),
internal_name=asset['filename'], **kwargs
)
ret_assets.append(thumb)
else:
asset = AssetMetadata(
course_key.make_asset_key('asset', asset['filename']),
basename=asset['filename'],
edited_on=asset['edit_info']['edited_on'],
contenttype=asset['contenttype'],
md5=str(asset['md5']), **kwargs
)
ret_assets.append(asset)
return ret_assets
def _import_course_asset_metadata(store, data_dir, course_id,
raise_on_failure):
"""
Read in assets XML file, parse it, and add all asset metadata to the modulestore.
"""
asset_dir = path(data_dir) / AssetMetadata.EXPORTED_ASSET_DIR
assets_filename = AssetMetadata.EXPORTED_ASSET_FILENAME
asset_xml_file = asset_dir / assets_filename
def make_asset_id(course_id, asset_xml):
"""
Construct an asset ID out of a complete asset XML section.
"""
asset_type = None
asset_name = None
for child in asset_xml.iterchildren():
if child.tag == AssetMetadata.ASSET_TYPE_ATTR:
asset_type = child.text
elif child.tag == AssetMetadata.ASSET_BASENAME_ATTR:
asset_name = child.text
return course_id.make_asset_key(asset_type, asset_name)
all_assets = []
try:
xml_data = etree.parse(asset_xml_file).getroot()
assert (xml_data.tag == AssetMetadata.ALL_ASSETS_XML_TAG)
for asset in xml_data.iterchildren():
if asset.tag == AssetMetadata.ASSET_XML_TAG:
# Construct the asset key.
asset_key = make_asset_id(course_id, asset)
asset_md = AssetMetadata(asset_key)
asset_md.from_xml(asset)
all_assets.append(asset_md)
except IOError:
logging.info('No {} file is present with asset metadata.'.format(
assets_filename))
return
except Exception: # pylint: disable=W0703
logging.exception('Error while parsing asset xml.')
if raise_on_failure:
raise
else:
return
# Now add all asset metadata to the modulestore.
if len(all_assets) > 0:
store.save_asset_metadata_list(all_assets,
all_assets[0].edited_by,
import_only=True)
def test_metadata_found_in_modulestore(self):
# Insert asset metadata into the modulestore (with no accompanying asset).
asset_key = self.course.id.make_asset_key(AssetMetadata.GENERAL_ASSET_TYPE, 'pic1.jpg')
asset_md = AssetMetadata(asset_key, {
'internal_name': 'EKMND332DDBK',
'basename': 'pix/archive',
'locked': False,
'curr_version': '14',
'prev_version': '13'
})
modulestore().save_asset_metadata(asset_md, 15)
# Get the asset metadata and have it be found in the modulestore.
# Currently, no asset metadata should be found in the modulestore. The code is not yet storing it there.
# If asset metadata *is* found there, an exception is raised. This test ensures the exception is indeed raised.
# THIS IS TEMPORARY. Soon, asset metadata *will* be stored in the modulestore.
with self.assertRaises((AssetMetadataFoundTemporary, NameError)):
self.client.get(unicode(asset_key), HTTP_ACCEPT='text/html')
def find_asset_metadata(self, asset_key, **kwargs):
"""
Find the metadata for a particular course asset.
Arguments:
asset_key (AssetKey): key containing original asset filename
Returns:
asset metadata (AssetMetadata) -or- None if not found
"""
course_assets, asset_idx = self._find_course_asset(asset_key)
if asset_idx is None:
return None
mdata = AssetMetadata(asset_key, asset_key.path, **kwargs)
all_assets = course_assets[asset_key.asset_type]
mdata.from_storable(all_assets[asset_idx])
return mdata
def test_export_single_asset_to_from_xml(self):
"""
Export a single AssetMetadata to XML and verify the structure and fields.
"""
asset_md = self.course_assets[0]
root = etree.Element("assets")
asset = etree.SubElement(root, "asset")
asset_md.to_xml(asset)
# If this line does *not* raise, the XML is valid.
etree.fromstring(etree.tostring(root), self.xmlparser)
new_asset_key = self.course_id.make_asset_key('tmp', 'tmp')
new_asset_md = AssetMetadata(new_asset_key)
new_asset_md.from_xml(asset)
# Compare asset_md to new_asset_md.
for attr in AssetMetadata.ALL_ATTRS:
orig_value = getattr(asset_md, attr)
new_value = getattr(new_asset_md, attr)
self.assertEqual(orig_value, new_value)
def setUp(self):
super(TestAssetXml, self).setUp()
xsd_filename = "assets.xsd"
self.course_id = CourseLocator('org1', 'course1', 'run1')
self.course_assets = []
for asset in AssetStoreTestData.all_asset_data:
asset_dict = dict(list(zip(AssetStoreTestData.asset_fields[1:], asset[1:])))
asset_md = AssetMetadata(self.course_id.make_asset_key('asset', asset[0]), **asset_dict)
self.course_assets.append(asset_md)
# Read in the XML schema definition and make a validator.
xsd_path = path(__file__).realpath().parent / xsd_filename
with open(xsd_path, 'r') as f:
schema_root = etree.XML(f.read())
schema = etree.XMLSchema(schema_root)
self.xmlparser = etree.XMLParser(schema=schema)
def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
"""
Find the info for a particular course asset/thumbnail.
Arguments:
asset_key (AssetKey): key containing original asset filename
thumbnail (bool): True if finding thumbnail, False if finding asset metadata
Returns:
asset/thumbnail metadata (AssetMetadata/AssetThumbnailMetadata) -or- None if not found
"""
course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path, thumbnail)
if asset_idx is None:
return None
if thumbnail:
info = 'thumbnails'
mdata = AssetThumbnailMetadata(asset_key, asset_key.path, **kwargs)
else:
info = 'assets'
mdata = AssetMetadata(asset_key, asset_key.path, **kwargs)
all_assets = course_assets[info]
mdata.from_mongo(all_assets[asset_idx])
return mdata
def generate_random_asset_md():
"""
Generates a single AssetMetadata object with semi-random data.
"""
course_key = CourseKey.from_string('org/course/run')
asset_key = course_key.make_asset_key(asset_type(), filename())
(curr_version, prev_version) = versions()
return AssetMetadata(
asset_key,
pathname=pathname(),
internal_name=str([filename() for __ in xrange(10)]),
locked=locked(),
contenttype=contenttype(),
thumbnail=filename(),
fields=fields(),
curr_version=curr_version,
prev_version=prev_version,
edited_by=user_id(),
edited_by_email='*****@*****.**',
edited_on=date_and_time(),
created_by=user_id(),
created_by_email='*****@*****.**',
created_on=date_and_time(),
)
def videos_post(course, request):
"""
Input (JSON):
{
"files": [{
"file_name": "video.mp4",
"content_type": "video/mp4"
}]
}
Returns (JSON):
{
"files": [{
"file_name": "video.mp4",
"upload_url": "http://example.com/put_video"
}]
}
The returned array corresponds exactly to the input array.
"""
error = None
if "files" not in request.json:
error = "Request object is not JSON or does not contain 'files'"
elif any("file_name" not in file or "content_type" not in file
for file in request.json["files"]):
error = "Request 'files' entry does not contain 'file_name' and 'content_type'"
if error:
return JsonResponse({"error": error}, status=400)
bucket = storage_service_bucket()
course_video_upload_token = course.video_upload_pipeline[
"course_video_upload_token"]
req_files = request.json["files"]
resp_files = []
for req_file in req_files:
file_name = req_file["file_name"]
edx_video_id = unicode(uuid4())
key = storage_service_key(bucket, file_name=edx_video_id)
for metadata_name, value in [
("course_video_upload_token", course_video_upload_token),
("client_video_id", file_name),
("course_key", unicode(course.id)),
]:
key.set_metadata(metadata_name, value)
upload_url = key.generate_url(
KEY_EXPIRATION_IN_SECONDS,
"PUT",
headers={"Content-Type": req_file["content_type"]})
# persist edx_video_id as uploaded through this course
video_meta_data = AssetMetadata(
course.id.make_asset_key(VIDEO_ASSET_TYPE, edx_video_id))
modulestore().save_asset_metadata(video_meta_data, request.user.id)
# persist edx_video_id in VAL
create_video({
"edx_video_id": edx_video_id,
"status": "upload",
"client_video_id": file_name,
"duration": 0,
"encoded_videos": [],
})
resp_files.append({"file_name": file_name, "upload_url": upload_url})
return JsonResponse({"files": resp_files}, status=200)
def get_all_asset_metadata(self,
course_key,
asset_type,
start=0,
maxresults=-1,
sort=None,
**kwargs):
"""
Returns a list of asset metadata for all assets of the given asset_type in the course.
Args:
course_key (CourseKey): course identifier
asset_type (str): the block_type of the assets to return. If None, return assets of all types.
start (int): optional - start at this asset number. Zero-based!
maxresults (int): optional - return at most this many, -1 means no limit
sort (array): optional - None means no sort
(sort_by (str), sort_order (str))
sort_by - one of 'uploadDate' or 'displayname'
sort_order - one of SortOrder.ascending or SortOrder.descending
Returns:
List of AssetMetadata objects.
"""
course_assets = self._find_course_assets(course_key)
# Determine the proper sort - with defaults of ('displayname', SortOrder.ascending).
key_func = None
sort_order = ModuleStoreEnum.SortOrder.ascending
if sort:
if sort[0] == 'uploadDate':
key_func = lambda x: x['edit_info']['edited_on']
if sort[1] == ModuleStoreEnum.SortOrder.descending:
sort_order = ModuleStoreEnum.SortOrder.descending
if asset_type is None:
# Add assets of all types to the sorted list.
all_assets = SortedAssetList(iterable=[], key=key_func)
for asset_type, val in course_assets.iteritems():
all_assets.update(val)
else:
# Add assets of a single type to the sorted list.
all_assets = SortedAssetList(iterable=course_assets.get(
asset_type, []),
key=key_func)
num_assets = len(all_assets)
start_idx = start
end_idx = min(num_assets, start + maxresults)
if maxresults < 0:
# No limit on the results.
end_idx = num_assets
step_incr = 1
if sort_order == ModuleStoreEnum.SortOrder.descending:
# Flip the indices and iterate backwards.
step_incr = -1
start_idx = (num_assets - 1) - start_idx
end_idx = (num_assets - 1) - end_idx
ret_assets = []
for idx in xrange(start_idx, end_idx, step_incr):
raw_asset = all_assets[idx]
asset_key = course_key.make_asset_key(raw_asset['asset_type'],
raw_asset['filename'])
new_asset = AssetMetadata(asset_key)
new_asset.from_storable(raw_asset)
ret_assets.append(new_asset)
return ret_assets
def setUp(self):
super(VideoUploadTestMixin, self).setUp()
self.url = self.get_url_for_course_key(self.course.id)
self.test_token = "test_token"
self.course.video_upload_pipeline = {
"course_video_upload_token": self.test_token,
}
self.save_course()
self.profiles = ["profile1", "profile2"]
self.previous_uploads = [
{
"edx_video_id": "test1",
"client_video_id": "test1.mp4",
"duration": 42.0,
"status": "upload",
"encoded_videos": [],
},
{
"edx_video_id":
"test2",
"client_video_id":
"test2.mp4",
"duration":
128.0,
"status":
"file_complete",
"encoded_videos": [
{
"profile": "profile1",
"url": "http://example.com/profile1/test2.mp4",
"file_size": 1600,
"bitrate": 100,
},
{
"profile": "profile2",
"url": "http://example.com/profile2/test2.mov",
"file_size": 16000,
"bitrate": 1000,
},
],
},
{
"edx_video_id":
"non-ascii",
"client_video_id":
u"nón-ascii-näme.mp4",
"duration":
256.0,
"status":
"transcode_active",
"encoded_videos": [
{
"profile": "profile1",
"url":
u"http://example.com/profile1/nón-ascii-näme.mp4",
"file_size": 3200,
"bitrate": 100,
},
]
},
]
# Ensure every status string is tested
self.previous_uploads += [
{
"edx_video_id": "status_test_{}".format(status),
"client_video_id": "status_test.mp4",
"duration": 3.14,
"status": status,
"encoded_videos": [],
} for status in (StatusDisplayStrings._STATUS_MAP.keys() + # pylint:disable=protected-access
["non_existent_status"])
]
for profile in self.profiles:
create_profile(profile)
for video in self.previous_uploads:
create_video(video)
modulestore().save_asset_metadata(
AssetMetadata(
self.course.id.make_asset_key(VIDEO_ASSET_TYPE,
video["edx_video_id"])),
self.user.id)
def setUp(self):
super(VideoUploadTestMixin, self).setUp()
self.url = self.get_url_for_course_key(self.course.id)
self.test_token = "test_token"
self.course.video_upload_pipeline = {
"course_video_upload_token": self.test_token,
}
self.save_course()
self.profiles = [
{
"profile_name": "profile1",
"extension": "mp4",
"width": 640,
"height": 480,
},
{
"profile_name": "profile2",
"extension": "mp4",
"width": 1920,
"height": 1080,
},
]
self.previous_uploads = [
{
"edx_video_id": "test1",
"client_video_id": "test1.mp4",
"duration": 42.0,
"status": "upload",
"encoded_videos": [],
},
{
"edx_video_id":
"test2",
"client_video_id":
"test2.mp4",
"duration":
128.0,
"status":
"file_complete",
"encoded_videos": [
{
"profile": "profile1",
"url": "http://example.com/profile1/test2.mp4",
"file_size": 1600,
"bitrate": 100,
},
{
"profile": "profile2",
"url": "http://example.com/profile2/test2.mov",
"file_size": 16000,
"bitrate": 1000,
},
],
},
{
"edx_video_id":
"non-ascii",
"client_video_id":
u"nón-ascii-näme.mp4",
"duration":
256.0,
"status":
"transcode_active",
"encoded_videos": [
{
"profile": "profile1",
"url":
u"http://example.com/profile1/nón-ascii-näme.mp4",
"file_size": 3200,
"bitrate": 100,
},
]
},
]
for profile in self.profiles:
create_profile(profile)
for video in self.previous_uploads:
create_video(video)
modulestore().save_asset_metadata(
AssetMetadata(
self.course.id.make_asset_key(VIDEO_ASSET_TYPE,
video["edx_video_id"])),
self.user.id)