def _get_all_content_for_course(self, course_key, get_thumbnails=False, start=0, maxresults=-1, sort=None):
'''
Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example:
[
{u'displayname': u'profile.jpg', u'chunkSize': 262144, u'length': 85374,
u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 183000), u'contentType': u'image/jpeg',
u'_id': {u'category': u'asset', u'name': u'profile.jpg', u'course': u'6.002x', u'tag': u'c4x',
u'org': u'MITx', u'revision': None}, u'md5': u'36dc53519d4b735eb6beba51cd686a0e'},
{u'displayname': u'profile.thumbnail.jpg', u'chunkSize': 262144, u'length': 4073,
u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 196000), u'contentType': u'image/jpeg',
u'_id': {u'category': u'asset', u'name': u'profile.thumbnail.jpg', u'course': u'6.002x', u'tag': u'c4x',
u'org': u'MITx', u'revision': None}, u'md5': u'ff1532598830e3feac91c2449eaa60d6'},
....
]
'''
course_filter = course_key.make_asset_key(
"asset" if not get_thumbnails else "thumbnail",
None
)
# 'borrow' the function 'location_to_query' from the Mongo modulestore implementation
if maxresults > 0:
items = self.fs_files.find(
location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG),
skip=start, limit=maxresults, sort=sort
)
else:
items = self.fs_files.find(location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG), sort=sort)
count = items.count()
return list(items), count
def _get_all_content_for_course(self, location, get_thumbnails=False, start=0, maxresults=-1, sort=None):
'''
Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example:
[
{u'displayname': u'profile.jpg', u'chunkSize': 262144, u'length': 85374,
u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 183000), u'contentType': u'image/jpeg',
u'_id': {u'category': u'asset', u'name': u'profile.jpg', u'course': u'6.002x', u'tag': u'c4x',
u'org': u'MITx', u'revision': None}, u'md5': u'36dc53519d4b735eb6beba51cd686a0e'},
{u'displayname': u'profile.thumbnail.jpg', u'chunkSize': 262144, u'length': 4073,
u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 196000), u'contentType': u'image/jpeg',
u'_id': {u'category': u'asset', u'name': u'profile.thumbnail.jpg', u'course': u'6.002x', u'tag': u'c4x',
u'org': u'MITx', u'revision': None}, u'md5': u'ff1532598830e3feac91c2449eaa60d6'},
....
]
'''
course_filter = Location(XASSET_LOCATION_TAG, category="asset" if not get_thumbnails else "thumbnail",
course=location.course, org=location.org)
# 'borrow' the function 'location_to_query' from the Mongo modulestore implementation
if maxresults > 0:
items = self.fs_files.find(
location_to_query(course_filter),
skip=start, limit=maxresults, sort=sort
)
else:
items = self.fs_files.find(location_to_query(course_filter), sort=sort)
count = items.count()
return list(items), count
def _get_all_content_for_course(self, location, get_thumbnails=False):
'''
Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example:
[
{u'displayname': u'profile.jpg', u'chunkSize': 262144, u'length': 85374,
u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 183000), u'contentType': u'image/jpeg',
u'_id': {u'category': u'asset', u'name': u'profile.jpg', u'course': u'6.002x', u'tag': u'c4x',
u'org': u'MITx', u'revision': None}, u'md5': u'36dc53519d4b735eb6beba51cd686a0e'},
{u'displayname': u'profile.thumbnail.jpg', u'chunkSize': 262144, u'length': 4073,
u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 196000), u'contentType': u'image/jpeg',
u'_id': {u'category': u'asset', u'name': u'profile.thumbnail.jpg', u'course': u'6.002x', u'tag': u'c4x',
u'org': u'MITx', u'revision': None}, u'md5': u'ff1532598830e3feac91c2449eaa60d6'},
....
]
'''
course_filter = Location(
XASSET_LOCATION_TAG,
category="asset" if not get_thumbnails else "thumbnail",
course=location.course,
org=location.org)
# 'borrow' the function 'location_to_query' from the Mongo modulestore implementation
items = self.fs_files.find(location_to_query(course_filter))
return list(items)
def test_export_all_courses(self):
"""
This test validates that redundant Mac metadata files ('._example.txt', '.DS_Store') are
cleaned up on import
"""
import_course_from_xml(self.module_store,
'**replace_user**',
TEST_DATA_DIR, ['course_ignore'],
static_content_store=self.content_store,
do_import_static=True,
verbose=True)
course = self.module_store.get_course(
CourseKey.from_string('/'.join(
['edX', 'course_ignore', '2014_Fall'])))
self.assertIsNotNone(course)
# check that there are two assets ['example.txt', '.example.txt'] in contentstore for imported course
all_assets, count = self.content_store.get_all_content_for_course(
course.id)
self.assertEqual(count, 2)
self.assertEqual(set([asset['_id']['name'] for asset in all_assets]),
set([u'.example.txt', u'example.txt']))
# manually add redundant assets (file ".DS_Store" and filename starts with "._")
course_filter = course.id.make_asset_key("asset", None)
query = location_to_query(course_filter,
wildcard=True,
tag=XASSET_LOCATION_TAG)
query['_id.name'] = all_assets[0]['_id']['name']
asset_doc = self.content_store.fs_files.find_one(query)
asset_doc['_id']['name'] = u'._example_test.txt'
self.content_store.fs_files.insert(asset_doc)
asset_doc['_id']['name'] = u'.DS_Store'
self.content_store.fs_files.insert(asset_doc)
# check that now course has four assets
all_assets, count = self.content_store.get_all_content_for_course(
course.id)
self.assertEqual(count, 4)
self.assertEqual(
set([asset['_id']['name'] for asset in all_assets]),
set([
u'.example.txt', u'example.txt', u'._example_test.txt',
u'.DS_Store'
]))
# now call asset_cleanup command and check that there is only two proper assets in contentstore for the course
call_command('cleanup_assets')
all_assets, count = self.content_store.get_all_content_for_course(
course.id)
self.assertEqual(count, 2)
self.assertEqual(set([asset['_id']['name'] for asset in all_assets]),
set([u'.example.txt', u'example.txt']))
def get_attrs(self, location):
"""
Gets all of the attributes associated with the given asset. Note, returns even built in attrs
such as md5 which you cannot resubmit in an update; so, don't call set_attrs with the result of this
but only with the set of attrs you want to explicitly update.
The attrs will be a superset of _id, contentType, chunkSize, filename, uploadDate, & md5
:param location: a c4x asset location
"""
item = self.fs_files.find_one(location_to_query(location))
if item is None:
raise NotFoundError()
return item
def get_attrs(self, location):
"""
Gets all of the attributes associated with the given asset. Note, returns even built in attrs
such as md5 which you cannot resubmit in an update; so, don't call set_attrs with the result of this
but only with the set of attrs you want to explicitly update.
The attrs will be a superset of _id, contentType, chunkSize, filename, uploadDate, & md5
:param location: a c4x asset location
"""
item = self.fs_files.find_one(location_to_query(location))
if item is None:
raise NotFoundError()
return item
def test_export_all_courses(self):
"""
This test validates that redundant Mac metadata files ('._example.txt', '.DS_Store') are
cleaned up on import
"""
import_from_xml(
self.module_store,
'**replace_user**',
'common/test/data/',
['dot-underscore'],
static_content_store=self.content_store,
do_import_static=True,
verbose=True
)
course = self.module_store.get_course(SlashSeparatedCourseKey('edX', 'dot-underscore', '2014_Fall'))
self.assertIsNotNone(course)
# check that there are two assets ['example.txt', '.example.txt'] in contentstore for imported course
all_assets, count = self.content_store.get_all_content_for_course(course.id)
self.assertEqual(count, 2)
self.assertEqual(all_assets[0]['_id']['name'], u'.example.txt')
self.assertEqual(all_assets[1]['_id']['name'], u'example.txt')
# manually add redundant assets (file ".DS_Store" and filename starts with "._")
course_filter = course.id.make_asset_key("asset", None)
query = location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG)
query['_id.name'] = all_assets[0]['_id']['name']
asset_doc = self.content_store.fs_files.find_one(query)
asset_doc['_id']['name'] = u'._example_test.txt'
self.content_store.fs_files.insert(asset_doc)
asset_doc['_id']['name'] = u'.DS_Store'
self.content_store.fs_files.insert(asset_doc)
# check that now course has four assets
all_assets, count = self.content_store.get_all_content_for_course(course.id)
self.assertEqual(count, 4)
self.assertEqual(all_assets[0]['_id']['name'], u'.example.txt')
self.assertEqual(all_assets[1]['_id']['name'], u'example.txt')
self.assertEqual(all_assets[2]['_id']['name'], u'._example_test.txt')
self.assertEqual(all_assets[3]['_id']['name'], u'.DS_Store')
# now call asset_cleanup command and check that there is only two proper assets in contentstore for the course
call_command('cleanup_assets')
all_assets, count = self.content_store.get_all_content_for_course(course.id)
self.assertEqual(count, 2)
self.assertEqual(all_assets[0]['_id']['name'], u'.example.txt')
self.assertEqual(all_assets[1]['_id']['name'], u'example.txt')
def set_attrs(self, location, attr_dict):
"""
Like set_attr but sets multiple key value pairs.
Returns nothing.
Raises NotFoundError if no such item exists
Raises AttributeError is attr_dict has any attrs which are one of the build in attrs.
:param location: a c4x asset location
"""
for attr in attr_dict.iterkeys():
if attr in ['_id', 'md5', 'uploadDate', 'length']:
raise AttributeError("{} is a protected attribute.".format(attr))
item = self.fs_files.find_one(location_to_query(location))
if item is None:
raise NotFoundError()
self.fs_files.update({"_id": item["_id"]}, {"$set": attr_dict})
def set_attrs(self, location, attr_dict):
"""
Like set_attr but sets multiple key value pairs.
Returns nothing.
Raises NotFoundError if no such item exists
Raises AttributeError is attr_dict has any attrs which are one of the build in attrs.
:param location: a c4x asset location
"""
for attr in attr_dict.iterkeys():
if attr in ['_id', 'md5', 'uploadDate', 'length']:
raise AttributeError(
"{} is a protected attribute.".format(attr))
item = self.fs_files.find_one(location_to_query(location))
if item is None:
raise NotFoundError()
self.fs_files.update({"_id": item["_id"]}, {"$set": attr_dict})
def convert_to_draft(self, source_location):
"""
Create a copy of the source and mark its revision as draft.
:param source: the location of the source (its revision must be None)
"""
original = self.collection.find_one(location_to_query(source_location))
draft_location = as_draft(source_location)
if draft_location.category in DIRECT_ONLY_CATEGORIES:
raise InvalidVersionError(source_location)
original['_id'] = draft_location.dict()
try:
self.collection.insert(original)
except pymongo.errors.DuplicateKeyError:
raise DuplicateItemError(original['_id'])
self.refresh_cached_metadata_inheritance_tree(draft_location)
self.fire_updated_modulestore_signal(get_course_id_no_run(draft_location), draft_location)
return self._load_items([original])[0]
def convert_to_draft(self, source_location):
"""
Create a copy of the source and mark its revision as draft.
:param source: the location of the source (its revision must be None)
"""
original = self.collection.find_one(location_to_query(source_location))
draft_location = as_draft(source_location)
if draft_location.category in DIRECT_ONLY_CATEGORIES:
raise InvalidVersionError(source_location)
original['_id'] = draft_location.dict()
try:
self.collection.insert(original)
except pymongo.errors.DuplicateKeyError:
raise DuplicateItemError(original['_id'])
self.refresh_cached_metadata_inheritance_tree(draft_location)
self.fire_updated_modulestore_signal(
get_course_id_no_run(draft_location), draft_location)
return self._load_items([original])[0]
def create_asset_entries(self, cstore, number):
"""
Create the fake entries
"""
course_filter = Location(
XASSET_LOCATION_TAG, category='asset', course=self.course.location.course, org=self.course.location.org
)
# purge existing entries (a bit brutal but hopefully tests are independent enuf to not trip on this)
cstore.fs_files.remove(location_to_query(course_filter))
base_entry = {
'displayname': 'foo.jpg',
'chunkSize': 262144,
'length': 0,
'uploadDate': datetime(2012, 1, 2, 0, 0),
'contentType': 'image/jpeg',
}
for i in range(number):
base_entry['displayname'] = '{:03x}.jpeg'.format(i)
base_entry['uploadDate'] += timedelta(hours=i)
base_entry['_id'] = course_filter.replace(name=base_entry['displayname']).dict()
cstore.fs_files.insert(base_entry)
return course_filter.dict()