def _get_all_content_for_course(self, course_key, get_thumbnails=False, start=0, maxresults=-1, sort=None): ''' Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example: [ {u'displayname': u'profile.jpg', u'chunkSize': 262144, u'length': 85374, u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 183000), u'contentType': u'image/jpeg', u'_id': {u'category': u'asset', u'name': u'profile.jpg', u'course': u'6.002x', u'tag': u'c4x', u'org': u'MITx', u'revision': None}, u'md5': u'36dc53519d4b735eb6beba51cd686a0e'}, {u'displayname': u'profile.thumbnail.jpg', u'chunkSize': 262144, u'length': 4073, u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 196000), u'contentType': u'image/jpeg', u'_id': {u'category': u'asset', u'name': u'profile.thumbnail.jpg', u'course': u'6.002x', u'tag': u'c4x', u'org': u'MITx', u'revision': None}, u'md5': u'ff1532598830e3feac91c2449eaa60d6'}, .... ] ''' course_filter = course_key.make_asset_key( "asset" if not get_thumbnails else "thumbnail", None ) # 'borrow' the function 'location_to_query' from the Mongo modulestore implementation if maxresults > 0: items = self.fs_files.find( location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG), skip=start, limit=maxresults, sort=sort ) else: items = self.fs_files.find(location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG), sort=sort) count = items.count() return list(items), count
def _get_all_content_for_course(self, location, get_thumbnails=False, start=0, maxresults=-1, sort=None): ''' Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example: [ {u'displayname': u'profile.jpg', u'chunkSize': 262144, u'length': 85374, u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 183000), u'contentType': u'image/jpeg', u'_id': {u'category': u'asset', u'name': u'profile.jpg', u'course': u'6.002x', u'tag': u'c4x', u'org': u'MITx', u'revision': None}, u'md5': u'36dc53519d4b735eb6beba51cd686a0e'}, {u'displayname': u'profile.thumbnail.jpg', u'chunkSize': 262144, u'length': 4073, u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 196000), u'contentType': u'image/jpeg', u'_id': {u'category': u'asset', u'name': u'profile.thumbnail.jpg', u'course': u'6.002x', u'tag': u'c4x', u'org': u'MITx', u'revision': None}, u'md5': u'ff1532598830e3feac91c2449eaa60d6'}, .... ] ''' course_filter = Location(XASSET_LOCATION_TAG, category="asset" if not get_thumbnails else "thumbnail", course=location.course, org=location.org) # 'borrow' the function 'location_to_query' from the Mongo modulestore implementation if maxresults > 0: items = self.fs_files.find( location_to_query(course_filter), skip=start, limit=maxresults, sort=sort ) else: items = self.fs_files.find(location_to_query(course_filter), sort=sort) count = items.count() return list(items), count
def _get_all_content_for_course(self, location, get_thumbnails=False): ''' Returns a list of all static assets for a course. The return format is a list of dictionary elements. Example: [ {u'displayname': u'profile.jpg', u'chunkSize': 262144, u'length': 85374, u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 183000), u'contentType': u'image/jpeg', u'_id': {u'category': u'asset', u'name': u'profile.jpg', u'course': u'6.002x', u'tag': u'c4x', u'org': u'MITx', u'revision': None}, u'md5': u'36dc53519d4b735eb6beba51cd686a0e'}, {u'displayname': u'profile.thumbnail.jpg', u'chunkSize': 262144, u'length': 4073, u'uploadDate': datetime.datetime(2012, 10, 3, 5, 41, 54, 196000), u'contentType': u'image/jpeg', u'_id': {u'category': u'asset', u'name': u'profile.thumbnail.jpg', u'course': u'6.002x', u'tag': u'c4x', u'org': u'MITx', u'revision': None}, u'md5': u'ff1532598830e3feac91c2449eaa60d6'}, .... ] ''' course_filter = Location( XASSET_LOCATION_TAG, category="asset" if not get_thumbnails else "thumbnail", course=location.course, org=location.org) # 'borrow' the function 'location_to_query' from the Mongo modulestore implementation items = self.fs_files.find(location_to_query(course_filter)) return list(items)
def test_export_all_courses(self): """ This test validates that redundant Mac metadata files ('._example.txt', '.DS_Store') are cleaned up on import """ import_course_from_xml(self.module_store, '**replace_user**', TEST_DATA_DIR, ['course_ignore'], static_content_store=self.content_store, do_import_static=True, verbose=True) course = self.module_store.get_course( CourseKey.from_string('/'.join( ['edX', 'course_ignore', '2014_Fall']))) self.assertIsNotNone(course) # check that there are two assets ['example.txt', '.example.txt'] in contentstore for imported course all_assets, count = self.content_store.get_all_content_for_course( course.id) self.assertEqual(count, 2) self.assertEqual(set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt'])) # manually add redundant assets (file ".DS_Store" and filename starts with "._") course_filter = course.id.make_asset_key("asset", None) query = location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG) query['_id.name'] = all_assets[0]['_id']['name'] asset_doc = self.content_store.fs_files.find_one(query) asset_doc['_id']['name'] = u'._example_test.txt' self.content_store.fs_files.insert(asset_doc) asset_doc['_id']['name'] = u'.DS_Store' self.content_store.fs_files.insert(asset_doc) # check that now course has four assets all_assets, count = self.content_store.get_all_content_for_course( course.id) self.assertEqual(count, 4) self.assertEqual( set([asset['_id']['name'] for asset in all_assets]), set([ u'.example.txt', u'example.txt', u'._example_test.txt', u'.DS_Store' ])) # now call asset_cleanup command and check that there is only two proper assets in contentstore for the course call_command('cleanup_assets') all_assets, count = self.content_store.get_all_content_for_course( course.id) self.assertEqual(count, 2) self.assertEqual(set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt']))
def get_attrs(self, location): """ Gets all of the attributes associated with the given asset. Note, returns even built in attrs such as md5 which you cannot resubmit in an update; so, don't call set_attrs with the result of this but only with the set of attrs you want to explicitly update. The attrs will be a superset of _id, contentType, chunkSize, filename, uploadDate, & md5 :param location: a c4x asset location """ item = self.fs_files.find_one(location_to_query(location)) if item is None: raise NotFoundError() return item
def test_export_all_courses(self): """ This test validates that redundant Mac metadata files ('._example.txt', '.DS_Store') are cleaned up on import """ import_from_xml( self.module_store, '**replace_user**', 'common/test/data/', ['dot-underscore'], static_content_store=self.content_store, do_import_static=True, verbose=True ) course = self.module_store.get_course(SlashSeparatedCourseKey('edX', 'dot-underscore', '2014_Fall')) self.assertIsNotNone(course) # check that there are two assets ['example.txt', '.example.txt'] in contentstore for imported course all_assets, count = self.content_store.get_all_content_for_course(course.id) self.assertEqual(count, 2) self.assertEqual(all_assets[0]['_id']['name'], u'.example.txt') self.assertEqual(all_assets[1]['_id']['name'], u'example.txt') # manually add redundant assets (file ".DS_Store" and filename starts with "._") course_filter = course.id.make_asset_key("asset", None) query = location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG) query['_id.name'] = all_assets[0]['_id']['name'] asset_doc = self.content_store.fs_files.find_one(query) asset_doc['_id']['name'] = u'._example_test.txt' self.content_store.fs_files.insert(asset_doc) asset_doc['_id']['name'] = u'.DS_Store' self.content_store.fs_files.insert(asset_doc) # check that now course has four assets all_assets, count = self.content_store.get_all_content_for_course(course.id) self.assertEqual(count, 4) self.assertEqual(all_assets[0]['_id']['name'], u'.example.txt') self.assertEqual(all_assets[1]['_id']['name'], u'example.txt') self.assertEqual(all_assets[2]['_id']['name'], u'._example_test.txt') self.assertEqual(all_assets[3]['_id']['name'], u'.DS_Store') # now call asset_cleanup command and check that there is only two proper assets in contentstore for the course call_command('cleanup_assets') all_assets, count = self.content_store.get_all_content_for_course(course.id) self.assertEqual(count, 2) self.assertEqual(all_assets[0]['_id']['name'], u'.example.txt') self.assertEqual(all_assets[1]['_id']['name'], u'example.txt')
def set_attrs(self, location, attr_dict): """ Like set_attr but sets multiple key value pairs. Returns nothing. Raises NotFoundError if no such item exists Raises AttributeError is attr_dict has any attrs which are one of the build in attrs. :param location: a c4x asset location """ for attr in attr_dict.iterkeys(): if attr in ['_id', 'md5', 'uploadDate', 'length']: raise AttributeError("{} is a protected attribute.".format(attr)) item = self.fs_files.find_one(location_to_query(location)) if item is None: raise NotFoundError() self.fs_files.update({"_id": item["_id"]}, {"$set": attr_dict})
def set_attrs(self, location, attr_dict): """ Like set_attr but sets multiple key value pairs. Returns nothing. Raises NotFoundError if no such item exists Raises AttributeError is attr_dict has any attrs which are one of the build in attrs. :param location: a c4x asset location """ for attr in attr_dict.iterkeys(): if attr in ['_id', 'md5', 'uploadDate', 'length']: raise AttributeError( "{} is a protected attribute.".format(attr)) item = self.fs_files.find_one(location_to_query(location)) if item is None: raise NotFoundError() self.fs_files.update({"_id": item["_id"]}, {"$set": attr_dict})
def convert_to_draft(self, source_location): """ Create a copy of the source and mark its revision as draft. :param source: the location of the source (its revision must be None) """ original = self.collection.find_one(location_to_query(source_location)) draft_location = as_draft(source_location) if draft_location.category in DIRECT_ONLY_CATEGORIES: raise InvalidVersionError(source_location) original['_id'] = draft_location.dict() try: self.collection.insert(original) except pymongo.errors.DuplicateKeyError: raise DuplicateItemError(original['_id']) self.refresh_cached_metadata_inheritance_tree(draft_location) self.fire_updated_modulestore_signal(get_course_id_no_run(draft_location), draft_location) return self._load_items([original])[0]
def convert_to_draft(self, source_location): """ Create a copy of the source and mark its revision as draft. :param source: the location of the source (its revision must be None) """ original = self.collection.find_one(location_to_query(source_location)) draft_location = as_draft(source_location) if draft_location.category in DIRECT_ONLY_CATEGORIES: raise InvalidVersionError(source_location) original['_id'] = draft_location.dict() try: self.collection.insert(original) except pymongo.errors.DuplicateKeyError: raise DuplicateItemError(original['_id']) self.refresh_cached_metadata_inheritance_tree(draft_location) self.fire_updated_modulestore_signal( get_course_id_no_run(draft_location), draft_location) return self._load_items([original])[0]
def create_asset_entries(self, cstore, number): """ Create the fake entries """ course_filter = Location( XASSET_LOCATION_TAG, category='asset', course=self.course.location.course, org=self.course.location.org ) # purge existing entries (a bit brutal but hopefully tests are independent enuf to not trip on this) cstore.fs_files.remove(location_to_query(course_filter)) base_entry = { 'displayname': 'foo.jpg', 'chunkSize': 262144, 'length': 0, 'uploadDate': datetime(2012, 1, 2, 0, 0), 'contentType': 'image/jpeg', } for i in range(number): base_entry['displayname'] = '{:03x}.jpeg'.format(i) base_entry['uploadDate'] += timedelta(hours=i) base_entry['_id'] = course_filter.replace(name=base_entry['displayname']).dict() cstore.fs_files.insert(base_entry) return course_filter.dict()