def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ if CodeBlockTimer is None: raise SkipTest("CodeBlockTimer undefined.") desc = "XMLRoundTrip:{}->{}:{}".format( SHORT_NAME_MAP[source_ms], SHORT_NAME_MAP[dest_ms], num_assets ) with CodeBlockTimer(desc): with CodeBlockTimer("fake_assets"): # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) with source_ms.build() as (source_content, source_store): with dest_ms.build() as (dest_content, dest_store): source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') with CodeBlockTimer("initial_import"): import_course_from_xml( source_store, 'test_user', TEST_DATA_ROOT, source_dirs=TEST_COURSE, static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with CodeBlockTimer("export"): export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with CodeBlockTimer("second_import"): import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, create_if_not_present=True, raise_on_failure=True, )
def test_generate_find_timings(self, source_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ if CodeBlockTimer is None: raise SkipTest("CodeBlockTimer undefined.") desc = "FindAssetTest:{}:{}".format( SHORT_NAME_MAP[source_ms], num_assets, ) with CodeBlockTimer(desc): with CodeBlockTimer("fake_assets"): # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) with source_ms.build() as (source_content, source_store): source_course_key = source_store.make_course_key('a', 'course', 'course') asset_key = source_course_key.make_asset_key( AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif' ) with CodeBlockTimer("initial_import"): import_course_from_xml( source_store, 'test_user', TEST_DATA_ROOT, source_dirs=TEST_COURSE, static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with CodeBlockTimer("find_nonexistent_asset"): # More correct would be using the AssetManager.find() - but since the test # has created its own test modulestore, the AssetManager can't be used. __ = source_store.find_asset_metadata(asset_key) # Perform get_all_asset_metadata for each sort. for sort in ALL_SORTS: with CodeBlockTimer("get_asset_list:{}-{}".format( sort[0], 'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc' )): # Grab two ranges of 50 assets using different sorts. # Why 50? That's how many are displayed on the current Studio "Files & Uploads" page. start_middle = num_assets / 2 __ = source_store.get_all_asset_metadata( source_course_key, 'asset', start=0, sort=sort, maxresults=50 ) __ = source_store.get_all_asset_metadata( source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50 )
def test_asset_sizes(self, source_ms, num_assets): """ Generate timings for different amounts of asset metadata and different modulestores. """ # First, make the fake asset metadata. make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) with source_ms.build() as (source_content, source_store): source_course_key = source_store.make_course_key('a', 'course', 'course') import_course_from_xml( source_store, 'test_user', TEST_DATA_ROOT, source_dirs=TEST_COURSE, static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) asset_collection = source_ms.asset_collection() # Ensure the asset collection exists. if asset_collection.name in asset_collection.database.collection_names(): # Map gets the size of each structure. mapper = Code(""" function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) } """) # Reduce finds the largest structure size and returns only it. reducer = Code(""" function(key, values) { var max_size = 0; for (var i=0; i < values.length; i++) { if (values[i] > max_size) { max_size = values[i]; } } return max_size; } """) results = asset_collection.map_reduce(mapper, reducer, "size_results") result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format( self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()] ) with open("bson_sizes.txt", "a") as f: f.write(result_str)
def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes): with store_builder.build() as (source_content, source_store): with store_builder.build() as (dest_content, dest_store): source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') # An extra import write occurs in the first Split import due to the mismatch between # the course id and the wiki_slug in the test XML course. The course must be updated # with the correct wiki_slug during import. with check_mongo_calls(import_reads, first_import_writes): import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['manual-testing-complete'], static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) with check_mongo_calls(export_reads): export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, 'exported_source_course', ) with check_mongo_calls(import_reads, second_import_writes): import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=['exported_source_course'], static_content_store=dest_content, target_id=dest_course_key, create_if_not_present=True, raise_on_failure=True, )
def test_number_mongo_calls(self, store, depth, lazy, access_all_block_fields, num_mongo_calls): with store.build() as (source_content, source_store): source_course_key = source_store.make_course_key('a', 'course', 'course') # First, import a course. import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['manual-testing-complete'], static_content_store=source_content, target_id=source_course_key, create_if_not_present=True, raise_on_failure=True, ) # Course traversal modeled after the traversal done here: # lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline # Starting at the root course block, do a breadth-first traversal using # get_children() to retrieve each block's children. with check_mongo_calls(num_mongo_calls): with source_store.bulk_operations(source_course_key): start_block = source_store.get_course(source_course_key, depth=depth, lazy=lazy) all_blocks = [] stack = [start_block] while stack: curr_block = stack.pop() all_blocks.append(curr_block) if curr_block.has_children: for block in reversed(curr_block.get_children()): stack.append(block) if access_all_block_fields: # Read the fields on each block in order to ensure each block and its definition is loaded. for xblock in all_blocks: for __, field in xblock.fields.iteritems(): if field.is_set_on(xblock): __ = field.read_from(xblock)
def initdb(cls): # connect to the db doc_store_config = { 'host': HOST, 'port': PORT, 'db': DB, 'collection': COLLECTION, } cls.add_asset_collection(doc_store_config) # since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class # as well content_store = MongoContentStore(HOST, DB, port=PORT) # # Also test draft store imports # draft_store = DraftModuleStore( content_store, doc_store_config, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS, branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred, xblock_mixins=(EditInfoMixin, InheritanceMixin, LocationMixin, XModuleMixin)) with patch('xmodule.tabs.CourseTab.from_json', side_effect=mock_tab_from_json): import_course_from_xml(draft_store, 999, DATA_DIR, cls.courses, static_content_store=content_store) # also test a course with no importing of static content import_course_from_xml(draft_store, 999, DATA_DIR, ['test_import_course'], static_content_store=content_store, do_import_static=False, verbose=True) # also import a course under a different course_id (especially ORG) import_course_from_xml(draft_store, 999, DATA_DIR, ['test_import_course'], static_content_store=content_store, do_import_static=False, verbose=True, target_id=SlashSeparatedCourseKey( 'guestx', 'foo', 'bar')) return content_store, draft_store
def test_round_trip(self, source_builder, dest_builder, source_content_builder, dest_content_builder, course_data_name, _mock_tab_from_json): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with source_builder.build( contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with dest_builder.build( contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key( 'a', 'course', 'course') dest_course_key = dest_store.make_course_key( 'a', 'course', 'course') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=[course_data_name], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME, ) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=[EXPORTED_COURSE_DIR_NAME], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) # NOT CURRENTLY USED # export_course_to_xml( # dest_store, # dest_content, # dest_course_key, # self.export_dir, # 'exported_dest_course', # ) self.exclude_field(None, 'wiki_slug') self.exclude_field(None, 'xml_attributes') self.exclude_field(None, 'parent') # discussion_ids are auto-generated based on usage_id, so they should change across # modulestores - see TNL-5001 self.exclude_field(None, 'discussion_id') self.ignore_asset_key('_id') self.ignore_asset_key('uploadDate') self.ignore_asset_key('content_son') self.ignore_asset_key('thumbnail_location') self.assertCoursesEqual( source_store, source_course_key, dest_store, dest_course_key, ) self.assertAssetsEqual( source_content, source_course_key, dest_content, dest_course_key, ) self.assertAssetsMetadataEqual( source_store, source_course_key, dest_store, dest_course_key, )
def test_split_course_export_import(self): # Construct the contentstore for storing the first import with MongoContentstoreBuilder().build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) with SPLIT_MODULESTORE_SETUP.build( contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with MongoContentstoreBuilder().build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) with SPLIT_MODULESTORE_SETUP.build( contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key( 'a', 'source', '2015_Fall') dest_course_key = dest_store.make_course_key( 'a', 'dest', '2015_Fall') import_course_from_xml( source_store, 'test_user', TEST_DATA_DIR, source_dirs=['split_course_with_static_tabs'], static_content_store=source_content, target_id=source_course_key, raise_on_failure=True, create_if_not_present=True, ) export_course_to_xml( source_store, source_content, source_course_key, self.export_dir, EXPORTED_COURSE_DIR_NAME, ) source_course = source_store.get_course( source_course_key, depth=None, lazy=False) self.assertEqual(source_course.url_name, 'course') export_dir_path = path(self.export_dir) policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course.url_name policy_path = policy_dir / 'policy.json' self.assertTrue(os.path.exists(policy_path)) import_course_from_xml( dest_store, 'test_user', self.export_dir, source_dirs=[EXPORTED_COURSE_DIR_NAME], static_content_store=dest_content, target_id=dest_course_key, raise_on_failure=True, create_if_not_present=True, ) dest_course = dest_store.get_course(dest_course_key, depth=None, lazy=False) self.assertEqual(dest_course.url_name, 'course')