def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
if CodeBlockTimer is None:
raise SkipTest("CodeBlockTimer undefined.")
desc = "XMLRoundTrip:{}->{}:{}".format(
SHORT_NAME_MAP[source_ms],
SHORT_NAME_MAP[dest_ms],
num_assets
)
with CodeBlockTimer(desc):
with CodeBlockTimer("fake_assets"):
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
with source_ms.build() as (source_content, source_store):
with dest_ms.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
with CodeBlockTimer("initial_import"):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
source_dirs=TEST_COURSE,
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with CodeBlockTimer("export"):
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with CodeBlockTimer("second_import"):
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
def test_generate_find_timings(self, source_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
if CodeBlockTimer is None:
raise SkipTest("CodeBlockTimer undefined.")
desc = "FindAssetTest:{}:{}".format(
SHORT_NAME_MAP[source_ms],
num_assets,
)
with CodeBlockTimer(desc):
with CodeBlockTimer("fake_assets"):
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
with source_ms.build() as (source_content, source_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
asset_key = source_course_key.make_asset_key(
AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif'
)
with CodeBlockTimer("initial_import"):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
source_dirs=TEST_COURSE,
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with CodeBlockTimer("find_nonexistent_asset"):
# More correct would be using the AssetManager.find() - but since the test
# has created its own test modulestore, the AssetManager can't be used.
__ = source_store.find_asset_metadata(asset_key)
# Perform get_all_asset_metadata for each sort.
for sort in ALL_SORTS:
with CodeBlockTimer("get_asset_list:{}-{}".format(
sort[0],
'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc'
)):
# Grab two ranges of 50 assets using different sorts.
# Why 50? That's how many are displayed on the current Studio "Files & Uploads" page.
start_middle = num_assets / 2
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=0, sort=sort, maxresults=50
)
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50
)
def test_asset_sizes(self, source_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
with source_ms.build() as (source_content, source_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
source_dirs=TEST_COURSE,
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
asset_collection = source_ms.asset_collection()
# Ensure the asset collection exists.
if asset_collection.name in asset_collection.database.collection_names():
# Map gets the size of each structure.
mapper = Code("""
function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) }
""")
# Reduce finds the largest structure size and returns only it.
reducer = Code("""
function(key, values) {
var max_size = 0;
for (var i=0; i < values.length; i++) {
if (values[i] > max_size) {
max_size = values[i];
}
}
return max_size;
}
""")
results = asset_collection.map_reduce(mapper, reducer, "size_results")
result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format(
self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()]
)
with open("bson_sizes.txt", "a") as f:
f.write(result_str)
def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes):
with store_builder.build() as (source_content, source_store):
with store_builder.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
# An extra import write occurs in the first Split import due to the mismatch between
# the course id and the wiki_slug in the test XML course. The course must be updated
# with the correct wiki_slug during import.
with check_mongo_calls(import_reads, first_import_writes):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['manual-testing-complete'],
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with check_mongo_calls(export_reads):
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with check_mongo_calls(import_reads, second_import_writes):
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
def test_number_mongo_calls(self, store, depth, lazy, access_all_block_fields, num_mongo_calls):
with store.build() as (source_content, source_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
# First, import a course.
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['manual-testing-complete'],
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
# Course traversal modeled after the traversal done here:
# lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline
# Starting at the root course block, do a breadth-first traversal using
# get_children() to retrieve each block's children.
with check_mongo_calls(num_mongo_calls):
with source_store.bulk_operations(source_course_key):
start_block = source_store.get_course(source_course_key, depth=depth, lazy=lazy)
all_blocks = []
stack = [start_block]
while stack:
curr_block = stack.pop()
all_blocks.append(curr_block)
if curr_block.has_children:
for block in reversed(curr_block.get_children()):
stack.append(block)
if access_all_block_fields:
# Read the fields on each block in order to ensure each block and its definition is loaded.
for xblock in all_blocks:
for __, field in xblock.fields.iteritems():
if field.is_set_on(xblock):
__ = field.read_from(xblock)
def initdb(cls):
# connect to the db
doc_store_config = {
'host': HOST,
'port': PORT,
'db': DB,
'collection': COLLECTION,
}
cls.add_asset_collection(doc_store_config)
# since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
# as well
content_store = MongoContentStore(HOST, DB, port=PORT)
#
# Also test draft store imports
#
draft_store = DraftModuleStore(
content_store,
doc_store_config,
FS_ROOT,
RENDER_TEMPLATE,
default_class=DEFAULT_CLASS,
branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred,
xblock_mixins=(EditInfoMixin, InheritanceMixin, LocationMixin,
XModuleMixin))
with patch('xmodule.tabs.CourseTab.from_json',
side_effect=mock_tab_from_json):
import_course_from_xml(draft_store,
999,
DATA_DIR,
cls.courses,
static_content_store=content_store)
# also test a course with no importing of static content
import_course_from_xml(draft_store,
999,
DATA_DIR, ['test_import_course'],
static_content_store=content_store,
do_import_static=False,
verbose=True)
# also import a course under a different course_id (especially ORG)
import_course_from_xml(draft_store,
999,
DATA_DIR, ['test_import_course'],
static_content_store=content_store,
do_import_static=False,
verbose=True,
target_id=SlashSeparatedCourseKey(
'guestx', 'foo', 'bar'))
return content_store, draft_store
def test_round_trip(self, source_builder, dest_builder,
source_content_builder, dest_content_builder,
course_data_name, _mock_tab_from_json):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(
contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(
contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=[course_data_name],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
EXPORTED_COURSE_DIR_NAME,
)
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=[EXPORTED_COURSE_DIR_NAME],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
# NOT CURRENTLY USED
# export_course_to_xml(
# dest_store,
# dest_content,
# dest_course_key,
# self.export_dir,
# 'exported_dest_course',
# )
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.exclude_field(None, 'parent')
# discussion_ids are auto-generated based on usage_id, so they should change across
# modulestores - see TNL-5001
self.exclude_field(None, 'discussion_id')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
self.assertAssetsMetadataEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
def test_split_course_export_import(self):
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with SPLIT_MODULESTORE_SETUP.build(
contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with SPLIT_MODULESTORE_SETUP.build(
contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key(
'a', 'source', '2015_Fall')
dest_course_key = dest_store.make_course_key(
'a', 'dest', '2015_Fall')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['split_course_with_static_tabs'],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
EXPORTED_COURSE_DIR_NAME,
)
source_course = source_store.get_course(
source_course_key, depth=None, lazy=False)
self.assertEqual(source_course.url_name, 'course')
export_dir_path = path(self.export_dir)
policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course.url_name
policy_path = policy_dir / 'policy.json'
self.assertTrue(os.path.exists(policy_path))
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=[EXPORTED_COURSE_DIR_NAME],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
dest_course = dest_store.get_course(dest_course_key,
depth=None,
lazy=False)
self.assertEqual(dest_course.url_name, 'course')