def setUpClass(cls):
super(ContentStoreToyCourseTest, cls).setUpClass()
cls.contentstore = contentstore()
cls.modulestore = modulestore()
cls.course_key = cls.modulestore.make_course_key('edX', 'toy', '2012_Fall')
import_course_from_xml(
cls.modulestore, 1, TEST_DATA_DIR, ['toy'],
static_content_store=cls.contentstore, verbose=True
)
# A locked asset
cls.locked_asset = cls.course_key.make_asset_key('asset', 'sample_static.html')
cls.url_locked = six.text_type(cls.locked_asset)
cls.url_locked_versioned = get_versioned_asset_url(cls.url_locked)
cls.url_locked_versioned_old_style = get_old_style_versioned_asset_url(cls.url_locked)
cls.contentstore.set_attr(cls.locked_asset, 'locked', True)
# An unlocked asset
cls.unlocked_asset = cls.course_key.make_asset_key('asset', 'another_static.txt')
cls.url_unlocked = six.text_type(cls.unlocked_asset)
cls.url_unlocked_versioned = get_versioned_asset_url(cls.url_unlocked)
cls.url_unlocked_versioned_old_style = get_old_style_versioned_asset_url(cls.url_unlocked)
cls.length_unlocked = cls.contentstore.get_attr(cls.unlocked_asset, 'length')
def test_rewrite_reference_list(self):
# This test fails with split modulestore (the HTML component is not in "different_course_id" namespace).
# More investigation needs to be done.
module_store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
target_id = module_store.make_course_key('testX', 'conditional_copy', 'copy_run')
import_course_from_xml(
module_store,
self.user.id,
TEST_DATA_DIR,
['conditional'],
target_id=target_id
)
conditional_module = module_store.get_item(
target_id.make_usage_key('conditional', 'condone')
)
self.assertIsNotNone(conditional_module)
different_course_id = module_store.make_course_key('edX', 'different_course', None)
self.assertListEqual(
[
target_id.make_usage_key('problem', 'choiceprob'),
different_course_id.make_usage_key('html', 'for_testing_import_rewrites')
],
conditional_module.sources_list
)
self.assertListEqual(
[
target_id.make_usage_key('html', 'congrats'),
target_id.make_usage_key('html', 'secret_page')
],
conditional_module.show_tag_list
)
def load_courses(cls):
"""
Load test courses and return list of ids
"""
store = modulestore()
unique_org = factory.Sequence(lambda n: 'edX.%d' % n)
cls.course = CourseFactory.create(
emit_signals=True,
org=unique_org,
course='simple',
run="run",
display_name=u'2012_Fáĺĺ',
modulestore=store
)
cls.discussion = ItemFactory.create(
category='discussion', parent_location=cls.course.location
)
courses = store.get_courses()
# NOTE: if xml store owns these, it won't import them into mongo
if cls.test_course_key not in [c.id for c in courses]:
import_course_from_xml(
store, ModuleStoreEnum.UserID.mgmt_command, DATA_DIR, XML_COURSE_DIRS, create_if_not_present=True
)
return [course.id for course in store.get_courses()]
def setUp(self):
"""
Set up the tests
"""
super(StaticTabDateTestCaseXML, self).setUp()
# The following XML test course (which lives at common/test/data/2014)
# is closed; we're testing that tabs still appear when
# the course is already closed
self.xml_course_key = self.store.make_course_key('edX', 'detached_pages', '2014')
import_course_from_xml(
self.store,
'test_user',
TEST_DATA_DIR,
source_dirs=['2014'],
static_content_store=None,
target_id=self.xml_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
# this text appears in the test course's tab
# common/test/data/2014/tabs/8e4cce2b4aaf4ba28b1220804619e41f.html
self.xml_data = "static 463139"
self.xml_url = "8e4cce2b4aaf4ba28b1220804619e41f"
def import_test_course(self, solution_attribute=None, solution_element=None):
"""
Import the test course with the sga unit
"""
# adapted from edx-platform/cms/djangoapps/contentstore/management/commands/tests/test_cleanup_assets.py
root = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
input_dir = os.path.join(root, "test_data")
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(temp_dir))
xml_dir = os.path.join(temp_dir, "xml")
shutil.copytree(input_dir, xml_dir)
with open(os.path.join(xml_dir, "2017_SGA", "vertical", "vertical.xml"), "w") as f:
f.write(self.make_test_vertical(solution_attribute, solution_element))
store = modulestore()
import_course_from_xml(
store,
'sga_user',
xml_dir,
)
return store.get_course(CourseLocator.from_string('SGAU/SGA101/course'))
def setUp(self):
"""
Create user and login.
"""
self.staff_pwd = super(ContentStoreToyCourseTest, self).setUp()
self.staff_usr = self.user
self.non_staff_usr, self.non_staff_pwd = self.create_non_staff_user()
self.client = Client()
self.contentstore = contentstore()
store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) # pylint: disable=protected-access
self.course_key = store.make_course_key('edX', 'toy', '2012_Fall')
import_course_from_xml(
store, self.user.id, TEST_DATA_DIR, ['toy'],
static_content_store=self.contentstore, verbose=True
)
# A locked asset
self.locked_asset = self.course_key.make_asset_key('asset', 'sample_static.txt')
self.url_locked = unicode(self.locked_asset)
self.contentstore.set_attr(self.locked_asset, 'locked', True)
# An unlocked asset
self.unlocked_asset = self.course_key.make_asset_key('asset', 'another_static.txt')
self.url_unlocked = unicode(self.unlocked_asset)
self.length_unlocked = self.contentstore.get_attr(self.unlocked_asset, 'length')
def test_generate_import_export_timings(self, source_ms, dest_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
if CodeBlockTimer is None:
raise SkipTest("CodeBlockTimer undefined.")
desc = "XMLRoundTrip:{}->{}:{}".format(
SHORT_NAME_MAP[source_ms],
SHORT_NAME_MAP[dest_ms],
num_assets
)
with CodeBlockTimer(desc):
with CodeBlockTimer("fake_assets"):
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
with source_ms.build() as (source_content, source_store):
with dest_ms.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
with CodeBlockTimer("initial_import"):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
source_dirs=TEST_COURSE,
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with CodeBlockTimer("export"):
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with CodeBlockTimer("second_import"):
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
def test_generate_find_timings(self, source_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
if CodeBlockTimer is None:
raise SkipTest("CodeBlockTimer undefined.")
desc = "FindAssetTest:{}:{}".format(
SHORT_NAME_MAP[source_ms],
num_assets,
)
with CodeBlockTimer(desc):
with CodeBlockTimer("fake_assets"):
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
with source_ms.build() as (source_content, source_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
asset_key = source_course_key.make_asset_key(
AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif'
)
with CodeBlockTimer("initial_import"):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
source_dirs=TEST_COURSE,
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with CodeBlockTimer("find_nonexistent_asset"):
# More correct would be using the AssetManager.find() - but since the test
# has created its own test modulestore, the AssetManager can't be used.
__ = source_store.find_asset_metadata(asset_key)
# Perform get_all_asset_metadata for each sort.
for sort in ALL_SORTS:
with CodeBlockTimer("get_asset_list:{}-{}".format(
sort[0],
'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc'
)):
# Grab two ranges of 50 assets using different sorts.
# Why 50? That's how many are displayed on the current Studio "Files & Uploads" page.
start_middle = num_assets / 2
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=0, sort=sort, maxresults=50
)
__ = source_store.get_all_asset_metadata(
source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50
)
def test_split_course_export_import(self):
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with SPLIT_MODULESTORE_SETUP.build(contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with SPLIT_MODULESTORE_SETUP.build(contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key('a', 'source', '2015_Fall')
dest_course_key = dest_store.make_course_key('a', 'dest', '2015_Fall')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['split_course_with_static_tabs'],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
EXPORTED_COURSE_DIR_NAME,
)
source_course = source_store.get_course(source_course_key, depth=None, lazy=False)
self.assertEqual(source_course.url_name, 'course')
export_dir_path = path(self.export_dir)
policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course_key.run
policy_path = policy_dir / 'policy.json'
self.assertTrue(os.path.exists(policy_path))
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=[EXPORTED_COURSE_DIR_NAME],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
dest_course = dest_store.get_course(dest_course_key, depth=None, lazy=False)
self.assertEqual(dest_course.url_name, 'course')
def test_rewrite_reference(self):
module_store = modulestore()
target_id = module_store.make_course_key("testX", "peergrading_copy", "copy_run")
import_course_from_xml(
module_store, self.user.id, TEST_DATA_DIR, ["open_ended"], target_id=target_id, create_if_not_present=True
)
peergrading_module = module_store.get_item(target_id.make_usage_key("peergrading", "PeerGradingLinked"))
self.assertIsNotNone(peergrading_module)
self.assertEqual(
target_id.make_usage_key("combinedopenended", "SampleQuestion"), peergrading_module.link_to_location
)
def test_asset_sizes(self, source_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
with source_ms.build() as (source_content, source_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
source_dirs=TEST_COURSE,
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
asset_collection = source_ms.asset_collection()
# Ensure the asset collection exists.
if asset_collection.name in asset_collection.database.collection_names():
# Map gets the size of each structure.
mapper = Code("""
function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) }
""")
# Reduce finds the largest structure size and returns only it.
reducer = Code("""
function(key, values) {
var max_size = 0;
for (var i=0; i < values.length; i++) {
if (values[i] > max_size) {
max_size = values[i];
}
}
return max_size;
}
""")
results = asset_collection.map_reduce(mapper, reducer, "size_results")
result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format(
self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()]
)
with open("bson_sizes.txt", "a") as f:
f.write(result_str)
def setUpClass(cls):
super(CommandExecutionTestCase, cls).setUpClass()
cls.course_key = cls.store.make_course_key(u'edX', u'lti_provider', u'3000')
import_course_from_xml(
cls.store,
u'test_user',
TEST_DATA_DIR,
source_dirs=[u'simple'],
static_content_store=None,
target_id=cls.course_key,
raise_on_failure=True,
create_if_not_present=True,
)
cls.lti_block = u'block-v1:edX+lti_provider+3000+type@chapter+block@chapter_2'
def setUp(self):
super(TestPeerGradingFound, self).setUp()
self.user = factories.UserFactory()
store = modulestore()
course_items = import_course_from_xml(store, self.user.id, TEST_DATA_DIR, ['open_ended_nopath']) # pylint: disable=maybe-no-member
self.course = course_items[0]
self.course_key = self.course.id
def handle(self, *args, **options):
"Execute the command"
if len(args) == 0:
raise CommandError("import requires at least one argument: <data directory> [--nostatic] [<course dir>...]")
data_dir = args[0]
do_import_static = not options.get('nostatic', False)
if len(args) > 1:
source_dirs = args[1:]
else:
source_dirs = None
self.stdout.write("Importing. Data_dir={data}, source_dirs={courses}\n".format(
data=data_dir,
courses=source_dirs,
))
mstore = modulestore()
course_items = import_course_from_xml(
mstore, ModuleStoreEnum.UserID.mgmt_command, data_dir, source_dirs, load_error_modules=False,
static_content_store=contentstore(), verbose=True,
do_import_static=do_import_static,
create_if_not_present=True,
)
for course in course_items:
course_id = course.id
if not are_permissions_roles_seeded(course_id):
self.stdout.write('Seeding forum roles for course {0}\n'.format(course_id))
seed_permissions_roles(course_id)
def test_unicode_chars_in_course_name_import(self):
"""
# Test that importing course with unicode 'id' and 'display name' doesn't give UnicodeEncodeError
"""
# Test with the split modulestore because store.has_course fails in old mongo with unicode characters.
with modulestore().default_store(ModuleStoreEnum.Type.split):
module_store = modulestore()
course_id = module_store.make_course_key(u"Юникода", u"unicode_course", u"échantillon")
import_course_from_xml(
module_store, self.user.id, TEST_DATA_DIR, ["2014_Uni"], target_id=course_id, create_if_not_present=True
)
course = module_store.get_course(course_id)
self.assertIsNotNone(course)
# test that course 'display_name' same as imported course 'display_name'
self.assertEqual(course.display_name, u"Φυσικά το όνομα Unicode")
def _import_course(self, content_store, modulestore):
"""
Imports a course for testing.
Returns the course key.
"""
course_key = modulestore.make_course_key('a', 'course', 'course')
import_course_from_xml(
modulestore,
'test_user',
TEST_DATA_DIR,
source_dirs=['manual-testing-complete'],
static_content_store=content_store,
target_id=course_key,
create_if_not_present=True,
raise_on_failure=True,
)
return course_key
def test_export_all_courses(self):
"""
This test validates that redundant Mac metadata files ('._example.txt', '.DS_Store') are
cleaned up on import
"""
import_course_from_xml(
self.module_store,
'**replace_user**',
TEST_DATA_DIR,
['course_ignore'],
static_content_store=self.content_store,
do_import_static=True,
verbose=True
)
course = self.module_store.get_course(CourseKey.from_string('/'.join(['edX', 'course_ignore', '2014_Fall'])))
self.assertIsNotNone(course)
# check that there are two assets ['example.txt', '.example.txt'] in contentstore for imported course
all_assets, count = self.content_store.get_all_content_for_course(course.id)
self.assertEqual(count, 2)
self.assertEqual(set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt']))
# manually add redundant assets (file ".DS_Store" and filename starts with "._")
course_filter = course.id.make_asset_key("asset", None)
query = location_to_query(course_filter, wildcard=True, tag=XASSET_LOCATION_TAG)
query['_id.name'] = all_assets[0]['_id']['name']
asset_doc = self.content_store.fs_files.find_one(query)
asset_doc['_id']['name'] = u'._example_test.txt'
self.content_store.fs_files.insert(asset_doc)
asset_doc['_id']['name'] = u'.DS_Store'
self.content_store.fs_files.insert(asset_doc)
# check that now course has four assets
all_assets, count = self.content_store.get_all_content_for_course(course.id)
self.assertEqual(count, 4)
self.assertEqual(
set([asset['_id']['name'] for asset in all_assets]),
set([u'.example.txt', u'example.txt', u'._example_test.txt', u'.DS_Store'])
)
# now call asset_cleanup command and check that there is only two proper assets in contentstore for the course
call_command('cleanup_assets')
all_assets, count = self.content_store.get_all_content_for_course(course.id)
self.assertEqual(count, 2)
self.assertEqual(set([asset['_id']['name'] for asset in all_assets]), set([u'.example.txt', u'example.txt']))
def test_import_course_into_similar_namespace(self):
# Checks to make sure that a course with an org/course like
# edx/course can be imported into a namespace with an org/course
# like edx/course_name
module_store, __, course = self.load_test_import_course()
course_items = import_course_from_xml(
module_store, self.user.id, TEST_DATA_DIR, ["test_import_course_2"], target_id=course.id, verbose=True
)
self.assertEqual(len(course_items), 1)
def add_mobile_available_toy_course(self):
""" use toy course with handouts, and make it mobile_available """
course_items = import_course_from_xml(
self.store, self.user.id, settings.COMMON_TEST_DATA_ROOT, ["toy"], create_if_not_present=True
)
self.course = course_items[0]
self.course.mobile_available = True
self.store.update_item(self.course, self.user.id)
self.login_and_enroll()
def _verify_split_test_import(self, target_course_name, source_course_name, split_test_name, groups_to_verticals):
module_store = modulestore()
target_id = module_store.make_course_key("testX", target_course_name, "copy_run")
import_course_from_xml(
module_store,
self.user.id,
TEST_DATA_DIR,
[source_course_name],
target_id=target_id,
create_if_not_present=True,
)
split_test_module = module_store.get_item(target_id.make_usage_key("split_test", split_test_name))
self.assertIsNotNone(split_test_module)
remapped_verticals = {
key: target_id.make_usage_key("vertical", value) for key, value in groups_to_verticals.iteritems()
}
self.assertEqual(remapped_verticals, split_test_module.group_id_to_child)
def test_asset_import_nostatic(self):
'''
This test validates that an image asset is NOT imported when do_import_static=False
'''
content_store = contentstore()
module_store = modulestore()
import_course_from_xml(
module_store, self.user.id, TEST_DATA_DIR, ['toy'],
static_content_store=content_store, do_import_static=False,
create_if_not_present=True, verbose=True
)
course = module_store.get_course(module_store.make_course_key('edX', 'toy', '2012_Fall'))
# make sure we have NO assets in our contentstore
all_assets, count = content_store.get_all_content_for_course(course.id)
self.assertEqual(len(all_assets), 0)
self.assertEqual(count, 0)
def test_rewrite_reference_list(self):
# This test fails with split modulestore (the HTML component is not in "different_course_id" namespace).
# More investigation needs to be done.
module_store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
target_id = module_store.make_course_key("testX", "conditional_copy", "copy_run")
import_course_from_xml(module_store, self.user.id, TEST_DATA_DIR, ["conditional"], target_id=target_id)
conditional_module = module_store.get_item(target_id.make_usage_key("conditional", "condone"))
self.assertIsNotNone(conditional_module)
different_course_id = module_store.make_course_key("edX", "different_course", None)
self.assertListEqual(
[
target_id.make_usage_key("problem", "choiceprob"),
different_course_id.make_usage_key("html", "for_testing_import_rewrites"),
],
conditional_module.sources_list,
)
self.assertListEqual(
[target_id.make_usage_key("html", "congrats"), target_id.make_usage_key("html", "secret_page")],
conditional_module.show_tag_list,
)
def setUp(self):
"""
Set up the course and user context
"""
super(CoursesRenderTest, self).setUp()
store = modulestore()
course_items = import_course_from_xml(store, self.user.id, TEST_DATA_DIR, ['toy'])
course_key = course_items[0].id
self.course = get_course_by_id(course_key)
self.request = get_request_for_user(UserFactory.create())
def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes):
with store_builder.build() as (source_content, source_store):
with store_builder.build() as (dest_content, dest_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
dest_course_key = dest_store.make_course_key('a', 'course', 'course')
# An extra import write occurs in the first Split import due to the mismatch between
# the course id and the wiki_slug in the test XML course. The course must be updated
# with the correct wiki_slug during import.
with check_mongo_calls(import_reads, first_import_writes):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['manual-testing-complete'],
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with check_mongo_calls(export_reads):
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
with check_mongo_calls(import_reads, second_import_writes):
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
def load_courses(self):
"""Load test courses and return list of ids"""
store = modulestore()
# Add a course with a unicode name.
unique_org = factory.Sequence(lambda n: u'ëḋẌ.%d' % n)
CourseFactory.create(
org=unique_org,
course=u'śíḿṕĺé',
display_name=u'2012_Fáĺĺ',
modulestore=store
)
courses = store.get_courses()
# NOTE: if xml store owns these, it won't import them into mongo
if self.test_course_key not in [c.id for c in courses]:
import_course_from_xml(
store, ModuleStoreEnum.UserID.mgmt_command, DATA_DIR, XML_COURSE_DIRS, create_if_not_present=True
)
return [course.id for course in store.get_courses()]
def setUp(self):
super(TestStudentProblemList, self).setUp()
# Load an open ended course with several problems.
self.user = factories.UserFactory()
store = modulestore()
course_items = import_course_from_xml(store, self.user.id, TEST_DATA_DIR, ['open_ended']) # pylint: disable=maybe-no-member
self.course = course_items[0]
self.course_key = self.course.id
# Enroll our user in our course and make them an instructor.
make_instructor(self.course, self.user.email)
def setUp(self):
super(TestHandouts, self).setUp()
# Deleting handouts fails with split modulestore because the handout has no parent.
# This needs further investigation to determine if it is a bug in the split modulestore.
# pylint: disable=protected-access
self.store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo)
# use toy course with handouts, and make it mobile_available
course_items = import_course_from_xml(self.store, self.user.id, settings.COMMON_TEST_DATA_ROOT, ['toy'])
self.course = course_items[0]
self.course.mobile_available = True
self.store.update_item(self.course, self.user.id)
def test_number_mongo_calls(self, store, depth, lazy, access_all_block_fields, num_mongo_calls):
with store.build() as (source_content, source_store):
source_course_key = source_store.make_course_key('a', 'course', 'course')
# First, import a course.
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['manual-testing-complete'],
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
# Course traversal modeled after the traversal done here:
# lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline
# Starting at the root course block, do a breadth-first traversal using
# get_children() to retrieve each block's children.
with check_mongo_calls(num_mongo_calls):
with source_store.bulk_operations(source_course_key):
start_block = source_store.get_course(source_course_key, depth=depth, lazy=lazy)
all_blocks = []
stack = [start_block]
while stack:
curr_block = stack.pop()
all_blocks.append(curr_block)
if curr_block.has_children:
for block in reversed(curr_block.get_children()):
stack.append(block)
if access_all_block_fields:
# Read the fields on each block in order to ensure each block and its definition is loaded.
for xblock in all_blocks:
for __, field in xblock.fields.iteritems():
if field.is_set_on(xblock):
__ = field.read_from(xblock)
def test_no_static_link_rewrites_on_import(self):
module_store = modulestore()
courses = import_course_from_xml(
module_store, self.user.id, TEST_DATA_DIR, ['toy'], do_import_static=False, verbose=True,
create_if_not_present=True
)
course_key = courses[0].id
handouts = module_store.get_item(course_key.make_usage_key('course_info', 'handouts'))
self.assertIn('/static/', handouts.data)
handouts = module_store.get_item(course_key.make_usage_key('html', 'toyhtml'))
self.assertIn('/static/', handouts.data)
def initdb(cls):
# connect to the db
doc_store_config = {
'host': HOST,
'port': PORT,
'db': DB,
'collection': COLLECTION,
}
cls.add_asset_collection(doc_store_config)
# since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
# as well
content_store = MongoContentStore(HOST, DB, port=PORT)
#
# Also test draft store imports
#
draft_store = DraftModuleStore(
content_store,
doc_store_config, FS_ROOT, RENDER_TEMPLATE,
default_class=DEFAULT_CLASS,
branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred,
xblock_mixins=(EditInfoMixin, InheritanceMixin, LocationMixin, XModuleMixin)
)
with patch('xmodule.tabs.CourseTab.from_json', side_effect=mock_tab_from_json):
import_course_from_xml(
draft_store,
999,
DATA_DIR,
cls.courses,
static_content_store=content_store
)
# also test a course with no importing of static content
import_course_from_xml(
draft_store,
999,
DATA_DIR,
['test_import_course'],
static_content_store=content_store,
do_import_static=False,
verbose=True
)
# also import a course under a different course_id (especially ORG)
import_course_from_xml(
draft_store,
999,
DATA_DIR,
['test_import_course'],
static_content_store=content_store,
do_import_static=False,
verbose=True,
target_id=SlashSeparatedCourseKey('guestx', 'foo', 'bar')
)
return content_store, draft_store
def initdb(cls):
# connect to the db
doc_store_config = {
'host': HOST,
'port': PORT,
'db': DB,
'collection': COLLECTION,
}
cls.add_asset_collection(doc_store_config)
# since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class
# as well
content_store = MongoContentStore(HOST, DB, port=PORT)
#
# Also test draft store imports
#
draft_store = DraftModuleStore(
content_store,
doc_store_config,
FS_ROOT,
RENDER_TEMPLATE,
default_class=DEFAULT_CLASS,
branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred,
xblock_mixins=(EditInfoMixin, InheritanceMixin, LocationMixin,
XModuleMixin))
with patch('xmodule.tabs.CourseTab.from_json',
side_effect=mock_tab_from_json):
import_course_from_xml(draft_store,
999,
DATA_DIR,
cls.courses,
static_content_store=content_store)
# also test a course with no importing of static content
import_course_from_xml(draft_store,
999,
DATA_DIR, ['test_import_course'],
static_content_store=content_store,
do_import_static=False,
verbose=True)
# also import a course under a different course_id (especially ORG)
import_course_from_xml(draft_store,
999,
DATA_DIR, ['test_import_course'],
static_content_store=content_store,
do_import_static=False,
verbose=True,
target_id=SlashSeparatedCourseKey(
'guestx', 'foo', 'bar'))
return content_store, draft_store
def setUp(self):
super(OpenEndedStatsTest, self).setUp()
self.user = UserFactory()
store = modulestore()
course_items = import_course_from_xml(store, self.user.id,
TEST_DATA_DIR, ['open_ended']) # pylint: disable=maybe-no-member
self.course = course_items[0]
self.course_id = self.course.id
self.problem_location = Location("edX", "open_ended", "2012_Fall",
"combinedopenended", "SampleQuestion")
self.task_number = 1
self.invalid_task_number = 3
self.student_on_initial = UserFactory()
self.student_on_accessing = UserFactory()
self.student_on_post_assessment = UserFactory()
StudentModuleFactory.create(course_id=self.course_id,
module_state_key=self.problem_location,
student=self.student_on_initial,
grade=0,
max_grade=1,
state=STATE_INITIAL)
StudentModuleFactory.create(course_id=self.course_id,
module_state_key=self.problem_location,
student=self.student_on_accessing,
grade=0,
max_grade=1,
state=STATE_ACCESSING)
StudentModuleFactory.create(course_id=self.course_id,
module_state_key=self.problem_location,
student=self.student_on_post_assessment,
grade=0,
max_grade=1,
state=STATE_POST_ASSESSMENT)
self.students = [
self.student_on_initial, self.student_on_accessing,
self.student_on_post_assessment
]
def handle(self, *args, **options):
data_dir = options['data_directory']
source_dirs = options['course_dirs']
if len(source_dirs) == 0:
source_dirs = None
do_import_static = not options.get('nostatic', False)
# If the static content is not skipped, the python lib should be imported regardless
# of the 'nopythonlib' flag.
do_import_python_lib = do_import_static or not options.get(
'nopythonlib', False)
python_lib_filename = options.get('python_lib_filename')
output = (u"Importing...\n"
u" data_dir={data}, source_dirs={courses}\n"
u" Importing static content? {import_static}\n"
u" Importing python lib? {import_python_lib}").format(
data=data_dir,
courses=source_dirs,
import_static=do_import_static,
import_python_lib=do_import_python_lib)
self.stdout.write(output)
mstore = modulestore()
course_items = import_course_from_xml(
mstore,
ModuleStoreEnum.UserID.mgmt_command,
data_dir,
source_dirs,
load_error_modules=False,
static_content_store=contentstore(),
verbose=True,
do_import_static=do_import_static,
do_import_python_lib=do_import_python_lib,
create_if_not_present=True,
python_lib_filename=python_lib_filename,
)
for course in course_items:
course_id = course.id
if not are_permissions_roles_seeded(course_id):
self.stdout.write(
u'Seeding forum roles for course {0}\n'.format(course_id))
seed_permissions_roles(course_id)
def load_scoreable_course(cls):
"""
This test course lives at `common/test/data/scoreable`.
For details on the contents and structure of the file, see
`common/test/data/scoreable/README`.
"""
course_items = import_course_from_xml(
cls.store,
'test_user',
TEST_DATA_DIR,
source_dirs=['scoreable'],
static_content_store=None,
target_id=cls.store.make_course_key('edX', 'scoreable', '3000'),
raise_on_failure=True,
create_if_not_present=True,
)
cls.course = course_items[0]
def import_test_course(self):
"""
Import the test course with the sga unit
"""
# adapted from edx-platform/cms/djangoapps/contentstore/
# management/commands/tests/test_cleanup_assets.py
input_dir = os.path.join(BASE_DIR, "..", "test_data")
temp_dir = tempfile.mkdtemp()
self.addCleanup(lambda: shutil.rmtree(temp_dir))
xml_dir = os.path.join(temp_dir, "xml")
shutil.copytree(input_dir, xml_dir)
store = modulestore()
courses = import_course_from_xml(
store,
'sga_user',
xml_dir,
)
return courses[0]
def test_pdf_asset(self):
module_store = modulestore()
course_items = import_course_from_xml(
module_store,
self.user.id,
TEST_DATA_DIR, ['toy'],
static_content_store=contentstore(),
verbose=True)
course = course_items[0]
url = reverse_course_url('assets_handler', course.id)
# Test valid contentType for pdf asset (textbook.pdf)
resp = self.client.get(url, HTTP_ACCEPT='application/json')
self.assertContains(resp, "/c4x/edX/toy/asset/textbook.pdf")
asset_location = AssetKey.from_string(
'/c4x/edX/toy/asset/textbook.pdf')
content = contentstore().find(asset_location)
# Check after import textbook.pdf has valid contentType ('application/pdf')
# Note: Actual contentType for textbook.pdf in asset.json is 'text/pdf'
self.assertEqual(content.content_type, 'application/pdf')
def _assert_import(self, course_dir, expected_field_val, has_draft=False):
"""
Import a course from XML, then verify that the XBlock was loaded
with the correct field value.
Args:
course_dir (str): The name of the course directory (relative to the test data directory)
expected_xblock_loc (str): The location of the XBlock in the course.
expected_field_val (str): The expected value of the XBlock's test field.
Kwargs:
has_draft (bool): If true, check that a draft of the XBlock exists with
the expected field value set.
"""
# It is necessary to use the "old mongo" modulestore because split doesn't work
# with the "has_draft" logic below.
store = modulestore()._get_modulestore_by_type(
ModuleStoreEnum.Type.mongo) # pylint: disable=protected-access
courses = import_course_from_xml(store,
self.user.id,
TEST_DATA_DIR, [course_dir],
create_if_not_present=True)
xblock_location = courses[0].id.make_usage_key('stubxblock',
'xblock_test')
if has_draft:
xblock_location = as_draft(xblock_location)
xblock = store.get_item(xblock_location)
self.assertTrue(isinstance(xblock, StubXBlock))
self.assertEqual(xblock.test_field, expected_field_val)
if has_draft:
draft_xblock = store.get_item(xblock_location)
self.assertTrue(getattr(draft_xblock, 'is_draft', False))
self.assertTrue(isinstance(draft_xblock, StubXBlock))
self.assertEqual(draft_xblock.test_field, expected_field_val)
def handle(self, *args, **options):
"Execute the command"
if len(args) == 0:
raise CommandError(
"import requires at least one argument: <data directory> [--nostatic] [<course dir>...]"
)
data_dir = args[0]
do_import_static = not (options.get('nostatic', False))
if len(args) > 1:
source_dirs = args[1:]
else:
source_dirs = None
self.stdout.write(
"Importing. Data_dir={data}, source_dirs={courses}\n".format(
data=data_dir,
courses=source_dirs,
))
mstore = modulestore()
course_items = import_course_from_xml(
mstore,
ModuleStoreEnum.UserID.mgmt_command,
data_dir,
source_dirs,
load_error_modules=False,
static_content_store=contentstore(),
verbose=True,
do_import_static=do_import_static,
create_if_not_present=True,
)
for course in course_items:
course_id = course.id
if not are_permissions_roles_seeded(course_id):
self.stdout.write(
'Seeding forum roles for course {0}\n'.format(course_id))
seed_permissions_roles(course_id)
def import_single_course(filename):
print >> sys.stderr, 'IMPORTING course:', filename
org, course_id, course_run = _filename_to_org_id_and_run(filename)
course_full_id = 'course-v1:{org}+{id}+{run}'.format(org=org,
id=course_id,
run=course_run)
course_xml_dir = path.join(
XML_EXTRACT_DIR, '{id}-{run}'.format(id=course_id, run=course_run))
mkdir(course_xml_dir)
subprocess.call(['tar', '-xzf', filename, '-C', course_xml_dir])
_fix_library_source_bug(course_xml_dir)
print >> sys.stderr, 'IMPORTING course:', course_full_id
course_items = import_course_from_xml(
store=MOD_STORE,
user_id=ModuleStoreEnum.UserID.mgmt_command,
data_dir=DATA_DIR,
source_dirs=[path.join(course_xml_dir,
'course')], # Open edX needs `course` dir
load_error_modules=False,
static_content_store=contentstore(),
verbose=True,
do_import_static=True,
target_id=CourseKey.from_string(course_full_id),
create_if_not_present=True,
)
for course in course_items:
course_id = course.id
if not are_permissions_roles_seeded(course_id):
print >> sys.stderr, 'Seeding forum roles for course', course_id
seed_permissions_roles(course_id)
_set_course_dates(course_id)
def test_generate_find_timings(self, source_ms, num_assets):
"""
Generate timings for different amounts of asset metadata and different modulestores.
"""
if CodeBlockTimer is None:
raise SkipTest("CodeBlockTimer undefined.")
desc = "FindAssetTest:{}:{}".format(
SHORT_NAME_MAP[source_ms],
num_assets,
)
with CodeBlockTimer(desc):
with CodeBlockTimer("fake_assets"):
# First, make the fake asset metadata.
make_asset_xml(num_assets, ASSET_XML_PATH)
validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH)
with source_ms.build() as (source_content, source_store):
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
asset_key = source_course_key.make_asset_key(
AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif')
with CodeBlockTimer("initial_import"):
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_ROOT,
source_dirs=TEST_COURSE,
static_content_store=source_content,
target_id=source_course_key,
create_if_not_present=True,
raise_on_failure=True,
)
with CodeBlockTimer("find_nonexistent_asset"):
# More correct would be using the AssetManager.find() - but since the test
# has created its own test modulestore, the AssetManager can't be used.
__ = source_store.find_asset_metadata(asset_key)
# Perform get_all_asset_metadata for each sort.
for sort in ALL_SORTS:
with CodeBlockTimer("get_asset_list:{}-{}".format(
sort[0], 'asc'
if sort[1] == ModuleStoreEnum.SortOrder.ascending
else 'desc')):
# Grab two ranges of 50 assets using different sorts.
# Why 50? That's how many are displayed on the current Studio "Files & Uploads" page.
start_middle = num_assets / 2
__ = source_store.get_all_asset_metadata(
source_course_key,
'asset',
start=0,
sort=sort,
maxresults=50)
__ = source_store.get_all_asset_metadata(
source_course_key,
'asset',
start=start_middle,
sort=sort,
maxresults=50)
def test_round_trip(self, source_builder, dest_builder,
source_content_builder, dest_content_builder,
course_data_name):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(
contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(
contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=[course_data_name],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
'exported_source_course',
)
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=['exported_source_course'],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
# NOT CURRENTLY USED
# export_course_to_xml(
# dest_store,
# dest_content,
# dest_course_key,
# self.export_dir,
# 'exported_dest_course',
# )
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.exclude_field(None, 'parent')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
self.assertAssetsMetadataEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
def test_round_trip(self, source_builder, dest_builder,
source_content_builder, dest_content_builder,
course_data_name, _mock_tab_from_json):
# Construct the contentstore for storing the first import
with source_content_builder.build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with source_builder.build(
contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with dest_content_builder.build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with dest_builder.build(
contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key(
'a', 'course', 'course')
dest_course_key = dest_store.make_course_key(
'a', 'course', 'course')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=[course_data_name],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
EXPORTED_COURSE_DIR_NAME,
)
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=[EXPORTED_COURSE_DIR_NAME],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
# NOT CURRENTLY USED
# export_course_to_xml(
# dest_store,
# dest_content,
# dest_course_key,
# self.export_dir,
# 'exported_dest_course',
# )
self.exclude_field(None, 'wiki_slug')
self.exclude_field(None, 'xml_attributes')
self.exclude_field(None, 'parent')
# discussion_ids are auto-generated based on usage_id, so they should change across
# modulestores - see TNL-5001
self.exclude_field(None, 'discussion_id')
self.ignore_asset_key('_id')
self.ignore_asset_key('uploadDate')
self.ignore_asset_key('content_son')
self.ignore_asset_key('thumbnail_location')
self.assertCoursesEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
self.assertAssetsEqual(
source_content,
source_course_key,
dest_content,
dest_course_key,
)
self.assertAssetsMetadataEqual(
source_store,
source_course_key,
dest_store,
dest_course_key,
)
def test_split_course_export_import(self):
# Construct the contentstore for storing the first import
with MongoContentstoreBuilder().build() as source_content:
# Construct the modulestore for storing the first import (using the previously created contentstore)
with SPLIT_MODULESTORE_SETUP.build(
contentstore=source_content) as source_store:
# Construct the contentstore for storing the second import
with MongoContentstoreBuilder().build() as dest_content:
# Construct the modulestore for storing the second import (using the second contentstore)
with SPLIT_MODULESTORE_SETUP.build(
contentstore=dest_content) as dest_store:
source_course_key = source_store.make_course_key(
'a', 'source', '2015_Fall')
dest_course_key = dest_store.make_course_key(
'a', 'dest', '2015_Fall')
import_course_from_xml(
source_store,
'test_user',
TEST_DATA_DIR,
source_dirs=['split_course_with_static_tabs'],
static_content_store=source_content,
target_id=source_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
export_course_to_xml(
source_store,
source_content,
source_course_key,
self.export_dir,
EXPORTED_COURSE_DIR_NAME,
)
source_course = source_store.get_course(
source_course_key, depth=None, lazy=False)
self.assertEqual(source_course.url_name, 'course')
export_dir_path = path(self.export_dir)
policy_dir = export_dir_path / 'exported_source_course' / 'policies' / source_course_key.run
policy_path = policy_dir / 'policy.json'
self.assertTrue(os.path.exists(policy_path))
import_course_from_xml(
dest_store,
'test_user',
self.export_dir,
source_dirs=[EXPORTED_COURSE_DIR_NAME],
static_content_store=dest_content,
target_id=dest_course_key,
raise_on_failure=True,
create_if_not_present=True,
)
dest_course = dest_store.get_course(dest_course_key,
depth=None,
lazy=False)
self.assertEqual(dest_course.url_name, 'course')
def import_and_populate_course(self):
"""
Imports the test toy course and populates it with additional test data
"""
content_store = contentstore()
import_course_from_xml(self.store,
self.user.id,
TEST_DATA_DIR, ['toy'],
static_content_store=content_store)
course_id = CourseKey.from_string('/'.join(['edX', 'toy',
'2012_Fall']))
# create an Orphan
# We had a bug where orphaned draft nodes caused export to fail. This is here to cover that case.
vertical = self.store.get_item(course_id.make_usage_key(
'vertical', self.TEST_VERTICAL),
depth=1)
vertical.location = vertical.location.replace(name='no_references')
self.store.update_item(vertical, self.user.id, allow_not_found=True)
orphan_vertical = self.store.get_item(vertical.location)
self.assertEqual(orphan_vertical.location.name, 'no_references')
self.assertEqual(len(orphan_vertical.children), len(vertical.children))
# create an orphan vertical and html; we already don't try to import
# the orphaned vertical, but we should make sure we don't import
# the orphaned vertical's child html, too
orphan_draft_vertical = self.store.create_item(
self.user.id, course_id, 'vertical', self.ORPHAN_DRAFT_VERTICAL)
orphan_draft_html = self.store.create_item(self.user.id, course_id,
'html',
self.ORPHAN_DRAFT_HTML)
orphan_draft_vertical.children.append(orphan_draft_html.location)
self.store.update_item(orphan_draft_vertical, self.user.id)
# create a Draft vertical
vertical = self.store.get_item(course_id.make_usage_key(
'vertical', self.TEST_VERTICAL),
depth=1)
draft_vertical = self.store.convert_to_draft(vertical.location,
self.user.id)
self.assertTrue(self.store.has_published_version(draft_vertical))
# create a Private (draft only) vertical
private_vertical = self.store.create_item(self.user.id, course_id,
'vertical',
self.PRIVATE_VERTICAL)
self.assertFalse(self.store.has_published_version(private_vertical))
# create a Published (no draft) vertical
public_vertical = self.store.create_item(self.user.id, course_id,
'vertical',
self.PUBLISHED_VERTICAL)
public_vertical = self.store.publish(public_vertical.location,
self.user.id)
self.assertTrue(self.store.has_published_version(public_vertical))
# add the new private and new public as children of the sequential
sequential = self.store.get_item(
course_id.make_usage_key('sequential', self.SEQUENTIAL))
sequential.children.append(private_vertical.location)
sequential.children.append(public_vertical.location)
self.store.update_item(sequential, self.user.id)
# create an html and video component to make drafts:
draft_html = self.store.create_item(self.user.id, course_id, 'html',
self.DRAFT_HTML)
draft_video = self.store.create_item(self.user.id, course_id, 'video',
self.DRAFT_VIDEO)
# add them as children to the public_vertical
public_vertical.children.append(draft_html.location)
public_vertical.children.append(draft_video.location)
self.store.update_item(public_vertical, self.user.id)
# publish changes to vertical
self.store.publish(public_vertical.location, self.user.id)
# convert html/video to draft
self.store.convert_to_draft(draft_html.location, self.user.id)
self.store.convert_to_draft(draft_video.location, self.user.id)
# lock an asset
content_store.set_attr(self.LOCKED_ASSET_KEY, 'locked', True)
# create a non-portable link - should be rewritten in new courses
html_module = self.store.get_item(
course_id.make_usage_key('html', 'nonportable'))
new_data = html_module.data = html_module.data.replace(
'/static/', '/c4x/{0}/{1}/asset/'.format(course_id.org,
course_id.course))
self.store.update_item(html_module, self.user.id)
html_module = self.store.get_item(html_module.location)
self.assertEqual(new_data, html_module.data)
return course_id
