def initdb(self, default):
"""
Initialize the database and create one test course in it
"""
# set the default modulestore
self.options['stores']['default'] = self.options['stores'][default]
self.store = MixedModuleStore(**self.options)
self.addCleanup(self.store.close_all_connections)
self.course_locations = {
course_id: generate_location(course_id)
for course_id in [self.MONGO_COURSEID, self.XML_COURSEID1, self.XML_COURSEID2]
}
self.fake_location = Location('i4x', 'foo', 'bar', 'vertical', 'baz')
self.import_chapter_location = self.course_locations[self.MONGO_COURSEID].replace(
category='chapter', name='Overview'
)
self.xml_chapter_location = self.course_locations[self.XML_COURSEID1].replace(
category='chapter', name='Overview'
)
# get Locators and set up the loc mapper if app is Locator based
if default == 'split':
self.fake_location = loc_mapper().translate_location('foo/bar/2012_Fall', self.fake_location)
self._create_course(default, self.MONGO_COURSEID)
def initdb(self, default):
"""
Initialize the database and create one test course in it
"""
# set the default modulestore
self.options['stores']['default'] = self.options['stores'][default]
self.store = MixedModuleStore(**self.options)
self.addCleanup(self.store.close_all_connections)
# convert to CourseKeys
self.course_locations = {
course_id:
SlashSeparatedCourseKey.from_deprecated_string(course_id)
for course_id in
[self.MONGO_COURSEID, self.XML_COURSEID1, self.XML_COURSEID2]
}
# and then to the root UsageKey
self.course_locations = {
course_id: course_key.make_usage_key('course', course_key.run)
for course_id, course_key in self.course_locations.iteritems() # pylint: disable=maybe-no-member
}
self.fake_location = Location('foo', 'bar', 'slowly', 'vertical',
'baz')
self.import_chapter_location = self.course_locations[
self.MONGO_COURSEID].replace(category='chapter', name='Overview')
self.xml_chapter_location = self.course_locations[
self.XML_COURSEID1].replace(category='chapter', name='Overview')
# get Locators and set up the loc mapper if app is Locator based
if default == 'split':
self.fake_location = loc_mapper().translate_location(
self.fake_location)
self._create_course(
default, self.course_locations[self.MONGO_COURSEID].course_key)
def _perform_test_using_store(self, store_type, test_to_perform):
""" Helper method to run a test function that uses a specific store """
with MongoContentstoreBuilder().build() as contentstore:
store = MixedModuleStore(
contentstore=contentstore,
create_modulestore_instance=create_modulestore_instance,
mappings={},
**self.OPTIONS)
self.addCleanup(store.close_all_connections)
with store.default_store(store_type):
self.setup_course_base(store)
test_to_perform(store)
def _perform_test_using_store(self, store_type, test_to_perform):
""" Helper method to run a test function that uses a specific store """
with MongoContentstoreBuilder().build() as contentstore:
store = MixedModuleStore(
contentstore=contentstore,
create_modulestore_instance=create_modulestore_instance,
mappings={},
**self.OPTIONS
)
self.addCleanup(store.close_all_connections)
with store.default_store(store_type):
self.setup_course_base(store)
test_to_perform(store)
def initdb():
# connect to the db
_options = {}
_options.update(OPTIONS)
store = MixedModuleStore(**_options)
import_from_xml(store._get_modulestore_for_courseid(IMPORT_COURSEID),
DATA_DIR, ['toy'],
target_location_namespace=Location(
'i4x', TestMixedModuleStore.import_org,
TestMixedModuleStore.import_course, 'course',
TestMixedModuleStore.import_run))
return store
def build_with_contentstore(self, contentstore):
"""
A contextmanager that returns a mixed modulestore built on top of modulestores
generated by other builder classes.
Args:
contentstore: The contentstore that this modulestore should use to store
all of its assets.
"""
names, generators = zip(*self.store_builders)
with nested(*(gen.build_with_contentstore(contentstore)
for gen in generators)) as modulestores:
# Make the modulestore creation function just return the already-created modulestores
store_iterator = iter(modulestores)
next_modulestore = lambda *args, **kwargs: store_iterator.next()
# Generate a fake list of stores to give the already generated stores appropriate names
stores = [{
'NAME': name,
'ENGINE': 'This space deliberately left blank'
} for name in names]
self.mixed_modulestore = MixedModuleStore(
contentstore,
self.mappings,
stores,
create_modulestore_instance=next_modulestore,
xblock_mixins=XBLOCK_MIXINS,
)
yield self.mixed_modulestore
def initdb(self, default):
"""
Initialize the database and create one test course in it
"""
# set the default modulestore
store_configs = self.options['stores']
for index in range(len(store_configs)):
if store_configs[index]['NAME'] == default:
if index > 0:
store_configs[index], store_configs[0] = store_configs[0], store_configs[index]
break
self.store = MixedModuleStore(None, create_modulestore_instance=create_modulestore_instance, **self.options)
self.addCleanup(self.store.close_all_connections)
# convert to CourseKeys
self.course_locations = {
course_id: CourseLocator.from_string(course_id)
for course_id in [self.MONGO_COURSEID, self.XML_COURSEID1, self.XML_COURSEID2]
}
# and then to the root UsageKey
self.course_locations = {
course_id: course_key.make_usage_key('course', course_key.run)
for course_id, course_key in self.course_locations.iteritems() # pylint: disable=maybe-no-member
}
if default == 'split':
self.fake_location = CourseLocator(
'foo', 'bar', 'slowly', branch=ModuleStoreEnum.BranchName.draft
).make_usage_key('vertical', 'baz')
else:
self.fake_location = Location('foo', 'bar', 'slowly', 'vertical', 'baz')
self.xml_chapter_location = self.course_locations[self.XML_COURSEID1].replace(
category='chapter', name='Overview'
)
self._create_course(default, self.course_locations[self.MONGO_COURSEID].course_key)
def initdb(self, default):
"""
Initialize the database and create one test course in it
"""
# set the default modulestore
self.options['stores']['default'] = self.options['stores'][default]
self.store = MixedModuleStore(**self.options)
self.addCleanup(self.store.close_all_connections)
# convert to CourseKeys
self.course_locations = {
course_id: SlashSeparatedCourseKey.from_deprecated_string(course_id)
for course_id in [self.MONGO_COURSEID, self.XML_COURSEID1, self.XML_COURSEID2]
}
# and then to the root UsageKey
self.course_locations = {
course_id: course_key.make_usage_key('course', course_key.run)
for course_id, course_key in self.course_locations.iteritems() # pylint: disable=maybe-no-member
}
self.fake_location = Location('foo', 'bar', 'slowly', 'vertical', 'baz')
self.import_chapter_location = self.course_locations[self.MONGO_COURSEID].replace(
category='chapter', name='Overview'
)
self.xml_chapter_location = self.course_locations[self.XML_COURSEID1].replace(
category='chapter', name='Overview'
)
# get Locators and set up the loc mapper if app is Locator based
if default == 'split':
self.fake_location = loc_mapper().translate_location(self.fake_location)
self._create_course(default, self.course_locations[self.MONGO_COURSEID].course_key)
def setUp(self):
unittest.TestCase.setUp(self)
options = copy.copy(OPTIONS)
del options['reference_type']
self.connection = pymongo.MongoClient(
host=HOST,
port=PORT,
tz_aware=True,
)
self.store = MixedModuleStore(**options)
def initdb():
# connect to the db
_options = {}
_options.update(OPTIONS)
store = MixedModuleStore(**_options)
import_from_xml(
store._get_modulestore_for_courseid(IMPORT_COURSEID),
DATA_DIR,
['toy'],
target_location_namespace=Location(
'i4x',
TestMixedModuleStore.import_org,
TestMixedModuleStore.import_course,
'course',
TestMixedModuleStore.import_run
)
)
return store
def setUp(self):
"""
Set up requirements for testing: a user ID and a modulestore
"""
super(MixedSplitTestCase, self).setUp()
self.user_id = ModuleStoreEnum.UserID.test
self.store = MixedModuleStore(
None,
create_modulestore_instance=create_modulestore_instance,
mappings={},
**self.MIXED_OPTIONS)
self.addCleanup(self.store.close_all_connections)
self.addCleanup(self.store._drop_database) # pylint: disable=protected-access
def initdb():
"""
Initialize the database and import one test course into it
"""
# connect to the db
_options = {}
_options.update(OPTIONS)
store = MixedModuleStore(**_options)
import_from_xml(
store._get_modulestore_for_courseid(IMPORT_COURSEID),
DATA_DIR,
["toy"],
target_location_namespace=Location(
"i4x",
TestMixedModuleStore.import_org,
TestMixedModuleStore.import_course,
"course",
TestMixedModuleStore.import_run,
),
)
return store
class TestMixedModuleStore(LocMapperSetupSansDjango):
"""
Quasi-superclass which tests Location based apps against both split and mongo dbs (Locator and
Location-based dbs)
"""
HOST = 'localhost'
PORT = 27017
DB = 'test_mongo_%s' % uuid4().hex[:5]
COLLECTION = 'modulestore'
FS_ROOT = DATA_DIR
DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor'
RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': ''
MONGO_COURSEID = 'MITx/999/2013_Spring'
XML_COURSEID1 = 'edX/toy/2012_Fall'
XML_COURSEID2 = 'edX/simple/2012_Fall'
modulestore_options = {
'default_class': DEFAULT_CLASS,
'fs_root': DATA_DIR,
'render_template': RENDER_TEMPLATE,
}
DOC_STORE_CONFIG = {
'host': HOST,
'db': DB,
'collection': COLLECTION,
}
OPTIONS = {
'mappings': {
XML_COURSEID1: 'xml',
XML_COURSEID2: 'xml',
MONGO_COURSEID: 'default'
},
'stores': {
'xml': {
'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
'OPTIONS': {
'data_dir': DATA_DIR,
'default_class': 'xmodule.hidden_module.HiddenDescriptor',
}
},
'direct': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
'draft': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
'split': {
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
}
}
}
def _compareIgnoreVersion(self, loc1, loc2, msg=None):
"""
AssertEqual replacement for CourseLocator
"""
if not (loc1.package_id == loc2.package_id and loc1.branch == loc2.branch and loc1.block_id == loc2.block_id):
self.fail(self._formatMessage(msg, u"{} != {}".format(unicode(loc1), unicode(loc2))))
def setUp(self):
"""
Set up the database for testing
"""
self.options = getattr(self, 'options', self.OPTIONS)
self.connection = pymongo.MongoClient(
host=self.HOST,
port=self.PORT,
tz_aware=True,
)
self.connection.drop_database(self.DB)
self.addCleanup(self.connection.drop_database, self.DB)
self.addCleanup(self.connection.close)
super(TestMixedModuleStore, self).setUp()
patcher = patch.multiple(
'xmodule.modulestore.mixed',
loc_mapper=Mock(return_value=LocMapperSetupSansDjango.loc_store),
create_modulestore_instance=create_modulestore_instance,
)
patcher.start()
self.addCleanup(patcher.stop)
self.addTypeEqualityFunc(BlockUsageLocator, '_compareIgnoreVersion')
# define attrs which get set in initdb to quell pylint
self.import_chapter_location = self.store = self.fake_location = self.xml_chapter_location = None
self.course_locations = []
# pylint: disable=invalid-name
def _create_course(self, default, course_id):
"""
Create a course w/ one item in the persistence store using the given course & item location.
"""
course = self.store.create_course(course_id, store_name=default)
category = self.import_chapter_location.category
block_id = self.import_chapter_location.name
chapter = self.store.create_item(
# don't use course_location as it may not be the repr
course.location, category, location=self.import_chapter_location, block_id=block_id
)
if isinstance(course.location, CourseLocator):
self.course_locations[self.MONGO_COURSEID] = course.location.version_agnostic()
self.import_chapter_location = chapter.location.version_agnostic()
else:
self.assertEqual(course.location.course_id, course_id)
self.assertEqual(chapter.location, self.import_chapter_location)
def initdb(self, default):
"""
Initialize the database and create one test course in it
"""
# set the default modulestore
self.options['stores']['default'] = self.options['stores'][default]
self.store = MixedModuleStore(**self.options)
self.addCleanup(self.store.close_all_connections)
self.course_locations = {
course_id: generate_location(course_id)
for course_id in [self.MONGO_COURSEID, self.XML_COURSEID1, self.XML_COURSEID2]
}
self.fake_location = Location('i4x', 'foo', 'bar', 'vertical', 'baz')
self.import_chapter_location = self.course_locations[self.MONGO_COURSEID].replace(
category='chapter', name='Overview'
)
self.xml_chapter_location = self.course_locations[self.XML_COURSEID1].replace(
category='chapter', name='Overview'
)
# get Locators and set up the loc mapper if app is Locator based
if default == 'split':
self.fake_location = loc_mapper().translate_location('foo/bar/2012_Fall', self.fake_location)
self._create_course(default, self.MONGO_COURSEID)
@ddt.data('direct', 'split')
def test_get_modulestore_type(self, default_ms):
"""
Make sure we get back the store type we expect for given mappings
"""
self.initdb(default_ms)
self.assertEqual(self.store.get_modulestore_type(self.XML_COURSEID1), XML_MODULESTORE_TYPE)
self.assertEqual(self.store.get_modulestore_type(self.XML_COURSEID2), XML_MODULESTORE_TYPE)
mongo_ms_type = MONGO_MODULESTORE_TYPE if default_ms == 'direct' else SPLIT_MONGO_MODULESTORE_TYPE
self.assertEqual(self.store.get_modulestore_type(self.MONGO_COURSEID), mongo_ms_type)
# try an unknown mapping, it should be the 'default' store
self.assertEqual(self.store.get_modulestore_type('foo/bar/2012_Fall'), mongo_ms_type)
@ddt.data('direct', 'split')
def test_has_item(self, default_ms):
self.initdb(default_ms)
for course_id, course_locn in self.course_locations.iteritems():
self.assertTrue(self.store.has_item(course_id, course_locn))
# try negative cases
self.assertFalse(self.store.has_item(
self.XML_COURSEID1,
self.course_locations[self.XML_COURSEID1].replace(name='not_findable', category='problem')
))
self.assertFalse(self.store.has_item(self.MONGO_COURSEID, self.fake_location))
@ddt.data('direct', 'split')
def test_get_item(self, default_ms):
self.initdb(default_ms)
with self.assertRaises(NotImplementedError):
self.store.get_item(self.fake_location)
@ddt.data('direct', 'split')
def test_get_instance(self, default_ms):
self.initdb(default_ms)
for course_id, course_locn in self.course_locations.iteritems():
self.assertIsNotNone(self.store.get_instance(course_id, course_locn))
# try negative cases
with self.assertRaises(ItemNotFoundError):
self.store.get_instance(
self.XML_COURSEID1,
self.course_locations[self.XML_COURSEID1].replace(name='not_findable', category='problem')
)
with self.assertRaises(ItemNotFoundError):
self.store.get_instance(self.MONGO_COURSEID, self.fake_location)
@ddt.data('direct', 'split')
def test_get_items(self, default_ms):
self.initdb(default_ms)
for course_id, course_locn in self.course_locations.iteritems():
if hasattr(course_locn, 'as_course_locator'):
locn = course_locn.as_course_locator()
else:
locn = course_locn.replace(org=None, course=None, name=None)
# NOTE: use get_course if you just want the course. get_items is expensive
modules = self.store.get_items(locn, course_id, qualifiers={'category': 'course'})
self.assertEqual(len(modules), 1)
self.assertEqual(modules[0].location, course_locn)
@ddt.data('direct', 'split')
def test_update_item(self, default_ms):
"""
Update should fail for r/o dbs and succeed for r/w ones
"""
self.initdb(default_ms)
course_id = self.XML_COURSEID1
course = self.store.get_course(course_id)
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
with self.assertRaises(NotImplementedError):
self.store.update_item(course, None)
# now do it for a r/w db
# get_course api's are inconsistent: one takes Locators the other an old style course id
if hasattr(self.course_locations[self.MONGO_COURSEID], 'as_course_locator'):
locn = self.course_locations[self.MONGO_COURSEID]
else:
locn = self.MONGO_COURSEID
course = self.store.get_course(locn)
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
self.store.update_item(course, None)
course = self.store.get_course(locn)
self.assertTrue(course.show_calculator)
@ddt.data('direct', 'split')
def test_delete_item(self, default_ms):
"""
Delete should reject on r/o db and work on r/w one
"""
self.initdb(default_ms)
# r/o try deleting the course
with self.assertRaises(NotImplementedError):
self.store.delete_item(self.xml_chapter_location)
self.store.delete_item(self.import_chapter_location, '**replace_user**')
# verify it's gone
with self.assertRaises(ItemNotFoundError):
self.store.get_instance(self.MONGO_COURSEID, self.import_chapter_location)
@ddt.data('direct', 'split')
def test_get_courses(self, default_ms):
self.initdb(default_ms)
# we should have 3 total courses across all stores
courses = self.store.get_courses()
course_ids = [
course.location.version_agnostic()
if hasattr(course.location, 'version_agnostic') else course.location
for course in courses
]
self.assertEqual(len(courses), 3, "Not 3 courses: {}".format(course_ids))
self.assertIn(self.course_locations[self.MONGO_COURSEID], course_ids)
self.assertIn(self.course_locations[self.XML_COURSEID1], course_ids)
self.assertIn(self.course_locations[self.XML_COURSEID2], course_ids)
def test_xml_get_courses(self):
"""
Test that the xml modulestore only loaded the courses from the maps.
"""
self.initdb('direct')
courses = self.store.modulestores['xml'].get_courses()
self.assertEqual(len(courses), 2)
course_ids = [course.location.course_id for course in courses]
self.assertIn(self.XML_COURSEID1, course_ids)
self.assertIn(self.XML_COURSEID2, course_ids)
# this course is in the directory from which we loaded courses but not in the map
self.assertNotIn("edX/toy/TT_2012_Fall", course_ids)
def test_xml_no_write(self):
"""
Test that the xml modulestore doesn't allow write ops.
"""
self.initdb('direct')
with self.assertRaises(NotImplementedError):
self.store.create_course("org/course/run", store_name='xml')
@ddt.data('direct', 'split')
def test_get_course(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues():
if hasattr(course_locn, 'as_course_locator'):
locn = course_locn.as_course_locator()
else:
locn = course_locn.course_id
# NOTE: use get_course if you just want the course. get_items is expensive
course = self.store.get_course(locn)
self.assertIsNotNone(course)
self.assertEqual(course.location, course_locn)
@ddt.data('direct', 'split')
def test_get_parent_locations(self, default_ms):
self.initdb(default_ms)
parents = self.store.get_parent_locations(
self.import_chapter_location,
self.MONGO_COURSEID
)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0], self.course_locations[self.MONGO_COURSEID])
parents = self.store.get_parent_locations(
self.xml_chapter_location,
self.XML_COURSEID1
)
self.assertEqual(len(parents), 1)
self.assertEqual(parents[0], self.course_locations[self.XML_COURSEID1])
@ddt.data('direct', 'split')
def test_get_orphans(self, default_ms):
self.initdb(default_ms)
# create an orphan
if default_ms == 'split':
course_id = self.course_locations[self.MONGO_COURSEID].as_course_locator()
branch = course_id.branch
else:
course_id = self.MONGO_COURSEID
branch = None
orphan = self.store.create_item(course_id, 'problem', block_id='orphan')
found_orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID], branch)
if default_ms == 'split':
self.assertEqual(found_orphans, [orphan.location.version_agnostic()])
else:
self.assertEqual(found_orphans, [unicode(orphan.location)])
@ddt.data('split')
def test_create_item_from_course_id(self, default_ms):
"""
Test code paths missed by the above:
* passing an old-style course_id which has a loc map to split's create_item
"""
self.initdb(default_ms)
# create loc_map entry
loc_mapper().translate_location(self.MONGO_COURSEID, generate_location(self.MONGO_COURSEID))
orphan = self.store.create_item(self.MONGO_COURSEID, 'problem', block_id='orphan')
self.assertEqual(
orphan.location.version_agnostic().as_course_locator(),
self.course_locations[self.MONGO_COURSEID].as_course_locator()
)
@ddt.data('direct')
def test_create_item_from_parent_location(self, default_ms):
"""
Test a code path missed by the above: passing an old-style location as parent but no
new location for the child
"""
self.initdb(default_ms)
self.store.create_item(self.course_locations[self.MONGO_COURSEID], 'problem', block_id='orphan')
orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID], None)
self.assertEqual(len(orphans), 0, "unexpected orphans: {}".format(orphans))
@ddt.data('direct')
def test_get_courses_for_wiki(self, default_ms):
"""
Test the get_courses_for_wiki method
"""
self.initdb(default_ms)
course_locations = self.store.get_courses_for_wiki('toy')
self.assertEqual(len(course_locations), 1)
self.assertIn(Location('i4x', 'edX', 'toy', 'course', '2012_Fall'), course_locations)
course_locations = self.store.get_courses_for_wiki('simple')
self.assertEqual(len(course_locations), 1)
self.assertIn(Location('i4x', 'edX', 'simple', 'course', '2012_Fall'), course_locations)
self.assertEqual(len(self.store.get_courses_for_wiki('edX.simple.2012_Fall')), 0)
self.assertEqual(len(self.store.get_courses_for_wiki('no_such_wiki')), 0)
class TestMixedModuleStore(unittest.TestCase):
"""
Quasi-superclass which tests Location based apps against both split and mongo dbs (Locator and
Location-based dbs)
"""
HOST = 'localhost'
PORT = 27017
DB = 'test_mongo_%s' % uuid4().hex[:5]
COLLECTION = 'modulestore'
FS_ROOT = DATA_DIR
DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor'
RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': ''
MONGO_COURSEID = 'MITx/999/2013_Spring'
XML_COURSEID1 = 'edX/toy/2012_Fall'
XML_COURSEID2 = 'edX/simple/2012_Fall'
BAD_COURSE_ID = 'edX/simple'
modulestore_options = {
'default_class': DEFAULT_CLASS,
'fs_root': DATA_DIR,
'render_template': RENDER_TEMPLATE,
}
DOC_STORE_CONFIG = {
'host': HOST,
'db': DB,
'collection': COLLECTION,
}
OPTIONS = {
'mappings': {
XML_COURSEID1: 'xml',
XML_COURSEID2: 'xml',
BAD_COURSE_ID: 'xml',
},
'stores': [
{
'NAME': 'draft',
'ENGINE': 'xmodule.modulestore.mongo.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
{
'NAME': 'split',
'ENGINE': 'xmodule.modulestore.split_mongo.split_draft.DraftVersioningModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
{
'NAME': 'xml',
'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
'OPTIONS': {
'data_dir': DATA_DIR,
'default_class': 'xmodule.hidden_module.HiddenDescriptor',
}
},
]
}
def _compareIgnoreVersion(self, loc1, loc2, msg=None):
"""
AssertEqual replacement for CourseLocator
"""
if loc1.version_agnostic() != loc2.version_agnostic():
self.fail(self._formatMessage(msg, u"{} != {}".format(unicode(loc1), unicode(loc2))))
def setUp(self):
"""
Set up the database for testing
"""
self.options = getattr(self, 'options', self.OPTIONS)
self.connection = pymongo.MongoClient(
host=self.HOST,
port=self.PORT,
tz_aware=True,
)
self.connection.drop_database(self.DB)
self.addCleanup(self.connection.drop_database, self.DB)
self.addCleanup(self.connection.close)
super(TestMixedModuleStore, self).setUp()
self.addTypeEqualityFunc(BlockUsageLocator, '_compareIgnoreVersion')
self.addTypeEqualityFunc(CourseLocator, '_compareIgnoreVersion')
# define attrs which get set in initdb to quell pylint
self.writable_chapter_location = self.store = self.fake_location = self.xml_chapter_location = None
self.course_locations = []
self.user_id = ModuleStoreEnum.UserID.test
# pylint: disable=invalid-name
def _create_course(self, default, course_key):
"""
Create a course w/ one item in the persistence store using the given course & item location.
"""
# create course
self.course = self.store.create_course(course_key.org, course_key.course, course_key.run, self.user_id)
if isinstance(self.course.id, CourseLocator):
self.course_locations[self.MONGO_COURSEID] = self.course.location.version_agnostic()
else:
self.assertEqual(self.course.id, course_key)
# create chapter
chapter = self.store.create_child(self.user_id, self.course.location, 'chapter', block_id='Overview')
self.writable_chapter_location = chapter.location.version_agnostic()
def _create_block_hierarchy(self):
"""
Creates a hierarchy of blocks for testing
Each block's (version_agnostic) location is assigned as a field of the class and can be easily accessed
"""
BlockInfo = namedtuple('BlockInfo', 'field_name, category, display_name, sub_tree')
trees = [
BlockInfo(
'chapter_x', 'chapter', 'Chapter_x', [
BlockInfo(
'sequential_x1', 'sequential', 'Sequential_x1', [
BlockInfo(
'vertical_x1a', 'vertical', 'Vertical_x1a', [
BlockInfo('problem_x1a_1', 'problem', 'Problem_x1a_1', []),
BlockInfo('problem_x1a_2', 'problem', 'Problem_x1a_2', []),
BlockInfo('problem_x1a_3', 'problem', 'Problem_x1a_3', []),
BlockInfo('html_x1a_1', 'html', 'HTML_x1a_1', []),
]
)
]
)
]
),
BlockInfo(
'chapter_y', 'chapter', 'Chapter_y', [
BlockInfo(
'sequential_y1', 'sequential', 'Sequential_y1', [
BlockInfo(
'vertical_y1a', 'vertical', 'Vertical_y1a', [
BlockInfo('problem_y1a_1', 'problem', 'Problem_y1a_1', []),
BlockInfo('problem_y1a_2', 'problem', 'Problem_y1a_2', []),
BlockInfo('problem_y1a_3', 'problem', 'Problem_y1a_3', []),
]
)
]
)
]
)
]
def create_sub_tree(parent, block_info):
block = self.store.create_child(
self.user_id, parent.location.version_agnostic(),
block_info.category, block_id=block_info.display_name
)
for tree in block_info.sub_tree:
create_sub_tree(block, tree)
setattr(self, block_info.field_name, block.location.version_agnostic())
for tree in trees:
create_sub_tree(self.course, tree)
def _course_key_from_string(self, string):
"""
Get the course key for the given course string
"""
return self.course_locations[string].course_key
def initdb(self, default):
"""
Initialize the database and create one test course in it
"""
# set the default modulestore
store_configs = self.options['stores']
for index in range(len(store_configs)):
if store_configs[index]['NAME'] == default:
if index > 0:
store_configs[index], store_configs[0] = store_configs[0], store_configs[index]
break
self.store = MixedModuleStore(None, create_modulestore_instance=create_modulestore_instance, **self.options)
self.addCleanup(self.store.close_all_connections)
# convert to CourseKeys
self.course_locations = {
course_id: CourseLocator.from_string(course_id)
for course_id in [self.MONGO_COURSEID, self.XML_COURSEID1, self.XML_COURSEID2]
}
# and then to the root UsageKey
self.course_locations = {
course_id: course_key.make_usage_key('course', course_key.run)
for course_id, course_key in self.course_locations.iteritems() # pylint: disable=maybe-no-member
}
if default == 'split':
self.fake_location = CourseLocator(
'foo', 'bar', 'slowly', branch=ModuleStoreEnum.BranchName.draft
).make_usage_key('vertical', 'baz')
else:
self.fake_location = Location('foo', 'bar', 'slowly', 'vertical', 'baz')
self.xml_chapter_location = self.course_locations[self.XML_COURSEID1].replace(
category='chapter', name='Overview'
)
self._create_course(default, self.course_locations[self.MONGO_COURSEID].course_key)
@ddt.data('draft', 'split')
def test_get_modulestore_type(self, default_ms):
"""
Make sure we get back the store type we expect for given mappings
"""
self.initdb(default_ms)
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.XML_COURSEID1)), ModuleStoreEnum.Type.xml
)
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.XML_COURSEID2)), ModuleStoreEnum.Type.xml
)
mongo_ms_type = ModuleStoreEnum.Type.mongo if default_ms == 'draft' else ModuleStoreEnum.Type.split
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.MONGO_COURSEID)), mongo_ms_type
)
# try an unknown mapping, it should be the 'default' store
self.assertEqual(self.store.get_modulestore_type(
SlashSeparatedCourseKey('foo', 'bar', '2012_Fall')), mongo_ms_type
)
@ddt.data('draft', 'split')
def test_has_item(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
self.assertTrue(self.store.has_item(course_locn))
# try negative cases
self.assertFalse(self.store.has_item(
self.course_locations[self.XML_COURSEID1].replace(name='not_findable', category='problem')
))
self.assertFalse(self.store.has_item(self.fake_location))
@ddt.data('draft', 'split')
def test_get_item(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
self.assertIsNotNone(self.store.get_item(course_locn))
# try negative cases
with self.assertRaises(ItemNotFoundError):
self.store.get_item(
self.course_locations[self.XML_COURSEID1].replace(name='not_findable', category='problem')
)
with self.assertRaises(ItemNotFoundError):
self.store.get_item(self.fake_location)
@ddt.data('draft', 'split')
def test_get_items(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
locn = course_locn.course_key
# NOTE: use get_course if you just want the course. get_items is expensive
modules = self.store.get_items(locn, category='course')
self.assertEqual(len(modules), 1)
self.assertEqual(modules[0].location, course_locn)
@ddt.data('draft', 'split')
def test_update_item(self, default_ms):
"""
Update should fail for r/o dbs and succeed for r/w ones
"""
self.initdb(default_ms)
course = self.store.get_course(self.course_locations[self.XML_COURSEID1].course_key)
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
with self.assertRaises(NotImplementedError): # ensure it doesn't allow writing
self.store.update_item(course, self.user_id)
# now do it for a r/w db
course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key)
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
self.store.update_item(course, self.user_id)
course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertTrue(course.show_calculator)
@ddt.data('draft', 'split')
def test_delete_item(self, default_ms):
"""
Delete should reject on r/o db and work on r/w one
"""
self.initdb(default_ms)
# r/o try deleting the chapter (is here to ensure it can't be deleted)
with self.assertRaises(NotImplementedError):
self.store.delete_item(self.xml_chapter_location, self.user_id)
self.store.delete_item(self.writable_chapter_location, self.user_id)
# verify it's gone
with self.assertRaises(ItemNotFoundError):
self.store.get_item(self.writable_chapter_location)
# create and delete a private vertical with private children
private_vert = self.store.create_child(
# don't use course_location as it may not be the repr
self.user_id, self.course_locations[self.MONGO_COURSEID],
'vertical', block_id='private'
)
private_leaf = self.store.create_child(
# don't use course_location as it may not be the repr
self.user_id, private_vert.location, 'html', block_id='private_leaf'
)
# verify pre delete state (just to verify that the test is valid)
self.assertTrue(hasattr(private_vert, 'is_draft') or private_vert.location.branch == ModuleStoreEnum.BranchName.draft)
if hasattr(private_vert.location, 'version_guid'):
# change to the HEAD version
vert_loc = private_vert.location.for_version(private_leaf.location.version_guid)
else:
vert_loc = private_vert.location
self.assertTrue(self.store.has_item(vert_loc))
self.assertTrue(self.store.has_item(private_leaf.location))
course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key, 0)
self.assertIn(vert_loc, course.children)
# update the component to force it to draft w/o forcing the unit to draft
# delete the vertical and ensure the course no longer points to it
self.store.delete_item(vert_loc, self.user_id)
course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key, 0)
if hasattr(private_vert.location, 'version_guid'):
# change to the HEAD version
vert_loc = private_vert.location.for_version(course.location.version_guid)
leaf_loc = private_leaf.location.for_version(course.location.version_guid)
else:
vert_loc = private_vert.location
leaf_loc = private_leaf.location
self.assertFalse(self.store.has_item(vert_loc))
self.assertFalse(self.store.has_item(leaf_loc))
self.assertNotIn(vert_loc, course.children)
# TODO can remove this once LMS-2869 is implemented
# first create a Published branch
self.store.publish(self.course_locations[self.MONGO_COURSEID], self.user_id)
# reproduce bug STUD-1965
# create and delete a private vertical with private children
private_vert = self.store.create_child(
# don't use course_location as it may not be the repr
self.user_id, self.course_locations[self.MONGO_COURSEID], 'vertical', block_id='publish'
)
private_leaf = self.store.create_child(
self.user_id, private_vert.location, 'html', block_id='bug_leaf'
)
self.store.publish(private_vert.location, self.user_id)
private_leaf.display_name = 'change me'
private_leaf = self.store.update_item(private_leaf, self.user_id)
# test succeeds if delete succeeds w/o error
self.store.delete_item(private_leaf.location, self.user_id)
@ddt.data('draft', 'split')
def test_get_courses(self, default_ms):
self.initdb(default_ms)
# we should have 3 total courses across all stores
courses = self.store.get_courses()
course_ids = [
course.location.version_agnostic()
if hasattr(course.location, 'version_agnostic') else course.location
for course in courses
]
self.assertEqual(len(courses), 3, "Not 3 courses: {}".format(course_ids))
self.assertIn(self.course_locations[self.MONGO_COURSEID], course_ids)
self.assertIn(self.course_locations[self.XML_COURSEID1], course_ids)
self.assertIn(self.course_locations[self.XML_COURSEID2], course_ids)
def test_xml_get_courses(self):
"""
Test that the xml modulestore only loaded the courses from the maps.
"""
self.initdb('draft')
xml_store = self.store._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
courses = xml_store.get_courses()
self.assertEqual(len(courses), 2)
course_ids = [course.id for course in courses]
self.assertIn(self.course_locations[self.XML_COURSEID1].course_key, course_ids)
self.assertIn(self.course_locations[self.XML_COURSEID2].course_key, course_ids)
# this course is in the directory from which we loaded courses but not in the map
self.assertNotIn("edX/toy/TT_2012_Fall", course_ids)
def test_xml_no_write(self):
"""
Test that the xml modulestore doesn't allow write ops.
"""
self.initdb('draft')
xml_store = self.store._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
# the important thing is not which exception it raises but that it raises an exception
with self.assertRaises(AttributeError):
xml_store.create_course("org", "course", "run", self.user_id)
@ddt.data('draft', 'split')
def test_get_course(self, default_ms):
self.initdb(default_ms)
for course_location in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
# NOTE: use get_course if you just want the course. get_items is expensive
course = self.store.get_course(course_location.course_key)
self.assertIsNotNone(course)
self.assertEqual(course.id, course_location.course_key)
@ddt.data('draft', 'split')
def test_get_parent_locations(self, default_ms):
self.initdb(default_ms)
parent = self.store.get_parent_location(self.writable_chapter_location)
self.assertEqual(parent, self.course_locations[self.MONGO_COURSEID])
parent = self.store.get_parent_location(self.xml_chapter_location)
self.assertEqual(parent, self.course_locations[self.XML_COURSEID1])
def verify_get_parent_locations_results(self, expected_results):
for child_location, parent_location, revision in expected_results:
self.assertEqual(
parent_location,
self.store.get_parent_location(child_location, revision=revision)
)
@ddt.data('draft', 'split')
def test_get_parent_locations_moved_child(self, default_ms):
self.initdb(default_ms)
self._create_block_hierarchy()
# publish the course
self.store.publish(self.course.location, self.user_id)
# make drafts of verticals
self.store.convert_to_draft(self.vertical_x1a, self.user_id)
self.store.convert_to_draft(self.vertical_y1a, self.user_id)
# move child problem_x1a_1 to vertical_y1a
child_to_move_location = self.problem_x1a_1
new_parent_location = self.vertical_y1a
old_parent_location = self.vertical_x1a
old_parent = self.store.get_item(old_parent_location)
old_parent.children.remove(child_to_move_location.replace(version_guid=old_parent.location.version_guid))
self.store.update_item(old_parent, self.user_id)
new_parent = self.store.get_item(new_parent_location)
new_parent.children.append(child_to_move_location.replace(version_guid=new_parent.location.version_guid))
self.store.update_item(new_parent, self.user_id)
self.verify_get_parent_locations_results([
(child_to_move_location, new_parent_location, None),
(child_to_move_location, new_parent_location, ModuleStoreEnum.RevisionOption.draft_preferred),
(child_to_move_location, old_parent_location.for_branch(ModuleStoreEnum.BranchName.published), ModuleStoreEnum.RevisionOption.published_only),
])
# publish the course again
self.store.publish(self.course.location, self.user_id)
self.verify_get_parent_locations_results([
(child_to_move_location, new_parent_location, None),
(child_to_move_location, new_parent_location, ModuleStoreEnum.RevisionOption.draft_preferred),
(child_to_move_location, new_parent_location.for_branch(ModuleStoreEnum.BranchName.published), ModuleStoreEnum.RevisionOption.published_only),
])
@ddt.data('draft')
def test_get_parent_locations_deleted_child(self, default_ms):
self.initdb(default_ms)
self._create_block_hierarchy()
# publish the course
self.store.publish(self.course.location, self.user_id)
# make draft of vertical
self.store.convert_to_draft(self.vertical_y1a, self.user_id)
# delete child problem_y1a_1
child_to_delete_location = self.problem_y1a_1
old_parent_location = self.vertical_y1a
self.store.delete_item(child_to_delete_location, self.user_id)
self.verify_get_parent_locations_results([
(child_to_delete_location, old_parent_location, None),
# Note: The following could be an unexpected result, but we want to avoid an extra database call
(child_to_delete_location, old_parent_location, ModuleStoreEnum.RevisionOption.draft_preferred),
(child_to_delete_location, old_parent_location, ModuleStoreEnum.RevisionOption.published_only),
])
# publish the course again
self.store.publish(self.course.location, self.user_id)
self.verify_get_parent_locations_results([
(child_to_delete_location, None, None),
(child_to_delete_location, None, ModuleStoreEnum.RevisionOption.draft_preferred),
(child_to_delete_location, None, ModuleStoreEnum.RevisionOption.published_only),
])
@ddt.data('draft')
def test_revert_to_published_root_draft(self, default_ms):
"""
Test calling revert_to_published on draft vertical.
"""
self.initdb(default_ms)
self._create_block_hierarchy()
vertical = self.store.get_item(self.vertical_x1a)
vertical_children_num = len(vertical.children)
self.store.publish(self.course.location, self.user_id)
# delete leaf problem (will make parent vertical a draft)
self.store.delete_item(self.problem_x1a_1, self.user_id)
draft_parent = self.store.get_item(self.vertical_x1a)
self.assertEqual(vertical_children_num - 1, len(draft_parent.children))
published_parent = self.store.get_item(
self.vertical_x1a,
revision=ModuleStoreEnum.RevisionOption.published_only
)
self.assertEqual(vertical_children_num, len(published_parent.children))
self.store.revert_to_published(self.vertical_x1a, self.user_id)
reverted_parent = self.store.get_item(self.vertical_x1a)
self.assertEqual(vertical_children_num, len(published_parent.children))
self.assertEqual(reverted_parent, published_parent)
@ddt.data('draft')
def test_revert_to_published_root_published(self, default_ms):
"""
Test calling revert_to_published on a published vertical with a draft child.
"""
self.initdb(default_ms)
self._create_block_hierarchy()
self.store.publish(self.course.location, self.user_id)
problem = self.store.get_item(self.problem_x1a_1)
orig_display_name = problem.display_name
# Change display name of problem and update just it (so parent remains published)
problem.display_name = "updated before calling revert"
self.store.update_item(problem, self.user_id)
self.store.revert_to_published(self.vertical_x1a, self.user_id)
reverted_problem = self.store.get_item(self.problem_x1a_1)
self.assertEqual(orig_display_name, reverted_problem.display_name)
@ddt.data('draft')
def test_revert_to_published_no_draft(self, default_ms):
"""
Test calling revert_to_published on vertical with no draft content does nothing.
"""
self.initdb(default_ms)
self._create_block_hierarchy()
self.store.publish(self.course.location, self.user_id)
orig_vertical = self.store.get_item(self.vertical_x1a)
self.store.revert_to_published(self.vertical_x1a, self.user_id)
reverted_vertical = self.store.get_item(self.vertical_x1a)
self.assertEqual(orig_vertical, reverted_vertical)
@ddt.data('draft')
def test_revert_to_published_no_published(self, default_ms):
"""
Test calling revert_to_published on vertical with no published version errors.
"""
self.initdb(default_ms)
self._create_block_hierarchy()
with self.assertRaises(InvalidVersionError):
self.store.revert_to_published(self.vertical_x1a, self.user_id)
@ddt.data('draft')
def test_revert_to_published_direct_only(self, default_ms):
"""
Test calling revert_to_published on a direct-only item is a no-op.
"""
self.initdb(default_ms)
self._create_block_hierarchy()
self.store.revert_to_published(self.sequential_x1, self.user_id)
reverted_parent = self.store.get_item(self.sequential_x1)
# It does not discard the child vertical, even though that child is a draft (with no published version)
self.assertEqual(1, len(reverted_parent.children))
@ddt.data('draft', 'split')
def test_get_orphans(self, default_ms):
self.initdb(default_ms)
course_id = self.course_locations[self.MONGO_COURSEID].course_key
# create parented children
self._create_block_hierarchy()
# orphans
orphan_locations = [
course_id.make_usage_key('chapter', 'OrphanChapter'),
course_id.make_usage_key('vertical', 'OrphanVertical'),
course_id.make_usage_key('problem', 'OrphanProblem'),
course_id.make_usage_key('html', 'OrphanHTML'),
]
# detached items (not considered as orphans)
detached_locations = [
course_id.make_usage_key('static_tab', 'StaticTab'),
course_id.make_usage_key('about', 'overview'),
course_id.make_usage_key('course_info', 'updates'),
]
for location in (orphan_locations + detached_locations):
self.store.create_item(
self.user_id,
location.course_key,
location.block_type,
block_id=location.block_id
)
found_orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertEqual(set(found_orphans), set(orphan_locations))
@ddt.data('draft')
def test_create_item_from_parent_location(self, default_ms):
"""
Test a code path missed by the above: passing an old-style location as parent but no
new location for the child
"""
self.initdb(default_ms)
self.store.create_child(
self.user_id,
self.course_locations[self.MONGO_COURSEID],
'problem',
block_id='orphan'
)
orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertEqual(len(orphans), 0, "unexpected orphans: {}".format(orphans))
@ddt.data('draft')
def test_get_courses_for_wiki(self, default_ms):
"""
Test the get_courses_for_wiki method
"""
self.initdb(default_ms)
course_locations = self.store.get_courses_for_wiki('toy')
self.assertEqual(len(course_locations), 1)
self.assertIn(self.course_locations[self.XML_COURSEID1], course_locations)
course_locations = self.store.get_courses_for_wiki('simple')
self.assertEqual(len(course_locations), 1)
self.assertIn(self.course_locations[self.XML_COURSEID2], course_locations)
self.assertEqual(len(self.store.get_courses_for_wiki('edX.simple.2012_Fall')), 0)
self.assertEqual(len(self.store.get_courses_for_wiki('no_such_wiki')), 0)
@ddt.data('draft', 'split')
def test_unpublish(self, default_ms):
"""
Test calling unpublish
"""
self.initdb(default_ms)
self._create_block_hierarchy()
# publish
self.store.publish(self.course.location, self.user_id)
published_xblock = self.store.get_item(
self.vertical_x1a,
revision=ModuleStoreEnum.RevisionOption.published_only
)
self.assertIsNotNone(published_xblock)
# unpublish
self.store.unpublish(self.vertical_x1a, self.user_id)
with self.assertRaises(ItemNotFoundError):
self.store.get_item(
self.vertical_x1a,
revision=ModuleStoreEnum.RevisionOption.published_only
)
# make sure draft version still exists
draft_xblock = self.store.get_item(
self.vertical_x1a,
revision=ModuleStoreEnum.RevisionOption.draft_only
)
self.assertIsNotNone(draft_xblock)
@ddt.data('draft', 'split')
def test_compute_publish_state(self, default_ms):
"""
Test the compute_publish_state method
"""
self.initdb(default_ms)
self._create_block_hierarchy()
# TODO - Remove this call to explicitly Publish the course once LMS-2869 is implemented
# For now, we need this since we can't publish a child item without its course already been published
course_location = self.course_locations[self.MONGO_COURSEID]
self.store.publish(course_location, self.user_id)
# start off as Private
item = self.store.create_child(self.user_id, self.writable_chapter_location, 'problem', 'test_compute_publish_state')
item_location = item.location.version_agnostic()
self.assertEquals(self.store.compute_publish_state(item), PublishState.private)
# Private -> Public
self.store.publish(item_location, self.user_id)
item = self.store.get_item(item_location)
self.assertEquals(self.store.compute_publish_state(item), PublishState.public)
# Public -> Private
self.store.unpublish(item_location, self.user_id)
item = self.store.get_item(item_location)
self.assertEquals(self.store.compute_publish_state(item), PublishState.private)
# Private -> Public
self.store.publish(item_location, self.user_id)
item = self.store.get_item(item_location)
self.assertEquals(self.store.compute_publish_state(item), PublishState.public)
# Public -> Draft with NO changes
# Note: This is where Split and Mongo differ
self.store.convert_to_draft(item_location, self.user_id)
item = self.store.get_item(item_location)
self.assertEquals(
self.store.compute_publish_state(item),
PublishState.draft if default_ms == 'draft' else PublishState.public
)
# Draft WITH changes
item.display_name = 'new name'
item = self.store.update_item(item, self.user_id)
self.assertTrue(self.store.has_changes(item.location))
self.assertEquals(self.store.compute_publish_state(item), PublishState.draft)
class TestMixedModuleStore(LocMapperSetupSansDjango):
"""
Quasi-superclass which tests Location based apps against both split and mongo dbs (Locator and
Location-based dbs)
"""
HOST = 'localhost'
PORT = 27017
DB = 'test_mongo_%s' % uuid4().hex[:5]
COLLECTION = 'modulestore'
FS_ROOT = DATA_DIR
DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor'
RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': ''
MONGO_COURSEID = 'MITx/999/2013_Spring'
XML_COURSEID1 = 'edX/toy/2012_Fall'
XML_COURSEID2 = 'edX/simple/2012_Fall'
BAD_COURSE_ID = 'edX/simple'
modulestore_options = {
'default_class': DEFAULT_CLASS,
'fs_root': DATA_DIR,
'render_template': RENDER_TEMPLATE,
}
DOC_STORE_CONFIG = {
'host': HOST,
'db': DB,
'collection': COLLECTION,
}
OPTIONS = {
'mappings': {
XML_COURSEID1: 'xml',
XML_COURSEID2: 'xml',
BAD_COURSE_ID: 'xml',
},
'stores': [
{
'NAME': 'draft',
'ENGINE': 'xmodule.modulestore.mongo.draft.DraftModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
{
'NAME': 'split',
'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
'OPTIONS': modulestore_options
},
{
'NAME': 'xml',
'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
'OPTIONS': {
'data_dir': DATA_DIR,
'default_class': 'xmodule.hidden_module.HiddenDescriptor',
}
},
]
}
def _compareIgnoreVersion(self, loc1, loc2, msg=None):
"""
AssertEqual replacement for CourseLocator
"""
if loc1.version_agnostic() != loc2.version_agnostic():
self.fail(self._formatMessage(msg, u"{} != {}".format(unicode(loc1), unicode(loc2))))
def setUp(self):
"""
Set up the database for testing
"""
self.options = getattr(self, 'options', self.OPTIONS)
self.connection = pymongo.MongoClient(
host=self.HOST,
port=self.PORT,
tz_aware=True,
)
self.connection.drop_database(self.DB)
self.addCleanup(self.connection.drop_database, self.DB)
self.addCleanup(self.connection.close)
super(TestMixedModuleStore, self).setUp()
patcher = patch.multiple(
'xmodule.modulestore.mixed',
loc_mapper=Mock(return_value=LocMapperSetupSansDjango.loc_store),
create_modulestore_instance=create_modulestore_instance,
)
patcher.start()
self.addCleanup(patcher.stop)
self.addTypeEqualityFunc(BlockUsageLocator, '_compareIgnoreVersion')
self.addTypeEqualityFunc(CourseLocator, '_compareIgnoreVersion')
# define attrs which get set in initdb to quell pylint
self.writable_chapter_location = self.store = self.fake_location = self.xml_chapter_location = None
self.course_locations = []
self.user_id = 0
# pylint: disable=invalid-name
def _create_course(self, default, course_key):
"""
Create a course w/ one item in the persistence store using the given course & item location.
"""
if default == 'split':
offering = course_key.offering.replace('/', '.')
else:
offering = course_key.offering
course = self.store.create_course(course_key.org, offering)
category = self.writable_chapter_location.category
block_id = self.writable_chapter_location.name
chapter = self.store.create_item(
# don't use course_location as it may not be the repr
course.location, category, location=self.writable_chapter_location, block_id=block_id
)
if isinstance(course.id, CourseLocator):
self.course_locations[self.MONGO_COURSEID] = course.location.version_agnostic()
self.writable_chapter_location = chapter.location.version_agnostic()
else:
self.assertEqual(course.id, course_key)
self.assertEqual(chapter.location, self.writable_chapter_location)
self.course = course
def _create_block_hierarchy(self):
"""
Creates a hierarchy of blocks for testing
Each block is assigned as a field of the class and can be easily accessed
"""
BlockInfo = namedtuple('BlockInfo', 'field_name, category, display_name, sub_tree')
trees = [
BlockInfo(
'chapter_x', 'chapter', 'Chapter_x', [
BlockInfo(
'sequential_x1', 'sequential', 'Sequential_x1', [
BlockInfo(
'vertical_x1a', 'vertical', 'Vertical_x1a', [
BlockInfo('problem_x1a_1', 'problem', 'Problem_x1a_1', []),
BlockInfo('problem_x1a_2', 'problem', 'Problem_x1a_2', []),
BlockInfo('problem_x1a_3', 'problem', 'Problem_x1a_3', []),
]
)
]
)
]
),
BlockInfo(
'chapter_y', 'chapter', 'Chapter_y', [
BlockInfo(
'sequential_y1', 'sequential', 'Sequential_y1', [
BlockInfo(
'vertical_y1a', 'vertical', 'Vertical_y1a', [
BlockInfo('problem_y1a_1', 'problem', 'Problem_y1a_1', []),
BlockInfo('problem_y1a_2', 'problem', 'Problem_y1a_2', []),
BlockInfo('problem_y1a_3', 'problem', 'Problem_y1a_3', []),
]
)
]
)
]
)
]
def create_sub_tree(parent, block_info):
block = self.store.create_item(parent.location, category=block_info.category, block_id=block_info.display_name)
for tree in block_info.sub_tree:
create_sub_tree(block, tree)
# reload the block to update its children field
block = self.store.get_item(block.location)
setattr(self, block_info.field_name, block)
for tree in trees:
create_sub_tree(self.course, tree)
def _course_key_from_string(self, string):
"""
Get the course key for the given course string
"""
return self.course_locations[string].course_key
def initdb(self, default):
"""
Initialize the database and create one test course in it
"""
# set the default modulestore
store_configs = self.options['stores']
for index in range(len(store_configs)):
if store_configs[index]['NAME'] == default:
if index > 0:
store_configs[index], store_configs[0] = store_configs[0], store_configs[index]
break
self.store = MixedModuleStore(**self.options)
self.addCleanup(self.store.close_all_connections)
# convert to CourseKeys
self.course_locations = {
course_id: SlashSeparatedCourseKey.from_deprecated_string(course_id)
for course_id in [self.MONGO_COURSEID, self.XML_COURSEID1, self.XML_COURSEID2]
}
# and then to the root UsageKey
self.course_locations = {
course_id: course_key.make_usage_key('course', course_key.run)
for course_id, course_key in self.course_locations.iteritems() # pylint: disable=maybe-no-member
}
self.fake_location = Location('foo', 'bar', 'slowly', 'vertical', 'baz')
self.writable_chapter_location = self.course_locations[self.MONGO_COURSEID].replace(
category='chapter', name='Overview'
)
self.xml_chapter_location = self.course_locations[self.XML_COURSEID1].replace(
category='chapter', name='Overview'
)
# get Locators and set up the loc mapper if app is Locator based
if default == 'split':
self.fake_location = loc_mapper().translate_location(self.fake_location)
self._create_course(default, self.course_locations[self.MONGO_COURSEID].course_key)
@ddt.data('draft', 'split')
def test_get_modulestore_type(self, default_ms):
"""
Make sure we get back the store type we expect for given mappings
"""
self.initdb(default_ms)
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.XML_COURSEID1)), ModuleStoreEnum.Type.xml
)
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.XML_COURSEID2)), ModuleStoreEnum.Type.xml
)
mongo_ms_type = ModuleStoreEnum.Type.mongo if default_ms == 'draft' else ModuleStoreEnum.Type.split
self.assertEqual(self.store.get_modulestore_type(
self._course_key_from_string(self.MONGO_COURSEID)), mongo_ms_type
)
# try an unknown mapping, it should be the 'default' store
self.assertEqual(self.store.get_modulestore_type(
SlashSeparatedCourseKey('foo', 'bar', '2012_Fall')), mongo_ms_type
)
@ddt.data('draft', 'split')
def test_has_item(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
self.assertTrue(self.store.has_item(course_locn))
# try negative cases
self.assertFalse(self.store.has_item(
self.course_locations[self.XML_COURSEID1].replace(name='not_findable', category='problem')
))
self.assertFalse(self.store.has_item(self.fake_location))
@ddt.data('draft', 'split')
def test_get_item(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
self.assertIsNotNone(self.store.get_item(course_locn))
# try negative cases
with self.assertRaises(ItemNotFoundError):
self.store.get_item(
self.course_locations[self.XML_COURSEID1].replace(name='not_findable', category='problem')
)
with self.assertRaises(ItemNotFoundError):
self.store.get_item(self.fake_location)
@ddt.data('draft', 'split')
def test_get_items(self, default_ms):
self.initdb(default_ms)
for course_locn in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
locn = course_locn.course_key
# NOTE: use get_course if you just want the course. get_items is expensive
modules = self.store.get_items(locn, category='course')
self.assertEqual(len(modules), 1)
self.assertEqual(modules[0].location, course_locn)
@ddt.data('draft', 'split')
def test_update_item(self, default_ms):
"""
Update should fail for r/o dbs and succeed for r/w ones
"""
self.initdb(default_ms)
course = self.store.get_course(self.course_locations[self.XML_COURSEID1].course_key)
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
with self.assertRaises(NotImplementedError): # ensure it doesn't allow writing
self.store.update_item(course, None)
# now do it for a r/w db
course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key)
# if following raised, then the test is really a noop, change it
self.assertFalse(course.show_calculator, "Default changed making test meaningless")
course.show_calculator = True
self.store.update_item(course, None)
course = self.store.get_course(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertTrue(course.show_calculator)
@ddt.data('draft', 'split')
def test_delete_item(self, default_ms):
"""
Delete should reject on r/o db and work on r/w one
"""
self.initdb(default_ms)
# r/o try deleting the course (is here to ensure it can't be deleted)
with self.assertRaises(NotImplementedError):
self.store.delete_item(self.xml_chapter_location, 13)
self.store.delete_item(self.writable_chapter_location, 9)
# verify it's gone
with self.assertRaises(ItemNotFoundError):
self.store.get_item(self.writable_chapter_location)
@ddt.data('draft', 'split')
def test_get_courses(self, default_ms):
self.initdb(default_ms)
# we should have 3 total courses across all stores
courses = self.store.get_courses()
course_ids = [
course.location.version_agnostic()
if hasattr(course.location, 'version_agnostic') else course.location
for course in courses
]
self.assertEqual(len(courses), 3, "Not 3 courses: {}".format(course_ids))
self.assertIn(self.course_locations[self.MONGO_COURSEID], course_ids)
self.assertIn(self.course_locations[self.XML_COURSEID1], course_ids)
self.assertIn(self.course_locations[self.XML_COURSEID2], course_ids)
def test_xml_get_courses(self):
"""
Test that the xml modulestore only loaded the courses from the maps.
"""
self.initdb('draft')
xml_store = self.store._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
courses = xml_store.get_courses()
self.assertEqual(len(courses), 2)
course_ids = [course.id for course in courses]
self.assertIn(self.course_locations[self.XML_COURSEID1].course_key, course_ids)
self.assertIn(self.course_locations[self.XML_COURSEID2].course_key, course_ids)
# this course is in the directory from which we loaded courses but not in the map
self.assertNotIn("edX/toy/TT_2012_Fall", course_ids)
def test_xml_no_write(self):
"""
Test that the xml modulestore doesn't allow write ops.
"""
self.initdb('draft')
xml_store = self.store._get_modulestore_by_type(ModuleStoreEnum.Type.xml)
# the important thing is not which exception it raises but that it raises an exception
with self.assertRaises(AttributeError):
xml_store.create_course("org", "course/run", 999)
@ddt.data('draft', 'split')
def test_get_course(self, default_ms):
self.initdb(default_ms)
for course_location in self.course_locations.itervalues(): # pylint: disable=maybe-no-member
# NOTE: use get_course if you just want the course. get_items is expensive
course = self.store.get_course(course_location.course_key)
self.assertIsNotNone(course)
self.assertEqual(course.id, course_location.course_key)
@ddt.data('draft', 'split')
def test_get_parent_locations(self, default_ms):
self.initdb(default_ms)
parent = self.store.get_parent_location(self.writable_chapter_location)
self.assertEqual(parent, self.course_locations[self.MONGO_COURSEID])
parent = self.store.get_parent_location(self.xml_chapter_location)
self.assertEqual(parent, self.course_locations[self.XML_COURSEID1])
def verify_get_parent_locations_results(self, expected_results):
# expected_results should be a list of (child, parent, revision)
for test in expected_results:
self.assertEqual(
test[1].location if test[1] else None,
self.store.get_parent_location(test[0].location, revision=test[2])
)
@ddt.data('draft')
def test_get_parent_locations_moved_child(self, default_ms):
self.initdb(default_ms)
self._create_block_hierarchy()
# publish the course
self.store.publish(self.course.location, self.user_id)
# make drafts of verticals
self.store.convert_to_draft(self.vertical_x1a.location, self.user_id)
self.store.convert_to_draft(self.vertical_y1a.location, self.user_id)
# move child problem_x1a_1 to vertical_y1a
child_to_move = self.problem_x1a_1
old_parent = self.vertical_x1a
new_parent = self.vertical_y1a
old_parent.children.remove(child_to_move.location)
new_parent.children.append(child_to_move.location)
self.store.update_item(old_parent, self.user_id)
self.store.update_item(new_parent, self.user_id)
self.verify_get_parent_locations_results([
(child_to_move, new_parent, None),
(child_to_move, new_parent, ModuleStoreEnum.RevisionOption.draft_preferred),
(child_to_move, old_parent, ModuleStoreEnum.RevisionOption.published_only),
])
# publish the course again
self.store.publish(self.course.location, self.user_id)
self.verify_get_parent_locations_results([
(child_to_move, new_parent, None),
(child_to_move, new_parent, ModuleStoreEnum.RevisionOption.draft_preferred),
(child_to_move, new_parent, ModuleStoreEnum.RevisionOption.published_only),
])
@ddt.data('draft')
def test_get_parent_locations_deleted_child(self, default_ms):
self.initdb(default_ms)
self._create_block_hierarchy()
# publish the course
self.store.publish(self.course.location, self.user_id)
# make draft of vertical
self.store.convert_to_draft(self.vertical_y1a.location, self.user_id)
# delete child problem_y1a_1
child_to_delete = self.problem_y1a_1
old_parent = self.vertical_y1a
self.store.delete_item(child_to_delete.location, self.user_id)
self.verify_get_parent_locations_results([
(child_to_delete, old_parent, None),
# Note: The following could be an unexpected result, but we want to avoid an extra database call
(child_to_delete, old_parent, ModuleStoreEnum.RevisionOption.draft_preferred),
(child_to_delete, old_parent, ModuleStoreEnum.RevisionOption.published_only),
])
# publish the course again
self.store.publish(self.course.location, self.user_id)
self.verify_get_parent_locations_results([
(child_to_delete, None, None),
(child_to_delete, None, ModuleStoreEnum.RevisionOption.draft_preferred),
(child_to_delete, None, ModuleStoreEnum.RevisionOption.published_only),
])
@ddt.data('draft', 'split')
def test_get_orphans(self, default_ms):
self.initdb(default_ms)
# create an orphan
course_id = self.course_locations[self.MONGO_COURSEID].course_key
orphan = self.store.create_item(course_id, 'problem', block_id='orphan')
found_orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID].course_key)
if default_ms == 'split':
self.assertEqual(found_orphans, [orphan.location.version_agnostic()])
else:
self.assertEqual(found_orphans, [orphan.location.to_deprecated_string()])
@ddt.data('draft')
def test_create_item_from_parent_location(self, default_ms):
"""
Test a code path missed by the above: passing an old-style location as parent but no
new location for the child
"""
self.initdb(default_ms)
self.store.create_item(self.course_locations[self.MONGO_COURSEID], 'problem', block_id='orphan')
orphans = self.store.get_orphans(self.course_locations[self.MONGO_COURSEID].course_key)
self.assertEqual(len(orphans), 0, "unexpected orphans: {}".format(orphans))
@ddt.data('draft')
def test_get_courses_for_wiki(self, default_ms):
"""
Test the get_courses_for_wiki method
"""
self.initdb(default_ms)
course_locations = self.store.get_courses_for_wiki('toy')
self.assertEqual(len(course_locations), 1)
self.assertIn(self.course_locations[self.XML_COURSEID1], course_locations)
course_locations = self.store.get_courses_for_wiki('simple')
self.assertEqual(len(course_locations), 1)
self.assertIn(self.course_locations[self.XML_COURSEID2], course_locations)
self.assertEqual(len(self.store.get_courses_for_wiki('edX.simple.2012_Fall')), 0)
self.assertEqual(len(self.store.get_courses_for_wiki('no_such_wiki')), 0)