def clone_course(self, source_course_id, dest_course_id, user_id):
"""
See the superclass for the general documentation.
If cloning w/in a store, delegates to that store's clone_course which, in order to be self-
sufficient, should handle the asset copying (call the same method as this one does)
If cloning between stores,
* copy the assets
* migrate the courseware
"""
source_modulestore = self._get_modulestore_for_courseid(source_course_id)
# for a temporary period of time, we may want to hardcode dest_modulestore as split if there's a split
# to have only course re-runs go to split. This code, however, uses the config'd priority
dest_modulestore = self._get_modulestore_for_courseid(dest_course_id)
if source_modulestore == dest_modulestore:
return source_modulestore.clone_course(source_course_id, dest_course_id, user_id)
# ensure super's only called once. The delegation above probably calls it; so, don't move
# the invocation above the delegation call
super(MixedModuleStore, self).clone_course(source_course_id, dest_course_id, user_id)
if dest_modulestore.get_modulestore_type() == ModuleStoreEnum.Type.split:
split_migrator = SplitMigrator(dest_modulestore, source_modulestore)
split_migrator.migrate_mongo_course(
source_course_id, user_id, dest_course_id.org, dest_course_id.course, dest_course_id.run
)
def setUp(self):
super(TestMigration, self).setUp()
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(),
**self.db_config)
self.split_mongo.loc_mapper = self.loc_mapper
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo,
self.draft_mongo, self.loc_mapper)
def handle(self, *args, **options):
course_key, user, org, course, run = self.parse_args(**options)
migrator = SplitMigrator(
source_modulestore=modulestore(),
split_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.split),
)
migrator.migrate_mongo_course(course_key, user, org, course, run)
def handle(self, *args, **options):
course_key, user, org, course, run = self.parse_args(**options)
migrator = SplitMigrator(
source_modulestore=modulestore(),
split_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.split), # lint-amnesty, pylint: disable=protected-access
)
migrator.migrate_mongo_course(course_key, user, org, course, run)
def handle(self, *args, **options):
course_key, user, org, course, run = self.parse_args(*args)
migrator = SplitMigrator(
source_modulestore=modulestore(),
split_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.split),
)
migrator.migrate_mongo_course(course_key, user, org, course, run)
def handle(self, *args, **options):
course_key, user, org, offering = self.parse_args(*args)
migrator = SplitMigrator(
draft_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo),
split_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.split),
loc_mapper=loc_mapper(),
)
migrator.migrate_mongo_course(course_key, user, org, offering)
def setUp(self):
super(TestMigration, self).setUp()
self.loc_mapper = LocMapperStore(**self.db_config)
self.old_mongo = MongoModuleStore(**self.modulestore_options)
self.draft_mongo = DraftModuleStore(**self.modulestore_options)
self.split_mongo = SplitMongoModuleStore(
loc_mapper=self.loc_mapper, **self.modulestore_options
)
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo, self.draft_mongo, self.loc_mapper)
self.course_location = None
self.create_source_course()
def handle(self, *args, **options):
course_key, user, org, offering = self.parse_args(*args)
migrator = SplitMigrator(
draft_modulestore=modulestore('default'),
direct_modulestore=modulestore('direct'),
split_modulestore=modulestore('split'),
loc_mapper=loc_mapper(),
)
migrator.migrate_mongo_course(course_key, user, org, offering)
def handle(self, *args, **options):
location, user, package_id = self.parse_args(*args)
migrator = SplitMigrator(
draft_modulestore=modulestore('default'),
direct_modulestore=modulestore('direct'),
split_modulestore=modulestore('split'),
loc_mapper=loc_mapper(),
)
migrator.migrate_mongo_course(location, user, package_id)
def handle(self, *args, **options):
location, user, package_id = self.parse_args(*args)
migrator = SplitMigrator(
draft_modulestore=modulestore('default'),
direct_modulestore=modulestore('direct'),
split_modulestore=modulestore('split'),
loc_mapper=loc_mapper(),
)
migrator.migrate_mongo_course(location, user, package_id)
def setUp(self):
super(TestRollbackSplitCourse, self).setUp()
self.old_course = CourseFactory()
uname = 'testuser'
email = '*****@*****.**'
password = '******'
self.user = User.objects.create_user(uname, email, password)
# migrate old course to split
migrator = SplitMigrator(
draft_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo),
split_modulestore=modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.split),
loc_mapper=loc_mapper(),
)
migrator.migrate_mongo_course(self.old_course.location, self.user)
self.course = modulestore('split').get_course(self.old_course.id)
def setUp(self):
super(TestRollbackSplitCourse, self).setUp()
self.old_course = CourseFactory()
uname = 'testuser'
email = '*****@*****.**'
password = '******'
self.user = User.objects.create_user(uname, email, password)
# migrate old course to split
migrator = SplitMigrator(
draft_modulestore=modulestore('default'),
direct_modulestore=modulestore('direct'),
split_modulestore=modulestore('split'),
loc_mapper=loc_mapper(),
)
migrator.migrate_mongo_course(self.old_course.location, self.user)
self.course = modulestore('split').get_course(self.old_course.id)
def setUp(self):
super(TestRollbackSplitCourse, self).setUp()
self.old_course = CourseFactory()
uname = "testuser"
email = "*****@*****.**"
password = "******"
self.user = User.objects.create_user(uname, email, password)
# migrate old course to split
migrator = SplitMigrator(
draft_modulestore=modulestore("default"),
direct_modulestore=modulestore("direct"),
split_modulestore=modulestore("split"),
loc_mapper=loc_mapper(),
)
migrator.migrate_mongo_course(self.old_course.location, self.user)
self.course = modulestore("split").get_course(self.old_course.id)
def setUp(self):
super(TestMigration, self).setUp()
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(),
**self.db_config)
self.old_mongo = MongoModuleStore(self.db_config,
**self.modulestore_options)
self.draft_mongo = DraftModuleStore(self.db_config,
**self.modulestore_options)
self.split_mongo = SplitMongoModuleStore(
doc_store_config=self.db_config,
loc_mapper=self.loc_mapper,
**self.modulestore_options)
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo,
self.draft_mongo, self.loc_mapper)
self.course_location = None
self.create_source_course()
def setUp(self):
super(TestMigration, self).setUp()
noop_cache = mock.Mock(spec=['get', 'set_many'])
noop_cache.configure_mock(**{'get.return_value': None})
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(noop_cache, **self.db_config)
self.old_mongo = MongoModuleStore(self.db_config,
**self.modulestore_options)
self.draft_mongo = DraftModuleStore(self.db_config,
**self.modulestore_options)
self.split_mongo = SplitMongoModuleStore(
doc_store_config=self.db_config,
loc_mapper=self.loc_mapper,
**self.modulestore_options)
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo,
self.draft_mongo, self.loc_mapper)
self.course_location = None
self.create_source_course()
def setUp(self):
super(TestRollbackSplitCourse, self).setUp()
self.old_course = CourseFactory()
uname = 'testuser'
email = '*****@*****.**'
password = '******'
self.user = User.objects.create_user(uname, email, password)
# migrate old course to split
migrator = SplitMigrator(
draft_modulestore=modulestore('default'),
direct_modulestore=modulestore('direct'),
split_modulestore=modulestore('split'),
loc_mapper=loc_mapper(),
)
migrator.migrate_mongo_course(self.old_course.location, self.user)
locator = loc_mapper().translate_location(self.old_course.id, self.old_course.location)
self.course = modulestore('split').get_course(locator)
def setUp(self):
super(TestMigration, self).setUp()
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(), **self.db_config)
self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
self.draft_mongo = DraftModuleStore(self.db_config, **self.modulestore_options)
self.split_mongo = SplitMongoModuleStore(
doc_store_config=self.db_config, loc_mapper=self.loc_mapper, **self.modulestore_options
)
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo, self.draft_mongo, self.loc_mapper)
self.course_location = None
self.create_source_course()
def setUp(self):
super(TestMigration, self).setUp()
noop_cache = mock.Mock(spec=['get', 'set_many'])
noop_cache.configure_mock(**{'get.return_value': None})
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(noop_cache, **self.db_config)
self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
self.draft_mongo = DraftModuleStore(self.db_config, **self.modulestore_options)
self.split_mongo = SplitMongoModuleStore(
doc_store_config=self.db_config,
loc_mapper=self.loc_mapper,
**self.modulestore_options
)
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo, self.draft_mongo, self.loc_mapper)
self.course_location = None
self.create_source_course()
class TestMigration(unittest.TestCase):
"""
Test the split migrator
"""
# Snippet of what would be in the django settings envs file
db_config = {
'host': 'localhost',
'db': 'test_xmodule',
'collection': 'modulestore{0}'.format(uuid.uuid4().hex[:5]),
}
modulestore_options = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': '',
'render_template': mock.Mock(return_value=""),
'xblock_mixins': (InheritanceMixin,)
}
def setUp(self):
super(TestMigration, self).setUp()
self.loc_mapper = LocMapperStore(**self.db_config)
self.old_mongo = MongoModuleStore(self.db_config, **self.modulestore_options)
self.draft_mongo = DraftModuleStore(self.db_config, **self.modulestore_options)
self.split_mongo = SplitMongoModuleStore(
doc_store_config=self.db_config,
loc_mapper=self.loc_mapper,
**self.modulestore_options
)
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo, self.draft_mongo, self.loc_mapper)
self.course_location = None
self.create_source_course()
def tearDown(self):
dbref = self.loc_mapper.db
dbref.drop_collection(self.loc_mapper.location_map)
split_db = self.split_mongo.db
split_db.drop_collection(self.split_mongo.db_connection.course_index)
split_db.drop_collection(self.split_mongo.db_connection.structures)
split_db.drop_collection(self.split_mongo.db_connection.definitions)
# old_mongo doesn't give a db attr, but all of the dbs are the same
dbref.drop_collection(self.old_mongo.collection)
dbref.connection.close()
super(TestMigration, self).tearDown()
def _create_and_get_item(self, store, location, data, metadata, runtime=None):
store.create_and_save_xmodule(location, data, metadata, runtime)
return store.get_item(location)
def create_source_course(self):
"""
A course testing all of the conversion mechanisms:
* some inheritable settings
* sequences w/ draft and live intermixed children to ensure all get to the draft but
only the live ones get to published. Some are only draft, some are both, some are only live.
* about, static_tab, and conditional documents
"""
location = Location('i4x', 'test_org', 'test_course', 'course', 'runid')
self.course_location = location
date_proxy = Date()
metadata = {
'start': date_proxy.to_json(datetime.datetime(2000, 3, 13, 4)),
'display_name': 'Migration test course',
}
data = {
'wiki_slug': 'test_course_slug'
}
course_root = self._create_and_get_item(self.old_mongo, location, data, metadata)
runtime = course_root.runtime
# chapters
location = location.replace(category='chapter', name=uuid.uuid4().hex)
chapter1 = self._create_and_get_item(self.old_mongo, location, {}, {'display_name': 'Chapter 1'}, runtime)
course_root.children.append(chapter1.location.url())
location = location.replace(category='chapter', name=uuid.uuid4().hex)
chapter2 = self._create_and_get_item(self.old_mongo, location, {}, {'display_name': 'Chapter 2'}, runtime)
course_root.children.append(chapter2.location.url())
self.old_mongo.update_children(course_root.location, course_root.children)
# vertical in live only
location = location.replace(category='vertical', name=uuid.uuid4().hex)
live_vert = self._create_and_get_item(self.old_mongo, location, {}, {'display_name': 'Live vertical'}, runtime)
chapter1.children.append(live_vert.location.url())
self.create_random_units(self.old_mongo, live_vert)
# vertical in both live and draft
location = location.replace(category='vertical', name=uuid.uuid4().hex)
both_vert = self._create_and_get_item(
self.old_mongo, location, {}, {'display_name': 'Both vertical'}, runtime
)
draft_both = self._create_and_get_item(
self.draft_mongo, location, {}, {'display_name': 'Both vertical renamed'}, runtime
)
chapter1.children.append(both_vert.location.url())
self.create_random_units(self.old_mongo, both_vert, self.draft_mongo, draft_both)
# vertical in draft only (x2)
location = location.replace(category='vertical', name=uuid.uuid4().hex)
draft_vert = self._create_and_get_item(
self.draft_mongo,
location, {}, {'display_name': 'Draft vertical'}, runtime)
chapter1.children.append(draft_vert.location.url())
self.create_random_units(self.draft_mongo, draft_vert)
location = location.replace(category='vertical', name=uuid.uuid4().hex)
draft_vert = self._create_and_get_item(
self.draft_mongo,
location, {}, {'display_name': 'Draft vertical2'}, runtime)
chapter1.children.append(draft_vert.location.url())
self.create_random_units(self.draft_mongo, draft_vert)
# and finally one in live only (so published has to skip 2)
location = location.replace(category='vertical', name=uuid.uuid4().hex)
live_vert = self._create_and_get_item(
self.old_mongo,
location, {}, {'display_name': 'Live vertical end'}, runtime)
chapter1.children.append(live_vert.location.url())
self.create_random_units(self.old_mongo, live_vert)
# update the chapter
self.old_mongo.update_children(chapter1.location, chapter1.children)
# now the other one w/ the conditional
# first create some show children
indirect1 = self._create_and_get_item(
self.old_mongo,
location.replace(category='discussion', name=uuid.uuid4().hex),
"", {'display_name': 'conditional show 1'}, runtime
)
indirect2 = self._create_and_get_item(
self.old_mongo,
location.replace(category='html', name=uuid.uuid4().hex),
"", {'display_name': 'conditional show 2'}, runtime
)
location = location.replace(category='conditional', name=uuid.uuid4().hex)
metadata = {
'xml_attributes': {
'sources': [live_vert.location.url(), ],
'completed': True,
},
}
data = {
'show_tag_list': [indirect1.location.url(), indirect2.location.url()]
}
conditional = self._create_and_get_item(self.old_mongo, location, data, metadata, runtime)
conditional.children = [indirect1.location.url(), indirect2.location.url()]
# add direct children
self.create_random_units(self.old_mongo, conditional)
chapter2.children.append(conditional.location.url())
self.old_mongo.update_children(chapter2.location, chapter2.children)
# and the ancillary docs (not children)
location = location.replace(category='static_tab', name=uuid.uuid4().hex)
# the below automatically adds the tab to the course
_tab = self._create_and_get_item(self.old_mongo, location, "", {'display_name': 'Tab uno'}, runtime)
location = location.replace(category='about', name='overview')
_overview = self._create_and_get_item(self.old_mongo, location, "<p>test</p>", {}, runtime)
location = location.replace(category='course_info', name='updates')
_overview = self._create_and_get_item(
self.old_mongo,
location, "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, runtime
)
def create_random_units(self, store, parent, cc_store=None, cc_parent=None):
"""
Create a random selection of units under the given parent w/ random names & attrs
:param store: which store (e.g., direct/draft) to create them in
:param parent: the parent to have point to them
:param cc_store: (optional) if given, make a small change and save also to this store but w/ same location
(only makes sense if store is 'direct' and this is 'draft' or vice versa)
"""
for _ in range(random.randrange(6)):
location = parent.location.replace(
category=random.choice(['html', 'video', 'problem', 'discussion']),
name=uuid.uuid4().hex
)
metadata = {'display_name': str(uuid.uuid4()), 'graded': True}
data = {}
element = self._create_and_get_item(store, location, data, metadata, parent.runtime)
parent.children.append(element.location.url())
if cc_store is not None:
# change display_name and remove graded to test the delta
element = self._create_and_get_item(
cc_store, location, data, {'display_name': str(uuid.uuid4())}, parent.runtime
)
cc_parent.children.append(element.location.url())
store.update_children(parent.location, parent.children)
if cc_store is not None:
cc_store.update_children(cc_parent.location, cc_parent.children)
def compare_courses(self, presplit, published):
# descend via children to do comparison
old_root = presplit.get_item(self.course_location, depth=None)
new_root_locator = self.loc_mapper.translate_location(
self.course_location.course_id, self.course_location, published, add_entry_if_missing=False
)
new_root = self.split_mongo.get_course(new_root_locator)
self.compare_dags(presplit, old_root, new_root, published)
# grab the detached items to compare they should be in both published and draft
for category in ['conditional', 'about', 'course_info', 'static_tab']:
location = self.course_location.replace(name=None, category=category)
for conditional in presplit.get_items(location):
locator = self.loc_mapper.translate_location(
self.course_location.course_id,
conditional.location, published, add_entry_if_missing=False
)
self.compare_dags(presplit, conditional, self.split_mongo.get_item(locator), published)
def compare_dags(self, presplit, presplit_dag_root, split_dag_root, published):
# check that locations match
self.assertEqual(
presplit_dag_root.location,
self.loc_mapper.translate_locator_to_location(split_dag_root.location).replace(revision=None)
)
# compare all fields but children
for name in presplit_dag_root.fields.iterkeys():
if name != 'children':
self.assertEqual(
getattr(presplit_dag_root, name),
getattr(split_dag_root, name),
"{}/{}: {} != {}".format(
split_dag_root.location, name, getattr(presplit_dag_root, name), getattr(split_dag_root, name)
)
)
# test split get_item using old Location: old draft store didn't set revision for things above vertical
# but split does distinguish these; so, set revision if not published
if not published:
location = draft.as_draft(presplit_dag_root.location)
else:
location = presplit_dag_root.location
refetched = self.split_mongo.get_item(location)
self.assertEqual(
refetched.location, split_dag_root.location,
"Fetch from split via old Location {} not same as new {}".format(
refetched.location, split_dag_root.location
)
)
# compare children
if presplit_dag_root.has_children:
self.assertEqual(
len(presplit_dag_root.get_children()), len(split_dag_root.get_children()),
"{0.category} '{0.display_name}': children count {1} != {2}".format(
presplit_dag_root, len(presplit_dag_root.get_children()), split_dag_root.children
)
)
for pre_child, split_child in zip(presplit_dag_root.get_children(), split_dag_root.get_children()):
self.compare_dags(presplit, pre_child, split_child, published)
def test_migrator(self):
self.migrator.migrate_mongo_course(self.course_location, random.getrandbits(32))
# now compare the migrated to the original course
self.compare_courses(self.old_mongo, True)
self.compare_courses(self.draft_mongo, False)
def setUp(self):
super(TestMigration, self).setUp()
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(), **self.db_config)
self.split_mongo.loc_mapper = self.loc_mapper
self.migrator = SplitMigrator(self.split_mongo, self.draft_mongo, self.loc_mapper)
class TestMigration(SplitWMongoCourseBoostrapper):
"""
Test the split migrator
"""
def setUp(self):
super(TestMigration, self).setUp()
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(), **self.db_config)
self.split_mongo.loc_mapper = self.loc_mapper
self.migrator = SplitMigrator(self.split_mongo, self.draft_mongo, self.loc_mapper)
def tearDown(self):
dbref = self.loc_mapper.db
dbref.drop_collection(self.loc_mapper.location_map)
super(TestMigration, self).tearDown()
def _create_course(self):
"""
A course testing all of the conversion mechanisms:
* some inheritable settings
* sequences w/ draft and live intermixed children to ensure all get to the draft but
only the live ones get to published. Some are only draft, some are both, some are only live.
* about, static_tab, and conditional documents
"""
super(TestMigration, self)._create_course(split=False)
# chapters
chapter1_name = uuid.uuid4().hex
self._create_item('chapter', chapter1_name, {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
chap2_loc = self.old_course_key.make_usage_key('chapter', uuid.uuid4().hex)
self._create_item(
chap2_loc.category, chap2_loc.name, {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False
)
# vertical in live only
live_vert_name = uuid.uuid4().hex
self._create_item(
'vertical', live_vert_name, {}, {'display_name': 'Live vertical'}, 'chapter', chapter1_name,
draft=False, split=False
)
self.create_random_units(False, self.old_course_key.make_usage_key('vertical', live_vert_name))
# vertical in both live and draft
both_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
both_vert_loc.category, both_vert_loc.name, {}, {'display_name': 'Both vertical'}, 'chapter', chapter1_name,
draft=False, split=False
)
self.create_random_units(False, both_vert_loc)
draft_both = self.draft_mongo.get_item(both_vert_loc)
draft_both.display_name = 'Both vertical renamed'
self.draft_mongo.update_item(draft_both, ModuleStoreEnum.UserID.test)
self.create_random_units(True, both_vert_loc)
# vertical in draft only (x2)
draft_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
draft_vert_loc.category, draft_vert_loc.name, {}, {'display_name': 'Draft vertical'}, 'chapter', chapter1_name,
draft=True, split=False
)
self.create_random_units(True, draft_vert_loc)
draft_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
draft_vert_loc.category, draft_vert_loc.name, {}, {'display_name': 'Draft vertical2'}, 'chapter', chapter1_name,
draft=True, split=False
)
self.create_random_units(True, draft_vert_loc)
# and finally one in live only (so published has to skip 2 preceding sibs)
live_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
live_vert_loc.category, live_vert_loc.name, {}, {'display_name': 'Live vertical end'}, 'chapter', chapter1_name,
draft=False, split=False
)
self.create_random_units(True, draft_vert_loc)
# now the other chapter w/ the conditional
# create pointers to children (before the children exist)
indirect1_loc = self.old_course_key.make_usage_key('discussion', uuid.uuid4().hex)
indirect2_loc = self.old_course_key.make_usage_key('html', uuid.uuid4().hex)
conditional_loc = self.old_course_key.make_usage_key('conditional', uuid.uuid4().hex)
self._create_item(
conditional_loc.category, conditional_loc.name,
{
'show_tag_list': [indirect1_loc, indirect2_loc],
'sources_list': [live_vert_loc, ],
},
{
'xml_attributes': {
'completed': True,
},
},
chap2_loc.category, chap2_loc.name,
draft=False, split=False
)
# create the children
self._create_item(
indirect1_loc.category, indirect1_loc.name, {'data': ""}, {'display_name': 'conditional show 1'},
conditional_loc.category, conditional_loc.name,
draft=False, split=False
)
self._create_item(
indirect2_loc.category, indirect2_loc.name, {'data': ""}, {'display_name': 'conditional show 2'},
conditional_loc.category, conditional_loc.name,
draft=False, split=False
)
# add direct children
self.create_random_units(False, conditional_loc)
# and the ancillary docs (not children)
self._create_item(
'static_tab', uuid.uuid4().hex, {'data': ""}, {'display_name': 'Tab uno'},
None, None, draft=False, split=False
)
self._create_item(
'about', 'overview', {'data': "<p>test</p>"}, {},
None, None, draft=False, split=False
)
self._create_item(
'course_info', 'updates', {'data': "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>"}, {},
None, None, draft=False, split=False
)
def create_random_units(self, draft, parent_loc):
"""
Create a random selection of units under the given parent w/ random names & attrs
:param store: which store (e.g., direct/draft) to create them in
:param parent: the parent to have point to them
(only makes sense if store is 'direct' and this is 'draft' or vice versa)
"""
for _ in range(random.randrange(6)):
location = parent_loc.replace(
category=random.choice(['html', 'video', 'problem', 'discussion']),
name=uuid.uuid4().hex
)
metadata = {'display_name': str(uuid.uuid4()), 'graded': True}
data = {}
self._create_item(
location.category, location.name, data, metadata, parent_loc.category, parent_loc.name,
draft=draft, split=False
)
def compare_courses(self, presplit, published):
# descend via children to do comparison
old_root = presplit.get_course(self.old_course_key)
new_root_locator = self.loc_mapper.translate_location_to_course_locator(
old_root.id, published
)
new_root = self.split_mongo.get_course(new_root_locator)
self.compare_dags(presplit, old_root, new_root, published)
# grab the detached items to compare they should be in both published and draft
for category in ['conditional', 'about', 'course_info', 'static_tab']:
for conditional in presplit.get_items(self.old_course_key, category=category):
locator = self.loc_mapper.translate_location(
conditional.location,
published,
add_entry_if_missing=False
)
self.compare_dags(presplit, conditional, self.split_mongo.get_item(locator), published)
def compare_dags(self, presplit, presplit_dag_root, split_dag_root, published):
# check that locations match
self.assertEqual(
presplit_dag_root.location,
self.loc_mapper.translate_locator_to_location(split_dag_root.location).replace(
revision=MongoRevisionKey.published
)
)
# compare all fields but children
for name, field in presplit_dag_root.fields.iteritems():
if not isinstance(field, (Reference, ReferenceList, ReferenceValueDict)):
self.assertEqual(
getattr(presplit_dag_root, name),
getattr(split_dag_root, name),
"{}/{}: {} != {}".format(
split_dag_root.location, name, getattr(presplit_dag_root, name), getattr(split_dag_root, name)
)
)
# compare children
if presplit_dag_root.has_children:
self.assertEqual(
# need get_children to filter out drafts
len(presplit_dag_root.get_children()), len(split_dag_root.children),
"{0.category} '{0.display_name}': children {1} != {2}".format(
presplit_dag_root, presplit_dag_root.children, split_dag_root.children
)
)
for pre_child, split_child in zip(presplit_dag_root.get_children(), split_dag_root.get_children()):
self.compare_dags(presplit, pre_child, split_child, published)
def test_migrator(self):
user = mock.Mock(id=1)
self.migrator.migrate_mongo_course(self.old_course_key, user)
# now compare the migrated to the original course
self.compare_courses(self.old_mongo, True)
self.compare_courses(self.draft_mongo, False)
def setUp(self):
super(TestMigration, self).setUp()
self.migrator = SplitMigrator(self.split_mongo, self.draft_mongo)
class TestMigration(SplitWMongoCourseBoostrapper):
"""
Test the split migrator
"""
def setUp(self):
super(TestMigration, self).setUp()
self.migrator = SplitMigrator(self.split_mongo, self.draft_mongo)
def _create_course(self):
"""
A course testing all of the conversion mechanisms:
* some inheritable settings
* sequences w/ draft and live intermixed children to ensure all get to the draft but
only the live ones get to published. Some are only draft, some are both, some are only live.
* about, static_tab, and conditional documents
"""
super(TestMigration, self)._create_course(split=False)
# chapters
chapter1_name = uuid.uuid4().hex
self._create_item('chapter',
chapter1_name, {}, {'display_name': 'Chapter 1'},
'course',
'runid',
split=False)
chap2_loc = self.old_course_key.make_usage_key('chapter',
uuid.uuid4().hex)
self._create_item(chap2_loc.category,
chap2_loc.name, {}, {'display_name': 'Chapter 2'},
'course',
'runid',
split=False)
# vertical in live only
live_vert_name = uuid.uuid4().hex
self._create_item('vertical',
live_vert_name, {},
{'display_name': 'Live vertical'},
'chapter',
chapter1_name,
draft=False,
split=False)
self.create_random_units(
False,
self.old_course_key.make_usage_key('vertical', live_vert_name))
# vertical in both live and draft
both_vert_loc = self.old_course_key.make_usage_key(
'vertical',
uuid.uuid4().hex)
self._create_item(both_vert_loc.category,
both_vert_loc.name, {},
{'display_name': 'Both vertical'},
'chapter',
chapter1_name,
draft=False,
split=False)
self.create_random_units(False, both_vert_loc)
draft_both = self.draft_mongo.get_item(both_vert_loc)
draft_both.display_name = 'Both vertical renamed'
self.draft_mongo.update_item(draft_both, self.user_id)
self.create_random_units(True, both_vert_loc)
# vertical in draft only (x2)
draft_vert_loc = self.old_course_key.make_usage_key(
'vertical',
uuid.uuid4().hex)
self._create_item(draft_vert_loc.category,
draft_vert_loc.name, {},
{'display_name': 'Draft vertical'},
'chapter',
chapter1_name,
draft=True,
split=False)
self.create_random_units(True, draft_vert_loc)
draft_vert_loc = self.old_course_key.make_usage_key(
'vertical',
uuid.uuid4().hex)
self._create_item(draft_vert_loc.category,
draft_vert_loc.name, {},
{'display_name': 'Draft vertical2'},
'chapter',
chapter1_name,
draft=True,
split=False)
self.create_random_units(True, draft_vert_loc)
# and finally one in live only (so published has to skip 2 preceding sibs)
live_vert_loc = self.old_course_key.make_usage_key(
'vertical',
uuid.uuid4().hex)
self._create_item(live_vert_loc.category,
live_vert_loc.name, {},
{'display_name': 'Live vertical end'},
'chapter',
chapter1_name,
draft=False,
split=False)
self.create_random_units(False, live_vert_loc)
# now the other chapter w/ the conditional
# create pointers to children (before the children exist)
indirect1_loc = self.old_course_key.make_usage_key(
'discussion',
uuid.uuid4().hex)
indirect2_loc = self.old_course_key.make_usage_key(
'html',
uuid.uuid4().hex)
conditional_loc = self.old_course_key.make_usage_key(
'conditional',
uuid.uuid4().hex)
self._create_item(conditional_loc.category,
conditional_loc.name, {
'show_tag_list': [indirect1_loc, indirect2_loc],
'sources_list': [
live_vert_loc,
],
}, {
'xml_attributes': {
'completed': True,
},
},
chap2_loc.category,
chap2_loc.name,
draft=False,
split=False)
# create the children
self._create_item(indirect1_loc.category,
indirect1_loc.name, {'data': ""},
{'display_name': 'conditional show 1'},
conditional_loc.category,
conditional_loc.name,
draft=False,
split=False)
self._create_item(indirect2_loc.category,
indirect2_loc.name, {'data': ""},
{'display_name': 'conditional show 2'},
conditional_loc.category,
conditional_loc.name,
draft=False,
split=False)
# add direct children
self.create_random_units(False, conditional_loc)
# and the ancillary docs (not children)
self._create_item('static_tab',
uuid.uuid4().hex, {'data': ""},
{'display_name': 'Tab uno'},
None,
None,
draft=False,
split=False)
self._create_item('about',
'overview', {'data': "<p>test</p>"}, {},
None,
None,
draft=False,
split=False)
self._create_item(
'course_info',
'updates',
{'data': "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>"}, {},
None,
None,
draft=False,
split=False)
def create_random_units(self, draft, parent_loc):
"""
Create a random selection of units under the given parent w/ random names & attrs
:param store: which store (e.g., direct/draft) to create them in
:param parent: the parent to have point to them
(only makes sense if store is 'direct' and this is 'draft' or vice versa)
"""
for _ in range(random.randrange(6)):
location = parent_loc.replace(category=random.choice(
['html', 'video', 'problem', 'discussion']),
name=uuid.uuid4().hex)
metadata = {'display_name': str(uuid.uuid4()), 'graded': True}
data = {}
self._create_item(location.category,
location.name,
data,
metadata,
parent_loc.category,
parent_loc.name,
draft=draft,
split=False)
def compare_courses(self, presplit, new_course_key, published):
# descend via children to do comparison
old_root = presplit.get_course(self.old_course_key)
new_root = self.split_mongo.get_course(new_course_key)
self.compare_dags(presplit, old_root, new_root, published)
# grab the detached items to compare they should be in both published and draft
for category in ['conditional', 'about', 'course_info', 'static_tab']:
for conditional in presplit.get_items(
self.old_course_key, qualifiers={'category': category}):
locator = new_course_key.make_usage_key(
category, conditional.location.block_id)
self.compare_dags(presplit, conditional,
self.split_mongo.get_item(locator),
published)
def compare_dags(self, presplit, presplit_dag_root, split_dag_root,
published):
if split_dag_root.category != 'course':
self.assertEqual(presplit_dag_root.location.block_id,
split_dag_root.location.block_id)
# compare all fields but references
for name, field in presplit_dag_root.fields.iteritems():
# fields generated from UNIQUE_IDs are unique to an XBlock's scope,
# so if such a field is unset on an XBlock, we don't expect it
# to persist across courses
field_generated_from_unique_id = not field.is_set_on(
presplit_dag_root) and field.default == UNIQUE_ID
should_check_field = not (
field_generated_from_unique_id
or isinstance(field,
(Reference, ReferenceList, ReferenceValueDict)))
if should_check_field:
self.assertEqual(
getattr(presplit_dag_root, name),
getattr(split_dag_root, name),
u"{}/{}: {} != {}".format(split_dag_root.location, name,
getattr(presplit_dag_root, name),
getattr(split_dag_root, name)))
# compare children
if presplit_dag_root.has_children:
self.assertEqual(
# need get_children to filter out drafts
len(presplit_dag_root.get_children()),
len(split_dag_root.children),
u"{0.category} '{0.display_name}': children {1} != {2}".
format(presplit_dag_root, presplit_dag_root.children,
split_dag_root.children))
for pre_child, split_child in zip(presplit_dag_root.get_children(),
split_dag_root.get_children()):
self.compare_dags(presplit, pre_child, split_child, published)
def test_migrator(self):
user = mock.Mock(id=1)
new_course_key = self.migrator.migrate_mongo_course(
self.old_course_key, user.id, new_run='new_run')
# now compare the migrated to the original course
self.compare_courses(self.draft_mongo, new_course_key,
True) # published
self.compare_courses(self.draft_mongo, new_course_key, False) # draft
def setUp(self):
super(TestMigration, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
self.migrator = SplitMigrator(self.split_mongo, self.draft_mongo)
class TestMigration(unittest.TestCase):
"""
Test the split migrator
"""
# Snippet of what would be in the django settings envs file
db_config = {
'host': 'localhost',
'db': 'test_xmodule',
'collection': 'modulestore{0}'.format(uuid.uuid4().hex[:5]),
}
modulestore_options = {
'default_class': 'xmodule.raw_module.RawDescriptor',
'fs_root': '',
'render_template': mock.Mock(return_value=""),
'xblock_mixins': (InheritanceMixin, )
}
def setUp(self):
super(TestMigration, self).setUp()
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(),
**self.db_config)
self.old_mongo = MongoModuleStore(self.db_config,
**self.modulestore_options)
self.draft_mongo = DraftModuleStore(self.db_config,
**self.modulestore_options)
self.split_mongo = SplitMongoModuleStore(
doc_store_config=self.db_config,
loc_mapper=self.loc_mapper,
**self.modulestore_options)
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo,
self.draft_mongo, self.loc_mapper)
self.course_location = None
self.create_source_course()
def tearDown(self):
dbref = self.loc_mapper.db
dbref.drop_collection(self.loc_mapper.location_map)
split_db = self.split_mongo.db
split_db.drop_collection(self.split_mongo.db_connection.course_index)
split_db.drop_collection(self.split_mongo.db_connection.structures)
split_db.drop_collection(self.split_mongo.db_connection.definitions)
# old_mongo doesn't give a db attr, but all of the dbs are the same
dbref.drop_collection(self.old_mongo.collection)
dbref.connection.close()
super(TestMigration, self).tearDown()
def _create_and_get_item(self,
store,
location,
data,
metadata,
runtime=None):
store.create_and_save_xmodule(location, data, metadata, runtime)
return store.get_item(location)
def create_source_course(self):
"""
A course testing all of the conversion mechanisms:
* some inheritable settings
* sequences w/ draft and live intermixed children to ensure all get to the draft but
only the live ones get to published. Some are only draft, some are both, some are only live.
* about, static_tab, and conditional documents
"""
location = Location('i4x', 'test_org', 'test_course', 'course',
'runid')
self.course_location = location
date_proxy = Date()
metadata = {
'start': date_proxy.to_json(datetime.datetime(2000, 3, 13, 4)),
'display_name': 'Migration test course',
}
data = {'wiki_slug': 'test_course_slug'}
course_root = self._create_and_get_item(self.old_mongo, location, data,
metadata)
runtime = course_root.runtime
# chapters
location = location.replace(category='chapter', name=uuid.uuid4().hex)
chapter1 = self._create_and_get_item(self.old_mongo, location, {},
{'display_name': 'Chapter 1'},
runtime)
course_root.children.append(chapter1.location.url())
location = location.replace(category='chapter', name=uuid.uuid4().hex)
chapter2 = self._create_and_get_item(self.old_mongo, location, {},
{'display_name': 'Chapter 2'},
runtime)
course_root.children.append(chapter2.location.url())
self.old_mongo.update_item(course_root, '**replace_user**')
# vertical in live only
location = location.replace(category='vertical', name=uuid.uuid4().hex)
live_vert = self._create_and_get_item(
self.old_mongo, location, {}, {'display_name': 'Live vertical'},
runtime)
chapter1.children.append(live_vert.location.url())
self.create_random_units(self.old_mongo, live_vert)
# vertical in both live and draft
location = location.replace(category='vertical', name=uuid.uuid4().hex)
both_vert = self._create_and_get_item(
self.old_mongo, location, {}, {'display_name': 'Both vertical'},
runtime)
draft_both = self._create_and_get_item(
self.draft_mongo, location, {},
{'display_name': 'Both vertical renamed'}, runtime)
chapter1.children.append(both_vert.location.url())
self.create_random_units(self.old_mongo, both_vert, self.draft_mongo,
draft_both)
# vertical in draft only (x2)
location = location.replace(category='vertical', name=uuid.uuid4().hex)
draft_vert = self._create_and_get_item(
self.draft_mongo, location, {}, {'display_name': 'Draft vertical'},
runtime)
chapter1.children.append(draft_vert.location.url())
self.create_random_units(self.draft_mongo, draft_vert)
location = location.replace(category='vertical', name=uuid.uuid4().hex)
draft_vert = self._create_and_get_item(
self.draft_mongo, location, {},
{'display_name': 'Draft vertical2'}, runtime)
chapter1.children.append(draft_vert.location.url())
self.create_random_units(self.draft_mongo, draft_vert)
# and finally one in live only (so published has to skip 2)
location = location.replace(category='vertical', name=uuid.uuid4().hex)
live_vert = self._create_and_get_item(
self.old_mongo, location, {},
{'display_name': 'Live vertical end'}, runtime)
chapter1.children.append(live_vert.location.url())
self.create_random_units(self.old_mongo, live_vert)
# update the chapter
self.old_mongo.update_item(chapter1, '**replace_user**')
# now the other one w/ the conditional
# first create some show children
indirect1 = self._create_and_get_item(
self.old_mongo,
location.replace(category='discussion', name=uuid.uuid4().hex), "",
{'display_name': 'conditional show 1'}, runtime)
indirect2 = self._create_and_get_item(
self.old_mongo,
location.replace(category='html', name=uuid.uuid4().hex), "",
{'display_name': 'conditional show 2'}, runtime)
location = location.replace(category='conditional',
name=uuid.uuid4().hex)
metadata = {
'xml_attributes': {
'sources': [
live_vert.location.url(),
],
'completed': True,
},
}
data = {
'show_tag_list':
[indirect1.location.url(),
indirect2.location.url()]
}
conditional = self._create_and_get_item(self.old_mongo, location, data,
metadata, runtime)
conditional.children = [
indirect1.location.url(),
indirect2.location.url()
]
# add direct children
self.create_random_units(self.old_mongo, conditional)
chapter2.children.append(conditional.location.url())
self.old_mongo.update_item(chapter2, '**replace_user**')
# and the ancillary docs (not children)
location = location.replace(category='static_tab',
name=uuid.uuid4().hex)
# the below automatically adds the tab to the course
_tab = self._create_and_get_item(self.old_mongo, location, "",
{'display_name': 'Tab uno'}, runtime)
location = location.replace(category='about', name='overview')
_overview = self._create_and_get_item(self.old_mongo, location,
"<p>test</p>", {}, runtime)
location = location.replace(category='course_info', name='updates')
_overview = self._create_and_get_item(
self.old_mongo, location,
"<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, runtime)
def create_random_units(self,
store,
parent,
cc_store=None,
cc_parent=None):
"""
Create a random selection of units under the given parent w/ random names & attrs
:param store: which store (e.g., direct/draft) to create them in
:param parent: the parent to have point to them
:param cc_store: (optional) if given, make a small change and save also to this store but w/ same location
(only makes sense if store is 'direct' and this is 'draft' or vice versa)
"""
for _ in range(random.randrange(6)):
location = parent.location.replace(category=random.choice(
['html', 'video', 'problem', 'discussion']),
name=uuid.uuid4().hex)
metadata = {'display_name': str(uuid.uuid4()), 'graded': True}
data = {}
element = self._create_and_get_item(store, location, data,
metadata, parent.runtime)
parent.children.append(element.location.url())
if cc_store is not None:
# change display_name and remove graded to test the delta
element = self._create_and_get_item(
cc_store, location, data,
{'display_name': str(uuid.uuid4())}, parent.runtime)
cc_parent.children.append(element.location.url())
store.update_item(parent, '**replace_user**')
if cc_store is not None:
cc_store.update_item(cc_parent, '**replace_user**')
def compare_courses(self, presplit, published):
# descend via children to do comparison
old_root = presplit.get_item(self.course_location, depth=None)
new_root_locator = self.loc_mapper.translate_location(
self.course_location.course_id,
self.course_location,
published,
add_entry_if_missing=False)
new_root = self.split_mongo.get_course(new_root_locator)
self.compare_dags(presplit, old_root, new_root, published)
# grab the detached items to compare they should be in both published and draft
for category in ['conditional', 'about', 'course_info', 'static_tab']:
location = self.course_location.replace(name=None,
category=category)
for conditional in presplit.get_items(location):
locator = self.loc_mapper.translate_location(
self.course_location.course_id,
conditional.location,
published,
add_entry_if_missing=False)
self.compare_dags(presplit, conditional,
self.split_mongo.get_item(locator),
published)
def compare_dags(self, presplit, presplit_dag_root, split_dag_root,
published):
# check that locations match
self.assertEqual(
presplit_dag_root.location,
self.loc_mapper.translate_locator_to_location(
split_dag_root.location).replace(revision=None))
# compare all fields but children
for name in presplit_dag_root.fields.iterkeys():
if name != 'children':
self.assertEqual(
getattr(presplit_dag_root, name),
getattr(split_dag_root, name),
"{}/{}: {} != {}".format(split_dag_root.location, name,
getattr(presplit_dag_root, name),
getattr(split_dag_root, name)))
# test split get_item using old Location: old draft store didn't set revision for things above vertical
# but split does distinguish these; so, set revision if not published
if not published:
location = draft.as_draft(presplit_dag_root.location)
else:
location = presplit_dag_root.location
refetched = self.split_mongo.get_item(location)
self.assertEqual(
refetched.location, split_dag_root.location,
"Fetch from split via old Location {} not same as new {}".format(
refetched.location, split_dag_root.location))
# compare children
if presplit_dag_root.has_children:
self.assertEqual(
len(presplit_dag_root.get_children()),
len(split_dag_root.get_children()),
"{0.category} '{0.display_name}': children count {1} != {2}".
format(presplit_dag_root,
len(presplit_dag_root.get_children()),
split_dag_root.children))
for pre_child, split_child in zip(presplit_dag_root.get_children(),
split_dag_root.get_children()):
self.compare_dags(presplit, pre_child, split_child, published)
def test_migrator(self):
user = mock.Mock(id=1)
self.migrator.migrate_mongo_course(self.course_location, user)
# now compare the migrated to the original course
self.compare_courses(self.old_mongo, True)
self.compare_courses(self.draft_mongo, False)
class TestMigration(SplitWMongoCourseBoostrapper):
"""
Test the split migrator
"""
def setUp(self):
super(TestMigration, self).setUp()
# pylint: disable=W0142
self.loc_mapper = LocMapperStore(test_location_mapper.TrivialCache(),
**self.db_config)
self.split_mongo.loc_mapper = self.loc_mapper
self.migrator = SplitMigrator(self.split_mongo, self.old_mongo,
self.draft_mongo, self.loc_mapper)
def tearDown(self):
dbref = self.loc_mapper.db
dbref.drop_collection(self.loc_mapper.location_map)
super(TestMigration, self).tearDown()
def _create_course(self):
"""
A course testing all of the conversion mechanisms:
* some inheritable settings
* sequences w/ draft and live intermixed children to ensure all get to the draft but
only the live ones get to published. Some are only draft, some are both, some are only live.
* about, static_tab, and conditional documents
"""
super(TestMigration, self)._create_course(split=False)
# chapters
chapter1_name = uuid.uuid4().hex
self._create_item('chapter',
chapter1_name, {}, {'display_name': 'Chapter 1'},
'course',
'runid',
split=False)
chap2_loc = self.old_course_key.make_usage_key('chapter',
uuid.uuid4().hex)
self._create_item(chap2_loc.category,
chap2_loc.name, {}, {'display_name': 'Chapter 2'},
'course',
'runid',
split=False)
# vertical in live only
live_vert_name = uuid.uuid4().hex
self._create_item('vertical',
live_vert_name, {},
{'display_name': 'Live vertical'},
'chapter',
chapter1_name,
draft=False,
split=False)
self.create_random_units(
False,
self.old_course_key.make_usage_key('vertical', live_vert_name))
# vertical in both live and draft
both_vert_loc = self.old_course_key.make_usage_key(
'vertical',
uuid.uuid4().hex)
self._create_item(both_vert_loc.category,
both_vert_loc.name, {},
{'display_name': 'Both vertical'},
'chapter',
chapter1_name,
draft=False,
split=False)
self.create_random_units(False, both_vert_loc)
draft_both = self.draft_mongo.get_item(both_vert_loc)
draft_both.display_name = 'Both vertical renamed'
self.draft_mongo.update_item(draft_both)
self.create_random_units(True, both_vert_loc)
# vertical in draft only (x2)
draft_vert_loc = self.old_course_key.make_usage_key(
'vertical',
uuid.uuid4().hex)
self._create_item(draft_vert_loc.category,
draft_vert_loc.name, {},
{'display_name': 'Draft vertical'},
'chapter',
chapter1_name,
draft=True,
split=False)
self.create_random_units(True, draft_vert_loc)
draft_vert_loc = self.old_course_key.make_usage_key(
'vertical',
uuid.uuid4().hex)
self._create_item(draft_vert_loc.category,
draft_vert_loc.name, {},
{'display_name': 'Draft vertical2'},
'chapter',
chapter1_name,
draft=True,
split=False)
self.create_random_units(True, draft_vert_loc)
# and finally one in live only (so published has to skip 2 preceding sibs)
live_vert_loc = self.old_course_key.make_usage_key(
'vertical',
uuid.uuid4().hex)
self._create_item(live_vert_loc.category,
live_vert_loc.name, {},
{'display_name': 'Live vertical end'},
'chapter',
chapter1_name,
draft=False,
split=False)
self.create_random_units(True, draft_vert_loc)
# now the other chapter w/ the conditional
# create pointers to children (before the children exist)
indirect1_loc = self.old_course_key.make_usage_key(
'discussion',
uuid.uuid4().hex)
indirect2_loc = self.old_course_key.make_usage_key(
'html',
uuid.uuid4().hex)
conditional_loc = self.old_course_key.make_usage_key(
'conditional',
uuid.uuid4().hex)
self._create_item(conditional_loc.category,
conditional_loc.name, {
'show_tag_list': [indirect1_loc, indirect2_loc],
'sources_list': [
live_vert_loc,
],
}, {
'xml_attributes': {
'completed': True,
},
},
chap2_loc.category,
chap2_loc.name,
draft=False,
split=False)
# create the children
self._create_item(indirect1_loc.category,
indirect1_loc.name, {'data': ""},
{'display_name': 'conditional show 1'},
conditional_loc.category,
conditional_loc.name,
draft=False,
split=False)
self._create_item(indirect2_loc.category,
indirect2_loc.name, {'data': ""},
{'display_name': 'conditional show 2'},
conditional_loc.category,
conditional_loc.name,
draft=False,
split=False)
# add direct children
self.create_random_units(False, conditional_loc)
# and the ancillary docs (not children)
self._create_item('static_tab',
uuid.uuid4().hex, {'data': ""},
{'display_name': 'Tab uno'},
None,
None,
draft=False,
split=False)
self._create_item('about',
'overview', {'data': "<p>test</p>"}, {},
None,
None,
draft=False,
split=False)
self._create_item(
'course_info',
'updates',
{'data': "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>"}, {},
None,
None,
draft=False,
split=False)
def create_random_units(self, draft, parent_loc):
"""
Create a random selection of units under the given parent w/ random names & attrs
:param store: which store (e.g., direct/draft) to create them in
:param parent: the parent to have point to them
(only makes sense if store is 'direct' and this is 'draft' or vice versa)
"""
for _ in range(random.randrange(6)):
location = parent_loc.replace(category=random.choice(
['html', 'video', 'problem', 'discussion']),
name=uuid.uuid4().hex)
metadata = {'display_name': str(uuid.uuid4()), 'graded': True}
data = {}
self._create_item(location.category,
location.name,
data,
metadata,
parent_loc.category,
parent_loc.name,
draft=draft,
split=False)
def compare_courses(self, presplit, published):
# descend via children to do comparison
old_root = presplit.get_course(self.old_course_key)
new_root_locator = self.loc_mapper.translate_location_to_course_locator(
old_root.id, published)
new_root = self.split_mongo.get_course(new_root_locator)
self.compare_dags(presplit, old_root, new_root, published)
# grab the detached items to compare they should be in both published and draft
for category in ['conditional', 'about', 'course_info', 'static_tab']:
for conditional in presplit.get_items(self.old_course_key,
category=category):
locator = self.loc_mapper.translate_location(
conditional.location,
published,
add_entry_if_missing=False)
self.compare_dags(presplit, conditional,
self.split_mongo.get_item(locator),
published)
def compare_dags(self, presplit, presplit_dag_root, split_dag_root,
published):
# check that locations match
self.assertEqual(
presplit_dag_root.location,
self.loc_mapper.translate_locator_to_location(
split_dag_root.location).replace(revision=None))
# compare all fields but children
for name in presplit_dag_root.fields.iterkeys():
if name != 'children':
self.assertEqual(
getattr(presplit_dag_root, name),
getattr(split_dag_root, name),
"{}/{}: {} != {}".format(split_dag_root.location, name,
getattr(presplit_dag_root, name),
getattr(split_dag_root, name)))
# test split get_item using old Location: old draft store didn't set revision for things above vertical
# but split does distinguish these; so, set revision if not published
if not published:
location = draft.as_draft(presplit_dag_root.location)
else:
location = presplit_dag_root.location
refetched = self.split_mongo.get_item(location)
self.assertEqual(
refetched.location, split_dag_root.location,
"Fetch from split via old Location {} not same as new {}".format(
refetched.location, split_dag_root.location))
# compare children
if presplit_dag_root.has_children:
self.assertEqual(
# need get_children to filter out drafts
len(presplit_dag_root.get_children()),
len(split_dag_root.children),
"{0.category} '{0.display_name}': children {1} != {2}".format(
presplit_dag_root, presplit_dag_root.children,
split_dag_root.children))
for pre_child, split_child in zip(presplit_dag_root.get_children(),
split_dag_root.get_children()):
self.compare_dags(presplit, pre_child, split_child, published)
def test_migrator(self):
user = mock.Mock(id=1)
self.migrator.migrate_mongo_course(self.old_course_key, user)
# now compare the migrated to the original course
self.compare_courses(self.old_mongo, True)
self.compare_courses(self.draft_mongo, False)
def setUp(self):
super(TestMigration, self).setUp()
self.migrator = SplitMigrator(self.split_mongo, self.draft_mongo)
class TestMigration(SplitWMongoCourseBootstrapper):
"""
Test the split migrator
"""
def setUp(self):
super(TestMigration, self).setUp()
self.migrator = SplitMigrator(self.split_mongo, self.draft_mongo)
def _create_course(self):
"""
A course testing all of the conversion mechanisms:
* some inheritable settings
* sequences w/ draft and live intermixed children to ensure all get to the draft but
only the live ones get to published. Some are only draft, some are both, some are only live.
* about, static_tab, and conditional documents
"""
super(TestMigration, self)._create_course(split=False)
# chapters
chapter1_name = uuid.uuid4().hex
self._create_item('chapter', chapter1_name, {}, {'display_name': 'Chapter 1'}, 'course', 'runid', split=False)
chap2_loc = self.old_course_key.make_usage_key('chapter', uuid.uuid4().hex)
self._create_item(
chap2_loc.block_type, chap2_loc.block_id, {}, {'display_name': 'Chapter 2'}, 'course', 'runid', split=False
)
# vertical in live only
live_vert_name = uuid.uuid4().hex
self._create_item(
'vertical', live_vert_name, {}, {'display_name': 'Live vertical'}, 'chapter', chapter1_name,
draft=False, split=False
)
self.create_random_units(False, self.old_course_key.make_usage_key('vertical', live_vert_name))
# vertical in both live and draft
both_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
both_vert_loc.block_type, both_vert_loc.block_id, {}, {'display_name': 'Both vertical'}, 'chapter', chapter1_name,
draft=False, split=False
)
self.create_random_units(False, both_vert_loc)
draft_both = self.draft_mongo.get_item(both_vert_loc)
draft_both.display_name = 'Both vertical renamed'
self.draft_mongo.update_item(draft_both, self.user_id)
self.create_random_units(True, both_vert_loc)
# vertical in draft only (x2)
draft_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
draft_vert_loc.block_type, draft_vert_loc.block_id, {}, {'display_name': 'Draft vertical'}, 'chapter', chapter1_name,
draft=True, split=False
)
self.create_random_units(True, draft_vert_loc)
draft_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
draft_vert_loc.block_type, draft_vert_loc.block_id, {}, {'display_name': 'Draft vertical2'}, 'chapter', chapter1_name,
draft=True, split=False
)
self.create_random_units(True, draft_vert_loc)
# and finally one in live only (so published has to skip 2 preceding sibs)
live_vert_loc = self.old_course_key.make_usage_key('vertical', uuid.uuid4().hex)
self._create_item(
live_vert_loc.block_type, live_vert_loc.block_id, {}, {'display_name': 'Live vertical end'}, 'chapter', chapter1_name,
draft=False, split=False
)
self.create_random_units(False, live_vert_loc)
# now the other chapter w/ the conditional
# create pointers to children (before the children exist)
indirect1_loc = self.old_course_key.make_usage_key('discussion', uuid.uuid4().hex)
indirect2_loc = self.old_course_key.make_usage_key('html', uuid.uuid4().hex)
conditional_loc = self.old_course_key.make_usage_key('conditional', uuid.uuid4().hex)
self._create_item(
conditional_loc.block_type, conditional_loc.block_id,
{
'show_tag_list': [indirect1_loc, indirect2_loc],
'sources_list': [live_vert_loc, ],
},
{
'xml_attributes': {
'completed': True,
},
},
chap2_loc.block_type, chap2_loc.block_id,
draft=False, split=False
)
# create the children
self._create_item(
indirect1_loc.block_type, indirect1_loc.block_id, {'data': ""}, {'display_name': 'conditional show 1'},
conditional_loc.block_type, conditional_loc.block_id,
draft=False, split=False
)
self._create_item(
indirect2_loc.block_type, indirect2_loc.block_id, {'data': ""}, {'display_name': 'conditional show 2'},
conditional_loc.block_type, conditional_loc.block_id,
draft=False, split=False
)
# add direct children
self.create_random_units(False, conditional_loc)
# and the ancillary docs (not children)
self._create_item(
'static_tab', uuid.uuid4().hex, {'data': ""}, {'display_name': 'Tab uno'},
None, None, draft=False, split=False
)
self._create_item(
'about', 'overview', {'data': "<p>test</p>"}, {},
None, None, draft=False, split=False
)
self._create_item(
'course_info', 'updates', {'data': "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>"}, {},
None, None, draft=False, split=False
)
def create_random_units(self, draft, parent_loc):
"""
Create a random selection of units under the given parent w/ random names & attrs
:param store: which store (e.g., direct/draft) to create them in
:param parent: the parent to have point to them
(only makes sense if store is 'direct' and this is 'draft' or vice versa)
"""
for _ in range(random.randrange(6)):
location = parent_loc.replace(
category=random.choice(['html', 'video', 'problem', 'discussion']),
name=uuid.uuid4().hex
)
metadata = {'display_name': str(uuid.uuid4()), 'graded': True}
data = {}
self._create_item(
location.block_type, location.block_id, data, metadata, parent_loc.block_type, parent_loc.block_id,
draft=draft, split=False
)
def compare_courses(self, presplit, new_course_key, published):
# descend via children to do comparison
old_root = presplit.get_course(self.old_course_key)
new_root = self.split_mongo.get_course(new_course_key)
self.compare_dags(presplit, old_root, new_root, published)
# grab the detached items to compare they should be in both published and draft
for category in ['conditional', 'about', 'course_info', 'static_tab']:
for conditional in presplit.get_items(self.old_course_key, qualifiers={'category': category}):
locator = new_course_key.make_usage_key(category, conditional.location.block_id)
self.compare_dags(presplit, conditional, self.split_mongo.get_item(locator), published)
def compare_dags(self, presplit, presplit_dag_root, split_dag_root, published):
if split_dag_root.category != 'course':
self.assertEqual(presplit_dag_root.location.block_id, split_dag_root.location.block_id)
# compare all fields but references
for name, field in presplit_dag_root.fields.iteritems():
# fields generated from UNIQUE_IDs are unique to an XBlock's scope,
# so if such a field is unset on an XBlock, we don't expect it
# to persist across courses
field_generated_from_unique_id = not field.is_set_on(presplit_dag_root) and field.default == UNIQUE_ID
should_check_field = not (
field_generated_from_unique_id or isinstance(field, (Reference, ReferenceList, ReferenceValueDict))
)
if should_check_field:
self.assertEqual(
getattr(presplit_dag_root, name),
getattr(split_dag_root, name),
u"{}/{}: {} != {}".format(
split_dag_root.location, name, getattr(presplit_dag_root, name), getattr(split_dag_root, name)
)
)
# compare children
if presplit_dag_root.has_children:
self.assertEqual(
# need get_children to filter out drafts
len(presplit_dag_root.get_children()), len(split_dag_root.children),
u"{0.category} '{0.display_name}': children {1} != {2}".format(
presplit_dag_root, presplit_dag_root.children, split_dag_root.children
)
)
for pre_child, split_child in zip(presplit_dag_root.get_children(), split_dag_root.get_children()):
self.compare_dags(presplit, pre_child, split_child, published)
def test_migrator(self):
user = mock.Mock(id=1)
new_course_key = self.migrator.migrate_mongo_course(self.old_course_key, user.id, new_run='new_run')
# now compare the migrated to the original course
self.compare_courses(self.draft_mongo, new_course_key, True) # published
self.compare_courses(self.draft_mongo, new_course_key, False) # draft
class TestMigration(SplitWMongoCourseBoostrapper):
"""
Test the split migrator
"""
def setUp(self):
super(TestMigration, self).setUp()
self.migrator = SplitMigrator(self.split_mongo, self.draft_mongo)
def _create_course(self):
"""
A course testing all of the conversion mechanisms:
* some inheritable settings
* sequences w/ draft and live intermixed children to ensure all get to the draft but
only the live ones get to published. Some are only draft, some are both, some are only live.
* about, static_tab, and conditional documents
"""
super(TestMigration, self)._create_course(split=False)
# chapters
chapter1_name = uuid.uuid4().hex
self._create_item("chapter", chapter1_name, {}, {"display_name": "Chapter 1"}, "course", "runid", split=False)
chap2_loc = self.old_course_key.make_usage_key("chapter", uuid.uuid4().hex)
self._create_item(
chap2_loc.category, chap2_loc.name, {}, {"display_name": "Chapter 2"}, "course", "runid", split=False
)
# vertical in live only
live_vert_name = uuid.uuid4().hex
self._create_item(
"vertical",
live_vert_name,
{},
{"display_name": "Live vertical"},
"chapter",
chapter1_name,
draft=False,
split=False,
)
self.create_random_units(False, self.old_course_key.make_usage_key("vertical", live_vert_name))
# vertical in both live and draft
both_vert_loc = self.old_course_key.make_usage_key("vertical", uuid.uuid4().hex)
self._create_item(
both_vert_loc.category,
both_vert_loc.name,
{},
{"display_name": "Both vertical"},
"chapter",
chapter1_name,
draft=False,
split=False,
)
self.create_random_units(False, both_vert_loc)
draft_both = self.draft_mongo.get_item(both_vert_loc)
draft_both.display_name = "Both vertical renamed"
self.draft_mongo.update_item(draft_both, self.user_id)
self.create_random_units(True, both_vert_loc)
# vertical in draft only (x2)
draft_vert_loc = self.old_course_key.make_usage_key("vertical", uuid.uuid4().hex)
self._create_item(
draft_vert_loc.category,
draft_vert_loc.name,
{},
{"display_name": "Draft vertical"},
"chapter",
chapter1_name,
draft=True,
split=False,
)
self.create_random_units(True, draft_vert_loc)
draft_vert_loc = self.old_course_key.make_usage_key("vertical", uuid.uuid4().hex)
self._create_item(
draft_vert_loc.category,
draft_vert_loc.name,
{},
{"display_name": "Draft vertical2"},
"chapter",
chapter1_name,
draft=True,
split=False,
)
self.create_random_units(True, draft_vert_loc)
# and finally one in live only (so published has to skip 2 preceding sibs)
live_vert_loc = self.old_course_key.make_usage_key("vertical", uuid.uuid4().hex)
self._create_item(
live_vert_loc.category,
live_vert_loc.name,
{},
{"display_name": "Live vertical end"},
"chapter",
chapter1_name,
draft=False,
split=False,
)
self.create_random_units(False, live_vert_loc)
# now the other chapter w/ the conditional
# create pointers to children (before the children exist)
indirect1_loc = self.old_course_key.make_usage_key("discussion", uuid.uuid4().hex)
indirect2_loc = self.old_course_key.make_usage_key("html", uuid.uuid4().hex)
conditional_loc = self.old_course_key.make_usage_key("conditional", uuid.uuid4().hex)
self._create_item(
conditional_loc.category,
conditional_loc.name,
{"show_tag_list": [indirect1_loc, indirect2_loc], "sources_list": [live_vert_loc]},
{"xml_attributes": {"completed": True}},
chap2_loc.category,
chap2_loc.name,
draft=False,
split=False,
)
# create the children
self._create_item(
indirect1_loc.category,
indirect1_loc.name,
{"data": ""},
{"display_name": "conditional show 1"},
conditional_loc.category,
conditional_loc.name,
draft=False,
split=False,
)
self._create_item(
indirect2_loc.category,
indirect2_loc.name,
{"data": ""},
{"display_name": "conditional show 2"},
conditional_loc.category,
conditional_loc.name,
draft=False,
split=False,
)
# add direct children
self.create_random_units(False, conditional_loc)
# and the ancillary docs (not children)
self._create_item(
"static_tab",
uuid.uuid4().hex,
{"data": ""},
{"display_name": "Tab uno"},
None,
None,
draft=False,
split=False,
)
self._create_item("about", "overview", {"data": "<p>test</p>"}, {}, None, None, draft=False, split=False)
self._create_item(
"course_info",
"updates",
{"data": "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>"},
{},
None,
None,
draft=False,
split=False,
)
def create_random_units(self, draft, parent_loc):
"""
Create a random selection of units under the given parent w/ random names & attrs
:param store: which store (e.g., direct/draft) to create them in
:param parent: the parent to have point to them
(only makes sense if store is 'direct' and this is 'draft' or vice versa)
"""
for _ in range(random.randrange(6)):
location = parent_loc.replace(
category=random.choice(["html", "video", "problem", "discussion"]), name=uuid.uuid4().hex
)
metadata = {"display_name": str(uuid.uuid4()), "graded": True}
data = {}
self._create_item(
location.category,
location.name,
data,
metadata,
parent_loc.category,
parent_loc.name,
draft=draft,
split=False,
)
def compare_courses(self, presplit, new_course_key, published):
# descend via children to do comparison
old_root = presplit.get_course(self.old_course_key)
new_root = self.split_mongo.get_course(new_course_key)
self.compare_dags(presplit, old_root, new_root, published)
# grab the detached items to compare they should be in both published and draft
for category in ["conditional", "about", "course_info", "static_tab"]:
for conditional in presplit.get_items(self.old_course_key, category=category):
locator = new_course_key.make_usage_key(category, conditional.location.block_id)
self.compare_dags(presplit, conditional, self.split_mongo.get_item(locator), published)
def compare_dags(self, presplit, presplit_dag_root, split_dag_root, published):
if split_dag_root.category != "course":
self.assertEqual(presplit_dag_root.location.block_id, split_dag_root.location.block_id)
# compare all fields but references
for name, field in presplit_dag_root.fields.iteritems():
if not isinstance(field, (Reference, ReferenceList, ReferenceValueDict)):
self.assertEqual(
getattr(presplit_dag_root, name),
getattr(split_dag_root, name),
u"{}/{}: {} != {}".format(
split_dag_root.location, name, getattr(presplit_dag_root, name), getattr(split_dag_root, name)
),
)
# compare children
if presplit_dag_root.has_children:
self.assertEqual(
# need get_children to filter out drafts
len(presplit_dag_root.get_children()),
len(split_dag_root.children),
u"{0.category} '{0.display_name}': children {1} != {2}".format(
presplit_dag_root, presplit_dag_root.children, split_dag_root.children
),
)
for pre_child, split_child in zip(presplit_dag_root.get_children(), split_dag_root.get_children()):
self.compare_dags(presplit, pre_child, split_child, published)
def test_migrator(self):
user = mock.Mock(id=1)
new_course_key = self.migrator.migrate_mongo_course(self.old_course_key, user.id, new_run="new_run")
# now compare the migrated to the original course
self.compare_courses(self.draft_mongo, new_course_key, True) # published
self.compare_courses(self.draft_mongo, new_course_key, False) # draft
