def test_library_export(self):
"""
Verify that useable library data can be exported.
"""
youtube_id = "qS4NO9MNC6w"
library = LibraryFactory.create(modulestore=self.store)
video_block = ItemFactory.create(
category="video",
parent_location=library.location,
user_id=self.user.id,
publish_item=False,
youtube_id_1_0=youtube_id
)
name = library.url_name
lib_key = library.location.library_key
root_dir = path(mkdtemp_clean())
export_library_to_xml(self.store, contentstore(), lib_key, root_dir, name)
lib_xml = lxml.etree.XML(open(root_dir / name / LIBRARY_ROOT).read()) # pylint: disable=no-member
self.assertEqual(lib_xml.get('org'), lib_key.org)
self.assertEqual(lib_xml.get('library'), lib_key.library)
block = lib_xml.find('video')
self.assertIsNotNone(block)
self.assertEqual(block.get('url_name'), video_block.url_name)
video_xml = lxml.etree.XML( # pylint: disable=no-member
open(root_dir / name / 'video' / video_block.url_name + '.xml').read()
)
self.assertEqual(video_xml.tag, 'video')
self.assertEqual(video_xml.get('youtube_id_1_0'), youtube_id)
def setUp(self):
super(ImportTestCase, self).setUp()
self.url = course_url('course_import_export_handler', self.course)
self.content_dir = path(mkdtemp_clean())
# Create tar test files -----------------------------------------------
# OK course:
good_dir = tempfile.mkdtemp(dir=self.content_dir)
# test course being deeper down than top of tar file
embedded_dir = os.path.join(good_dir, "grandparent", "parent")
os.makedirs(os.path.join(embedded_dir, "course"))
with open(os.path.join(embedded_dir, "course.xml"), "w+") as f:
f.write('<course url_name="2013_Spring" org="EDx" course="0.00x"/>')
with open(os.path.join(embedded_dir, "course", "2013_Spring.xml"), "w+") as f:
f.write('<course></course>')
self.good_tar = os.path.join(self.content_dir, "good.tar.gz")
with tarfile.open(self.good_tar, "w:gz") as gtar:
gtar.add(good_dir)
# Bad course (no 'course.xml' file):
bad_dir = tempfile.mkdtemp(dir=self.content_dir)
path.joinpath(bad_dir, "bad.xml").touch()
self.bad_tar = os.path.join(self.content_dir, "bad.tar.gz")
with tarfile.open(self.bad_tar, "w:gz") as btar:
btar.add(bad_dir)
self.unsafe_common_dir = path(tempfile.mkdtemp(dir=self.content_dir))
def test_theme_outside_repo(self):
# Need to create a temporary theme, and defer decorating the function
# until it is done, which leads to this strange nested-function style
# of test.
# Make a temp directory as a theme.
themes_dir = path(mkdtemp_clean())
tmp_theme = "temp_theme"
template_dir = themes_dir / tmp_theme / "lms/templates"
template_dir.makedirs()
with open(template_dir / "footer.html", "w") as footer:
footer.write("<footer>TEMPORARY THEME</footer>")
dest_path = path(settings.COMPREHENSIVE_THEME_DIR) / tmp_theme
create_symlink(themes_dir / tmp_theme, dest_path)
@with_comprehensive_theme(tmp_theme)
def do_the_test(self):
"""A function to do the work so we can use the decorator."""
resp = self.client.get('/')
self.assertEqual(resp.status_code, 200)
self.assertContains(resp, "TEMPORARY THEME")
do_the_test(self)
# remove symlinks before running subsequent tests
delete_symlink(dest_path)
def __init__(self, base_loader):
# base_loader is an instance of a BaseLoader subclass
self.base_loader = base_loader
module_directory = getattr(settings, 'MAKO_MODULE_DIR', None)
if module_directory is None:
log.warning("For more caching of mako templates, set the MAKO_MODULE_DIR in settings!")
module_directory = mkdtemp_clean()
self.module_directory = module_directory
def __init__(self, base_loader):
# base_loader is an instance of a BaseLoader subclass
self.base_loader = base_loader
module_directory = getattr(settings, 'MAKO_MODULE_DIR', None)
if module_directory is None:
log.warning("For more caching of mako templates, set the MAKO_MODULE_DIR in settings!")
module_directory = mkdtemp_clean()
self.module_directory = module_directory
def test_theme_outside_repo(self):
# Need to create a temporary theme, and defer decorating the function
# until it is done, which leads to this strange nested-function style
# of test.
# Make a temp directory as a theme.
tmp_theme = path(mkdtemp_clean())
template_dir = tmp_theme / "lms/templates"
template_dir.makedirs()
with open(template_dir / "footer.html", "w") as footer:
footer.write("<footer>TEMPORARY THEME</footer>")
@with_comp_theme(tmp_theme)
def do_the_test(self):
"""A function to do the work so we can use the decorator."""
resp = self.client.get('/')
self.assertEqual(resp.status_code, 200)
self.assertContains(resp, "TEMPORARY THEME")
do_the_test(self)
def test_theme_outside_repo(self):
# Need to create a temporary theme, and defer decorating the function
# until it is done, which leads to this strange nested-function style
# of test.
# Make a temp directory as a theme.
tmp_theme = path(mkdtemp_clean())
template_dir = tmp_theme / "lms/templates"
template_dir.makedirs()
with open(template_dir / "footer.html", "w") as footer:
footer.write("<footer>TEMPORARY THEME</footer>")
@with_comprehensive_theme(tmp_theme)
def do_the_test(self):
"""A function to do the work so we can use the decorator."""
resp = self.client.get('/')
self.assertEqual(resp.status_code, 200)
self.assertContains(resp, "TEMPORARY THEME")
do_the_test(self)
# This modulestore will provide both a mixed mongo editable modulestore, and
# an XML store with common/test/data/2014 loaded, which is a course that is closed.
TEST_DATA_MIXED_CLOSED_MODULESTORE = mixed_store_config(
TEST_DATA_DIR, {'edX/detached_pages/2014': 'xml', }, include_xml=True, xml_source_dirs=['2014']
)
# This modulestore will provide both a mixed mongo editable modulestore, and
# an XML store with common/test/data/graded loaded, which is a course that is graded.
TEST_DATA_MIXED_GRADED_MODULESTORE = mixed_store_config(
TEST_DATA_DIR, {'edX/graded/2012_Fall': 'xml', }, include_xml=True, xml_source_dirs=['graded']
)
# All store requests now go through mixed
# Use this modulestore if you specifically want to test mongo and not a mocked modulestore.
# This modulestore definition below will not load any xml courses.
TEST_DATA_MONGO_MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
# All store requests now go through mixed
# Use this modulestore if you specifically want to test split-mongo and not a mocked modulestore.
# This modulestore definition below will not load any xml courses.
TEST_DATA_SPLIT_MODULESTORE = mixed_store_config(
mkdtemp_clean(),
{},
include_xml=False,
store_order=[StoreConstructors.split, StoreConstructors.draft]
)
class ModuleStoreTestCase(TestCase):
"""
Subclass for any test case that uses a ModuleStore.
TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
# This modulestore will provide a mixed mongo editable modulestore.
# If your test uses the 'toy' course, use the the ToyCourseFactory to construct it.
# If your test needs a closed course to test against, import the common/test/data/2014
# test course into this modulestore.
# If your test needs a graded course to test against, import the common/test/data/graded
# test course into this modulestore.
TEST_DATA_MIXED_MODULESTORE = mixed_store_config(
TEST_DATA_DIR, {}
)
# All store requests now go through mixed
# Use this modulestore if you specifically want to test mongo and not a mocked modulestore.
TEST_DATA_MONGO_MODULESTORE = mixed_store_config(mkdtemp_clean(), {})
# All store requests now go through mixed
# Use this modulestore if you specifically want to test split-mongo and not a mocked modulestore.
TEST_DATA_SPLIT_MODULESTORE = mixed_store_config(
mkdtemp_clean(),
{},
store_order=[StoreConstructors.split, StoreConstructors.draft]
)
class ModuleStoreIsolationMixin(CacheIsolationMixin):
"""
A mixin to be used by TestCases that want to isolate their use of the
Modulestore.
class ModuleStoreIsolationMixin(CacheIsolationMixin, SignalIsolationMixin):
"""
A mixin to be used by TestCases that want to isolate their use of the
Modulestore.
How to use::
class MyTestCase(ModuleStoreMixin, TestCase):
MODULESTORE = <settings for the modulestore to test>
ENABLED_SIGNALS = ['course_published']
def my_test(self):
self.start_modulestore_isolation()
self.addCleanup(self.end_modulestore_isolation)
modulestore.create_course(...)
...
"""
MODULESTORE = functools.partial(mixed_store_config, mkdtemp_clean(), {})
CONTENTSTORE = functools.partial(contentstore_config)
ENABLED_CACHES = [
'default', 'mongo_metadata_inheritance', 'loc_cache',
'course_index_cache'
]
# List of modulestore signals enabled for this test. Defaults to an empty
# list. The list of signals available is found on the SignalHandler class,
# in /common/lib/xmodule/xmodule/modulestore/xmodule_django.py
#
# You must use the signal itself, and not its name. So for example:
#
# class MyPublishTestCase(ModuleStoreTestCase):
# ENABLED_SIGNALS = ['course_published', 'pre_publish']
#
ENABLED_SIGNALS = []
__settings_overrides = []
__old_modulestores = []
__old_contentstores = []
@classmethod
def start_modulestore_isolation(cls):
"""
Isolate uses of the modulestore after this call. Once
:py:meth:`end_modulestore_isolation` is called, this modulestore will
be flushed (all content will be deleted).
"""
cls.disable_all_signals()
cls.enable_signals_by_name(*cls.ENABLED_SIGNALS)
cls.start_cache_isolation()
override = override_settings(
MODULESTORE=cls.MODULESTORE(),
CONTENTSTORE=cls.CONTENTSTORE(),
)
cls.__old_modulestores.append(copy.deepcopy(settings.MODULESTORE))
cls.__old_contentstores.append(copy.deepcopy(settings.CONTENTSTORE))
override.__enter__()
cls.__settings_overrides.append(override)
XMODULE_FACTORY_LOCK.enable()
clear_existing_modulestores()
cls.store = modulestore()
@classmethod
def end_modulestore_isolation(cls):
"""
Delete all content in the Modulestore, and reset the Modulestore
settings from before :py:meth:`start_modulestore_isolation` was
called.
"""
drop_mongo_collections() # pylint: disable=no-value-for-parameter
XMODULE_FACTORY_LOCK.disable()
cls.__settings_overrides.pop().__exit__(None, None, None)
assert settings.MODULESTORE == cls.__old_modulestores.pop()
assert settings.CONTENTSTORE == cls.__old_contentstores.pop()
cls.end_cache_isolation()
cls.enable_all_signals()
@staticmethod
def allow_transaction_exception():
"""
Context manager to wrap modulestore-using test code that may throw an exception.
(Use this if a modulestore test is failing with TransactionManagementError during cleanup.)
Details:
Some test cases that purposely throw an exception may normally cause the end_modulestore_isolation() cleanup
step to fail with
TransactionManagementError:
An error occurred in the current transaction. You can't execute queries until the end of the 'atomic' block.
This happens because the test is wrapped in an implicit transaction and when the exception occurs, django won't
allow any subsequent database queries in the same transaction - in particular, the queries needed to clean up
split modulestore's SplitModulestoreCourseIndex table after the test.
By wrapping the inner part of the test in this atomic() call, we create a savepoint so that if an exception is
thrown, Django merely rolls back to the savepoint and the overall transaction continues, including the eventual
cleanup step.
This method mostly exists to provide this docstring/explanation; the code itself is trivial.
"""
return transaction.atomic()
def test_library_import(self):
"""
Try importing a known good library archive, and verify that the
contents of the library have completely replaced the old contents.
"""
# Create some blocks to overwrite
library = LibraryFactory.create(modulestore=self.store)
lib_key = library.location.library_key
test_block = ItemFactory.create(
category="vertical",
parent_location=library.location,
user_id=self.user.id,
publish_item=False,
)
test_block2 = ItemFactory.create(
category="vertical",
parent_location=library.location,
user_id=self.user.id,
publish_item=False
)
# Create a library and blocks that should remain unmolested.
unchanged_lib = LibraryFactory.create()
unchanged_key = unchanged_lib.location.library_key
test_block3 = ItemFactory.create(
category="vertical",
parent_location=unchanged_lib.location,
user_id=self.user.id,
publish_item=False
)
test_block4 = ItemFactory.create(
category="vertical",
parent_location=unchanged_lib.location,
user_id=self.user.id,
publish_item=False
)
# Refresh library.
library = self.store.get_library(lib_key)
children = [self.store.get_item(child).url_name for child in library.children]
self.assertEqual(len(children), 2)
self.assertIn(test_block.url_name, children)
self.assertIn(test_block2.url_name, children)
unchanged_lib = self.store.get_library(unchanged_key)
children = [self.store.get_item(child).url_name for child in unchanged_lib.children]
self.assertEqual(len(children), 2)
self.assertIn(test_block3.url_name, children)
self.assertIn(test_block4.url_name, children)
extract_dir = path(mkdtemp_clean())
tar = tarfile.open(path(TEST_DATA_DIR) / 'imports' / 'library.HhJfPD.tar.gz')
safetar_extractall(tar, extract_dir)
library_items = import_library_from_xml(
self.store, self.user.id,
settings.GITHUB_REPO_ROOT, [extract_dir / 'library'],
load_error_modules=False,
static_content_store=contentstore(),
target_id=lib_key
)
self.assertEqual(lib_key, library_items[0].location.library_key)
library = self.store.get_library(lib_key)
children = [self.store.get_item(child).url_name for child in library.children]
self.assertEqual(len(children), 3)
self.assertNotIn(test_block.url_name, children)
self.assertNotIn(test_block2.url_name, children)
unchanged_lib = self.store.get_library(unchanged_key)
children = [self.store.get_item(child).url_name for child in unchanged_lib.children]
self.assertEqual(len(children), 2)
self.assertIn(test_block3.url_name, children)
self.assertIn(test_block4.url_name, children)
class ModuleStoreTestCase(TestCase):
"""
Subclass for any test case that uses a ModuleStore.
Ensures that the ModuleStore is cleaned before/after each test.
Usage:
1. Create a subclass of `ModuleStoreTestCase`
2. (optional) If you need a specific variety of modulestore, or particular ModuleStore
options, set the MODULESTORE class attribute of your test class to the
appropriate modulestore config.
For example:
class FooTest(ModuleStoreTestCase):
MODULESTORE = mixed_store_config(data_dir, mappings)
# ...
3. Use factories (e.g. `CourseFactory`, `ItemFactory`) to populate
the modulestore with test data.
NOTE:
* For Mongo-backed courses (created with `CourseFactory`),
the state of the course will be reset before/after each
test method executes.
* For XML-backed courses, the course state will NOT
reset between test methods (although it will reset
between test classes)
The reason is: XML courses are not editable, so to reset
a course you have to reload it from disk, which is slow.
If you do need to reset an XML course, use
`clear_existing_modulestores()` directly in
your `setUp()` method.
"""
MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
# Tell Django to clean out all databases, not just default
multi_db = True
def setUp(self, **kwargs):
"""
Creates a test User if `create_user` is True.
Returns the password for the test User.
Args:
create_user - specifies whether or not to create a test User. Default is True.
"""
settings_override = override_settings(MODULESTORE=self.MODULESTORE)
settings_override.__enter__()
self.addCleanup(settings_override.__exit__, None, None, None)
# Clear out any existing modulestores,
# which will cause them to be re-created
clear_existing_modulestores()
self.addCleanup(drop_mongo_collections)
self.addCleanup(clear_all_caches)
# Enable XModuleFactories for the space of this test (and its setUp).
self.addCleanup(XMODULE_FACTORY_LOCK.disable)
XMODULE_FACTORY_LOCK.enable()
# When testing CCX, we should make sure that
# OverrideFieldData.provider_classes is always reset to `None` so
# that they're recalculated for every test
OverrideFieldData.provider_classes = None
super(ModuleStoreTestCase, self).setUp()
SignalHandler.course_published.disconnect(
trigger_update_xblocks_cache_task)
self.store = modulestore()
uname = 'testuser'
email = '*****@*****.**'
password = '******'
if kwargs.pop('create_user', True):
# Create the user so we can log them in.
self.user = User.objects.create_user(uname, email, password)
# Note that we do not actually need to do anything
# for registration if we directly mark them active.
self.user.is_active = True
# Staff has access to view all courses
self.user.is_staff = True
self.user.save()
return password
def create_non_staff_user(self):
"""
Creates a non-staff test user.
Returns the non-staff test user and its password.
"""
uname = 'teststudent'
password = '******'
nonstaff_user = User.objects.create_user(uname, '*****@*****.**',
password)
# Note that we do not actually need to do anything
# for registration if we directly mark them active.
nonstaff_user.is_active = True
nonstaff_user.is_staff = False
nonstaff_user.save()
return nonstaff_user, password
def update_course(self, course, user_id):
"""
Updates the version of course in the modulestore
'course' is an instance of CourseDescriptor for which we want
to update metadata.
"""
with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred,
course.id):
self.store.update_item(course, user_id)
updated_course = self.store.get_course(course.id)
return updated_course
def get(self, request, course_key_string):
"""
The restful handler for exporting a full course or content library.
GET
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request
header can specify application/x-tgz via HTTP_ACCEPT, or a query
parameter can be used (?accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails,
a JSON string will be returned which describes the error
"""
redirect_url = request.QUERY_PARAMS.get('redirect', None)
courselike_key = CourseKey.from_string(course_key_string)
library = isinstance(courselike_key, LibraryLocator)
if library:
courselike_module = modulestore().get_library(courselike_key)
else:
courselike_module = modulestore().get_course(courselike_key)
name = courselike_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp_clean())
try:
if library:
export_library_to_xml(
modulestore(),
contentstore(),
courselike_key,
root_dir,
name
)
else:
export_course_to_xml(
modulestore(),
contentstore(),
courselike_module.id,
root_dir,
name
)
logging.debug(
u'tar file being generated at %s', export_file.name
)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(
u'There was an error exporting course %s',
courselike_key
)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(
failed_item.location
)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except Exception: # pylint: disable=broad-except
# if we have a nested exception, then we'll show the more
# generic error message
pass
return self._export_error_response(
{
"context_course": str(courselike_module.location),
"error": True,
"error_message": str(exc),
"failed_module":
str(failed_item.location) if failed_item else None,
"unit":
str(unit.location) if unit else None
},
redirect_url=redirect_url
)
except Exception as exc: # pylint: disable=broad-except
log.exception(
'There was an error exporting course %s',
courselike_key
)
return self._export_error_response(
{
"context_course": courselike_module.url_name,
"error": True,
"error_message": str(exc),
"unit": None
},
redirect_url=redirect_url
)
# The course is all set; return the tar.gz
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response['Content-Disposition'] = 'attachment; filename={}'.format(
os.path.basename(
export_file.name.encode('utf-8')
)
)
response['Content-Length'] = os.path.getsize(export_file.name)
return response
# Avoid having to run collectstatic before the unit test suite
# If we don't add these settings, then Django templates that can't
# find pipelined assets will raise a ValueError.
# http://stackoverflow.com/questions/12816941/unit-testing-with-django-pipeline
STATICFILES_STORAGE = 'pipeline.storage.NonPackagingPipelineStorage'
# Don't use compression during tests
PIPELINE_JS_COMPRESSOR = None
update_module_store_settings(
MODULESTORE,
module_store_options={
'fs_root': TEST_ROOT / "data",
},
xml_store_options={
'data_dir': mkdtemp_clean(
dir=TEST_ROOT), # never inadvertently load all the XML courses
},
doc_store_settings={
'host': MONGO_HOST,
'port': MONGO_PORT_NUM,
'db': 'test_xmodule',
'collection': 'test_modulestore{0}'.format(THIS_UUID),
},
)
CONTENTSTORE = {
'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
'DOC_STORE_CONFIG': {
'host': MONGO_HOST,
'db': 'xcontent',
'port': MONGO_PORT_NUM,
class ModuleStoreIsolationMixin(CacheIsolationMixin, SignalIsolationMixin):
"""
A mixin to be used by TestCases that want to isolate their use of the
Modulestore.
How to use::
class MyTestCase(ModuleStoreMixin, TestCase):
MODULESTORE = <settings for the modulestore to test>
ENABLED_SIGNALS = ['course_published']
def my_test(self):
self.start_modulestore_isolation()
self.addCleanup(self.end_modulestore_isolation)
modulestore.create_course(...)
...
"""
MODULESTORE = functools.partial(mixed_store_config, mkdtemp_clean(), {})
CONTENTSTORE = functools.partial(contentstore_config)
ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
# List of modulestore signals enabled for this test. Defaults to an empty
# list. The list of signals available is found on the SignalHandler class,
# in /common/lib/xmodule/xmodule/modulestore/xmodule_django.py
#
# You must use the signal itself, and not its name. So for example:
#
# class MyPublishTestCase(ModuleStoreTestCase):
# ENABLED_SIGNALS = ['course_published', 'pre_publish']
#
ENABLED_SIGNALS = []
__settings_overrides = []
__old_modulestores = []
__old_contentstores = []
@classmethod
def start_modulestore_isolation(cls):
"""
Isolate uses of the modulestore after this call. Once
:py:meth:`end_modulestore_isolation` is called, this modulestore will
be flushed (all content will be deleted).
"""
cls.disable_all_signals()
cls.enable_signals_by_name(*cls.ENABLED_SIGNALS)
cls.start_cache_isolation()
override = override_settings(
MODULESTORE=cls.MODULESTORE(),
CONTENTSTORE=cls.CONTENTSTORE(),
)
cls.__old_modulestores.append(copy.deepcopy(settings.MODULESTORE))
cls.__old_contentstores.append(copy.deepcopy(settings.CONTENTSTORE))
override.__enter__()
cls.__settings_overrides.append(override)
XMODULE_FACTORY_LOCK.enable()
clear_existing_modulestores()
cls.store = modulestore()
@classmethod
def end_modulestore_isolation(cls):
"""
Delete all content in the Modulestore, and reset the Modulestore
settings from before :py:meth:`start_modulestore_isolation` was
called.
"""
drop_mongo_collections() # pylint: disable=no-value-for-parameter
XMODULE_FACTORY_LOCK.disable()
cls.__settings_overrides.pop().__exit__(None, None, None)
assert settings.MODULESTORE == cls.__old_modulestores.pop()
assert settings.CONTENTSTORE == cls.__old_contentstores.pop()
cls.end_cache_isolation()
cls.enable_all_signals()
class ModuleStoreSettingsMigration(TestCase):
"""
Tests for the migration code for the module store settings
"""
OLD_CONFIG = {
"default": {
"ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
"OPTIONS": {
"data_dir": "directory",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
},
"DOC_STORE_CONFIG": {},
}
}
OLD_CONFIG_WITH_DIRECT_MONGO = {
"default": {
"ENGINE": "xmodule.modulestore.mongo.MongoModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "edxapp",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
"fs_root": mkdtemp_clean(),
"host": "localhost",
"password": "******",
"port": 27017,
"render_template": "edxmako.shortcuts.render_to_string",
"user": "******"
},
"DOC_STORE_CONFIG": {},
}
}
OLD_MIXED_CONFIG_WITH_DICT = {
"default": {
"ENGINE": "xmodule.modulestore.mixed.MixedModuleStore",
"OPTIONS": {
"mappings": {},
"stores": {
"an_old_mongo_store": {
"DOC_STORE_CONFIG": {},
"ENGINE": "xmodule.modulestore.mongo.MongoModuleStore",
"OPTIONS": {
"collection":
"modulestore",
"db":
"test",
"default_class":
"xmodule.hidden_module.HiddenDescriptor",
}
},
"default": {
"ENGINE": "the_default_store",
"OPTIONS": {
"option1": "value1",
"option2": "value2"
},
"DOC_STORE_CONFIG": {}
},
"xml": {
"ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
"OPTIONS": {
"data_dir":
"directory",
"default_class":
"xmodule.hidden_module.HiddenDescriptor"
},
"DOC_STORE_CONFIG": {}
}
}
}
}
}
ALREADY_UPDATED_MIXED_CONFIG = {
'default': {
'ENGINE': 'xmodule.modulestore.mixed.MixedModuleStore',
'OPTIONS': {
'mappings': {},
'stores': [
{
'NAME': 'split',
'ENGINE':
'xmodule.modulestore.split_mongo.split_draft.DraftVersioningModuleStore',
'DOC_STORE_CONFIG': {},
'OPTIONS': {
'default_class':
'xmodule.hidden_module.HiddenDescriptor',
'fs_root':
"fs_root",
'render_template':
'edxmako.shortcuts.render_to_string',
}
},
{
'NAME': 'draft',
'ENGINE':
'xmodule.modulestore.mongo.draft.DraftModuleStore',
'DOC_STORE_CONFIG': {},
'OPTIONS': {
'default_class':
'xmodule.hidden_module.HiddenDescriptor',
'fs_root':
"fs_root",
'render_template':
'edxmako.shortcuts.render_to_string',
}
},
]
}
}
}
def assertStoreValuesEqual(self, store_setting1, store_setting2):
"""
Tests whether the fields in the given store_settings are equal.
"""
store_fields = ["OPTIONS", "DOC_STORE_CONFIG"]
for field in store_fields:
self.assertEqual(store_setting1[field], store_setting2[field])
def assertMigrated(self, old_setting):
"""
Migrates the given setting and checks whether it correctly converted
to an ordered list of stores within Mixed.
"""
# pass a copy of the old setting since the migration modifies the given setting
new_mixed_setting = convert_module_store_setting_if_needed(
copy.deepcopy(old_setting))
# check whether the configuration is encapsulated within Mixed.
self.assertEqual(new_mixed_setting["default"]["ENGINE"],
"xmodule.modulestore.mixed.MixedModuleStore")
# check whether the stores are in an ordered list
new_stores = get_mixed_stores(new_mixed_setting)
self.assertIsInstance(new_stores, list)
return new_mixed_setting, new_stores[0]
def is_split_configured(self, mixed_setting):
"""
Tests whether the split module store is configured in the given setting.
"""
stores = get_mixed_stores(mixed_setting)
split_settings = [
store for store in stores
if store['ENGINE'].endswith('.DraftVersioningModuleStore')
]
if len(split_settings):
# there should only be one setting for split
self.assertEqual(len(split_settings), 1)
# verify name
self.assertEqual(split_settings[0]['NAME'], 'split')
# verify split config settings equal those of mongo
self.assertStoreValuesEqual(
split_settings[0],
next((store for store in stores
if 'DraftModuleStore' in store['ENGINE']), None))
return len(split_settings) > 0
def test_convert_into_mixed(self):
old_setting = self.OLD_CONFIG
new_mixed_setting, new_default_store_setting = self.assertMigrated(
old_setting)
self.assertStoreValuesEqual(new_default_store_setting,
old_setting["default"])
self.assertEqual(new_default_store_setting["ENGINE"],
old_setting["default"]["ENGINE"])
self.assertFalse(self.is_split_configured(new_mixed_setting))
def test_convert_from_old_mongo_to_draft_store(self):
old_setting = self.OLD_CONFIG_WITH_DIRECT_MONGO
new_mixed_setting, new_default_store_setting = self.assertMigrated(
old_setting)
self.assertStoreValuesEqual(new_default_store_setting,
old_setting["default"])
self.assertEqual(new_default_store_setting["ENGINE"],
"xmodule.modulestore.mongo.draft.DraftModuleStore")
self.assertTrue(self.is_split_configured(new_mixed_setting))
def test_convert_from_dict_to_list(self):
old_mixed_setting = self.OLD_MIXED_CONFIG_WITH_DICT
new_mixed_setting, new_default_store_setting = self.assertMigrated(
old_mixed_setting)
self.assertEqual(new_default_store_setting["ENGINE"],
"the_default_store")
self.assertTrue(self.is_split_configured(new_mixed_setting))
# exclude split when comparing old and new, since split was added as part of the migration
new_stores = [
store for store in get_mixed_stores(new_mixed_setting)
if store['NAME'] != 'split'
]
old_stores = get_mixed_stores(self.OLD_MIXED_CONFIG_WITH_DICT)
# compare each store configured in mixed
self.assertEqual(len(new_stores), len(old_stores))
for new_store in new_stores:
self.assertStoreValuesEqual(new_store,
old_stores[new_store['NAME']])
def test_no_conversion(self):
# make sure there is no migration done on an already updated config
old_mixed_setting = self.ALREADY_UPDATED_MIXED_CONFIG
new_mixed_setting, new_default_store_setting = self.assertMigrated(
old_mixed_setting)
self.assertTrue(self.is_split_configured(new_mixed_setting))
self.assertEqual(old_mixed_setting, new_mixed_setting)
@ddt.data('draft', 'split')
def test_update_settings(self, default_store):
mixed_setting = self.ALREADY_UPDATED_MIXED_CONFIG
update_module_store_settings(mixed_setting,
default_store=default_store)
self.assertEqual(
get_mixed_stores(mixed_setting)[0]['NAME'], default_store)
def test_update_settings_error(self):
mixed_setting = self.ALREADY_UPDATED_MIXED_CONFIG
with self.assertRaises(Exception):
update_module_store_settings(mixed_setting,
default_store='non-existent store')
TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
# This modulestore will provide a mixed mongo editable modulestore.
# If your test uses the 'toy' course, use the the ToyCourseFactory to construct it.
# If your test needs a closed course to test against, import the common/test/data/2014
# test course into this modulestore.
# If your test needs a graded course to test against, import the common/test/data/graded
# test course into this modulestore.
TEST_DATA_MIXED_MODULESTORE = functools.partial(mixed_store_config,
TEST_DATA_DIR, {})
# All store requests now go through mixed
# Use this modulestore if you specifically want to test mongo and not a mocked modulestore.
TEST_DATA_MONGO_MODULESTORE = functools.partial(mixed_store_config,
mkdtemp_clean(), {})
# All store requests now go through mixed
# Use this modulestore if you specifically want to test split-mongo and not a mocked modulestore.
TEST_DATA_SPLIT_MODULESTORE = functools.partial(
mixed_store_config,
mkdtemp_clean(), {},
store_order=[StoreConstructors.split, StoreConstructors.draft])
class SignalIsolationMixin(object):
"""
Simple utility mixin class to toggle ModuleStore signals on and off. This
class operates on `SwitchedSignal` objects on the modulestore's
`SignalHandler`.
"""
class ModuleStoreIsolationMixin(CacheIsolationMixin):
"""
A mixin to be used by TestCases that want to isolate their use of the
Modulestore.
How to use::
class MyTestCase(ModuleStoreMixin, TestCase):
MODULESTORE = <settings for the modulestore to test>
def my_test(self):
self.start_modulestore_isolation()
self.addCleanup(self.end_modulestore_isolation)
modulestore.create_course(...)
...
"""
MODULESTORE = functools.partial(mixed_store_config, mkdtemp_clean(), {})
CONTENTSTORE = functools.partial(contentstore_config)
ENABLED_CACHES = ['default', 'mongo_metadata_inheritance', 'loc_cache']
__settings_overrides = []
__old_modulestores = []
__old_contentstores = []
@classmethod
def start_modulestore_isolation(cls):
"""
Isolate uses of the modulestore after this call. Once
:py:meth:`end_modulestore_isolation` is called, this modulestore will
be flushed (all content will be deleted).
"""
cls.start_cache_isolation()
override = override_settings(
MODULESTORE=cls.MODULESTORE(),
CONTENTSTORE=cls.CONTENTSTORE(),
)
cls.__old_modulestores.append(copy.deepcopy(settings.MODULESTORE))
cls.__old_contentstores.append(copy.deepcopy(settings.CONTENTSTORE))
override.__enter__()
cls.__settings_overrides.append(override)
XMODULE_FACTORY_LOCK.enable()
clear_existing_modulestores()
cls.store = modulestore()
@classmethod
def end_modulestore_isolation(cls):
"""
Delete all content in the Modulestore, and reset the Modulestore
settings from before :py:meth:`start_modulestore_isolation` was
called.
"""
drop_mongo_collections() # pylint: disable=no-value-for-parameter
XMODULE_FACTORY_LOCK.disable()
cls.__settings_overrides.pop().__exit__(None, None, None)
assert settings.MODULESTORE == cls.__old_modulestores.pop()
assert settings.CONTENTSTORE == cls.__old_contentstores.pop()
cls.end_cache_isolation()
def get(self, request, course_key_string):
"""
The restful handler for exporting a full course or content library.
GET
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request
header can specify application/x-tgz via HTTP_ACCEPT, or a query
parameter can be used (?accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails,
a JSON string will be returned which describes the error
"""
redirect_url = request.QUERY_PARAMS.get('redirect', None)
courselike_key = CourseKey.from_string(course_key_string)
library = isinstance(courselike_key, LibraryLocator)
if library:
courselike_module = modulestore().get_library(courselike_key)
else:
courselike_module = modulestore().get_course(courselike_key)
name = courselike_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp_clean())
try:
if library:
export_library_to_xml(modulestore(), contentstore(),
courselike_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(),
courselike_module.id, root_dir, name)
logging.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(u'There was an error exporting course %s',
courselike_key)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(
failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except Exception: # pylint: disable=broad-except
# if we have a nested exception, then we'll show the more
# generic error message
pass
return self._export_error_response(
{
"context_course": str(courselike_module.location),
"error": True,
"error_message": str(exc),
"failed_module":
str(failed_item.location) if failed_item else None,
"unit": str(unit.location) if unit else None
},
redirect_url=redirect_url)
except Exception as exc: # pylint: disable=broad-except
log.exception('There was an error exporting course %s',
courselike_key)
return self._export_error_response(
{
"context_course": courselike_module.url_name,
"error": True,
"error_message": str(exc),
"unit": None
},
redirect_url=redirect_url)
# The course is all set; return the tar.gz
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response['Content-Disposition'] = 'attachment; filename={}'.format(
os.path.basename(export_file.name.encode('utf-8')))
response['Content-Length'] = os.path.getsize(export_file.name)
return response
if os.path.isdir(COMMON_TEST_DATA_ROOT / course_dir)
]
# Avoid having to run collectstatic before the unit test suite
# If we don't add these settings, then Django templates that can't
# find pipelined assets will raise a ValueError.
# http://stackoverflow.com/questions/12816941/unit-testing-with-django-pipeline
STATICFILES_STORAGE = "pipeline.storage.NonPackagingPipelineStorage"
# Don't use compression during tests
PIPELINE_JS_COMPRESSOR = None
update_module_store_settings(
MODULESTORE,
module_store_options={"fs_root": TEST_ROOT / "data"},
xml_store_options={"data_dir": mkdtemp_clean(dir=TEST_ROOT)}, # never inadvertently load all the XML courses
doc_store_settings={
"host": MONGO_HOST,
"port": MONGO_PORT_NUM,
"db": "test_xmodule",
"collection": "test_modulestore{0}".format(THIS_UUID),
},
)
CONTENTSTORE = {
"ENGINE": "xmodule.contentstore.mongo.MongoContentStore",
"DOC_STORE_CONFIG": {"host": MONGO_HOST, "db": "xcontent", "port": MONGO_PORT_NUM},
}
DATABASES = {
"default": {"ENGINE": "django.db.backends.sqlite3", "NAME": TEST_ROOT / "db" / "edx.db", "ATOMIC_REQUESTS": True}
class SharedModuleStoreTestCase(TestCase):
"""
Subclass for any test case that uses a ModuleStore that can be shared
between individual tests. This class ensures that the ModuleStore is cleaned
before/after the entire test case has run. Use this class if your tests
set up one or a small number of courses that individual tests do not modify
(or modify extermely rarely -- see @modifies_courseware).
If your tests modify contents in the ModuleStore, you should use
ModuleStoreTestCase instead.
How to use::
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from student.tests.factories import CourseEnrollmentFactory, UserFactory
class MyModuleStoreTestCase(SharedModuleStoreTestCase):
@classmethod
def setUpClass(cls):
super(MyModuleStoreTestCase, cls).setUpClass()
cls.course = CourseFactory.create()
def setUp(self):
super(MyModuleStoreTestCase, self).setUp()
self.user = UserFactory.create()
CourseEnrollmentFactory.create(
user=self.user, course_id=self.course.id
)
Important things to note:
1. You're creating the course in setUpClass(), *not* in setUp().
2. Any Django ORM operations should still happen in setUp(). Models created
in setUpClass() will *not* be cleaned up, and will leave side-effects
that can break other, completely unrelated test cases.
In Django 1.8, we will be able to use setUpTestData() to do class level init
for Django ORM models that will get cleaned up properly.
"""
MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
@classmethod
def setUpClass(cls):
super(SharedModuleStoreTestCase, cls).setUpClass()
cls._settings_override = override_settings(MODULESTORE=cls.MODULESTORE)
cls._settings_override.__enter__()
XMODULE_FACTORY_LOCK.enable()
clear_existing_modulestores()
cls.store = modulestore()
@classmethod
def tearDownClass(cls):
drop_mongo_collections() # pylint: disable=no-value-for-parameter
RequestCache().clear_request_cache()
XMODULE_FACTORY_LOCK.disable()
cls._settings_override.__exit__(None, None, None)
super(SharedModuleStoreTestCase, cls).tearDownClass()
def setUp(self):
# OverrideFieldData.provider_classes is always reset to `None` so
# that they're recalculated for every test
OverrideFieldData.provider_classes = None
super(SharedModuleStoreTestCase, self).setUp()
def reset(self):
"""
Manually run tearDownClass/setUpClass again.
This is so that if you have a mostly read-only course that you're just
modifying in one test, you can write `self.reset()` at the
end of that test and reset the state of the world for other tests in
the class.
"""
self.tearDownClass()
self.setUpClass()
@staticmethod
def modifies_courseware(f):
"""
Decorator to place around tests that modify course content.
For performance reasons, SharedModuleStoreTestCase intentionally does
not reset the modulestore between individual tests. However, sometimes
you might have a test case where the vast majority of tests treat a
course as read-only, but one or two want to modify it. In that case, you
can do this:
class MyTestCase(SharedModuleStoreTestCase):
# ...
@SharedModuleStoreTestCase.modifies_courseware
def test_that_edits_modulestore(self):
do_something()
This is equivalent to calling `self.reset()` at the end of
your test.
If you find yourself using this functionality a lot, it might indicate
that you should be using ModuleStoreTestCase instead, or that you should
break up your tests into different TestCases.
"""
@functools.wraps(f)
def wrapper(*args, **kwargs):
"""Call the object method, and reset the test case afterwards."""
return_val = f(*args, **kwargs)
obj = args[0]
obj.reset()
return return_val
return wrapper
class ModuleStoreTestCase(TestCase):
"""
Subclass for any test case that uses a ModuleStore.
Ensures that the ModuleStore is cleaned before/after each test.
Usage:
1. Create a subclass of `ModuleStoreTestCase`
2. (optional) If you need a specific variety of modulestore, or particular ModuleStore
options, set the MODULESTORE class attribute of your test class to the
appropriate modulestore config.
For example:
class FooTest(ModuleStoreTestCase):
MODULESTORE = mixed_store_config(data_dir, mappings)
# ...
3. Use factories (e.g. `CourseFactory`, `ItemFactory`) to populate
the modulestore with test data.
NOTE:
* For Mongo-backed courses (created with `CourseFactory`),
the state of the course will be reset before/after each
test method executes.
* For XML-backed courses, the course state will NOT
reset between test methods (although it will reset
between test classes)
The reason is: XML courses are not editable, so to reset
a course you have to reload it from disk, which is slow.
If you do need to reset an XML course, use
`clear_existing_modulestores()` directly in
your `setUp()` method.
"""
MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
def setUp(self, **kwargs):
"""
Creates a test User if `create_user` is True.
Returns the password for the test User.
Args:
create_user - specifies whether or not to create a test User. Default is True.
"""
settings_override = override_settings(MODULESTORE=self.MODULESTORE)
settings_override.__enter__()
self.addCleanup(settings_override.__exit__, None, None, None)
# Clear out any existing modulestores,
# which will cause them to be re-created
clear_existing_modulestores()
self.addCleanup(drop_mongo_collections)
self.addCleanup(RequestCache().clear_request_cache)
# Enable XModuleFactories for the space of this test (and its setUp).
self.addCleanup(XMODULE_FACTORY_LOCK.disable)
XMODULE_FACTORY_LOCK.enable()
# When testing CCX, we should make sure that
# OverrideFieldData.provider_classes is always reset to `None` so
# that they're recalculated for every test
OverrideFieldData.provider_classes = None
super(ModuleStoreTestCase, self).setUp()
self.store = modulestore()
uname = 'testuser'
email = '*****@*****.**'
password = '******'
if kwargs.pop('create_user', True):
# Create the user so we can log them in.
self.user = User.objects.create_user(uname, email, password)
# Note that we do not actually need to do anything
# for registration if we directly mark them active.
self.user.is_active = True
# Staff has access to view all courses
self.user.is_staff = True
self.user.save()
return password
def create_non_staff_user(self):
"""
Creates a non-staff test user.
Returns the non-staff test user and its password.
"""
uname = 'teststudent'
password = '******'
nonstaff_user = User.objects.create_user(uname, '*****@*****.**',
password)
# Note that we do not actually need to do anything
# for registration if we directly mark them active.
nonstaff_user.is_active = True
nonstaff_user.is_staff = False
nonstaff_user.save()
return nonstaff_user, password
def update_course(self, course, user_id):
"""
Updates the version of course in the modulestore
'course' is an instance of CourseDescriptor for which we want
to update metadata.
"""
with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred,
course.id):
self.store.update_item(course, user_id)
updated_course = self.store.get_course(course.id)
return updated_course
def create_sample_course(self,
org,
course,
run,
block_info_tree=None,
course_fields=None):
"""
create a course in the default modulestore from the collection of BlockInfo
records defining the course tree
Returns:
course_loc: the CourseKey for the created course
"""
if block_info_tree is None:
block_info_tree = default_block_info_tree
with self.store.branch_setting(ModuleStoreEnum.Branch.draft_preferred,
None):
course = self.store.create_course(org,
course,
run,
self.user.id,
fields=course_fields)
self.course_loc = course.location # pylint: disable=attribute-defined-outside-init
def create_sub_tree(parent_loc, block_info):
"""Recursively creates a sub_tree on this parent_loc with this block."""
block = self.store.create_child(
self.user.id,
# TODO remove version_agnostic() when we impl the single transaction
parent_loc.version_agnostic(),
block_info.category,
block_id=block_info.block_id,
fields=block_info.fields,
)
for tree in block_info.sub_tree:
create_sub_tree(block.location, tree)
setattr(self, block_info.block_id,
block.location.version_agnostic())
for tree in block_info_tree:
create_sub_tree(self.course_loc, tree)
# remove version_agnostic when bulk write works
self.store.publish(self.course_loc.version_agnostic(),
self.user.id)
return self.course_loc.course_key.version_agnostic()
def create_toy_course(self, org='edX', course='toy', run='2012_Fall'):
"""
Create an equivalent to the toy xml course
"""
with self.store.bulk_operations(self.store.make_course_key(
org, course, run),
emit_signals=False):
self.toy_loc = self.create_sample_course( # pylint: disable=attribute-defined-outside-init
org, course, run, TOY_BLOCK_INFO_TREE,
{
"textbooks": [["Textbook", "https://s3.amazonaws.com/edx-textbooks/guttag_computation_v3/"]],
"wiki_slug": "toy",
"display_name": "Toy Course",
"graded": True,
"discussion_topics": {"General": {"id": "i4x-edX-toy-course-2012_Fall"}},
"graceperiod": datetime.timedelta(days=2, seconds=21599),
"start": datetime.datetime(2015, 07, 17, 12, tzinfo=pytz.utc),
"xml_attributes": {"filename": ["course/2012_Fall.xml", "course/2012_Fall.xml"]},
"pdf_textbooks": [
{
"tab_title": "Sample Multi Chapter Textbook",
"id": "MyTextbook",
"chapters": [
{"url": "/static/Chapter1.pdf", "title": "Chapter 1"},
{"url": "/static/Chapter2.pdf", "title": "Chapter 2"}
]
}
],
"course_image": "just_a_test.jpg",
}
)
with self.store.branch_setting(
ModuleStoreEnum.Branch.draft_preferred, self.toy_loc):
self.store.create_item(self.user.id,
self.toy_loc,
"about",
block_id="short_description",
fields={"data": "A course about toys."})
self.store.create_item(self.user.id,
self.toy_loc,
"about",
block_id="effort",
fields={"data": "6 hours"})
self.store.create_item(self.user.id,
self.toy_loc,
"about",
block_id="end_date",
fields={"data": "TBD"})
self.store.create_item(
self.user.id,
self.toy_loc,
"course_info",
"handouts",
fields={
"data":
"<a href='/static/handouts/sample_handout.txt'>Sample</a>"
})
self.store.create_item(
self.user.id,
self.toy_loc,
"static_tab",
"resources",
fields={"display_name": "Resources"},
)
self.store.create_item(
self.user.id,
self.toy_loc,
"static_tab",
"syllabus",
fields={"display_name": "Syllabus"},
)
return self.toy_loc
# This modulestore will provide a mixed mongo editable modulestore.
# If your test uses the 'toy' course, use the the ToyCourseFactory to construct it.
# If your test needs a closed course to test against, import the common/test/data/2014
# test course into this modulestore.
# If your test needs a graded course to test against, import the common/test/data/graded
# test course into this modulestore.
TEST_DATA_MIXED_MODULESTORE = functools.partial(
mixed_store_config,
TEST_DATA_DIR,
{}
)
# All store requests now go through mixed
# Use this modulestore if you specifically want to test mongo and not a mocked modulestore.
TEST_DATA_MONGO_MODULESTORE = functools.partial(mixed_store_config, mkdtemp_clean(), {})
# All store requests now go through mixed
# Use this modulestore if you specifically want to test split-mongo and not a mocked modulestore.
TEST_DATA_SPLIT_MODULESTORE = functools.partial(
mixed_store_config,
mkdtemp_clean(),
{},
store_order=[StoreConstructors.split, StoreConstructors.draft]
)
class SignalIsolationMixin(object):
"""
Simple utility mixin class to toggle ModuleStore signals on and off. This
class operates on `SwitchedSignal` objects on the modulestore's
include_xml=True,
xml_source_dirs=['2014'])
# This modulestore will provide both a mixed mongo editable modulestore, and
# an XML store with common/test/data/graded loaded, which is a course that is graded.
TEST_DATA_MIXED_GRADED_MODULESTORE = mixed_store_config(
TEST_DATA_DIR, {
'edX/graded/2012_Fall': 'xml',
},
include_xml=True,
xml_source_dirs=['graded'])
# All store requests now go through mixed
# Use this modulestore if you specifically want to test mongo and not a mocked modulestore.
# This modulestore definition below will not load any xml courses.
TEST_DATA_MONGO_MODULESTORE = mixed_store_config(mkdtemp_clean(), {},
include_xml=False)
# All store requests now go through mixed
# Use this modulestore if you specifically want to test split-mongo and not a mocked modulestore.
# This modulestore definition below will not load any xml courses.
TEST_DATA_SPLIT_MODULESTORE = mixed_store_config(
mkdtemp_clean(), {},
include_xml=False,
store_order=[StoreConstructors.split, StoreConstructors.draft])
def clear_all_caches():
"""Clear all caches so that cache info doesn't leak across test cases."""
# This will no longer be necessary when Django adds (in Django 1.10?):
# https://code.djangoproject.com/ticket/11505
# Avoid having to run collectstatic before the unit test suite
# If we don't add these settings, then Django templates that can't
# find pipelined assets will raise a ValueError.
# http://stackoverflow.com/questions/12816941/unit-testing-with-django-pipeline
STATICFILES_STORAGE = 'pipeline.storage.NonPackagingPipelineStorage'
# Don't use compression during tests
PIPELINE_JS_COMPRESSOR = None
update_module_store_settings(
MODULESTORE,
module_store_options={
'fs_root': TEST_ROOT / "data",
},
xml_store_options={
'data_dir': mkdtemp_clean(dir=TEST_ROOT), # never inadvertently load all the XML courses
},
doc_store_settings={
'host': MONGO_HOST,
'port': MONGO_PORT_NUM,
'db': 'test_xmodule_{}'.format(THIS_UUID),
'collection': 'test_modulestore',
},
)
CONTENTSTORE = {
'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
'DOC_STORE_CONFIG': {
'host': MONGO_HOST,
'db': 'test_xcontent_{}'.format(THIS_UUID),
'port': MONGO_PORT_NUM,
class SharedModuleStoreTestCase(TestCase):
"""
Subclass for any test case that uses a ModuleStore that can be shared
between individual tests. This class ensures that the ModuleStore is cleaned
before/after the entire test case has run. Use this class if your tests
set up one or a small number of courses that individual tests do not modify
(or modify extermely rarely -- see @modifies_courseware).
If your tests modify contents in the ModuleStore, you should use
ModuleStoreTestCase instead.
How to use::
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from student.tests.factories import CourseEnrollmentFactory, UserFactory
class MyModuleStoreTestCase(SharedModuleStoreTestCase):
@classmethod
def setUpClass(cls):
super(MyModuleStoreTestCase, cls).setUpClass()
cls.course = CourseFactory.create()
def setUp(self):
super(MyModuleStoreTestCase, self).setUp()
self.user = UserFactory.create()
CourseEnrollmentFactory.create(
user=self.user, course_id=self.course.id
)
Important things to note:
1. You're creating the course in setUpClass(), *not* in setUp().
2. Any Django ORM operations should still happen in setUp(). Models created
in setUpClass() will *not* be cleaned up, and will leave side-effects
that can break other, completely unrelated test cases.
In Django 1.8, we will be able to use setUpTestData() to do class level init
for Django ORM models that will get cleaned up properly.
"""
MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
# Tell Django to clean out all databases, not just default
multi_db = True
@classmethod
def _setUpModuleStore(cls): # pylint: disable=invalid-name
"""
Set up the modulestore for an entire test class.
"""
cls._settings_override = override_settings(MODULESTORE=cls.MODULESTORE)
cls._settings_override.__enter__()
XMODULE_FACTORY_LOCK.enable()
clear_existing_modulestores()
cls.store = modulestore()
@classmethod
@contextmanager
def setUpClassAndTestData(cls): # pylint: disable=invalid-name
"""
For use when the test class has a setUpTestData() method that uses variables
that are setup during setUpClass() of the same test class.
Use it like so:
@classmethod
def setUpClass(cls):
with super(MyTestClass, cls).setUpClassAndTestData():
<all the cls.setUpClass() setup code that performs modulestore setup...>
@classmethod
def setUpTestData(cls):
<all the setup code that creates Django models per test class...>
<these models can use variables (courses) setup in setUpClass() above>
"""
cls._setUpModuleStore()
# Now yield to allow the test class to run its setUpClass() setup code.
yield
# Now call the base class, which calls back into the test class's setUpTestData().
super(SharedModuleStoreTestCase, cls).setUpClass()
@classmethod
def setUpClass(cls):
"""
For use when the test class has no setUpTestData() method -or-
when that method does not use variable set up in setUpClass().
"""
super(SharedModuleStoreTestCase, cls).setUpClass()
cls._setUpModuleStore()
@classmethod
def tearDownClass(cls):
drop_mongo_collections() # pylint: disable=no-value-for-parameter
clear_all_caches()
XMODULE_FACTORY_LOCK.disable()
cls._settings_override.__exit__(None, None, None)
super(SharedModuleStoreTestCase, cls).tearDownClass()
def setUp(self):
# OverrideFieldData.provider_classes is always reset to `None` so
# that they're recalculated for every test
OverrideFieldData.provider_classes = None
super(SharedModuleStoreTestCase, self).setUp()
def tearDown(self):
"""Reset caches."""
clear_all_caches()
super(SharedModuleStoreTestCase, self).tearDown()
def reset(self):
"""
Manually run tearDownClass/setUpClass again.
This is so that if you have a mostly read-only course that you're just
modifying in one test, you can write `self.reset()` at the
end of that test and reset the state of the world for other tests in
the class.
"""
self.tearDownClass()
self.setUpClass()
@staticmethod
def modifies_courseware(f):
"""
Decorator to place around tests that modify course content.
For performance reasons, SharedModuleStoreTestCase intentionally does
not reset the modulestore between individual tests. However, sometimes
you might have a test case where the vast majority of tests treat a
course as read-only, but one or two want to modify it. In that case, you
can do this:
class MyTestCase(SharedModuleStoreTestCase):
# ...
@SharedModuleStoreTestCase.modifies_courseware
def test_that_edits_modulestore(self):
do_something()
This is equivalent to calling `self.reset()` at the end of
your test.
If you find yourself using this functionality a lot, it might indicate
that you should be using ModuleStoreTestCase instead, or that you should
break up your tests into different TestCases.
"""
@functools.wraps(f)
def wrapper(*args, **kwargs):
"""Call the object method, and reset the test case afterwards."""
try:
# Attempt execution of the test.
return_val = f(*args, **kwargs)
except:
# If the test raises an exception, re-raise it.
raise
else:
# Otherwise, return the test's return value.
return return_val
finally:
# In either case, call SharedModuleStoreTestCase.reset() "on the way out."
# For more, see here: https://docs.python.org/2/tutorial/errors.html#defining-clean-up-actions.
obj = args[0]
obj.reset()
return wrapper
# This modulestore will provide a mixed mongo editable modulestore.
# If your test uses the 'toy' course, use the the ToyCourseFactory to construct it.
# If your test needs a closed course to test against, import the common/test/data/2014
# test course into this modulestore.
# If your test needs a graded course to test against, import the common/test/data/graded
# test course into this modulestore.
TEST_DATA_MIXED_MODULESTORE = functools.partial(
mixed_store_config,
TEST_DATA_DIR,
{}
)
# All store requests now go through mixed
# Use this modulestore if you specifically want to test mongo and not a mocked modulestore.
TEST_DATA_MONGO_MODULESTORE = functools.partial(mixed_store_config, mkdtemp_clean(), {})
# All store requests now go through mixed
# Use this modulestore if you specifically want to test split-mongo and not a mocked modulestore.
TEST_DATA_SPLIT_MODULESTORE = functools.partial(
mixed_store_config,
mkdtemp_clean(),
{},
store_order=[StoreConstructors.split, StoreConstructors.draft]
)
class SignalIsolationMixin(object):
"""
Simple utility mixin class to toggle ModuleStore signals on and off. This
class operates on `SwitchedSignal` objects on the modulestore's
class SharedModuleStoreTestCase(TestCase):
"""
Subclass for any test case that uses a ModuleStore that can be shared
between individual tests. This class ensures that the ModuleStore is cleaned
before/after the entire test case has run. Use this class if your tests
set up one or a small number of courses that individual tests do not modify.
If your tests modify contents in the ModuleStore, you should use
ModuleStoreTestCase instead.
How to use::
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from student.tests.factories import CourseEnrollmentFactory, UserFactory
class MyModuleStoreTestCase(SharedModuleStoreTestCase):
@classmethod
def setUpClass(cls):
super(MyModuleStoreTestCase, cls).setUpClass()
cls.course = CourseFactory.create()
def setUp(self):
super(MyModuleStoreTestCase, self).setUp()
self.user = UserFactory.create()
CourseEnrollmentFactory.create(
user=self.user, course_id=self.course.id
)
Important things to note:
1. You're creating the course in setUpClass(), *not* in setUp().
2. Any Django ORM operations should still happen in setUp(). Models created
in setUpClass() will *not* be cleaned up, and will leave side-effects
that can break other, completely unrelated test cases.
In Django 1.8, we will be able to use setUpTestData() to do class level init
for Django ORM models that will get cleaned up properly.
"""
MODULESTORE = mixed_store_config(mkdtemp_clean(), {}, include_xml=False)
@classmethod
def setUpClass(cls):
super(SharedModuleStoreTestCase, cls).setUpClass()
cls._settings_override = override_settings(MODULESTORE=cls.MODULESTORE)
cls._settings_override.__enter__()
XMODULE_FACTORY_LOCK.enable()
clear_existing_modulestores()
cls.store = modulestore()
@classmethod
def tearDownClass(cls):
drop_mongo_collections() # pylint: disable=no-value-for-parameter
RequestCache().clear_request_cache()
XMODULE_FACTORY_LOCK.disable()
cls._settings_override.__exit__(None, None, None)
super(SharedModuleStoreTestCase, cls).tearDownClass()
def setUp(self):
# OverrideFieldData.provider_classes is always reset to `None` so
# that they're recalculated for every test
OverrideFieldData.provider_classes = None
super(SharedModuleStoreTestCase, self).setUp()
