def make_block():
""" Instantiate a DragAndDropBlock XBlock inside a WorkbenchRuntime """
block_type = 'drag_and_drop_v2'
key_store = DictKeyValueStore()
field_data = KvsFieldData(key_store)
runtime = WorkbenchRuntime()
def_id = runtime.id_generator.create_definition(block_type)
usage_id = runtime.id_generator.create_usage(def_id)
scope_ids = ScopeIds('user', block_type, def_id, usage_id)
return drag_and_drop_v2.DragAndDropBlock(runtime,
field_data,
scope_ids=scope_ids)
def __init__(
self, data_dir, default_class=None, course_dirs=None, course_ids=None,
load_error_modules=True, i18n_service=None, **kwargs
):
"""
Initialize an XMLModuleStore from data_dir
Args:
data_dir (str): path to data directory containing the course directories
default_class (str): dot-separated string defining the default descriptor
class to use if none is specified in entry_points
course_dirs or course_ids (list of str): If specified, the list of course_dirs or course_ids to load. Otherwise,
load all courses. Note, providing both
"""
super(XMLModuleStore, self).__init__(**kwargs)
self.data_dir = path(data_dir)
self.modules = defaultdict(dict) # course_id -> dict(location -> XBlock)
self.courses = {} # course_dir -> XBlock for the course
self.errored_courses = {} # course_dir -> errorlog, for dirs that failed to load
if course_ids is not None:
course_ids = [SlashSeparatedCourseKey.from_deprecated_string(course_id) for course_id in course_ids]
self.load_error_modules = load_error_modules
if default_class is None:
self.default_class = None
else:
module_path, _, class_name = default_class.rpartition('.')
class_ = getattr(import_module(module_path), class_name)
self.default_class = class_
self.parent_trackers = defaultdict(ParentTracker)
self.reference_type = Location
# All field data will be stored in an inheriting field data.
self.field_data = inheriting_field_data(kvs=DictKeyValueStore())
self.i18n_service = i18n_service
# If we are specifically asked for missing courses, that should
# be an error. If we are asked for "all" courses, find the ones
# that have a course.xml. We sort the dirs in alpha order so we always
# read things in the same order (OS differences in load order have
# bitten us in the past.)
if course_dirs is None:
course_dirs = sorted([d for d in os.listdir(self.data_dir) if
os.path.exists(self.data_dir / d / "course.xml")])
for course_dir in course_dirs:
self.try_load_course(course_dir, course_ids)
def setUp(self):
block_type = 'hastexo'
key_store = DictKeyValueStore()
field_data = KvsFieldData(key_store)
runtime = WorkbenchRuntime()
def_id = runtime.id_generator.create_definition(block_type)
usage_id = runtime.id_generator.create_usage(def_id)
scope_ids = ScopeIds('user', block_type, def_id, usage_id)
self.block = HastexoXBlock(runtime,
field_data,
scope_ids=scope_ids)
def _prepare_asides(self, scope_ids):
"""
Return list with connected aside xblocks
"""
key_store = DictKeyValueStore()
field_data = KvsFieldData(key_store)
aside = AsideTest(
scope_ids=scope_ids,
runtime=TestRuntime(services={'field-data': field_data}))
aside.fields[self.ASIDE_DATA_FIELD.field_name].write_to(
aside, self.ASIDE_DATA_FIELD.initial)
return [aside]
def test_view_counter_state():
key_store = DictKeyValueStore()
db_model = KvsFieldData(key_store)
tester = ViewCounter(Mock(), db_model, Mock())
assert_equals(tester.views, 0)
# View the XBlock five times
for i in xrange(5):
generated_html = tester.student_view({})
# Make sure the html fragment we're expecting appears in the body_html
assert_in('<span class="views">{0}</span>'.format(i + 1),
generated_html.body_html())
assert_equals(tester.views, i + 1)
def test_view_counter_state():
key_store = DictKeyValueStore()
field_data = KvsFieldData(key_store)
runtime = Runtime(services={'field-data': field_data})
tester = ViewCounter(runtime, scope_ids=Mock())
assert tester.views == 0
# View the XBlock five times
for i in range(5):
generated_html = tester.student_view({})
# Make sure the html fragment we're expecting appears in the body_html
assert f'<span class="views">{i + 1}</span>' in generated_html.body_html()
assert tester.views == (i + 1)
def test_aside_contains_tags(self):
"""
Checks that available_tags list is not empty
"""
sids = ScopeIds(user_id="bob",
block_type="bobs-type",
def_id="definition-id",
usage_id="usage-id")
key_store = DictKeyValueStore()
field_data = KvsFieldData(key_store)
runtime = TestRuntime(services={'field-data': field_data}) # pylint: disable=abstract-class-instantiated
xblock_aside = StructuredTagsAside(scope_ids=sids, runtime=runtime)
available_tags = xblock_aside.get_available_tags()
self.assertEquals(len(available_tags), 2, "StructuredTagsAside should contains two tag categories")
def setUp(self):
"""
Create a stub XBlock backed by in-memory storage.
"""
self.runtime = mock.MagicMock(Runtime)
self.field_data = KvsFieldData(kvs=DictKeyValueStore())
self.scope_ids = ScopeIds('Bob', 'mutablestubxblock', '123', 'import')
self.xblock = StubXBlockWithMutableFields(self.runtime, self.field_data, self.scope_ids)
self.fake_children_locations = [
BlockUsageLocator(CourseLocator('org', 'course', 'run'), 'mutablestubxblock', 'child1'),
BlockUsageLocator(CourseLocator('org', 'course', 'run'), 'mutablestubxblock', 'child2'),
]
super(UpdateLocationTest, self).setUp()
def make_xblock(xblock_name, xblock_cls, attributes):
"""
Helper to construct XBlock objects
"""
runtime = WorkbenchRuntime()
key_store = DictKeyValueStore()
db_model = KvsFieldData(key_store)
ids = generate_scope_ids(runtime, xblock_name)
xblock = xblock_cls(runtime, db_model, scope_ids=ids)
xblock.category = Mock()
xblock.location = Mock(html_id=Mock(return_value='sample_element_id'), )
xblock.runtime = Mock(hostname='localhost', )
xblock.course_id = 'course-v1:edX+DemoX+Demo_Course'
for key, value in attributes.items():
setattr(xblock, key, value)
return xblock
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir", source_dirs=[],
load_error_modules=load_error_modules)
course_id = CourseKey.from_string('/'.join([ORG, COURSE, 'test_run']))
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
)
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir",
course_dirs=[],
load_error_modules=load_error_modules)
course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run')
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
)
def __init__(self, load_error_modules, course_id=None):
xmlstore = XMLModuleStore("data_dir", source_dirs=[],
load_error_modules=load_error_modules)
if course_id is None:
course_id = CourseKey.from_string('/'.join([ORG, COURSE, 'test_run']))
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
)
def __init__(self, load_error_modules, library=False):
if library:
xmlstore = LibraryXMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules)
else:
xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules)
course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run')
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
mixins=(InheritanceMixin, XModuleMixin),
field_data=KvsFieldData(DictKeyValueStore()),
)
def test_runtime_handle():
# Test a simple handler and a fallback handler
key_store = DictKeyValueStore()
field_data = KvsFieldData(key_store)
test_runtime = TestRuntime(services={'field-data': field_data})
basic_tester = TestXBlock(test_runtime, scope_ids=Mock(spec=ScopeIds))
runtime = MockRuntimeForQuerying()
# string we want to update using the handler
update_string = "user state update"
assert_equals(
runtime.handle(basic_tester, 'existing_handler', update_string),
'I am the existing test handler')
assert_equals(basic_tester.user_state, update_string)
# when the handler needs to use the fallback as given name can't be found
new_update_string = "new update"
assert_equals(
runtime.handle(basic_tester, 'test_fallback_handler',
new_update_string), 'I have been handled')
assert_equals(basic_tester.user_state, new_update_string)
# request to use a handler which doesn't have XBlock.handler decoration
# should use the fallback
new_update_string = "new update"
assert_equals(
runtime.handle(basic_tester, 'handler_without_correct_decoration',
new_update_string), 'gone to fallback')
assert_equals(basic_tester.user_state, new_update_string)
# handler can't be found & no fallback handler supplied, should throw an exception
no_fallback_tester = TestXBlockNoFallback(runtime,
scope_ids=Mock(spec=ScopeIds))
ultimate_string = "ultimate update"
with assert_raises(NoSuchHandlerError):
runtime.handle(no_fallback_tester, 'test_nonexistant_fallback_handler',
ultimate_string)
# request to use a handler which doesn't have XBlock.handler decoration
# and no fallback should raise NoSuchHandlerError
with assert_raises(NoSuchHandlerError):
runtime.handle(no_fallback_tester,
'handler_without_correct_decoration', 'handled')
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir", course_dirs=[],
load_error_modules=load_error_modules)
course_id = "/".join([ORG, COURSE, 'test_run'])
course_dir = "test_dir"
error_tracker = Mock()
parent_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
parent_tracker=parent_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
id_reader=LocationReader(),
)
def __init__(self, load_error_modules, library=False):
if library:
xmlstore = LibraryXMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules)
else:
xmlstore = XMLModuleStore("data_dir", source_dirs=[], load_error_modules=load_error_modules)
course_id = CourseKey.from_string('/'.join([ORG, COURSE, RUN]))
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
mixins=(InheritanceMixin, XModuleMixin),
field_data=KvsFieldData(DictKeyValueStore()),
)
def __init__(self, xml_import_data):
self.course_id = CourseKey.from_string(xml_import_data.course_id)
self.default_class = xml_import_data.default_class
self._descriptors = {}
def get_policy(usage_id):
"""Return the policy data for the specified usage"""
return xml_import_data.policy.get(policy_key(usage_id), {})
super(InMemorySystem, self).__init__(
get_policy=get_policy,
process_xml=self.process_xml,
load_item=self.load_item,
error_tracker=Mock(),
resources_fs=xml_import_data.filesystem,
mixins=xml_import_data.xblock_mixins,
select=xml_import_data.xblock_select,
render_template=lambda template, context: pprint.pformat((template, context)),
field_data=KvsFieldData(DictKeyValueStore()),
)
def test_runtime_render():
key_store = DictKeyValueStore()
field_data = KvsFieldData(key_store)
runtime = MockRuntimeForQuerying(services={'field-data': field_data})
block_type = 'test'
def_id = runtime.id_generator.create_definition(block_type)
usage_id = runtime.id_generator.create_usage(def_id)
tester = TestXBlock(runtime,
scope_ids=ScopeIds('user', block_type, def_id,
usage_id))
# string we want to update using the handler
update_string = "user state update"
# test against the student view
frag = runtime.render(tester, 'student_view', [update_string])
assert update_string in frag.body_html()
assert tester.preferences == update_string
# test against the fallback view
update_string = "new update"
frag = runtime.render(tester, 'test_fallback_view', [update_string])
assert update_string in frag.body_html()
assert tester.preferences == update_string
# test block-first
update_string = "penultimate update"
frag = tester.render('student_view', [update_string])
assert update_string in frag.body_html()
assert tester.preferences == update_string
# test against the no-fallback XBlock
update_string = "ultimate update"
no_fallback_tester = TestXBlockNoFallback(Mock(),
scope_ids=Mock(spec=ScopeIds))
with pytest.raises(NoSuchViewError):
runtime.render(no_fallback_tester, 'test_nonexistent_view',
[update_string])
def test_db_model_keys():
# Tests that updates to fields are properly recorded in the KeyValueStore,
# and that the keys have been constructed correctly
key_store = DictKeyValueStore()
field_data = KvsFieldData(key_store)
runtime = TestRuntime(Mock(), field_data, [TestMixin])
tester = runtime.construct_xblock_from_class(
TestXBlock, ScopeIds('s0', 'TestXBlock', 'd0', 'u0'))
assert_false(field_data.has(tester, 'not a field'))
for field in tester.fields.values():
new_value = 'new ' + field.name
assert_false(field_data.has(tester, field.name))
setattr(tester, field.name, new_value)
# Write out the values
tester.save()
# Make sure everything saved correctly
for field in tester.fields.values():
assert_true(field_data.has(tester, field.name))
def get_key_value(scope, user_id, block_scope_id, field_name):
"""Gets the value, from `key_store`, of a Key with the given values."""
new_key = KeyValueStore.Key(scope, user_id, block_scope_id, field_name)
return key_store.db_dict[new_key]
# Examine each value in the database and ensure that keys were constructed correctly
assert_equals('new content',
get_key_value(Scope.content, None, 'd0', 'content'))
assert_equals('new settings',
get_key_value(Scope.settings, None, 'u0', 'settings'))
assert_equals('new user_state',
get_key_value(Scope.user_state, 's0', 'u0', 'user_state'))
assert_equals(
'new preferences',
get_key_value(Scope.preferences, 's0', 'TestXBlock', 'preferences'))
assert_equals('new user_info',
get_key_value(Scope.user_info, 's0', None, 'user_info'))
assert_equals(
'new by_type',
get_key_value(Scope(UserScope.NONE, BlockScope.TYPE), None,
'TestXBlock', 'by_type'))
assert_equals(
'new for_all',
get_key_value(Scope(UserScope.NONE, BlockScope.ALL), None, None,
'for_all'))
assert_equals(
'new user_def',
get_key_value(Scope(UserScope.ONE, BlockScope.DEFINITION), 's0', 'd0',
'user_def'))
assert_equals(
'new agg_global',
get_key_value(Scope(UserScope.ALL, BlockScope.ALL), None, None,
'agg_global'))
assert_equals(
'new agg_type',
get_key_value(Scope(UserScope.ALL, BlockScope.TYPE), None,
'TestXBlock', 'agg_type'))
assert_equals(
'new agg_def',
get_key_value(Scope(UserScope.ALL, BlockScope.DEFINITION), None, 'd0',
'agg_def'))
assert_equals(
'new agg_usage',
get_key_value(Scope.user_state_summary, None, 'u0', 'agg_usage'))
assert_equals('new mixin_content',
get_key_value(Scope.content, None, 'd0', 'mixin_content'))
assert_equals('new mixin_settings',
get_key_value(Scope.settings, None, 'u0', 'mixin_settings'))
assert_equals(
'new mixin_user_state',
get_key_value(Scope.user_state, 's0', 'u0', 'mixin_user_state'))
assert_equals(
'new mixin_preferences',
get_key_value(Scope.preferences, 's0', 'TestXBlock',
'mixin_preferences'))
assert_equals(
'new mixin_user_info',
get_key_value(Scope.user_info, 's0', None, 'mixin_user_info'))
assert_equals(
'new mixin_by_type',
get_key_value(Scope(UserScope.NONE, BlockScope.TYPE), None,
'TestXBlock', 'mixin_by_type'))
assert_equals(
'new mixin_for_all',
get_key_value(Scope(UserScope.NONE, BlockScope.ALL), None, None,
'mixin_for_all'))
assert_equals(
'new mixin_user_def',
get_key_value(Scope(UserScope.ONE, BlockScope.DEFINITION), 's0', 'd0',
'mixin_user_def'))
assert_equals(
'new mixin_agg_global',
get_key_value(Scope(UserScope.ALL, BlockScope.ALL), None, None,
'mixin_agg_global'))
assert_equals(
'new mixin_agg_type',
get_key_value(Scope(UserScope.ALL, BlockScope.TYPE), None,
'TestXBlock', 'mixin_agg_type'))
assert_equals(
'new mixin_agg_def',
get_key_value(Scope(UserScope.ALL, BlockScope.DEFINITION), None, 'd0',
'mixin_agg_def'))
assert_equals(
'new mixin_agg_usage',
get_key_value(Scope.user_state_summary, None, 'u0', 'mixin_agg_usage'))
def setUp(self):
key_store = DictKeyValueStore()
field_data = KvsFieldData(key_store)
self.runtime = TestRuntime(services={'field-data': field_data})
def make_xblock(self):
key_store = DictKeyValueStore()
field_data = KvsFieldData(key_store)
runtime = TestRuntime(services={'field-data': field_data})
xblock = WistiaVideoXBlock(runtime, scope_ids=Mock())
return xblock
def import_course_draft(
xml_module_store, store, draft_store, course_data_path,
static_content_store, source_location_namespace,
target_location_namespace):
'''
This will import all the content inside of the 'drafts' folder, if it exists
NOTE: This is not a full course import, basically in our current
application only verticals (and downwards) can be in draft.
Therefore, we need to use slightly different call points into
the import process_xml as we can't simply call XMLModuleStore() constructor
(like we do for importing public content)
'''
draft_dir = course_data_path + "/drafts"
if not os.path.exists(draft_dir):
return
# create a new 'System' object which will manage the importing
errorlog = make_error_tracker()
# The course_dir as passed to ImportSystem is expected to just be relative, not
# the complete path including data_dir. ImportSystem will concatenate the two together.
data_dir = xml_module_store.data_dir
# Whether or not data_dir ends with a "/" differs in production vs. test.
if not data_dir.endswith("/"):
data_dir += "/"
draft_course_dir = draft_dir.replace(data_dir, '', 1)
system = ImportSystem(
xmlstore=xml_module_store,
course_id=target_location_namespace.course_id,
course_dir=draft_course_dir,
error_tracker=errorlog.tracker,
parent_tracker=ParentTracker(),
load_error_modules=False,
mixins=xml_module_store.xblock_mixins,
field_data=KvsFieldData(kvs=DictKeyValueStore()),
)
# now walk the /vertical directory where each file in there
# will be a draft copy of the Vertical
# First it is necessary to order the draft items by their desired index in the child list
# (order os.walk returns them in is not guaranteed).
drafts = dict()
for dirname, _dirnames, filenames in os.walk(draft_dir + "/vertical"):
for filename in filenames:
module_path = os.path.join(dirname, filename)
with open(module_path, 'r') as f:
try:
# note, on local dev it seems like OSX will put
# some extra files in the directory with "quarantine"
# information. These files are binary files and will
# throw exceptions when we try to parse the file
# as an XML string. Let's make sure we're
# dealing with a string before ingesting
data = f.read()
try:
xml = data.decode('utf-8')
except UnicodeDecodeError, err:
# seems like on OSX localdev, the OS is making
# quarantine files in the unzip directory
# when importing courses so if we blindly try to
# enumerate through the directory, we'll try
# to process a bunch of binary quarantine files
# (which are prefixed with a '._' character which
# will dump a bunch of exceptions to the output,
# although they are harmless.
#
# Reading online docs there doesn't seem to be
# a good means to detect a 'hidden' file that works
# well across all OS environments. So for now, I'm using
# OSX's utilization of a leading '.' in the filename
# to indicate a system hidden file.
#
# Better yet would be a way to figure out if this is
# a binary file, but I haven't found a good way
# to do this yet.
if filename.startswith('._'):
continue
# Not a 'hidden file', then re-raise exception
raise err
descriptor = system.process_xml(xml)
# HACK: since we are doing partial imports of drafts
# the vertical doesn't have the 'url-name' set in the
# attributes (they are normally in the parent object,
# aka sequential), so we have to replace the location.name
# with the XML filename that is part of the pack
fn, fileExtension = os.path.splitext(filename)
descriptor.location = descriptor.location.replace(name=fn)
index = int(descriptor.xml_attributes['index_in_children_list'])
if index in drafts:
drafts[index].append(descriptor)
else:
drafts[index] = [descriptor]
except Exception, e:
logging.exception('There was an error. {err}'.format(
err=unicode(e)
))
def _import_course_draft(
xml_module_store,
store,
user_id,
course_data_path,
source_course_id,
target_course_id,
mongo_runtime
):
'''
This will import all the content inside of the 'drafts' folder, if it exists
NOTE: This is not a full course import, basically in our current
application only verticals (and downwards) can be in draft.
Therefore, we need to use slightly different call points into
the import process_xml as we can't simply call XMLModuleStore() constructor
(like we do for importing public content)
'''
draft_dir = course_data_path + "/drafts"
if not os.path.exists(draft_dir):
return
# create a new 'System' object which will manage the importing
errorlog = make_error_tracker()
# The course_dir as passed to ImportSystem is expected to just be relative, not
# the complete path including data_dir. ImportSystem will concatenate the two together.
data_dir = xml_module_store.data_dir
# Whether or not data_dir ends with a "/" differs in production vs. test.
if not data_dir.endswith("/"):
data_dir += "/"
draft_course_dir = draft_dir.replace(data_dir, '', 1)
system = ImportSystem(
xmlstore=xml_module_store,
course_id=source_course_id,
course_dir=draft_course_dir,
error_tracker=errorlog.tracker,
parent_tracker=ParentTracker(),
load_error_modules=False,
mixins=xml_module_store.xblock_mixins,
field_data=KvsFieldData(kvs=DictKeyValueStore()),
)
def _import_module(module):
# IMPORTANT: Be sure to update the module location in the NEW namespace
module_location = module.location.map_into_course(target_course_id)
# Update the module's location to DRAFT revision
# We need to call this method (instead of updating the location directly)
# to ensure that pure XBlock field data is updated correctly.
_update_module_location(module, module_location.replace(revision=MongoRevisionKey.draft))
parent_url = get_parent_url(module)
index = index_in_children_list(module)
# make sure our parent has us in its list of children
# this is to make sure private only modules show up
# in the list of children since they would have been
# filtered out from the non-draft store export.
if parent_url is not None and index is not None:
course_key = descriptor.location.course_key
parent_location = course_key.make_usage_key_from_deprecated_string(parent_url)
# IMPORTANT: Be sure to update the parent in the NEW namespace
parent_location = parent_location.map_into_course(target_course_id)
parent = store.get_item(parent_location, depth=0)
non_draft_location = module.location.map_into_course(target_course_id)
if not any(child.block_id == module.location.block_id for child in parent.children):
parent.children.insert(index, non_draft_location)
store.update_item(parent, user_id)
_import_module_and_update_references(
module, store, user_id,
source_course_id,
target_course_id,
runtime=mongo_runtime,
)
for child in module.get_children():
_import_module(child)
# now walk the /vertical directory where each file in there
# will be a draft copy of the Vertical
# First it is necessary to order the draft items by their desired index in the child list
# (order os.walk returns them in is not guaranteed).
drafts = []
for dirname, _dirnames, filenames in os.walk(draft_dir):
for filename in filenames:
module_path = os.path.join(dirname, filename)
with open(module_path, 'r') as f:
try:
# note, on local dev it seems like OSX will put
# some extra files in the directory with "quarantine"
# information. These files are binary files and will
# throw exceptions when we try to parse the file
# as an XML string. Let's make sure we're
# dealing with a string before ingesting
data = f.read()
try:
xml = data.decode('utf-8')
except UnicodeDecodeError, err:
# seems like on OSX localdev, the OS is making
# quarantine files in the unzip directory
# when importing courses so if we blindly try to
# enumerate through the directory, we'll try
# to process a bunch of binary quarantine files
# (which are prefixed with a '._' character which
# will dump a bunch of exceptions to the output,
# although they are harmless.
#
# Reading online docs there doesn't seem to be
# a good means to detect a 'hidden' file that works
# well across all OS environments. So for now, I'm using
# OSX's utilization of a leading '.' in the filename
# to indicate a system hidden file.
#
# Better yet would be a way to figure out if this is
# a binary file, but I haven't found a good way
# to do this yet.
if filename.startswith('._'):
continue
# Not a 'hidden file', then re-raise exception
raise err
# process_xml call below recursively processes all descendants. If
# we call this on all verticals in a course with verticals nested below
# the unit level, we try to import the same content twice, causing naming conflicts.
# Therefore only process verticals at the unit level, assuming that any other
# verticals must be descendants.
if 'index_in_children_list' in xml:
descriptor = system.process_xml(xml)
# HACK: since we are doing partial imports of drafts
# the vertical doesn't have the 'url-name' set in the
# attributes (they are normally in the parent object,
# aka sequential), so we have to replace the location.name
# with the XML filename that is part of the pack
filename, __ = os.path.splitext(filename)
descriptor.location = descriptor.location.replace(name=filename)
index = index_in_children_list(descriptor)
parent_url = get_parent_url(descriptor, xml)
draft_url = descriptor.location.to_deprecated_string()
draft = draft_node_constructor(
module=descriptor, url=draft_url, parent_url=parent_url, index=index
)
drafts.append(draft)
except Exception: # pylint: disable=broad-except
logging.exception('Error while parsing course xml.')
def make_block():
runtime = WorkbenchRuntime()
key_store = DictKeyValueStore()
db_model = KvsFieldData(key_store)
return drag_and_drop_v2.DragAndDropBlock(runtime, db_model, Mock())
def _import_course_draft(
xml_module_store,
store,
user_id,
course_data_path,
source_course_id,
target_id,
mongo_runtime
):
"""
This method will import all the content inside of the 'drafts' folder, if content exists.
NOTE: This is not a full course import! In our current application, only verticals
(and blocks beneath) can be in draft. Therefore, different call points into the import
process_xml are used as the XMLModuleStore() constructor cannot simply be called
(as is done for importing public content).
"""
draft_dir = course_data_path + "/drafts"
if not os.path.exists(draft_dir):
return
# create a new 'System' object which will manage the importing
errorlog = make_error_tracker()
# The course_dir as passed to ImportSystem is expected to just be relative, not
# the complete path including data_dir. ImportSystem will concatenate the two together.
data_dir = xml_module_store.data_dir
# Whether or not data_dir ends with a "/" differs in production vs. test.
if not data_dir.endswith("/"):
data_dir += "/"
# Remove absolute path, leaving relative <course_name>/drafts.
draft_course_dir = draft_dir.replace(data_dir, '', 1)
system = ImportSystem(
xmlstore=xml_module_store,
course_id=source_course_id,
course_dir=draft_course_dir,
error_tracker=errorlog.tracker,
load_error_modules=False,
mixins=xml_module_store.xblock_mixins,
field_data=KvsFieldData(kvs=DictKeyValueStore()),
target_course_id=target_id,
)
def _import_module(module):
# IMPORTANT: Be sure to update the module location in the NEW namespace
module_location = module.location.map_into_course(target_id)
# Update the module's location to DRAFT revision
# We need to call this method (instead of updating the location directly)
# to ensure that pure XBlock field data is updated correctly.
_update_module_location(module, module_location.replace(revision=MongoRevisionKey.draft))
parent_url = get_parent_url(module)
index = index_in_children_list(module)
# make sure our parent has us in its list of children
# this is to make sure private only modules show up
# in the list of children since they would have been
# filtered out from the non-draft store export.
if parent_url is not None and index is not None:
course_key = descriptor.location.course_key
parent_location = course_key.make_usage_key_from_deprecated_string(parent_url)
# IMPORTANT: Be sure to update the parent in the NEW namespace
parent_location = parent_location.map_into_course(target_id)
parent = store.get_item(parent_location, depth=0)
non_draft_location = module.location.map_into_course(target_id)
if not any(child.block_id == module.location.block_id for child in parent.children):
parent.children.insert(index, non_draft_location)
store.update_item(parent, user_id)
_update_and_import_module(
module, store, user_id,
source_course_id,
target_id,
runtime=mongo_runtime,
)
for child in module.get_children():
_import_module(child)
# Now walk the /drafts directory.
# Each file in the directory will be a draft copy of the vertical.
# First it is necessary to order the draft items by their desired index in the child list,
# since the order in which os.walk() returns the files is not guaranteed.
drafts = []
for rootdir, __, filenames in os.walk(draft_dir):
for filename in filenames:
if filename.startswith('._'):
# Skip any OSX quarantine files, prefixed with a '._'.
continue
module_path = os.path.join(rootdir, filename)
with open(module_path, 'r') as f:
try:
xml = f.read().decode('utf-8')
# The process_xml() call below recursively processes all descendants. If
# we call this on all verticals in a course with verticals nested below
# the unit level, we try to import the same content twice, causing naming conflicts.
# Therefore only process verticals at the unit level, assuming that any other
# verticals must be descendants.
if 'index_in_children_list' in xml:
descriptor = system.process_xml(xml)
# HACK: since we are doing partial imports of drafts
# the vertical doesn't have the 'url-name' set in the
# attributes (they are normally in the parent object,
# aka sequential), so we have to replace the location.name
# with the XML filename that is part of the pack
filename, __ = os.path.splitext(filename)
descriptor.location = descriptor.location.replace(name=filename)
index = index_in_children_list(descriptor)
parent_url = get_parent_url(descriptor, xml)
draft_url = unicode(descriptor.location)
draft = draft_node_constructor(
module=descriptor, url=draft_url, parent_url=parent_url, index=index
)
drafts.append(draft)
except Exception: # pylint: disable=broad-except
logging.exception('Error while parsing course drafts xml.')
# Sort drafts by `index_in_children_list` attribute.
drafts.sort(key=lambda x: x.index)
for draft in get_draft_subtree_roots(drafts):
try:
_import_module(draft.module)
except Exception: # pylint: disable=broad-except
logging.exception('while importing draft descriptor %s', draft.module)
def __init__(self, **kwargs):
field_data = kwargs.get('field_data',
KvsFieldData(DictKeyValueStore()))
super(MockRuntime, self).__init__(field_data=field_data)
def setUp(self):
self.runtime = TestRuntime(field_data=KvsFieldData(DictKeyValueStore()))
