def get_block_display_name(block_or_key):
"""
Efficiently get the display name of the specified block. This is done in a
way that avoids having to load and parse the block's entire XML field data
using its parse_xml() method, which may be very expensive (e.g. the video
XBlock parse_xml leads to various slow edxval API calls in some cases).
This method also defines and implements various fallback mechanisms in case
the ID can't be loaded.
block_or_key can be an XBlock instance, a usage key or a definition key.
Returns the display name as a string
"""
def_key = resolve_definition(block_or_key)
use_draft = get_xblock_app_config().get_learning_context_params().get('use_draft')
cache = BundleCache(def_key.bundle_uuid, draft_name=use_draft)
cache_key = ('block_display_name', str(def_key))
display_name = cache.get(cache_key)
if display_name is None:
# Instead of loading the block, just load its XML and parse it
try:
olx_node = xml_for_definition(def_key)
except Exception: # pylint: disable=broad-except
log.exception("Error when trying to get display_name for block definition %s", def_key)
# Return now so we don't cache the error result
return xblock_type_display_name(def_key.block_type)
try:
display_name = olx_node.attrib['display_name']
except KeyError:
display_name = xblock_type_display_name(def_key.block_type)
cache.set(cache_key, display_name)
return display_name
def __init__(self, library_key, bundle_uuid, draft_name=None):
"""
Instantiate this wrapper for the bundle with the specified library_key,
UUID, and optionally the specified draft name.
"""
self.library_key = library_key
self.bundle_uuid = bundle_uuid
self.draft_name = draft_name
self.cache = BundleCache(bundle_uuid, draft_name)
def test_bundle_draft_cache(self):
"""
Test caching data related to a bundle draft
"""
cache = BundleCache(self.bundle.uuid, draft_name=self.draft.name)
key1 = ("some", "key", "1")
key2 = ("key2", )
value1 = "value1"
cache.set(key1, value1)
value2 = {"this is": "a dict", "for": "key2"}
cache.set(key2, value2)
self.assertEqual(cache.get(key1), value1)
self.assertEqual(cache.get(key2), value2)
# Now make a change to the draft (doesn't matter if we commit it or not)
api.write_draft_file(
self.draft.uuid, "test.txt",
"we need a changed file in order to publish a new version")
# Now the cache should be invalidated
# (immediately since we set MAX_BLOCKSTORE_CACHE_DELAY to 0)
self.assertEqual(cache.get(key1), None)
self.assertEqual(cache.get(key2), None)
def test_bundle_cache(self):
"""
Test caching data related to a bundle (no draft)
"""
cache = BundleCache(self.bundle.uuid)
key1 = ("some", "key", "1")
key2 = ("key2", )
value1 = "value1"
cache.set(key1, value1)
value2 = {"this is": "a dict", "for": "key2"}
cache.set(key2, value2)
self.assertEqual(cache.get(key1), value1)
self.assertEqual(cache.get(key2), value2)
# Now publish a new version of the bundle:
api.write_draft_file(
self.draft.uuid, "test.txt",
"we need a changed file in order to publish a new version")
api.commit_draft(self.draft.uuid)
# Now the cache should be invalidated
# (immediately since we set MAX_BLOCKSTORE_CACHE_DELAY to 0)
self.assertEqual(cache.get(key1), None)
self.assertEqual(cache.get(key2), None)
def set_library_block_olx(usage_key, new_olx_str):
"""
Replace the OLX source of the given XBlock.
This is only meant for use by developers or API client applications, as
very little validation is done and this can easily result in a broken XBlock
that won't load.
"""
# because this old pylint can't understand attr.ib() objects, pylint: disable=no-member
assert isinstance(usage_key, LibraryUsageLocatorV2)
# Make sure the block exists:
metadata = get_library_block(usage_key)
block_type = usage_key.block_type
# Verify that the OLX parses, at least as generic XML:
node = etree.fromstring(new_olx_str)
if node.tag != block_type:
raise ValueError(
"Invalid root tag in OLX, expected {}".format(block_type))
# Write the new XML/OLX file into the library bundle's draft
draft = get_or_create_bundle_draft(metadata.def_key.bundle_uuid,
DRAFT_NAME)
write_draft_file(draft.uuid, metadata.def_key.olx_path,
new_olx_str.encode('utf-8'))
# Clear the bundle cache so everyone sees the new block immediately:
BundleCache(metadata.def_key.bundle_uuid, draft_name=DRAFT_NAME).clear()
LIBRARY_BLOCK_UPDATED.send(sender=None,
library_key=usage_key.context_key,
usage_key=usage_key)
def save_block(self, block):
"""
Save any pending field data values to Blockstore.
This gets called by block.save() - do not call this directly.
"""
if not self.system.authored_data_store.has_changes(block):
return # No changes, so no action needed.
definition_key = block.scope_ids.def_id
if definition_key.draft_name is None:
raise RuntimeError(
"The Blockstore runtime does not support saving changes to blockstore without a draft. "
"Are you making changes to UserScope.NONE fields from the LMS rather than Studio?"
)
olx_str, static_files = serialize_xblock(block)
# Write the OLX file to the bundle:
draft_uuid = blockstore_api.get_or_create_bundle_draft(
definition_key.bundle_uuid, definition_key.draft_name).uuid
olx_path = definition_key.olx_path
blockstore_api.write_draft_file(draft_uuid, olx_path, olx_str)
# And the other files, if any:
olx_static_path = os.path.dirname(olx_path) + '/static/'
for fh in static_files:
new_path = olx_static_path + fh.name
blockstore_api.write_draft_file(draft_uuid, new_path, fh.data)
# Now invalidate the blockstore data cache for the bundle:
BundleCache(definition_key.bundle_uuid,
draft_name=definition_key.draft_name).clear()
def create_library_block(library_key, block_type, definition_id):
"""
Create a new XBlock in this library of the specified type (e.g. "html").
The 'definition_id' value (which should be a string like "problem1") will be
used as both the definition_id and the usage_id.
"""
assert isinstance(library_key, LibraryLocatorV2)
ref = ContentLibrary.objects.get_by_key(library_key)
# Make sure the proposed ID will be valid:
validate_unicode_slug(definition_id)
# Ensure the XBlock type is valid and installed:
XBlock.load_class(block_type) # Will raise an exception if invalid
# Make sure the new ID is not taken already:
new_usage_id = definition_id # Since this is a top level XBlock, usage_id == definition_id
usage_key = LibraryUsageLocatorV2(
library_org=library_key.org,
library_slug=library_key.slug,
block_type=block_type,
usage_id=new_usage_id,
)
library_context = get_learning_context_impl(usage_key)
if library_context.definition_for_usage(usage_key) is not None:
raise LibraryBlockAlreadyExists("An XBlock with ID '{}' already exists".format(new_usage_id))
new_definition_xml = '<{}/>'.format(block_type) # xss-lint: disable=python-wrap-html
path = "{}/{}/definition.xml".format(block_type, definition_id)
# Write the new XML/OLX file into the library bundle's draft
draft = get_or_create_bundle_draft(ref.bundle_uuid, DRAFT_NAME)
write_draft_file(draft.uuid, path, new_definition_xml)
# Clear the bundle cache so everyone sees the new block immediately:
BundleCache(ref.bundle_uuid, draft_name=DRAFT_NAME).clear()
# Now return the metadata about the new block:
return get_library_block(usage_key)
def save_block(self, block):
"""
Save any pending field data values to Blockstore.
This gets called by block.save() - do not call this directly.
"""
if not self.system.authored_data_store.has_changes(block):
return # No changes, so no action needed.
definition_key = block.scope_ids.def_id
if definition_key.draft_name is None:
raise RuntimeError(
"The Blockstore runtime does not support saving changes to blockstore without a draft. "
"Are you making changes to UserScope.NONE fields from the LMS rather than Studio?"
)
# Verify that the user has permission to write to authored data in this
# learning context:
if self.user is not None:
learning_context = get_learning_context_impl(block.scope_ids.usage_id)
if not learning_context.can_edit_block(self.user, block.scope_ids.usage_id):
log.warning("User %s does not have permission to edit %s", self.user.username, block.scope_ids.usage_id)
raise RuntimeError("You do not have permission to edit this XBlock")
olx_str, static_files = serialize_xblock(block)
# Write the OLX file to the bundle:
draft_uuid = blockstore_api.get_or_create_bundle_draft(
definition_key.bundle_uuid, definition_key.draft_name
).uuid
olx_path = definition_key.olx_path
blockstore_api.write_draft_file(draft_uuid, olx_path, olx_str)
# And the other files, if any:
olx_static_path = os.path.dirname(olx_path) + '/static/'
for fh in static_files:
new_path = olx_static_path + fh.name
blockstore_api.write_draft_file(draft_uuid, new_path, fh.data)
# Now invalidate the blockstore data cache for the bundle:
BundleCache(definition_key.bundle_uuid, draft_name=definition_key.draft_name).clear()
def create_library_block(library_key, block_type, definition_id):
"""
Create a new XBlock in this library of the specified type (e.g. "html").
The 'definition_id' value (which should be a string like "problem1") will be
used as both the definition_id and the usage_id.
"""
assert isinstance(library_key, LibraryLocatorV2)
ref = ContentLibrary.objects.get_by_key(library_key)
if ref.type != COMPLEX:
if block_type != ref.type:
raise IncompatibleTypesError(
_('Block type "{block_type}" is not compatible with library type "{library_type}".'
).format(
block_type=block_type,
library_type=ref.type,
))
lib_bundle = LibraryBundle(library_key,
ref.bundle_uuid,
draft_name=DRAFT_NAME)
# Total number of blocks should not exceed the maximum allowed
total_blocks = len(lib_bundle.get_top_level_usages())
if total_blocks + 1 > settings.MAX_BLOCKS_PER_CONTENT_LIBRARY:
raise BlockLimitReachedError(
_(u"Library cannot have more than {} XBlocks").format(
settings.MAX_BLOCKS_PER_CONTENT_LIBRARY))
# Make sure the proposed ID will be valid:
validate_unicode_slug(definition_id)
# Ensure the XBlock type is valid and installed:
XBlock.load_class(block_type) # Will raise an exception if invalid
# Make sure the new ID is not taken already:
new_usage_id = definition_id # Since this is a top level XBlock, usage_id == definition_id
usage_key = LibraryUsageLocatorV2(
lib_key=library_key,
block_type=block_type,
usage_id=new_usage_id,
)
library_context = get_learning_context_impl(usage_key)
if library_context.definition_for_usage(usage_key) is not None:
raise LibraryBlockAlreadyExists(
"An XBlock with ID '{}' already exists".format(new_usage_id))
new_definition_xml = '<{}/>'.format(
block_type) # xss-lint: disable=python-wrap-html
path = "{}/{}/definition.xml".format(block_type, definition_id)
# Write the new XML/OLX file into the library bundle's draft
draft = get_or_create_bundle_draft(ref.bundle_uuid, DRAFT_NAME)
write_draft_file(draft.uuid, path, new_definition_xml.encode('utf-8'))
# Clear the bundle cache so everyone sees the new block immediately:
BundleCache(ref.bundle_uuid, draft_name=DRAFT_NAME).clear()
# Now return the metadata about the new block:
LIBRARY_BLOCK_CREATED.send(sender=None,
library_key=ref.library_key,
usage_key=usage_key)
return get_library_block(usage_key)
def test_bundle_cache_clear(self):
"""
Test the cache clear() method
"""
cache = BundleCache(self.bundle.uuid)
key1 = ("some", "key", "1")
value1 = "value1"
cache.set(key1, value1)
assert cache.get(key1) == value1
# Now publish a new version of the bundle:
api.write_draft_file(self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version")
api.commit_draft(self.draft.uuid)
# Now the cache will not be immediately invalidated; it takes up to MAX_BLOCKSTORE_CACHE_DELAY seconds.
# Since this is a new bundle and we _just_ accessed the cache for the first time, we can be confident
# it won't yet be automatically invalidated.
assert cache.get(key1) == value1
# Now "clear" the cache, forcing the check of the new version:
cache.clear()
assert cache.get(key1) is None
class LibraryBundle(object):
"""
Wrapper around a Content Library Blockstore bundle that contains OLX.
"""
def __init__(self, library_key, bundle_uuid, draft_name=None):
"""
Instantiate this wrapper for the bundle with the specified library_key,
UUID, and optionally the specified draft name.
"""
self.library_key = library_key
self.bundle_uuid = bundle_uuid
self.draft_name = draft_name
self.cache = BundleCache(bundle_uuid, draft_name)
def get_olx_files(self):
"""
Get the list of OLX files in this bundle (using a heuristic)
Because this uses a heuristic, it will only return files with filenames
that seem like OLX files that are in the expected locations of OLX
files. They are not guaranteed to be valid OLX nor will OLX files in
nonstandard locations be returned.
Example return value: [
'html/intro/definition.xml',
'unit/unit1/definition.xml',
]
"""
bundle_files = get_bundle_files_cached(self.bundle_uuid,
draft_name=self.draft_name)
return [
f.path for f in bundle_files if f.path.endswith("/definition.xml")
]
def definition_for_usage(self, usage_key):
"""
Given the usage key for an XBlock in this library bundle, return the
BundleDefinitionLocator which specifies the actual XBlock definition (as
a path to an OLX in a specific blockstore bundle).
Must return a BundleDefinitionLocator if the XBlock exists in this
context, or None otherwise.
For a content library, the rules are simple:
* If the usage key points to a block in this library, the filename
(definition) of the OLX file is always
{block_type}/{usage_id}/definition.xml
Each library has exactly one usage per definition for its own blocks.
* However, block definitions from other content libraries may be linked
into this library via <xblock-include ... /> directives. In that case,
it's necessary to inspect every OLX file in this library that might
have an <xblock-include /> directive in order to find what external
block the usage ID refers to.
"""
# Now that we know the library/bundle, find the block's definition
if self.draft_name:
version_arg = {"draft_name": self.draft_name}
else:
version_arg = {
"bundle_version": get_bundle_version_number(self.bundle_uuid)
}
olx_path = "{}/{}/definition.xml".format(usage_key.block_type,
usage_key.usage_id)
try:
get_bundle_file_metadata_with_cache(self.bundle_uuid, olx_path,
**version_arg)
return BundleDefinitionLocator(self.bundle_uuid,
usage_key.block_type, olx_path,
**version_arg)
except blockstore_api.BundleFileNotFound:
# This must be a usage of a block from a linked bundle. One of the
# OLX files in this bundle contains an <xblock-include usage="..."/>
bundle_includes = self.get_bundle_includes()
try:
return bundle_includes[usage_key]
except KeyError:
return None
def get_top_level_usages(self):
"""
Get the set of usage keys in this bundle that have no parent.
"""
own_usage_keys = []
for olx_file_path in self.get_olx_files():
block_type, usage_id, _unused = olx_file_path.split('/')
usage_key = LibraryUsageLocatorV2(self.library_key, block_type,
usage_id)
own_usage_keys.append(usage_key)
usage_keys_with_parents = self.get_bundle_includes().keys()
return [
usage_key for usage_key in own_usage_keys
if usage_key not in usage_keys_with_parents
]
def get_bundle_includes(self):
"""
Scan through the bundle and all linked bundles as needed to generate
a complete list of all the blocks that are included as
child/grandchild/... blocks of the blocks in this bundle.
Returns a dict of {usage_key -> BundleDefinitionLocator}
Blocks in the bundle that have no parent are not included.
"""
cache_key = ("bundle_includes", )
usages_found = self.cache.get(cache_key)
if usages_found is not None:
return usages_found
usages_found = {}
def add_definitions_children(usage_key, def_key):
"""
Recursively add any children of the given XBlock usage+definition to
usages_found.
"""
if not does_block_type_support_children(def_key.block_type):
return
try:
xml_node = xml_for_definition(def_key)
except: # pylint:disable=bare-except
log.exception("Unable to load definition {}".format(def_key))
return
for child in xml_node:
if child.tag != 'xblock-include':
continue
try:
parsed_include = parse_xblock_include(child)
child_usage = usage_for_child_include(
usage_key, def_key, parsed_include)
child_def_key = definition_for_include(
parsed_include, def_key)
except BundleFormatException:
log.exception(
"Unable to parse a child of {}".format(def_key))
continue
usages_found[child_usage] = child_def_key
add_definitions_children(child_usage, child_def_key)
# Find all the definitions in this bundle and recursively add all their descendants:
bundle_files = get_bundle_files_cached(self.bundle_uuid,
draft_name=self.draft_name)
if self.draft_name:
version_arg = {"draft_name": self.draft_name}
else:
version_arg = {
"bundle_version": get_bundle_version_number(self.bundle_uuid)
}
for bfile in bundle_files:
if not bfile.path.endswith(
"/definition.xml") or bfile.path.count('/') != 2:
continue # Not an OLX file.
block_type, usage_id, _unused = bfile.path.split('/')
def_key = BundleDefinitionLocator(bundle_uuid=self.bundle_uuid,
block_type=block_type,
olx_path=bfile.path,
**version_arg)
usage_key = LibraryUsageLocatorV2(self.library_key, block_type,
usage_id)
add_definitions_children(usage_key, def_key)
self.cache.set(cache_key, usages_found)
return usages_found
def does_definition_have_unpublished_changes(self, definition_key):
"""
Given the defnition key of an XBlock, which exists in an OLX file like
problem/quiz1/definition.xml
Check if the bundle's draft has _any_ unpublished changes in the
problem/quiz1/
directory.
"""
if self.draft_name is None:
return False # No active draft so can't be changes
prefix = self.olx_prefix(definition_key)
return prefix in self._get_changed_definitions()
def _get_changed_definitions(self):
"""
Helper method to get a list of all paths with changes, where a path is
problem/quiz1/
Or similar (a type and an ID), excluding 'definition.xml'
"""
cached_result = self.cache.get(('changed_definition_prefixes', ))
if cached_result is not None:
return cached_result
changed = []
bundle_files = get_bundle_files_cached(self.bundle_uuid,
draft_name=self.draft_name)
for file_ in bundle_files:
if getattr(file_, 'modified',
False) and file_.path.count('/') >= 2:
(type_part, id_part, _rest) = file_.path.split('/', 2)
prefix = type_part + '/' + id_part + '/'
if prefix not in changed:
changed.append(prefix)
self.cache.set(('changed_definition_prefixes', ), changed)
return changed
def has_changes(self):
"""
Helper method to check if this OLX bundle has any pending changes,
including any deleted blocks.
Returns a tuple of (
has_unpublished_changes,
has_unpublished_deletes,
)
Where has_unpublished_changes is true if there is any type of change,
including deletes, and has_unpublished_deletes is only true if one or
more blocks has been deleted since the last publish.
"""
if not self.draft_name:
return (False, False)
cached_result = self.cache.get(('has_changes', ))
if cached_result is not None:
return cached_result
draft_files = get_bundle_files_cached(self.bundle_uuid,
draft_name=self.draft_name)
has_unpublished_changes = False
has_unpublished_deletes = False
for file_ in draft_files:
if getattr(file_, 'modified', False):
has_unpublished_changes = True
break
published_file_paths = set(
f.path for f in get_bundle_files_cached(self.bundle_uuid))
draft_file_paths = set(f.path for f in draft_files)
for file_path in published_file_paths:
if file_path not in draft_file_paths:
has_unpublished_changes = True
if file_path.endswith('/definition.xml'):
# only set 'has_unpublished_deletes' if the actual main definition XML
# file was deleted, not if only some asset file was deleted, etc.
has_unpublished_deletes = True
break
result = (has_unpublished_changes, has_unpublished_deletes)
self.cache.set(('has_changes', ), result)
return result
def get_static_prefix_for_definition(self, definition_key):
"""
Given a definition key, get the path prefix used for all (public) static
asset files.
Example: problem/quiz1/static/
"""
return self.olx_prefix(definition_key) + 'static/'
def get_static_files_for_definition(self, definition_key):
"""
Return a list of the static asset files related with a particular XBlock
definition.
If the bundle contains files like:
problem/quiz1/definition.xml
problem/quiz1/static/image1.png
Then this will return
[BundleFile(path="image1.png", size, url, hash_digest)]
"""
path_prefix = self.get_static_prefix_for_definition(definition_key)
path_prefix_len = len(path_prefix)
return [
blockstore_api.BundleFile(path=f.path[path_prefix_len:],
size=f.size,
url=f.url,
hash_digest=f.hash_digest)
for f in get_bundle_files_cached(self.bundle_uuid,
draft_name=self.draft_name)
if f.path.startswith(path_prefix)
]
@staticmethod
def olx_prefix(definition_key):
"""
Given a definition key in a compatible bundle, whose olx_path refers to
block_type/some_id/definition.xml
Return the "folder name" / "path prefix"
block-type/some_id/
This method is here rather than a method of BundleDefinitionLocator
because BundleDefinitionLocator is more generic and doesn't require
that its olx_path always ends in /definition.xml
"""
if not definition_key.olx_path.endswith('/definition.xml'):
raise ValueError
return definition_key.olx_path[:
-14] # Remove 'definition.xml', keep trailing slash
