def get_block_display_name(block_or_key): """ Efficiently get the display name of the specified block. This is done in a way that avoids having to load and parse the block's entire XML field data using its parse_xml() method, which may be very expensive (e.g. the video XBlock parse_xml leads to various slow edxval API calls in some cases). This method also defines and implements various fallback mechanisms in case the ID can't be loaded. block_or_key can be an XBlock instance, a usage key or a definition key. Returns the display name as a string """ def_key = resolve_definition(block_or_key) use_draft = get_xblock_app_config().get_learning_context_params().get('use_draft') cache = BundleCache(def_key.bundle_uuid, draft_name=use_draft) cache_key = ('block_display_name', str(def_key)) display_name = cache.get(cache_key) if display_name is None: # Instead of loading the block, just load its XML and parse it try: olx_node = xml_for_definition(def_key) except Exception: # pylint: disable=broad-except log.exception("Error when trying to get display_name for block definition %s", def_key) # Return now so we don't cache the error result return xblock_type_display_name(def_key.block_type) try: display_name = olx_node.attrib['display_name'] except KeyError: display_name = xblock_type_display_name(def_key.block_type) cache.set(cache_key, display_name) return display_name
def __init__(self, library_key, bundle_uuid, draft_name=None): """ Instantiate this wrapper for the bundle with the specified library_key, UUID, and optionally the specified draft name. """ self.library_key = library_key self.bundle_uuid = bundle_uuid self.draft_name = draft_name self.cache = BundleCache(bundle_uuid, draft_name)
def test_bundle_draft_cache(self): """ Test caching data related to a bundle draft """ cache = BundleCache(self.bundle.uuid, draft_name=self.draft.name) key1 = ("some", "key", "1") key2 = ("key2", ) value1 = "value1" cache.set(key1, value1) value2 = {"this is": "a dict", "for": "key2"} cache.set(key2, value2) self.assertEqual(cache.get(key1), value1) self.assertEqual(cache.get(key2), value2) # Now make a change to the draft (doesn't matter if we commit it or not) api.write_draft_file( self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version") # Now the cache should be invalidated # (immediately since we set MAX_BLOCKSTORE_CACHE_DELAY to 0) self.assertEqual(cache.get(key1), None) self.assertEqual(cache.get(key2), None)
def test_bundle_cache(self): """ Test caching data related to a bundle (no draft) """ cache = BundleCache(self.bundle.uuid) key1 = ("some", "key", "1") key2 = ("key2", ) value1 = "value1" cache.set(key1, value1) value2 = {"this is": "a dict", "for": "key2"} cache.set(key2, value2) self.assertEqual(cache.get(key1), value1) self.assertEqual(cache.get(key2), value2) # Now publish a new version of the bundle: api.write_draft_file( self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version") api.commit_draft(self.draft.uuid) # Now the cache should be invalidated # (immediately since we set MAX_BLOCKSTORE_CACHE_DELAY to 0) self.assertEqual(cache.get(key1), None) self.assertEqual(cache.get(key2), None)
def set_library_block_olx(usage_key, new_olx_str): """ Replace the OLX source of the given XBlock. This is only meant for use by developers or API client applications, as very little validation is done and this can easily result in a broken XBlock that won't load. """ # because this old pylint can't understand attr.ib() objects, pylint: disable=no-member assert isinstance(usage_key, LibraryUsageLocatorV2) # Make sure the block exists: metadata = get_library_block(usage_key) block_type = usage_key.block_type # Verify that the OLX parses, at least as generic XML: node = etree.fromstring(new_olx_str) if node.tag != block_type: raise ValueError( "Invalid root tag in OLX, expected {}".format(block_type)) # Write the new XML/OLX file into the library bundle's draft draft = get_or_create_bundle_draft(metadata.def_key.bundle_uuid, DRAFT_NAME) write_draft_file(draft.uuid, metadata.def_key.olx_path, new_olx_str.encode('utf-8')) # Clear the bundle cache so everyone sees the new block immediately: BundleCache(metadata.def_key.bundle_uuid, draft_name=DRAFT_NAME).clear() LIBRARY_BLOCK_UPDATED.send(sender=None, library_key=usage_key.context_key, usage_key=usage_key)
def save_block(self, block): """ Save any pending field data values to Blockstore. This gets called by block.save() - do not call this directly. """ if not self.system.authored_data_store.has_changes(block): return # No changes, so no action needed. definition_key = block.scope_ids.def_id if definition_key.draft_name is None: raise RuntimeError( "The Blockstore runtime does not support saving changes to blockstore without a draft. " "Are you making changes to UserScope.NONE fields from the LMS rather than Studio?" ) olx_str, static_files = serialize_xblock(block) # Write the OLX file to the bundle: draft_uuid = blockstore_api.get_or_create_bundle_draft( definition_key.bundle_uuid, definition_key.draft_name).uuid olx_path = definition_key.olx_path blockstore_api.write_draft_file(draft_uuid, olx_path, olx_str) # And the other files, if any: olx_static_path = os.path.dirname(olx_path) + '/static/' for fh in static_files: new_path = olx_static_path + fh.name blockstore_api.write_draft_file(draft_uuid, new_path, fh.data) # Now invalidate the blockstore data cache for the bundle: BundleCache(definition_key.bundle_uuid, draft_name=definition_key.draft_name).clear()
def create_library_block(library_key, block_type, definition_id): """ Create a new XBlock in this library of the specified type (e.g. "html"). The 'definition_id' value (which should be a string like "problem1") will be used as both the definition_id and the usage_id. """ assert isinstance(library_key, LibraryLocatorV2) ref = ContentLibrary.objects.get_by_key(library_key) # Make sure the proposed ID will be valid: validate_unicode_slug(definition_id) # Ensure the XBlock type is valid and installed: XBlock.load_class(block_type) # Will raise an exception if invalid # Make sure the new ID is not taken already: new_usage_id = definition_id # Since this is a top level XBlock, usage_id == definition_id usage_key = LibraryUsageLocatorV2( library_org=library_key.org, library_slug=library_key.slug, block_type=block_type, usage_id=new_usage_id, ) library_context = get_learning_context_impl(usage_key) if library_context.definition_for_usage(usage_key) is not None: raise LibraryBlockAlreadyExists("An XBlock with ID '{}' already exists".format(new_usage_id)) new_definition_xml = '<{}/>'.format(block_type) # xss-lint: disable=python-wrap-html path = "{}/{}/definition.xml".format(block_type, definition_id) # Write the new XML/OLX file into the library bundle's draft draft = get_or_create_bundle_draft(ref.bundle_uuid, DRAFT_NAME) write_draft_file(draft.uuid, path, new_definition_xml) # Clear the bundle cache so everyone sees the new block immediately: BundleCache(ref.bundle_uuid, draft_name=DRAFT_NAME).clear() # Now return the metadata about the new block: return get_library_block(usage_key)
def save_block(self, block): """ Save any pending field data values to Blockstore. This gets called by block.save() - do not call this directly. """ if not self.system.authored_data_store.has_changes(block): return # No changes, so no action needed. definition_key = block.scope_ids.def_id if definition_key.draft_name is None: raise RuntimeError( "The Blockstore runtime does not support saving changes to blockstore without a draft. " "Are you making changes to UserScope.NONE fields from the LMS rather than Studio?" ) # Verify that the user has permission to write to authored data in this # learning context: if self.user is not None: learning_context = get_learning_context_impl(block.scope_ids.usage_id) if not learning_context.can_edit_block(self.user, block.scope_ids.usage_id): log.warning("User %s does not have permission to edit %s", self.user.username, block.scope_ids.usage_id) raise RuntimeError("You do not have permission to edit this XBlock") olx_str, static_files = serialize_xblock(block) # Write the OLX file to the bundle: draft_uuid = blockstore_api.get_or_create_bundle_draft( definition_key.bundle_uuid, definition_key.draft_name ).uuid olx_path = definition_key.olx_path blockstore_api.write_draft_file(draft_uuid, olx_path, olx_str) # And the other files, if any: olx_static_path = os.path.dirname(olx_path) + '/static/' for fh in static_files: new_path = olx_static_path + fh.name blockstore_api.write_draft_file(draft_uuid, new_path, fh.data) # Now invalidate the blockstore data cache for the bundle: BundleCache(definition_key.bundle_uuid, draft_name=definition_key.draft_name).clear()
def create_library_block(library_key, block_type, definition_id): """ Create a new XBlock in this library of the specified type (e.g. "html"). The 'definition_id' value (which should be a string like "problem1") will be used as both the definition_id and the usage_id. """ assert isinstance(library_key, LibraryLocatorV2) ref = ContentLibrary.objects.get_by_key(library_key) if ref.type != COMPLEX: if block_type != ref.type: raise IncompatibleTypesError( _('Block type "{block_type}" is not compatible with library type "{library_type}".' ).format( block_type=block_type, library_type=ref.type, )) lib_bundle = LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME) # Total number of blocks should not exceed the maximum allowed total_blocks = len(lib_bundle.get_top_level_usages()) if total_blocks + 1 > settings.MAX_BLOCKS_PER_CONTENT_LIBRARY: raise BlockLimitReachedError( _(u"Library cannot have more than {} XBlocks").format( settings.MAX_BLOCKS_PER_CONTENT_LIBRARY)) # Make sure the proposed ID will be valid: validate_unicode_slug(definition_id) # Ensure the XBlock type is valid and installed: XBlock.load_class(block_type) # Will raise an exception if invalid # Make sure the new ID is not taken already: new_usage_id = definition_id # Since this is a top level XBlock, usage_id == definition_id usage_key = LibraryUsageLocatorV2( lib_key=library_key, block_type=block_type, usage_id=new_usage_id, ) library_context = get_learning_context_impl(usage_key) if library_context.definition_for_usage(usage_key) is not None: raise LibraryBlockAlreadyExists( "An XBlock with ID '{}' already exists".format(new_usage_id)) new_definition_xml = '<{}/>'.format( block_type) # xss-lint: disable=python-wrap-html path = "{}/{}/definition.xml".format(block_type, definition_id) # Write the new XML/OLX file into the library bundle's draft draft = get_or_create_bundle_draft(ref.bundle_uuid, DRAFT_NAME) write_draft_file(draft.uuid, path, new_definition_xml.encode('utf-8')) # Clear the bundle cache so everyone sees the new block immediately: BundleCache(ref.bundle_uuid, draft_name=DRAFT_NAME).clear() # Now return the metadata about the new block: LIBRARY_BLOCK_CREATED.send(sender=None, library_key=ref.library_key, usage_key=usage_key) return get_library_block(usage_key)
def test_bundle_cache_clear(self): """ Test the cache clear() method """ cache = BundleCache(self.bundle.uuid) key1 = ("some", "key", "1") value1 = "value1" cache.set(key1, value1) assert cache.get(key1) == value1 # Now publish a new version of the bundle: api.write_draft_file(self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version") api.commit_draft(self.draft.uuid) # Now the cache will not be immediately invalidated; it takes up to MAX_BLOCKSTORE_CACHE_DELAY seconds. # Since this is a new bundle and we _just_ accessed the cache for the first time, we can be confident # it won't yet be automatically invalidated. assert cache.get(key1) == value1 # Now "clear" the cache, forcing the check of the new version: cache.clear() assert cache.get(key1) is None
class LibraryBundle(object): """ Wrapper around a Content Library Blockstore bundle that contains OLX. """ def __init__(self, library_key, bundle_uuid, draft_name=None): """ Instantiate this wrapper for the bundle with the specified library_key, UUID, and optionally the specified draft name. """ self.library_key = library_key self.bundle_uuid = bundle_uuid self.draft_name = draft_name self.cache = BundleCache(bundle_uuid, draft_name) def get_olx_files(self): """ Get the list of OLX files in this bundle (using a heuristic) Because this uses a heuristic, it will only return files with filenames that seem like OLX files that are in the expected locations of OLX files. They are not guaranteed to be valid OLX nor will OLX files in nonstandard locations be returned. Example return value: [ 'html/intro/definition.xml', 'unit/unit1/definition.xml', ] """ bundle_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) return [ f.path for f in bundle_files if f.path.endswith("/definition.xml") ] def definition_for_usage(self, usage_key): """ Given the usage key for an XBlock in this library bundle, return the BundleDefinitionLocator which specifies the actual XBlock definition (as a path to an OLX in a specific blockstore bundle). Must return a BundleDefinitionLocator if the XBlock exists in this context, or None otherwise. For a content library, the rules are simple: * If the usage key points to a block in this library, the filename (definition) of the OLX file is always {block_type}/{usage_id}/definition.xml Each library has exactly one usage per definition for its own blocks. * However, block definitions from other content libraries may be linked into this library via <xblock-include ... /> directives. In that case, it's necessary to inspect every OLX file in this library that might have an <xblock-include /> directive in order to find what external block the usage ID refers to. """ # Now that we know the library/bundle, find the block's definition if self.draft_name: version_arg = {"draft_name": self.draft_name} else: version_arg = { "bundle_version": get_bundle_version_number(self.bundle_uuid) } olx_path = "{}/{}/definition.xml".format(usage_key.block_type, usage_key.usage_id) try: get_bundle_file_metadata_with_cache(self.bundle_uuid, olx_path, **version_arg) return BundleDefinitionLocator(self.bundle_uuid, usage_key.block_type, olx_path, **version_arg) except blockstore_api.BundleFileNotFound: # This must be a usage of a block from a linked bundle. One of the # OLX files in this bundle contains an <xblock-include usage="..."/> bundle_includes = self.get_bundle_includes() try: return bundle_includes[usage_key] except KeyError: return None def get_top_level_usages(self): """ Get the set of usage keys in this bundle that have no parent. """ own_usage_keys = [] for olx_file_path in self.get_olx_files(): block_type, usage_id, _unused = olx_file_path.split('/') usage_key = LibraryUsageLocatorV2(self.library_key, block_type, usage_id) own_usage_keys.append(usage_key) usage_keys_with_parents = self.get_bundle_includes().keys() return [ usage_key for usage_key in own_usage_keys if usage_key not in usage_keys_with_parents ] def get_bundle_includes(self): """ Scan through the bundle and all linked bundles as needed to generate a complete list of all the blocks that are included as child/grandchild/... blocks of the blocks in this bundle. Returns a dict of {usage_key -> BundleDefinitionLocator} Blocks in the bundle that have no parent are not included. """ cache_key = ("bundle_includes", ) usages_found = self.cache.get(cache_key) if usages_found is not None: return usages_found usages_found = {} def add_definitions_children(usage_key, def_key): """ Recursively add any children of the given XBlock usage+definition to usages_found. """ if not does_block_type_support_children(def_key.block_type): return try: xml_node = xml_for_definition(def_key) except: # pylint:disable=bare-except log.exception("Unable to load definition {}".format(def_key)) return for child in xml_node: if child.tag != 'xblock-include': continue try: parsed_include = parse_xblock_include(child) child_usage = usage_for_child_include( usage_key, def_key, parsed_include) child_def_key = definition_for_include( parsed_include, def_key) except BundleFormatException: log.exception( "Unable to parse a child of {}".format(def_key)) continue usages_found[child_usage] = child_def_key add_definitions_children(child_usage, child_def_key) # Find all the definitions in this bundle and recursively add all their descendants: bundle_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) if self.draft_name: version_arg = {"draft_name": self.draft_name} else: version_arg = { "bundle_version": get_bundle_version_number(self.bundle_uuid) } for bfile in bundle_files: if not bfile.path.endswith( "/definition.xml") or bfile.path.count('/') != 2: continue # Not an OLX file. block_type, usage_id, _unused = bfile.path.split('/') def_key = BundleDefinitionLocator(bundle_uuid=self.bundle_uuid, block_type=block_type, olx_path=bfile.path, **version_arg) usage_key = LibraryUsageLocatorV2(self.library_key, block_type, usage_id) add_definitions_children(usage_key, def_key) self.cache.set(cache_key, usages_found) return usages_found def does_definition_have_unpublished_changes(self, definition_key): """ Given the defnition key of an XBlock, which exists in an OLX file like problem/quiz1/definition.xml Check if the bundle's draft has _any_ unpublished changes in the problem/quiz1/ directory. """ if self.draft_name is None: return False # No active draft so can't be changes prefix = self.olx_prefix(definition_key) return prefix in self._get_changed_definitions() def _get_changed_definitions(self): """ Helper method to get a list of all paths with changes, where a path is problem/quiz1/ Or similar (a type and an ID), excluding 'definition.xml' """ cached_result = self.cache.get(('changed_definition_prefixes', )) if cached_result is not None: return cached_result changed = [] bundle_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) for file_ in bundle_files: if getattr(file_, 'modified', False) and file_.path.count('/') >= 2: (type_part, id_part, _rest) = file_.path.split('/', 2) prefix = type_part + '/' + id_part + '/' if prefix not in changed: changed.append(prefix) self.cache.set(('changed_definition_prefixes', ), changed) return changed def has_changes(self): """ Helper method to check if this OLX bundle has any pending changes, including any deleted blocks. Returns a tuple of ( has_unpublished_changes, has_unpublished_deletes, ) Where has_unpublished_changes is true if there is any type of change, including deletes, and has_unpublished_deletes is only true if one or more blocks has been deleted since the last publish. """ if not self.draft_name: return (False, False) cached_result = self.cache.get(('has_changes', )) if cached_result is not None: return cached_result draft_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) has_unpublished_changes = False has_unpublished_deletes = False for file_ in draft_files: if getattr(file_, 'modified', False): has_unpublished_changes = True break published_file_paths = set( f.path for f in get_bundle_files_cached(self.bundle_uuid)) draft_file_paths = set(f.path for f in draft_files) for file_path in published_file_paths: if file_path not in draft_file_paths: has_unpublished_changes = True if file_path.endswith('/definition.xml'): # only set 'has_unpublished_deletes' if the actual main definition XML # file was deleted, not if only some asset file was deleted, etc. has_unpublished_deletes = True break result = (has_unpublished_changes, has_unpublished_deletes) self.cache.set(('has_changes', ), result) return result def get_static_prefix_for_definition(self, definition_key): """ Given a definition key, get the path prefix used for all (public) static asset files. Example: problem/quiz1/static/ """ return self.olx_prefix(definition_key) + 'static/' def get_static_files_for_definition(self, definition_key): """ Return a list of the static asset files related with a particular XBlock definition. If the bundle contains files like: problem/quiz1/definition.xml problem/quiz1/static/image1.png Then this will return [BundleFile(path="image1.png", size, url, hash_digest)] """ path_prefix = self.get_static_prefix_for_definition(definition_key) path_prefix_len = len(path_prefix) return [ blockstore_api.BundleFile(path=f.path[path_prefix_len:], size=f.size, url=f.url, hash_digest=f.hash_digest) for f in get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) if f.path.startswith(path_prefix) ] @staticmethod def olx_prefix(definition_key): """ Given a definition key in a compatible bundle, whose olx_path refers to block_type/some_id/definition.xml Return the "folder name" / "path prefix" block-type/some_id/ This method is here rather than a method of BundleDefinitionLocator because BundleDefinitionLocator is more generic and doesn't require that its olx_path always ends in /definition.xml """ if not definition_key.olx_path.endswith('/definition.xml'): raise ValueError return definition_key.olx_path[: -14] # Remove 'definition.xml', keep trailing slash