def test_bundle_draft_cache(self): """ Test caching data related to a bundle draft """ cache = BundleCache(self.bundle.uuid, draft_name=self.draft.name) key1 = ("some", "key", "1") key2 = ("key2", ) value1 = "value1" cache.set(key1, value1) value2 = {"this is": "a dict", "for": "key2"} cache.set(key2, value2) self.assertEqual(cache.get(key1), value1) self.assertEqual(cache.get(key2), value2) # Now make a change to the draft (doesn't matter if we commit it or not) api.write_draft_file( self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version") # Now the cache should be invalidated # (immediately since we set MAX_BLOCKSTORE_CACHE_DELAY to 0) self.assertEqual(cache.get(key1), None) self.assertEqual(cache.get(key2), None)
def test_bundle_cache(self): """ Test caching data related to a bundle (no draft) """ cache = BundleCache(self.bundle.uuid) key1 = ("some", "key", "1") key2 = ("key2", ) value1 = "value1" cache.set(key1, value1) value2 = {"this is": "a dict", "for": "key2"} cache.set(key2, value2) self.assertEqual(cache.get(key1), value1) self.assertEqual(cache.get(key2), value2) # Now publish a new version of the bundle: api.write_draft_file( self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version") api.commit_draft(self.draft.uuid) # Now the cache should be invalidated # (immediately since we set MAX_BLOCKSTORE_CACHE_DELAY to 0) self.assertEqual(cache.get(key1), None) self.assertEqual(cache.get(key2), None)
def get_block_display_name(block_or_key): """ Efficiently get the display name of the specified block. This is done in a way that avoids having to load and parse the block's entire XML field data using its parse_xml() method, which may be very expensive (e.g. the video XBlock parse_xml leads to various slow edxval API calls in some cases). This method also defines and implements various fallback mechanisms in case the ID can't be loaded. block_or_key can be an XBlock instance, a usage key or a definition key. Returns the display name as a string """ def_key = resolve_definition(block_or_key) use_draft = get_xblock_app_config().get_learning_context_params().get('use_draft') cache = BundleCache(def_key.bundle_uuid, draft_name=use_draft) cache_key = ('block_display_name', str(def_key)) display_name = cache.get(cache_key) if display_name is None: # Instead of loading the block, just load its XML and parse it try: olx_node = xml_for_definition(def_key) except Exception: # pylint: disable=broad-except log.exception("Error when trying to get display_name for block definition %s", def_key) # Return now so we don't cache the error result return xblock_type_display_name(def_key.block_type) try: display_name = olx_node.attrib['display_name'] except KeyError: display_name = xblock_type_display_name(def_key.block_type) cache.set(cache_key, display_name) return display_name
def test_bundle_cache_clear(self): """ Test the cache clear() method """ cache = BundleCache(self.bundle.uuid) key1 = ("some", "key", "1") value1 = "value1" cache.set(key1, value1) assert cache.get(key1) == value1 # Now publish a new version of the bundle: api.write_draft_file(self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version") api.commit_draft(self.draft.uuid) # Now the cache will not be immediately invalidated; it takes up to MAX_BLOCKSTORE_CACHE_DELAY seconds. # Since this is a new bundle and we _just_ accessed the cache for the first time, we can be confident # it won't yet be automatically invalidated. assert cache.get(key1) == value1 # Now "clear" the cache, forcing the check of the new version: cache.clear() assert cache.get(key1) is None
class LibraryBundle(object): """ Wrapper around a Content Library Blockstore bundle that contains OLX. """ def __init__(self, library_key, bundle_uuid, draft_name=None): """ Instantiate this wrapper for the bundle with the specified library_key, UUID, and optionally the specified draft name. """ self.library_key = library_key self.bundle_uuid = bundle_uuid self.draft_name = draft_name self.cache = BundleCache(bundle_uuid, draft_name) def get_olx_files(self): """ Get the list of OLX files in this bundle (using a heuristic) Because this uses a heuristic, it will only return files with filenames that seem like OLX files that are in the expected locations of OLX files. They are not guaranteed to be valid OLX nor will OLX files in nonstandard locations be returned. Example return value: [ 'html/intro/definition.xml', 'unit/unit1/definition.xml', ] """ bundle_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) return [ f.path for f in bundle_files if f.path.endswith("/definition.xml") ] def definition_for_usage(self, usage_key): """ Given the usage key for an XBlock in this library bundle, return the BundleDefinitionLocator which specifies the actual XBlock definition (as a path to an OLX in a specific blockstore bundle). Must return a BundleDefinitionLocator if the XBlock exists in this context, or None otherwise. For a content library, the rules are simple: * If the usage key points to a block in this library, the filename (definition) of the OLX file is always {block_type}/{usage_id}/definition.xml Each library has exactly one usage per definition for its own blocks. * However, block definitions from other content libraries may be linked into this library via <xblock-include ... /> directives. In that case, it's necessary to inspect every OLX file in this library that might have an <xblock-include /> directive in order to find what external block the usage ID refers to. """ # Now that we know the library/bundle, find the block's definition if self.draft_name: version_arg = {"draft_name": self.draft_name} else: version_arg = { "bundle_version": get_bundle_version_number(self.bundle_uuid) } olx_path = "{}/{}/definition.xml".format(usage_key.block_type, usage_key.usage_id) try: get_bundle_file_metadata_with_cache(self.bundle_uuid, olx_path, **version_arg) return BundleDefinitionLocator(self.bundle_uuid, usage_key.block_type, olx_path, **version_arg) except blockstore_api.BundleFileNotFound: # This must be a usage of a block from a linked bundle. One of the # OLX files in this bundle contains an <xblock-include usage="..."/> bundle_includes = self.get_bundle_includes() try: return bundle_includes[usage_key] except KeyError: return None def get_top_level_usages(self): """ Get the set of usage keys in this bundle that have no parent. """ own_usage_keys = [] for olx_file_path in self.get_olx_files(): block_type, usage_id, _unused = olx_file_path.split('/') usage_key = LibraryUsageLocatorV2(self.library_key, block_type, usage_id) own_usage_keys.append(usage_key) usage_keys_with_parents = self.get_bundle_includes().keys() return [ usage_key for usage_key in own_usage_keys if usage_key not in usage_keys_with_parents ] def get_bundle_includes(self): """ Scan through the bundle and all linked bundles as needed to generate a complete list of all the blocks that are included as child/grandchild/... blocks of the blocks in this bundle. Returns a dict of {usage_key -> BundleDefinitionLocator} Blocks in the bundle that have no parent are not included. """ cache_key = ("bundle_includes", ) usages_found = self.cache.get(cache_key) if usages_found is not None: return usages_found usages_found = {} def add_definitions_children(usage_key, def_key): """ Recursively add any children of the given XBlock usage+definition to usages_found. """ if not does_block_type_support_children(def_key.block_type): return try: xml_node = xml_for_definition(def_key) except: # pylint:disable=bare-except log.exception("Unable to load definition {}".format(def_key)) return for child in xml_node: if child.tag != 'xblock-include': continue try: parsed_include = parse_xblock_include(child) child_usage = usage_for_child_include( usage_key, def_key, parsed_include) child_def_key = definition_for_include( parsed_include, def_key) except BundleFormatException: log.exception( "Unable to parse a child of {}".format(def_key)) continue usages_found[child_usage] = child_def_key add_definitions_children(child_usage, child_def_key) # Find all the definitions in this bundle and recursively add all their descendants: bundle_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) if self.draft_name: version_arg = {"draft_name": self.draft_name} else: version_arg = { "bundle_version": get_bundle_version_number(self.bundle_uuid) } for bfile in bundle_files: if not bfile.path.endswith( "/definition.xml") or bfile.path.count('/') != 2: continue # Not an OLX file. block_type, usage_id, _unused = bfile.path.split('/') def_key = BundleDefinitionLocator(bundle_uuid=self.bundle_uuid, block_type=block_type, olx_path=bfile.path, **version_arg) usage_key = LibraryUsageLocatorV2(self.library_key, block_type, usage_id) add_definitions_children(usage_key, def_key) self.cache.set(cache_key, usages_found) return usages_found def does_definition_have_unpublished_changes(self, definition_key): """ Given the defnition key of an XBlock, which exists in an OLX file like problem/quiz1/definition.xml Check if the bundle's draft has _any_ unpublished changes in the problem/quiz1/ directory. """ if self.draft_name is None: return False # No active draft so can't be changes prefix = self.olx_prefix(definition_key) return prefix in self._get_changed_definitions() def _get_changed_definitions(self): """ Helper method to get a list of all paths with changes, where a path is problem/quiz1/ Or similar (a type and an ID), excluding 'definition.xml' """ cached_result = self.cache.get(('changed_definition_prefixes', )) if cached_result is not None: return cached_result changed = [] bundle_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) for file_ in bundle_files: if getattr(file_, 'modified', False) and file_.path.count('/') >= 2: (type_part, id_part, _rest) = file_.path.split('/', 2) prefix = type_part + '/' + id_part + '/' if prefix not in changed: changed.append(prefix) self.cache.set(('changed_definition_prefixes', ), changed) return changed def has_changes(self): """ Helper method to check if this OLX bundle has any pending changes, including any deleted blocks. Returns a tuple of ( has_unpublished_changes, has_unpublished_deletes, ) Where has_unpublished_changes is true if there is any type of change, including deletes, and has_unpublished_deletes is only true if one or more blocks has been deleted since the last publish. """ if not self.draft_name: return (False, False) cached_result = self.cache.get(('has_changes', )) if cached_result is not None: return cached_result draft_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) has_unpublished_changes = False has_unpublished_deletes = False for file_ in draft_files: if getattr(file_, 'modified', False): has_unpublished_changes = True break published_file_paths = set( f.path for f in get_bundle_files_cached(self.bundle_uuid)) draft_file_paths = set(f.path for f in draft_files) for file_path in published_file_paths: if file_path not in draft_file_paths: has_unpublished_changes = True if file_path.endswith('/definition.xml'): # only set 'has_unpublished_deletes' if the actual main definition XML # file was deleted, not if only some asset file was deleted, etc. has_unpublished_deletes = True break result = (has_unpublished_changes, has_unpublished_deletes) self.cache.set(('has_changes', ), result) return result def get_static_prefix_for_definition(self, definition_key): """ Given a definition key, get the path prefix used for all (public) static asset files. Example: problem/quiz1/static/ """ return self.olx_prefix(definition_key) + 'static/' def get_static_files_for_definition(self, definition_key): """ Return a list of the static asset files related with a particular XBlock definition. If the bundle contains files like: problem/quiz1/definition.xml problem/quiz1/static/image1.png Then this will return [BundleFile(path="image1.png", size, url, hash_digest)] """ path_prefix = self.get_static_prefix_for_definition(definition_key) path_prefix_len = len(path_prefix) return [ blockstore_api.BundleFile(path=f.path[path_prefix_len:], size=f.size, url=f.url, hash_digest=f.hash_digest) for f in get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name) if f.path.startswith(path_prefix) ] @staticmethod def olx_prefix(definition_key): """ Given a definition key in a compatible bundle, whose olx_path refers to block_type/some_id/definition.xml Return the "folder name" / "path prefix" block-type/some_id/ This method is here rather than a method of BundleDefinitionLocator because BundleDefinitionLocator is more generic and doesn't require that its olx_path always ends in /definition.xml """ if not definition_key.olx_path.endswith('/definition.xml'): raise ValueError return definition_key.olx_path[: -14] # Remove 'definition.xml', keep trailing slash