def test_bundle_draft_cache(self):
"""
Test caching data related to a bundle draft
"""
cache = BundleCache(self.bundle.uuid, draft_name=self.draft.name)
key1 = ("some", "key", "1")
key2 = ("key2", )
value1 = "value1"
cache.set(key1, value1)
value2 = {"this is": "a dict", "for": "key2"}
cache.set(key2, value2)
self.assertEqual(cache.get(key1), value1)
self.assertEqual(cache.get(key2), value2)
# Now make a change to the draft (doesn't matter if we commit it or not)
api.write_draft_file(
self.draft.uuid, "test.txt",
"we need a changed file in order to publish a new version")
# Now the cache should be invalidated
# (immediately since we set MAX_BLOCKSTORE_CACHE_DELAY to 0)
self.assertEqual(cache.get(key1), None)
self.assertEqual(cache.get(key2), None)
def set_library_block_olx(usage_key, new_olx_str):
"""
Replace the OLX source of the given XBlock.
This is only meant for use by developers or API client applications, as
very little validation is done and this can easily result in a broken XBlock
that won't load.
"""
# because this old pylint can't understand attr.ib() objects, pylint: disable=no-member
assert isinstance(usage_key, LibraryUsageLocatorV2)
# Make sure the block exists:
metadata = get_library_block(usage_key)
block_type = usage_key.block_type
# Verify that the OLX parses, at least as generic XML:
node = etree.fromstring(new_olx_str)
if node.tag != block_type:
raise ValueError(
"Invalid root tag in OLX, expected {}".format(block_type))
# Write the new XML/OLX file into the library bundle's draft
draft = get_or_create_bundle_draft(metadata.def_key.bundle_uuid,
DRAFT_NAME)
write_draft_file(draft.uuid, metadata.def_key.olx_path,
new_olx_str.encode('utf-8'))
# Clear the bundle cache so everyone sees the new block immediately:
BundleCache(metadata.def_key.bundle_uuid, draft_name=DRAFT_NAME).clear()
LIBRARY_BLOCK_UPDATED.send(sender=None,
library_key=usage_key.context_key,
usage_key=usage_key)
def test_bundle_cache(self):
"""
Test caching data related to a bundle (no draft)
"""
cache = BundleCache(self.bundle.uuid)
key1 = ("some", "key", "1")
key2 = ("key2", )
value1 = "value1"
cache.set(key1, value1)
value2 = {"this is": "a dict", "for": "key2"}
cache.set(key2, value2)
self.assertEqual(cache.get(key1), value1)
self.assertEqual(cache.get(key2), value2)
# Now publish a new version of the bundle:
api.write_draft_file(
self.draft.uuid, "test.txt",
"we need a changed file in order to publish a new version")
api.commit_draft(self.draft.uuid)
# Now the cache should be invalidated
# (immediately since we set MAX_BLOCKSTORE_CACHE_DELAY to 0)
self.assertEqual(cache.get(key1), None)
self.assertEqual(cache.get(key2), None)
def save_block(self, block):
"""
Save any pending field data values to Blockstore.
This gets called by block.save() - do not call this directly.
"""
if not self.system.authored_data_store.has_changes(block):
return # No changes, so no action needed.
definition_key = block.scope_ids.def_id
if definition_key.draft_name is None:
raise RuntimeError(
"The Blockstore runtime does not support saving changes to blockstore without a draft. "
"Are you making changes to UserScope.NONE fields from the LMS rather than Studio?"
)
olx_str, static_files = serialize_xblock(block)
# Write the OLX file to the bundle:
draft_uuid = blockstore_api.get_or_create_bundle_draft(
definition_key.bundle_uuid, definition_key.draft_name).uuid
olx_path = definition_key.olx_path
blockstore_api.write_draft_file(draft_uuid, olx_path, olx_str)
# And the other files, if any:
olx_static_path = os.path.dirname(olx_path) + '/static/'
for fh in static_files:
new_path = olx_static_path + fh.name
blockstore_api.write_draft_file(draft_uuid, new_path, fh.data)
# Now invalidate the blockstore data cache for the bundle:
BundleCache(definition_key.bundle_uuid,
draft_name=definition_key.draft_name).clear()
def add_library_block_static_asset_file(usage_key, file_name, file_content):
"""
Upload a static asset file into the library, to be associated with the
specified XBlock. Will silently overwrite an existing file of the same name.
file_name should be a name like "doc.pdf". It may optionally contain slashes
like 'en/doc.pdf'
file_content should be a binary string.
Returns a LibraryXBlockStaticFile object.
Example:
video_block = UsageKey.from_string("lb:VideoTeam:python-intro:video:1")
add_library_block_static_asset_file(video_block, "subtitles-en.srt", subtitles.encode('utf-8'))
"""
assert isinstance(file_content, six.binary_type)
def_key, lib_bundle = _lookup_usage_key(usage_key)
if file_name != file_name.strip().strip('/'):
raise InvalidNameError("file name cannot start/end with / or whitespace.")
if '//' in file_name or '..' in file_name:
raise InvalidNameError("Invalid sequence (// or ..) in filename.")
file_path = lib_bundle.get_static_prefix_for_definition(def_key) + file_name
# Write the new static file into the library bundle's draft
draft = get_or_create_bundle_draft(def_key.bundle_uuid, DRAFT_NAME)
write_draft_file(draft.uuid, file_path, file_content)
# Clear the bundle cache so everyone sees the new file immediately:
lib_bundle.cache.clear()
file_metadata = blockstore_cache.get_bundle_file_metadata_with_cache(
bundle_uuid=def_key.bundle_uuid, path=file_path, draft_name=DRAFT_NAME,
)
LIBRARY_BLOCK_UPDATED.send(sender=None, library_key=lib_bundle.library_key, usage_key=usage_key)
return LibraryXBlockStaticFile(path=file_metadata.path, url=file_metadata.url, size=file_metadata.size)
def create_library_block(library_key, block_type, definition_id):
"""
Create a new XBlock in this library of the specified type (e.g. "html").
The 'definition_id' value (which should be a string like "problem1") will be
used as both the definition_id and the usage_id.
"""
assert isinstance(library_key, LibraryLocatorV2)
ref = ContentLibrary.objects.get_by_key(library_key)
# Make sure the proposed ID will be valid:
validate_unicode_slug(definition_id)
# Ensure the XBlock type is valid and installed:
XBlock.load_class(block_type) # Will raise an exception if invalid
# Make sure the new ID is not taken already:
new_usage_id = definition_id # Since this is a top level XBlock, usage_id == definition_id
usage_key = LibraryUsageLocatorV2(
library_org=library_key.org,
library_slug=library_key.slug,
block_type=block_type,
usage_id=new_usage_id,
)
library_context = get_learning_context_impl(usage_key)
if library_context.definition_for_usage(usage_key) is not None:
raise LibraryBlockAlreadyExists("An XBlock with ID '{}' already exists".format(new_usage_id))
new_definition_xml = '<{}/>'.format(block_type) # xss-lint: disable=python-wrap-html
path = "{}/{}/definition.xml".format(block_type, definition_id)
# Write the new XML/OLX file into the library bundle's draft
draft = get_or_create_bundle_draft(ref.bundle_uuid, DRAFT_NAME)
write_draft_file(draft.uuid, path, new_definition_xml)
# Clear the bundle cache so everyone sees the new block immediately:
BundleCache(ref.bundle_uuid, draft_name=DRAFT_NAME).clear()
# Now return the metadata about the new block:
return get_library_block(usage_key)
def delete_library_block(usage_key, remove_from_parent=True):
"""
Delete the specified block from this library (and any children it has).
If the block's definition (OLX file) is within this same library as the
usage key, both the definition and the usage will be deleted.
If the usage points to a definition in a linked bundle, the usage will be
deleted but the link and the linked bundle will be unaffected.
If the block is in use by some other bundle that links to this one, that
will not prevent deletion of the definition.
remove_from_parent: modify the parent to remove the reference to this
delete block. This should always be true except when this function
calls itself recursively.
"""
assert isinstance(usage_key, LibraryUsageLocatorV2)
library_context = get_learning_context_impl(usage_key)
library_ref = ContentLibrary.objects.get_by_key(usage_key.context_key)
def_key = library_context.definition_for_usage(usage_key)
if def_key is None:
raise ContentLibraryBlockNotFound(usage_key)
lib_bundle = LibraryBundle(usage_key.context_key,
library_ref.bundle_uuid,
draft_name=DRAFT_NAME)
# Create a draft:
draft_uuid = get_or_create_bundle_draft(def_key.bundle_uuid,
DRAFT_NAME).uuid
# Does this block have a parent?
if usage_key not in lib_bundle.get_top_level_usages(
) and remove_from_parent:
# Yes: this is not a top-level block.
# First need to modify the parent to remove this block as a child.
raise NotImplementedError
# Does this block have children?
block = load_block(usage_key, user=None)
if block.has_children:
# Next, recursively call delete_library_block(...) on each child usage
for child_usage in block.children:
# Specify remove_from_parent=False to avoid unnecessary work to
# modify this block's children list when deleting each child, since
# we're going to delete this block anyways.
delete_library_block(child_usage, remove_from_parent=False)
# Delete the definition:
if def_key.bundle_uuid == library_ref.bundle_uuid:
# This definition is in the library, so delete it:
path_prefix = lib_bundle.olx_prefix(def_key)
for bundle_file in get_bundle_files(def_key.bundle_uuid,
use_draft=DRAFT_NAME):
if bundle_file.path.startswith(path_prefix):
# Delete this file, within this definition's "folder"
write_draft_file(draft_uuid, bundle_file.path, contents=None)
else:
# The definition must be in a linked bundle, so we don't want to delete
# it; just the <xblock-include /> in the parent, which was already
# deleted above.
pass
# Clear the bundle cache so everyone sees the deleted block immediately:
lib_bundle.cache.clear()
def save_block(self, block):
"""
Save any pending field data values to Blockstore.
This gets called by block.save() - do not call this directly.
"""
if not self.system.authored_data_store.has_changes(block):
return # No changes, so no action needed.
definition_key = block.scope_ids.def_id
if definition_key.draft_name is None:
raise RuntimeError(
"The Blockstore runtime does not support saving changes to blockstore without a draft. "
"Are you making changes to UserScope.NONE fields from the LMS rather than Studio?"
)
# Verify that the user has permission to write to authored data in this
# learning context:
if self.user is not None:
learning_context = get_learning_context_impl(block.scope_ids.usage_id)
if not learning_context.can_edit_block(self.user, block.scope_ids.usage_id):
log.warning("User %s does not have permission to edit %s", self.user.username, block.scope_ids.usage_id)
raise RuntimeError("You do not have permission to edit this XBlock")
olx_str, static_files = serialize_xblock(block)
# Write the OLX file to the bundle:
draft_uuid = blockstore_api.get_or_create_bundle_draft(
definition_key.bundle_uuid, definition_key.draft_name
).uuid
olx_path = definition_key.olx_path
blockstore_api.write_draft_file(draft_uuid, olx_path, olx_str)
# And the other files, if any:
olx_static_path = os.path.dirname(olx_path) + '/static/'
for fh in static_files:
new_path = olx_static_path + fh.name
blockstore_api.write_draft_file(draft_uuid, new_path, fh.data)
# Now invalidate the blockstore data cache for the bundle:
BundleCache(definition_key.bundle_uuid, draft_name=definition_key.draft_name).clear()
def test_drafts_and_files(self):
"""
Test creating, reading, writing, committing, and reverting drafts and
files.
"""
coll = api.create_collection("Test Collection")
bundle = api.create_bundle(coll.uuid,
title="Earth 🗿 Bundle",
slug="earth",
description="another test bundle")
# Create a draft
draft = api.get_or_create_bundle_draft(bundle.uuid,
draft_name="test-draft")
self.assertEqual(draft.bundle_uuid, bundle.uuid)
self.assertEqual(draft.name, "test-draft")
self.assertGreaterEqual(draft.updated_at.year, 2019)
# And retrieve it again:
draft2 = api.get_or_create_bundle_draft(bundle.uuid,
draft_name="test-draft")
self.assertEqual(draft, draft2)
# Also test retrieving using get_draft
draft3 = api.get_draft(draft.uuid)
self.assertEqual(draft, draft3)
# Write a file into the bundle:
api.write_draft_file(draft.uuid, "test.txt", "initial version")
# Now the file should be visible in the draft:
draft_contents = api.get_bundle_file_data(bundle.uuid,
"test.txt",
use_draft=draft.name)
self.assertEqual(draft_contents, "initial version")
api.commit_draft(draft.uuid)
# Write a new version into the draft:
api.write_draft_file(draft.uuid, "test.txt", "modified version")
published_contents = api.get_bundle_file_data(bundle.uuid, "test.txt")
self.assertEqual(published_contents, "initial version")
draft_contents2 = api.get_bundle_file_data(bundle.uuid,
"test.txt",
use_draft=draft.name)
self.assertEqual(draft_contents2, "modified version")
# Now delete the draft:
api.delete_draft(draft.uuid)
draft_contents3 = api.get_bundle_file_data(bundle.uuid,
"test.txt",
use_draft=draft.name)
# Confirm the file is now reset:
self.assertEqual(draft_contents3, "initial version")
# Finaly, test the get_bundle_file* methods:
file_info1 = api.get_bundle_file_metadata(bundle.uuid, "test.txt")
self.assertEqual(file_info1.path, "test.txt")
self.assertEqual(file_info1.size, len("initial version"))
self.assertEqual(file_info1.hash_digest,
"a45a5c6716276a66c4005534a51453ab16ea63c4")
self.assertEqual(api.get_bundle_files(bundle.uuid), [file_info1])
self.assertEqual(api.get_bundle_files_dict(bundle.uuid), {
"test.txt": file_info1,
})
def test_drafts_and_files(self):
"""
Test creating, reading, writing, committing, and reverting drafts and
files.
"""
coll = api.create_collection("Test Collection")
bundle = api.create_bundle(coll.uuid,
title="Earth 🗿 Bundle",
slug="earth",
description="another test bundle")
# Create a draft
draft = api.get_or_create_bundle_draft(bundle.uuid,
draft_name="test-draft")
assert draft.bundle_uuid == bundle.uuid
assert draft.name == 'test-draft'
assert draft.updated_at.year >= 2019
# And retrieve it again:
draft2 = api.get_or_create_bundle_draft(bundle.uuid,
draft_name="test-draft")
assert draft == draft2
# Also test retrieving using get_draft
draft3 = api.get_draft(draft.uuid)
assert draft == draft3
# Write a file into the bundle:
api.write_draft_file(draft.uuid, "test.txt", b"initial version")
# Now the file should be visible in the draft:
draft_contents = api.get_bundle_file_data(bundle.uuid,
"test.txt",
use_draft=draft.name)
assert draft_contents == b'initial version'
api.commit_draft(draft.uuid)
# Write a new version into the draft:
api.write_draft_file(draft.uuid, "test.txt", b"modified version")
published_contents = api.get_bundle_file_data(bundle.uuid, "test.txt")
assert published_contents == b'initial version'
draft_contents2 = api.get_bundle_file_data(bundle.uuid,
"test.txt",
use_draft=draft.name)
assert draft_contents2 == b'modified version'
# Now delete the draft:
api.delete_draft(draft.uuid)
draft_contents3 = api.get_bundle_file_data(bundle.uuid,
"test.txt",
use_draft=draft.name)
# Confirm the file is now reset:
assert draft_contents3 == b'initial version'
# Finaly, test the get_bundle_file* methods:
file_info1 = api.get_bundle_file_metadata(bundle.uuid, "test.txt")
assert file_info1.path == 'test.txt'
assert file_info1.size == len(b'initial version')
assert file_info1.hash_digest == 'a45a5c6716276a66c4005534a51453ab16ea63c4'
assert list(api.get_bundle_files(bundle.uuid)) == [file_info1]
assert api.get_bundle_files_dict(bundle.uuid) == {
'test.txt': file_info1
}
def create_library_block(library_key, block_type, definition_id):
"""
Create a new XBlock in this library of the specified type (e.g. "html").
The 'definition_id' value (which should be a string like "problem1") will be
used as both the definition_id and the usage_id.
"""
assert isinstance(library_key, LibraryLocatorV2)
ref = ContentLibrary.objects.get_by_key(library_key)
if ref.type != COMPLEX:
if block_type != ref.type:
raise IncompatibleTypesError(
_('Block type "{block_type}" is not compatible with library type "{library_type}".'
).format(
block_type=block_type,
library_type=ref.type,
))
lib_bundle = LibraryBundle(library_key,
ref.bundle_uuid,
draft_name=DRAFT_NAME)
# Total number of blocks should not exceed the maximum allowed
total_blocks = len(lib_bundle.get_top_level_usages())
if total_blocks + 1 > settings.MAX_BLOCKS_PER_CONTENT_LIBRARY:
raise BlockLimitReachedError(
_(u"Library cannot have more than {} XBlocks").format(
settings.MAX_BLOCKS_PER_CONTENT_LIBRARY))
# Make sure the proposed ID will be valid:
validate_unicode_slug(definition_id)
# Ensure the XBlock type is valid and installed:
XBlock.load_class(block_type) # Will raise an exception if invalid
# Make sure the new ID is not taken already:
new_usage_id = definition_id # Since this is a top level XBlock, usage_id == definition_id
usage_key = LibraryUsageLocatorV2(
lib_key=library_key,
block_type=block_type,
usage_id=new_usage_id,
)
library_context = get_learning_context_impl(usage_key)
if library_context.definition_for_usage(usage_key) is not None:
raise LibraryBlockAlreadyExists(
"An XBlock with ID '{}' already exists".format(new_usage_id))
new_definition_xml = '<{}/>'.format(
block_type) # xss-lint: disable=python-wrap-html
path = "{}/{}/definition.xml".format(block_type, definition_id)
# Write the new XML/OLX file into the library bundle's draft
draft = get_or_create_bundle_draft(ref.bundle_uuid, DRAFT_NAME)
write_draft_file(draft.uuid, path, new_definition_xml.encode('utf-8'))
# Clear the bundle cache so everyone sees the new block immediately:
BundleCache(ref.bundle_uuid, draft_name=DRAFT_NAME).clear()
# Now return the metadata about the new block:
LIBRARY_BLOCK_CREATED.send(sender=None,
library_key=ref.library_key,
usage_key=usage_key)
return get_library_block(usage_key)
def test_links(self):
"""
Test operations involving bundle links.
"""
coll = api.create_collection("Test Collection")
# Create two library bundles and a course bundle:
lib1_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib1")
lib1_draft = api.get_or_create_bundle_draft(lib1_bundle.uuid, draft_name="test-draft")
lib2_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib2")
lib2_draft = api.get_or_create_bundle_draft(lib2_bundle.uuid, draft_name="other-draft")
course_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="course")
course_draft = api.get_or_create_bundle_draft(course_bundle.uuid, draft_name="test-draft")
# To create links, we need valid BundleVersions, which requires having committed at least one change:
api.write_draft_file(lib1_draft.uuid, "lib1-data.txt", "hello world")
api.commit_draft(lib1_draft.uuid) # Creates version 1
api.write_draft_file(lib2_draft.uuid, "lib2-data.txt", "hello world")
api.commit_draft(lib2_draft.uuid) # Creates version 1
# Lib2 has no links:
self.assertFalse(api.get_bundle_links(lib2_bundle.uuid))
# Create a link from lib2 to lib1
link1_name = "lib2_to_lib1"
api.set_draft_link(lib2_draft.uuid, link1_name, lib1_bundle.uuid, version=1)
# Now confirm the link exists in the draft:
lib2_draft_links = api.get_bundle_links(lib2_bundle.uuid, use_draft=lib2_draft.name)
self.assertIn(link1_name, lib2_draft_links)
self.assertEqual(lib2_draft_links[link1_name].direct.bundle_uuid, lib1_bundle.uuid)
self.assertEqual(lib2_draft_links[link1_name].direct.version, 1)
# Now commit the change to lib2:
api.commit_draft(lib2_draft.uuid) # Creates version 2
# Now create a link from course to lib2
link2_name = "course_to_lib2"
api.set_draft_link(course_draft.uuid, link2_name, lib2_bundle.uuid, version=2)
api.commit_draft(course_draft.uuid)
# And confirm the link exists in the resulting bundle version:
course_links = api.get_bundle_links(course_bundle.uuid)
self.assertIn(link2_name, course_links)
self.assertEqual(course_links[link2_name].direct.bundle_uuid, lib2_bundle.uuid)
self.assertEqual(course_links[link2_name].direct.version, 2)
# And since the links go course->lib2->lib1, course has an indirect link to lib1:
self.assertEqual(course_links[link2_name].indirect[0].bundle_uuid, lib1_bundle.uuid)
self.assertEqual(course_links[link2_name].indirect[0].version, 1)
# Finally, test deleting a link from course's draft:
api.set_draft_link(course_draft.uuid, link2_name, None, None)
self.assertFalse(api.get_bundle_links(course_bundle.uuid, use_draft=course_draft.name))
def delete_library_block_static_asset_file(usage_key, file_name):
"""
Delete a static asset file from the library.
Example:
video_block = UsageKey.from_string("lb:VideoTeam:python-intro:video:1")
delete_library_block_static_asset_file(video_block, "subtitles-en.srt")
"""
def_key, lib_bundle = _lookup_usage_key(usage_key)
if '..' in file_name:
raise InvalidNameError("Invalid .. in file name.")
file_path = lib_bundle.get_static_prefix_for_definition(def_key) + file_name
# Delete the file from the library bundle's draft
draft = get_or_create_bundle_draft(def_key.bundle_uuid, DRAFT_NAME)
write_draft_file(draft.uuid, file_path, contents=None)
# Clear the bundle cache so everyone sees the new file immediately:
lib_bundle.cache.clear()
def test_bundle_cache_clear(self):
"""
Test the cache clear() method
"""
cache = BundleCache(self.bundle.uuid)
key1 = ("some", "key", "1")
value1 = "value1"
cache.set(key1, value1)
assert cache.get(key1) == value1
# Now publish a new version of the bundle:
api.write_draft_file(self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version")
api.commit_draft(self.draft.uuid)
# Now the cache will not be immediately invalidated; it takes up to MAX_BLOCKSTORE_CACHE_DELAY seconds.
# Since this is a new bundle and we _just_ accessed the cache for the first time, we can be confident
# it won't yet be automatically invalidated.
assert cache.get(key1) == value1
# Now "clear" the cache, forcing the check of the new version:
cache.clear()
assert cache.get(key1) is None
