def setUp(self):
super(ResourceSizeCacheTestCase, self).setUp()
self.node = mock.Mock(spec_set=ContentNode())
self.node.pk = "abcdefghijklmnopqrstuvwxyz"
self.redis_client = mock_class_instance("redis.client.StrictRedis")
self.cache_client = mock_class_instance(
"django_redis.client.DefaultClient")
self.cache_client.get_client.return_value = self.redis_client
self.cache = mock.Mock(client=self.cache_client)
self.helper = ResourceSizeCache(self.node, self.cache)
def update(self, instance, validated_data):
if "contentnode" in validated_data:
# if we're updating the file's related node, we'll trigger a reset for the
# old channel's cache modified date
update_node = validated_data.get("contentnode", None)
if not update_node or update_node.id != instance.contentnode_id:
ResourceSizeCache.reset_modified_for_file(instance)
results = super(FileSerializer, self).update(instance, validated_data)
if instance.uploaded_by_id:
calculate_user_storage(instance.uploaded_by_id)
return results
def move_node(self, node, target, position="last-child"):
"""
Vendored from mptt - by default mptt moves then saves
This is updated to call the save with the skip_lock kwarg
to prevent a second atomic transaction and tree locking context
being opened.
Moves ``node`` relative to a given ``target`` node as specified
by ``position`` (when appropriate), by examining both nodes and
calling the appropriate method to perform the move.
A ``target`` of ``None`` indicates that ``node`` should be
turned into a root node.
Valid values for ``position`` are ``'first-child'``,
``'last-child'``, ``'left'`` or ``'right'``.
``node`` will be modified to reflect its new tree state in the
database.
This method explicitly checks for ``node`` being made a sibling
of a root node, as this is a special case due to our use of tree
ids to order root nodes.
NOTE: This is a low-level method; it does NOT respect
``MPTTMeta.order_insertion_by``. In most cases you should just
move the node yourself by setting node.parent.
"""
old_parent = node.parent
with self.lock_mptt(node.tree_id, target.tree_id):
# Call _mptt_refresh to ensure that the mptt fields on
# these nodes are up to date once we have acquired a lock
# on the associated trees. This means that the mptt data
# will remain fresh until the lock is released at the end
# of the context manager.
self._mptt_refresh(node, target)
# N.B. this only calls save if we are running inside a
# delay MPTT updates context
self._move_node(node, target, position=position)
node.save(skip_lock=True)
node_moved.send(
sender=node.__class__,
instance=node,
target=target,
position=position,
)
# when moving to a new tree, like trash, we'll blanket reset the modified for the
# new root and the old root nodes
if old_parent.tree_id != target.tree_id:
for size_cache in [
ResourceSizeCache(target.get_root()),
ResourceSizeCache(old_parent.get_root())
]:
size_cache.reset_modified(None)
def calculate_resource_size(node, force=False):
"""
Function that calculates the total file size of all files of the specified node and it's
descendants, if they're marked complete
:param node: The ContentNode for which to calculate resource size.
:param force: A boolean to force calculation if node is too big and would otherwise do so async
:return: A tuple of (size, stale)
:rtype: (int, bool)
"""
cache = ResourceSizeCache(node)
db = ResourceSizeHelper(node)
size = None if force else cache.get_size()
modified = None if force else cache.get_modified()
# since we added file.modified as nullable, if the result is None/Null, then we know that it
# hasn't been modified since our last cached value, so we only need to check is False
if size is not None and modified is not None and db.modified_since(
modified) is False:
# use cache if not modified since cache modified timestamp
return size, False
# if the node is too big to calculate its size right away, we return "stale"
if not force and node.get_descendant_count() > STALE_MAX_CALCULATION_SIZE:
return size, True
start = time.time()
# do recalculation, marking modified time before starting
now = timezone.now()
size = db.get_size()
cache.set_size(size)
cache.set_modified(now)
elapsed = time.time() - start
if not force and elapsed > SLOW_UNFORCED_CALC_THRESHOLD:
# warn us in Sentry if an unforced recalculation took too long
try:
# we need to raise it to get Python to fill out the stack trace.
raise SlowCalculationError(node.pk, elapsed)
except SlowCalculationError as e:
report_exception(e)
return size, False
def delete_from_changes(self, changes):
try:
# reset channel resource size cache
keys = [change["key"] for change in changes]
queryset = self.filter_queryset_from_keys(
self.get_edit_queryset(), keys
).order_by()
# find all root nodes for files, and reset the cache modified date
root_nodes = ContentNode.objects.filter(
parent__isnull=True,
tree_id__in=queryset.values_list('contentnode__tree_id', flat=True).distinct(),
)
for root_node in root_nodes:
ResourceSizeCache(root_node).reset_modified(None)
except Exception as e:
report_exception(e)
return super(FileViewSet, self).delete_from_changes(changes)
class ResourceSizeCacheTestCase(SimpleTestCase):
def setUp(self):
super(ResourceSizeCacheTestCase, self).setUp()
self.node = mock.Mock(spec_set=ContentNode())
self.node.pk = "abcdefghijklmnopqrstuvwxyz"
self.redis_client = mock_class_instance("redis.client.StrictRedis")
self.cache_client = mock_class_instance(
"django_redis.client.DefaultClient")
self.cache_client.get_client.return_value = self.redis_client
self.cache = mock.Mock(client=self.cache_client)
self.helper = ResourceSizeCache(self.node, self.cache)
def test_redis_client(self):
self.assertEqual(self.helper.redis_client, self.redis_client)
self.cache_client.get_client.assert_called_once_with(write=True)
def test_redis_client__not_redis(self):
self.cache.client = mock.Mock()
self.assertIsNone(self.helper.redis_client)
def test_hash_key(self):
self.assertEqual("resource_size:abcd", self.helper.hash_key)
def test_size_key(self):
self.assertEqual("abcdefghijklmnopqrstuvwxyz:value",
self.helper.size_key)
def test_modified_key(self):
self.assertEqual("abcdefghijklmnopqrstuvwxyz:modified",
self.helper.modified_key)
def test_cache_get(self):
self.redis_client.hget.return_value = 123
self.assertEqual(123, self.helper.cache_get("test_key"))
self.redis_client.hget.assert_called_once_with(self.helper.hash_key,
"test_key")
def test_cache_get__not_redis(self):
self.cache.client = mock.Mock()
self.cache.get.return_value = 123
self.assertEqual(123, self.helper.cache_get("test_key"))
self.cache.get.assert_called_once_with("{}:{}".format(
self.helper.hash_key, "test_key"))
def test_cache_set(self):
self.helper.cache_set("test_key", 123)
self.redis_client.hset.assert_called_once_with(self.helper.hash_key,
"test_key", 123)
def test_cache_set__delete(self):
self.helper.cache_set("test_key", None)
self.redis_client.hdel.assert_called_once_with(self.helper.hash_key,
"test_key")
def test_cache_set__not_redis(self):
self.cache.client = mock.Mock()
self.helper.cache_set("test_key", 123)
self.cache.set.assert_called_once_with(
"{}:{}".format(self.helper.hash_key, "test_key"), 123)
def test_get_size(self):
with mock.patch.object(self.helper, 'cache_get') as cache_get:
cache_get.return_value = 123
self.assertEqual(123, self.helper.get_size())
cache_get.assert_called_once_with(self.helper.size_key)
def test_set_size(self):
with mock.patch.object(self.helper, 'cache_set') as cache_set:
self.helper.set_size(123)
cache_set.assert_called_once_with(self.helper.size_key, 123)
def test_get_modified(self):
with mock.patch.object(self.helper, 'cache_get') as cache_get:
cache_get.return_value = '2021-01-01 00:00:00'
modified = self.helper.get_modified()
self.assertIsNotNone(modified)
self.assertEqual('2021-01-01T00:00:00', modified.isoformat())
cache_get.assert_called_once_with(self.helper.modified_key)
def test_set_modified(self):
with mock.patch.object(self.helper, 'cache_set') as cache_set:
self.helper.set_modified('2021-01-01 00:00:00')
cache_set.assert_called_once_with(self.helper.modified_key,
'2021-01-01 00:00:00')