def update_storage_usage_cache(target_id, target_guid, per_page=500000):
if not settings.ENABLE_STORAGE_USAGE_CACHE:
return
sql = """
SELECT count(size), sum(size) from
(SELECT size FROM osf_basefileversionsthrough AS obfnv
LEFT JOIN osf_basefilenode file ON obfnv.basefilenode_id = file.id
LEFT JOIN osf_fileversion version ON obfnv.fileversion_id = version.id
LEFT JOIN django_content_type type on file.target_content_type_id = type.id
WHERE file.provider = 'osfstorage'
AND type.model = 'abstractnode'
AND file.deleted_on IS NULL
AND file.target_object_id=%s
ORDER BY version.id
LIMIT %s OFFSET %s) file_page
"""
count = per_page
offset = 0
storage_usage_total = 0
with connection.cursor() as cursor:
while count:
cursor.execute(sql, [target_id, per_page, offset])
result = cursor.fetchall()
storage_usage_total += int(result[0][1]) if result[0][1] else 0
count = int(result[0][0]) if result[0][0] else 0
offset += count
key = cache_settings.STORAGE_USAGE_KEY.format(target_id=target_guid)
storage_usage_cache.set(key, storage_usage_total,
settings.STORAGE_USAGE_CACHE_TIMEOUT)
def test_limit_default(self, node):
assert node.storage_usage is 0
key = cache_settings.STORAGE_USAGE_KEY.format(target_id=node._id)
storage_usage_cache.set(key, 0)
assert node.storage_limit_status == StorageLimits.DEFAULT
def update_storage_usage_with_size(payload):
BaseFileNode = apps.get_model('osf.basefilenode')
AbstractNode = apps.get_model('osf.abstractnode')
metadata = payload.get('metadata') or payload.get('destination')
if not metadata.get('nid'):
return
target_node = AbstractNode.load(metadata['nid'])
if target_node.is_quickfiles:
return
action = payload['action']
provider = metadata.get('provider', 'osfstorage')
target_file_id = metadata['path'].replace('/', '')
target_file_size = metadata.get('size', 0)
current_usage = target_node.storage_usage
target_file = BaseFileNode.load(target_file_id)
if target_file and action in ['copy', 'delete', 'move']:
target_file_size = target_file.versions.aggregate(
Sum('size'))['size__sum'] or target_file_size
if action in ['create', 'update', 'copy'] and provider == 'osfstorage':
current_usage += target_file_size
elif action == 'delete' and provider == 'osfstorage':
current_usage = max(current_usage - target_file_size, 0)
elif action in 'move':
source_node = AbstractNode.load(
payload['source']['nid']) # Getting the 'from' node
source_provider = payload['source']['provider']
if target_node == source_node and source_provider == provider:
return # Its not going anywhere.
if source_provider == 'osfstorage' and not source_node.is_quickfiles:
source_node_usage = source_node.storage_usage
source_node_usage = max(source_node_usage - target_file_size, 0)
key = cache_settings.STORAGE_USAGE_KEY.format(
target_id=source_node._id)
storage_usage_cache.set(key, source_node_usage,
settings.STORAGE_USAGE_CACHE_TIMEOUT)
current_usage += target_file_size
if provider != 'osfstorage':
return # We don't want to update the destination node if the provider isn't osfstorage
else:
return
key = cache_settings.STORAGE_USAGE_KEY.format(target_id=target_node._id)
storage_usage_cache.set(key, current_usage,
settings.STORAGE_USAGE_CACHE_TIMEOUT)
def update_storage_usage_cache(target_id):
AbstractNode = apps.get_model('osf.AbstractNode')
storage_usage_total = AbstractNode.objects.get(
guids___id=target_id,
).files.aggregate(sum=models.Sum('versions__size'))['sum'] or 0
key = cache_settings.STORAGE_USAGE_KEY.format(target_id=target_id)
storage_usage_cache.set(key, storage_usage_total, cache_settings.FIVE_MIN_TIMEOUT)
def test_limit_custom(self, node):
node.custom_storage_usage_limit_private = 7
node.save()
key = cache_settings.STORAGE_USAGE_KEY.format(target_id=node._id)
storage_usage_cache.set(key,
node.custom_storage_usage_limit_private * GBs)
assert node.storage_limit_status is StorageLimits.OVER_PRIVATE
storage_usage_cache.set(
key, node.custom_storage_usage_limit_private * GBs - 1)
assert node.storage_limit_status is StorageLimits.APPROACHING_PRIVATE
node.custom_storage_usage_limit_public = 142
node.save()
storage_usage_cache.set(key,
node.custom_storage_usage_limit_public * GBs)
assert node.storage_limit_status is StorageLimits.OVER_PUBLIC
storage_usage_cache.set(
key, node.custom_storage_usage_limit_public * GBs - 1)
assert node.storage_limit_status is StorageLimits.APPROACHING_PUBLIC
def test_node_storage_with_storage_usage(self, app, url, project,
admin_contributor):
# Test Node Storage with OSFStorage Usage
storage_usage = (settings.STORAGE_LIMIT_PRIVATE + 1) * settings.GBs
key = cache_settings.STORAGE_USAGE_KEY.format(target_id=project._id)
storage_usage_cache.set(key, storage_usage,
settings.STORAGE_USAGE_CACHE_TIMEOUT)
res = app.get(url, auth=admin_contributor.auth)
assert res.status_code == 200
data = res.json['data']
assert data['attributes']['storage_limit_status'] == 'OVER_PRIVATE'
assert data['attributes']['storage_usage'] == str(storage_usage)
def test_storage_limits(self, node):
assert node.storage_limit_status is StorageLimits.NOT_CALCULATED
key = cache_settings.STORAGE_USAGE_KEY.format(target_id=node._id)
storage_usage_cache.set(
key, int(STORAGE_LIMIT_PUBLIC * STORAGE_WARNING_THRESHOLD * GBs))
assert node.storage_limit_status is StorageLimits.APPROACHING_PUBLIC
storage_usage_cache.set(
key, int(STORAGE_LIMIT_PRIVATE * STORAGE_WARNING_THRESHOLD * GBs))
assert node.storage_limit_status is StorageLimits.APPROACHING_PRIVATE
storage_usage_cache.set(key, int(STORAGE_LIMIT_PUBLIC * GBs))
assert node.storage_limit_status is StorageLimits.OVER_PUBLIC
