def test_meta_size(testfile, cache, transfer):
t = transfer
size = os.path.getsize(testfile)
assert size != 0
storage_object = StorageObject('foo', 'bar', local_file=testfile)
transfer.delete(storage_object)
uploaded, size_diff = t.store(storage_object)
uploaded_size = t.meta(StorageObject('foo', 'bar')).size
assert size == uploaded_size
assert uploaded.size == size
def test_basic(testfile, cache, transfer):
t = transfer
size = os.path.getsize(testfile)
storage_object = StorageObject('foo', 'bar', local_file=testfile)
t.delete(storage_object)
uploaded, size_diff = t.store(storage_object)
assert size_diff == size
assert uploaded.etag is not None
downloaded = t.retrieve(StorageObject('foo', 'bar'))
assert uploaded.etag == downloaded.etag
delete_size = t.delete(storage_object)
assert size == delete_size
def test_meta(testfile, cache, transfer):
size = os.path.getsize(testfile)
named_object = StorageObject('fus-roh', 'bar')
transfer.delete(named_object) # XXX tests should clean up
storage_object = named_object._replace(local_file=testfile)
assert transfer.meta(named_object) is None
uploaded, _ = transfer.store(storage_object)
meta = transfer.meta(named_object)
assert meta.size == size
assert meta.size == uploaded.size
assert meta.etag == uploaded.etag
def test_meta(testfile, cache, transfer):
size = os.path.getsize(testfile)
named_object = StorageObject('fus-roh', 'bar')
transfer.delete(named_object) # XXX tests should clean up
storage_object = named_object._replace(local_file=testfile)
assert transfer.meta(named_object) is None
uploaded, _ = transfer.store(storage_object)
meta = transfer.meta(named_object)
assert meta.size == size
assert meta.size == uploaded.size
assert meta.etag == uploaded.etag
def test_cache_for_etag(testfile, cache, transfer):
storage_object = StorageObject('foo', 'bar', local_file=testfile)
transfer.delete(storage_object)
uploaded, _ = transfer.store(storage_object)
# the request should work without connection to the backend
# this is simulated by disabling the reference to the internal backend
transfer_module.files = {}
transfer.s3 = None
assert transfer.retrieve(uploaded)
def get_storage(self, storage_object: StorageObject) -> StorageObject:
"""
Gets the etag and size of a StorageObject according to the cache
Raises a KeyError if the etag is not known
"""
key = self._storage_key(storage_object)
etag, size = self._get(key, 'etag', 'size')
if etag is None or size is None:
raise KeyError("Element not found")
etag = etag.decode('UTF-8')
size = int(size)
return storage_object._replace(etag=etag, size=size)
def get_storage(self, storage_object: StorageObject) -> StorageObject:
"""
Gets the etag and size of a StorageObject according to the cache
Raises a KeyError if the etag is not known
"""
key = self._storage_key(storage_object)
etag, size = self._get(key, 'etag', 'size')
if etag is None or size is None:
raise KeyError("Element not found")
etag = etag.decode('UTF-8')
size = int(size)
return storage_object._replace(etag=etag, size=size)
async def _authorize_upload_request(self, file_path, file_size, prefix):
used_quota = (await self.get_database()).get_size(self.user.user_id)
quota_reached = used_quota + file_size > self.user.quota
is_block = file_path.startswith('block/')
old_size = await self.transfer_connector.get_size(
StorageObject(prefix, file_path))
if old_size is None:
is_overwrite = False
size_change = file_size
else:
is_overwrite = True
size_change = file_size - old_size
if not QuotaPolicy.upload(quota_reached, size_change, is_block,
is_overwrite):
self.temp.close()
self._quota_error()
async def check_post_etag(self, prefix, file_path, etag):
if not etag:
return True
stored_object = await self.transfer_connector.meta(
StorageObject(prefix, file_path))
if not stored_object:
self.set_status(
412,
reason='If-Match ETag did not match: object does not exist.')
await self.finish()
return False
elif stored_object.etag != etag:
self.set_status(412,
reason='If-Match ETag did not match stored object')
self.set_header('ETag', stored_object.etag)
await self.finish()
return False
return True
def test_meta_non_existing_file(cache, transfer):
assert transfer.meta(StorageObject('making-things', 'up')) is None
def test_meta_does_not_corrupt_cache(cache, transfer, testfile):
storage_object = StorageObject('foo', 'baz', local_file=testfile)
transfer.store(storage_object)
cache.flush()
transfer.meta(StorageObject('foo', 'baz'))
def store_file(self, prefix, file_path, filename):
return self.transfer.store(
StorageObject(prefix, file_path, None, filename))
def test_delete_non_existing_file(testfile, cache, transfer):
t = transfer
storage_object = StorageObject('foo', 'bar')
t.delete(storage_object)
assert t.delete(storage_object) == 0
def test_cache_is_filled(testfile, cache, transfer):
storage_object = StorageObject('foo', 'bar', local_file=testfile)
transfer.delete(storage_object)
uploaded, size_diff = transfer.store(storage_object)
assert cache.get_storage(storage_object) == storage_object._replace(
etag=uploaded.etag, size=uploaded.size)
def retrieve_file(self, prefix, file_path, etag):
return self.transfer.retrieve(
StorageObject(prefix, file_path, etag, None))
import pytest
from blockserver.backend.transfer import StorageObject
from blockserver.backend.auth import User
with_etag = StorageObject('foo', 'bar', 'etag', size=10)
without_etag = with_etag._replace(etag=None, size=None) # type: StorageObject
def test_storage_cache_basics(cache):
with pytest.raises(KeyError):
cache.get_storage(without_etag)
with pytest.raises(ValueError):
cache.set_storage(without_etag)
cache.set_storage(with_etag)
assert cache.get_storage(without_etag) == with_etag
AUTH_TOKEN = 'MAGICFAIRYTALE'
def test_auth_cache_basics(cache):
with pytest.raises(KeyError):
cache.get_auth(AUTH_TOKEN)
with pytest.raises(ValueError):
cache.set_auth(AUTH_TOKEN, None)
user = User(0, True, 123, 456)
cache.set_auth(AUTH_TOKEN, user)
assert cache.get_auth(AUTH_TOKEN) == user
user_2 = User(2, False, 456, 123789)
cache.set_auth(AUTH_TOKEN, user_2)
def test_cache_is_filled(testfile, cache, transfer):
storage_object = StorageObject('foo', 'bar', local_file=testfile)
transfer.delete(storage_object)
uploaded, size_diff = transfer.store(storage_object)
assert cache.get_storage(storage_object) == storage_object._replace(
etag=uploaded.etag, size=uploaded.size)
def delete_file(self, prefix, file_path):
return self.transfer.delete(
StorageObject(prefix, file_path, None, None))
