def test_get_range(self):
user_id = uuid4().hex
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, user_id)
self.addCleanup(manager.close)
blob_id, content = 'blob_id', '0123456789'
doc = BlobDoc(BytesIO(content), blob_id)
yield manager.put(doc, len(content))
uri = urljoin(self.uri, '%s/%s' % (user_id, blob_id))
res = yield _get(uri, headers={'Range': 'bytes=10-20'})
text = yield res.text()
self.assertTrue(res.headers.hasHeader('content-range'))
content_range = res.headers.getRawHeaders('content-range').pop()
self.assertIsNotNone(re.match('^bytes 10-20/[0-9]+$', content_range))
self.assertEqual(10, len(text))
def test_refresh_deletions_from_server(self):
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, uuid4().hex)
self.addCleanup(manager.close)
blob_id, content = 'delete_me', 'content'
blob_id2 = 'dont_delete_me'
doc1 = BlobDoc(BytesIO(content), blob_id)
doc2 = BlobDoc(BytesIO(content), blob_id2)
yield manager.put(doc1, len(content))
yield manager.put(doc2, len(content))
yield manager._delete_from_remote(blob_id) # remote only deletion
self.assertTrue((yield manager.local.exists(blob_id)))
yield manager.sync()
self.assertFalse((yield manager.local.exists(blob_id)))
self.assertTrue((yield manager.local.exists(blob_id2)))
def test_get_range_not_satisfiable(self):
# put a blob in place
user_id = uuid4().hex
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, user_id)
self.addCleanup(manager.close)
blob_id, content = uuid4().hex, 'content'
doc = BlobDoc(BytesIO(content), blob_id)
yield manager.put(doc, len(content))
# and check possible parsing errors
uri = urljoin(self.uri, '%s/%s' % (user_id, blob_id))
ranges = [
'bytes',
'bytes=',
'bytes=1',
'bytes=blah-100',
'potatoes=10-100'
'blah'
]
for range in ranges:
res = yield _get(uri, headers={'Range': range})
self.assertEqual(416, res.code)
content_range = res.headers.getRawHeaders('content-range').pop()
self.assertIsNotNone(re.match('^bytes \*/[0-9]+$', content_range))
class BlobManagerTestCase(unittest.TestCase):
class doc_info:
doc_id = 'D-deadbeef'
rev = FIXED_REV
def setUp(self):
self.cleartext = BytesIO('rosa de foc')
self.secret = 'A' * 96
self.manager = BlobManager(self.tempdir, '', 'A' * 32, self.secret,
'uuid', 'token', None)
self.addCleanup(self.manager.close)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_get_inexistent(self):
self.manager._download_and_decrypt = Mock(return_value=None)
bad_blob_id = 'inexsistent_id'
result = yield self.manager.get(bad_blob_id)
self.assertIsNone(result)
args = bad_blob_id, ''
self.manager._download_and_decrypt.assert_called_once_with(*args)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_get_from_existing_value(self):
self.manager._download_and_decrypt = Mock(return_value=None)
msg, blob_id = "It's me, M4r10!", uuid4().hex
yield self.manager.local.put(blob_id, BytesIO(msg), size=len(msg))
result = yield self.manager.get(blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertNot(self.manager._download_and_decrypt.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_stores_on_local_db(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
msg, blob_id = "Hey Joe", uuid4().hex
doc = BlobDoc(BytesIO(msg), blob_id=blob_id)
yield self.manager.put(doc, size=len(msg))
result = yield self.manager.local.get(blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertTrue(self.manager._encrypt_and_upload.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_then_get_using_real_file_descriptor(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._download_and_decrypt = Mock(return_value=None)
msg, blob_id = "Fuuuuull cycleee! \o/", uuid4().hex
tmpfile = os.tmpfile()
tmpfile.write(msg)
tmpfile.seek(0)
doc = BlobDoc(tmpfile, blob_id)
yield self.manager.put(doc, size=len(msg))
result = yield self.manager.get(doc.blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertTrue(self.manager._encrypt_and_upload.called)
self.assertFalse(self.manager._download_and_decrypt.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_local_list_blobs(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
msg, blob_id1, blob_id2 = "1337", uuid4().hex, uuid4().hex
doc = BlobDoc(BytesIO(msg), blob_id1)
yield self.manager.put(doc, size=len(msg))
doc2 = BlobDoc(BytesIO(msg), blob_id2)
yield self.manager.put(doc2, size=len(msg))
blobs_list = yield self.manager.local_list()
self.assertEquals(set([blob_id1, blob_id2]), set(blobs_list))
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_send_missing(self):
fd = BytesIO('test')
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager.remote_list = Mock(return_value=[])
yield self.manager.local.put('missing_id', fd, 4)
yield self.manager.send_missing()
call_list = self.manager._encrypt_and_upload.call_args_list
self.assertEquals(1, len(call_list))
call_blob_id, call_fd = call_list[0][0]
self.assertEquals('missing_id', call_blob_id)
self.assertEquals('test', call_fd.getvalue())
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_duplicated_blob_error_on_put(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
content = "Blob content"
doc1 = BlobDoc(BytesIO(content), 'existing_id')
yield self.manager.put(doc1, len(content))
doc2 = BlobDoc(BytesIO(content), 'existing_id')
self.manager._encrypt_and_upload.reset_mock()
with pytest.raises(BlobAlreadyExistsError):
yield self.manager.put(doc2, len(content))
self.assertFalse(self.manager._encrypt_and_upload.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_delete_from_local_and_remote(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._delete_from_remote = Mock(return_value=None)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
yield self.manager.put(doc1, len(content))
yield self.manager.delete(blob_id)
local_list = yield self.manager.local_list()
self.assertEquals(0, len(local_list))
params = {'namespace': ''}
self.manager._delete_from_remote.assert_called_with(blob_id, **params)
class BlobManagerTestCase(unittest.TestCase):
class doc_info:
doc_id = 'D-deadbeef'
rev = FIXED_REV
def setUp(self):
self.cleartext = BytesIO('rosa de foc')
self.secret = 'A' * 96
self.manager = BlobManager(self.tempdir, '', 'A' * 32, self.secret,
uuid4().hex, 'token', None)
self.addCleanup(self.manager.close)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_get_missing(self):
self.manager._download_and_decrypt = Mock(return_value=None)
missing_blob_id = uuid4().hex
result = yield self.manager.get(missing_blob_id)
self.assertIsNone(result)
args = missing_blob_id, ''
self.manager._download_and_decrypt.assert_called_once_with(*args)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_get_from_existing_value(self):
self.manager._download_and_decrypt = Mock(return_value=None)
msg, blob_id = "It's me, M4r10!", uuid4().hex
yield self.manager.local.put(blob_id, BytesIO(msg), size=len(msg))
result = yield self.manager.get(blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertNot(self.manager._download_and_decrypt.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_stores_on_local_db(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
msg, blob_id = "Hey Joe", uuid4().hex
doc = BlobDoc(BytesIO(msg), blob_id=blob_id)
yield self.manager.put(doc, size=len(msg))
result = yield self.manager.local.get(blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertTrue(self.manager._encrypt_and_upload.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_stores_on_local_db_with_namespace(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._download_and_decrypt = Mock(return_value=None)
msg, blob_id = "Hey Joe", uuid4().hex
doc = BlobDoc(BytesIO(msg), blob_id=blob_id)
yield self.manager.put(doc, size=len(msg), namespace='custom')
self.assertTrue(self.manager._encrypt_and_upload.called)
arg1, arg2 = self.manager._encrypt_and_upload.call_args[0]
kwargs = self.manager._encrypt_and_upload.call_args[1]
self.assertEquals(arg1, blob_id)
self.assertTrue(isinstance(arg2, BytesIO))
self.assertEquals(kwargs, {'namespace': 'custom'})
result = yield self.manager.local.get(blob_id)
self.assertEquals(result, None)
result = yield self.manager.local.get(blob_id, namespace='custom')
self.assertEquals(result.getvalue(), msg)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_local_only_doesnt_send_to_server(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
msg, blob_id = "Hey Joe", uuid4().hex
doc = BlobDoc(BytesIO(msg), blob_id=blob_id)
yield self.manager.put(doc, size=len(msg), local_only=True)
result = yield self.manager.local.get(blob_id)
status, _ = yield self.manager.local.get_sync_status(blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertEquals(status, SyncStatus.LOCAL_ONLY)
self.assertFalse(self.manager._encrypt_and_upload.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_then_get_using_real_file_descriptor(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._download_and_decrypt = Mock(return_value=None)
msg, blob_id = "Fuuuuull cycleee! \o/", uuid4().hex
tmpfile = os.tmpfile()
tmpfile.write(msg)
tmpfile.seek(0)
doc = BlobDoc(tmpfile, blob_id)
yield self.manager.put(doc, size=len(msg))
result = yield self.manager.get(doc.blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertTrue(self.manager._encrypt_and_upload.called)
self.assertFalse(self.manager._download_and_decrypt.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_local_list_blobs(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
msg, blob_id1, blob_id2 = "1337", uuid4().hex, uuid4().hex
doc = BlobDoc(BytesIO(msg), blob_id1)
yield self.manager.put(doc, size=len(msg))
doc2 = BlobDoc(BytesIO(msg), blob_id2)
yield self.manager.put(doc2, size=len(msg))
blobs_list = yield self.manager.local_list()
self.assertEquals(set([blob_id1, blob_id2]), set(blobs_list))
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_send_missing(self):
fd, missing_id = BytesIO('test'), uuid4().hex
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager.remote_list = Mock(return_value=[])
doc1 = BlobDoc(fd, missing_id)
yield self.manager.put(doc1, 4)
yield self.manager.send_missing()
call_list = self.manager._encrypt_and_upload.call_args_list
self.assertEquals(1, len(call_list))
call_blob_id, call_fd = call_list[0][0]
self.assertEquals(missing_id, call_blob_id)
self.assertEquals('test', call_fd.getvalue())
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_sync_progress(self):
deferreds = []
local = self.manager.local
pending_download = SyncStatus.PENDING_DOWNLOAD
pending_upload = SyncStatus.PENDING_UPLOAD
synced = SyncStatus.SYNCED
for status in [pending_download, pending_upload, synced, synced]:
deferreds.append(local.update_sync_status(uuid4().hex, status))
yield defer.gatherResults(deferreds)
progress = yield self.manager.sync_progress
self.assertEquals(progress, {
'PENDING_DOWNLOAD': 1,
'PENDING_UPLOAD': 1,
'SYNCED': 2
})
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_duplicated_blob_error_on_put(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
content, existing_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), existing_id)
yield self.manager.put(doc1, len(content))
doc2 = BlobDoc(BytesIO(content), existing_id)
self.manager._encrypt_and_upload.reset_mock()
with pytest.raises(BlobAlreadyExistsError):
yield self.manager.put(doc2, len(content))
self.assertFalse(self.manager._encrypt_and_upload.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_delete_from_local_and_remote(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._delete_from_remote = Mock(return_value=None)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
yield self.manager.put(doc1, len(content))
yield self.manager.delete(blob_id)
local_list = yield self.manager.local_list()
self.assertEquals(0, len(local_list))
params = {'namespace': ''}
self.manager._delete_from_remote.assert_called_with(blob_id, **params)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_offline_delete_marks_as_pending_delete(self):
deletion_failure = defer.fail(Exception())
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._delete_from_remote = Mock(return_value=deletion_failure)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
yield self.manager.put(doc1, len(content))
with pytest.raises(Exception):
yield self.manager.delete(blob_id)
sync_progress = yield self.manager.sync_progress
expected = {'PENDING_DELETE': 1}
self.assertEquals(expected, sync_progress)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_online_delete_marks_as_synced(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._delete_from_remote = Mock(return_value=None)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
yield self.manager.put(doc1, len(content))
yield self.manager.delete(blob_id)
sync_progress = yield self.manager.sync_progress
expected = {'SYNCED': 1}
self.assertEquals(expected, sync_progress)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_local_sync_status_pending_upload(self):
upload_failure = defer.fail(Exception())
self.manager._encrypt_and_upload = Mock(return_value=upload_failure)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
with pytest.raises(Exception):
yield self.manager.put(doc1, len(content))
pending_upload = SyncStatus.PENDING_UPLOAD
local_list = yield self.manager.local_list_status(pending_upload)
self.assertIn(blob_id, local_list)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_local_list_doesnt_include_unavailable_blobs(self):
local = self.manager.local
unavailable_ids, deferreds = [], []
for status in SyncStatus.UNAVAILABLE_STATUSES:
blob_id = uuid4().hex
deferreds.append(local.update_sync_status(blob_id, status))
unavailable_ids.append(blob_id)
available_blob_id = uuid4().hex
content, length = self.cleartext, len(self.cleartext.getvalue())
deferreds.append(local.put(available_blob_id, content, length))
yield defer.gatherResults(deferreds)
local_list = yield local.list()
message = 'Unavailable blob showing up on listing!'
for blob_id in unavailable_ids:
self.assertNotIn(blob_id, local_list, message)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_get_doesnt_include_unavailable_blobs(self):
local = self.manager.local
unavailable_ids, deferreds = [], []
for status in SyncStatus.UNAVAILABLE_STATUSES:
blob_id = uuid4().hex
deferreds.append(local.update_sync_status(blob_id, status))
unavailable_ids.append(blob_id)
available_blob_id = uuid4().hex
content, length = self.cleartext, len(self.cleartext.getvalue())
deferreds.append(local.put(available_blob_id, content, length))
yield defer.gatherResults(deferreds)
message = 'Unavailable blob showing up on GET!'
for blob_id in unavailable_ids:
blob = yield local.get(blob_id)
self.assertFalse(blob, message)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_persist_sync_statuses_listing_from_server(self):
local = self.manager.local
remote_ids = [uuid4().hex for _ in range(10)]
local_ids = [uuid4().hex for _ in range(10)]
self.manager.remote_list = Mock(return_value=defer.succeed(remote_ids))
content, pending = self.cleartext, SyncStatus.PENDING_UPLOAD
length, deferreds = len(content.getvalue()), []
for blob_id in local_ids:
d = local.put(blob_id, content, length)
deferreds.append(d)
d = local.update_sync_status(blob_id, pending)
deferreds.append(d)
yield defer.gatherResults(deferreds)
yield self.manager.refresh_sync_status_from_server()
d = self.manager.local_list_status(SyncStatus.PENDING_UPLOAD)
pending_upload_list = yield d
d = self.manager.local_list_status(SyncStatus.PENDING_DOWNLOAD)
pending_download_list = yield d
self.assertEquals(set(pending_upload_list), set(local_ids))
self.assertEquals(set(pending_download_list), set(remote_ids))
class BlobManagerTestCase(unittest.TestCase):
class doc_info:
doc_id = 'D-deadbeef'
rev = FIXED_REV
def setUp(self):
self.cleartext = BytesIO('rosa de foc')
self.secret = 'A' * 96
self.manager = BlobManager(
self.tempdir, '',
'A' * 32, self.secret,
uuid4().hex, 'token', None)
self.addCleanup(self.manager.close)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_get_missing(self):
self.manager._download_and_decrypt = Mock(return_value=None)
missing_blob_id = uuid4().hex
result = yield self.manager.get(missing_blob_id)
self.assertIsNone(result)
args = missing_blob_id, ''
self.manager._download_and_decrypt.assert_called_once_with(*args)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_get_from_existing_value(self):
self.manager._download_and_decrypt = Mock(return_value=None)
msg, blob_id = "It's me, M4r10!", uuid4().hex
yield self.manager.local.put(blob_id, BytesIO(msg),
size=len(msg))
result = yield self.manager.get(blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertNot(self.manager._download_and_decrypt.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_stores_on_local_db(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
msg, blob_id = "Hey Joe", uuid4().hex
doc = BlobDoc(BytesIO(msg), blob_id=blob_id)
yield self.manager.put(doc, size=len(msg))
result = yield self.manager.local.get(blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertTrue(self.manager._encrypt_and_upload.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_stores_on_local_db_with_namespace(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._download_and_decrypt = Mock(return_value=None)
msg, blob_id = "Hey Joe", uuid4().hex
doc = BlobDoc(BytesIO(msg), blob_id=blob_id)
yield self.manager.put(doc, size=len(msg), namespace='custom')
self.assertTrue(self.manager._encrypt_and_upload.called)
arg1, arg2 = self.manager._encrypt_and_upload.call_args[0]
kwargs = self.manager._encrypt_and_upload.call_args[1]
self.assertEquals(arg1, blob_id)
self.assertTrue(isinstance(arg2, BytesIO))
self.assertEquals(kwargs, {'namespace': 'custom'})
result = yield self.manager.local.get(blob_id)
self.assertEquals(result, None)
result = yield self.manager.local.get(blob_id, namespace='custom')
self.assertEquals(result.getvalue(), msg)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_local_only_doesnt_send_to_server(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
msg, blob_id = "Hey Joe", uuid4().hex
doc = BlobDoc(BytesIO(msg), blob_id=blob_id)
yield self.manager.put(doc, size=len(msg), local_only=True)
result = yield self.manager.local.get(blob_id)
status, _ = yield self.manager.local.get_sync_status(blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertEquals(status, SyncStatus.LOCAL_ONLY)
self.assertFalse(self.manager._encrypt_and_upload.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_then_get_using_real_file_descriptor(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._download_and_decrypt = Mock(return_value=None)
msg, blob_id = "Fuuuuull cycleee! \o/", uuid4().hex
tmpfile = os.tmpfile()
tmpfile.write(msg)
tmpfile.seek(0)
doc = BlobDoc(tmpfile, blob_id)
yield self.manager.put(doc, size=len(msg))
result = yield self.manager.get(doc.blob_id)
self.assertEquals(result.getvalue(), msg)
self.assertTrue(self.manager._encrypt_and_upload.called)
self.assertFalse(self.manager._download_and_decrypt.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_local_list_blobs(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
msg, blob_id1, blob_id2 = "1337", uuid4().hex, uuid4().hex
doc = BlobDoc(BytesIO(msg), blob_id1)
yield self.manager.put(doc, size=len(msg))
doc2 = BlobDoc(BytesIO(msg), blob_id2)
yield self.manager.put(doc2, size=len(msg))
blobs_list = yield self.manager.local_list()
self.assertEquals(set([blob_id1, blob_id2]), set(blobs_list))
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_send_missing(self):
fd, missing_id = BytesIO('test'), uuid4().hex
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager.remote_list = Mock(return_value=[])
doc1 = BlobDoc(fd, missing_id)
yield self.manager.put(doc1, 4)
yield self.manager.send_missing()
call_list = self.manager._encrypt_and_upload.call_args_list
self.assertEquals(1, len(call_list))
call_blob_id, call_fd = call_list[0][0]
self.assertEquals(missing_id, call_blob_id)
self.assertEquals('test', call_fd.getvalue())
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_sync_progress(self):
deferreds = []
local = self.manager.local
pending_download = SyncStatus.PENDING_DOWNLOAD
pending_upload = SyncStatus.PENDING_UPLOAD
synced = SyncStatus.SYNCED
for status in [pending_download, pending_upload, synced, synced]:
deferreds.append(local.update_sync_status(uuid4().hex, status))
yield defer.gatherResults(deferreds)
progress = yield self.manager.sync_progress
self.assertEquals(progress, {
'PENDING_DOWNLOAD': 1, 'PENDING_UPLOAD': 1, 'SYNCED': 2})
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_duplicated_blob_error_on_put(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
content, existing_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), existing_id)
yield self.manager.put(doc1, len(content))
doc2 = BlobDoc(BytesIO(content), existing_id)
self.manager._encrypt_and_upload.reset_mock()
with pytest.raises(BlobAlreadyExistsError):
yield self.manager.put(doc2, len(content))
self.assertFalse(self.manager._encrypt_and_upload.called)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_delete_from_local_and_remote(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._delete_from_remote = Mock(return_value=None)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
yield self.manager.put(doc1, len(content))
yield self.manager.delete(blob_id)
local_list = yield self.manager.local_list()
self.assertEquals(0, len(local_list))
params = {'namespace': ''}
self.manager._delete_from_remote.assert_called_with(blob_id, **params)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_offline_delete_marks_as_pending_delete(self):
deletion_failure = defer.fail(Exception())
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._delete_from_remote = Mock(return_value=deletion_failure)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
yield self.manager.put(doc1, len(content))
with pytest.raises(Exception):
yield self.manager.delete(blob_id)
sync_progress = yield self.manager.sync_progress
expected = {'PENDING_DELETE': 1}
self.assertEquals(expected, sync_progress)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_online_delete_marks_as_synced(self):
self.manager._encrypt_and_upload = Mock(return_value=None)
self.manager._delete_from_remote = Mock(return_value=None)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
yield self.manager.put(doc1, len(content))
yield self.manager.delete(blob_id)
sync_progress = yield self.manager.sync_progress
expected = {'SYNCED': 1}
self.assertEquals(expected, sync_progress)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_local_sync_status_pending_upload(self):
upload_failure = defer.fail(Exception())
self.manager._encrypt_and_upload = Mock(return_value=upload_failure)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
with pytest.raises(Exception):
yield self.manager.put(doc1, len(content))
pending_upload = SyncStatus.PENDING_UPLOAD
local_list = yield self.manager.local_list_status(pending_upload)
self.assertIn(blob_id, local_list)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_local_list_doesnt_include_unavailable_blobs(self):
local = self.manager.local
unavailable_ids, deferreds = [], []
for status in SyncStatus.UNAVAILABLE_STATUSES:
blob_id = uuid4().hex
deferreds.append(local.update_sync_status(blob_id, status))
unavailable_ids.append(blob_id)
available_blob_id = uuid4().hex
content, length = self.cleartext, len(self.cleartext.getvalue())
deferreds.append(local.put(available_blob_id, content, length))
yield defer.gatherResults(deferreds)
local_list = yield local.list()
message = 'Unavailable blob showing up on listing!'
for blob_id in unavailable_ids:
self.assertNotIn(blob_id, local_list, message)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_get_doesnt_include_unavailable_blobs(self):
local = self.manager.local
unavailable_ids, deferreds = [], []
for status in SyncStatus.UNAVAILABLE_STATUSES:
blob_id = uuid4().hex
deferreds.append(local.update_sync_status(blob_id, status))
unavailable_ids.append(blob_id)
available_blob_id = uuid4().hex
content, length = self.cleartext, len(self.cleartext.getvalue())
deferreds.append(local.put(available_blob_id, content, length))
yield defer.gatherResults(deferreds)
message = 'Unavailable blob showing up on GET!'
for blob_id in unavailable_ids:
blob = yield local.get(blob_id)
self.assertFalse(blob, message)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_persist_sync_statuses_listing_from_server(self):
local = self.manager.local
remote_ids = [uuid4().hex for _ in range(10)]
local_ids = [uuid4().hex for _ in range(10)]
self.manager.remote_list = Mock(return_value=defer.succeed(remote_ids))
content, pending = self.cleartext, SyncStatus.PENDING_UPLOAD
length, deferreds = len(content.getvalue()), []
for blob_id in local_ids:
d = local.put(blob_id, content, length)
deferreds.append(d)
d = local.update_sync_status(blob_id, pending)
deferreds.append(d)
yield defer.gatherResults(deferreds)
yield self.manager.refresh_sync_status_from_server()
d = self.manager.local_list_status(SyncStatus.PENDING_UPLOAD)
pending_upload_list = yield d
d = self.manager.local_list_status(SyncStatus.PENDING_DOWNLOAD)
pending_download_list = yield d
self.assertEquals(set(pending_upload_list), set(local_ids))
self.assertEquals(set(pending_download_list), set(remote_ids))
class BlobPrioritiesTests(unittest.TestCase):
def setUp(self):
self.cleartext = BytesIO('patriarchy is opression')
self.secret = 'A' * 96
self.manager = BlobManager(self.tempdir, '', 'A' * 32, self.secret,
uuid4().hex, 'token', None)
self.addCleanup(self.manager.close)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_get_sets_default_priority(self):
self.manager._download_and_decrypt = Mock(return_value=None)
missing_blob_id = uuid4().hex
result = yield self.manager.get(missing_blob_id)
self.assertIsNone(result)
priority = yield self.manager._get_priority(missing_blob_id)
self.assertEqual(Priority.DEFAULT, priority)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_get_sets_priority(self):
self.manager._download_and_decrypt = Mock(return_value=None)
missing_blob_id = uuid4().hex
urgent = 'urgent'
result = yield self.manager.get(missing_blob_id, priority=urgent)
self.assertIsNone(result)
priority = yield self.manager._get_priority(missing_blob_id)
self.assertEqual(Priority.URGENT, priority)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_sets_default_priority(self):
upload_failure = defer.fail(Exception())
self.manager._encrypt_and_upload = Mock(return_value=upload_failure)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
with pytest.raises(Exception):
yield self.manager.put(doc1, len(content))
priority = yield self.manager._get_priority(blob_id)
self.assertEqual(Priority.DEFAULT, priority)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_put_sets_priority(self):
upload_failure = defer.fail(Exception())
self.manager._encrypt_and_upload = Mock(return_value=upload_failure)
content, blob_id = "Blob content", uuid4().hex
doc1 = BlobDoc(BytesIO(content), blob_id)
with pytest.raises(Exception):
yield self.manager.put(doc1, len(content), priority='urgent')
priority = yield self.manager._get_priority(blob_id)
self.assertEqual(Priority.URGENT, priority)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_set_priority_sets_priority(self):
self.manager._download_and_decrypt = Mock(return_value=None)
missing_blob_id = uuid4().hex
result = yield self.manager.get(missing_blob_id)
self.assertIsNone(result)
urgent = 'urgent'
yield self.manager._set_priority(missing_blob_id, urgent)
priority = yield self.manager._get_priority(missing_blob_id)
self.assertEqual(Priority.URGENT, priority)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_local_list_status_orders_by_priority(self):
self.manager._download_and_decrypt = Mock(return_value=None)
def _get(priority):
missing_blob_id = uuid4().hex
d = self.manager.get(missing_blob_id, priority=priority)
d.addCallback(lambda _: missing_blob_id)
return d
# get some blobs in arbitrary order
low = yield _get('low')
high = yield _get('high')
medium = yield _get('medium')
urgent = yield _get('urgent')
# make sure they are ordered by priority
status = SyncStatus.PENDING_DOWNLOAD
pending = yield self.manager.local_list_status(status)
self.assertEqual([urgent, high, medium, low], pending)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_refresh_sync_status_from_server_saves_default_priorities(self):
remote_ids = [uuid4().hex for _ in range(10)]
self.manager.remote_list = Mock(return_value=defer.succeed(remote_ids))
yield self.manager.refresh_sync_status_from_server()
for blob_id in remote_ids:
priority = yield self.manager._get_priority(blob_id)
self.assertEquals(Priority.DEFAULT, priority)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_fetch_missing_fetches_with_priority(self):
# pretend we have some pending downloads
status = SyncStatus.PENDING_DOWNLOAD
update_meth = self.manager.local.update_sync_status
priorities = [
('low', Priority.LOW),
('high', Priority.HIGH),
('medium', Priority.MEDIUM),
('urgent', Priority.URGENT),
]
deferreds = []
for blob_id, priority in priorities:
d = update_meth(blob_id, status, priority=priority)
deferreds.append(d)
yield defer.gatherResults(deferreds)
# make sure download "succeeds" so fetching works
content = 'vegan muffin'
fd = BytesIO(content)
size = len(content)
self.manager._download_and_decrypt = Mock(return_value=(fd, size))
self.manager.concurrent_transfers_limit = 1
# this is the operation we are interested in
yield self.manager.fetch_missing()
# retrieve the order in which blob transfers were made
calls = self.manager._download_and_decrypt.mock_calls
order = map(lambda c: c[1][0], calls)
self.assertEqual(['urgent', 'high', 'medium', 'low'], order)
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_send_missing_sends_with_priority(self):
# pretend we have some pending uploads
_send = self.manager._send
self.manager._send = Mock(return_value=None)
content = "vegan cake"
length = len(content)
priorities = [
('low', Priority.LOW),
('high', Priority.HIGH),
('medium', Priority.MEDIUM),
('urgent', Priority.URGENT),
]
deferreds = []
for blob_id, priority in priorities:
doc = BlobDoc(BytesIO(content), blob_id)
d = self.manager.put(doc, length, priority=priority)
deferreds.append(d)
yield defer.gatherResults(deferreds)
# make sure upload "succeeds" so sending works
self.manager._send = _send
self.manager._encrypt_and_upload = Mock(return_value=None)
# this is the operation we are interested in
self.manager.concurrent_transfers_limit = 1
yield self.manager.send_missing()
# retrieve the order in which blob transfers were made
calls = self.manager._encrypt_and_upload.mock_calls
order = map(lambda c: c[1][0], calls)
self.assertEqual(['urgent', 'high', 'medium', 'low'], order)