def test_upload_download(self):
manager = BlobManager('', self.uri, self.secret,
self.secret, uuid4().hex)
fd = BytesIO("save me")
yield manager._encrypt_and_upload('blob_id', fd)
blob, size = yield manager._download_and_decrypt('blob_id')
self.assertEquals(blob.getvalue(), "save me")
def test_upload_then_delete_updates_list(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
yield manager._encrypt_and_upload('blob_id1', BytesIO("1"))
yield manager._encrypt_and_upload('blob_id2', BytesIO("2"))
yield manager._delete_from_remote('blob_id1')
blobs_list = yield manager.remote_list()
self.assertEquals(set(['blob_id2']), set(blobs_list))
def test_upload_changes_remote_list(self):
manager = BlobManager('', self.uri, self.secret,
self.secret, uuid4().hex)
yield manager._encrypt_and_upload('blob_id1', BytesIO("1"))
yield manager._encrypt_and_upload('blob_id2', BytesIO("2"))
blobs_list = yield manager.remote_list()
self.assertEquals(set(['blob_id1', 'blob_id2']), set(blobs_list))
def test_get_empty_flags(self):
manager = BlobManager('', self.uri, self.secret,
self.secret, uuid4().hex)
fd = BytesIO("flag me")
yield manager._encrypt_and_upload('blob_id', fd)
flags = yield manager.get_flags('blob_id')
self.assertEquals([], flags)
def test_set_get_flags(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
fd = BytesIO("flag me")
yield manager._encrypt_and_upload('blob_id', fd)
yield manager.set_flags('blob_id', [Flags.PROCESSING])
flags = yield manager.get_flags('blob_id')
self.assertEquals([Flags.PROCESSING], flags)
def test_flags_ignored_by_listing(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
fd = BytesIO("flag me")
yield manager._encrypt_and_upload('blob_id', fd)
yield manager.set_flags('blob_id', [Flags.PROCESSING])
blobs_list = yield manager.remote_list()
self.assertEquals(['blob_id'], blobs_list)
def test_upload_deny_duplicates(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
fd = BytesIO("save me")
yield manager._encrypt_and_upload('blob_id', fd)
fd = BytesIO("save me")
with pytest.raises(BlobAlreadyExistsError):
yield manager._encrypt_and_upload('blob_id', fd)
def test_cant_set_invalid_flags(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
fd = BytesIO("flag me")
yield manager._encrypt_and_upload('blob_id', fd)
with pytest.raises(InvalidFlagsError):
yield manager.set_flags('blob_id', ['invalid'])
flags = yield manager.get_flags('blob_id')
self.assertEquals([], flags)
def test_download_from_namespace(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
namespace, blob_id, content = 'incoming', 'blob_id1', 'test'
yield manager._encrypt_and_upload(blob_id,
BytesIO(content),
namespace=namespace)
got_blob = yield manager._download_and_decrypt(blob_id, namespace)
self.assertEquals(content, got_blob[0].getvalue())
def test_list_restricted_by_namespace(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
namespace = 'incoming'
yield manager._encrypt_and_upload('blob_id1',
BytesIO("1"),
namespace=namespace)
yield manager._encrypt_and_upload('blob_id2', BytesIO("2"))
blobs_list = yield manager.remote_list(namespace=namespace)
self.assertEquals(['blob_id1'], blobs_list)
def test_list_default_doesnt_list_other_namespaces(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
namespace = 'incoming'
yield manager._encrypt_and_upload('blob_id1',
BytesIO("1"),
namespace=namespace)
yield manager._encrypt_and_upload('blob_id2', BytesIO("2"))
blobs_list = yield manager.remote_list()
self.assertEquals(['blob_id2'], blobs_list)
def test_count(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
deferreds = []
for i in range(10):
deferreds.append(manager._encrypt_and_upload(str(i), BytesIO("1")))
yield defer.gatherResults(deferreds)
result = yield manager.count()
self.assertEquals({"count": len(deferreds)}, result)
def test_send_missing(self):
manager = BlobManager(self.tempdir, self.uri, self.secret, self.secret,
'user')
self.addCleanup(manager.close)
blob_id = 'local_only_blob_id'
yield manager.local.put(blob_id, BytesIO("X"), size=1)
yield manager.send_missing()
result = yield manager._download_and_decrypt(blob_id)
self.assertIsNotNone(result)
self.assertEquals(result[0].getvalue(), "X")
def test_fetch_missing(self):
manager = BlobManager(self.tempdir, self.uri, self.secret, self.secret,
'user')
self.addCleanup(manager.close)
blob_id = 'remote_only_blob_id'
yield manager._encrypt_and_upload(blob_id, BytesIO("X"))
yield manager.fetch_missing()
result = yield manager.local.get(blob_id)
self.assertIsNotNone(result)
self.assertEquals(result.getvalue(), "X")
def test_send_missing(self):
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, uuid4().hex)
self.addCleanup(manager.close)
blob_id = 'local_only_blob_id'
yield manager.local.put(blob_id, BytesIO("X"), size=1)
pending = SyncStatus.PENDING_UPLOAD
yield manager.local.update_sync_status(blob_id, pending)
yield manager.send_missing()
result = yield manager._download_and_decrypt(blob_id)
self.assertIsNotNone(result)
self.assertEquals(result[0].getvalue(), "X")
def test_upstream_from_namespace(self):
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, uuid4().hex,
remote_stream=self.stream_uri)
self.addCleanup(manager.close)
blob_ids = [uuid4().hex for _ in range(5)]
for i, blob_id in enumerate(blob_ids):
yield manager.local.put(blob_id, BytesIO("X" * i), size=i,
namespace='test')
yield manager._upstream(blob_ids, namespace='test')
for i, blob_id in enumerate(blob_ids):
got_blob = yield manager._download_and_decrypt(blob_id,
namespace='test')
self.assertEquals(got_blob[0].getvalue(), "X" * i)
def test_decrypt_uploading_encrypted_blob(self):
@defer.inlineCallbacks
def _check_result(uri, data, *args, **kwargs):
decryptor = _crypto.BlobDecryptor(self.doc_info,
data,
armor=False,
secret=self.secret)
decrypted = yield decryptor.decrypt()
self.assertEquals(decrypted.getvalue(), 'up and up')
defer.returnValue(Mock(code=200))
manager = BlobManager('', '', self.secret, self.secret, 'user')
fd = BytesIO('up and up')
manager._client.put = _check_result
yield manager._encrypt_and_upload(self.doc_info.doc_id, fd)
def test_get_range(self):
user_id = uuid4().hex
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, user_id)
self.addCleanup(manager.close)
blob_id, content = 'blob_id', '0123456789'
doc = BlobDoc(BytesIO(content), blob_id)
yield manager.put(doc, len(content))
uri = urljoin(self.uri, '%s/%s' % (user_id, blob_id))
res = yield _get(uri, headers={'Range': 'bytes=10-20'})
text = yield res.text()
self.assertTrue(res.headers.hasHeader('content-range'))
content_range = res.headers.getRawHeaders('content-range').pop()
self.assertIsNotNone(re.match('^bytes 10-20/[0-9]+$', content_range))
self.assertEqual(10, len(text))
def test_refresh_deletions_from_server(self):
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, uuid4().hex)
self.addCleanup(manager.close)
blob_id, content = 'delete_me', 'content'
blob_id2 = 'dont_delete_me'
doc1 = BlobDoc(BytesIO(content), blob_id)
doc2 = BlobDoc(BytesIO(content), blob_id2)
yield manager.put(doc1, len(content))
yield manager.put(doc2, len(content))
yield manager._delete_from_remote(blob_id) # remote only deletion
self.assertTrue((yield manager.local.exists(blob_id)))
yield manager.sync()
self.assertFalse((yield manager.local.exists(blob_id)))
self.assertTrue((yield manager.local.exists(blob_id2)))
def test_sync_fetch_missing_retry(self):
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, uuid4().hex)
self.addCleanup(manager.close)
blob_id = 'remote_only_blob_id'
yield manager._encrypt_and_upload(blob_id, BytesIO("X"))
yield manager.refresh_sync_status_from_server()
yield self.port.stopListening()
d = manager.fetch_missing()
yield sleep(0.1)
self.port = reactor.listenTCP(
self.host.port, self.site, interface='127.0.0.1')
yield d
result = yield manager.local.get(blob_id)
self.assertIsNotNone(result)
self.assertEquals(result.getvalue(), "X")
def test_downstream_from_namespace(self):
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, uuid4().hex,
remote_stream=self.stream_uri)
self.addCleanup(manager.close)
namespace, blob_id, content = 'incoming', 'blob_id1', 'test'
yield manager._encrypt_and_upload(blob_id, BytesIO(content),
namespace=namespace)
blob_id2, content2 = 'blob_id2', 'second test'
yield manager._encrypt_and_upload(blob_id2, BytesIO(content2),
namespace=namespace)
blobs_list = [blob_id, blob_id2]
yield manager._downstream(blobs_list, namespace)
result = yield manager.local.get(blob_id, namespace)
self.assertEquals(content, result.getvalue())
result = yield manager.local.get(blob_id2, namespace)
self.assertEquals(content2, result.getvalue())
class IncomingFlowIntegrationTestCase(unittest.TestCase):
def setUp(self):
root = Resource()
state = BlobsServerState('filesystem', blobs_path=self.tempdir)
incoming_resource = IncomingResource(state)
blobs_resource = server_blobs.BlobsResource("filesystem", self.tempdir)
root.putChild('blobs', blobs_resource)
root.putChild('incoming', incoming_resource)
site = Site(root)
self.port = reactor.listenTCP(0, site, interface='127.0.0.1')
self.host = self.port.getHost()
self.uri = 'http://%s:%s/' % (self.host.host, self.host.port)
self.blobs_uri = self.uri + 'blobs/'
self.incoming_uri = self.uri + 'incoming'
self.user_id = 'user-' + uuid4().hex
self.secret = 'A' * 96
self.blob_manager = BlobManager(self.tempdir, self.blobs_uri,
self.secret, self.secret,
self.user_id)
self.box = IncomingBox(self.blob_manager, 'MX')
self.loop = IncomingBoxProcessingLoop(self.box)
# FIXME: We use blob_manager client only to avoid DelayedCalls
# Somehow treq being used here keeps a connection pool open
self.client = self.blob_manager._client
def fill(self, messages):
deferreds = []
for message_id, message in messages:
uri = '%s/%s/%s' % (self.incoming_uri, self.user_id, message_id)
deferreds.append(self.blob_manager._client.put(uri, data=message))
return defer.gatherResults(deferreds)
def tearDown(self):
self.port.stopListening()
self.blob_manager.close()
@defer.inlineCallbacks
@pytest.mark.usefixtures("method_tmpdir")
def test_consume_a_incoming_message(self):
yield self.fill([('msg1', 'blob')])
consumer = GoodConsumer()
self.loop.add_consumer(consumer)
yield self.loop()
self.assertIn('msg1', consumer.processed)
def test_send_missing_retry(self):
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, uuid4().hex)
self.addCleanup(manager.close)
blob_id = 'remote_only_blob_id'
yield manager.local.put(blob_id, BytesIO("X"), size=1)
pending = SyncStatus.PENDING_UPLOAD
yield manager.local.update_sync_status(blob_id, pending)
yield self.port.stopListening()
d = manager.send_missing()
yield sleep(0.1)
self.port = reactor.listenTCP(
self.host.port, self.site, interface='127.0.0.1')
yield d
result = yield manager._download_and_decrypt(blob_id)
self.assertIsNotNone(result)
self.assertEquals(result[0].getvalue(), "X")
def test_download_corrupted_tag_marks_blob_as_failed(self):
user_id = uuid4().hex
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, user_id)
self.addCleanup(manager.close)
blob_id = 'corrupted'
yield manager._encrypt_and_upload(blob_id, BytesIO("corrupted"))
parts = ['default'] + [blob_id[0], blob_id[0:3], blob_id[0:6]]
parts += [blob_id]
corrupted_blob_path = os.path.join(self.tempdir, user_id, *parts)
with open(corrupted_blob_path, 'r+b') as corrupted_blob:
# Corrupt the tag (last 16 bytes)
corrupted_blob.seek(-16, 2)
corrupted_blob.write('x' * 16)
with pytest.raises(MaximumRetriesError):
yield manager.sync()
status, retries = yield manager.local.get_sync_status(blob_id)
self.assertEquals(status, SyncStatus.FAILED_DOWNLOAD)
self.assertEquals(retries, 3)
def test_list_filter_flag_order_by_date(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
yield manager._encrypt_and_upload('blob_id1', BytesIO("x"))
yield manager._encrypt_and_upload('blob_id2', BytesIO("x"))
yield manager._encrypt_and_upload('blob_id3', BytesIO("x"))
yield manager.set_flags('blob_id1', [Flags.PROCESSING])
yield manager.set_flags('blob_id2', [Flags.PROCESSING])
yield manager.set_flags('blob_id3', [Flags.PROCESSING])
blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING,
order_by='+date')
expected_list = ['blob_id1', 'blob_id2', 'blob_id3']
self.assertEquals(expected_list, blobs_list)
blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING,
order_by='-date')
self.assertEquals(list(reversed(expected_list)), blobs_list)
def setUp(self):
root = Resource()
state = BlobsServerState('filesystem', blobs_path=self.tempdir)
incoming_resource = IncomingResource(state)
blobs_resource = server_blobs.BlobsResource("filesystem", self.tempdir)
root.putChild('blobs', blobs_resource)
root.putChild('incoming', incoming_resource)
site = Site(root)
self.port = reactor.listenTCP(0, site, interface='127.0.0.1')
self.host = self.port.getHost()
self.uri = 'http://%s:%s/' % (self.host.host, self.host.port)
self.blobs_uri = self.uri + 'blobs/'
self.incoming_uri = self.uri + 'incoming'
self.user_id = 'user-' + uuid4().hex
self.secret = 'A' * 96
self.blob_manager = BlobManager(self.tempdir, self.blobs_uri,
self.secret, self.secret,
self.user_id)
self.box = IncomingBox(self.blob_manager, 'MX')
self.loop = IncomingBoxProcessingLoop(self.box)
# FIXME: We use blob_manager client only to avoid DelayedCalls
# Somehow treq being used here keeps a connection pool open
self.client = self.blob_manager._client
def test_upload_then_delete_updates_list_using_namespace(self):
manager = BlobManager('', self.uri, self.secret,
self.secret, uuid4().hex)
namespace = 'special_archives'
yield manager._encrypt_and_upload('blob_id1', BytesIO("1"),
namespace=namespace)
yield manager._encrypt_and_upload('blob_id2', BytesIO("2"),
namespace=namespace)
yield manager._delete_from_remote('blob_id1', namespace=namespace)
blobs_list = yield manager.remote_list(namespace=namespace)
deleted_blobs_list = yield manager.remote_list(namespace, deleted=True)
self.assertEquals(set(['blob_id2']), set(blobs_list))
self.assertEquals(set(['blob_id1']), set(deleted_blobs_list))
def test_get_range_not_satisfiable(self):
# put a blob in place
user_id = uuid4().hex
manager = BlobManager(self.tempdir, self.uri, self.secret,
self.secret, user_id)
self.addCleanup(manager.close)
blob_id, content = uuid4().hex, 'content'
doc = BlobDoc(BytesIO(content), blob_id)
yield manager.put(doc, len(content))
# and check possible parsing errors
uri = urljoin(self.uri, '%s/%s' % (user_id, blob_id))
ranges = [
'bytes',
'bytes=',
'bytes=1',
'bytes=blah-100',
'potatoes=10-100'
'blah'
]
for range in ranges:
res = yield _get(uri, headers={'Range': range})
self.assertEqual(416, res.code)
content_range = res.headers.getRawHeaders('content-range').pop()
self.assertIsNotNone(re.match('^bytes \*/[0-9]+$', content_range))
def test_list_filter_flag(self):
manager = BlobManager('', self.uri, self.secret,
self.secret, uuid4().hex)
fd = BytesIO("flag me")
yield manager._encrypt_and_upload('blob_id', fd)
yield manager.set_flags('blob_id', [Flags.PROCESSING])
blobs_list = yield manager.remote_list(filter_flag=Flags.PENDING)
self.assertEquals([], blobs_list)
blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING)
self.assertEquals(['blob_id'], blobs_list)
def test_list_orders_by_date(self):
manager = BlobManager('', self.uri, self.secret, self.secret, 'user')
yield manager._encrypt_and_upload('blob_id1', BytesIO("1"))
yield manager._encrypt_and_upload('blob_id2', BytesIO("2"))
blobs_list = yield manager.remote_list(order_by='date')
self.assertEquals(['blob_id1', 'blob_id2'], blobs_list)
parts = ['user', 'default', 'b', 'blo', 'blob_i', 'blob_id1']
self.__touch(self.tempdir, *parts)
blobs_list = yield manager.remote_list(order_by='+date')
self.assertEquals(['blob_id2', 'blob_id1'], blobs_list)
blobs_list = yield manager.remote_list(order_by='-date')
self.assertEquals(['blob_id1', 'blob_id2'], blobs_list)