def test_upload_deny_duplicates(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') fd = BytesIO("save me") yield manager._encrypt_and_upload('blob_id', fd) fd = BytesIO("save me") with pytest.raises(BlobAlreadyExistsError): yield manager._encrypt_and_upload('blob_id', fd)
def test_upload_deny_duplicates(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) fd = BytesIO("save me") yield manager._encrypt_and_upload('blob_id', fd) fd = BytesIO("save me") with pytest.raises(BlobAlreadyExistsError): yield manager._encrypt_and_upload('blob_id', fd)
def test_list_restricted_by_namespace(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') namespace = 'incoming' yield manager._encrypt_and_upload('blob_id1', BytesIO("1"), namespace=namespace) yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) blobs_list = yield manager.remote_list(namespace=namespace) self.assertEquals(['blob_id1'], blobs_list)
def test_list_default_doesnt_list_other_namespaces(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') namespace = 'incoming' yield manager._encrypt_and_upload('blob_id1', BytesIO("1"), namespace=namespace) yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) blobs_list = yield manager.remote_list() self.assertEquals(['blob_id2'], blobs_list)
def test_list_default_doesnt_list_other_namespaces(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) namespace = 'incoming' yield manager._encrypt_and_upload('blob_id1', BytesIO("1"), namespace=namespace) yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) blobs_list = yield manager.remote_list() self.assertEquals(['blob_id2'], blobs_list)
def test_upload_then_delete_updates_list_using_namespace(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') namespace = 'special_archives' yield manager._encrypt_and_upload('blob_id1', BytesIO("1"), namespace=namespace) yield manager._encrypt_and_upload('blob_id2', BytesIO("2"), namespace=namespace) yield manager._delete_from_remote('blob_id1', namespace=namespace) blobs_list = yield manager.remote_list(namespace=namespace) self.assertEquals(set(['blob_id2']), set(blobs_list))
def test_upload_then_delete_updates_list_using_namespace(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) namespace = 'special_archives' yield manager._encrypt_and_upload('blob_id1', BytesIO("1"), namespace=namespace) yield manager._encrypt_and_upload('blob_id2', BytesIO("2"), namespace=namespace) yield manager._delete_from_remote('blob_id1', namespace=namespace) blobs_list = yield manager.remote_list(namespace=namespace) deleted_blobs_list = yield manager.remote_list(namespace, deleted=True) self.assertEquals(set(['blob_id2']), set(blobs_list)) self.assertEquals(set(['blob_id1']), set(deleted_blobs_list))
def test_list_filter_flag_order_by_date(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') yield manager._encrypt_and_upload('blob_id1', BytesIO("x")) yield manager._encrypt_and_upload('blob_id2', BytesIO("x")) yield manager._encrypt_and_upload('blob_id3', BytesIO("x")) yield manager.set_flags('blob_id1', [Flags.PROCESSING]) yield manager.set_flags('blob_id2', [Flags.PROCESSING]) yield manager.set_flags('blob_id3', [Flags.PROCESSING]) blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING, order_by='+date') expected_list = ['blob_id1', 'blob_id2', 'blob_id3'] self.assertEquals(expected_list, blobs_list) blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING, order_by='-date') self.assertEquals(list(reversed(expected_list)), blobs_list)
def test_flags_ignored_by_listing(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') fd = BytesIO("flag me") yield manager._encrypt_and_upload('blob_id', fd) yield manager.set_flags('blob_id', [Flags.PROCESSING]) blobs_list = yield manager.remote_list() self.assertEquals(['blob_id'], blobs_list)
def test_list_filter_flag_order_by_date(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) yield manager._encrypt_and_upload('blob_id1', BytesIO("x")) yield manager._encrypt_and_upload('blob_id2', BytesIO("x")) yield manager._encrypt_and_upload('blob_id3', BytesIO("x")) yield manager.set_flags('blob_id1', [Flags.PROCESSING]) yield manager.set_flags('blob_id2', [Flags.PROCESSING]) yield manager.set_flags('blob_id3', [Flags.PROCESSING]) blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING, order_by='+date') expected_list = ['blob_id1', 'blob_id2', 'blob_id3'] self.assertEquals(expected_list, blobs_list) blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING, order_by='-date') self.assertEquals(list(reversed(expected_list)), blobs_list)
def test_upload_then_delete_updates_list(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') yield manager._encrypt_and_upload('blob_id1', BytesIO("1")) yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) yield manager._delete_from_remote('blob_id1') blobs_list = yield manager.remote_list() self.assertEquals(set(['blob_id2']), set(blobs_list))
def test_upload_download(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) fd = BytesIO("save me") yield manager._encrypt_and_upload('blob_id', fd) blob, size = yield manager._download_and_decrypt('blob_id') self.assertEquals(blob.getvalue(), "save me")
def test_set_get_flags(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') fd = BytesIO("flag me") yield manager._encrypt_and_upload('blob_id', fd) yield manager.set_flags('blob_id', [Flags.PROCESSING]) flags = yield manager.get_flags('blob_id') self.assertEquals([Flags.PROCESSING], flags)
def test_upload_changes_remote_list(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) yield manager._encrypt_and_upload('blob_id1', BytesIO("1")) yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) blobs_list = yield manager.remote_list() self.assertEquals(set(['blob_id1', 'blob_id2']), set(blobs_list))
def test_get_empty_flags(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) fd = BytesIO("flag me") yield manager._encrypt_and_upload('blob_id', fd) flags = yield manager.get_flags('blob_id') self.assertEquals([], flags)
def test_download_from_namespace(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') namespace, blob_id, content = 'incoming', 'blob_id1', 'test' yield manager._encrypt_and_upload(blob_id, BytesIO(content), namespace=namespace) got_blob = yield manager._download_and_decrypt(blob_id, namespace) self.assertEquals(content, got_blob[0].getvalue())
def test_cant_set_invalid_flags(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') fd = BytesIO("flag me") yield manager._encrypt_and_upload('blob_id', fd) with pytest.raises(InvalidFlagsError): yield manager.set_flags('blob_id', ['invalid']) flags = yield manager.get_flags('blob_id') self.assertEquals([], flags)
def test_flags_ignored_by_listing(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) fd = BytesIO("flag me") yield manager._encrypt_and_upload('blob_id', fd) yield manager.set_flags('blob_id', [Flags.PROCESSING]) blobs_list = yield manager.remote_list() self.assertEquals(['blob_id'], blobs_list)
def test_download_from_namespace(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) namespace, blob_id, content = 'incoming', 'blob_id1', 'test' yield manager._encrypt_and_upload(blob_id, BytesIO(content), namespace=namespace) got_blob = yield manager._download_and_decrypt(blob_id, namespace) self.assertEquals(content, got_blob[0].getvalue())
def test_downstream_from_namespace(self): manager = BlobManager(self.tempdir, self.uri, self.secret, self.secret, uuid4().hex, remote_stream=self.stream_uri) self.addCleanup(manager.close) namespace, blob_id, content = 'incoming', 'blob_id1', 'test' yield manager._encrypt_and_upload(blob_id, BytesIO(content), namespace=namespace) blob_id2, content2 = 'blob_id2', 'second test' yield manager._encrypt_and_upload(blob_id2, BytesIO(content2), namespace=namespace) blobs_list = [blob_id, blob_id2] yield manager._downstream(blobs_list, namespace) result = yield manager.local.get(blob_id, namespace) self.assertEquals(content, result.getvalue()) result = yield manager.local.get(blob_id2, namespace) self.assertEquals(content2, result.getvalue())
def test_count(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') deferreds = [] for i in range(10): deferreds.append(manager._encrypt_and_upload(str(i), BytesIO("1"))) yield defer.gatherResults(deferreds) result = yield manager.count() self.assertEquals({"count": len(deferreds)}, result)
def test_cant_set_invalid_flags(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) fd = BytesIO("flag me") yield manager._encrypt_and_upload('blob_id', fd) with pytest.raises(InvalidFlagsError): yield manager.set_flags('blob_id', ['invalid']) flags = yield manager.get_flags('blob_id') self.assertEquals([], flags)
def test_sync_fetch_missing(self): manager = BlobManager(self.tempdir, self.uri, self.secret, self.secret, uuid4().hex) self.addCleanup(manager.close) blob_id = 'remote_only_blob_id' yield manager._encrypt_and_upload(blob_id, BytesIO("X")) yield manager.sync() result = yield manager.local.get(blob_id) self.assertIsNotNone(result) self.assertEquals(result.getvalue(), "X")
def test_list_filter_flag(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) fd = BytesIO("flag me") yield manager._encrypt_and_upload('blob_id', fd) yield manager.set_flags('blob_id', [Flags.PROCESSING]) blobs_list = yield manager.remote_list(filter_flag=Flags.PENDING) self.assertEquals([], blobs_list) blobs_list = yield manager.remote_list(filter_flag=Flags.PROCESSING) self.assertEquals(['blob_id'], blobs_list)
def test_count(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) deferreds = [] for i in range(10): deferreds.append(manager._encrypt_and_upload(str(i), BytesIO("1"))) yield defer.gatherResults(deferreds) result = yield manager.count() self.assertEquals({"count": len(deferreds)}, result)
def test_upload_then_delete_updates_list(self): manager = BlobManager('', self.uri, self.secret, self.secret, uuid4().hex) yield manager._encrypt_and_upload('blob_id1', BytesIO("1")) yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) yield manager._delete_from_remote('blob_id1') blobs_list = yield manager.remote_list() deleted_blobs_list = yield manager.remote_list(deleted=True) self.assertEquals(set(['blob_id2']), set(blobs_list)) self.assertEquals(set(['blob_id1']), set(deleted_blobs_list))
def test_fetch_missing(self): manager = BlobManager(self.tempdir, self.uri, self.secret, self.secret, 'user') self.addCleanup(manager.close) blob_id = 'remote_only_blob_id' yield manager._encrypt_and_upload(blob_id, BytesIO("X")) yield manager.fetch_missing() result = yield manager.local.get(blob_id) self.assertIsNotNone(result) self.assertEquals(result.getvalue(), "X")
def test_list_orders_by_date(self): manager = BlobManager('', self.uri, self.secret, self.secret, 'user') yield manager._encrypt_and_upload('blob_id1', BytesIO("1")) yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) blobs_list = yield manager.remote_list(order_by='date') self.assertEquals(['blob_id1', 'blob_id2'], blobs_list) parts = ['user', 'default', 'b', 'blo', 'blob_i', 'blob_id1'] self.__touch(self.tempdir, *parts) blobs_list = yield manager.remote_list(order_by='+date') self.assertEquals(['blob_id2', 'blob_id1'], blobs_list) blobs_list = yield manager.remote_list(order_by='-date') self.assertEquals(['blob_id1', 'blob_id2'], blobs_list)
def test_list_orders_by_date(self): user_uid = uuid4().hex manager = BlobManager('', self.uri, self.secret, self.secret, user_uid) yield manager._encrypt_and_upload('blob_id1', BytesIO("1")) yield manager._encrypt_and_upload('blob_id2', BytesIO("2")) blobs_list = yield manager.remote_list(order_by='date') self.assertEquals(['blob_id1', 'blob_id2'], blobs_list) parts = [user_uid, 'default', 'b', 'blo', 'blob_i', 'blob_id1'] self.__touch(self.tempdir, *parts) blobs_list = yield manager.remote_list(order_by='+date') self.assertEquals(['blob_id2', 'blob_id1'], blobs_list) blobs_list = yield manager.remote_list(order_by='-date') self.assertEquals(['blob_id1', 'blob_id2'], blobs_list)
def test_decrypt_uploading_encrypted_blob(self): @defer.inlineCallbacks def _check_result(uri, data, *args, **kwargs): decryptor = _crypto.BlobDecryptor(self.doc_info, data, armor=False, secret=self.secret) decrypted = yield decryptor.decrypt() self.assertEquals(decrypted.getvalue(), 'up and up') defer.returnValue(Mock(code=200)) manager = BlobManager('', '', self.secret, self.secret, 'user') fd = BytesIO('up and up') manager._client.put = _check_result yield manager._encrypt_and_upload(self.doc_info.doc_id, fd)
def test_decrypt_uploading_encrypted_blob(self): @defer.inlineCallbacks def _check_result(uri, data, *args, **kwargs): decryptor = _crypto.BlobDecryptor( self.doc_info, data, armor=False, secret=self.secret) decrypted = yield decryptor.decrypt() self.assertEquals(decrypted.getvalue(), 'up and up') defer.returnValue(Mock(code=200)) manager = BlobManager('', '', self.secret, self.secret, 'user') fd = BytesIO('up and up') manager._client.put = _check_result yield manager._encrypt_and_upload(self.doc_info.doc_id, fd)
def test_sync_fetch_missing_retry(self): manager = BlobManager(self.tempdir, self.uri, self.secret, self.secret, uuid4().hex) self.addCleanup(manager.close) blob_id = 'remote_only_blob_id' yield manager._encrypt_and_upload(blob_id, BytesIO("X")) yield manager.refresh_sync_status_from_server() yield self.port.stopListening() d = manager.fetch_missing() yield sleep(0.1) self.port = reactor.listenTCP( self.host.port, self.site, interface='127.0.0.1') yield d result = yield manager.local.get(blob_id) self.assertIsNotNone(result) self.assertEquals(result.getvalue(), "X")
def test_download_corrupted_tag_marks_blob_as_failed(self): user_id = uuid4().hex manager = BlobManager(self.tempdir, self.uri, self.secret, self.secret, user_id) self.addCleanup(manager.close) blob_id = 'corrupted' yield manager._encrypt_and_upload(blob_id, BytesIO("corrupted")) parts = ['default'] + [blob_id[0], blob_id[0:3], blob_id[0:6]] parts += [blob_id] corrupted_blob_path = os.path.join(self.tempdir, user_id, *parts) with open(corrupted_blob_path, 'r+b') as corrupted_blob: # Corrupt the tag (last 16 bytes) corrupted_blob.seek(-16, 2) corrupted_blob.write('x' * 16) with pytest.raises(MaximumRetriesError): yield manager.sync() status, retries = yield manager.local.get_sync_status(blob_id) self.assertEquals(status, SyncStatus.FAILED_DOWNLOAD) self.assertEquals(retries, 3)