async def upload(self, bundle: Bundle, identity: Identity) -> Revision: vault = cast( Vault, self.vault) # We can savely cast because of @require_vault logger.info("Uploading %s", bundle) dest_path = os.path.join(self.path, bundle.store_hash) shutil.copyfile(bundle.path, dest_path) metadata = await bundle.encrypted_metadata_reader().readall() len(metadata) if bundle.local_hash is None: raise ValueError("Please update bundle before upload.") await bundle.load_key() s = vault.crypt_engine.read_encrypted_stream(bundle) >> FileWriter( dest_path) await s.consume() # s = bundle.encrypted_metadata_reader() >> FileWriter(dest_path + ".metadata") # await s.consume() with open(dest_path + ".hash", "w") as hashfile: hashfile.write(bundle.local_hash) revision = Revision(operation=RevisionOp.Upload) revision.vault_id = vault.config.id revision.parent_id = vault.revision #revision.user_id = "user@localhost" revision.file_hash = bundle.store_hash revision.revision_metadata = metadata revision.crypt_hash = bundle.local_hash revision.file_size_crypt = bundle.file_size_crypt revision.sign(identity=identity) return self.add_revision(revision)
async def remove_user_vault_key(self, identity: Identity, user_id: str, user_identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") self.logger.debug('Removing user vault key') revision = Revision(operation=RevisionOp.RemoveUserKey) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.user_id = user_id revision.user_public_key = user_identity.public_key.exportKey("DER") revision.sign(identity=identity) # upload metadata await self.write_term('remove_user_vault_key', user_id, revision.user_public_key, user_identity.get_fingerprint(), revision.user_fingerprint, revision.parent_id, revision.signature) # assert :ok response = await self.read_response() ret_revision = self.server_info_to_revision( rewrite_atoms_dict(response), vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at return revision
async def remove_user_vault_key(self, identity: Identity, user_id: str, user_identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") self.logger.debug('Removing user vault key') revision = Revision(operation=RevisionOp.RemoveUserKey) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.user_id = user_id revision.user_public_key = user_identity.public_key.exportKey("DER") revision.sign(identity=identity) # upload metadata await self.write_term('remove_user_vault_key', user_id, revision.user_public_key, user_identity.get_fingerprint(), revision.user_fingerprint, revision.parent_id, revision.signature ) # assert :ok response = await self.read_response() ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at return revision
async def upload(self, bundle: Bundle, identity: Identity) -> Revision: vault = cast(Vault, self.vault) # We can savely cast because of @require_vault logger.info("Uploading %s", bundle) dest_path = os.path.join(self.path, bundle.store_hash) shutil.copyfile(bundle.path, dest_path) metadata = await bundle.encrypted_metadata_reader().readall() len(metadata) if bundle.local_hash is None: raise ValueError("Please update bundle before upload.") await bundle.load_key() s = vault.crypt_engine.read_encrypted_stream(bundle) >> FileWriter(dest_path) await s.consume() # s = bundle.encrypted_metadata_reader() >> FileWriter(dest_path + ".metadata") # await s.consume() with open(dest_path + ".hash", "w") as hashfile: hashfile.write(bundle.local_hash) revision = Revision(operation=RevisionOp.Upload) revision.vault_id = vault.config.id revision.parent_id = vault.revision #revision.user_id = "user@localhost" revision.file_hash = bundle.store_hash revision.revision_metadata = metadata revision.crypt_hash = bundle.local_hash revision.file_size_crypt = bundle.file_size_crypt revision.sign(identity=identity) return self.add_revision(revision)
async def add_vault_user(self, user_id: str, identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") revision = Revision(operation=RevisionOp.AddUser) revision.vault_id = vault.config.id revision.user_id = user_id revision.parent_id = vault.revision revision.sign(identity=identity) await self.write_term('add_vault_user', revision.user_id, revision.user_fingerprint, revision.parent_id, revision.signature) # assert :ok response = await self.read_response() ret_revision = self.server_info_to_revision( rewrite_atoms_dict(response), vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at return revision
async def set_vault_metadata(self, identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") metadata = await vault.encrypted_metadata_reader().readall() self.logger.debug('Setting metadata for %s (%d bytes)', self.vault, len(metadata)) revision = Revision(operation=RevisionOp.SetMetadata) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.revision_metadata = metadata revision.sign(identity=identity) # upload metadata await self.write_term('set_vault_metadata', metadata, revision.user_fingerprint, revision.parent_id, revision.signature) # assert :ok response = await self.read_response() ret_revision = self.server_info_to_revision( rewrite_atoms_dict(response), vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at return revision
async def remove_file(self, bundle, identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") self.logger.info('Removing %s', bundle) revision = Revision(operation=RevisionOp.RemoveFile) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.file_hash = bundle.store_hash revision.sign(identity=identity) # upload key and file await self.write_term('remove_file', revision.file_hash, revision.user_fingerprint, revision.signature, revision.parent_id) # assert :ok response = await self.read_response() ret_revision = self.server_info_to_revision( rewrite_atoms_dict(response), vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at return revision
async def set_vault_metadata(self, identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") metadata = await vault.encrypted_metadata_reader().readall() self.logger.debug('Setting metadata for %s (%d bytes)', self.vault, len(metadata)) revision = Revision(operation=RevisionOp.SetMetadata) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.revision_metadata = metadata revision.sign(identity=identity) # upload metadata await self.write_term('set_vault_metadata', metadata, revision.user_fingerprint, revision.parent_id, revision.signature) # assert :ok response = await self.read_response() ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at return revision
async def add_vault_user(self, user_id: str, identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") revision = Revision(operation=RevisionOp.AddUser) revision.vault_id = vault.config.id revision.user_id = user_id revision.parent_id = vault.revision revision.sign(identity=identity) await self.write_term('add_vault_user', revision.user_id, revision.user_fingerprint, revision.parent_id, revision.signature) # assert :ok response = await self.read_response() ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at return revision
def generate_fake_revision(vault): revision = Revision(operation=RevisionOp.SetMetadata) revision.vault_id = vault.id revision.parent_id = vault.revision revision.user_id = "user@localhost" revision.user_fingerprint = "aabbcc" revision.revision_metadata = b"123456" revision.signature = b"12345" return revision
def server_info_to_revision(self, server_info, vault: Vault, parent_id: Optional[str] = None): operation = server_info['operation'].decode() vault_public_key = None user_id = None if operation == 'store': operation = RevisionOp.Upload elif operation == 'create_vault': operation = RevisionOp.CreateVault vault_public_key = vault.identity.export_public_key() user_id = server_info['metadata'].decode() elif operation == 'set_metadata': operation = RevisionOp.SetMetadata vault_public_key = vault.identity.export_public_key() elif operation == 'add_user': operation = RevisionOp.AddUser user_id = server_info['metadata'].decode() elif operation == 'remove_user': operation = RevisionOp.RemoveUser user_id = server_info['metadata'].decode() elif operation == 'add_user_vault_key': operation = RevisionOp.AddUserKey user_id = server_info['metadata'].decode() elif operation == 'remove_user_vault_key': operation = RevisionOp.RemoveUserKey user_id = server_info['metadata'].decode() else: raise ServerError("Unknown operation: " + operation) user_fingerprint = server_info['user_key_fingerprint'].decode() signature = server_info['signature'] file_hash = server_info['file_hash'].decode( ) if server_info['file_hash'] is not None else None crypt_hash = server_info['content_hash'].decode( ) if server_info['content_hash'] is not None else None file_size_crypt = server_info['size'] metadata = server_info['metadata'] user_public_key = server_info['user_public_key'] revision_id = server_info['id'].decode() created_at = \ iso8601.parse_date(server_info['created_at'].decode()) return Revision(operation=operation, revision_id=revision_id, parent_id=parent_id, vault_id=vault.config.id, created_at=created_at, revision_metadata=metadata, file_size_crypt=file_size_crypt, file_hash=file_hash, crypt_hash=crypt_hash, user_id=user_id, signature=signature, user_public_key=user_public_key, vault_public_key=vault_public_key, user_fingerprint=user_fingerprint)
async def create_vault(self, identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") revision = Revision(operation=RevisionOp.CreateVault) revision.vault_public_key = vault.identity.public_key.exportKey("DER") revision.user_public_key = identity.public_key.exportKey("DER") user_info = await self.user_info() revision.user_id = user_info['email'] revision.sign(identity=identity) await self.write_term('create_vault', revision.vault_public_key, revision.user_public_key, revision.user_fingerprint, revision.signature) response = await self.read_term() vault_id = response[1].decode(vault.config.encoding) auth = response[2].decode(vault.config.encoding) server_info = rewrite_atoms_dict(response[3]) if not vault_id: raise ServerError("Invalid vault ID: {0}".format(vault_id)) if not auth: raise ServerError("Invalid auth token: {0}".format(auth)) revision.vault_id = vault_id # assert :ok ret_revision = self.server_info_to_revision(server_info, vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at self.logger.info('Successfully created vault %s', vault_id) with vault.config.update_context(): vault.config.update('remote', {'auth': auth}) vault.config.update( 'vault', {'id': response[1].decode(vault.config.encoding)}) return revision
async def create_vault(self, identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") revision = Revision(operation=RevisionOp.CreateVault) revision.vault_public_key = vault.identity.public_key.exportKey("DER") revision.user_public_key = identity.public_key.exportKey("DER") revision.user_id = '*****@*****.**' # TBD revision.sign(identity=identity) await self.write_term('create_vault', revision.vault_public_key, revision.user_public_key, revision.user_fingerprint, revision.signature) response = await self.read_term() vault_id = response[1].decode(vault.config.encoding) auth = response[2].decode(vault.config.encoding) server_info = rewrite_atoms_dict(response[3]) if not vault_id: raise ServerError("Invalid vault ID: {0}".format(vault_id)) if not auth: raise ServerError("Invalid auth token: {0}".format(auth)) revision.vault_id = vault_id # assert :ok ret_revision = self.server_info_to_revision(server_info, vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at self.logger.info('Successfully created vault %s', vault_id) with vault.config.update_context(): vault.config.update('remote', { 'auth': auth }) vault.config.update('vault', { 'id': response[1].decode(vault.config.encoding) }) return revision
def add_revision(self, revision: Revision) -> Revision: "Persist the revision in the local storage. This will also generate a revision id." if revision.revision_id is not None: raise ValueError("Revision already has an id.") if revision.signature is None: raise ValueError("Revision is not signed.") revision.revision_id = str(uuid4()) revision.created_at = datetime.utcnow() with open(os.path.join(self.path, "txchain"), "ab") as txchain: logger.debug( "Adding revision %s to signchain (%s)", revision.revision_id, os.path.join(self.path, "txchain"), ) binary_tx = pickle.dumps(revision) txchain.write(binary_tx) return revision
def add_revision(self, revision: Revision) -> Revision: "Persist the revision in the local storage. This will also generate a revision id." if revision.revision_id is not None: raise ValueError("Revision already has an id.") if revision.signature is None: raise ValueError("Revision is not signed.") revision.revision_id = str(uuid4()) revision.created_at = datetime.utcnow() with open(os.path.join(self.path, "txchain"), "ab") as txchain: logger.debug( "Adding revision %s to signchain (%s)", revision.revision_id, os.path.join(self.path, "txchain"), ) binary_tx = pickle.dumps(revision) txchain.write(binary_tx) return revision
async def remove_user_vault_key(self, identity: Identity, user_id: str, user_identity: Identity): vault = cast( Vault, self.vault) # We can savely cast because of @require_vault logger.info("Removing user vault key %s", user_id) revision = Revision(operation=RevisionOp.RemoveUserKey) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.user_public_key = user_identity.public_key.exportKey('DER') revision.user_id = user_id revision.sign(identity=identity) return self.add_revision(revision)
async def remove_file(self, bundle: Bundle, identity: Identity) -> Revision: vault = cast( Vault, self.vault) # We can savely cast because of @require_vault assert bundle.store_hash logger.info("Deleting %s", bundle) revision = Revision(operation=RevisionOp.RemoveFile) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.user_id = "user@localhost" revision.file_hash = bundle.store_hash revision.sign(identity=identity) return self.add_revision(revision)
async def remove_vault_user(self, user_id: str, identity: Identity) -> Revision: vault = cast( Vault, self.vault) # We can savely cast because of @require_vault logger.info("Remove user %s", user_id) revision = Revision(operation=RevisionOp.RemoveUser) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.user_id = user_id revision.sign(identity=identity) return self.add_revision(revision)
async def remove_user_vault_key(self, identity: Identity, user_id: str, user_identity: Identity): vault = cast(Vault, self.vault) # We can savely cast because of @require_vault logger.info("Removing user vault key %s", user_id) revision = Revision(operation=RevisionOp.RemoveUserKey) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.user_public_key = user_identity.public_key.exportKey('DER') revision.user_id = user_id revision.sign(identity=identity) return self.add_revision(revision)
async def set_vault_metadata(self, identity: Identity) -> Revision: vault = cast( Vault, self.vault) # We can savely cast because of @require_vault dest_path = os.path.join(self.path, "metadata") writer = vault.encrypted_metadata_reader() >> FileWriter( dest_path, create_dirs=True) await writer.consume() metadata = await vault.encrypted_metadata_reader().readall() revision = Revision(operation=RevisionOp.SetMetadata) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.user_id = "user@localhost" revision.revision_metadata = metadata revision.sign(identity) return self.add_revision(revision)
async def remove_file(self, bundle: Bundle, identity: Identity) -> Revision: vault = cast(Vault, self.vault) # We can savely cast because of @require_vault assert bundle.store_hash logger.info("Deleting %s", bundle) revision = Revision(operation=RevisionOp.RemoveFile) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.user_id = "user@localhost" revision.file_hash = bundle.store_hash revision.sign(identity=identity) return self.add_revision(revision)
async def remove_vault_user(self, user_id: str, identity: Identity) -> Revision: vault = cast(Vault, self.vault) # We can savely cast because of @require_vault logger.info("Remove user %s", user_id) revision = Revision(operation=RevisionOp.RemoveUser) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.user_id = user_id revision.sign(identity=identity) return self.add_revision(revision)
async def set_vault_metadata(self, identity: Identity) -> Revision: vault = cast(Vault, self.vault) # We can savely cast because of @require_vault dest_path = os.path.join(self.path, "metadata") writer = vault.encrypted_metadata_reader() >> FileWriter( dest_path, create_dirs=True ) await writer.consume() metadata = await vault.encrypted_metadata_reader().readall() revision = Revision(operation=RevisionOp.SetMetadata) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.user_id = "user@localhost" revision.revision_metadata = metadata revision.sign(identity) return self.add_revision(revision)
async def init(self, identity: Identity) -> Revision: vault = cast( Vault, self.vault) # We can savely cast because of @require_vault new_vault_id = str(uuid4()) if not vault.config.get("vault.id"): with vault.config.update_context(): vault.config.update("vault", {"id": new_vault_id}) await self.open() # create directory # create txchain store with open(os.path.join(self.path, "txchain"), "wb"): pass revision = Revision(operation=RevisionOp.CreateVault) revision.vault_id = new_vault_id revision.user_id = "user@localhost" revision.vault_public_key = vault.identity.public_key.exportKey("DER") revision.user_public_key = identity.public_key.exportKey("DER") revision.sign(identity=identity) return self.add_revision(revision)
async def init(self, identity: Identity) -> Revision: vault = cast(Vault, self.vault) # We can savely cast because of @require_vault new_vault_id = str(uuid4()) if not vault.config.get("vault.id"): with vault.config.update_context(): vault.config.update("vault", {"id": new_vault_id}) await self.open() # create directory # create txchain store with open(os.path.join(self.path, "txchain"), "wb"): pass revision = Revision(operation=RevisionOp.CreateVault) revision.vault_id = new_vault_id revision.user_id = "user@localhost" revision.vault_public_key = vault.identity.public_key.exportKey("DER") revision.user_public_key = identity.public_key.exportKey("DER") revision.sign(identity=identity) return self.add_revision(revision)
async def apply(self, revision: Revision, vault: Vault): if inspect(vault).session: raise ValueError('Vault object is bound to a session') revision.assert_valid() # 1. Check preconditions for this to be a valid revision (current revision must be parent) if vault.revision != revision.parent_id: raise UnexpectedParentInRevision("Expected parent to be {0}, but is {1}"\ .format(revision.parent_id, vault.revision)) smokesignal.emit('pre_apply_revision', vault=vault, revision=revision) with store.session() as session: # 2. Check if signing user's key is in the user vault key list if revision.operation != RevisionOp.CreateVault: signer_key = self.app.user_vault_keys.find_key( vault, revision.user_fingerprint) if not signer_key: raise InvalidRevision( "Key {0} is not allowed to generate revisions for vault {1}" .format(revision.user_fingerprint, vault)) else: # CreateVault is the only operation that is allowed to provide its own key signer_key = UserVaultKey( vault_id=vault.id, user_id=revision.user_id, fingerprint=revision.user_fingerprint, public_key=revision.user_public_key, ) # 3. Verify revision signature revision.verify(signer_key.get_identity(self.app.config)) # 4. Based on the revision type, perform an action to our state of the vault logger.debug( "Applying %s (%s) to %s", revision.operation, revision.revision_id, vault.id, ) if revision.operation == RevisionOp.CreateVault: session.add(vault) session.add(signer_key) session.add( VaultUser(vault_id=vault.id, user_id=revision.user_id)) session.commit() elif revision.operation == RevisionOp.Upload: try: bundle = await self.app.bundles.get_bundle_by_hash( vault, revision.file_hash) session.delete(bundle) except FileNotFoundError: pass bundle = await self.create_bundle_from_revision( revision, vault) session.add(bundle) session.commit() revision.path = bundle.relpath elif revision.operation == RevisionOp.SetMetadata: await vault.write_encrypted_metadata( Once(revision.revision_metadata)) elif revision.operation == RevisionOp.RemoveFile: bundle = await self.app.bundles.get_bundle_by_hash( vault, revision.file_hash) session.delete(bundle) session.commit() revision.path = bundle.relpath elif revision.operation == RevisionOp.AddUser: self.app.vault_users.add(vault, revision.user_id) elif revision.operation == RevisionOp.RemoveUser: self.app.vault_users.remove(vault, revision.user_id) elif revision.operation == RevisionOp.AddUserKey: new_identity = Identity.from_key(revision.user_public_key, self.app.config) self.app.user_vault_keys.add(vault, revision.user_id, new_identity) elif revision.operation == RevisionOp.RemoveUserKey: new_identity = Identity.from_key(revision.user_public_key, self.app.config) self.app.user_vault_keys.remove(vault, revision.user_id, new_identity) else: raise NotImplementedError(revision.operation) # 5. Store the revision in config and db revision.local_vault_id = vault.id revision.creator_id = signer_key.user_id session.add(revision) session.commit() vault.revision_count = (session.query(Revision).filter( Revision.local_vault_id == vault.id).count()) if revision.operation in (RevisionOp.Upload, RevisionOp.RemoveFile): vault.file_count = (session.query(Bundle).filter( Bundle.vault_id == vault.id).count()) if revision.operation in (RevisionOp.CreateVault, RevisionOp.AddUser, RevisionOp.RemoveUser): vault.user_count = (session.query(VaultUser).filter( VaultUser.vault_id == vault.id).count()) vault.modification_date = revision.created_at logger.debug( "Vault state revision_count=%s file_count=%s user_count=%s", vault.revision_count, vault.file_count, vault.user_count) # vault.revision = revision.id session.add(vault) vault.update_revision(revision) session.commit() smokesignal.emit('post_apply_revision', vault=vault, revision=revision)
async def apply(self, revision: Revision, vault: Vault): if inspect(vault).session: raise ValueError('Vault object is bound to a session') revision.assert_valid() # 1. Check preconditions for this to be a valid revision (current revision must be parent) if vault.revision != revision.parent_id: raise UnexpectedParentInRevision("Expected parent to be {0}, but is {1}"\ .format(revision.parent_id, vault.revision)) smokesignal.emit('pre_apply_revision', vault=vault, revision=revision) with store.session() as session: # 2. Check if signing user's key is in the user vault key list if revision.operation != RevisionOp.CreateVault: signer_key = self.app.user_vault_keys.find_key( vault, revision.user_fingerprint ) if not signer_key: raise InvalidRevision( "Key {0} is not allowed to generate revisions for vault {1}" .format(revision.user_fingerprint, vault) ) else: # CreateVault is the only operation that is allowed to provide its own key signer_key = UserVaultKey( vault_id=vault.id, user_id=revision.user_id, fingerprint=revision.user_fingerprint, public_key=revision.user_public_key, ) # 3. Verify revision signature revision.verify(signer_key.get_identity(self.app.config)) # 4. Based on the revision type, perform an action to our state of the vault logger.debug( "Applying %s (%s) to %s", revision.operation, revision.revision_id, vault.id, ) if revision.operation == RevisionOp.CreateVault: session.add(vault) session.add(signer_key) session.add(VaultUser(vault_id=vault.id, user_id=revision.user_id)) session.commit() elif revision.operation == RevisionOp.Upload: try: bundle = await self.app.bundles.get_bundle_by_hash(vault, revision.file_hash) session.delete(bundle) except FileNotFoundError: pass bundle = await self.create_bundle_from_revision(revision, vault) session.add(bundle) session.commit() revision.path = bundle.relpath elif revision.operation == RevisionOp.SetMetadata: await vault.write_encrypted_metadata(Once(revision.revision_metadata)) elif revision.operation == RevisionOp.RemoveFile: bundle = await self.app.bundles.get_bundle_by_hash(vault, revision.file_hash) session.delete(bundle) session.commit() revision.path = bundle.relpath elif revision.operation == RevisionOp.AddUser: self.app.vault_users.add(vault, revision.user_id) elif revision.operation == RevisionOp.RemoveUser: self.app.vault_users.remove(vault, revision.user_id) elif revision.operation == RevisionOp.AddUserKey: new_identity = Identity.from_key(revision.user_public_key, self.app.config) self.app.user_vault_keys.add(vault, revision.user_id, new_identity) elif revision.operation == RevisionOp.RemoveUserKey: new_identity = Identity.from_key(revision.user_public_key, self.app.config) self.app.user_vault_keys.remove(vault, revision.user_id, new_identity) else: raise NotImplementedError(revision.operation) # 5. Store the revision in config and db revision.local_vault_id = vault.id revision.creator_id = signer_key.user_id session.add(revision) session.commit() vault.revision_count = ( session.query(Revision) .filter(Revision.local_vault_id == vault.id) .count() ) if revision.operation in (RevisionOp.Upload, RevisionOp.RemoveFile): vault.file_count = ( session.query(Bundle) .filter(Bundle.vault_id == vault.id) .count() ) if revision.operation in (RevisionOp.CreateVault, RevisionOp.AddUser, RevisionOp.RemoveUser): vault.user_count = ( session.query(VaultUser) .filter(VaultUser.vault_id == vault.id) .count() ) vault.modification_date = revision.created_at logger.debug("Vault state revision_count=%s file_count=%s user_count=%s", vault.revision_count, vault.file_count, vault.user_count) # vault.revision = revision.id session.add(vault) vault.update_revision(revision) session.commit() smokesignal.emit('post_apply_revision', vault=vault, revision=revision)
async def upload(self, bundle, identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") self.logger.info('Uploading %s', bundle) assert bundle.uptodate metadata = await bundle.encrypted_metadata_reader().readall() metadata_size = len(metadata) while True: revision = Revision(operation=RevisionOp.Upload) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.crypt_hash = bundle.local_hash revision.file_hash = bundle.store_hash revision.file_size_crypt = bundle.file_size_crypt revision.revision_metadata = metadata revision.sign(identity=identity) # upload key and file await self.write_term('upload', revision.file_hash, revision.crypt_hash, revision.revision_metadata, revision.file_size_crypt, revision.user_fingerprint, revision.signature, revision.parent_id ) response = await self.read_term(assert_ok=False) if response[0] == Atom('ok'): break elif response[0] == Atom('error') and \ isinstance(response[1], (list, tuple)) and \ response[1][0] == Atom('parent_revision_outdated'): logger.info('Revision outdated') await trio.sleep(10.0) continue else: raise ServerError(response) self.logger.debug('Uploading bundle (metadata: {0} bytes, content: {1} bytes)'\ .format(metadata_size, bundle.file_size_crypt)) bundle.bytes_written = 0 upload_id = None urls = None reader = vault.crypt_engine.read_encrypted_stream(bundle) response = response[1] if len(response) > 1 else None if isinstance(response, tuple) and len(response) > 0 and response[0] == Atom('url'): if isinstance(response[1], tuple) and response[1][0] == Atom('multi'): _, upload_id, urls = response[1] chunksize = int(math.ceil(bundle.file_size_crypt * 1.0 / len(urls))) self.logger.info('Chunked URL upload to %d urls. chunksize=%d', len(urls), chunksize) writer = reader >> ChunkedURLWriter([u.decode() for u in urls], chunksize,\ total_size=bundle.file_size_crypt) url = None else: url = response[1].decode() self.logger.info('Non-chunked URL upload to %s.', url) writer = reader >> URLWriter(url, size=bundle.file_size_crypt) upload_id = None await writer.consume() if writer.bytes_written != bundle.file_size_crypt: self.logger.error('Uploaded size did not match: should be %d, is %d (diff %d)', bundle.file_size_crypt, writer.bytes_written, writer.bytes_written - bundle.file_size_crypt) raise Exception('Uploaded size did not match') if upload_id: await self.write_term('uploaded', (Atom('multi'), upload_id, writer.etags)) else: await self.write_term('uploaded', url) else: self.logger.debug('Streaming upload requested.') writer = reader >> TrioStreamWriter(self.stream) await writer.consume() if writer.bytes_written != bundle.file_size_crypt: self.logger.error('Uploaded size did not match: should be %d, is %d (diff %d)', bundle.file_size_crypt, writer.bytes_written, writer.bytes_written - bundle.file_size_crypt) raise Exception('Uploaded size did not match') # server should return the response response = await self.read_response() ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at return revision
async def upload(self, bundle, identity: Identity) -> Revision: vault = self.vault if vault is None: raise ValueError("Invalid argument") self.logger.info('Uploading %s', bundle) assert bundle.uptodate metadata = await bundle.encrypted_metadata_reader().readall() metadata_size = len(metadata) while True: revision = Revision(operation=RevisionOp.Upload) revision.vault_id = vault.config.id revision.parent_id = vault.revision revision.crypt_hash = bundle.local_hash revision.file_hash = bundle.store_hash revision.file_size_crypt = bundle.file_size_crypt revision.revision_metadata = metadata revision.sign(identity=identity) # upload key and file await self.write_term('upload', revision.file_hash, revision.crypt_hash, revision.revision_metadata, revision.file_size_crypt, revision.user_fingerprint, revision.signature, revision.parent_id) response = await self.read_term(assert_ok=False) if response[0] == Atom('ok'): break elif response[0] == Atom('error') and \ isinstance(response[1], (list, tuple)) and \ response[1][0] == Atom('parent_revision_outdated'): logger.info('Revision outdated') await trio.sleep(10.0) continue else: raise ServerError(response) self.logger.debug('Uploading bundle (metadata: {0} bytes, content: {1} bytes)'\ .format(metadata_size, bundle.file_size_crypt)) bundle.bytes_written = 0 upload_id = None urls = None reader = vault.crypt_engine.read_encrypted_stream(bundle) response = response[1] if len(response) > 1 else None if isinstance( response, tuple) and len(response) > 0 and response[0] == Atom('url'): if isinstance(response[1], tuple) and response[1][0] == Atom('multi'): _, upload_id, urls = response[1] chunksize = int( math.ceil(bundle.file_size_crypt * 1.0 / len(urls))) self.logger.info('Chunked URL upload to %d urls. chunksize=%d', len(urls), chunksize) writer = reader >> ChunkedURLWriter([u.decode() for u in urls], chunksize,\ total_size=bundle.file_size_crypt) url = None else: url = response[1].decode() self.logger.info('Non-chunked URL upload to %s.', url) writer = reader >> URLWriter(url, size=bundle.file_size_crypt) upload_id = None await writer.consume() if writer.bytes_written != bundle.file_size_crypt: self.logger.error( 'Uploaded size did not match: should be %d, is %d (diff %d)', bundle.file_size_crypt, writer.bytes_written, writer.bytes_written - bundle.file_size_crypt) raise Exception('Uploaded size did not match') if upload_id: await self.write_term('uploaded', (Atom('multi'), upload_id, writer.etags)) else: await self.write_term('uploaded', url) else: self.logger.debug('Streaming upload requested.') writer = reader >> TrioStreamWriter(self.stream) await writer.consume() if writer.bytes_written != bundle.file_size_crypt: self.logger.error( 'Uploaded size did not match: should be %d, is %d (diff %d)', bundle.file_size_crypt, writer.bytes_written, writer.bytes_written - bundle.file_size_crypt) raise Exception('Uploaded size did not match') # server should return the response response = await self.read_response() ret_revision = self.server_info_to_revision( rewrite_atoms_dict(response), vault) revision.revision_id = ret_revision.revision_id revision.created_at = ret_revision.created_at return revision