async def upload(self, bundle: Bundle, identity: Identity) -> Revision:
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Uploading %s", bundle)
dest_path = os.path.join(self.path, bundle.store_hash)
shutil.copyfile(bundle.path, dest_path)
metadata = await bundle.encrypted_metadata_reader().readall()
len(metadata)
if bundle.local_hash is None:
raise ValueError("Please update bundle before upload.")
await bundle.load_key()
s = vault.crypt_engine.read_encrypted_stream(bundle) >> FileWriter(
dest_path)
await s.consume()
# s = bundle.encrypted_metadata_reader() >> FileWriter(dest_path + ".metadata")
# await s.consume()
with open(dest_path + ".hash", "w") as hashfile:
hashfile.write(bundle.local_hash)
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
#revision.user_id = "user@localhost"
revision.file_hash = bundle.store_hash
revision.revision_metadata = metadata
revision.crypt_hash = bundle.local_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_user_vault_key(self, identity: Identity, user_id: str,
user_identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.debug('Removing user vault key')
revision = Revision(operation=RevisionOp.RemoveUserKey)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = user_id
revision.user_public_key = user_identity.public_key.exportKey("DER")
revision.sign(identity=identity)
# upload metadata
await self.write_term('remove_user_vault_key', user_id,
revision.user_public_key,
user_identity.get_fingerprint(),
revision.user_fingerprint, revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def remove_user_vault_key(self, identity: Identity, user_id: str,
user_identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.debug('Removing user vault key')
revision = Revision(operation=RevisionOp.RemoveUserKey)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = user_id
revision.user_public_key = user_identity.public_key.exportKey("DER")
revision.sign(identity=identity)
# upload metadata
await self.write_term('remove_user_vault_key',
user_id,
revision.user_public_key,
user_identity.get_fingerprint(),
revision.user_fingerprint,
revision.parent_id,
revision.signature
)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def upload(self, bundle: Bundle, identity: Identity) -> Revision:
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Uploading %s", bundle)
dest_path = os.path.join(self.path, bundle.store_hash)
shutil.copyfile(bundle.path, dest_path)
metadata = await bundle.encrypted_metadata_reader().readall()
len(metadata)
if bundle.local_hash is None:
raise ValueError("Please update bundle before upload.")
await bundle.load_key()
s = vault.crypt_engine.read_encrypted_stream(bundle) >> FileWriter(dest_path)
await s.consume()
# s = bundle.encrypted_metadata_reader() >> FileWriter(dest_path + ".metadata")
# await s.consume()
with open(dest_path + ".hash", "w") as hashfile:
hashfile.write(bundle.local_hash)
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
#revision.user_id = "user@localhost"
revision.file_hash = bundle.store_hash
revision.revision_metadata = metadata
revision.crypt_hash = bundle.local_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.sign(identity=identity)
return self.add_revision(revision)
async def add_vault_user(self, user_id: str,
identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
revision = Revision(operation=RevisionOp.AddUser)
revision.vault_id = vault.config.id
revision.user_id = user_id
revision.parent_id = vault.revision
revision.sign(identity=identity)
await self.write_term('add_vault_user', revision.user_id,
revision.user_fingerprint, revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def set_vault_metadata(self, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
metadata = await vault.encrypted_metadata_reader().readall()
self.logger.debug('Setting metadata for %s (%d bytes)', self.vault,
len(metadata))
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload metadata
await self.write_term('set_vault_metadata', metadata,
revision.user_fingerprint, revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def remove_file(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Removing %s', bundle)
revision = Revision(operation=RevisionOp.RemoveFile)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.file_hash = bundle.store_hash
revision.sign(identity=identity)
# upload key and file
await self.write_term('remove_file', revision.file_hash,
revision.user_fingerprint, revision.signature,
revision.parent_id)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def set_vault_metadata(self, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
metadata = await vault.encrypted_metadata_reader().readall()
self.logger.debug('Setting metadata for %s (%d bytes)', self.vault,
len(metadata))
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload metadata
await self.write_term('set_vault_metadata', metadata,
revision.user_fingerprint,
revision.parent_id, revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def add_vault_user(self, user_id: str, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
revision = Revision(operation=RevisionOp.AddUser)
revision.vault_id = vault.config.id
revision.user_id = user_id
revision.parent_id = vault.revision
revision.sign(identity=identity)
await self.write_term('add_vault_user',
revision.user_id,
revision.user_fingerprint,
revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
def generate_fake_revision(vault):
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.id
revision.parent_id = vault.revision
revision.user_id = "user@localhost"
revision.user_fingerprint = "aabbcc"
revision.revision_metadata = b"123456"
revision.signature = b"12345"
return revision
def server_info_to_revision(self,
server_info,
vault: Vault,
parent_id: Optional[str] = None):
operation = server_info['operation'].decode()
vault_public_key = None
user_id = None
if operation == 'store':
operation = RevisionOp.Upload
elif operation == 'create_vault':
operation = RevisionOp.CreateVault
vault_public_key = vault.identity.export_public_key()
user_id = server_info['metadata'].decode()
elif operation == 'set_metadata':
operation = RevisionOp.SetMetadata
vault_public_key = vault.identity.export_public_key()
elif operation == 'add_user':
operation = RevisionOp.AddUser
user_id = server_info['metadata'].decode()
elif operation == 'remove_user':
operation = RevisionOp.RemoveUser
user_id = server_info['metadata'].decode()
elif operation == 'add_user_vault_key':
operation = RevisionOp.AddUserKey
user_id = server_info['metadata'].decode()
elif operation == 'remove_user_vault_key':
operation = RevisionOp.RemoveUserKey
user_id = server_info['metadata'].decode()
else:
raise ServerError("Unknown operation: " + operation)
user_fingerprint = server_info['user_key_fingerprint'].decode()
signature = server_info['signature']
file_hash = server_info['file_hash'].decode(
) if server_info['file_hash'] is not None else None
crypt_hash = server_info['content_hash'].decode(
) if server_info['content_hash'] is not None else None
file_size_crypt = server_info['size']
metadata = server_info['metadata']
user_public_key = server_info['user_public_key']
revision_id = server_info['id'].decode()
created_at = \
iso8601.parse_date(server_info['created_at'].decode())
return Revision(operation=operation,
revision_id=revision_id,
parent_id=parent_id,
vault_id=vault.config.id,
created_at=created_at,
revision_metadata=metadata,
file_size_crypt=file_size_crypt,
file_hash=file_hash,
crypt_hash=crypt_hash,
user_id=user_id,
signature=signature,
user_public_key=user_public_key,
vault_public_key=vault_public_key,
user_fingerprint=user_fingerprint)
async def create_vault(self, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
revision = Revision(operation=RevisionOp.CreateVault)
revision.vault_public_key = vault.identity.public_key.exportKey("DER")
revision.user_public_key = identity.public_key.exportKey("DER")
user_info = await self.user_info()
revision.user_id = user_info['email']
revision.sign(identity=identity)
await self.write_term('create_vault', revision.vault_public_key,
revision.user_public_key,
revision.user_fingerprint, revision.signature)
response = await self.read_term()
vault_id = response[1].decode(vault.config.encoding)
auth = response[2].decode(vault.config.encoding)
server_info = rewrite_atoms_dict(response[3])
if not vault_id:
raise ServerError("Invalid vault ID: {0}".format(vault_id))
if not auth:
raise ServerError("Invalid auth token: {0}".format(auth))
revision.vault_id = vault_id
# assert :ok
ret_revision = self.server_info_to_revision(server_info, vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
self.logger.info('Successfully created vault %s', vault_id)
with vault.config.update_context():
vault.config.update('remote', {'auth': auth})
vault.config.update(
'vault', {'id': response[1].decode(vault.config.encoding)})
return revision
async def create_vault(self, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
revision = Revision(operation=RevisionOp.CreateVault)
revision.vault_public_key = vault.identity.public_key.exportKey("DER")
revision.user_public_key = identity.public_key.exportKey("DER")
revision.user_id = '*****@*****.**' # TBD
revision.sign(identity=identity)
await self.write_term('create_vault',
revision.vault_public_key,
revision.user_public_key,
revision.user_fingerprint,
revision.signature)
response = await self.read_term()
vault_id = response[1].decode(vault.config.encoding)
auth = response[2].decode(vault.config.encoding)
server_info = rewrite_atoms_dict(response[3])
if not vault_id:
raise ServerError("Invalid vault ID: {0}".format(vault_id))
if not auth:
raise ServerError("Invalid auth token: {0}".format(auth))
revision.vault_id = vault_id
# assert :ok
ret_revision = self.server_info_to_revision(server_info, vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
self.logger.info('Successfully created vault %s', vault_id)
with vault.config.update_context():
vault.config.update('remote', {
'auth': auth
})
vault.config.update('vault', {
'id': response[1].decode(vault.config.encoding)
})
return revision
def add_revision(self, revision: Revision) -> Revision:
"Persist the revision in the local storage. This will also generate a revision id."
if revision.revision_id is not None:
raise ValueError("Revision already has an id.")
if revision.signature is None:
raise ValueError("Revision is not signed.")
revision.revision_id = str(uuid4())
revision.created_at = datetime.utcnow()
with open(os.path.join(self.path, "txchain"), "ab") as txchain:
logger.debug(
"Adding revision %s to signchain (%s)",
revision.revision_id,
os.path.join(self.path, "txchain"),
)
binary_tx = pickle.dumps(revision)
txchain.write(binary_tx)
return revision
def add_revision(self, revision: Revision) -> Revision:
"Persist the revision in the local storage. This will also generate a revision id."
if revision.revision_id is not None:
raise ValueError("Revision already has an id.")
if revision.signature is None:
raise ValueError("Revision is not signed.")
revision.revision_id = str(uuid4())
revision.created_at = datetime.utcnow()
with open(os.path.join(self.path, "txchain"), "ab") as txchain:
logger.debug(
"Adding revision %s to signchain (%s)",
revision.revision_id,
os.path.join(self.path, "txchain"),
)
binary_tx = pickle.dumps(revision)
txchain.write(binary_tx)
return revision
async def remove_user_vault_key(self, identity: Identity, user_id: str,
user_identity: Identity):
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Removing user vault key %s", user_id)
revision = Revision(operation=RevisionOp.RemoveUserKey)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_public_key = user_identity.public_key.exportKey('DER')
revision.user_id = user_id
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_file(self, bundle: Bundle,
identity: Identity) -> Revision:
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
assert bundle.store_hash
logger.info("Deleting %s", bundle)
revision = Revision(operation=RevisionOp.RemoveFile)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = "user@localhost"
revision.file_hash = bundle.store_hash
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_vault_user(self, user_id: str,
identity: Identity) -> Revision:
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Remove user %s", user_id)
revision = Revision(operation=RevisionOp.RemoveUser)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = user_id
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_user_vault_key(self, identity: Identity, user_id: str,
user_identity: Identity):
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Removing user vault key %s", user_id)
revision = Revision(operation=RevisionOp.RemoveUserKey)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_public_key = user_identity.public_key.exportKey('DER')
revision.user_id = user_id
revision.sign(identity=identity)
return self.add_revision(revision)
async def set_vault_metadata(self, identity: Identity) -> Revision:
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
dest_path = os.path.join(self.path, "metadata")
writer = vault.encrypted_metadata_reader() >> FileWriter(
dest_path, create_dirs=True)
await writer.consume()
metadata = await vault.encrypted_metadata_reader().readall()
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = "user@localhost"
revision.revision_metadata = metadata
revision.sign(identity)
return self.add_revision(revision)
async def remove_file(self, bundle: Bundle, identity: Identity) -> Revision:
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
assert bundle.store_hash
logger.info("Deleting %s", bundle)
revision = Revision(operation=RevisionOp.RemoveFile)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = "user@localhost"
revision.file_hash = bundle.store_hash
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_vault_user(self, user_id: str, identity: Identity) -> Revision:
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Remove user %s", user_id)
revision = Revision(operation=RevisionOp.RemoveUser)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = user_id
revision.sign(identity=identity)
return self.add_revision(revision)
async def set_vault_metadata(self, identity: Identity) -> Revision:
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
dest_path = os.path.join(self.path, "metadata")
writer = vault.encrypted_metadata_reader() >> FileWriter(
dest_path, create_dirs=True
)
await writer.consume()
metadata = await vault.encrypted_metadata_reader().readall()
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = "user@localhost"
revision.revision_metadata = metadata
revision.sign(identity)
return self.add_revision(revision)
async def init(self, identity: Identity) -> Revision:
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
new_vault_id = str(uuid4())
if not vault.config.get("vault.id"):
with vault.config.update_context():
vault.config.update("vault", {"id": new_vault_id})
await self.open() # create directory
# create txchain store
with open(os.path.join(self.path, "txchain"), "wb"):
pass
revision = Revision(operation=RevisionOp.CreateVault)
revision.vault_id = new_vault_id
revision.user_id = "user@localhost"
revision.vault_public_key = vault.identity.public_key.exportKey("DER")
revision.user_public_key = identity.public_key.exportKey("DER")
revision.sign(identity=identity)
return self.add_revision(revision)
async def init(self, identity: Identity) -> Revision:
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
new_vault_id = str(uuid4())
if not vault.config.get("vault.id"):
with vault.config.update_context():
vault.config.update("vault", {"id": new_vault_id})
await self.open() # create directory
# create txchain store
with open(os.path.join(self.path, "txchain"), "wb"):
pass
revision = Revision(operation=RevisionOp.CreateVault)
revision.vault_id = new_vault_id
revision.user_id = "user@localhost"
revision.vault_public_key = vault.identity.public_key.exportKey("DER")
revision.user_public_key = identity.public_key.exportKey("DER")
revision.sign(identity=identity)
return self.add_revision(revision)
async def apply(self, revision: Revision, vault: Vault):
if inspect(vault).session:
raise ValueError('Vault object is bound to a session')
revision.assert_valid()
# 1. Check preconditions for this to be a valid revision (current revision must be parent)
if vault.revision != revision.parent_id:
raise UnexpectedParentInRevision("Expected parent to be {0}, but is {1}"\
.format(revision.parent_id, vault.revision))
smokesignal.emit('pre_apply_revision', vault=vault, revision=revision)
with store.session() as session:
# 2. Check if signing user's key is in the user vault key list
if revision.operation != RevisionOp.CreateVault:
signer_key = self.app.user_vault_keys.find_key(
vault, revision.user_fingerprint)
if not signer_key:
raise InvalidRevision(
"Key {0} is not allowed to generate revisions for vault {1}"
.format(revision.user_fingerprint, vault))
else:
# CreateVault is the only operation that is allowed to provide its own key
signer_key = UserVaultKey(
vault_id=vault.id,
user_id=revision.user_id,
fingerprint=revision.user_fingerprint,
public_key=revision.user_public_key,
)
# 3. Verify revision signature
revision.verify(signer_key.get_identity(self.app.config))
# 4. Based on the revision type, perform an action to our state of the vault
logger.debug(
"Applying %s (%s) to %s",
revision.operation,
revision.revision_id,
vault.id,
)
if revision.operation == RevisionOp.CreateVault:
session.add(vault)
session.add(signer_key)
session.add(
VaultUser(vault_id=vault.id, user_id=revision.user_id))
session.commit()
elif revision.operation == RevisionOp.Upload:
try:
bundle = await self.app.bundles.get_bundle_by_hash(
vault, revision.file_hash)
session.delete(bundle)
except FileNotFoundError:
pass
bundle = await self.create_bundle_from_revision(
revision, vault)
session.add(bundle)
session.commit()
revision.path = bundle.relpath
elif revision.operation == RevisionOp.SetMetadata:
await vault.write_encrypted_metadata(
Once(revision.revision_metadata))
elif revision.operation == RevisionOp.RemoveFile:
bundle = await self.app.bundles.get_bundle_by_hash(
vault, revision.file_hash)
session.delete(bundle)
session.commit()
revision.path = bundle.relpath
elif revision.operation == RevisionOp.AddUser:
self.app.vault_users.add(vault, revision.user_id)
elif revision.operation == RevisionOp.RemoveUser:
self.app.vault_users.remove(vault, revision.user_id)
elif revision.operation == RevisionOp.AddUserKey:
new_identity = Identity.from_key(revision.user_public_key,
self.app.config)
self.app.user_vault_keys.add(vault, revision.user_id,
new_identity)
elif revision.operation == RevisionOp.RemoveUserKey:
new_identity = Identity.from_key(revision.user_public_key,
self.app.config)
self.app.user_vault_keys.remove(vault, revision.user_id,
new_identity)
else:
raise NotImplementedError(revision.operation)
# 5. Store the revision in config and db
revision.local_vault_id = vault.id
revision.creator_id = signer_key.user_id
session.add(revision)
session.commit()
vault.revision_count = (session.query(Revision).filter(
Revision.local_vault_id == vault.id).count())
if revision.operation in (RevisionOp.Upload,
RevisionOp.RemoveFile):
vault.file_count = (session.query(Bundle).filter(
Bundle.vault_id == vault.id).count())
if revision.operation in (RevisionOp.CreateVault,
RevisionOp.AddUser,
RevisionOp.RemoveUser):
vault.user_count = (session.query(VaultUser).filter(
VaultUser.vault_id == vault.id).count())
vault.modification_date = revision.created_at
logger.debug(
"Vault state revision_count=%s file_count=%s user_count=%s",
vault.revision_count, vault.file_count, vault.user_count)
# vault.revision = revision.id
session.add(vault)
vault.update_revision(revision)
session.commit()
smokesignal.emit('post_apply_revision', vault=vault, revision=revision)
async def apply(self, revision: Revision, vault: Vault):
if inspect(vault).session:
raise ValueError('Vault object is bound to a session')
revision.assert_valid()
# 1. Check preconditions for this to be a valid revision (current revision must be parent)
if vault.revision != revision.parent_id:
raise UnexpectedParentInRevision("Expected parent to be {0}, but is {1}"\
.format(revision.parent_id, vault.revision))
smokesignal.emit('pre_apply_revision', vault=vault, revision=revision)
with store.session() as session:
# 2. Check if signing user's key is in the user vault key list
if revision.operation != RevisionOp.CreateVault:
signer_key = self.app.user_vault_keys.find_key(
vault, revision.user_fingerprint
)
if not signer_key:
raise InvalidRevision(
"Key {0} is not allowed to generate revisions for vault {1}"
.format(revision.user_fingerprint, vault)
)
else:
# CreateVault is the only operation that is allowed to provide its own key
signer_key = UserVaultKey(
vault_id=vault.id,
user_id=revision.user_id,
fingerprint=revision.user_fingerprint,
public_key=revision.user_public_key,
)
# 3. Verify revision signature
revision.verify(signer_key.get_identity(self.app.config))
# 4. Based on the revision type, perform an action to our state of the vault
logger.debug(
"Applying %s (%s) to %s",
revision.operation,
revision.revision_id,
vault.id,
)
if revision.operation == RevisionOp.CreateVault:
session.add(vault)
session.add(signer_key)
session.add(VaultUser(vault_id=vault.id, user_id=revision.user_id))
session.commit()
elif revision.operation == RevisionOp.Upload:
try:
bundle = await self.app.bundles.get_bundle_by_hash(vault, revision.file_hash)
session.delete(bundle)
except FileNotFoundError:
pass
bundle = await self.create_bundle_from_revision(revision, vault)
session.add(bundle)
session.commit()
revision.path = bundle.relpath
elif revision.operation == RevisionOp.SetMetadata:
await vault.write_encrypted_metadata(Once(revision.revision_metadata))
elif revision.operation == RevisionOp.RemoveFile:
bundle = await self.app.bundles.get_bundle_by_hash(vault, revision.file_hash)
session.delete(bundle)
session.commit()
revision.path = bundle.relpath
elif revision.operation == RevisionOp.AddUser:
self.app.vault_users.add(vault, revision.user_id)
elif revision.operation == RevisionOp.RemoveUser:
self.app.vault_users.remove(vault, revision.user_id)
elif revision.operation == RevisionOp.AddUserKey:
new_identity = Identity.from_key(revision.user_public_key, self.app.config)
self.app.user_vault_keys.add(vault, revision.user_id, new_identity)
elif revision.operation == RevisionOp.RemoveUserKey:
new_identity = Identity.from_key(revision.user_public_key, self.app.config)
self.app.user_vault_keys.remove(vault, revision.user_id, new_identity)
else:
raise NotImplementedError(revision.operation)
# 5. Store the revision in config and db
revision.local_vault_id = vault.id
revision.creator_id = signer_key.user_id
session.add(revision)
session.commit()
vault.revision_count = (
session.query(Revision)
.filter(Revision.local_vault_id == vault.id)
.count()
)
if revision.operation in (RevisionOp.Upload, RevisionOp.RemoveFile):
vault.file_count = (
session.query(Bundle)
.filter(Bundle.vault_id == vault.id)
.count()
)
if revision.operation in (RevisionOp.CreateVault, RevisionOp.AddUser, RevisionOp.RemoveUser):
vault.user_count = (
session.query(VaultUser)
.filter(VaultUser.vault_id == vault.id)
.count()
)
vault.modification_date = revision.created_at
logger.debug("Vault state revision_count=%s file_count=%s user_count=%s",
vault.revision_count, vault.file_count, vault.user_count)
# vault.revision = revision.id
session.add(vault)
vault.update_revision(revision)
session.commit()
smokesignal.emit('post_apply_revision', vault=vault, revision=revision)
async def upload(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Uploading %s', bundle)
assert bundle.uptodate
metadata = await bundle.encrypted_metadata_reader().readall()
metadata_size = len(metadata)
while True:
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.crypt_hash = bundle.local_hash
revision.file_hash = bundle.store_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload key and file
await self.write_term('upload',
revision.file_hash,
revision.crypt_hash,
revision.revision_metadata,
revision.file_size_crypt,
revision.user_fingerprint,
revision.signature,
revision.parent_id
)
response = await self.read_term(assert_ok=False)
if response[0] == Atom('ok'):
break
elif response[0] == Atom('error') and \
isinstance(response[1], (list, tuple)) and \
response[1][0] == Atom('parent_revision_outdated'):
logger.info('Revision outdated')
await trio.sleep(10.0)
continue
else:
raise ServerError(response)
self.logger.debug('Uploading bundle (metadata: {0} bytes, content: {1} bytes)'\
.format(metadata_size, bundle.file_size_crypt))
bundle.bytes_written = 0
upload_id = None
urls = None
reader = vault.crypt_engine.read_encrypted_stream(bundle)
response = response[1] if len(response) > 1 else None
if isinstance(response, tuple) and len(response) > 0 and response[0] == Atom('url'):
if isinstance(response[1], tuple) and response[1][0] == Atom('multi'):
_, upload_id, urls = response[1]
chunksize = int(math.ceil(bundle.file_size_crypt * 1.0 / len(urls)))
self.logger.info('Chunked URL upload to %d urls. chunksize=%d', len(urls), chunksize)
writer = reader >> ChunkedURLWriter([u.decode() for u in urls], chunksize,\
total_size=bundle.file_size_crypt)
url = None
else:
url = response[1].decode()
self.logger.info('Non-chunked URL upload to %s.', url)
writer = reader >> URLWriter(url, size=bundle.file_size_crypt)
upload_id = None
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error('Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
if upload_id:
await self.write_term('uploaded', (Atom('multi'), upload_id, writer.etags))
else:
await self.write_term('uploaded', url)
else:
self.logger.debug('Streaming upload requested.')
writer = reader >> TrioStreamWriter(self.stream)
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error('Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
# server should return the response
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def upload(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Uploading %s', bundle)
assert bundle.uptodate
metadata = await bundle.encrypted_metadata_reader().readall()
metadata_size = len(metadata)
while True:
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.crypt_hash = bundle.local_hash
revision.file_hash = bundle.store_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload key and file
await self.write_term('upload', revision.file_hash,
revision.crypt_hash,
revision.revision_metadata,
revision.file_size_crypt,
revision.user_fingerprint,
revision.signature, revision.parent_id)
response = await self.read_term(assert_ok=False)
if response[0] == Atom('ok'):
break
elif response[0] == Atom('error') and \
isinstance(response[1], (list, tuple)) and \
response[1][0] == Atom('parent_revision_outdated'):
logger.info('Revision outdated')
await trio.sleep(10.0)
continue
else:
raise ServerError(response)
self.logger.debug('Uploading bundle (metadata: {0} bytes, content: {1} bytes)'\
.format(metadata_size, bundle.file_size_crypt))
bundle.bytes_written = 0
upload_id = None
urls = None
reader = vault.crypt_engine.read_encrypted_stream(bundle)
response = response[1] if len(response) > 1 else None
if isinstance(
response,
tuple) and len(response) > 0 and response[0] == Atom('url'):
if isinstance(response[1],
tuple) and response[1][0] == Atom('multi'):
_, upload_id, urls = response[1]
chunksize = int(
math.ceil(bundle.file_size_crypt * 1.0 / len(urls)))
self.logger.info('Chunked URL upload to %d urls. chunksize=%d',
len(urls), chunksize)
writer = reader >> ChunkedURLWriter([u.decode() for u in urls], chunksize,\
total_size=bundle.file_size_crypt)
url = None
else:
url = response[1].decode()
self.logger.info('Non-chunked URL upload to %s.', url)
writer = reader >> URLWriter(url, size=bundle.file_size_crypt)
upload_id = None
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error(
'Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
if upload_id:
await self.write_term('uploaded',
(Atom('multi'), upload_id, writer.etags))
else:
await self.write_term('uploaded', url)
else:
self.logger.debug('Streaming upload requested.')
writer = reader >> TrioStreamWriter(self.stream)
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error(
'Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
# server should return the response
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
