async def remove_user_vault_key(self, identity: Identity, user_id: str,
user_identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.debug('Removing user vault key')
revision = Revision(operation=RevisionOp.RemoveUserKey)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = user_id
revision.user_public_key = user_identity.public_key.exportKey("DER")
revision.sign(identity=identity)
# upload metadata
await self.write_term('remove_user_vault_key',
user_id,
revision.user_public_key,
user_identity.get_fingerprint(),
revision.user_fingerprint,
revision.parent_id,
revision.signature
)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def add_vault_user(self, user_id: str, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
revision = Revision(operation=RevisionOp.AddUser)
revision.vault_id = vault.config.id
revision.user_id = user_id
revision.parent_id = vault.revision
revision.sign(identity=identity)
await self.write_term('add_vault_user',
revision.user_id,
revision.user_fingerprint,
revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def add_vault_user(self, user_id: str,
identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
revision = Revision(operation=RevisionOp.AddUser)
revision.vault_id = vault.config.id
revision.user_id = user_id
revision.parent_id = vault.revision
revision.sign(identity=identity)
await self.write_term('add_vault_user', revision.user_id,
revision.user_fingerprint, revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def set_vault_metadata(self, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
metadata = await vault.encrypted_metadata_reader().readall()
self.logger.debug('Setting metadata for %s (%d bytes)', self.vault,
len(metadata))
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload metadata
await self.write_term('set_vault_metadata', metadata,
revision.user_fingerprint,
revision.parent_id, revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def remove_user_vault_key(self, identity: Identity, user_id: str,
user_identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.debug('Removing user vault key')
revision = Revision(operation=RevisionOp.RemoveUserKey)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = user_id
revision.user_public_key = user_identity.public_key.exportKey("DER")
revision.sign(identity=identity)
# upload metadata
await self.write_term('remove_user_vault_key', user_id,
revision.user_public_key,
user_identity.get_fingerprint(),
revision.user_fingerprint, revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def set_vault_metadata(self, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
metadata = await vault.encrypted_metadata_reader().readall()
self.logger.debug('Setting metadata for %s (%d bytes)', self.vault,
len(metadata))
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload metadata
await self.write_term('set_vault_metadata', metadata,
revision.user_fingerprint, revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def remove_file(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Removing %s', bundle)
revision = Revision(operation=RevisionOp.RemoveFile)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.file_hash = bundle.store_hash
revision.sign(identity=identity)
# upload key and file
await self.write_term('remove_file', revision.file_hash,
revision.user_fingerprint, revision.signature,
revision.parent_id)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def create_vault(self, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
revision = Revision(operation=RevisionOp.CreateVault)
revision.vault_public_key = vault.identity.public_key.exportKey("DER")
revision.user_public_key = identity.public_key.exportKey("DER")
revision.user_id = '*****@*****.**' # TBD
revision.sign(identity=identity)
await self.write_term('create_vault',
revision.vault_public_key,
revision.user_public_key,
revision.user_fingerprint,
revision.signature)
response = await self.read_term()
vault_id = response[1].decode(vault.config.encoding)
auth = response[2].decode(vault.config.encoding)
server_info = rewrite_atoms_dict(response[3])
if not vault_id:
raise ServerError("Invalid vault ID: {0}".format(vault_id))
if not auth:
raise ServerError("Invalid auth token: {0}".format(auth))
revision.vault_id = vault_id
# assert :ok
ret_revision = self.server_info_to_revision(server_info, vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
self.logger.info('Successfully created vault %s', vault_id)
with vault.config.update_context():
vault.config.update('remote', {
'auth': auth
})
vault.config.update('vault', {
'id': response[1].decode(vault.config.encoding)
})
return revision
async def create_vault(self, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
revision = Revision(operation=RevisionOp.CreateVault)
revision.vault_public_key = vault.identity.public_key.exportKey("DER")
revision.user_public_key = identity.public_key.exportKey("DER")
user_info = await self.user_info()
revision.user_id = user_info['email']
revision.sign(identity=identity)
await self.write_term('create_vault', revision.vault_public_key,
revision.user_public_key,
revision.user_fingerprint, revision.signature)
response = await self.read_term()
vault_id = response[1].decode(vault.config.encoding)
auth = response[2].decode(vault.config.encoding)
server_info = rewrite_atoms_dict(response[3])
if not vault_id:
raise ServerError("Invalid vault ID: {0}".format(vault_id))
if not auth:
raise ServerError("Invalid auth token: {0}".format(auth))
revision.vault_id = vault_id
# assert :ok
ret_revision = self.server_info_to_revision(server_info, vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
self.logger.info('Successfully created vault %s', vault_id)
with vault.config.update_context():
vault.config.update('remote', {'auth': auth})
vault.config.update(
'vault', {'id': response[1].decode(vault.config.encoding)})
return revision
def add_revision(self, revision: Revision) -> Revision:
"Persist the revision in the local storage. This will also generate a revision id."
if revision.revision_id is not None:
raise ValueError("Revision already has an id.")
if revision.signature is None:
raise ValueError("Revision is not signed.")
revision.revision_id = str(uuid4())
revision.created_at = datetime.utcnow()
with open(os.path.join(self.path, "txchain"), "ab") as txchain:
logger.debug(
"Adding revision %s to signchain (%s)",
revision.revision_id,
os.path.join(self.path, "txchain"),
)
binary_tx = pickle.dumps(revision)
txchain.write(binary_tx)
return revision
def add_revision(self, revision: Revision) -> Revision:
"Persist the revision in the local storage. This will also generate a revision id."
if revision.revision_id is not None:
raise ValueError("Revision already has an id.")
if revision.signature is None:
raise ValueError("Revision is not signed.")
revision.revision_id = str(uuid4())
revision.created_at = datetime.utcnow()
with open(os.path.join(self.path, "txchain"), "ab") as txchain:
logger.debug(
"Adding revision %s to signchain (%s)",
revision.revision_id,
os.path.join(self.path, "txchain"),
)
binary_tx = pickle.dumps(revision)
txchain.write(binary_tx)
return revision
async def upload(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Uploading %s', bundle)
assert bundle.uptodate
metadata = await bundle.encrypted_metadata_reader().readall()
metadata_size = len(metadata)
while True:
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.crypt_hash = bundle.local_hash
revision.file_hash = bundle.store_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload key and file
await self.write_term('upload',
revision.file_hash,
revision.crypt_hash,
revision.revision_metadata,
revision.file_size_crypt,
revision.user_fingerprint,
revision.signature,
revision.parent_id
)
response = await self.read_term(assert_ok=False)
if response[0] == Atom('ok'):
break
elif response[0] == Atom('error') and \
isinstance(response[1], (list, tuple)) and \
response[1][0] == Atom('parent_revision_outdated'):
logger.info('Revision outdated')
await trio.sleep(10.0)
continue
else:
raise ServerError(response)
self.logger.debug('Uploading bundle (metadata: {0} bytes, content: {1} bytes)'\
.format(metadata_size, bundle.file_size_crypt))
bundle.bytes_written = 0
upload_id = None
urls = None
reader = vault.crypt_engine.read_encrypted_stream(bundle)
response = response[1] if len(response) > 1 else None
if isinstance(response, tuple) and len(response) > 0 and response[0] == Atom('url'):
if isinstance(response[1], tuple) and response[1][0] == Atom('multi'):
_, upload_id, urls = response[1]
chunksize = int(math.ceil(bundle.file_size_crypt * 1.0 / len(urls)))
self.logger.info('Chunked URL upload to %d urls. chunksize=%d', len(urls), chunksize)
writer = reader >> ChunkedURLWriter([u.decode() for u in urls], chunksize,\
total_size=bundle.file_size_crypt)
url = None
else:
url = response[1].decode()
self.logger.info('Non-chunked URL upload to %s.', url)
writer = reader >> URLWriter(url, size=bundle.file_size_crypt)
upload_id = None
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error('Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
if upload_id:
await self.write_term('uploaded', (Atom('multi'), upload_id, writer.etags))
else:
await self.write_term('uploaded', url)
else:
self.logger.debug('Streaming upload requested.')
writer = reader >> TrioStreamWriter(self.stream)
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error('Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
# server should return the response
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def upload(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Uploading %s', bundle)
assert bundle.uptodate
metadata = await bundle.encrypted_metadata_reader().readall()
metadata_size = len(metadata)
while True:
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.crypt_hash = bundle.local_hash
revision.file_hash = bundle.store_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload key and file
await self.write_term('upload', revision.file_hash,
revision.crypt_hash,
revision.revision_metadata,
revision.file_size_crypt,
revision.user_fingerprint,
revision.signature, revision.parent_id)
response = await self.read_term(assert_ok=False)
if response[0] == Atom('ok'):
break
elif response[0] == Atom('error') and \
isinstance(response[1], (list, tuple)) and \
response[1][0] == Atom('parent_revision_outdated'):
logger.info('Revision outdated')
await trio.sleep(10.0)
continue
else:
raise ServerError(response)
self.logger.debug('Uploading bundle (metadata: {0} bytes, content: {1} bytes)'\
.format(metadata_size, bundle.file_size_crypt))
bundle.bytes_written = 0
upload_id = None
urls = None
reader = vault.crypt_engine.read_encrypted_stream(bundle)
response = response[1] if len(response) > 1 else None
if isinstance(
response,
tuple) and len(response) > 0 and response[0] == Atom('url'):
if isinstance(response[1],
tuple) and response[1][0] == Atom('multi'):
_, upload_id, urls = response[1]
chunksize = int(
math.ceil(bundle.file_size_crypt * 1.0 / len(urls)))
self.logger.info('Chunked URL upload to %d urls. chunksize=%d',
len(urls), chunksize)
writer = reader >> ChunkedURLWriter([u.decode() for u in urls], chunksize,\
total_size=bundle.file_size_crypt)
url = None
else:
url = response[1].decode()
self.logger.info('Non-chunked URL upload to %s.', url)
writer = reader >> URLWriter(url, size=bundle.file_size_crypt)
upload_id = None
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error(
'Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
if upload_id:
await self.write_term('uploaded',
(Atom('multi'), upload_id, writer.etags))
else:
await self.write_term('uploaded', url)
else:
self.logger.debug('Streaming upload requested.')
writer = reader >> TrioStreamWriter(self.stream)
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error(
'Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
# server should return the response
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
