async def upload(self, bundle: Bundle, identity: Identity) -> Revision:
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Uploading %s", bundle)
dest_path = os.path.join(self.path, bundle.store_hash)
shutil.copyfile(bundle.path, dest_path)
metadata = await bundle.encrypted_metadata_reader().readall()
len(metadata)
if bundle.local_hash is None:
raise ValueError("Please update bundle before upload.")
await bundle.load_key()
s = vault.crypt_engine.read_encrypted_stream(bundle) >> FileWriter(
dest_path)
await s.consume()
# s = bundle.encrypted_metadata_reader() >> FileWriter(dest_path + ".metadata")
# await s.consume()
with open(dest_path + ".hash", "w") as hashfile:
hashfile.write(bundle.local_hash)
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
#revision.user_id = "user@localhost"
revision.file_hash = bundle.store_hash
revision.revision_metadata = metadata
revision.crypt_hash = bundle.local_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_user_vault_key(self, identity: Identity, user_id: str,
user_identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.debug('Removing user vault key')
revision = Revision(operation=RevisionOp.RemoveUserKey)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = user_id
revision.user_public_key = user_identity.public_key.exportKey("DER")
revision.sign(identity=identity)
# upload metadata
await self.write_term('remove_user_vault_key',
user_id,
revision.user_public_key,
user_identity.get_fingerprint(),
revision.user_fingerprint,
revision.parent_id,
revision.signature
)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def remove_file(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Removing %s', bundle)
revision = Revision(operation=RevisionOp.RemoveFile)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.file_hash = bundle.store_hash
revision.sign(identity=identity)
# upload key and file
await self.write_term('remove_file', revision.file_hash,
revision.user_fingerprint, revision.signature,
revision.parent_id)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def add_vault_user(self, user_id: str, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
revision = Revision(operation=RevisionOp.AddUser)
revision.vault_id = vault.config.id
revision.user_id = user_id
revision.parent_id = vault.revision
revision.sign(identity=identity)
await self.write_term('add_vault_user',
revision.user_id,
revision.user_fingerprint,
revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def add_vault_user(self, user_id: str,
identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
revision = Revision(operation=RevisionOp.AddUser)
revision.vault_id = vault.config.id
revision.user_id = user_id
revision.parent_id = vault.revision
revision.sign(identity=identity)
await self.write_term('add_vault_user', revision.user_id,
revision.user_fingerprint, revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def set_vault_metadata(self, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
metadata = await vault.encrypted_metadata_reader().readall()
self.logger.debug('Setting metadata for %s (%d bytes)', self.vault,
len(metadata))
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload metadata
await self.write_term('set_vault_metadata', metadata,
revision.user_fingerprint,
revision.parent_id, revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def remove_user_vault_key(self, identity: Identity, user_id: str,
user_identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.debug('Removing user vault key')
revision = Revision(operation=RevisionOp.RemoveUserKey)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = user_id
revision.user_public_key = user_identity.public_key.exportKey("DER")
revision.sign(identity=identity)
# upload metadata
await self.write_term('remove_user_vault_key', user_id,
revision.user_public_key,
user_identity.get_fingerprint(),
revision.user_fingerprint, revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def set_vault_metadata(self, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
metadata = await vault.encrypted_metadata_reader().readall()
self.logger.debug('Setting metadata for %s (%d bytes)', self.vault,
len(metadata))
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload metadata
await self.write_term('set_vault_metadata', metadata,
revision.user_fingerprint, revision.parent_id,
revision.signature)
# assert :ok
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def upload(self, bundle: Bundle, identity: Identity) -> Revision:
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Uploading %s", bundle)
dest_path = os.path.join(self.path, bundle.store_hash)
shutil.copyfile(bundle.path, dest_path)
metadata = await bundle.encrypted_metadata_reader().readall()
len(metadata)
if bundle.local_hash is None:
raise ValueError("Please update bundle before upload.")
await bundle.load_key()
s = vault.crypt_engine.read_encrypted_stream(bundle) >> FileWriter(dest_path)
await s.consume()
# s = bundle.encrypted_metadata_reader() >> FileWriter(dest_path + ".metadata")
# await s.consume()
with open(dest_path + ".hash", "w") as hashfile:
hashfile.write(bundle.local_hash)
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
#revision.user_id = "user@localhost"
revision.file_hash = bundle.store_hash
revision.revision_metadata = metadata
revision.crypt_hash = bundle.local_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.sign(identity=identity)
return self.add_revision(revision)
def generate_fake_revision(vault):
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.id
revision.parent_id = vault.revision
revision.user_id = "user@localhost"
revision.user_fingerprint = "aabbcc"
revision.revision_metadata = b"123456"
revision.signature = b"12345"
return revision
async def remove_vault_user(self, user_id: str, identity: Identity) -> Revision:
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Remove user %s", user_id)
revision = Revision(operation=RevisionOp.RemoveUser)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = user_id
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_vault_user(self, user_id: str,
identity: Identity) -> Revision:
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Remove user %s", user_id)
revision = Revision(operation=RevisionOp.RemoveUser)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = user_id
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_user_vault_key(self, identity: Identity, user_id: str,
user_identity: Identity):
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Removing user vault key %s", user_id)
revision = Revision(operation=RevisionOp.RemoveUserKey)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_public_key = user_identity.public_key.exportKey('DER')
revision.user_id = user_id
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_file(self, bundle: Bundle, identity: Identity) -> Revision:
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
assert bundle.store_hash
logger.info("Deleting %s", bundle)
revision = Revision(operation=RevisionOp.RemoveFile)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = "user@localhost"
revision.file_hash = bundle.store_hash
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_user_vault_key(self, identity: Identity, user_id: str,
user_identity: Identity):
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Removing user vault key %s", user_id)
revision = Revision(operation=RevisionOp.RemoveUserKey)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_public_key = user_identity.public_key.exportKey('DER')
revision.user_id = user_id
revision.sign(identity=identity)
return self.add_revision(revision)
async def remove_file(self, bundle: Bundle,
identity: Identity) -> Revision:
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
assert bundle.store_hash
logger.info("Deleting %s", bundle)
revision = Revision(operation=RevisionOp.RemoveFile)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = "user@localhost"
revision.file_hash = bundle.store_hash
revision.sign(identity=identity)
return self.add_revision(revision)
async def set_vault_metadata(self, identity: Identity) -> Revision:
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
dest_path = os.path.join(self.path, "metadata")
writer = vault.encrypted_metadata_reader() >> FileWriter(
dest_path, create_dirs=True)
await writer.consume()
metadata = await vault.encrypted_metadata_reader().readall()
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = "user@localhost"
revision.revision_metadata = metadata
revision.sign(identity)
return self.add_revision(revision)
async def set_vault_metadata(self, identity: Identity) -> Revision:
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
dest_path = os.path.join(self.path, "metadata")
writer = vault.encrypted_metadata_reader() >> FileWriter(
dest_path, create_dirs=True
)
await writer.consume()
metadata = await vault.encrypted_metadata_reader().readall()
revision = Revision(operation=RevisionOp.SetMetadata)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.user_id = "user@localhost"
revision.revision_metadata = metadata
revision.sign(identity)
return self.add_revision(revision)
async def upload(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Uploading %s', bundle)
assert bundle.uptodate
metadata = await bundle.encrypted_metadata_reader().readall()
metadata_size = len(metadata)
while True:
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.crypt_hash = bundle.local_hash
revision.file_hash = bundle.store_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload key and file
await self.write_term('upload', revision.file_hash,
revision.crypt_hash,
revision.revision_metadata,
revision.file_size_crypt,
revision.user_fingerprint,
revision.signature, revision.parent_id)
response = await self.read_term(assert_ok=False)
if response[0] == Atom('ok'):
break
elif response[0] == Atom('error') and \
isinstance(response[1], (list, tuple)) and \
response[1][0] == Atom('parent_revision_outdated'):
logger.info('Revision outdated')
await trio.sleep(10.0)
continue
else:
raise ServerError(response)
self.logger.debug('Uploading bundle (metadata: {0} bytes, content: {1} bytes)'\
.format(metadata_size, bundle.file_size_crypt))
bundle.bytes_written = 0
upload_id = None
urls = None
reader = vault.crypt_engine.read_encrypted_stream(bundle)
response = response[1] if len(response) > 1 else None
if isinstance(
response,
tuple) and len(response) > 0 and response[0] == Atom('url'):
if isinstance(response[1],
tuple) and response[1][0] == Atom('multi'):
_, upload_id, urls = response[1]
chunksize = int(
math.ceil(bundle.file_size_crypt * 1.0 / len(urls)))
self.logger.info('Chunked URL upload to %d urls. chunksize=%d',
len(urls), chunksize)
writer = reader >> ChunkedURLWriter([u.decode() for u in urls], chunksize,\
total_size=bundle.file_size_crypt)
url = None
else:
url = response[1].decode()
self.logger.info('Non-chunked URL upload to %s.', url)
writer = reader >> URLWriter(url, size=bundle.file_size_crypt)
upload_id = None
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error(
'Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
if upload_id:
await self.write_term('uploaded',
(Atom('multi'), upload_id, writer.etags))
else:
await self.write_term('uploaded', url)
else:
self.logger.debug('Streaming upload requested.')
writer = reader >> TrioStreamWriter(self.stream)
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error(
'Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
# server should return the response
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def upload(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Uploading %s', bundle)
assert bundle.uptodate
metadata = await bundle.encrypted_metadata_reader().readall()
metadata_size = len(metadata)
while True:
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.crypt_hash = bundle.local_hash
revision.file_hash = bundle.store_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload key and file
await self.write_term('upload',
revision.file_hash,
revision.crypt_hash,
revision.revision_metadata,
revision.file_size_crypt,
revision.user_fingerprint,
revision.signature,
revision.parent_id
)
response = await self.read_term(assert_ok=False)
if response[0] == Atom('ok'):
break
elif response[0] == Atom('error') and \
isinstance(response[1], (list, tuple)) and \
response[1][0] == Atom('parent_revision_outdated'):
logger.info('Revision outdated')
await trio.sleep(10.0)
continue
else:
raise ServerError(response)
self.logger.debug('Uploading bundle (metadata: {0} bytes, content: {1} bytes)'\
.format(metadata_size, bundle.file_size_crypt))
bundle.bytes_written = 0
upload_id = None
urls = None
reader = vault.crypt_engine.read_encrypted_stream(bundle)
response = response[1] if len(response) > 1 else None
if isinstance(response, tuple) and len(response) > 0 and response[0] == Atom('url'):
if isinstance(response[1], tuple) and response[1][0] == Atom('multi'):
_, upload_id, urls = response[1]
chunksize = int(math.ceil(bundle.file_size_crypt * 1.0 / len(urls)))
self.logger.info('Chunked URL upload to %d urls. chunksize=%d', len(urls), chunksize)
writer = reader >> ChunkedURLWriter([u.decode() for u in urls], chunksize,\
total_size=bundle.file_size_crypt)
url = None
else:
url = response[1].decode()
self.logger.info('Non-chunked URL upload to %s.', url)
writer = reader >> URLWriter(url, size=bundle.file_size_crypt)
upload_id = None
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error('Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
if upload_id:
await self.write_term('uploaded', (Atom('multi'), upload_id, writer.etags))
else:
await self.write_term('uploaded', url)
else:
self.logger.debug('Streaming upload requested.')
writer = reader >> TrioStreamWriter(self.stream)
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error('Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
# server should return the response
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision