async def upload(self, bundle: Bundle, identity: Identity) -> Revision:
vault = cast(
Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Uploading %s", bundle)
dest_path = os.path.join(self.path, bundle.store_hash)
shutil.copyfile(bundle.path, dest_path)
metadata = await bundle.encrypted_metadata_reader().readall()
len(metadata)
if bundle.local_hash is None:
raise ValueError("Please update bundle before upload.")
await bundle.load_key()
s = vault.crypt_engine.read_encrypted_stream(bundle) >> FileWriter(
dest_path)
await s.consume()
# s = bundle.encrypted_metadata_reader() >> FileWriter(dest_path + ".metadata")
# await s.consume()
with open(dest_path + ".hash", "w") as hashfile:
hashfile.write(bundle.local_hash)
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
#revision.user_id = "user@localhost"
revision.file_hash = bundle.store_hash
revision.revision_metadata = metadata
revision.crypt_hash = bundle.local_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.sign(identity=identity)
return self.add_revision(revision)
async def upload(self, bundle: Bundle, identity: Identity) -> Revision:
vault = cast(Vault, self.vault) # We can savely cast because of @require_vault
logger.info("Uploading %s", bundle)
dest_path = os.path.join(self.path, bundle.store_hash)
shutil.copyfile(bundle.path, dest_path)
metadata = await bundle.encrypted_metadata_reader().readall()
len(metadata)
if bundle.local_hash is None:
raise ValueError("Please update bundle before upload.")
await bundle.load_key()
s = vault.crypt_engine.read_encrypted_stream(bundle) >> FileWriter(dest_path)
await s.consume()
# s = bundle.encrypted_metadata_reader() >> FileWriter(dest_path + ".metadata")
# await s.consume()
with open(dest_path + ".hash", "w") as hashfile:
hashfile.write(bundle.local_hash)
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
#revision.user_id = "user@localhost"
revision.file_hash = bundle.store_hash
revision.revision_metadata = metadata
revision.crypt_hash = bundle.local_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.sign(identity=identity)
return self.add_revision(revision)
async def upload(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Uploading %s', bundle)
assert bundle.uptodate
metadata = await bundle.encrypted_metadata_reader().readall()
metadata_size = len(metadata)
while True:
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.crypt_hash = bundle.local_hash
revision.file_hash = bundle.store_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload key and file
await self.write_term('upload',
revision.file_hash,
revision.crypt_hash,
revision.revision_metadata,
revision.file_size_crypt,
revision.user_fingerprint,
revision.signature,
revision.parent_id
)
response = await self.read_term(assert_ok=False)
if response[0] == Atom('ok'):
break
elif response[0] == Atom('error') and \
isinstance(response[1], (list, tuple)) and \
response[1][0] == Atom('parent_revision_outdated'):
logger.info('Revision outdated')
await trio.sleep(10.0)
continue
else:
raise ServerError(response)
self.logger.debug('Uploading bundle (metadata: {0} bytes, content: {1} bytes)'\
.format(metadata_size, bundle.file_size_crypt))
bundle.bytes_written = 0
upload_id = None
urls = None
reader = vault.crypt_engine.read_encrypted_stream(bundle)
response = response[1] if len(response) > 1 else None
if isinstance(response, tuple) and len(response) > 0 and response[0] == Atom('url'):
if isinstance(response[1], tuple) and response[1][0] == Atom('multi'):
_, upload_id, urls = response[1]
chunksize = int(math.ceil(bundle.file_size_crypt * 1.0 / len(urls)))
self.logger.info('Chunked URL upload to %d urls. chunksize=%d', len(urls), chunksize)
writer = reader >> ChunkedURLWriter([u.decode() for u in urls], chunksize,\
total_size=bundle.file_size_crypt)
url = None
else:
url = response[1].decode()
self.logger.info('Non-chunked URL upload to %s.', url)
writer = reader >> URLWriter(url, size=bundle.file_size_crypt)
upload_id = None
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error('Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
if upload_id:
await self.write_term('uploaded', (Atom('multi'), upload_id, writer.etags))
else:
await self.write_term('uploaded', url)
else:
self.logger.debug('Streaming upload requested.')
writer = reader >> TrioStreamWriter(self.stream)
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error('Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
# server should return the response
response = await self.read_response()
ret_revision = self.server_info_to_revision(rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision
async def upload(self, bundle, identity: Identity) -> Revision:
vault = self.vault
if vault is None:
raise ValueError("Invalid argument")
self.logger.info('Uploading %s', bundle)
assert bundle.uptodate
metadata = await bundle.encrypted_metadata_reader().readall()
metadata_size = len(metadata)
while True:
revision = Revision(operation=RevisionOp.Upload)
revision.vault_id = vault.config.id
revision.parent_id = vault.revision
revision.crypt_hash = bundle.local_hash
revision.file_hash = bundle.store_hash
revision.file_size_crypt = bundle.file_size_crypt
revision.revision_metadata = metadata
revision.sign(identity=identity)
# upload key and file
await self.write_term('upload', revision.file_hash,
revision.crypt_hash,
revision.revision_metadata,
revision.file_size_crypt,
revision.user_fingerprint,
revision.signature, revision.parent_id)
response = await self.read_term(assert_ok=False)
if response[0] == Atom('ok'):
break
elif response[0] == Atom('error') and \
isinstance(response[1], (list, tuple)) and \
response[1][0] == Atom('parent_revision_outdated'):
logger.info('Revision outdated')
await trio.sleep(10.0)
continue
else:
raise ServerError(response)
self.logger.debug('Uploading bundle (metadata: {0} bytes, content: {1} bytes)'\
.format(metadata_size, bundle.file_size_crypt))
bundle.bytes_written = 0
upload_id = None
urls = None
reader = vault.crypt_engine.read_encrypted_stream(bundle)
response = response[1] if len(response) > 1 else None
if isinstance(
response,
tuple) and len(response) > 0 and response[0] == Atom('url'):
if isinstance(response[1],
tuple) and response[1][0] == Atom('multi'):
_, upload_id, urls = response[1]
chunksize = int(
math.ceil(bundle.file_size_crypt * 1.0 / len(urls)))
self.logger.info('Chunked URL upload to %d urls. chunksize=%d',
len(urls), chunksize)
writer = reader >> ChunkedURLWriter([u.decode() for u in urls], chunksize,\
total_size=bundle.file_size_crypt)
url = None
else:
url = response[1].decode()
self.logger.info('Non-chunked URL upload to %s.', url)
writer = reader >> URLWriter(url, size=bundle.file_size_crypt)
upload_id = None
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error(
'Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
if upload_id:
await self.write_term('uploaded',
(Atom('multi'), upload_id, writer.etags))
else:
await self.write_term('uploaded', url)
else:
self.logger.debug('Streaming upload requested.')
writer = reader >> TrioStreamWriter(self.stream)
await writer.consume()
if writer.bytes_written != bundle.file_size_crypt:
self.logger.error(
'Uploaded size did not match: should be %d, is %d (diff %d)',
bundle.file_size_crypt, writer.bytes_written,
writer.bytes_written - bundle.file_size_crypt)
raise Exception('Uploaded size did not match')
# server should return the response
response = await self.read_response()
ret_revision = self.server_info_to_revision(
rewrite_atoms_dict(response), vault)
revision.revision_id = ret_revision.revision_id
revision.created_at = ret_revision.created_at
return revision