def diff_versions(self, old_version=None, new_version=None):
empty_entries = {'/': None}
empty_manifest = {'entries': empty_entries, 'groups': {}, 'dustbin': [], 'versions': {}}
# Old manifest
if old_version and old_version > 0:
old_vlob = yield Effect(EUserVlobRead(old_version))
old_blob = from_jsonb64(old_vlob['blob'])
content = self.encryptor.decrypt(old_blob)
old_manifest = ejson_loads(content.decode())
elif old_version == 0:
old_manifest = empty_manifest
else:
old_manifest = self.original_manifest
# New manifest
if new_version and new_version > 0:
new_vlob = yield Effect(EUserVlobRead(new_version))
new_blob = from_jsonb64(new_vlob['blob'])
content = self.encryptor.decrypt(new_blob)
new_manifest = ejson_loads(content.decode())
elif new_version == 0:
new_manifest = empty_manifest
else:
dump = yield self.dumps()
new_manifest = ejson_loads(dump)
return self.diff(old_manifest, new_manifest)
def perform_pubkey_get(intent):
msg = {'id': intent.id}
ret = yield Effect(BackendCmd('pubkey_get', msg))
status = ret['status']
if status != 'ok':
raise exception_from_status(status)(ret['label'])
return PubKey(intent.id, from_jsonb64(ret['key']))
def load(cls, id, key, read_trust_seed, write_trust_seed):
self = GroupManifest(id)
self.read_trust_seed = read_trust_seed
self.write_trust_seed = write_trust_seed
self.encryptor = load_sym_key(from_jsonb64(key))
yield self.reload(reset=True)
return self
def stat(self):
version = self.get_version()
vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version))
encrypted_blob = vlob['blob']
encrypted_blob = from_jsonb64(encrypted_blob)
blob = self.encryptor.decrypt(encrypted_blob)
blob = ejson_loads(blob.decode())
size = 0
for blocks_and_key in blob:
for block in blocks_and_key['blocks']:
size += block['size']
for modification in self.modifications:
if modification[0] == self.write:
end_offset = modification[2] + len(modification[1])
if size < end_offset:
size = end_offset
elif modification[0] == self.truncate:
if size > modification[1]:
size = modification[1]
else:
raise NotImplementedError()
# TODO don't provide atime field if we don't know it?
# TODO real date
return {
'id': self.id,
'type': 'file',
'created': '2012-01-01T00:00:00',
'updated': '2012-01-01T00:00:00',
'size': size,
'version': vlob['version']
}
def check_consistency(self, manifest):
entries = [entry for entry in list(manifest['entries'].values()) if entry]
entries += manifest['dustbin']
for entry in entries:
try:
vlob = yield Effect(EVlobRead(entry['id'],
entry['read_trust_seed'],
manifest['versions'][entry['id']]))
encrypted_blob = vlob['blob']
encrypted_blob = from_jsonb64(encrypted_blob)
key = from_jsonb64(entry['key']) if entry['key'] else None
encryptor = load_sym_key(key)
encryptor.decrypt(encrypted_blob) # TODO check exception
except VlobNotFound:
return False
return True
def read(self, size=None, offset=0):
self.flush()
# TODO use flags
response = self._operations.send_cmd(
cmd='file_read', path=self.path, size=size, offset=offset)
if response['status'] != 'ok':
raise FuseOSError(ENOENT)
return from_jsonb64(response['content'])
def perform_user_vlob_read(intent):
msg = {}
if intent.version is not None:
msg['version'] = intent.version
ret = yield Effect(BackendCmd('user_vlob_read', msg))
status = ret['status']
if status != 'ok':
raise exception_from_status(status)(ret['label'])
return UserVlobAtom(ret['version'], from_jsonb64(ret['blob']))
def perform_vlob_read(intent):
assert isinstance(intent.id, str)
msg = {'id': intent.id, 'trust_seed': intent.trust_seed}
if intent.version is not None:
msg['version'] = intent.version
ret = yield Effect(BackendCmd('vlob_read', msg))
status = ret['status']
if status != 'ok':
raise exception_from_status(status)(ret['label'])
return VlobAtom(ret['id'], ret['version'], from_jsonb64(ret['blob']))
def perform_message_get(intent):
msg = {'recipient': intent.recipient, 'offset': intent.offset}
ret = yield Effect(BackendCmd('message_get', msg))
status = ret['status']
if status != 'ok':
raise exception_from_status(status)(ret['label'])
return [
Message(msg['count'], from_jsonb64(msg['body']))
for msg in ret['messages']
]
def check_consistency(self, manifest):
consistency = yield super().check_consistency(manifest)
if consistency is False:
return False
for entry in manifest['groups'].values():
try:
vlob = yield Effect(EVlobRead(entry['id'], entry['read_trust_seed']))
encrypted_blob = vlob['blob']
key = from_jsonb64(entry['key']) if entry['key'] else None
encryptor = load_sym_key(key)
encryptor.decrypt(encrypted_blob)
except VlobNotFound:
return False
return True
def read(self, size=None, offset=0):
yield self.flush()
# Get data
matching_blocks = yield self._find_matching_blocks(size, offset)
data = matching_blocks['pre_included_data']
for blocks_and_key in matching_blocks['included_blocks']:
block_key = blocks_and_key['key']
decoded_block_key = from_jsonb64(block_key)
encryptor = load_sym_key(decoded_block_key)
for block_properties in blocks_and_key['blocks']:
block = yield Effect(EBlockRead(block_properties['block']))
# Decrypt
# TODO: clean this hack
if isinstance(block['content'], str):
block_content = from_jsonb64(block['content'])
else:
block_content = from_jsonb64(block['content'].decode())
chunk_data = encryptor.decrypt(block_content)
# Check integrity
assert digest(chunk_data) == block_properties['digest']
assert len(chunk_data) == block_properties['size']
data += chunk_data
data += matching_blocks['post_included_data']
return data
def reload(self, reset=False):
vlob = yield Effect(EUserVlobRead())
if not vlob['blob']:
raise ManifestNotFound('User manifest not found.')
blob = from_jsonb64(vlob['blob'])
content = self.encryptor.decrypt(blob)
if not reset and vlob['version'] <= self.version:
return
new_manifest = ejson_loads(content.decode())
backup_new_manifest = deepcopy(new_manifest)
consistency = yield self.check_consistency(new_manifest)
if not consistency:
raise ManifestError('not_consistent', 'User manifest not consistent.')
if not reset:
diff = yield self.diff_versions()
new_manifest = self.patch(new_manifest, diff)
self.entries = new_manifest['entries']
self.dustbin = new_manifest['dustbin']
self.version = vlob['version']
self.group_manifests = {}
for group, group_vlob in new_manifest['groups'].items():
self.import_group_vlob(group, group_vlob)
self.original_manifest = backup_new_manifest
versions = new_manifest['versions']
file_vlob = None
for vlob_id, version in sorted(versions.items()):
for entry in self.entries.values():
if entry and entry['id'] == vlob_id:
file_vlob = entry
break
if not file_vlob:
for entry in self.dustbin:
if entry['id'] == vlob_id:
file_vlob = {'id': entry['id'],
'read_trust_seed': entry['read_trust_seed'],
'write_trust_seed': entry['write_trust_seed'],
'key': entry['key']}
break
if file_vlob:
file_vlob = None
file = yield File.load(entry['id'],
entry['key'],
entry['read_trust_seed'],
entry['write_trust_seed'])
try:
yield file.restore(version)
except FileError:
pass
def reencrypt(self):
yield self.flush()
version = self.get_version()
old_vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version))
old_blob = old_vlob['blob']
old_encrypted_blob = from_jsonb64(old_blob)
new_blob = self.encryptor.decrypt(old_encrypted_blob)
self.encryptor = generate_sym_key()
new_encrypted_blob = self.encryptor.encrypt(new_blob)
new_encrypted_blob = to_jsonb64(new_encrypted_blob)
new_vlob = yield Effect(EVlobCreate(new_encrypted_blob))
del File.files[self.id]
self.id = new_vlob['id']
self.read_trust_seed = new_vlob['read_trust_seed']
self.write_trust_seed = new_vlob['write_trust_seed']
File.files[self.id] = self
self.dirty = True
def reload(self, reset=False):
# Subscribe to events
# yield Effect(EConnectEvent('on_vlob_updated', self.id, self.handler)) # TODO call
vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed))
blob = from_jsonb64(vlob['blob'])
content = self.encryptor.decrypt(blob)
if not reset and vlob['version'] <= self.version:
return
new_manifest = ejson_loads(content.decode())
backup_new_manifest = deepcopy(new_manifest)
consistency = yield self.check_consistency(new_manifest)
if not consistency:
raise ManifestError('not_consistent', 'Group manifest not consistent.')
if not reset:
diff = yield self.diff_versions()
new_manifest = self.patch(new_manifest, diff)
self.entries = new_manifest['entries']
self.dustbin = new_manifest['dustbin']
self.version = vlob['version']
self.original_manifest = backup_new_manifest
versions = new_manifest['versions']
file_vlob = None
for vlob_id, version in sorted(versions.items()):
for entry in self.entries.values():
if entry and entry['id'] == vlob_id:
file_vlob = entry
break
if not file_vlob:
for entry in self.dustbin:
if entry['id'] == vlob_id:
file_vlob = {'id': entry['id'],
'read_trust_seed': entry['read_trust_seed'],
'write_trust_seed': entry['write_trust_seed'],
'key': entry['key']}
break
if file_vlob:
file_vlob = None
file = yield File.load(entry['id'],
entry['key'],
entry['read_trust_seed'],
entry['write_trust_seed'])
try:
yield file.restore(version)
except FileError:
pass
def load(cls, id, key, read_trust_seed, write_trust_seed, version=None):
if id in File.files:
return File.files[id]
self = File()
self.id = id
self.read_trust_seed = read_trust_seed
self.write_trust_seed = write_trust_seed
self.encryptor = load_sym_key(from_jsonb64(key))
vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version))
self.version = vlob['version']
self.dirty = False
vlob_list = yield Effect(EVlobList())
if vlob['id'] in vlob_list:
self.dirty = True
self.version -= 1
self.modifications = []
File.files[self.id] = self
return self
def update_vlob(self, new_vlob):
self.id = new_vlob['id']
self.encryptor = load_sym_key(from_jsonb64(new_vlob['key']))
self.read_trust_seed = new_vlob['read_trust_seed']
self.write_trust_seed = new_vlob['write_trust_seed']
def _find_matching_blocks(self, size=None, offset=0):
if size is None:
size = sys.maxsize
pre_excluded_blocks = []
post_excluded_blocks = []
version = self.get_version()
vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version))
blob = vlob['blob']
encrypted_blob = from_jsonb64(blob)
blob = self.encryptor.decrypt(encrypted_blob)
blob = ejson_loads(blob.decode())
pre_excluded_blocks = []
included_blocks = []
post_excluded_blocks = []
cursor = 0
pre_excluded_data = b''
pre_included_data = b''
post_included_data = b''
post_excluded_data = b''
for blocks_and_key in blob:
block_key = blocks_and_key['key']
decoded_block_key = from_jsonb64(block_key)
encryptor = load_sym_key(decoded_block_key)
for block_properties in blocks_and_key['blocks']:
cursor += block_properties['size']
if cursor <= offset:
if len(pre_excluded_blocks) and pre_excluded_blocks[-1]['key'] == block_key:
pre_excluded_blocks[-1]['blocks'].append(block_properties)
else:
pre_excluded_blocks.append({'blocks': [block_properties], 'key': block_key})
elif cursor > offset and cursor - block_properties['size'] < offset:
delta = cursor - offset
block = yield Effect(EBlockRead(block_properties['block']))
content = from_jsonb64(block['content'])
block_data = encryptor.decrypt(content)
pre_excluded_data = block_data[:-delta]
pre_included_data = block_data[-delta:][:size]
if size < len(block_data[-delta:]):
post_excluded_data = block_data[-delta:][size:]
elif cursor > offset and cursor <= offset + size:
if len(included_blocks) and included_blocks[-1]['key'] == block_key:
included_blocks[-1]['blocks'].append(block_properties)
else:
included_blocks.append({'blocks': [block_properties], 'key': block_key})
elif cursor > offset + size and cursor - block_properties['size'] < offset + size:
delta = offset + size - (cursor - block_properties['size'])
block = yield Effect(EBlockRead(block_properties['block']))
content = from_jsonb64(block['content'])
block_data = encryptor.decrypt(content)
post_included_data = block_data[:delta]
post_excluded_data = block_data[delta:]
else:
if len(post_excluded_blocks) and post_excluded_blocks[-1]['key'] == block_key:
post_excluded_blocks[-1]['blocks'].append(block_properties)
else:
post_excluded_blocks.append({'blocks': [block_properties],
'key': block_key})
return {
'pre_excluded_blocks': pre_excluded_blocks,
'pre_excluded_data': pre_excluded_data,
'pre_included_data': pre_included_data,
'included_blocks': included_blocks,
'post_included_data': post_included_data,
'post_excluded_data': post_excluded_data,
'post_excluded_blocks': post_excluded_blocks
}
