def diff_versions(self, old_version=None, new_version=None): empty_entries = {'/': None} empty_manifest = {'entries': empty_entries, 'groups': {}, 'dustbin': [], 'versions': {}} # Old manifest if old_version and old_version > 0: old_vlob = yield Effect(EUserVlobRead(old_version)) old_blob = from_jsonb64(old_vlob['blob']) content = self.encryptor.decrypt(old_blob) old_manifest = ejson_loads(content.decode()) elif old_version == 0: old_manifest = empty_manifest else: old_manifest = self.original_manifest # New manifest if new_version and new_version > 0: new_vlob = yield Effect(EUserVlobRead(new_version)) new_blob = from_jsonb64(new_vlob['blob']) content = self.encryptor.decrypt(new_blob) new_manifest = ejson_loads(content.decode()) elif new_version == 0: new_manifest = empty_manifest else: dump = yield self.dumps() new_manifest = ejson_loads(dump) return self.diff(old_manifest, new_manifest)
def perform_pubkey_get(intent): msg = {'id': intent.id} ret = yield Effect(BackendCmd('pubkey_get', msg)) status = ret['status'] if status != 'ok': raise exception_from_status(status)(ret['label']) return PubKey(intent.id, from_jsonb64(ret['key']))
def load(cls, id, key, read_trust_seed, write_trust_seed): self = GroupManifest(id) self.read_trust_seed = read_trust_seed self.write_trust_seed = write_trust_seed self.encryptor = load_sym_key(from_jsonb64(key)) yield self.reload(reset=True) return self
def stat(self): version = self.get_version() vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version)) encrypted_blob = vlob['blob'] encrypted_blob = from_jsonb64(encrypted_blob) blob = self.encryptor.decrypt(encrypted_blob) blob = ejson_loads(blob.decode()) size = 0 for blocks_and_key in blob: for block in blocks_and_key['blocks']: size += block['size'] for modification in self.modifications: if modification[0] == self.write: end_offset = modification[2] + len(modification[1]) if size < end_offset: size = end_offset elif modification[0] == self.truncate: if size > modification[1]: size = modification[1] else: raise NotImplementedError() # TODO don't provide atime field if we don't know it? # TODO real date return { 'id': self.id, 'type': 'file', 'created': '2012-01-01T00:00:00', 'updated': '2012-01-01T00:00:00', 'size': size, 'version': vlob['version'] }
def check_consistency(self, manifest): entries = [entry for entry in list(manifest['entries'].values()) if entry] entries += manifest['dustbin'] for entry in entries: try: vlob = yield Effect(EVlobRead(entry['id'], entry['read_trust_seed'], manifest['versions'][entry['id']])) encrypted_blob = vlob['blob'] encrypted_blob = from_jsonb64(encrypted_blob) key = from_jsonb64(entry['key']) if entry['key'] else None encryptor = load_sym_key(key) encryptor.decrypt(encrypted_blob) # TODO check exception except VlobNotFound: return False return True
def read(self, size=None, offset=0): self.flush() # TODO use flags response = self._operations.send_cmd( cmd='file_read', path=self.path, size=size, offset=offset) if response['status'] != 'ok': raise FuseOSError(ENOENT) return from_jsonb64(response['content'])
def perform_user_vlob_read(intent): msg = {} if intent.version is not None: msg['version'] = intent.version ret = yield Effect(BackendCmd('user_vlob_read', msg)) status = ret['status'] if status != 'ok': raise exception_from_status(status)(ret['label']) return UserVlobAtom(ret['version'], from_jsonb64(ret['blob']))
def perform_vlob_read(intent): assert isinstance(intent.id, str) msg = {'id': intent.id, 'trust_seed': intent.trust_seed} if intent.version is not None: msg['version'] = intent.version ret = yield Effect(BackendCmd('vlob_read', msg)) status = ret['status'] if status != 'ok': raise exception_from_status(status)(ret['label']) return VlobAtom(ret['id'], ret['version'], from_jsonb64(ret['blob']))
def perform_message_get(intent): msg = {'recipient': intent.recipient, 'offset': intent.offset} ret = yield Effect(BackendCmd('message_get', msg)) status = ret['status'] if status != 'ok': raise exception_from_status(status)(ret['label']) return [ Message(msg['count'], from_jsonb64(msg['body'])) for msg in ret['messages'] ]
def check_consistency(self, manifest): consistency = yield super().check_consistency(manifest) if consistency is False: return False for entry in manifest['groups'].values(): try: vlob = yield Effect(EVlobRead(entry['id'], entry['read_trust_seed'])) encrypted_blob = vlob['blob'] key = from_jsonb64(entry['key']) if entry['key'] else None encryptor = load_sym_key(key) encryptor.decrypt(encrypted_blob) except VlobNotFound: return False return True
def read(self, size=None, offset=0): yield self.flush() # Get data matching_blocks = yield self._find_matching_blocks(size, offset) data = matching_blocks['pre_included_data'] for blocks_and_key in matching_blocks['included_blocks']: block_key = blocks_and_key['key'] decoded_block_key = from_jsonb64(block_key) encryptor = load_sym_key(decoded_block_key) for block_properties in blocks_and_key['blocks']: block = yield Effect(EBlockRead(block_properties['block'])) # Decrypt # TODO: clean this hack if isinstance(block['content'], str): block_content = from_jsonb64(block['content']) else: block_content = from_jsonb64(block['content'].decode()) chunk_data = encryptor.decrypt(block_content) # Check integrity assert digest(chunk_data) == block_properties['digest'] assert len(chunk_data) == block_properties['size'] data += chunk_data data += matching_blocks['post_included_data'] return data
def reload(self, reset=False): vlob = yield Effect(EUserVlobRead()) if not vlob['blob']: raise ManifestNotFound('User manifest not found.') blob = from_jsonb64(vlob['blob']) content = self.encryptor.decrypt(blob) if not reset and vlob['version'] <= self.version: return new_manifest = ejson_loads(content.decode()) backup_new_manifest = deepcopy(new_manifest) consistency = yield self.check_consistency(new_manifest) if not consistency: raise ManifestError('not_consistent', 'User manifest not consistent.') if not reset: diff = yield self.diff_versions() new_manifest = self.patch(new_manifest, diff) self.entries = new_manifest['entries'] self.dustbin = new_manifest['dustbin'] self.version = vlob['version'] self.group_manifests = {} for group, group_vlob in new_manifest['groups'].items(): self.import_group_vlob(group, group_vlob) self.original_manifest = backup_new_manifest versions = new_manifest['versions'] file_vlob = None for vlob_id, version in sorted(versions.items()): for entry in self.entries.values(): if entry and entry['id'] == vlob_id: file_vlob = entry break if not file_vlob: for entry in self.dustbin: if entry['id'] == vlob_id: file_vlob = {'id': entry['id'], 'read_trust_seed': entry['read_trust_seed'], 'write_trust_seed': entry['write_trust_seed'], 'key': entry['key']} break if file_vlob: file_vlob = None file = yield File.load(entry['id'], entry['key'], entry['read_trust_seed'], entry['write_trust_seed']) try: yield file.restore(version) except FileError: pass
def reencrypt(self): yield self.flush() version = self.get_version() old_vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version)) old_blob = old_vlob['blob'] old_encrypted_blob = from_jsonb64(old_blob) new_blob = self.encryptor.decrypt(old_encrypted_blob) self.encryptor = generate_sym_key() new_encrypted_blob = self.encryptor.encrypt(new_blob) new_encrypted_blob = to_jsonb64(new_encrypted_blob) new_vlob = yield Effect(EVlobCreate(new_encrypted_blob)) del File.files[self.id] self.id = new_vlob['id'] self.read_trust_seed = new_vlob['read_trust_seed'] self.write_trust_seed = new_vlob['write_trust_seed'] File.files[self.id] = self self.dirty = True
def reload(self, reset=False): # Subscribe to events # yield Effect(EConnectEvent('on_vlob_updated', self.id, self.handler)) # TODO call vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed)) blob = from_jsonb64(vlob['blob']) content = self.encryptor.decrypt(blob) if not reset and vlob['version'] <= self.version: return new_manifest = ejson_loads(content.decode()) backup_new_manifest = deepcopy(new_manifest) consistency = yield self.check_consistency(new_manifest) if not consistency: raise ManifestError('not_consistent', 'Group manifest not consistent.') if not reset: diff = yield self.diff_versions() new_manifest = self.patch(new_manifest, diff) self.entries = new_manifest['entries'] self.dustbin = new_manifest['dustbin'] self.version = vlob['version'] self.original_manifest = backup_new_manifest versions = new_manifest['versions'] file_vlob = None for vlob_id, version in sorted(versions.items()): for entry in self.entries.values(): if entry and entry['id'] == vlob_id: file_vlob = entry break if not file_vlob: for entry in self.dustbin: if entry['id'] == vlob_id: file_vlob = {'id': entry['id'], 'read_trust_seed': entry['read_trust_seed'], 'write_trust_seed': entry['write_trust_seed'], 'key': entry['key']} break if file_vlob: file_vlob = None file = yield File.load(entry['id'], entry['key'], entry['read_trust_seed'], entry['write_trust_seed']) try: yield file.restore(version) except FileError: pass
def load(cls, id, key, read_trust_seed, write_trust_seed, version=None): if id in File.files: return File.files[id] self = File() self.id = id self.read_trust_seed = read_trust_seed self.write_trust_seed = write_trust_seed self.encryptor = load_sym_key(from_jsonb64(key)) vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version)) self.version = vlob['version'] self.dirty = False vlob_list = yield Effect(EVlobList()) if vlob['id'] in vlob_list: self.dirty = True self.version -= 1 self.modifications = [] File.files[self.id] = self return self
def update_vlob(self, new_vlob): self.id = new_vlob['id'] self.encryptor = load_sym_key(from_jsonb64(new_vlob['key'])) self.read_trust_seed = new_vlob['read_trust_seed'] self.write_trust_seed = new_vlob['write_trust_seed']
def _find_matching_blocks(self, size=None, offset=0): if size is None: size = sys.maxsize pre_excluded_blocks = [] post_excluded_blocks = [] version = self.get_version() vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version)) blob = vlob['blob'] encrypted_blob = from_jsonb64(blob) blob = self.encryptor.decrypt(encrypted_blob) blob = ejson_loads(blob.decode()) pre_excluded_blocks = [] included_blocks = [] post_excluded_blocks = [] cursor = 0 pre_excluded_data = b'' pre_included_data = b'' post_included_data = b'' post_excluded_data = b'' for blocks_and_key in blob: block_key = blocks_and_key['key'] decoded_block_key = from_jsonb64(block_key) encryptor = load_sym_key(decoded_block_key) for block_properties in blocks_and_key['blocks']: cursor += block_properties['size'] if cursor <= offset: if len(pre_excluded_blocks) and pre_excluded_blocks[-1]['key'] == block_key: pre_excluded_blocks[-1]['blocks'].append(block_properties) else: pre_excluded_blocks.append({'blocks': [block_properties], 'key': block_key}) elif cursor > offset and cursor - block_properties['size'] < offset: delta = cursor - offset block = yield Effect(EBlockRead(block_properties['block'])) content = from_jsonb64(block['content']) block_data = encryptor.decrypt(content) pre_excluded_data = block_data[:-delta] pre_included_data = block_data[-delta:][:size] if size < len(block_data[-delta:]): post_excluded_data = block_data[-delta:][size:] elif cursor > offset and cursor <= offset + size: if len(included_blocks) and included_blocks[-1]['key'] == block_key: included_blocks[-1]['blocks'].append(block_properties) else: included_blocks.append({'blocks': [block_properties], 'key': block_key}) elif cursor > offset + size and cursor - block_properties['size'] < offset + size: delta = offset + size - (cursor - block_properties['size']) block = yield Effect(EBlockRead(block_properties['block'])) content = from_jsonb64(block['content']) block_data = encryptor.decrypt(content) post_included_data = block_data[:delta] post_excluded_data = block_data[delta:] else: if len(post_excluded_blocks) and post_excluded_blocks[-1]['key'] == block_key: post_excluded_blocks[-1]['blocks'].append(block_properties) else: post_excluded_blocks.append({'blocks': [block_properties], 'key': block_key}) return { 'pre_excluded_blocks': pre_excluded_blocks, 'pre_excluded_data': pre_excluded_data, 'pre_included_data': pre_included_data, 'included_blocks': included_blocks, 'post_included_data': post_included_data, 'post_excluded_data': post_excluded_data, 'post_excluded_blocks': post_excluded_blocks }