def diff_versions(self, old_version=None, new_version=None): empty_entries = {'/': None} empty_manifest = {'entries': empty_entries, 'groups': {}, 'dustbin': [], 'versions': {}} # Old manifest if old_version and old_version > 0: old_vlob = yield Effect(EUserVlobRead(old_version)) old_blob = from_jsonb64(old_vlob['blob']) content = self.encryptor.decrypt(old_blob) old_manifest = ejson_loads(content.decode()) elif old_version == 0: old_manifest = empty_manifest else: old_manifest = self.original_manifest # New manifest if new_version and new_version > 0: new_vlob = yield Effect(EUserVlobRead(new_version)) new_blob = from_jsonb64(new_vlob['blob']) content = self.encryptor.decrypt(new_blob) new_manifest = ejson_loads(content.decode()) elif new_version == 0: new_manifest = empty_manifest else: dump = yield self.dumps() new_manifest = ejson_loads(dump) return self.diff(old_manifest, new_manifest)
async def open_connection(self, identity): logger.debug('Connection to backend opened') assert not self._websocket, "Connection to backend already opened" try: self._websocket = await websockets.connect(self.url) # Handle handshake raw = await self._websocket.recv() challenge = ejson_loads(raw) answer = identity.private_key.sign(challenge['challenge'].encode()) await self._websocket.send( ejson_dumps({ 'handshake': 'answer', 'identity': identity.id, 'answer': to_jsonb64(answer) })) resp = ejson_loads(await self._websocket.recv()) if resp['status'] != 'ok': await self.close_connection() raise exception_from_status(resp['status'])(resp['label']) self._ws_recv_handler_task = asyncio.ensure_future( self._ws_recv_handler(), loop=self.loop) if self.watchdog_time: self._watchdog_task = asyncio.ensure_future(self._watchdog(), loop=self.loop) except (ConnectionRefusedError, websockets.exceptions.ConnectionClosed) as exc: raise BackendConnectionError('Cannot connect to backend (%s)' % exc)
def commit(self): is_dirty = yield self.is_dirty() if self.version != 0 and not is_dirty: return # Update manifest entries with new file vlobs (dustbin entries are already commited) vlob_list = yield Effect(EVlobList()) for entry in self.entries.values(): if entry and entry['id'] in vlob_list: file = yield File.load(entry['id'], entry['key'], entry['read_trust_seed'], entry['write_trust_seed']) new_vlob = yield file.commit() if new_vlob and new_vlob is not True: entry['id'] = new_vlob['id'] entry['read_trust_seed'] = new_vlob['read_trust_seed'] entry['write_trust_seed'] = new_vlob['write_trust_seed'] # Commit manifest blob = yield self.dumps() encrypted_blob = self.encryptor.encrypt(blob.encode()) encrypted_blob = to_jsonb64(encrypted_blob) yield Effect(EVlobUpdate(self.id, self.write_trust_seed, self.version + 1, encrypted_blob)) self.original_manifest = ejson_loads(blob) new_vlob = yield Effect(EVlobSynchronize(self.id)) if new_vlob: if new_vlob is not True: self.id = new_vlob['id'] self.read_trust_seed = new_vlob['read_trust_seed'] self.write_trust_seed = new_vlob['write_trust_seed'] new_vlob = self.get_vlob() self.version += 1 return new_vlob
def stat(self): version = self.get_version() vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version)) encrypted_blob = vlob['blob'] encrypted_blob = from_jsonb64(encrypted_blob) blob = self.encryptor.decrypt(encrypted_blob) blob = ejson_loads(blob.decode()) size = 0 for blocks_and_key in blob: for block in blocks_and_key['blocks']: size += block['size'] for modification in self.modifications: if modification[0] == self.write: end_offset = modification[2] + len(modification[1]) if size < end_offset: size = end_offset elif modification[0] == self.truncate: if size > modification[1]: size = modification[1] else: raise NotImplementedError() # TODO don't provide atime field if we don't know it? # TODO real date return { 'id': self.id, 'type': 'file', 'created': '2012-01-01T00:00:00', 'updated': '2012-01-01T00:00:00', 'size': size, 'version': vlob['version'] }
def commit(self, recursive=True): is_dirty = yield self.is_dirty() if self.version != 0 and not is_dirty: return # Update manifest with new group vlobs vlob_list = yield Effect(EVlobList()) if recursive: for group_manifest in self.group_manifests.values(): new_vlob = yield group_manifest.commit() if new_vlob and new_vlob is not True: old_vlob = group_manifest.get_vlob() new_vlob['key'] = old_vlob['key'] group_manifest.update_vlob(new_vlob) # Update manifest entries with new file vlobs (dustbin entries are already commited) for entry in self.entries.values(): if entry and entry['id'] in vlob_list: file = yield File.load(entry['id'], entry['key'], entry['read_trust_seed'], entry['write_trust_seed']) new_vlob = yield file.commit() if new_vlob and new_vlob is not True: entry['id'] = new_vlob['id'] entry['read_trust_seed'] = new_vlob['read_trust_seed'] entry['write_trust_seed'] = new_vlob['write_trust_seed'] # Commit manifest blob = yield self.dumps() encrypted_blob = self.encryptor.pub_key.encrypt(blob.encode()) encrypted_blob = to_jsonb64(encrypted_blob) yield Effect(EUserVlobUpdate(self.version + 1, encrypted_blob)) self.original_manifest = ejson_loads(blob) synchronized = yield Effect(EUserVlobSynchronize()) if synchronized: self.version += 1
async def send_cmd(self, cmd, **kwargs): msg = {'cmd': cmd, **kwargs} raw_msg = ejson_dumps(msg).encode() self.writer.write(raw_msg) self.writer.write(b'\n') raw_resp = await self.reader.readline() return ejson_loads(raw_resp.decode())
def is_dirty(self): dump = yield self.dumps() current_manifest = ejson_loads(dump) diff = self.diff(self.original_manifest, current_manifest) for category in diff.keys(): for operation in diff[category].keys(): if diff[category][operation]: return True return False
async def perform_group_read(self, intent): async with self.connection.acquire() as conn: async with conn.cursor() as cur: await cur.execute('SELECT body FROM groups WHERE id=%s', (intent.name, )) ret = await cur.fetchone() if ret is None: raise GroupNotFound('Group not found.') data = ejson_loads(ret[0]) return Group(name=intent.name, **data)
def send_cmd(self, **msg): with self._socket_lock: req = ejson_dumps(msg).encode() + b'\n' logger.debug('Send: %r' % req) self.sock.send(req) raw_reps = self.sock.recv(4096) while raw_reps[-1] != ord(b'\n'): raw_reps += self.sock.recv(4096) logger.debug('Received: %r' % raw_reps) return ejson_loads(raw_reps[:-1].decode())
async def perform_group_add_identities(self, intent): async with self.connection.acquire() as conn: async with conn.cursor() as cur: await cur.execute('SELECT body FROM groups WHERE id=%s', (intent.name, )) ret = await cur.fetchone() if ret is None: raise GroupNotFound('Group not found.') group = ejson_loads(ret[0]) group_entry = 'admins' if intent.admin else 'users' group[group_entry] = list(set(group[group_entry]) | set(intent.identities)) await cur.execute('UPDATE groups SET body=%s WHERE id=%s', (ejson_dumps(group), intent.name))
def execute_raw_cmd(raw_cmd: str): try: params = ejson_loads(raw_cmd) except json.decoder.JSONDecodeError: ret = {'status': 'bad_msg', 'label': 'Message is not a valid JSON.'} else: cmd_type = params.pop('cmd', None) if not isinstance(cmd_type, str): ret = { 'status': 'bad_msg', 'label': '`cmd` string field is mandatory.' } else: ret = yield execute_cmd(cmd_type, params) return ejson_dumps(ret)
def reload(self, reset=False): vlob = yield Effect(EUserVlobRead()) if not vlob['blob']: raise ManifestNotFound('User manifest not found.') blob = from_jsonb64(vlob['blob']) content = self.encryptor.decrypt(blob) if not reset and vlob['version'] <= self.version: return new_manifest = ejson_loads(content.decode()) backup_new_manifest = deepcopy(new_manifest) consistency = yield self.check_consistency(new_manifest) if not consistency: raise ManifestError('not_consistent', 'User manifest not consistent.') if not reset: diff = yield self.diff_versions() new_manifest = self.patch(new_manifest, diff) self.entries = new_manifest['entries'] self.dustbin = new_manifest['dustbin'] self.version = vlob['version'] self.group_manifests = {} for group, group_vlob in new_manifest['groups'].items(): self.import_group_vlob(group, group_vlob) self.original_manifest = backup_new_manifest versions = new_manifest['versions'] file_vlob = None for vlob_id, version in sorted(versions.items()): for entry in self.entries.values(): if entry and entry['id'] == vlob_id: file_vlob = entry break if not file_vlob: for entry in self.dustbin: if entry['id'] == vlob_id: file_vlob = {'id': entry['id'], 'read_trust_seed': entry['read_trust_seed'], 'write_trust_seed': entry['write_trust_seed'], 'key': entry['key']} break if file_vlob: file_vlob = None file = yield File.load(entry['id'], entry['key'], entry['read_trust_seed'], entry['write_trust_seed']) try: yield file.restore(version) except FileError: pass
def reload(self, reset=False): # Subscribe to events # yield Effect(EConnectEvent('on_vlob_updated', self.id, self.handler)) # TODO call vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed)) blob = from_jsonb64(vlob['blob']) content = self.encryptor.decrypt(blob) if not reset and vlob['version'] <= self.version: return new_manifest = ejson_loads(content.decode()) backup_new_manifest = deepcopy(new_manifest) consistency = yield self.check_consistency(new_manifest) if not consistency: raise ManifestError('not_consistent', 'Group manifest not consistent.') if not reset: diff = yield self.diff_versions() new_manifest = self.patch(new_manifest, diff) self.entries = new_manifest['entries'] self.dustbin = new_manifest['dustbin'] self.version = vlob['version'] self.original_manifest = backup_new_manifest versions = new_manifest['versions'] file_vlob = None for vlob_id, version in sorted(versions.items()): for entry in self.entries.values(): if entry and entry['id'] == vlob_id: file_vlob = entry break if not file_vlob: for entry in self.dustbin: if entry['id'] == vlob_id: file_vlob = {'id': entry['id'], 'read_trust_seed': entry['read_trust_seed'], 'write_trust_seed': entry['write_trust_seed'], 'key': entry['key']} break if file_vlob: file_vlob = None file = yield File.load(entry['id'], entry['key'], entry['read_trust_seed'], entry['write_trust_seed']) try: yield file.restore(version) except FileError: pass
async def run(): try: reader, writer = await asyncio.open_unix_connection(path=socket) except (FileNotFoundError, ConnectionRefusedError): raise SystemExit('ERROR: Cannot connect to parsec core at %s' % socket) msg = { 'cmd': 'identity_signup', 'id': identity, 'password': password, 'key_size': key_size } writer.write(ejson_dumps(msg).encode()) writer.write(b'\n') raw_resp = await reader.readline() resp = ejson_loads(raw_resp.decode()) writer.close() print(resp)
async def _ws_recv_handler(self): # Given command responses and notifications are all send through the # same websocket, separate them here, passing command response thanks # to a Queue. while True: raw = await self._websocket.recv() try: if isinstance(raw, bytes): raw = raw.decode() recv = ejson_loads(raw) if 'status' in recv: # Message response self._resp_queue.put_nowait(recv) else: # Event self._signal_ns.signal(recv['event']).send(recv['sender']) except (KeyError, TypeError, json.JSONDecodeError): # Dummy ??? logger.warning('Backend server sent invalid message: %s' % raw)
async def repl(socket_path): from parsec import __version__ print('Parsec shell version: %s' % __version__) print('Connecting to: %s' % socket_path) open_conn = partial(asyncio.open_unix_connection, path=socket_path) reader, writer = await open_conn() quit = False while not quit: data = input('>>> ') if data in ('quit', 'q'): writer.close() return elif data in ('help', 'h'): print('No help for the braves !') continue elif data in ('reload', 'r'): writer.close() reader, writer = await open_conn() continue writer.write(data.encode()) writer.write(b'\n') raw_resp = await reader.readline() resp = ejson_loads(raw_resp.decode()) print('Received: %r' % resp)
def handshake(self): if self.id: raise HandshakeError('Handshake already done.') challenge = _generate_challenge() query = {'handshake': 'challenge', 'challenge': challenge} yield Effect(EHandshakeSend(ejson_dumps(query))) raw_resp = yield Effect(EHandshakeRecv()) try: resp = ejson_loads(raw_resp) except (TypeError, json.JSONDecodeError): error = HandshakeError('Invalid challenge response format') yield Effect(EHandshakeSend(error.to_raw())) raise error resp = HandshakeAnswerSchema().load(resp) claimed_identity = resp['identity'] try: pubkey = yield Effect(EPubKeyGet(claimed_identity)) pubkey.verify(resp['answer'], challenge.encode()) yield Effect(EHandshakeSend('{"status": "ok", "handshake": "done"}')) self.id = claimed_identity except (TypeError, PubKeyNotFound, InvalidSignature): error = HandshakeError('Invalid signature, challenge or identity') yield Effect(EHandshakeSend(error.to_raw())) raise error
def parse_cmd(raw_cmd: bytes): try: return ejson_loads(raw_cmd.decode('utf-8')) except (json.decoder.JSONDecodeError, UnicodeDecodeError): pass
def _find_matching_blocks(self, size=None, offset=0): if size is None: size = sys.maxsize pre_excluded_blocks = [] post_excluded_blocks = [] version = self.get_version() vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version)) blob = vlob['blob'] encrypted_blob = from_jsonb64(blob) blob = self.encryptor.decrypt(encrypted_blob) blob = ejson_loads(blob.decode()) pre_excluded_blocks = [] included_blocks = [] post_excluded_blocks = [] cursor = 0 pre_excluded_data = b'' pre_included_data = b'' post_included_data = b'' post_excluded_data = b'' for blocks_and_key in blob: block_key = blocks_and_key['key'] decoded_block_key = from_jsonb64(block_key) encryptor = load_sym_key(decoded_block_key) for block_properties in blocks_and_key['blocks']: cursor += block_properties['size'] if cursor <= offset: if len(pre_excluded_blocks) and pre_excluded_blocks[-1]['key'] == block_key: pre_excluded_blocks[-1]['blocks'].append(block_properties) else: pre_excluded_blocks.append({'blocks': [block_properties], 'key': block_key}) elif cursor > offset and cursor - block_properties['size'] < offset: delta = cursor - offset block = yield Effect(EBlockRead(block_properties['block'])) content = from_jsonb64(block['content']) block_data = encryptor.decrypt(content) pre_excluded_data = block_data[:-delta] pre_included_data = block_data[-delta:][:size] if size < len(block_data[-delta:]): post_excluded_data = block_data[-delta:][size:] elif cursor > offset and cursor <= offset + size: if len(included_blocks) and included_blocks[-1]['key'] == block_key: included_blocks[-1]['blocks'].append(block_properties) else: included_blocks.append({'blocks': [block_properties], 'key': block_key}) elif cursor > offset + size and cursor - block_properties['size'] < offset + size: delta = offset + size - (cursor - block_properties['size']) block = yield Effect(EBlockRead(block_properties['block'])) content = from_jsonb64(block['content']) block_data = encryptor.decrypt(content) post_included_data = block_data[:delta] post_excluded_data = block_data[delta:] else: if len(post_excluded_blocks) and post_excluded_blocks[-1]['key'] == block_key: post_excluded_blocks[-1]['blocks'].append(block_properties) else: post_excluded_blocks.append({'blocks': [block_properties], 'key': block_key}) return { 'pre_excluded_blocks': pre_excluded_blocks, 'pre_excluded_data': pre_excluded_data, 'pre_included_data': pre_included_data, 'included_blocks': included_blocks, 'post_included_data': post_included_data, 'post_excluded_data': post_excluded_data, 'post_excluded_blocks': post_excluded_blocks }