def diff_versions(self, old_version=None, new_version=None):
empty_entries = {'/': None}
empty_manifest = {'entries': empty_entries, 'groups': {}, 'dustbin': [], 'versions': {}}
# Old manifest
if old_version and old_version > 0:
old_vlob = yield Effect(EUserVlobRead(old_version))
old_blob = from_jsonb64(old_vlob['blob'])
content = self.encryptor.decrypt(old_blob)
old_manifest = ejson_loads(content.decode())
elif old_version == 0:
old_manifest = empty_manifest
else:
old_manifest = self.original_manifest
# New manifest
if new_version and new_version > 0:
new_vlob = yield Effect(EUserVlobRead(new_version))
new_blob = from_jsonb64(new_vlob['blob'])
content = self.encryptor.decrypt(new_blob)
new_manifest = ejson_loads(content.decode())
elif new_version == 0:
new_manifest = empty_manifest
else:
dump = yield self.dumps()
new_manifest = ejson_loads(dump)
return self.diff(old_manifest, new_manifest)
async def open_connection(self, identity):
logger.debug('Connection to backend opened')
assert not self._websocket, "Connection to backend already opened"
try:
self._websocket = await websockets.connect(self.url)
# Handle handshake
raw = await self._websocket.recv()
challenge = ejson_loads(raw)
answer = identity.private_key.sign(challenge['challenge'].encode())
await self._websocket.send(
ejson_dumps({
'handshake': 'answer',
'identity': identity.id,
'answer': to_jsonb64(answer)
}))
resp = ejson_loads(await self._websocket.recv())
if resp['status'] != 'ok':
await self.close_connection()
raise exception_from_status(resp['status'])(resp['label'])
self._ws_recv_handler_task = asyncio.ensure_future(
self._ws_recv_handler(), loop=self.loop)
if self.watchdog_time:
self._watchdog_task = asyncio.ensure_future(self._watchdog(),
loop=self.loop)
except (ConnectionRefusedError,
websockets.exceptions.ConnectionClosed) as exc:
raise BackendConnectionError('Cannot connect to backend (%s)' %
exc)
def commit(self):
is_dirty = yield self.is_dirty()
if self.version != 0 and not is_dirty:
return
# Update manifest entries with new file vlobs (dustbin entries are already commited)
vlob_list = yield Effect(EVlobList())
for entry in self.entries.values():
if entry and entry['id'] in vlob_list:
file = yield File.load(entry['id'],
entry['key'],
entry['read_trust_seed'],
entry['write_trust_seed'])
new_vlob = yield file.commit()
if new_vlob and new_vlob is not True:
entry['id'] = new_vlob['id']
entry['read_trust_seed'] = new_vlob['read_trust_seed']
entry['write_trust_seed'] = new_vlob['write_trust_seed']
# Commit manifest
blob = yield self.dumps()
encrypted_blob = self.encryptor.encrypt(blob.encode())
encrypted_blob = to_jsonb64(encrypted_blob)
yield Effect(EVlobUpdate(self.id, self.write_trust_seed, self.version + 1, encrypted_blob))
self.original_manifest = ejson_loads(blob)
new_vlob = yield Effect(EVlobSynchronize(self.id))
if new_vlob:
if new_vlob is not True:
self.id = new_vlob['id']
self.read_trust_seed = new_vlob['read_trust_seed']
self.write_trust_seed = new_vlob['write_trust_seed']
new_vlob = self.get_vlob()
self.version += 1
return new_vlob
def stat(self):
version = self.get_version()
vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version))
encrypted_blob = vlob['blob']
encrypted_blob = from_jsonb64(encrypted_blob)
blob = self.encryptor.decrypt(encrypted_blob)
blob = ejson_loads(blob.decode())
size = 0
for blocks_and_key in blob:
for block in blocks_and_key['blocks']:
size += block['size']
for modification in self.modifications:
if modification[0] == self.write:
end_offset = modification[2] + len(modification[1])
if size < end_offset:
size = end_offset
elif modification[0] == self.truncate:
if size > modification[1]:
size = modification[1]
else:
raise NotImplementedError()
# TODO don't provide atime field if we don't know it?
# TODO real date
return {
'id': self.id,
'type': 'file',
'created': '2012-01-01T00:00:00',
'updated': '2012-01-01T00:00:00',
'size': size,
'version': vlob['version']
}
def commit(self, recursive=True):
is_dirty = yield self.is_dirty()
if self.version != 0 and not is_dirty:
return
# Update manifest with new group vlobs
vlob_list = yield Effect(EVlobList())
if recursive:
for group_manifest in self.group_manifests.values():
new_vlob = yield group_manifest.commit()
if new_vlob and new_vlob is not True:
old_vlob = group_manifest.get_vlob()
new_vlob['key'] = old_vlob['key']
group_manifest.update_vlob(new_vlob)
# Update manifest entries with new file vlobs (dustbin entries are already commited)
for entry in self.entries.values():
if entry and entry['id'] in vlob_list:
file = yield File.load(entry['id'],
entry['key'],
entry['read_trust_seed'],
entry['write_trust_seed'])
new_vlob = yield file.commit()
if new_vlob and new_vlob is not True:
entry['id'] = new_vlob['id']
entry['read_trust_seed'] = new_vlob['read_trust_seed']
entry['write_trust_seed'] = new_vlob['write_trust_seed']
# Commit manifest
blob = yield self.dumps()
encrypted_blob = self.encryptor.pub_key.encrypt(blob.encode())
encrypted_blob = to_jsonb64(encrypted_blob)
yield Effect(EUserVlobUpdate(self.version + 1, encrypted_blob))
self.original_manifest = ejson_loads(blob)
synchronized = yield Effect(EUserVlobSynchronize())
if synchronized:
self.version += 1
async def send_cmd(self, cmd, **kwargs):
msg = {'cmd': cmd, **kwargs}
raw_msg = ejson_dumps(msg).encode()
self.writer.write(raw_msg)
self.writer.write(b'\n')
raw_resp = await self.reader.readline()
return ejson_loads(raw_resp.decode())
def is_dirty(self):
dump = yield self.dumps()
current_manifest = ejson_loads(dump)
diff = self.diff(self.original_manifest, current_manifest)
for category in diff.keys():
for operation in diff[category].keys():
if diff[category][operation]:
return True
return False
async def perform_group_read(self, intent):
async with self.connection.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute('SELECT body FROM groups WHERE id=%s', (intent.name, ))
ret = await cur.fetchone()
if ret is None:
raise GroupNotFound('Group not found.')
data = ejson_loads(ret[0])
return Group(name=intent.name, **data)
def send_cmd(self, **msg):
with self._socket_lock:
req = ejson_dumps(msg).encode() + b'\n'
logger.debug('Send: %r' % req)
self.sock.send(req)
raw_reps = self.sock.recv(4096)
while raw_reps[-1] != ord(b'\n'):
raw_reps += self.sock.recv(4096)
logger.debug('Received: %r' % raw_reps)
return ejson_loads(raw_reps[:-1].decode())
async def perform_group_add_identities(self, intent):
async with self.connection.acquire() as conn:
async with conn.cursor() as cur:
await cur.execute('SELECT body FROM groups WHERE id=%s', (intent.name, ))
ret = await cur.fetchone()
if ret is None:
raise GroupNotFound('Group not found.')
group = ejson_loads(ret[0])
group_entry = 'admins' if intent.admin else 'users'
group[group_entry] = list(set(group[group_entry]) | set(intent.identities))
await cur.execute('UPDATE groups SET body=%s WHERE id=%s',
(ejson_dumps(group), intent.name))
def execute_raw_cmd(raw_cmd: str):
try:
params = ejson_loads(raw_cmd)
except json.decoder.JSONDecodeError:
ret = {'status': 'bad_msg', 'label': 'Message is not a valid JSON.'}
else:
cmd_type = params.pop('cmd', None)
if not isinstance(cmd_type, str):
ret = {
'status': 'bad_msg',
'label': '`cmd` string field is mandatory.'
}
else:
ret = yield execute_cmd(cmd_type, params)
return ejson_dumps(ret)
def reload(self, reset=False):
vlob = yield Effect(EUserVlobRead())
if not vlob['blob']:
raise ManifestNotFound('User manifest not found.')
blob = from_jsonb64(vlob['blob'])
content = self.encryptor.decrypt(blob)
if not reset and vlob['version'] <= self.version:
return
new_manifest = ejson_loads(content.decode())
backup_new_manifest = deepcopy(new_manifest)
consistency = yield self.check_consistency(new_manifest)
if not consistency:
raise ManifestError('not_consistent', 'User manifest not consistent.')
if not reset:
diff = yield self.diff_versions()
new_manifest = self.patch(new_manifest, diff)
self.entries = new_manifest['entries']
self.dustbin = new_manifest['dustbin']
self.version = vlob['version']
self.group_manifests = {}
for group, group_vlob in new_manifest['groups'].items():
self.import_group_vlob(group, group_vlob)
self.original_manifest = backup_new_manifest
versions = new_manifest['versions']
file_vlob = None
for vlob_id, version in sorted(versions.items()):
for entry in self.entries.values():
if entry and entry['id'] == vlob_id:
file_vlob = entry
break
if not file_vlob:
for entry in self.dustbin:
if entry['id'] == vlob_id:
file_vlob = {'id': entry['id'],
'read_trust_seed': entry['read_trust_seed'],
'write_trust_seed': entry['write_trust_seed'],
'key': entry['key']}
break
if file_vlob:
file_vlob = None
file = yield File.load(entry['id'],
entry['key'],
entry['read_trust_seed'],
entry['write_trust_seed'])
try:
yield file.restore(version)
except FileError:
pass
def reload(self, reset=False):
# Subscribe to events
# yield Effect(EConnectEvent('on_vlob_updated', self.id, self.handler)) # TODO call
vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed))
blob = from_jsonb64(vlob['blob'])
content = self.encryptor.decrypt(blob)
if not reset and vlob['version'] <= self.version:
return
new_manifest = ejson_loads(content.decode())
backup_new_manifest = deepcopy(new_manifest)
consistency = yield self.check_consistency(new_manifest)
if not consistency:
raise ManifestError('not_consistent', 'Group manifest not consistent.')
if not reset:
diff = yield self.diff_versions()
new_manifest = self.patch(new_manifest, diff)
self.entries = new_manifest['entries']
self.dustbin = new_manifest['dustbin']
self.version = vlob['version']
self.original_manifest = backup_new_manifest
versions = new_manifest['versions']
file_vlob = None
for vlob_id, version in sorted(versions.items()):
for entry in self.entries.values():
if entry and entry['id'] == vlob_id:
file_vlob = entry
break
if not file_vlob:
for entry in self.dustbin:
if entry['id'] == vlob_id:
file_vlob = {'id': entry['id'],
'read_trust_seed': entry['read_trust_seed'],
'write_trust_seed': entry['write_trust_seed'],
'key': entry['key']}
break
if file_vlob:
file_vlob = None
file = yield File.load(entry['id'],
entry['key'],
entry['read_trust_seed'],
entry['write_trust_seed'])
try:
yield file.restore(version)
except FileError:
pass
async def run():
try:
reader, writer = await asyncio.open_unix_connection(path=socket)
except (FileNotFoundError, ConnectionRefusedError):
raise SystemExit('ERROR: Cannot connect to parsec core at %s' %
socket)
msg = {
'cmd': 'identity_signup',
'id': identity,
'password': password,
'key_size': key_size
}
writer.write(ejson_dumps(msg).encode())
writer.write(b'\n')
raw_resp = await reader.readline()
resp = ejson_loads(raw_resp.decode())
writer.close()
print(resp)
async def _ws_recv_handler(self):
# Given command responses and notifications are all send through the
# same websocket, separate them here, passing command response thanks
# to a Queue.
while True:
raw = await self._websocket.recv()
try:
if isinstance(raw, bytes):
raw = raw.decode()
recv = ejson_loads(raw)
if 'status' in recv:
# Message response
self._resp_queue.put_nowait(recv)
else:
# Event
self._signal_ns.signal(recv['event']).send(recv['sender'])
except (KeyError, TypeError, json.JSONDecodeError):
# Dummy ???
logger.warning('Backend server sent invalid message: %s' % raw)
async def repl(socket_path):
from parsec import __version__
print('Parsec shell version: %s' % __version__)
print('Connecting to: %s' % socket_path)
open_conn = partial(asyncio.open_unix_connection, path=socket_path)
reader, writer = await open_conn()
quit = False
while not quit:
data = input('>>> ')
if data in ('quit', 'q'):
writer.close()
return
elif data in ('help', 'h'):
print('No help for the braves !')
continue
elif data in ('reload', 'r'):
writer.close()
reader, writer = await open_conn()
continue
writer.write(data.encode())
writer.write(b'\n')
raw_resp = await reader.readline()
resp = ejson_loads(raw_resp.decode())
print('Received: %r' % resp)
def handshake(self):
if self.id:
raise HandshakeError('Handshake already done.')
challenge = _generate_challenge()
query = {'handshake': 'challenge', 'challenge': challenge}
yield Effect(EHandshakeSend(ejson_dumps(query)))
raw_resp = yield Effect(EHandshakeRecv())
try:
resp = ejson_loads(raw_resp)
except (TypeError, json.JSONDecodeError):
error = HandshakeError('Invalid challenge response format')
yield Effect(EHandshakeSend(error.to_raw()))
raise error
resp = HandshakeAnswerSchema().load(resp)
claimed_identity = resp['identity']
try:
pubkey = yield Effect(EPubKeyGet(claimed_identity))
pubkey.verify(resp['answer'], challenge.encode())
yield Effect(EHandshakeSend('{"status": "ok", "handshake": "done"}'))
self.id = claimed_identity
except (TypeError, PubKeyNotFound, InvalidSignature):
error = HandshakeError('Invalid signature, challenge or identity')
yield Effect(EHandshakeSend(error.to_raw()))
raise error
def parse_cmd(raw_cmd: bytes):
try:
return ejson_loads(raw_cmd.decode('utf-8'))
except (json.decoder.JSONDecodeError, UnicodeDecodeError):
pass
def _find_matching_blocks(self, size=None, offset=0):
if size is None:
size = sys.maxsize
pre_excluded_blocks = []
post_excluded_blocks = []
version = self.get_version()
vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version))
blob = vlob['blob']
encrypted_blob = from_jsonb64(blob)
blob = self.encryptor.decrypt(encrypted_blob)
blob = ejson_loads(blob.decode())
pre_excluded_blocks = []
included_blocks = []
post_excluded_blocks = []
cursor = 0
pre_excluded_data = b''
pre_included_data = b''
post_included_data = b''
post_excluded_data = b''
for blocks_and_key in blob:
block_key = blocks_and_key['key']
decoded_block_key = from_jsonb64(block_key)
encryptor = load_sym_key(decoded_block_key)
for block_properties in blocks_and_key['blocks']:
cursor += block_properties['size']
if cursor <= offset:
if len(pre_excluded_blocks) and pre_excluded_blocks[-1]['key'] == block_key:
pre_excluded_blocks[-1]['blocks'].append(block_properties)
else:
pre_excluded_blocks.append({'blocks': [block_properties], 'key': block_key})
elif cursor > offset and cursor - block_properties['size'] < offset:
delta = cursor - offset
block = yield Effect(EBlockRead(block_properties['block']))
content = from_jsonb64(block['content'])
block_data = encryptor.decrypt(content)
pre_excluded_data = block_data[:-delta]
pre_included_data = block_data[-delta:][:size]
if size < len(block_data[-delta:]):
post_excluded_data = block_data[-delta:][size:]
elif cursor > offset and cursor <= offset + size:
if len(included_blocks) and included_blocks[-1]['key'] == block_key:
included_blocks[-1]['blocks'].append(block_properties)
else:
included_blocks.append({'blocks': [block_properties], 'key': block_key})
elif cursor > offset + size and cursor - block_properties['size'] < offset + size:
delta = offset + size - (cursor - block_properties['size'])
block = yield Effect(EBlockRead(block_properties['block']))
content = from_jsonb64(block['content'])
block_data = encryptor.decrypt(content)
post_included_data = block_data[:delta]
post_excluded_data = block_data[delta:]
else:
if len(post_excluded_blocks) and post_excluded_blocks[-1]['key'] == block_key:
post_excluded_blocks[-1]['blocks'].append(block_properties)
else:
post_excluded_blocks.append({'blocks': [block_properties],
'key': block_key})
return {
'pre_excluded_blocks': pre_excluded_blocks,
'pre_excluded_data': pre_excluded_data,
'pre_included_data': pre_included_data,
'included_blocks': included_blocks,
'post_included_data': post_included_data,
'post_excluded_data': post_excluded_data,
'post_excluded_blocks': post_excluded_blocks
}
