def test_perform_file_create(app, alice_identity, file): vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'} block_id = '4567' # Already exist blob = [{ 'blocks': [{ 'block': block_id, 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_file_create(EFileCreate('/foo')) sequence = [ (EBlockCreate(''), const(block_id)), (EVlobCreate(blob), const(vlob)), (EIdentityGet(), const(alice_identity)), (EVlobRead(vlob['id'], vlob['read_trust_seed'], 1), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EBlockDelete(block_id), noop), (EVlobDelete(vlob['id']), noop), ] with pytest.raises(ManifestError): perform_sequence(sequence, eff)
def test_perform_file_history(app, file, alice_identity): vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'} blob = [{ 'blocks': [{ 'block': '4567', 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_file_history(EFileHistory('/foo', 1, 1)) sequence = [ (EIdentityGet(), const(alice_identity)), (EIdentityGet(), const(alice_identity)), (EVlobRead(vlob['id'], vlob['read_trust_seed']), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EVlobList(), const([])), ] perform_sequence(sequence, eff)
def test_perform_undelete(app, alice_identity, file): vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'} blob = [{ 'blocks': [{ 'block': '4567', 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_delete(EDelete('/foo')) sequence = [ (EIdentityGet(), const(alice_identity)), (EVlobRead(vlob['id'], vlob['read_trust_seed']), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EVlobList(), const([])), (EVlobRead(vlob['id'], vlob['read_trust_seed'], 1), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EVlobDelete('2345'), conste(VlobNotFound('Vlob not found.'))) ] ret = perform_sequence(sequence, eff) eff = app.perform_undelete(EUndelete('2345')) sequence = [(EIdentityGet(), const(alice_identity))] ret = perform_sequence(sequence, eff) assert ret is None
def test_load_file(self, file): vlob_id = '1234' other_vlob_id = '5678' read_trust_seed = '42' version = 1 # Load from open files file2 = perform_sequence( [], File.load(vlob_id, to_jsonb64(b'<dummy-key-00000000000000000001>'), read_trust_seed, '43')) assert file == file2 File.files = {} # Test reloading commited and not commited file for synchronizer_vlob_list in [[vlob_id, other_vlob_id], [other_vlob_id]]: key = to_jsonb64(b'<dummy-key-00000000000000000001>') sequence = [ (EVlobRead(vlob_id, read_trust_seed, None), const({ 'id': vlob_id, 'blob': 'foo', 'version': version })), (EVlobList(), const(synchronizer_vlob_list)), ] file = perform_sequence( sequence, File.load(vlob_id, key, read_trust_seed, '43')) assert file.dirty is (vlob_id in synchronizer_vlob_list) assert file.version == (version - 1 if file.dirty else version) File.files = {}
def test_discard(self, file): content = b'This is a test content.' block_ids = ['4567', '5678', '6789'] # Original content chunk_1 = content[:5] chunk_2 = content[5:14] chunk_3 = content[14:] blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }, { 'block': block_ids[1], 'digest': digest(chunk_2), 'size': len(chunk_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': digest(chunk_3), 'size': len(chunk_3) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000004>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) # Already synchronized sequence = [ (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EBlockDelete('5678'), noop), (EBlockDelete('6789'), noop), (EVlobDelete('1234'), conste(VlobNotFound('Block not found.')) ) # TODO vlob OR block exceptin ] ret = perform_sequence(sequence, file.discard()) assert ret is False # Not already synchronized file.dirty = True file.version = 0 sequence = [(EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), noop), (EBlockDelete('5678'), noop), (EBlockDelete('6789'), noop), (EVlobDelete('1234'), noop)] ret = perform_sequence(sequence, file.discard()) assert ret is True assert file.dirty is False
def test_reencrypt(self, file): old_vlob = file.get_vlob() content = b'This is a test content.' block_ids = ['4567', '5678', '6789'] # Original content chunk_1 = content[:5] chunk_2 = content[5:14] chunk_3 = content[14:] blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }, { 'block': block_ids[1], 'digest': digest(chunk_2), 'size': len(chunk_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': digest(chunk_3), 'size': len(chunk_3) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000002>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) sequence = [ (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), ( EVlobCreate(blob), # TODO check re-encryption const({ 'id': '2345', 'read_trust_seed': '21', 'write_trust_seed': '22' })) ] ret = perform_sequence(sequence, file.reencrypt()) assert ret is None file.reencrypt() new_vlob = file.get_vlob() for property in old_vlob.keys(): assert old_vlob[property] != new_vlob[property]
def test_perform_message_get(): eff = perform_message_get(EBackendMessageGet('John', 1)) backend_response = { 'status': 'ok', 'messages': [ {'count': 1, 'body': to_jsonb64(b'body 1')}, {'count': 2, 'body': to_jsonb64(b'body 2')} ] } sequence = [ (BackendCmd('message_get', {'offset': 1, 'recipient': 'John'}), const(backend_response)) ] ret = perform_sequence(sequence, eff) assert ret == [Message(1, b'body 1'), Message(2, b'body 2')]
async def open_connection(self, identity): logger.debug('Connection to backend opened') assert not self._websocket, "Connection to backend already opened" try: self._websocket = await websockets.connect(self.url) # Handle handshake raw = await self._websocket.recv() challenge = ejson_loads(raw) answer = identity.private_key.sign(challenge['challenge'].encode()) await self._websocket.send( ejson_dumps({ 'handshake': 'answer', 'identity': identity.id, 'answer': to_jsonb64(answer) })) resp = ejson_loads(await self._websocket.recv()) if resp['status'] != 'ok': await self.close_connection() raise exception_from_status(resp['status'])(resp['label']) self._ws_recv_handler_task = asyncio.ensure_future( self._ws_recv_handler(), loop=self.loop) if self.watchdog_time: self._watchdog_task = asyncio.ensure_future(self._watchdog(), loop=self.loop) except (ConnectionRefusedError, websockets.exceptions.ConnectionClosed) as exc: raise BackendConnectionError('Cannot connect to backend (%s)' % exc)
def test_vlob_create_ok(self, id_and_blob): id, blob = id_and_blob intent = EVlobCreate(id, blob) intent_ret = VlobAtom(id, 'readtrustseed-123', 'writetrustseed-123', blob, 1) payload = {} if id: payload['id'] = id else: intent_ret.id = '123' if blob: payload['blob'] = to_jsonb64(blob) else: intent.blob = b'' intent_ret.blob = b'' eff = execute_cmd('vlob_create', payload) sequence = [ (intent, const(intent_ret)), ] ret = perform_sequence(sequence, eff) assert ret == { 'status': 'ok', 'id': intent_ret.id, 'read_trust_seed': 'readtrustseed-123', 'write_trust_seed': 'writetrustseed-123' }
def test_build_file_blocks(self, file, length): file.dirty = False block_size = 4096 content = b''.join( [str(random.randint(1, 9)).encode() for i in range(0, length)]) chunks = [ content[i:i + block_size] for i in range(0, len(content), block_size) ] if not chunks: chunks = [b''] sequence = [] for chunk in chunks: sequence.append((EBlockCreate(to_jsonb64(chunk)), const('4567'))) blocks = perform_sequence(sequence, file._build_file_blocks(content)) assert sorted(blocks.keys()) == ['blocks', 'key'] assert isinstance(blocks['blocks'], list) required_blocks = int(len(content) / block_size) if not len(content) or len(content) % block_size: required_blocks += 1 assert len(blocks['blocks']) == required_blocks for index, block in enumerate(blocks['blocks']): assert sorted(block.keys()) == ['block', 'digest', 'size'] assert block['block'] length = len(content) - index * block_size length = block_size if length > block_size else length assert block['size'] == length assert block['digest'] == digest(content[index * block_size:index + 1 * block_size]) assert file.dirty is True
def get_vlob(self): vlob = {} vlob['id'] = self.id vlob['read_trust_seed'] = self.read_trust_seed vlob['write_trust_seed'] = self.write_trust_seed vlob['key'] = to_jsonb64(self.encryptor.key) return vlob
def app(mock_crypto_passthrough, alice_identity): # app = FSComponent() # identity_component = IdentityComponent() fs_component = FSComponent() # synchronizer_component = SynchronizerComponent() # identity_component = IdentityComponent() # app = app_factory( # fs_component.get_dispatcher(), # synchronizer_component.get_dispatcher(), # identity_component.get_dispatcher() # ) blob = { 'dustbin': [], 'entries': { '/': None }, 'groups': {}, 'versions': {} } blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) sequence = [(EIdentityGet(), const(alice_identity)), (EUserVlobRead(), const({ 'blob': '', 'version': 0 })), (EUserVlobUpdate(1, blob), noop)] perform_sequence(sequence, fs_component._get_manifest()) return fs_component
def test_get_vlob(self, file): assert file.get_vlob() == { 'id': '1234', 'key': to_jsonb64(b'<dummy-key-00000000000000000002>'), 'read_trust_seed': '42', 'write_trust_seed': '43' }
def commit(self, recursive=True): is_dirty = yield self.is_dirty() if self.version != 0 and not is_dirty: return # Update manifest with new group vlobs vlob_list = yield Effect(EVlobList()) if recursive: for group_manifest in self.group_manifests.values(): new_vlob = yield group_manifest.commit() if new_vlob and new_vlob is not True: old_vlob = group_manifest.get_vlob() new_vlob['key'] = old_vlob['key'] group_manifest.update_vlob(new_vlob) # Update manifest entries with new file vlobs (dustbin entries are already commited) for entry in self.entries.values(): if entry and entry['id'] in vlob_list: file = yield File.load(entry['id'], entry['key'], entry['read_trust_seed'], entry['write_trust_seed']) new_vlob = yield file.commit() if new_vlob and new_vlob is not True: entry['id'] = new_vlob['id'] entry['read_trust_seed'] = new_vlob['read_trust_seed'] entry['write_trust_seed'] = new_vlob['write_trust_seed'] # Commit manifest blob = yield self.dumps() encrypted_blob = self.encryptor.pub_key.encrypt(blob.encode()) encrypted_blob = to_jsonb64(encrypted_blob) yield Effect(EUserVlobUpdate(self.version + 1, encrypted_blob)) self.original_manifest = ejson_loads(blob) synchronized = yield Effect(EUserVlobSynchronize()) if synchronized: self.version += 1
def flush(self): if not self.modifications: return # Merge all modifications to build final content builder = ContentBuilder() shortest_truncate = None for modification in self.modifications: if modification[0] == self.write: builder.write(*modification[1:]) elif modification[0] == self.truncate: builder.truncate(modification[1]) if not shortest_truncate or shortest_truncate > modification[1]: shortest_truncate = modification[1] else: raise NotImplementedError() self.modifications = [] # Truncate file if shortest_truncate is not None: response = self._operations.send_cmd( cmd='file_truncate', path=self.path, length=shortest_truncate) if response['status'] != 'ok': raise FuseOSError(ENOENT) # Write new contents for offset, content in builder.contents.items(): # TODO use flags response = self._operations.send_cmd( cmd='file_write', path=self.path, content=to_jsonb64(content), offset=offset) if response['status'] != 'ok': raise FuseOSError(ENOENT)
def commit(self): is_dirty = yield self.is_dirty() if self.version != 0 and not is_dirty: return # Update manifest entries with new file vlobs (dustbin entries are already commited) vlob_list = yield Effect(EVlobList()) for entry in self.entries.values(): if entry and entry['id'] in vlob_list: file = yield File.load(entry['id'], entry['key'], entry['read_trust_seed'], entry['write_trust_seed']) new_vlob = yield file.commit() if new_vlob and new_vlob is not True: entry['id'] = new_vlob['id'] entry['read_trust_seed'] = new_vlob['read_trust_seed'] entry['write_trust_seed'] = new_vlob['write_trust_seed'] # Commit manifest blob = yield self.dumps() encrypted_blob = self.encryptor.encrypt(blob.encode()) encrypted_blob = to_jsonb64(encrypted_blob) yield Effect(EVlobUpdate(self.id, self.write_trust_seed, self.version + 1, encrypted_blob)) self.original_manifest = ejson_loads(blob) new_vlob = yield Effect(EVlobSynchronize(self.id)) if new_vlob: if new_vlob is not True: self.id = new_vlob['id'] self.read_trust_seed = new_vlob['read_trust_seed'] self.write_trust_seed = new_vlob['write_trust_seed'] new_vlob = self.get_vlob() self.version += 1 return new_vlob
def reencrypt(self): # Reencrypt files for path, entry in self.entries.items(): if entry: file = yield File.load(**entry) yield file.reencrypt() new_vlob = file.get_vlob() self.entries[path] = new_vlob for index, entry in enumerate(self.dustbin): entry = deepcopy(entry) path = entry['path'] removed_date = entry['removed_date'] for key in ['path', 'removed_date']: del entry[key] file = yield File.load(**entry) yield file.reencrypt() new_vlob = file.get_vlob() new_vlob['path'] = path new_vlob['removed_date'] = removed_date self.dustbin[index] = new_vlob # Reencrypt manifest blob = yield self.dumps() self.encryptor = generate_sym_key() encrypted_blob = self.encryptor.encrypt(blob.encode()) encrypted_blob = to_jsonb64(encrypted_blob) new_vlob = yield Effect(EVlobCreate(encrypted_blob)) self.id = new_vlob['id'] self.read_trust_seed = new_vlob['read_trust_seed'] self.write_trust_seed = new_vlob['write_trust_seed'] self.version = 0
async def test_read_offset(johndoe_client): await mk_file(johndoe_client, '/foo.txt', data=b'1234567890') ret = await johndoe_client.send_cmd('file_read', path='/foo.txt', offset=3, size=4) assert ret == {'status': 'ok', 'content': to_jsonb64(b'4567')}
async def test_file_truncate(johndoe_client): await mk_file(johndoe_client, '/foo.txt', data=b'1234567890') ret = await johndoe_client.send_cmd('file_truncate', path='/foo.txt', length=4) assert ret == {'status': 'ok'} ret = await johndoe_client.send_cmd('file_read', path='/foo.txt') assert ret == {'status': 'ok', 'content': to_jsonb64(b'1234')}
def perform_vlob_create(intent): msg = {'blob': to_jsonb64(intent.blob)} ret = yield Effect(BackendCmd('vlob_create', msg)) status = ret['status'] if status != 'ok': raise exception_from_status(status)(ret['label']) return VlobAccess(ret['id'], ret['read_trust_seed'], ret['write_trust_seed'])
def api_user_vlob_read(msg): msg = cmd_READ_Schema().load(msg) atom = yield Effect(EUserVlobRead(**msg)) return { 'status': 'ok', 'blob': to_jsonb64(atom.blob), 'version': atom.version }
async def test_big_file(johndoe_client): data = b'x' * 10000 # 10ko await mk_file(johndoe_client, '/foo.txt', data=data) ret = await johndoe_client.send_cmd('file_read', path='/foo.txt', offset=42, size=1000) assert ret == {'status': 'ok', 'content': to_jsonb64(data[42:1042])}
async def test_write_offset(johndoe_client): await mk_file(johndoe_client, '/foo.txt', data=b'1234567890') ret = await johndoe_client.send_cmd('file_write', path='/foo.txt', content=b'abcd', offset=3) assert ret == {'status': 'ok'} ret = await johndoe_client.send_cmd('file_read', path='/foo.txt') assert ret == {'status': 'ok', 'content': to_jsonb64(b'123abcd890')}
async def test_write_file(johndoe_client): ret = await johndoe_client.send_cmd('file_create', path='/foo.txt') assert ret == {'status': 'ok'} ret = await johndoe_client.send_cmd('file_write', path='/foo.txt', content=b'fooo') assert ret == {'status': 'ok'} ret = await johndoe_client.send_cmd('file_read', path='/foo.txt') assert ret == {'status': 'ok', 'content': to_jsonb64(b'fooo')}
def test_create_file(self, file): assert file.dirty is True assert file.version == 0 assert file.get_vlob() == { 'id': '1234', 'key': to_jsonb64(b'<dummy-key-00000000000000000002>'), 'read_trust_seed': '42', 'write_trust_seed': '43' }
def test_user_vlob_update_bad_version(self): eff = execute_cmd('user_vlob_update', { 'version': 42, 'blob': to_jsonb64(b'Next version.') }) sequence = [(EUserVlobUpdate(version=42, blob=b'Next version.'), conste(UserVlobError('Wrong blob version.')))] ret = perform_sequence(sequence, eff) assert ret['status'] == 'user_vlob_error'
def test_message_get_ok(self): eff = execute_cmd('message_get', {}) sequence = [(EMessageGet(offset=0), const([b'zero', b'one', b'two']))] ret = perform_sequence(sequence, eff) assert ret == { 'status': 'ok', 'messages': [{ 'count': 1, 'body': to_jsonb64(b'zero') }, { 'count': 2, 'body': to_jsonb64(b'one') }, { 'count': 3, 'body': to_jsonb64(b'two') }] }
def test_perform_vlob_update(): eff = perform_user_vlob_update(EBackendUserVlobUpdate(3, b'bar')) backend_response = {'status': 'ok'} sequence = [(BackendCmd('user_vlob_update', { 'version': 3, 'blob': to_jsonb64(b'bar') }), const(backend_response))] ret = perform_sequence(sequence, eff) assert ret is None
def test_perform_message_new(): eff = perform_message_new(EBackendMessageNew('John', b'my body')) backend_response = {'status': 'ok'} sequence = [ (BackendCmd('message_new', {'recipient': 'John', 'body': to_jsonb64(b'my body')}), const(backend_response)) ] ret = perform_sequence(sequence, eff) assert ret is None
def test_perform_dustbin_show(app, alice_identity, file): with freeze_time('2012-01-01') as frozen_datetime: vlob = { 'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43' } blob = [{ 'blocks': [{ 'block': '4567', 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_delete(EDelete('/foo')) sequence = [ (EIdentityGet(), const(alice_identity)), (EVlobRead(vlob['id'], vlob['read_trust_seed']), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EVlobList(), const([])), (EVlobRead(vlob['id'], vlob['read_trust_seed'], 1), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EVlobDelete('2345'), conste(VlobNotFound('Vlob not found.'))), ] perform_sequence(sequence, eff) eff = app.perform_dustbin_show(EDustbinShow()) sequence = [(EIdentityGet(), const(alice_identity))] dustbin = perform_sequence(sequence, eff) vlob['path'] = '/foo' vlob['removed_date'] = frozen_datetime().isoformat() vlob['key'] = to_jsonb64(b'<dummy-key-00000000000000000002>') assert dustbin == [vlob]