def dumps(self, original_manifest=False): if original_manifest: return ejson_dumps(self.original_manifest) else: versions = yield self.get_vlobs_versions() return ejson_dumps({'entries': self.entries, 'dustbin': self.dustbin, 'versions': versions})
def dumps(self, original_manifest=False): if original_manifest: manifest = deepcopy(self.original_manifest) manifest['groups'] = self.get_group_vlobs() return ejson_dumps(manifest) else: versions = yield self.get_vlobs_versions() return ejson_dumps({'entries': self.entries, 'dustbin': self.dustbin, 'groups': self.get_group_vlobs(), 'versions': versions})
def test_perform_undelete(app, alice_identity, file): vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'} blob = [{ 'blocks': [{ 'block': '4567', 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_delete(EDelete('/foo')) sequence = [ (EIdentityGet(), const(alice_identity)), (EVlobRead(vlob['id'], vlob['read_trust_seed']), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EVlobList(), const([])), (EVlobRead(vlob['id'], vlob['read_trust_seed'], 1), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EVlobDelete('2345'), conste(VlobNotFound('Vlob not found.'))) ] ret = perform_sequence(sequence, eff) eff = app.perform_undelete(EUndelete('2345')) sequence = [(EIdentityGet(), const(alice_identity))] ret = perform_sequence(sequence, eff) assert ret is None
def test_perform_file_create(app, alice_identity, file): vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'} block_id = '4567' # Already exist blob = [{ 'blocks': [{ 'block': block_id, 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_file_create(EFileCreate('/foo')) sequence = [ (EBlockCreate(''), const(block_id)), (EVlobCreate(blob), const(vlob)), (EIdentityGet(), const(alice_identity)), (EVlobRead(vlob['id'], vlob['read_trust_seed'], 1), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EBlockDelete(block_id), noop), (EVlobDelete(vlob['id']), noop), ] with pytest.raises(ManifestError): perform_sequence(sequence, eff)
async def send_cmd(self, cmd, **kwargs): msg = {'cmd': cmd, **kwargs} raw_msg = ejson_dumps(msg).encode() self.writer.write(raw_msg) self.writer.write(b'\n') raw_resp = await self.reader.readline() return ejson_loads(raw_resp.decode())
async def open_connection(self, identity): logger.debug('Connection to backend opened') assert not self._websocket, "Connection to backend already opened" try: self._websocket = await websockets.connect(self.url) # Handle handshake raw = await self._websocket.recv() challenge = ejson_loads(raw) answer = identity.private_key.sign(challenge['challenge'].encode()) await self._websocket.send( ejson_dumps({ 'handshake': 'answer', 'identity': identity.id, 'answer': to_jsonb64(answer) })) resp = ejson_loads(await self._websocket.recv()) if resp['status'] != 'ok': await self.close_connection() raise exception_from_status(resp['status'])(resp['label']) self._ws_recv_handler_task = asyncio.ensure_future( self._ws_recv_handler(), loop=self.loop) if self.watchdog_time: self._watchdog_task = asyncio.ensure_future(self._watchdog(), loop=self.loop) except (ConnectionRefusedError, websockets.exceptions.ConnectionClosed) as exc: raise BackendConnectionError('Cannot connect to backend (%s)' % exc)
def app(mock_crypto_passthrough, alice_identity): # app = FSComponent() # identity_component = IdentityComponent() fs_component = FSComponent() # synchronizer_component = SynchronizerComponent() # identity_component = IdentityComponent() # app = app_factory( # fs_component.get_dispatcher(), # synchronizer_component.get_dispatcher(), # identity_component.get_dispatcher() # ) blob = { 'dustbin': [], 'entries': { '/': None }, 'groups': {}, 'versions': {} } blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) sequence = [(EIdentityGet(), const(alice_identity)), (EUserVlobRead(), const({ 'blob': '', 'version': 0 })), (EUserVlobUpdate(1, blob), noop)] perform_sequence(sequence, fs_component._get_manifest()) return fs_component
def test_perform_file_history(app, file, alice_identity): vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'} blob = [{ 'blocks': [{ 'block': '4567', 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_file_history(EFileHistory('/foo', 1, 1)) sequence = [ (EIdentityGet(), const(alice_identity)), (EIdentityGet(), const(alice_identity)), (EVlobRead(vlob['id'], vlob['read_trust_seed']), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EVlobList(), const([])), ] perform_sequence(sequence, eff)
def test_discard(self, file): content = b'This is a test content.' block_ids = ['4567', '5678', '6789'] # Original content chunk_1 = content[:5] chunk_2 = content[5:14] chunk_3 = content[14:] blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }, { 'block': block_ids[1], 'digest': digest(chunk_2), 'size': len(chunk_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': digest(chunk_3), 'size': len(chunk_3) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000004>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) # Already synchronized sequence = [ (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EBlockDelete('5678'), noop), (EBlockDelete('6789'), noop), (EVlobDelete('1234'), conste(VlobNotFound('Block not found.')) ) # TODO vlob OR block exceptin ] ret = perform_sequence(sequence, file.discard()) assert ret is False # Not already synchronized file.dirty = True file.version = 0 sequence = [(EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), noop), (EBlockDelete('5678'), noop), (EBlockDelete('6789'), noop), (EVlobDelete('1234'), noop)] ret = perform_sequence(sequence, file.discard()) assert ret is True assert file.dirty is False
def send_cmd(self, **msg): with self._socket_lock: req = ejson_dumps(msg).encode() + b'\n' logger.debug('Send: %r' % req) self.sock.send(req) raw_reps = self.sock.recv(4096) while raw_reps[-1] != ord(b'\n'): raw_reps += self.sock.recv(4096) logger.debug('Received: %r' % raw_reps) return ejson_loads(raw_reps[:-1].decode())
async def perform_group_add_identities(self, intent): async with self.connection.acquire() as conn: async with conn.cursor() as cur: await cur.execute('SELECT body FROM groups WHERE id=%s', (intent.name, )) ret = await cur.fetchone() if ret is None: raise GroupNotFound('Group not found.') group = ejson_loads(ret[0]) group_entry = 'admins' if intent.admin else 'users' group[group_entry] = list(set(group[group_entry]) | set(intent.identities)) await cur.execute('UPDATE groups SET body=%s WHERE id=%s', (ejson_dumps(group), intent.name))
def test_reencrypt(self, file): old_vlob = file.get_vlob() content = b'This is a test content.' block_ids = ['4567', '5678', '6789'] # Original content chunk_1 = content[:5] chunk_2 = content[5:14] chunk_3 = content[14:] blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }, { 'block': block_ids[1], 'digest': digest(chunk_2), 'size': len(chunk_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': digest(chunk_3), 'size': len(chunk_3) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000002>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) sequence = [ (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), ( EVlobCreate(blob), # TODO check re-encryption const({ 'id': '2345', 'read_trust_seed': '21', 'write_trust_seed': '22' })) ] ret = perform_sequence(sequence, file.reencrypt()) assert ret is None file.reencrypt() new_vlob = file.get_vlob() for property in old_vlob.keys(): assert old_vlob[property] != new_vlob[property]
def test_perform_group_create(app, alice_identity): blob = {'dustbin': [], 'entries': {'/': None}, 'versions': {}} blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_group_create(EGroupCreate('share')) sequence = [(EIdentityGet(), const(alice_identity)), (EVlobCreate(), const({ 'id': '1234', 'read_trust_seed': '42', 'write_trust_seed': '43' })), (EVlobUpdate('1234', '43', 1, blob), noop)] ret = perform_sequence(sequence, eff) assert ret is None
def execute_raw_cmd(raw_cmd): params = parse_cmd(raw_cmd) if not params: ret = {'status': 'bad_msg', 'label': 'Message is not a valid JSON.'} else: cmd_type = params.pop('cmd', None) if not isinstance(cmd_type, str): ret = { 'status': 'bad_msg', 'label': '`cmd` string field is mandatory.' } else: ret = yield execute_cmd(cmd_type, params) return ejson_dumps(ret).encode('utf-8')
async def send_cmd(self, msg): if not self._websocket: raise BackendConnectionError( 'BackendAPIService cannot send command in current state') try: await self._websocket.send(ejson_dumps(msg)) except websockets.exceptions.ConnectionClosed as exc: raise BackendConnectionError('Cannot connect to backend (%s)' % exc) ret = await self._resp_queue.get() status = ret['status'] if status == 'ok': return ret else: raise exception_from_status(status)(ret['label'])
def execute_raw_cmd(raw_cmd: str): try: params = ejson_loads(raw_cmd) except json.decoder.JSONDecodeError: ret = {'status': 'bad_msg', 'label': 'Message is not a valid JSON.'} else: cmd_type = params.pop('cmd', None) if not isinstance(cmd_type, str): ret = { 'status': 'bad_msg', 'label': '`cmd` string field is mandatory.' } else: ret = yield execute_cmd(cmd_type, params) return ejson_dumps(ret)
def create(cls): self = File() blob = yield self._build_file_blocks(b'') blob = [blob] blob = ejson_dumps(blob).encode() self.encryptor = generate_sym_key() encrypted_blob = self.encryptor.encrypt(blob) encrypted_blob = to_jsonb64(encrypted_blob) vlob = yield Effect(EVlobCreate(encrypted_blob)) self.id = vlob['id'] self.read_trust_seed = vlob['read_trust_seed'] self.write_trust_seed = vlob['write_trust_seed'] self.dirty = True self.version = 0 self.modifications = [] File.files[self.id] = self return self
def test_perform_synchronize(app, alice_identity): blob = { 'dustbin': [], 'entries': { '/': None }, 'groups': {}, 'versions': {} } blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_synchronize(ESynchronize()) sequence = [(EIdentityGet(), const(alice_identity)), (EVlobList(), const([])), (EUserVlobUpdate(1, blob), noop), (EUserVlobSynchronize(), noop)] ret = perform_sequence(sequence, eff) assert ret is None
async def run(): try: reader, writer = await asyncio.open_unix_connection(path=socket) except (FileNotFoundError, ConnectionRefusedError): raise SystemExit('ERROR: Cannot connect to parsec core at %s' % socket) msg = { 'cmd': 'identity_signup', 'id': identity, 'password': password, 'key_size': key_size } writer.write(ejson_dumps(msg).encode()) writer.write(b'\n') raw_resp = await reader.readline() resp = ejson_loads(raw_resp.decode()) writer.close() print(resp)
def test_perform_dustbin_show(app, alice_identity, file): with freeze_time('2012-01-01') as frozen_datetime: vlob = { 'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43' } blob = [{ 'blocks': [{ 'block': '4567', 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_delete(EDelete('/foo')) sequence = [ (EIdentityGet(), const(alice_identity)), (EVlobRead(vlob['id'], vlob['read_trust_seed']), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EVlobList(), const([])), (EVlobRead(vlob['id'], vlob['read_trust_seed'], 1), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EVlobDelete('2345'), conste(VlobNotFound('Vlob not found.'))), ] perform_sequence(sequence, eff) eff = app.perform_dustbin_show(EDustbinShow()) sequence = [(EIdentityGet(), const(alice_identity))] dustbin = perform_sequence(sequence, eff) vlob['path'] = '/foo' vlob['removed_date'] = frozen_datetime().isoformat() vlob['key'] = to_jsonb64(b'<dummy-key-00000000000000000002>') assert dustbin == [vlob]
def file(app, alice_identity, mock_crypto_passthrough): vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'} block_id = '4567' blob = [{ 'blocks': [{ 'block': block_id, 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_file_create(EFileCreate('/foo')) sequence = [(EBlockCreate(''), const(block_id)), (EVlobCreate(blob), const(vlob)), (EIdentityGet(), const(alice_identity))] ret = perform_sequence(sequence, eff) assert ret is None File.files = {}
def test_get_blocks(self, file): file.dirty = False file.version = 1 vlob_id = '1234' block_ids = ['4567', '5678', '6789'] chunk_digest = digest(b'') blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': chunk_digest, 'size': 0 }, { 'block': block_ids[1], 'digest': chunk_digest, 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': chunk_digest, 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000002>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) sequence = [ (EVlobRead(vlob_id, '42', 1), const({ 'id': vlob_id, 'blob': blob, 'version': 1 })), ] ret = perform_sequence(sequence, file.get_blocks()) assert ret == block_ids assert file.dirty is False assert file.version == 1
def file(mock_crypto_passthrough): block_id = '4567' blob = [{ 'blocks': [{ 'block': block_id, 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) sequence = [ (EBlockCreate(''), const(block_id)), (EVlobCreate(blob), const({ 'id': '1234', 'read_trust_seed': '42', 'write_trust_seed': '43' })), ] return perform_sequence(sequence, File.create())
def handshake(self): if self.id: raise HandshakeError('Handshake already done.') challenge = _generate_challenge() query = {'handshake': 'challenge', 'challenge': challenge} yield Effect(EHandshakeSend(ejson_dumps(query))) raw_resp = yield Effect(EHandshakeRecv()) try: resp = ejson_loads(raw_resp) except (TypeError, json.JSONDecodeError): error = HandshakeError('Invalid challenge response format') yield Effect(EHandshakeSend(error.to_raw())) raise error resp = HandshakeAnswerSchema().load(resp) claimed_identity = resp['identity'] try: pubkey = yield Effect(EPubKeyGet(claimed_identity)) pubkey.verify(resp['answer'], challenge.encode()) yield Effect(EHandshakeSend('{"status": "ok", "handshake": "done"}')) self.id = claimed_identity except (TypeError, PubKeyNotFound, InvalidSignature): error = HandshakeError('Invalid signature, challenge or identity') yield Effect(EHandshakeSend(error.to_raw())) raise error
def on_event(sender): payload = ejson_dumps({'event': intent.event, 'sender': sender}) context.queued_pushed_events.put_nowait(payload)
def test_commit(self, file): vlob_id = '1234' content = b'This is a test content.' block_ids = ['4567', '5678', '6789'] new_vlob = { 'id': '2345', 'read_trust_seed': 'ABC', 'write_trust_seed': 'DEF' } # Original content chunk_1 = content[:5] chunk_2 = content[5:14] chunk_3 = content[14:] blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }, { 'block': block_ids[1], 'digest': digest(chunk_2), 'size': len(chunk_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': digest(chunk_3), 'size': len(chunk_3) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000004>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) # New content after truncate new_chuck_2 = b'is a' new_block_id = '7654' new_blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }, { 'blocks': [{ 'block': new_block_id, 'digest': digest(new_chuck_2), 'size': len(new_chuck_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }] new_blob = ejson_dumps(new_blob).encode() new_blob = to_jsonb64(new_blob) file.truncate(9) sequence = [ (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), (EBlockRead(block_ids[1]), const({ 'content': to_jsonb64(chunk_2), 'creation_date': '2012-01-01T00:00:00' })), (EBlockCreate(to_jsonb64(new_chuck_2)), const(new_block_id)), (EVlobUpdate(vlob_id, '43', 1, new_blob), noop), (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': new_blob, 'version': 1 })), (EBlockDelete('5678'), conste(BlockNotFound('Block not found.'))), (EBlockDelete('6789'), noop), (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': new_blob, 'version': 1 })), (EBlockSynchronize('4567'), const(True)), (EBlockSynchronize('7654'), const(False)), (EVlobSynchronize('1234'), const(new_vlob)) ] ret = perform_sequence(sequence, file.commit()) new_vlob['key'] = to_jsonb64(b'<dummy-key-00000000000000000002>') assert ret == new_vlob assert file.dirty is False assert file.version == 1
def test_find_matching_blocks(self, file): vlob_id = '1234' block_size = 4096 # Contents contents = {} total_length = 0 for index, length in enumerate([ block_size + 1, block_size - 1, block_size, 2 * block_size + 2, 2 * block_size - 2, 2 * block_size ]): content = b''.join( [str(random.randint(1, 9)).encode() for i in range(0, length)]) contents[index] = content total_length += length # Blocks def generator(): i = 2000 while True: yield str(i) i += 1 gen = generator() blocks = {} block_contents = {} block_id = 2000 for index, content in contents.items(): chunks = [ content[i:i + block_size] for i in range(0, len(content), block_size) ] if not chunks: chunks = [b''] sequence = [] for chunk in chunks: encoded_chunk = to_jsonb64(chunk) sequence.append((EBlockCreate(encoded_chunk), lambda id=id: next(gen))) # TODO dirty block_contents[str(block_id)] = encoded_chunk block_id += 1 blocks[index] = perform_sequence(sequence, file._build_file_blocks(content)) # Create file blob = ejson_dumps([blocks[i] for i in range(0, len(blocks))]).encode() blob = to_jsonb64(blob) # All matching blocks sequence = [(EVlobRead(vlob_id, '42', 1), const({ 'id': vlob_id, 'blob': blob, 'version': 1 }))] matching_blocks = perform_sequence(sequence, file._find_matching_blocks()) assert matching_blocks == { 'pre_excluded_blocks': [], 'pre_excluded_data': b'', 'pre_included_data': b'', 'included_blocks': [blocks[i] for i in range(0, len(blocks))], 'post_included_data': b'', 'post_excluded_data': b'', 'post_excluded_blocks': [] } # With offset delta = 10 offset = (blocks[0]['blocks'][0]['size'] + blocks[0]['blocks'][1]['size'] + blocks[1]['blocks'][0]['size'] + blocks[2]['blocks'][0]['size'] - delta) sequence = [(EVlobRead(vlob_id, '42', 1), const({ 'id': vlob_id, 'blob': blob, 'version': 1 })), (EBlockRead('2003'), const({ 'content': block_contents['2003'], 'creation_date': '2012-01-01T00:00:00' }))] matching_blocks = perform_sequence( sequence, file._find_matching_blocks(None, offset)) pre_excluded_data = contents[2][:blocks[2]['blocks'][0]['size'] - delta] pre_included_data = contents[2][-delta:] assert matching_blocks == { 'pre_excluded_blocks': [blocks[0], blocks[1]], 'pre_excluded_data': pre_excluded_data, 'pre_included_data': pre_included_data, 'included_blocks': [blocks[i] for i in range(3, 6)], 'post_included_data': b'', 'post_excluded_data': b'', 'post_excluded_blocks': [] } # With small size delta = 10 size = 5 offset = (blocks[0]['blocks'][0]['size'] + blocks[0]['blocks'][1]['size'] + blocks[1]['blocks'][0]['size'] + blocks[2]['blocks'][0]['size'] - delta) sequence = [(EVlobRead(vlob_id, '42', 1), const({ 'id': vlob_id, 'blob': blob, 'version': 1 })), (EBlockRead(id='2003'), const({ 'content': block_contents['2003'], 'creation_date': '2012-01-01T00:00:00' }))] matching_blocks = perform_sequence( sequence, file._find_matching_blocks(size, offset)) pre_excluded_data = contents[2][:blocks[2]['blocks'][0]['size'] - delta] pre_included_data = contents[2][-delta:][:size] post_excluded_data = contents[2][-delta:][size:] assert matching_blocks == { 'pre_excluded_blocks': [blocks[0], blocks[1]], 'pre_excluded_data': pre_excluded_data, 'pre_included_data': pre_included_data, 'included_blocks': [], 'post_included_data': b'', 'post_excluded_data': post_excluded_data, 'post_excluded_blocks': [blocks[i] for i in range(3, 6)] } # With big size delta = 10 size = delta size += blocks[3]['blocks'][0]['size'] size += blocks[3]['blocks'][1]['size'] size += blocks[3]['blocks'][2]['size'] size += 2 * delta offset = (blocks[0]['blocks'][0]['size'] + blocks[0]['blocks'][1]['size'] + blocks[1]['blocks'][0]['size'] + blocks[2]['blocks'][0]['size'] - delta) sequence = [(EVlobRead(vlob_id, '42', 1), const({ 'id': vlob_id, 'blob': blob, 'version': 1 })), (EBlockRead('2003'), const({ 'content': block_contents['2003'], 'creation_date': '2012-01-01T00:00:00' })), (EBlockRead('2007'), const({ 'content': block_contents['2007'], 'creation_date': '2012-01-01T00:00:00' }))] matching_blocks = perform_sequence( sequence, file._find_matching_blocks(size, offset)) pre_excluded_data = contents[2][:-delta] pre_included_data = contents[2][-delta:] post_included_data = contents[4][:2 * delta] post_excluded_data = contents[4][:block_size][2 * delta:] partial_block_4 = deepcopy(blocks[4]) del partial_block_4['blocks'][0] assert matching_blocks == { 'pre_excluded_blocks': [blocks[0], blocks[1]], 'pre_excluded_data': pre_excluded_data, 'pre_included_data': pre_included_data, 'included_blocks': [blocks[3]], 'post_included_data': post_included_data, 'post_excluded_data': post_excluded_data, 'post_excluded_blocks': [partial_block_4, blocks[5]] } # With big size and no delta size = blocks[3]['blocks'][0]['size'] size += blocks[3]['blocks'][1]['size'] size += blocks[3]['blocks'][2]['size'] offset = (blocks[0]['blocks'][0]['size'] + blocks[0]['blocks'][1]['size'] + blocks[1]['blocks'][0]['size'] + blocks[2]['blocks'][0]['size']) sequence = [ (EVlobRead(vlob_id, '42', 1), const({ 'id': vlob_id, 'blob': blob, 'version': 1 })), ] matching_blocks = perform_sequence( sequence, file._find_matching_blocks(size, offset)) assert matching_blocks == { 'pre_excluded_blocks': [blocks[0], blocks[1], blocks[2]], 'pre_excluded_data': b'', 'pre_included_data': b'', 'included_blocks': [blocks[3]], 'post_included_data': b'', 'post_excluded_data': b'', 'post_excluded_blocks': [blocks[4], blocks[5]] } # With total size sequence = [ (EVlobRead(vlob_id, '42', 1), const({ 'id': vlob_id, 'blob': blob, 'version': 1 })), ] matching_blocks = perform_sequence( sequence, file._find_matching_blocks(total_length, 0)) assert matching_blocks == { 'pre_excluded_blocks': [], 'pre_excluded_data': b'', 'pre_included_data': b'', 'included_blocks': [blocks[i] for i in range(0, 6)], 'post_included_data': b'', 'post_excluded_data': b'', 'post_excluded_blocks': [] }
def to_raw(self): return tools.ejson_dumps(self.to_dict())
def perform_client_event(intent): payload = ejson_dumps({'event': intent.event, 'sender': intent.sender}) client_context.queued_pushed_events.put_nowait(payload)
def flush(self): if not self.modifications: return # Merge all modifications to build final content builder = ContentBuilder() shortest_truncate = None for modification in self.modifications: if modification[0] == self.write: builder.write(*modification[1:]) elif modification[0] == self.truncate: builder.truncate(modification[1]) if not shortest_truncate or shortest_truncate > modification[1]: shortest_truncate = modification[1] else: raise NotImplementedError() self.modifications = [] # Truncate file previous_block_ids = yield self.get_blocks() if shortest_truncate is not None: matching_blocks = yield self._find_matching_blocks(shortest_truncate, 0) blob = [] blob += matching_blocks['included_blocks'] new_blocks = yield self._build_file_blocks(matching_blocks['post_included_data']) blob.append(new_blocks) blob = ejson_dumps(blob) blob = blob.encode() encrypted_blob = self.encryptor.encrypt(blob) encrypted_blob = to_jsonb64(encrypted_blob) yield Effect(EVlobUpdate(self.id, self.write_trust_seed, self.version + 1, encrypted_blob)) self.dirty = True # Write new contents for offset, content in builder.contents.items(): matching_blocks = yield self._find_matching_blocks(len(content), offset) new_data = matching_blocks['pre_excluded_data'] new_data += content new_data += matching_blocks['post_excluded_data'] blob = [] blob += matching_blocks['pre_excluded_blocks'] new_blocks = yield self._build_file_blocks(new_data) blob.append(new_blocks) blob += matching_blocks['post_excluded_blocks'] blob = ejson_dumps(blob) blob = blob.encode() encrypted_blob = self.encryptor.encrypt(blob) encrypted_blob = to_jsonb64(encrypted_blob) yield Effect(EVlobUpdate(self.id, self.write_trust_seed, self.version + 1, encrypted_blob)) self.dirty = True # Clean blocks current_block_ids = yield self.get_blocks() for block_id in previous_block_ids: if block_id not in current_block_ids: try: yield Effect(EBlockDelete(block_id)) except BlockNotFound: pass