def test_perform_file_create(app, alice_identity, file):
vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'}
block_id = '4567'
# Already exist
blob = [{
'blocks': [{
'block': block_id,
'digest': digest(b''),
'size': 0
}],
'key': to_jsonb64(b'<dummy-key-00000000000000000003>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
eff = app.perform_file_create(EFileCreate('/foo'))
sequence = [
(EBlockCreate(''), const(block_id)),
(EVlobCreate(blob), const(vlob)),
(EIdentityGet(), const(alice_identity)),
(EVlobRead(vlob['id'], vlob['read_trust_seed'],
1), const({
'id': vlob['id'],
'blob': blob,
'version': 1
})),
(EBlockDelete(block_id), noop),
(EVlobDelete(vlob['id']), noop),
]
with pytest.raises(ManifestError):
perform_sequence(sequence, eff)
def test_perform_file_history(app, file, alice_identity):
vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'}
blob = [{
'blocks': [{
'block': '4567',
'digest': digest(b''),
'size': 0
}],
'key': to_jsonb64(b'<dummy-key-00000000000000000001>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
eff = app.perform_file_history(EFileHistory('/foo', 1, 1))
sequence = [
(EIdentityGet(), const(alice_identity)),
(EIdentityGet(), const(alice_identity)),
(EVlobRead(vlob['id'], vlob['read_trust_seed']),
const({
'id': vlob['id'],
'blob': blob,
'version': 1
})),
(EVlobList(), const([])),
]
perform_sequence(sequence, eff)
def test_perform_undelete(app, alice_identity, file):
vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'}
blob = [{
'blocks': [{
'block': '4567',
'digest': digest(b''),
'size': 0
}],
'key': to_jsonb64(b'<dummy-key-00000000000000000001>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
eff = app.perform_delete(EDelete('/foo'))
sequence = [
(EIdentityGet(), const(alice_identity)),
(EVlobRead(vlob['id'], vlob['read_trust_seed']),
const({
'id': vlob['id'],
'blob': blob,
'version': 1
})), (EVlobList(), const([])),
(EVlobRead(vlob['id'], vlob['read_trust_seed'],
1), const({
'id': vlob['id'],
'blob': blob,
'version': 1
})),
(EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))),
(EVlobDelete('2345'), conste(VlobNotFound('Vlob not found.')))
]
ret = perform_sequence(sequence, eff)
eff = app.perform_undelete(EUndelete('2345'))
sequence = [(EIdentityGet(), const(alice_identity))]
ret = perform_sequence(sequence, eff)
assert ret is None
def test_load_file(self, file):
vlob_id = '1234'
other_vlob_id = '5678'
read_trust_seed = '42'
version = 1
# Load from open files
file2 = perform_sequence(
[],
File.load(vlob_id, to_jsonb64(b'<dummy-key-00000000000000000001>'),
read_trust_seed, '43'))
assert file == file2
File.files = {}
# Test reloading commited and not commited file
for synchronizer_vlob_list in [[vlob_id, other_vlob_id],
[other_vlob_id]]:
key = to_jsonb64(b'<dummy-key-00000000000000000001>')
sequence = [
(EVlobRead(vlob_id, read_trust_seed, None),
const({
'id': vlob_id,
'blob': 'foo',
'version': version
})),
(EVlobList(), const(synchronizer_vlob_list)),
]
file = perform_sequence(
sequence, File.load(vlob_id, key, read_trust_seed, '43'))
assert file.dirty is (vlob_id in synchronizer_vlob_list)
assert file.version == (version - 1 if file.dirty else version)
File.files = {}
def test_discard(self, file):
content = b'This is a test content.'
block_ids = ['4567', '5678', '6789']
# Original content
chunk_1 = content[:5]
chunk_2 = content[5:14]
chunk_3 = content[14:]
blob = [{
'blocks': [{
'block': block_ids[0],
'digest': digest(chunk_1),
'size': len(chunk_1)
}, {
'block': block_ids[1],
'digest': digest(chunk_2),
'size': len(chunk_2)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000003>')
}, {
'blocks': [{
'block': block_ids[2],
'digest': digest(chunk_3),
'size': len(chunk_3)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000004>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
# Already synchronized
sequence = [
(EVlobRead('1234', '42',
1), const({
'id': '1234',
'blob': blob,
'version': 1
})),
(EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))),
(EBlockDelete('5678'), noop),
(EBlockDelete('6789'), noop),
(EVlobDelete('1234'), conste(VlobNotFound('Block not found.'))
) # TODO vlob OR block exceptin
]
ret = perform_sequence(sequence, file.discard())
assert ret is False
# Not already synchronized
file.dirty = True
file.version = 0
sequence = [(EVlobRead('1234', '42', 1),
const({
'id': '1234',
'blob': blob,
'version': 1
})), (EBlockDelete('4567'), noop),
(EBlockDelete('5678'), noop), (EBlockDelete('6789'), noop),
(EVlobDelete('1234'), noop)]
ret = perform_sequence(sequence, file.discard())
assert ret is True
assert file.dirty is False
def test_reencrypt(self, file):
old_vlob = file.get_vlob()
content = b'This is a test content.'
block_ids = ['4567', '5678', '6789']
# Original content
chunk_1 = content[:5]
chunk_2 = content[5:14]
chunk_3 = content[14:]
blob = [{
'blocks': [{
'block': block_ids[0],
'digest': digest(chunk_1),
'size': len(chunk_1)
}, {
'block': block_ids[1],
'digest': digest(chunk_2),
'size': len(chunk_2)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000001>')
}, {
'blocks': [{
'block': block_ids[2],
'digest': digest(chunk_3),
'size': len(chunk_3)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000002>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
sequence = [
(EVlobRead('1234', '42',
1), const({
'id': '1234',
'blob': blob,
'version': 1
})),
(
EVlobCreate(blob), # TODO check re-encryption
const({
'id': '2345',
'read_trust_seed': '21',
'write_trust_seed': '22'
}))
]
ret = perform_sequence(sequence, file.reencrypt())
assert ret is None
file.reencrypt()
new_vlob = file.get_vlob()
for property in old_vlob.keys():
assert old_vlob[property] != new_vlob[property]
def test_perform_message_get():
eff = perform_message_get(EBackendMessageGet('John', 1))
backend_response = {
'status': 'ok',
'messages': [
{'count': 1, 'body': to_jsonb64(b'body 1')},
{'count': 2, 'body': to_jsonb64(b'body 2')}
]
}
sequence = [
(BackendCmd('message_get', {'offset': 1, 'recipient': 'John'}), const(backend_response))
]
ret = perform_sequence(sequence, eff)
assert ret == [Message(1, b'body 1'), Message(2, b'body 2')]
async def open_connection(self, identity):
logger.debug('Connection to backend opened')
assert not self._websocket, "Connection to backend already opened"
try:
self._websocket = await websockets.connect(self.url)
# Handle handshake
raw = await self._websocket.recv()
challenge = ejson_loads(raw)
answer = identity.private_key.sign(challenge['challenge'].encode())
await self._websocket.send(
ejson_dumps({
'handshake': 'answer',
'identity': identity.id,
'answer': to_jsonb64(answer)
}))
resp = ejson_loads(await self._websocket.recv())
if resp['status'] != 'ok':
await self.close_connection()
raise exception_from_status(resp['status'])(resp['label'])
self._ws_recv_handler_task = asyncio.ensure_future(
self._ws_recv_handler(), loop=self.loop)
if self.watchdog_time:
self._watchdog_task = asyncio.ensure_future(self._watchdog(),
loop=self.loop)
except (ConnectionRefusedError,
websockets.exceptions.ConnectionClosed) as exc:
raise BackendConnectionError('Cannot connect to backend (%s)' %
exc)
def test_vlob_create_ok(self, id_and_blob):
id, blob = id_and_blob
intent = EVlobCreate(id, blob)
intent_ret = VlobAtom(id, 'readtrustseed-123', 'writetrustseed-123',
blob, 1)
payload = {}
if id:
payload['id'] = id
else:
intent_ret.id = '123'
if blob:
payload['blob'] = to_jsonb64(blob)
else:
intent.blob = b''
intent_ret.blob = b''
eff = execute_cmd('vlob_create', payload)
sequence = [
(intent, const(intent_ret)),
]
ret = perform_sequence(sequence, eff)
assert ret == {
'status': 'ok',
'id': intent_ret.id,
'read_trust_seed': 'readtrustseed-123',
'write_trust_seed': 'writetrustseed-123'
}
def test_build_file_blocks(self, file, length):
file.dirty = False
block_size = 4096
content = b''.join(
[str(random.randint(1, 9)).encode() for i in range(0, length)])
chunks = [
content[i:i + block_size]
for i in range(0, len(content), block_size)
]
if not chunks:
chunks = [b'']
sequence = []
for chunk in chunks:
sequence.append((EBlockCreate(to_jsonb64(chunk)), const('4567')))
blocks = perform_sequence(sequence, file._build_file_blocks(content))
assert sorted(blocks.keys()) == ['blocks', 'key']
assert isinstance(blocks['blocks'], list)
required_blocks = int(len(content) / block_size)
if not len(content) or len(content) % block_size:
required_blocks += 1
assert len(blocks['blocks']) == required_blocks
for index, block in enumerate(blocks['blocks']):
assert sorted(block.keys()) == ['block', 'digest', 'size']
assert block['block']
length = len(content) - index * block_size
length = block_size if length > block_size else length
assert block['size'] == length
assert block['digest'] == digest(content[index * block_size:index +
1 * block_size])
assert file.dirty is True
def get_vlob(self):
vlob = {}
vlob['id'] = self.id
vlob['read_trust_seed'] = self.read_trust_seed
vlob['write_trust_seed'] = self.write_trust_seed
vlob['key'] = to_jsonb64(self.encryptor.key)
return vlob
def app(mock_crypto_passthrough, alice_identity):
# app = FSComponent()
# identity_component = IdentityComponent()
fs_component = FSComponent()
# synchronizer_component = SynchronizerComponent()
# identity_component = IdentityComponent()
# app = app_factory(
# fs_component.get_dispatcher(),
# synchronizer_component.get_dispatcher(),
# identity_component.get_dispatcher()
# )
blob = {
'dustbin': [],
'entries': {
'/': None
},
'groups': {},
'versions': {}
}
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
sequence = [(EIdentityGet(), const(alice_identity)),
(EUserVlobRead(), const({
'blob': '',
'version': 0
})), (EUserVlobUpdate(1, blob), noop)]
perform_sequence(sequence, fs_component._get_manifest())
return fs_component
def test_get_vlob(self, file):
assert file.get_vlob() == {
'id': '1234',
'key': to_jsonb64(b'<dummy-key-00000000000000000002>'),
'read_trust_seed': '42',
'write_trust_seed': '43'
}
def commit(self, recursive=True):
is_dirty = yield self.is_dirty()
if self.version != 0 and not is_dirty:
return
# Update manifest with new group vlobs
vlob_list = yield Effect(EVlobList())
if recursive:
for group_manifest in self.group_manifests.values():
new_vlob = yield group_manifest.commit()
if new_vlob and new_vlob is not True:
old_vlob = group_manifest.get_vlob()
new_vlob['key'] = old_vlob['key']
group_manifest.update_vlob(new_vlob)
# Update manifest entries with new file vlobs (dustbin entries are already commited)
for entry in self.entries.values():
if entry and entry['id'] in vlob_list:
file = yield File.load(entry['id'],
entry['key'],
entry['read_trust_seed'],
entry['write_trust_seed'])
new_vlob = yield file.commit()
if new_vlob and new_vlob is not True:
entry['id'] = new_vlob['id']
entry['read_trust_seed'] = new_vlob['read_trust_seed']
entry['write_trust_seed'] = new_vlob['write_trust_seed']
# Commit manifest
blob = yield self.dumps()
encrypted_blob = self.encryptor.pub_key.encrypt(blob.encode())
encrypted_blob = to_jsonb64(encrypted_blob)
yield Effect(EUserVlobUpdate(self.version + 1, encrypted_blob))
self.original_manifest = ejson_loads(blob)
synchronized = yield Effect(EUserVlobSynchronize())
if synchronized:
self.version += 1
def flush(self):
if not self.modifications:
return
# Merge all modifications to build final content
builder = ContentBuilder()
shortest_truncate = None
for modification in self.modifications:
if modification[0] == self.write:
builder.write(*modification[1:])
elif modification[0] == self.truncate:
builder.truncate(modification[1])
if not shortest_truncate or shortest_truncate > modification[1]:
shortest_truncate = modification[1]
else:
raise NotImplementedError()
self.modifications = []
# Truncate file
if shortest_truncate is not None:
response = self._operations.send_cmd(
cmd='file_truncate', path=self.path, length=shortest_truncate)
if response['status'] != 'ok':
raise FuseOSError(ENOENT)
# Write new contents
for offset, content in builder.contents.items():
# TODO use flags
response = self._operations.send_cmd(
cmd='file_write',
path=self.path,
content=to_jsonb64(content),
offset=offset)
if response['status'] != 'ok':
raise FuseOSError(ENOENT)
def commit(self):
is_dirty = yield self.is_dirty()
if self.version != 0 and not is_dirty:
return
# Update manifest entries with new file vlobs (dustbin entries are already commited)
vlob_list = yield Effect(EVlobList())
for entry in self.entries.values():
if entry and entry['id'] in vlob_list:
file = yield File.load(entry['id'],
entry['key'],
entry['read_trust_seed'],
entry['write_trust_seed'])
new_vlob = yield file.commit()
if new_vlob and new_vlob is not True:
entry['id'] = new_vlob['id']
entry['read_trust_seed'] = new_vlob['read_trust_seed']
entry['write_trust_seed'] = new_vlob['write_trust_seed']
# Commit manifest
blob = yield self.dumps()
encrypted_blob = self.encryptor.encrypt(blob.encode())
encrypted_blob = to_jsonb64(encrypted_blob)
yield Effect(EVlobUpdate(self.id, self.write_trust_seed, self.version + 1, encrypted_blob))
self.original_manifest = ejson_loads(blob)
new_vlob = yield Effect(EVlobSynchronize(self.id))
if new_vlob:
if new_vlob is not True:
self.id = new_vlob['id']
self.read_trust_seed = new_vlob['read_trust_seed']
self.write_trust_seed = new_vlob['write_trust_seed']
new_vlob = self.get_vlob()
self.version += 1
return new_vlob
def reencrypt(self):
# Reencrypt files
for path, entry in self.entries.items():
if entry:
file = yield File.load(**entry)
yield file.reencrypt()
new_vlob = file.get_vlob()
self.entries[path] = new_vlob
for index, entry in enumerate(self.dustbin):
entry = deepcopy(entry)
path = entry['path']
removed_date = entry['removed_date']
for key in ['path', 'removed_date']:
del entry[key]
file = yield File.load(**entry)
yield file.reencrypt()
new_vlob = file.get_vlob()
new_vlob['path'] = path
new_vlob['removed_date'] = removed_date
self.dustbin[index] = new_vlob
# Reencrypt manifest
blob = yield self.dumps()
self.encryptor = generate_sym_key()
encrypted_blob = self.encryptor.encrypt(blob.encode())
encrypted_blob = to_jsonb64(encrypted_blob)
new_vlob = yield Effect(EVlobCreate(encrypted_blob))
self.id = new_vlob['id']
self.read_trust_seed = new_vlob['read_trust_seed']
self.write_trust_seed = new_vlob['write_trust_seed']
self.version = 0
async def test_read_offset(johndoe_client):
await mk_file(johndoe_client, '/foo.txt', data=b'1234567890')
ret = await johndoe_client.send_cmd('file_read',
path='/foo.txt',
offset=3,
size=4)
assert ret == {'status': 'ok', 'content': to_jsonb64(b'4567')}
async def test_file_truncate(johndoe_client):
await mk_file(johndoe_client, '/foo.txt', data=b'1234567890')
ret = await johndoe_client.send_cmd('file_truncate',
path='/foo.txt',
length=4)
assert ret == {'status': 'ok'}
ret = await johndoe_client.send_cmd('file_read', path='/foo.txt')
assert ret == {'status': 'ok', 'content': to_jsonb64(b'1234')}
def perform_vlob_create(intent):
msg = {'blob': to_jsonb64(intent.blob)}
ret = yield Effect(BackendCmd('vlob_create', msg))
status = ret['status']
if status != 'ok':
raise exception_from_status(status)(ret['label'])
return VlobAccess(ret['id'], ret['read_trust_seed'],
ret['write_trust_seed'])
def api_user_vlob_read(msg):
msg = cmd_READ_Schema().load(msg)
atom = yield Effect(EUserVlobRead(**msg))
return {
'status': 'ok',
'blob': to_jsonb64(atom.blob),
'version': atom.version
}
async def test_big_file(johndoe_client):
data = b'x' * 10000 # 10ko
await mk_file(johndoe_client, '/foo.txt', data=data)
ret = await johndoe_client.send_cmd('file_read',
path='/foo.txt',
offset=42,
size=1000)
assert ret == {'status': 'ok', 'content': to_jsonb64(data[42:1042])}
async def test_write_offset(johndoe_client):
await mk_file(johndoe_client, '/foo.txt', data=b'1234567890')
ret = await johndoe_client.send_cmd('file_write',
path='/foo.txt',
content=b'abcd',
offset=3)
assert ret == {'status': 'ok'}
ret = await johndoe_client.send_cmd('file_read', path='/foo.txt')
assert ret == {'status': 'ok', 'content': to_jsonb64(b'123abcd890')}
async def test_write_file(johndoe_client):
ret = await johndoe_client.send_cmd('file_create', path='/foo.txt')
assert ret == {'status': 'ok'}
ret = await johndoe_client.send_cmd('file_write',
path='/foo.txt',
content=b'fooo')
assert ret == {'status': 'ok'}
ret = await johndoe_client.send_cmd('file_read', path='/foo.txt')
assert ret == {'status': 'ok', 'content': to_jsonb64(b'fooo')}
def test_create_file(self, file):
assert file.dirty is True
assert file.version == 0
assert file.get_vlob() == {
'id': '1234',
'key': to_jsonb64(b'<dummy-key-00000000000000000002>'),
'read_trust_seed': '42',
'write_trust_seed': '43'
}
def test_user_vlob_update_bad_version(self):
eff = execute_cmd('user_vlob_update', {
'version': 42,
'blob': to_jsonb64(b'Next version.')
})
sequence = [(EUserVlobUpdate(version=42, blob=b'Next version.'),
conste(UserVlobError('Wrong blob version.')))]
ret = perform_sequence(sequence, eff)
assert ret['status'] == 'user_vlob_error'
def test_message_get_ok(self):
eff = execute_cmd('message_get', {})
sequence = [(EMessageGet(offset=0), const([b'zero', b'one', b'two']))]
ret = perform_sequence(sequence, eff)
assert ret == {
'status':
'ok',
'messages': [{
'count': 1,
'body': to_jsonb64(b'zero')
}, {
'count': 2,
'body': to_jsonb64(b'one')
}, {
'count': 3,
'body': to_jsonb64(b'two')
}]
}
def test_perform_vlob_update():
eff = perform_user_vlob_update(EBackendUserVlobUpdate(3, b'bar'))
backend_response = {'status': 'ok'}
sequence = [(BackendCmd('user_vlob_update', {
'version': 3,
'blob': to_jsonb64(b'bar')
}), const(backend_response))]
ret = perform_sequence(sequence, eff)
assert ret is None
def test_perform_message_new():
eff = perform_message_new(EBackendMessageNew('John', b'my body'))
backend_response = {'status': 'ok'}
sequence = [
(BackendCmd('message_new', {'recipient': 'John', 'body': to_jsonb64(b'my body')}),
const(backend_response))
]
ret = perform_sequence(sequence, eff)
assert ret is None
def test_perform_dustbin_show(app, alice_identity, file):
with freeze_time('2012-01-01') as frozen_datetime:
vlob = {
'id': '2345',
'read_trust_seed': '42',
'write_trust_seed': '43'
}
blob = [{
'blocks': [{
'block': '4567',
'digest': digest(b''),
'size': 0
}],
'key': to_jsonb64(b'<dummy-key-00000000000000000001>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
eff = app.perform_delete(EDelete('/foo'))
sequence = [
(EIdentityGet(), const(alice_identity)),
(EVlobRead(vlob['id'], vlob['read_trust_seed']),
const({
'id': vlob['id'],
'blob': blob,
'version': 1
})),
(EVlobList(), const([])),
(EVlobRead(vlob['id'], vlob['read_trust_seed'],
1), const({
'id': vlob['id'],
'blob': blob,
'version': 1
})),
(EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))),
(EVlobDelete('2345'), conste(VlobNotFound('Vlob not found.'))),
]
perform_sequence(sequence, eff)
eff = app.perform_dustbin_show(EDustbinShow())
sequence = [(EIdentityGet(), const(alice_identity))]
dustbin = perform_sequence(sequence, eff)
vlob['path'] = '/foo'
vlob['removed_date'] = frozen_datetime().isoformat()
vlob['key'] = to_jsonb64(b'<dummy-key-00000000000000000002>')
assert dustbin == [vlob]