def commit(self):
is_dirty = yield self.is_dirty()
if self.version != 0 and not is_dirty:
return
# Update manifest entries with new file vlobs (dustbin entries are already commited)
vlob_list = yield Effect(EVlobList())
for entry in self.entries.values():
if entry and entry['id'] in vlob_list:
file = yield File.load(entry['id'],
entry['key'],
entry['read_trust_seed'],
entry['write_trust_seed'])
new_vlob = yield file.commit()
if new_vlob and new_vlob is not True:
entry['id'] = new_vlob['id']
entry['read_trust_seed'] = new_vlob['read_trust_seed']
entry['write_trust_seed'] = new_vlob['write_trust_seed']
# Commit manifest
blob = yield self.dumps()
encrypted_blob = self.encryptor.encrypt(blob.encode())
encrypted_blob = to_jsonb64(encrypted_blob)
yield Effect(EVlobUpdate(self.id, self.write_trust_seed, self.version + 1, encrypted_blob))
self.original_manifest = ejson_loads(blob)
new_vlob = yield Effect(EVlobSynchronize(self.id))
if new_vlob:
if new_vlob is not True:
self.id = new_vlob['id']
self.read_trust_seed = new_vlob['read_trust_seed']
self.write_trust_seed = new_vlob['write_trust_seed']
new_vlob = self.get_vlob()
self.version += 1
return new_vlob
def commit(self, recursive=True):
is_dirty = yield self.is_dirty()
if self.version != 0 and not is_dirty:
return
# Update manifest with new group vlobs
vlob_list = yield Effect(EVlobList())
if recursive:
for group_manifest in self.group_manifests.values():
new_vlob = yield group_manifest.commit()
if new_vlob and new_vlob is not True:
old_vlob = group_manifest.get_vlob()
new_vlob['key'] = old_vlob['key']
group_manifest.update_vlob(new_vlob)
# Update manifest entries with new file vlobs (dustbin entries are already commited)
for entry in self.entries.values():
if entry and entry['id'] in vlob_list:
file = yield File.load(entry['id'],
entry['key'],
entry['read_trust_seed'],
entry['write_trust_seed'])
new_vlob = yield file.commit()
if new_vlob and new_vlob is not True:
entry['id'] = new_vlob['id']
entry['read_trust_seed'] = new_vlob['read_trust_seed']
entry['write_trust_seed'] = new_vlob['write_trust_seed']
# Commit manifest
blob = yield self.dumps()
encrypted_blob = self.encryptor.pub_key.encrypt(blob.encode())
encrypted_blob = to_jsonb64(encrypted_blob)
yield Effect(EUserVlobUpdate(self.version + 1, encrypted_blob))
self.original_manifest = ejson_loads(blob)
synchronized = yield Effect(EUserVlobSynchronize())
if synchronized:
self.version += 1
def test_perform_undelete(app, alice_identity, file):
vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'}
blob = [{
'blocks': [{
'block': '4567',
'digest': digest(b''),
'size': 0
}],
'key': to_jsonb64(b'<dummy-key-00000000000000000001>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
eff = app.perform_delete(EDelete('/foo'))
sequence = [
(EIdentityGet(), const(alice_identity)),
(EVlobRead(vlob['id'], vlob['read_trust_seed']),
const({
'id': vlob['id'],
'blob': blob,
'version': 1
})), (EVlobList(), const([])),
(EVlobRead(vlob['id'], vlob['read_trust_seed'],
1), const({
'id': vlob['id'],
'blob': blob,
'version': 1
})),
(EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))),
(EVlobDelete('2345'), conste(VlobNotFound('Vlob not found.')))
]
ret = perform_sequence(sequence, eff)
eff = app.perform_undelete(EUndelete('2345'))
sequence = [(EIdentityGet(), const(alice_identity))]
ret = perform_sequence(sequence, eff)
assert ret is None
def test_perform_file_history(app, file, alice_identity):
vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'}
blob = [{
'blocks': [{
'block': '4567',
'digest': digest(b''),
'size': 0
}],
'key': to_jsonb64(b'<dummy-key-00000000000000000001>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
eff = app.perform_file_history(EFileHistory('/foo', 1, 1))
sequence = [
(EIdentityGet(), const(alice_identity)),
(EIdentityGet(), const(alice_identity)),
(EVlobRead(vlob['id'], vlob['read_trust_seed']),
const({
'id': vlob['id'],
'blob': blob,
'version': 1
})),
(EVlobList(), const([])),
]
perform_sequence(sequence, eff)
def test_load_file(self, file):
vlob_id = '1234'
other_vlob_id = '5678'
read_trust_seed = '42'
version = 1
# Load from open files
file2 = perform_sequence(
[],
File.load(vlob_id, to_jsonb64(b'<dummy-key-00000000000000000001>'),
read_trust_seed, '43'))
assert file == file2
File.files = {}
# Test reloading commited and not commited file
for synchronizer_vlob_list in [[vlob_id, other_vlob_id],
[other_vlob_id]]:
key = to_jsonb64(b'<dummy-key-00000000000000000001>')
sequence = [
(EVlobRead(vlob_id, read_trust_seed, None),
const({
'id': vlob_id,
'blob': 'foo',
'version': version
})),
(EVlobList(), const(synchronizer_vlob_list)),
]
file = perform_sequence(
sequence, File.load(vlob_id, key, read_trust_seed, '43'))
assert file.dirty is (vlob_id in synchronizer_vlob_list)
assert file.version == (version - 1 if file.dirty else version)
File.files = {}
def test_perform_vlob_list(app):
blob = 'foo'
eff = app.perform_vlob_create(EVlobCreate(blob))
vlob = perform_sequence([], eff)
vlob_id = vlob['id']
eff = app.perform_vlob_create(EVlobCreate(blob))
vlob = perform_sequence([], eff)
vlob_2_id = vlob['id']
eff = app.perform_vlob_list(EVlobList())
vlob_list = perform_sequence([], eff)
assert isinstance(vlob_list, list)
assert set(vlob_list) == set([vlob_id, vlob_2_id])
# Synchronized vlobs are excluded
eff = app.perform_vlob_synchronize(EVlobSynchronize(vlob_2_id))
sequence = [(
EBackendVlobCreate(blob.encode()), # TODO encode ok ?
const(VlobAccess(vlob_2_id, '42', '42')))]
perform_sequence(sequence, eff)
eff = app.perform_vlob_list(EVlobList())
vlob_list = perform_sequence([], eff)
assert isinstance(vlob_list, list)
assert set(vlob_list) == set([vlob_id])
def test_perform_synchronize(app, alice_identity):
blob = {
'dustbin': [],
'entries': {
'/': None
},
'groups': {},
'versions': {}
}
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
eff = app.perform_synchronize(ESynchronize())
sequence = [(EIdentityGet(), const(alice_identity)),
(EVlobList(), const([])), (EUserVlobUpdate(1, blob), noop),
(EUserVlobSynchronize(), noop)]
ret = perform_sequence(sequence, eff)
assert ret is None
def load(cls, id, key, read_trust_seed, write_trust_seed, version=None):
if id in File.files:
return File.files[id]
self = File()
self.id = id
self.read_trust_seed = read_trust_seed
self.write_trust_seed = write_trust_seed
self.encryptor = load_sym_key(from_jsonb64(key))
vlob = yield Effect(EVlobRead(self.id, self.read_trust_seed, version))
self.version = vlob['version']
self.dirty = False
vlob_list = yield Effect(EVlobList())
if vlob['id'] in vlob_list:
self.dirty = True
self.version -= 1
self.modifications = []
File.files[self.id] = self
return self
def test_perform_dustbin_show(app, alice_identity, file):
with freeze_time('2012-01-01') as frozen_datetime:
vlob = {
'id': '2345',
'read_trust_seed': '42',
'write_trust_seed': '43'
}
blob = [{
'blocks': [{
'block': '4567',
'digest': digest(b''),
'size': 0
}],
'key': to_jsonb64(b'<dummy-key-00000000000000000001>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
eff = app.perform_delete(EDelete('/foo'))
sequence = [
(EIdentityGet(), const(alice_identity)),
(EVlobRead(vlob['id'], vlob['read_trust_seed']),
const({
'id': vlob['id'],
'blob': blob,
'version': 1
})),
(EVlobList(), const([])),
(EVlobRead(vlob['id'], vlob['read_trust_seed'],
1), const({
'id': vlob['id'],
'blob': blob,
'version': 1
})),
(EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))),
(EVlobDelete('2345'), conste(VlobNotFound('Vlob not found.'))),
]
perform_sequence(sequence, eff)
eff = app.perform_dustbin_show(EDustbinShow())
sequence = [(EIdentityGet(), const(alice_identity))]
dustbin = perform_sequence(sequence, eff)
vlob['path'] = '/foo'
vlob['removed_date'] = frozen_datetime().isoformat()
vlob['key'] = to_jsonb64(b'<dummy-key-00000000000000000002>')
assert dustbin == [vlob]
def test_perform_synchronize(app):
content = 'foo'
eff = app.perform_block_create(EBlockCreate(content))
block_id = perform_sequence([], eff)
eff = app.perform_block_create(EBlockCreate(content))
block_2_id = perform_sequence([], eff)
block_ids = sorted([block_id, block_2_id])
blob = 'foo'
eff = app.perform_vlob_create(EVlobCreate(blob))
perform_sequence([], eff)
eff = app.perform_vlob_create(EVlobCreate(blob))
perform_sequence([], eff)
eff = app.perform_synchronize(ESynchronize())
sequence = [
(EBackendBlockCreate(block_ids[0],
content), const(Block(block_ids[0], content))),
(EBackendBlockCreate(block_ids[1],
content), const(Block(block_ids[1], content))),
(
EBackendVlobCreate(blob.encode()), # TODO encode correct?
const(VlobAccess('345', 'ABC', 'DEF'))),
(
EBackendVlobCreate(blob.encode()), # TODO encode correct?
const(VlobAccess('678', 'ABC', 'DEF'))),
]
synchronization = perform_sequence(sequence, eff)
assert synchronization is True
eff = app.perform_block_list(EBlockList())
block_list = perform_sequence([], eff)
assert block_list == []
eff = app.perform_user_vlob_exist(EUserVlobExist())
exist = perform_sequence([], eff)
assert exist is False
eff = app.perform_vlob_list(EVlobList())
vlob_list = perform_sequence([], eff)
assert vlob_list == []
# Do nothing
eff = app.perform_synchronize(ESynchronize())
synchronization = perform_sequence([], eff)
assert synchronization is False
def test_perform_vlob_synchronize(app, app_no_cache):
# Create new vlob
blob = 'foo'
eff = app.perform_vlob_update(EVlobUpdate('123', 'ABC', 1, blob))
perform_sequence([], eff)
# With cache enabled
assert app.vlob_cache.currsize == 0
eff = app.perform_vlob_synchronize(EVlobSynchronize('123'))
sequence = [(
EBackendVlobCreate(blob.encode()), # TODO encode ok ?
const(VlobAccess('123', 'ABC', 'DEF')))]
synchronization = perform_sequence(sequence, eff)
assert synchronization == {
'id': '123',
'read_trust_seed': 'ABC',
'write_trust_seed': 'DEF'
}
eff = app.perform_vlob_list(EVlobList())
vlob_list = perform_sequence([], eff)
assert vlob_list == []
assert app.vlob_cache.currsize == 1
# With cache disabled
eff = app_no_cache.perform_vlob_update(EVlobUpdate('123', 'ABC', 1, blob))
perform_sequence([], eff)
assert app_no_cache.vlob_cache.currsize == 0
eff = app_no_cache.perform_vlob_synchronize(EVlobSynchronize('123'))
sequence = [(
EBackendVlobCreate(blob.encode()), # TODO encode ok ?
const(VlobAccess('123', 'ABC', 'DEF')))]
synchronization = perform_sequence(sequence, eff)
assert synchronization == {
'id': '123',
'read_trust_seed': 'ABC',
'write_trust_seed': 'DEF'
}
eff = app_no_cache.perform_vlob_list(EVlobList())
vlob_list = perform_sequence([], eff)
assert vlob_list == []
assert app_no_cache.vlob_cache.currsize == 0
# Update vlob
blob = 'bar'
eff = app.perform_vlob_update(EVlobUpdate('123', 'ABC', 2, blob))
perform_sequence([], eff)
vlob = app.vlobs['123']
# With cache enabled
assert app.vlob_cache.currsize == 1
eff = app.perform_vlob_synchronize(EVlobSynchronize('123'))
sequence = [(
EBackendVlobUpdate('123', 'ABC', 2, blob.encode()), # TODO encode ok ?
noop)]
synchronization = perform_sequence(sequence, eff)
assert synchronization is True
eff = app.perform_vlob_list(EVlobList())
vlob_list = perform_sequence([], eff)
assert vlob_list == []
assert app.vlob_cache.currsize == 2
# With cache disabled
app_no_cache.vlobs['123'] = vlob
assert app_no_cache.vlob_cache.currsize == 0
eff = app_no_cache.perform_vlob_synchronize(EVlobSynchronize('123'))
sequence = [(
EBackendVlobUpdate('123', 'ABC', 2, blob.encode()), # TODO encode ok ?
noop)]
synchronization = perform_sequence(sequence, eff)
assert synchronization is True
eff = app_no_cache.perform_vlob_list(EVlobList())
vlob_list = perform_sequence([], eff)
assert vlob_list == []
assert app_no_cache.vlob_cache.currsize == 0
# Vlob synchronized
eff = app.perform_vlob_synchronize(EVlobSynchronize('123'))
synchronization = perform_sequence([], eff)
assert synchronization is False
# Delete user vlob from cache
for version in [1, 2]:
eff = app.perform_vlob_delete(EVlobDelete('123', version))
perform_sequence([], eff)
# Do nothing if vlob not found
eff = app.perform_vlob_synchronize(EVlobSynchronize('123'))
synchronization = perform_sequence([], eff)
assert synchronization is False