def test_perform_block_synchronize(app, app_no_cache):
content = 'foo'
eff = app.perform_block_create(EBlockCreate(content))
block_id = perform_sequence([], eff)
eff = app_no_cache.perform_block_create(EBlockCreate(content))
block_2_id = perform_sequence([], eff)
# With cache enabled
assert app.block_cache.currsize == 0
eff = app.perform_block_synchronize(EBlockSynchronize(block_id))
sequence = [(EBackendBlockCreate(block_id,
content), const(Block(block_id,
content)))]
synchronization = perform_sequence(sequence, eff)
assert synchronization is True
assert block_id not in app.blocks
assert app.block_cache.currsize == 1
# With cache disabled
assert app_no_cache.block_cache.currsize == 0
eff = app_no_cache.perform_block_synchronize(EBlockSynchronize(block_2_id))
sequence = [(EBackendBlockCreate(block_2_id, content),
const(Block(block_2_id, content)))]
synchronization = perform_sequence(sequence, eff)
assert synchronization is True
assert block_2_id not in app_no_cache.blocks
assert app_no_cache.block_cache.currsize == 0
# Do nothing
eff = app.perform_block_synchronize(EBlockSynchronize(block_2_id))
synchronization = perform_sequence([], eff)
assert synchronization is False
def test_build_file_blocks(self, file, length):
file.dirty = False
block_size = 4096
content = b''.join(
[str(random.randint(1, 9)).encode() for i in range(0, length)])
chunks = [
content[i:i + block_size]
for i in range(0, len(content), block_size)
]
if not chunks:
chunks = [b'']
sequence = []
for chunk in chunks:
sequence.append((EBlockCreate(to_jsonb64(chunk)), const('4567')))
blocks = perform_sequence(sequence, file._build_file_blocks(content))
assert sorted(blocks.keys()) == ['blocks', 'key']
assert isinstance(blocks['blocks'], list)
required_blocks = int(len(content) / block_size)
if not len(content) or len(content) % block_size:
required_blocks += 1
assert len(blocks['blocks']) == required_blocks
for index, block in enumerate(blocks['blocks']):
assert sorted(block.keys()) == ['block', 'digest', 'size']
assert block['block']
length = len(content) - index * block_size
length = block_size if length > block_size else length
assert block['size'] == length
assert block['digest'] == digest(content[index * block_size:index +
1 * block_size])
assert file.dirty is True
def test_perform_file_create(app, alice_identity, file):
vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'}
block_id = '4567'
# Already exist
blob = [{
'blocks': [{
'block': block_id,
'digest': digest(b''),
'size': 0
}],
'key': to_jsonb64(b'<dummy-key-00000000000000000003>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
eff = app.perform_file_create(EFileCreate('/foo'))
sequence = [
(EBlockCreate(''), const(block_id)),
(EVlobCreate(blob), const(vlob)),
(EIdentityGet(), const(alice_identity)),
(EVlobRead(vlob['id'], vlob['read_trust_seed'],
1), const({
'id': vlob['id'],
'blob': blob,
'version': 1
})),
(EBlockDelete(block_id), noop),
(EVlobDelete(vlob['id']), noop),
]
with pytest.raises(ManifestError):
perform_sequence(sequence, eff)
def test_perform_block_create(app):
content = 'foo'
with freeze_time('2012-01-01') as frozen_datetime:
eff = app.perform_block_create(EBlockCreate(content))
block_id = perform_sequence([], eff)
assert app.last_modified == Arrow.fromdatetime(frozen_datetime())
eff = app.perform_block_read(EBlockRead(block_id))
block = perform_sequence([], eff)
assert block['content'] == content
def test_perform_block_list(app):
content = 'foo'
eff = app.perform_block_create(EBlockCreate(content))
block_id = perform_sequence([], eff)
eff = app.perform_block_create(EBlockCreate(content))
block_2_id = perform_sequence([], eff)
eff = app.perform_block_list(EBlockList())
block_list = perform_sequence([], eff)
assert isinstance(block_list, list)
assert set(block_list) == set([block_id, block_2_id])
# Synchronized blocks are excluded
eff = app.perform_block_synchronize(EBlockSynchronize(block_2_id))
sequence = [(EBackendBlockCreate(block_2_id, content),
const(Block(block_2_id, content)))]
perform_sequence(sequence, eff)
eff = app.perform_block_list(EBlockList())
block_list = perform_sequence([], eff)
assert isinstance(block_list, list)
assert set(block_list) == set([block_id])
def test_perform_synchronize(app):
content = 'foo'
eff = app.perform_block_create(EBlockCreate(content))
block_id = perform_sequence([], eff)
eff = app.perform_block_create(EBlockCreate(content))
block_2_id = perform_sequence([], eff)
block_ids = sorted([block_id, block_2_id])
blob = 'foo'
eff = app.perform_vlob_create(EVlobCreate(blob))
perform_sequence([], eff)
eff = app.perform_vlob_create(EVlobCreate(blob))
perform_sequence([], eff)
eff = app.perform_synchronize(ESynchronize())
sequence = [
(EBackendBlockCreate(block_ids[0],
content), const(Block(block_ids[0], content))),
(EBackendBlockCreate(block_ids[1],
content), const(Block(block_ids[1], content))),
(
EBackendVlobCreate(blob.encode()), # TODO encode correct?
const(VlobAccess('345', 'ABC', 'DEF'))),
(
EBackendVlobCreate(blob.encode()), # TODO encode correct?
const(VlobAccess('678', 'ABC', 'DEF'))),
]
synchronization = perform_sequence(sequence, eff)
assert synchronization is True
eff = app.perform_block_list(EBlockList())
block_list = perform_sequence([], eff)
assert block_list == []
eff = app.perform_user_vlob_exist(EUserVlobExist())
exist = perform_sequence([], eff)
assert exist is False
eff = app.perform_vlob_list(EVlobList())
vlob_list = perform_sequence([], eff)
assert vlob_list == []
# Do nothing
eff = app.perform_synchronize(ESynchronize())
synchronization = perform_sequence([], eff)
assert synchronization is False
def test_perform_block_read(app, app_no_cache):
local_content = 'foo'
eff = app.perform_block_create(EBlockCreate(local_content))
block_id = perform_sequence([], eff)
# Read block in new blocks
eff = app.perform_block_read(EBlockRead(block_id))
block = perform_sequence([], eff)
assert sorted(list(block.keys())) == ['content', 'id']
assert block['id']
assert block['content'] == local_content
remote_content = b'bar'
# Read remote block
assert app.block_cache.currsize == 0
eff = app.perform_block_read(EBlockRead('123'))
sequence = [(EBackendBlockRead('123'), const(Block('123',
remote_content)))]
block = perform_sequence(sequence, eff)
assert sorted(list(block.keys())) == ['content', 'id']
assert block['id']
assert block['content'] == remote_content
assert app.block_cache.currsize == 1
# Read remote block with cache disabled
assert app_no_cache.block_cache.currsize == 0
eff = app_no_cache.perform_block_read(EBlockRead('123'))
sequence = [(EBackendBlockRead('123'), const(Block('123',
remote_content)))]
block = perform_sequence(sequence, eff)
assert sorted(list(block.keys())) == ['content', 'id']
assert block['id']
assert block['content'] == remote_content
assert app_no_cache.block_cache.currsize == 0
# Read block in cache
eff = app.perform_block_read(EBlockRead('123'))
block = perform_sequence([], eff)
assert sorted(list(block.keys())) == ['content', 'id']
assert block['id']
assert block['content'] == remote_content
# Delete block from cache
eff = app.perform_block_delete(EBlockDelete('123'))
perform_sequence([], eff)
# Not found
eff = app.perform_block_read(EBlockRead('123'))
sequence = [(EBackendBlockRead('123'),
conste(BlockNotFound('Block not found.')))]
with pytest.raises(BlockNotFound):
block = perform_sequence(sequence, eff)
eff = app.perform_block_read(EBlockRead('123'))
sequence = [(EBackendBlockRead('123'), conste(BlockError('Block error.'))
) # TODO keep it? usefull with multiple backends...
]
with pytest.raises(BlockNotFound):
block = perform_sequence(sequence, eff)
def file(app, alice_identity, mock_crypto_passthrough):
vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'}
block_id = '4567'
blob = [{
'blocks': [{
'block': block_id,
'digest': digest(b''),
'size': 0
}],
'key': to_jsonb64(b'<dummy-key-00000000000000000001>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
eff = app.perform_file_create(EFileCreate('/foo'))
sequence = [(EBlockCreate(''), const(block_id)),
(EVlobCreate(blob), const(vlob)),
(EIdentityGet(), const(alice_identity))]
ret = perform_sequence(sequence, eff)
assert ret is None
File.files = {}
def test_perform_block_delete(app):
content = 'foo'
eff = app.perform_block_create(EBlockCreate(content))
block_id = perform_sequence([], eff)
# Delete from new blocks
with freeze_time('2012-01-01') as frozen_datetime:
eff = app.perform_block_delete(EBlockDelete(block_id))
perform_sequence([], eff)
assert app.last_modified == Arrow.fromdatetime(frozen_datetime())
# Delete in cache
app.block_cache[block_id] = {'foo': 'bar'}
assert app.block_cache.currsize == 1
with freeze_time('2012-01-01') as frozen_datetime:
eff = app.perform_block_delete(EBlockDelete(block_id))
perform_sequence([], eff)
assert app.last_modified == Arrow.fromdatetime(frozen_datetime())
assert app.block_cache.currsize == 0
# Not found
with pytest.raises(BlockNotFound):
eff = app.perform_block_delete(EBlockDelete(block_id))
perform_sequence([], eff)
def _build_file_blocks(self, data):
# Create chunks
chunk_size = 4096 # TODO modify size
chunks = [data[i:i + chunk_size] for i in range(0, len(data), chunk_size)]
# Force a chunk even if the data is empty
if not chunks:
chunks = [b'']
encryptor = generate_sym_key()
blocks = []
for chunk in chunks:
cypher_chunk = encryptor.encrypt(chunk)
cypher_chunk = to_jsonb64(cypher_chunk)
block_id = yield Effect(EBlockCreate(cypher_chunk))
blocks.append({'block': block_id,
'digest': digest(chunk),
'size': len(chunk)})
# New vlob atom
block_key = to_jsonb64(encryptor.key)
blob = {'blocks': blocks,
'key': block_key}
self.dirty = True
return blob
def file(mock_crypto_passthrough):
block_id = '4567'
blob = [{
'blocks': [{
'block': block_id,
'digest': digest(b''),
'size': 0
}],
'key': to_jsonb64(b'<dummy-key-00000000000000000001>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
sequence = [
(EBlockCreate(''), const(block_id)),
(EVlobCreate(blob),
const({
'id': '1234',
'read_trust_seed': '42',
'write_trust_seed': '43'
})),
]
return perform_sequence(sequence, File.create())
def test_find_matching_blocks(self, file):
vlob_id = '1234'
block_size = 4096
# Contents
contents = {}
total_length = 0
for index, length in enumerate([
block_size + 1, block_size - 1, block_size, 2 * block_size + 2,
2 * block_size - 2, 2 * block_size
]):
content = b''.join(
[str(random.randint(1, 9)).encode() for i in range(0, length)])
contents[index] = content
total_length += length
# Blocks
def generator():
i = 2000
while True:
yield str(i)
i += 1
gen = generator()
blocks = {}
block_contents = {}
block_id = 2000
for index, content in contents.items():
chunks = [
content[i:i + block_size]
for i in range(0, len(content), block_size)
]
if not chunks:
chunks = [b'']
sequence = []
for chunk in chunks:
encoded_chunk = to_jsonb64(chunk)
sequence.append((EBlockCreate(encoded_chunk),
lambda id=id: next(gen))) # TODO dirty
block_contents[str(block_id)] = encoded_chunk
block_id += 1
blocks[index] = perform_sequence(sequence,
file._build_file_blocks(content))
# Create file
blob = ejson_dumps([blocks[i] for i in range(0, len(blocks))]).encode()
blob = to_jsonb64(blob)
# All matching blocks
sequence = [(EVlobRead(vlob_id, '42', 1),
const({
'id': vlob_id,
'blob': blob,
'version': 1
}))]
matching_blocks = perform_sequence(sequence,
file._find_matching_blocks())
assert matching_blocks == {
'pre_excluded_blocks': [],
'pre_excluded_data': b'',
'pre_included_data': b'',
'included_blocks': [blocks[i] for i in range(0, len(blocks))],
'post_included_data': b'',
'post_excluded_data': b'',
'post_excluded_blocks': []
}
# With offset
delta = 10
offset = (blocks[0]['blocks'][0]['size'] +
blocks[0]['blocks'][1]['size'] +
blocks[1]['blocks'][0]['size'] +
blocks[2]['blocks'][0]['size'] - delta)
sequence = [(EVlobRead(vlob_id, '42', 1),
const({
'id': vlob_id,
'blob': blob,
'version': 1
})),
(EBlockRead('2003'),
const({
'content': block_contents['2003'],
'creation_date': '2012-01-01T00:00:00'
}))]
matching_blocks = perform_sequence(
sequence, file._find_matching_blocks(None, offset))
pre_excluded_data = contents[2][:blocks[2]['blocks'][0]['size'] -
delta]
pre_included_data = contents[2][-delta:]
assert matching_blocks == {
'pre_excluded_blocks': [blocks[0], blocks[1]],
'pre_excluded_data': pre_excluded_data,
'pre_included_data': pre_included_data,
'included_blocks': [blocks[i] for i in range(3, 6)],
'post_included_data': b'',
'post_excluded_data': b'',
'post_excluded_blocks': []
}
# With small size
delta = 10
size = 5
offset = (blocks[0]['blocks'][0]['size'] +
blocks[0]['blocks'][1]['size'] +
blocks[1]['blocks'][0]['size'] +
blocks[2]['blocks'][0]['size'] - delta)
sequence = [(EVlobRead(vlob_id, '42', 1),
const({
'id': vlob_id,
'blob': blob,
'version': 1
})),
(EBlockRead(id='2003'),
const({
'content': block_contents['2003'],
'creation_date': '2012-01-01T00:00:00'
}))]
matching_blocks = perform_sequence(
sequence, file._find_matching_blocks(size, offset))
pre_excluded_data = contents[2][:blocks[2]['blocks'][0]['size'] -
delta]
pre_included_data = contents[2][-delta:][:size]
post_excluded_data = contents[2][-delta:][size:]
assert matching_blocks == {
'pre_excluded_blocks': [blocks[0], blocks[1]],
'pre_excluded_data': pre_excluded_data,
'pre_included_data': pre_included_data,
'included_blocks': [],
'post_included_data': b'',
'post_excluded_data': post_excluded_data,
'post_excluded_blocks': [blocks[i] for i in range(3, 6)]
}
# With big size
delta = 10
size = delta
size += blocks[3]['blocks'][0]['size']
size += blocks[3]['blocks'][1]['size']
size += blocks[3]['blocks'][2]['size']
size += 2 * delta
offset = (blocks[0]['blocks'][0]['size'] +
blocks[0]['blocks'][1]['size'] +
blocks[1]['blocks'][0]['size'] +
blocks[2]['blocks'][0]['size'] - delta)
sequence = [(EVlobRead(vlob_id, '42', 1),
const({
'id': vlob_id,
'blob': blob,
'version': 1
})),
(EBlockRead('2003'),
const({
'content': block_contents['2003'],
'creation_date': '2012-01-01T00:00:00'
})),
(EBlockRead('2007'),
const({
'content': block_contents['2007'],
'creation_date': '2012-01-01T00:00:00'
}))]
matching_blocks = perform_sequence(
sequence, file._find_matching_blocks(size, offset))
pre_excluded_data = contents[2][:-delta]
pre_included_data = contents[2][-delta:]
post_included_data = contents[4][:2 * delta]
post_excluded_data = contents[4][:block_size][2 * delta:]
partial_block_4 = deepcopy(blocks[4])
del partial_block_4['blocks'][0]
assert matching_blocks == {
'pre_excluded_blocks': [blocks[0], blocks[1]],
'pre_excluded_data': pre_excluded_data,
'pre_included_data': pre_included_data,
'included_blocks': [blocks[3]],
'post_included_data': post_included_data,
'post_excluded_data': post_excluded_data,
'post_excluded_blocks': [partial_block_4, blocks[5]]
}
# With big size and no delta
size = blocks[3]['blocks'][0]['size']
size += blocks[3]['blocks'][1]['size']
size += blocks[3]['blocks'][2]['size']
offset = (blocks[0]['blocks'][0]['size'] +
blocks[0]['blocks'][1]['size'] +
blocks[1]['blocks'][0]['size'] +
blocks[2]['blocks'][0]['size'])
sequence = [
(EVlobRead(vlob_id, '42',
1), const({
'id': vlob_id,
'blob': blob,
'version': 1
})),
]
matching_blocks = perform_sequence(
sequence, file._find_matching_blocks(size, offset))
assert matching_blocks == {
'pre_excluded_blocks': [blocks[0], blocks[1], blocks[2]],
'pre_excluded_data': b'',
'pre_included_data': b'',
'included_blocks': [blocks[3]],
'post_included_data': b'',
'post_excluded_data': b'',
'post_excluded_blocks': [blocks[4], blocks[5]]
}
# With total size
sequence = [
(EVlobRead(vlob_id, '42',
1), const({
'id': vlob_id,
'blob': blob,
'version': 1
})),
]
matching_blocks = perform_sequence(
sequence, file._find_matching_blocks(total_length, 0))
assert matching_blocks == {
'pre_excluded_blocks': [],
'pre_excluded_data': b'',
'pre_included_data': b'',
'included_blocks': [blocks[i] for i in range(0, 6)],
'post_included_data': b'',
'post_excluded_data': b'',
'post_excluded_blocks': []
}
def test_commit(self, file):
vlob_id = '1234'
content = b'This is a test content.'
block_ids = ['4567', '5678', '6789']
new_vlob = {
'id': '2345',
'read_trust_seed': 'ABC',
'write_trust_seed': 'DEF'
}
# Original content
chunk_1 = content[:5]
chunk_2 = content[5:14]
chunk_3 = content[14:]
blob = [{
'blocks': [{
'block': block_ids[0],
'digest': digest(chunk_1),
'size': len(chunk_1)
}, {
'block': block_ids[1],
'digest': digest(chunk_2),
'size': len(chunk_2)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000003>')
}, {
'blocks': [{
'block': block_ids[2],
'digest': digest(chunk_3),
'size': len(chunk_3)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000004>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
# New content after truncate
new_chuck_2 = b'is a'
new_block_id = '7654'
new_blob = [{
'blocks': [{
'block': block_ids[0],
'digest': digest(chunk_1),
'size': len(chunk_1)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000003>')
}, {
'blocks': [{
'block': new_block_id,
'digest': digest(new_chuck_2),
'size': len(new_chuck_2)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000003>')
}]
new_blob = ejson_dumps(new_blob).encode()
new_blob = to_jsonb64(new_blob)
file.truncate(9)
sequence = [
(EVlobRead('1234', '42',
1), const({
'id': '1234',
'blob': blob,
'version': 1
})),
(EVlobRead('1234', '42',
1), const({
'id': '1234',
'blob': blob,
'version': 1
})),
(EBlockRead(block_ids[1]),
const({
'content': to_jsonb64(chunk_2),
'creation_date': '2012-01-01T00:00:00'
})), (EBlockCreate(to_jsonb64(new_chuck_2)), const(new_block_id)),
(EVlobUpdate(vlob_id, '43', 1, new_blob), noop),
(EVlobRead('1234', '42',
1), const({
'id': '1234',
'blob': new_blob,
'version': 1
})),
(EBlockDelete('5678'), conste(BlockNotFound('Block not found.'))),
(EBlockDelete('6789'), noop),
(EVlobRead('1234', '42',
1), const({
'id': '1234',
'blob': new_blob,
'version': 1
})), (EBlockSynchronize('4567'), const(True)),
(EBlockSynchronize('7654'), const(False)),
(EVlobSynchronize('1234'), const(new_vlob))
]
ret = perform_sequence(sequence, file.commit())
new_vlob['key'] = to_jsonb64(b'<dummy-key-00000000000000000002>')
assert ret == new_vlob
assert file.dirty is False
assert file.version == 1
def test_flush(self, file):
file.truncate(9)
file.write(b'IS', 5)
file.write(b'IS a nice test content.', 5)
file.dirty = False
file.version = 2
vlob_id = '1234'
content = b'This is a test content.'
block_ids = ['4567', '5678', '6789']
# Original content
chunk_1 = content[:5]
chunk_2 = content[5:14]
chunk_3 = content[14:]
blob = [{
'blocks': [{
'block': block_ids[0],
'digest': digest(chunk_1),
'size': len(chunk_1)
}, {
'block': block_ids[1],
'digest': digest(chunk_2),
'size': len(chunk_2)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000001>')
}, {
'blocks': [{
'block': block_ids[2],
'digest': digest(chunk_3),
'size': len(chunk_3)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000002>')
}]
blob = ejson_dumps(blob).encode()
blob = to_jsonb64(blob)
# New content after truncate
new_chuck_2 = b'is a'
new_block_id = '7654'
new_blob = [{
'blocks': [{
'block': block_ids[0],
'digest': digest(chunk_1),
'size': len(chunk_1)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000001>')
}, {
'blocks': [{
'block': new_block_id,
'digest': digest(new_chuck_2),
'size': len(new_chuck_2)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000003>')
}]
new_blob = ejson_dumps(new_blob).encode()
new_blob = to_jsonb64(new_blob)
# New content after write
new_block_2_id = '6543'
new_chunk_4 = b'IS a nice test content.'
new_blob_2 = [{
'blocks': [{
'block': block_ids[0],
'digest': digest(chunk_1),
'size': len(chunk_1)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000001>')
}, {
'blocks': [{
'block': new_block_2_id,
'digest': digest(new_chunk_4),
'size': len(new_chunk_4)
}],
'key':
to_jsonb64(b'<dummy-key-00000000000000000004>')
}]
new_blob_2 = ejson_dumps(new_blob_2).encode()
new_blob_2 = to_jsonb64(new_blob_2)
sequence = [
(
EVlobRead(vlob_id, '42', 2), # Get blocks
const({
'id': vlob_id,
'blob': blob,
'version': 2
})),
(
EVlobRead(vlob_id, '42', 2), # Matching blocks
const({
'id': vlob_id,
'blob': blob,
'version': 2
})),
(EBlockRead(block_ids[1]),
const({
'content': to_jsonb64(chunk_2),
'creation_date': '2012-01-01T00:00:00'
})),
(EBlockCreate(to_jsonb64(new_chuck_2)), const(new_block_id)),
(EVlobUpdate(vlob_id, '43', 3, new_blob), noop),
(
EVlobRead(vlob_id, '42', 3), # Matching blocks
const({
'id': vlob_id,
'blob': new_blob,
'version': 3
})),
(EBlockCreate(to_jsonb64(new_chunk_4)), const(new_block_2_id)),
(EVlobUpdate(vlob_id, '43', 3, new_blob_2), noop),
(EVlobRead(vlob_id, '42', 3),
const({
'id': vlob_id,
'blob': new_blob_2,
'version': 3
})),
(EBlockDelete('5678'), conste(BlockNotFound('Block not found.'))),
(EBlockDelete('6789'), noop),
]
ret = perform_sequence(sequence, file.flush())
assert ret is None
assert file.dirty is True
assert file.version == 2