def test_perform_undelete(app, alice_identity, file): vlob = {'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43'} blob = [{ 'blocks': [{ 'block': '4567', 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_delete(EDelete('/foo')) sequence = [ (EIdentityGet(), const(alice_identity)), (EVlobRead(vlob['id'], vlob['read_trust_seed']), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EVlobList(), const([])), (EVlobRead(vlob['id'], vlob['read_trust_seed'], 1), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EVlobDelete('2345'), conste(VlobNotFound('Vlob not found.'))) ] ret = perform_sequence(sequence, eff) eff = app.perform_undelete(EUndelete('2345')) sequence = [(EIdentityGet(), const(alice_identity))] ret = perform_sequence(sequence, eff) assert ret is None
async def read(self, id: str): func = partial(self.s3.get_object, Bucket=self.s3_bucket, Key=id) try: obj = await get_event_loop().run_in_executor(None, func) except (S3ClientError, S3EndpointConnectionError) as exc: raise BlockNotFound(str(exc)) return obj['Body'].read()
def test_discard(self, file): content = b'This is a test content.' block_ids = ['4567', '5678', '6789'] # Original content chunk_1 = content[:5] chunk_2 = content[5:14] chunk_3 = content[14:] blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }, { 'block': block_ids[1], 'digest': digest(chunk_2), 'size': len(chunk_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': digest(chunk_3), 'size': len(chunk_3) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000004>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) # Already synchronized sequence = [ (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EBlockDelete('5678'), noop), (EBlockDelete('6789'), noop), (EVlobDelete('1234'), conste(VlobNotFound('Block not found.')) ) # TODO vlob OR block exceptin ] ret = perform_sequence(sequence, file.discard()) assert ret is False # Not already synchronized file.dirty = True file.version = 0 sequence = [(EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), noop), (EBlockDelete('5678'), noop), (EBlockDelete('6789'), noop), (EVlobDelete('1234'), noop)] ret = perform_sequence(sequence, file.discard()) assert ret is True assert file.dirty is False
def perform_block_delete(self, intent): self.last_modified = arrow.utcnow() try: del self.blocks[intent.id] except KeyError: try: del self.block_cache[intent.id] except KeyError: raise BlockNotFound('Block not found.')
async def read(self, id: str): route = '%s/%s' % (self.url, id) async with aiohttp.ClientSession() as session: async with session.get(route) as resp: if resp.status == 200: return await resp.read() elif resp.status == 404: raise BlockNotFound('Block %s not found' % id) else: raise BlockError(await resp.text())
def test_perform_block_read(app, app_no_cache): local_content = 'foo' eff = app.perform_block_create(EBlockCreate(local_content)) block_id = perform_sequence([], eff) # Read block in new blocks eff = app.perform_block_read(EBlockRead(block_id)) block = perform_sequence([], eff) assert sorted(list(block.keys())) == ['content', 'id'] assert block['id'] assert block['content'] == local_content remote_content = b'bar' # Read remote block assert app.block_cache.currsize == 0 eff = app.perform_block_read(EBlockRead('123')) sequence = [(EBackendBlockRead('123'), const(Block('123', remote_content)))] block = perform_sequence(sequence, eff) assert sorted(list(block.keys())) == ['content', 'id'] assert block['id'] assert block['content'] == remote_content assert app.block_cache.currsize == 1 # Read remote block with cache disabled assert app_no_cache.block_cache.currsize == 0 eff = app_no_cache.perform_block_read(EBlockRead('123')) sequence = [(EBackendBlockRead('123'), const(Block('123', remote_content)))] block = perform_sequence(sequence, eff) assert sorted(list(block.keys())) == ['content', 'id'] assert block['id'] assert block['content'] == remote_content assert app_no_cache.block_cache.currsize == 0 # Read block in cache eff = app.perform_block_read(EBlockRead('123')) block = perform_sequence([], eff) assert sorted(list(block.keys())) == ['content', 'id'] assert block['id'] assert block['content'] == remote_content # Delete block from cache eff = app.perform_block_delete(EBlockDelete('123')) perform_sequence([], eff) # Not found eff = app.perform_block_read(EBlockRead('123')) sequence = [(EBackendBlockRead('123'), conste(BlockNotFound('Block not found.')))] with pytest.raises(BlockNotFound): block = perform_sequence(sequence, eff) eff = app.perform_block_read(EBlockRead('123')) sequence = [(EBackendBlockRead('123'), conste(BlockError('Block error.')) ) # TODO keep it? usefull with multiple backends... ] with pytest.raises(BlockNotFound): block = perform_sequence(sequence, eff)
def perform_block_read(self, intent): try: return self.blocks[intent.id] except KeyError: try: return self.block_cache[intent.id] except KeyError: try: block = yield Effect(EBackendBlockRead(intent.id)) block = {'id': block.id, 'content': block.content} except (BlockNotFound, BlockError): raise BlockNotFound('Block not found.') try: self.block_cache[intent.id] = block except ValueError: pass # Value too large if cache is disabled return block
def test_perform_dustbin_show(app, alice_identity, file): with freeze_time('2012-01-01') as frozen_datetime: vlob = { 'id': '2345', 'read_trust_seed': '42', 'write_trust_seed': '43' } blob = [{ 'blocks': [{ 'block': '4567', 'digest': digest(b''), 'size': 0 }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) eff = app.perform_delete(EDelete('/foo')) sequence = [ (EIdentityGet(), const(alice_identity)), (EVlobRead(vlob['id'], vlob['read_trust_seed']), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EVlobList(), const([])), (EVlobRead(vlob['id'], vlob['read_trust_seed'], 1), const({ 'id': vlob['id'], 'blob': blob, 'version': 1 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EVlobDelete('2345'), conste(VlobNotFound('Vlob not found.'))), ] perform_sequence(sequence, eff) eff = app.perform_dustbin_show(EDustbinShow()) sequence = [(EIdentityGet(), const(alice_identity))] dustbin = perform_sequence(sequence, eff) vlob['path'] = '/foo' vlob['removed_date'] = frozen_datetime().isoformat() vlob['key'] = to_jsonb64(b'<dummy-key-00000000000000000002>') assert dustbin == [vlob]
def test_commit(self, file): vlob_id = '1234' content = b'This is a test content.' block_ids = ['4567', '5678', '6789'] new_vlob = { 'id': '2345', 'read_trust_seed': 'ABC', 'write_trust_seed': 'DEF' } # Original content chunk_1 = content[:5] chunk_2 = content[5:14] chunk_3 = content[14:] blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }, { 'block': block_ids[1], 'digest': digest(chunk_2), 'size': len(chunk_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': digest(chunk_3), 'size': len(chunk_3) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000004>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) # New content after truncate new_chuck_2 = b'is a' new_block_id = '7654' new_blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }, { 'blocks': [{ 'block': new_block_id, 'digest': digest(new_chuck_2), 'size': len(new_chuck_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }] new_blob = ejson_dumps(new_blob).encode() new_blob = to_jsonb64(new_blob) file.truncate(9) sequence = [ (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': blob, 'version': 1 })), (EBlockRead(block_ids[1]), const({ 'content': to_jsonb64(chunk_2), 'creation_date': '2012-01-01T00:00:00' })), (EBlockCreate(to_jsonb64(new_chuck_2)), const(new_block_id)), (EVlobUpdate(vlob_id, '43', 1, new_blob), noop), (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': new_blob, 'version': 1 })), (EBlockDelete('5678'), conste(BlockNotFound('Block not found.'))), (EBlockDelete('6789'), noop), (EVlobRead('1234', '42', 1), const({ 'id': '1234', 'blob': new_blob, 'version': 1 })), (EBlockSynchronize('4567'), const(True)), (EBlockSynchronize('7654'), const(False)), (EVlobSynchronize('1234'), const(new_vlob)) ] ret = perform_sequence(sequence, file.commit()) new_vlob['key'] = to_jsonb64(b'<dummy-key-00000000000000000002>') assert ret == new_vlob assert file.dirty is False assert file.version == 1
def test_flush(self, file): file.truncate(9) file.write(b'IS', 5) file.write(b'IS a nice test content.', 5) file.dirty = False file.version = 2 vlob_id = '1234' content = b'This is a test content.' block_ids = ['4567', '5678', '6789'] # Original content chunk_1 = content[:5] chunk_2 = content[5:14] chunk_3 = content[14:] blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }, { 'block': block_ids[1], 'digest': digest(chunk_2), 'size': len(chunk_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': digest(chunk_3), 'size': len(chunk_3) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000002>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) # New content after truncate new_chuck_2 = b'is a' new_block_id = '7654' new_blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }, { 'blocks': [{ 'block': new_block_id, 'digest': digest(new_chuck_2), 'size': len(new_chuck_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }] new_blob = ejson_dumps(new_blob).encode() new_blob = to_jsonb64(new_blob) # New content after write new_block_2_id = '6543' new_chunk_4 = b'IS a nice test content.' new_blob_2 = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }, { 'blocks': [{ 'block': new_block_2_id, 'digest': digest(new_chunk_4), 'size': len(new_chunk_4) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000004>') }] new_blob_2 = ejson_dumps(new_blob_2).encode() new_blob_2 = to_jsonb64(new_blob_2) sequence = [ ( EVlobRead(vlob_id, '42', 2), # Get blocks const({ 'id': vlob_id, 'blob': blob, 'version': 2 })), ( EVlobRead(vlob_id, '42', 2), # Matching blocks const({ 'id': vlob_id, 'blob': blob, 'version': 2 })), (EBlockRead(block_ids[1]), const({ 'content': to_jsonb64(chunk_2), 'creation_date': '2012-01-01T00:00:00' })), (EBlockCreate(to_jsonb64(new_chuck_2)), const(new_block_id)), (EVlobUpdate(vlob_id, '43', 3, new_blob), noop), ( EVlobRead(vlob_id, '42', 3), # Matching blocks const({ 'id': vlob_id, 'blob': new_blob, 'version': 3 })), (EBlockCreate(to_jsonb64(new_chunk_4)), const(new_block_2_id)), (EVlobUpdate(vlob_id, '43', 3, new_blob_2), noop), (EVlobRead(vlob_id, '42', 3), const({ 'id': vlob_id, 'blob': new_blob_2, 'version': 3 })), (EBlockDelete('5678'), conste(BlockNotFound('Block not found.'))), (EBlockDelete('6789'), noop), ] ret = perform_sequence(sequence, file.flush()) assert ret is None assert file.dirty is True assert file.version == 2
def test_restore(self, file): vlob_id = '1234' content = b'This is a test content.' block_ids = ['4567', '5678', '6789'] # Original content chunk_1 = content[:5] chunk_2 = content[5:14] chunk_3 = content[14:] blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }, { 'block': block_ids[1], 'digest': digest(chunk_2), 'size': len(chunk_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': digest(chunk_3), 'size': len(chunk_3) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000002>') }] blob = ejson_dumps(blob).encode() blob = to_jsonb64(blob) # New content new_chuck_2 = b'is A test' new_block_id = '7654' new_blob = [{ 'blocks': [{ 'block': block_ids[0], 'digest': digest(chunk_1), 'size': len(chunk_1) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000001>') }, { 'blocks': [{ 'block': new_block_id, 'digest': digest(new_chuck_2), 'size': len(new_chuck_2) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000003>') }, { 'blocks': [{ 'block': block_ids[2], 'digest': digest(chunk_3), 'size': len(chunk_3) }], 'key': to_jsonb64(b'<dummy-key-00000000000000000002>') }] new_blob = ejson_dumps(new_blob).encode() new_blob = to_jsonb64(new_blob) # Restore not commited file with version = 1 file.dirty = False with pytest.raises(FileError): perform_sequence([], file.restore()) assert file.dirty is False file.dirty = True # Restore commited file with version = 1 file.dirty = False file.version = 1 with pytest.raises(FileError): perform_sequence([], file.restore()) assert file.dirty is False # Restore not commited file with version = current version file.dirty = True file.version = 5 with pytest.raises(FileError): perform_sequence([], file.restore(6)) assert file.dirty is True # Restore commited file with version = current version file.dirty = False file.version = 6 with pytest.raises(FileError): perform_sequence([], file.restore(6)) assert file.dirty is False # Restore previous version sequence = [ ( EVlobRead(vlob_id, '42', 6), # Discard const({ 'id': vlob_id, 'blob': blob, 'version': 6 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EBlockDelete('5678'), noop), (EBlockDelete('6789'), noop), (EVlobDelete('1234'), noop), (EVlobRead('1234', '42', 5), const({ 'id': vlob_id, 'blob': new_blob, 'version': 5 })), (EVlobUpdate(vlob_id, '43', 7, new_blob), noop) ] ret = perform_sequence(sequence, file.restore()) assert ret is None assert file.dirty is True assert file.version == 6 # Restore specific version sequence = [ (EVlobRead(vlob_id, '42', 7), const({ 'id': vlob_id, 'blob': new_blob, 'version': 7 })), (EBlockDelete('4567'), conste(BlockNotFound('Block not found.'))), (EBlockDelete('7654'), noop), (EBlockDelete('6789'), noop), (EVlobDelete('1234'), noop), (EVlobRead('1234', '42', 2), const({ 'id': vlob_id, 'blob': blob, 'version': 2 })), (EVlobUpdate(vlob_id, '43', 7, blob), noop) ] ret = perform_sequence(sequence, file.restore(2)) assert ret is None assert file.dirty is True assert file.version == 6