async def get_dirty_block(self, block_id: BlockID) -> bytes:
return await self.chunk_storage.get_chunk(ChunkID(block_id))
async def set_clean_block(self, block_id: BlockID, block: bytes) -> None:
assert isinstance(block_id, BlockID)
return await self.block_storage.set_chunk(ChunkID(block_id), block)
async def clear_clean_block(self, block_id: BlockID) -> None:
assert isinstance(block_id, BlockID)
try:
await self.block_storage.clear_chunk(ChunkID(block_id))
except FSLocalMissError:
pass
def test_local_file_manifest():
from parsec.api.data.manifest import BlockAccess
from parsec.core.types.manifest import (
_RsLocalFileManifest,
LocalFileManifest,
_PyLocalFileManifest,
Chunk,
)
assert LocalFileManifest is _RsLocalFileManifest
def _assert_local_file_manifest_eq(py,
rs,
exclude_base=False,
exclude_id=False):
assert isinstance(py, _PyLocalFileManifest)
assert isinstance(rs, _RsLocalFileManifest)
if not exclude_base:
assert py.base == rs.base
assert py.need_sync == rs.need_sync
assert py.updated == rs.updated
assert py.size == rs.size
assert py.blocksize == rs.blocksize
assert len(py.blocks) == len(rs.blocks)
assert isinstance(rs.blocks, type(py.blocks))
if len(py.blocks):
assert isinstance(rs.blocks[0], type(py.blocks[0]))
if not exclude_id:
assert py.id == rs.id
assert py.created == rs.created
assert py.base_version == rs.base_version
assert py.is_placeholder == rs.is_placeholder
assert py.is_reshaped() == rs.is_reshaped()
for (b1, b2) in zip(sorted(py.blocks), sorted(rs.blocks)):
assert len(b1) == len(b2)
assert all(
isinstance(c2, Chunk) and c1.id == c2.id and c1.start == c2.
start and c1.stop == c2.stop and c1.raw_offset == c2.raw_offset
and c1.raw_size == c2.raw_size and c1.access == c2.access
for (c1, c2) in zip(b1, b2))
def _assert_file_manifest_eq(py, rs):
assert py.author == rs.author
assert py.parent == rs.parent
assert py.version == rs.version
assert py.size == rs.size
assert py.blocksize == rs.blocksize
assert py.timestamp == rs.timestamp
assert py.created == rs.created
assert py.updated == rs.updated
assert len(py.blocks) == len(rs.blocks)
assert isinstance(rs.blocks, type(py.blocks))
assert all(
isinstance(b2, BlockAccess) and b1.id == b2.id
and b1.offset == b2.offset and b1.size == b2.size
for (b1, b2) in zip(sorted(py.blocks), sorted(rs.blocks)))
kwargs = {
"base":
FileManifest(
author=DeviceID("user@device"),
id=EntryID.new(),
parent=EntryID.new(),
version=42,
size=1337,
blocksize=85,
timestamp=pendulum.now(),
created=pendulum.now(),
updated=pendulum.now(),
blocks=(BlockAccess(
id=BlockID.new(),
key=SecretKey.generate(),
offset=0,
size=1024,
digest=HashDigest.from_data(b"a"),
), ),
),
"need_sync":
True,
"updated":
pendulum.now(),
"size":
42,
"blocksize":
64,
"blocks": ((
Chunk(
id=ChunkID.new(),
start=0,
stop=250,
raw_offset=0,
raw_size=512,
access=BlockAccess(
id=BlockID.new(),
key=SecretKey.generate(),
offset=0,
size=512,
digest=HashDigest.from_data(b"aa"),
),
),
Chunk(id=ChunkID.new(),
start=0,
stop=250,
raw_offset=250,
raw_size=250,
access=None),
), ),
}
py_lfm = _PyLocalFileManifest(**kwargs)
rs_lfm = LocalFileManifest(**kwargs)
_assert_local_file_manifest_eq(py_lfm, rs_lfm)
kwargs = {
"base":
kwargs["base"].evolve(
**{
"author":
DeviceID("a@b"),
"id":
EntryID.new(),
"parent":
EntryID.new(),
"version":
1337,
"size":
4096,
"blocksize":
512,
"timestamp":
pendulum.now(),
"created":
pendulum.now(),
"updated":
pendulum.now(),
"blocks": (BlockAccess(
id=BlockID.new(),
key=SecretKey.generate(),
offset=64,
size=2048,
digest=HashDigest.from_data(b"b"),
), ),
}),
"need_sync":
False,
"updated":
pendulum.now(),
"size":
2048,
"blocksize":
1024,
"blocks": ((
Chunk(
id=ChunkID.new(),
start=0,
stop=1024,
raw_offset=0,
raw_size=1024,
access=BlockAccess(
id=BlockID.new(),
key=SecretKey.generate(),
offset=0,
size=1024,
digest=HashDigest.from_data(b"bb"),
),
),
Chunk(
id=ChunkID.new(),
start=1024,
stop=2048,
raw_offset=1024,
raw_size=1024,
access=None,
),
), ),
}
py_lfm = py_lfm.evolve(**kwargs)
rs_lfm = rs_lfm.evolve(**kwargs)
_assert_local_file_manifest_eq(py_lfm, rs_lfm)
sk = SecretKey.generate()
py_enc = py_lfm.dump_and_encrypt(sk)
rs_enc = rs_lfm.dump_and_encrypt(sk)
# Decrypt rust encrypted with Python and vice versa
lfm1 = _PyLocalFileManifest.decrypt_and_load(rs_enc, sk)
lfm2 = LocalFileManifest.decrypt_and_load(py_enc, sk)
assert isinstance(lfm1, LocalFileManifest)
assert isinstance(lfm2, LocalFileManifest)
assert lfm1 == lfm2
py_lfm = py_lfm.evolve(**{"size": 1337})
rs_lfm = rs_lfm.evolve(**{"size": 1337})
_assert_local_file_manifest_eq(py_lfm, rs_lfm)
with pytest.raises(AssertionError):
py_lfm.assert_integrity()
with pytest.raises(AssertionError):
rs_lfm.assert_integrity()
assert py_lfm.to_stats() == rs_lfm.to_stats()
assert py_lfm.parent == rs_lfm.parent
assert py_lfm.get_chunks(0) == rs_lfm.get_chunks(0)
assert py_lfm.get_chunks(1000) == rs_lfm.get_chunks(1000)
assert py_lfm.asdict() == rs_lfm.asdict()
di = DeviceID("a@b")
ts = pendulum.now()
kwargs = {
"size":
1024,
"blocksize":
1024,
"blocks": ((Chunk(
id=ChunkID.new(),
start=0,
stop=1024,
raw_offset=0,
raw_size=1024,
access=BlockAccess(
id=BlockID.new(),
key=SecretKey.generate(),
offset=0,
size=1024,
digest=HashDigest.from_data(b"bb"),
),
), ), ),
}
py_lfm = py_lfm.evolve(**kwargs)
rs_lfm = rs_lfm.evolve(**kwargs)
_assert_local_file_manifest_eq(py_lfm, rs_lfm)
py_rfm = py_lfm.to_remote(author=di, timestamp=ts)
rs_rfm = rs_lfm.to_remote(author=di, timestamp=ts)
_assert_file_manifest_eq(py_rfm, rs_rfm)
py_lfm2 = _PyLocalFileManifest.from_remote(py_rfm)
rs_lfm2 = LocalFileManifest.from_remote(rs_rfm)
_assert_local_file_manifest_eq(py_lfm2,
rs_lfm2,
exclude_base=True,
exclude_id=True)
py_lfm2 = _PyLocalFileManifest.from_remote_with_local_context(
remote=py_rfm,
prevent_sync_pattern=r".+",
local_manifest=py_lfm2,
timestamp=ts)
rs_lfm2 = LocalFileManifest.from_remote_with_local_context(
remote=rs_rfm,
prevent_sync_pattern=r".+",
local_manifest=rs_lfm2,
timestamp=ts)
assert py_lfm.match_remote(py_rfm) == rs_lfm.match_remote(rs_rfm)
py_lfm = py_lfm.evolve_and_mark_updated(timestamp=ts, size=4096)
rs_lfm = rs_lfm.evolve_and_mark_updated(timestamp=ts, size=4096)
_assert_local_file_manifest_eq(py_lfm,
rs_lfm,
exclude_base=True,
exclude_id=True)
with pytest.raises(TypeError) as excinfo:
py_lfm.evolve_and_mark_updated(timestamp=ts, need_sync=True)
assert str(excinfo.value) == "Unexpected keyword argument `need_sync`"
with pytest.raises(TypeError) as excinfo:
rs_lfm.evolve_and_mark_updated(timestamp=ts, need_sync=True)
assert str(excinfo.value) == "Unexpected keyword argument `need_sync`"
ei = EntryID.new()
# Without blocksize
py_lfm = _PyLocalFileManifest.new_placeholder(author=di,
parent=ei,
timestamp=ts)
rs_lfm = LocalFileManifest.new_placeholder(author=di,
parent=ei,
timestamp=ts)
_assert_local_file_manifest_eq(py_lfm,
rs_lfm,
exclude_base=True,
exclude_id=True)
# With blocksize
py_lfm = _PyLocalFileManifest.new_placeholder(author=di,
parent=ei,
timestamp=ts,
blocksize=1024)
rs_lfm = LocalFileManifest.new_placeholder(author=di,
parent=ei,
timestamp=ts,
blocksize=1024)
_assert_local_file_manifest_eq(py_lfm,
rs_lfm,
exclude_base=True,
exclude_id=True)