async def test_realm_checkpoint(alice_workspace_storage):
aws = alice_workspace_storage
manifest = create_manifest(aws.device, LocalFileManifest)
assert await aws.get_realm_checkpoint() == 0
assert await aws.get_need_sync_entries() == (set(), set())
await aws.update_realm_checkpoint(11, {manifest.id: 22, EntryID(): 33})
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set(), set())
await aws.set_manifest(manifest.id, manifest, check_lock_status=False)
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set([manifest.id]), set())
await aws.set_manifest(manifest.id,
manifest.evolve(need_sync=False),
check_lock_status=False)
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set(), set())
await aws.update_realm_checkpoint(44, {manifest.id: 55, EntryID(): 66})
assert await aws.get_realm_checkpoint() == 44
assert await aws.get_need_sync_entries() == (set(), set([manifest.id]))
async def test_path_info_remote_loader_exceptions(monkeypatch, alice_workspace,
alice):
manifest, _ = await alice_workspace.transactions._get_manifest_from_path(
FsPath("/foo/bar"))
async with alice_workspace.local_storage.lock_entry_id(manifest.id):
await alice_workspace.local_storage.clear_manifest(manifest.id)
vanilla_file_manifest_deserialize = BaseRemoteManifest._deserialize
def mocked_file_manifest_deserialize(*args, **kwargs):
return vanilla_file_manifest_deserialize(
*args, **kwargs).evolve(**manifest_modifiers)
monkeypatch.setattr(BaseRemoteManifest, "_deserialize",
mocked_file_manifest_deserialize)
manifest_modifiers = {"id": EntryID()}
with pytest.raises(FSError) as exc:
await alice_workspace.path_info(FsPath("/foo/bar"))
assert f"Invalid entry ID: expected `{manifest.id}`, got `{manifest_modifiers['id']}`" in str(
exc.value)
manifest_modifiers = {"version": 4}
with pytest.raises(FSError) as exc:
await alice_workspace.path_info(FsPath("/foo/bar"))
assert "Invalid version: expected `1`, got `4`" in str(exc.value)
manifest_modifiers = {"author": DeviceID("mallory@pc1")}
with pytest.raises(FSError) as exc:
await alice_workspace.path_info(FsPath("/foo/bar"))
assert "Invalid author: expected `alice@dev1`, got `mallory@pc1`" in str(
exc.value)
def __init__(self) -> None:
super().__init__()
self.oracle = open(tmpdir / "oracle.txt", "w+b")
self.manifest = LocalFileManifest.new_placeholder(
DeviceID.new(), parent=EntryID(), blocksize=8
)
self.storage = Storage()
async def test_synchronization_step_transaction(alice_sync_transactions, type):
sync_transactions = alice_sync_transactions
synchronization_step = sync_transactions.synchronization_step
entry_id = sync_transactions.get_workspace_entry().id
# Sync a placeholder
manifest = await synchronization_step(entry_id)
# Acknowledge a successful synchronization
assert await synchronization_step(entry_id, manifest) is None
# Local change
if type == "file":
a_id, fd = await sync_transactions.file_create(FsPath("/a"))
await sync_transactions.fd_write(fd, b"abc", 0)
await sync_transactions.fd_close(fd)
else:
a_id = await sync_transactions.folder_create(FsPath("/a"))
# Sync parent with a placeholder child
manifest = await synchronization_step(entry_id)
children = []
async for child in sync_transactions.get_placeholder_children(manifest):
children.append(child)
(a_entry_id, ) = children
assert a_entry_id == a_id
# Sync child
if type == "file":
await synchronization_step(a_entry_id)
a_manifest = await synchronization_step(a_entry_id)
assert await synchronization_step(a_entry_id, a_manifest) is None
# Acknowledge the manifest
assert sorted(manifest.children) == ["a"]
assert await synchronization_step(entry_id, manifest) is None
# Local change
b_id = await sync_transactions.folder_create(FsPath("/b"))
# Remote change
children = {**manifest.children, "c": EntryID()}
manifest = manifest.evolve(version=5, children=children, author="b@b")
# Sync parent with a placeholder child
manifest = await synchronization_step(entry_id, manifest)
children = []
async for child in sync_transactions.get_placeholder_children(manifest):
children.append(child)
(b_entry_id, ) = children
assert b_entry_id == b_id
# Sync child
b_manifest = await synchronization_step(b_entry_id)
assert await synchronization_step(b_entry_id, b_manifest) is None
# Acknowledge the manifest
assert sorted(manifest.children) == ["a", "b", "c"]
assert await synchronization_step(entry_id, manifest) is None
async def test_unknown_workspace(alice_user_fs):
bad_wid = EntryID()
with pytest.raises(FSWorkspaceNotFoundError):
await alice_user_fs.workspace_start_reencryption(bad_wid)
with pytest.raises(FSWorkspaceNotFoundError):
await alice_user_fs.workspace_continue_reencryption(bad_wid)
async def _transactions_controlled_cb(started_cb):
async with WorkspaceStorage.run(alice, Path("/dummy"),
EntryID()) as local_storage:
file_transactions = await file_transactions_factory(
self.device,
alice_backend_cmds,
local_storage=local_storage)
await started_cb(file_transactions=file_transactions)
async def foo_txt(alice, alice_file_transactions):
local_storage = alice_file_transactions.local_storage
now = datetime(2000, 1, 2)
placeholder = LocalFileManifest.new_placeholder(alice.device_id,
parent=EntryID(),
now=now)
remote_v1 = placeholder.to_remote(author=alice.device_id, timestamp=now)
manifest = LocalFileManifest.from_remote(remote_v1)
async with local_storage.lock_entry_id(manifest.id):
await local_storage.set_manifest(manifest.id, manifest)
return File(local_storage, manifest)
async def make_workspace_dir_inconsistent(workspace: WorkspaceFS, dir: FsPath):
await workspace.mkdir(dir)
await workspace.touch(dir / "foo.txt")
rep_info = await workspace.transactions.entry_info(dir)
rep_manifest = await workspace.local_storage.get_manifest(rep_info["id"])
children = rep_manifest.children
children["newfail.txt"] = EntryID("b9295787-d9aa-6cbd-be27-1ff83ac72fa6")
rep_manifest.evolve(children=children)
async with workspace.local_storage.lock_manifest(rep_info["id"]):
await workspace.local_storage.set_manifest(rep_info["id"],
rep_manifest)
await workspace.sync()
def test_merge_folder_children():
m1 = EntryID()
m2 = EntryID()
a1 = {"a": m1}
a2 = {"a": m2}
b1 = {"b.txt": m1}
b2 = {"b.txt": m2}
c1 = {"c.tar.gz": m1}
c2 = {"c.tar.gz": m2}
# Empty folder
assert merge_folder_children({}, {}, {}, "a@a") == {}
# Adding children
assert merge_folder_children({}, a1, {}, "a@a") == a1
assert merge_folder_children({}, {}, a1, "a@a") == a1
assert merge_folder_children({}, a1, a1, "a@a") == a1
# Removing children
assert merge_folder_children(a1, {}, a1, "a@a") == {}
assert merge_folder_children(a1, a1, {}, "a@a") == {}
assert merge_folder_children(a1, {}, {}, "a@a") == {}
# Renaming children
assert merge_folder_children(a1, a1, b1, "a@a") == b1
assert merge_folder_children(a1, b1, a1, "a@a") == b1
assert merge_folder_children(a1, b1, b1, "a@a") == b1
# Conflicting renaming
result = merge_folder_children(a1, b1, c1, "a@a")
assert result == {"c (renamed by a@a).tar.gz": m1}
# Conflicting names
result = merge_folder_children({}, a1, a2, "a@a")
assert result == {"a": m2, "a (conflicting with a@a)": m1}
result = merge_folder_children({}, b1, b2, "a@a")
assert result == {"b.txt": m2, "b (conflicting with a@a).txt": m1}
result = merge_folder_children({}, c1, c2, "a@a")
assert result == {"c.tar.gz": m2, "c (conflicting with a@a).tar.gz": m1}
def test_merge_manifests_with_a_placeholder():
my_device = DeviceID("b@b")
other_device = DeviceID("a@a")
parent = EntryID()
m1 = LocalFolderManifest.new_placeholder(my_device, parent=parent)
m2 = merge_manifests(my_device, empty_filter, m1)
assert m2 == m1
v1 = m1.to_remote(author=my_device)
m2a = merge_manifests(my_device, empty_filter, m1, v1)
assert m2a == LocalFolderManifest.from_remote(v1, empty_filter)
m2b = m1.evolve_children_and_mark_updated({"a": EntryID()}, empty_filter)
m3b = merge_manifests(my_device, empty_filter, m2b, v1)
assert m3b == m2b.evolve(base=v1)
v2 = v1.evolve(version=2, author=other_device, children={"b": EntryID()})
m2c = m1.evolve_children_and_mark_updated({"a": EntryID()}, empty_filter)
m3c = merge_manifests(my_device, empty_filter, m2c, v2)
children = {**v2.children, **m2c.children}
assert m3c == m2c.evolve(base=v2, children=children, updated=m3c.updated)
def create_manifest(device,
type=LocalWorkspaceManifest,
use_legacy_none_author=False):
author = device.device_id
if type is LocalUserManifest:
manifest = LocalUserManifest.new_placeholder(author)
elif type is LocalWorkspaceManifest:
manifest = type.new_placeholder(author)
else:
manifest = type.new_placeholder(author, parent=EntryID())
if use_legacy_none_author:
base = manifest.base.evolve(author=None)
manifest = manifest.evolve(base=base)
return manifest
def test_merge_folder_manifests():
my_device = DeviceID("b@b")
other_device = DeviceID("a@a")
parent = EntryID()
v1 = LocalFolderManifest.new_placeholder(
my_device, parent=parent).to_remote(author=other_device)
# Initial base manifest
m1 = LocalFolderManifest.from_remote(v1, empty_filter)
assert merge_manifests(my_device, empty_filter, m1) == m1
# Local change
m2 = m1.evolve_children_and_mark_updated({"a": EntryID()}, empty_filter)
assert merge_manifests(my_device, empty_filter, m2) == m2
# Successful upload
v2 = m2.to_remote(author=my_device)
m3 = merge_manifests(my_device, empty_filter, m2, v2)
assert m3 == LocalFolderManifest.from_remote(v2, empty_filter)
# Two local changes
m4 = m3.evolve_children_and_mark_updated({"b": EntryID()}, empty_filter)
assert merge_manifests(my_device, empty_filter, m4) == m4
m5 = m4.evolve_children_and_mark_updated({"c": EntryID()}, empty_filter)
assert merge_manifests(my_device, empty_filter, m4) == m4
# M4 has been successfully uploaded
v3 = m4.to_remote(author=my_device)
m6 = merge_manifests(my_device, empty_filter, m5, v3)
assert m6 == m5.evolve(base=v3)
# The remote has changed
v4 = v3.evolve(version=4,
children={
"d": EntryID(),
**v3.children
},
author=other_device)
m7 = merge_manifests(my_device, empty_filter, m6, v4)
assert m7.base_version == 4
assert sorted(m7.children) == ["a", "b", "c", "d"]
assert m7.need_sync
# Successful upload
v5 = m7.to_remote(author=my_device)
m8 = merge_manifests(my_device, empty_filter, m7, v5)
assert m8 == LocalFolderManifest.from_remote(v5, empty_filter)
# The remote has changed
v6 = v5.evolve(version=6,
children={
"e": EntryID(),
**v5.children
},
author=other_device)
m9 = merge_manifests(my_device, empty_filter, m8, v6)
assert m9 == LocalFolderManifest.from_remote(v6, empty_filter)
async def test_timestamped_storage(alice_workspace_storage):
timestamp = now()
aws = alice_workspace_storage
taws = aws.to_timestamped(timestamp)
assert taws.timestamp == timestamp
assert taws.device == aws.device
assert taws.path == aws.path
assert taws.workspace_id == aws.workspace_id
assert taws.manifest_storage is None
assert taws.block_storage == aws.block_storage
assert taws.chunk_storage == aws.chunk_storage
with pytest.raises(FSError):
await taws.set_chunk("chunk id", "data")
with pytest.raises(FSError):
await taws.clear_chunk("chunk id")
with pytest.raises(FSError):
await taws.clear_manifest("manifest id")
with pytest.raises(FSError):
await taws.run_vacuum()
with pytest.raises(FSError):
await taws.get_need_sync_entries()
with pytest.raises(FSError):
await taws.get_realm_checkpoint()
with pytest.raises(FSError):
await taws.clear_memory_cache("flush")
with pytest.raises(FSLocalMissError):
await taws.get_manifest(EntryID())
manifest = create_manifest(aws.device)
with pytest.raises(FSError):
await taws.set_manifest(manifest.id, manifest)
manifest = manifest.evolve(need_sync=False)
async with taws.lock_entry_id(manifest.id):
await taws.set_manifest(manifest.id, manifest)
assert await taws.get_manifest(manifest.id) == manifest
# No-op
await taws.ensure_manifest_persistent(manifest.id)
async def test_realm_checkpoint(alice, alice_user_storage,
initial_user_manifest_state):
aws = alice_user_storage
user_manifest_id = alice.user_manifest_id
# Brand new user storage contains user manifest placeholder
assert await aws.get_realm_checkpoint() == 0
assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
# Modified entries not in storage should be ignored
await aws.update_realm_checkpoint(1, {EntryID(): 2})
assert await aws.get_realm_checkpoint() == 1
assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
# Another device updated the user manifest remotly
await aws.update_realm_checkpoint(2, {user_manifest_id: 1})
assert await aws.get_realm_checkpoint() == 2
assert await aws.get_need_sync_entries() == ({user_manifest_id},
{user_manifest_id})
# Load the up to date version of the user manifest
manifest_v1 = initial_user_manifest_state.get_user_manifest_v1_for_device(
alice)
await aws.set_user_manifest(manifest_v1)
assert await aws.get_realm_checkpoint() == 2
assert await aws.get_need_sync_entries() == (set(), set())
# Provide new remote changes
await aws.update_realm_checkpoint(3, {user_manifest_id: 2})
assert await aws.get_realm_checkpoint() == 3
assert await aws.get_need_sync_entries() == (set(), {user_manifest_id})
# Provide new local changes too
manifest_v1_modified = manifest_v1.evolve(need_sync=True)
await aws.set_user_manifest(manifest_v1_modified)
assert await aws.get_realm_checkpoint() == 3
assert await aws.get_need_sync_entries() == ({user_manifest_id},
{user_manifest_id})
# Checkpoint's remote version should be ignored if manifest base version is greater
manifest_v2 = manifest_v1_modified.evolve(base=manifest_v1.base.evolve(
version=4))
await aws.set_user_manifest(manifest_v2)
assert await aws.get_realm_checkpoint() == 3
assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
def generate_new_device(
organization_addr: BackendOrganizationAddr,
profile: UserProfile = UserProfile.STANDARD,
device_id: Optional[DeviceID] = None,
human_handle: Optional[HumanHandle] = None,
device_label: Optional[str] = None,
signing_key: Optional[SigningKey] = None,
private_key: Optional[PrivateKey] = None,
) -> LocalDevice:
return LocalDevice(
organization_addr=organization_addr,
device_id=device_id or DeviceID.new(),
device_label=device_label,
human_handle=human_handle,
signing_key=signing_key or SigningKey.generate(),
private_key=private_key or PrivateKey.generate(),
profile=profile,
user_manifest_id=EntryID(),
user_manifest_key=SecretKey.generate(),
local_symkey=SecretKey.generate(),
)
async def get_need_sync_entries(self) -> Tuple[Set[EntryID], Set[EntryID]]:
"""
Raises: Nothing !
"""
remote_changes = set()
local_changes = {
entry_id for entry_id, manifest in self._cache.items() if manifest.need_sync
}
async with self._open_cursor() as cursor:
cursor.execute(
"SELECT vlob_id, need_sync, base_version, remote_version "
"FROM vlobs WHERE need_sync = 1 OR base_version != remote_version"
)
for manifest_id, need_sync, bv, rv in cursor.fetchall():
manifest_id = EntryID(manifest_id)
if need_sync:
local_changes.add(manifest_id)
if bv != rv:
remote_changes.add(manifest_id)
return local_changes, remote_changes
def test_merge_file_manifests():
my_device = DeviceID("b@b")
other_device = DeviceID("a@a")
parent = EntryID()
v1 = LocalFileManifest.new_placeholder(
my_device, parent=parent).to_remote(author=other_device)
def evolve(m, n):
chunk = Chunk.new(0, n).evolve_as_block(b"a" * n)
blocks = ((chunk, ), )
return m1.evolve_and_mark_updated(size=n, blocks=blocks)
# Initial base manifest
m1 = LocalFileManifest.from_remote(v1)
assert merge_manifests(my_device, empty_filter, m1) == m1
# Local change
m2 = evolve(m1, 1)
assert merge_manifests(my_device, empty_filter, m2) == m2
# Successful upload
v2 = m2.to_remote(author=my_device)
m3 = merge_manifests(my_device, empty_filter, m2, v2)
assert m3 == LocalFileManifest.from_remote(v2)
# Two local changes
m4 = evolve(m3, 2)
assert merge_manifests(my_device, empty_filter, m4) == m4
m5 = evolve(m4, 3)
assert merge_manifests(my_device, empty_filter, m4) == m4
# M4 has been successfully uploaded
v3 = m4.to_remote(author=my_device)
m6 = merge_manifests(my_device, empty_filter, m5, v3)
assert m6 == m5.evolve(base=v3)
# The remote has changed
v4 = v3.evolve(version=4, size=0, author=other_device)
with pytest.raises(FSFileConflictError):
merge_manifests(my_device, empty_filter, m6, v4)
async def init(self):
nonlocal tentative
tentative += 1
await reset_testbed()
self.device = alice
await self.start_transactions()
self.file_transactions = self.transactions_controller.file_transactions
self.local_storage = self.file_transactions.local_storage
self.fresh_manifest = LocalFileManifest.new_placeholder(
alice.device_id, parent=EntryID())
self.entry_id = self.fresh_manifest.id
async with self.local_storage.lock_entry_id(self.entry_id):
await self.local_storage.set_manifest(self.entry_id,
self.fresh_manifest)
self.fd = self.local_storage.create_file_descriptor(
self.fresh_manifest)
self.file_oracle_path = tmpdir / f"oracle-test-{tentative}.txt"
self.file_oracle_fd = os.open(self.file_oracle_path,
os.O_RDWR | os.O_CREAT)
async def alice_transaction_local_storage(alice, persistent_mockup):
async with WorkspaceStorage.run(alice, Path("/dummy"),
EntryID()) as storage:
yield storage
def workspace_id():
return EntryID()
def test_complete_scenario():
storage = Storage()
with freeze_time("2000-01-01"):
base = manifest = LocalFileManifest.new_placeholder(
DeviceID.new(), parent=EntryID(), blocksize=16
)
assert manifest == base.evolve(size=0)
with freeze_time("2000-01-02") as t2:
manifest = storage.write(manifest, b"Hello ", 0)
assert storage.read(manifest, 6, 0) == b"Hello "
((chunk0,),) = manifest.blocks
assert manifest == base.evolve(size=6, blocks=((chunk0,),), updated=t2)
assert chunk0 == Chunk(id=chunk0.id, start=0, stop=6, raw_offset=0, raw_size=6, access=None)
assert storage[chunk0.id] == b"Hello "
with freeze_time("2000-01-03") as t3:
manifest = storage.write(manifest, b"world !", 6)
assert storage.read(manifest, 13, 0) == b"Hello world !"
((_, chunk1),) = manifest.blocks
assert manifest == base.evolve(size=13, blocks=((chunk0, chunk1),), updated=t3)
assert chunk1 == Chunk(id=chunk1.id, start=6, stop=13, raw_offset=6, raw_size=7, access=None)
assert storage[chunk1.id] == b"world !"
with freeze_time("2000-01-04") as t4:
manifest = storage.write(manifest, b"\n More kontent", 13)
assert storage.read(manifest, 27, 0) == b"Hello world !\n More kontent"
(_, _, chunk2), (chunk3,) = manifest.blocks
assert storage[chunk2.id] == b"\n M"
assert storage[chunk3.id] == b"ore kontent"
assert manifest == base.evolve(
size=27, blocks=((chunk0, chunk1, chunk2), (chunk3,)), updated=t4
)
with freeze_time("2000-01-05") as t5:
manifest = storage.write(manifest, b"c", 20)
assert storage.read(manifest, 27, 0) == b"Hello world !\n More content"
chunk4, chunk5, chunk6 = manifest.blocks[1]
assert chunk3.id == chunk4.id == chunk6.id
assert storage[chunk5.id] == b"c"
assert manifest == base.evolve(
size=27, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6)), updated=t5
)
with freeze_time("2000-01-06") as t6:
manifest = storage.resize(manifest, 40)
expected = b"Hello world !\n More content" + b"\x00" * 13
assert storage.read(manifest, 40, 0) == expected
(_, _, _, chunk7), (chunk8,) = manifest.blocks[1:]
assert storage[chunk7.id] == b"\x00" * 5
assert storage[chunk8.id] == b"\x00" * 8
assert manifest == base.evolve(
size=40,
blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6, chunk7), (chunk8,)),
updated=t6,
)
with freeze_time("2000-01-07") as t7:
manifest = storage.resize(manifest, 25)
expected = b"Hello world !\n More conte"
assert storage.read(manifest, 25, 0) == expected
((_, _, chunk9),) = manifest.blocks[1:]
assert chunk9.id == chunk6.id
assert manifest == base.evolve(
size=25, blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk9)), updated=t7
)
with freeze_time("2000-01-08"):
assert not manifest.is_reshaped()
manifest = storage.reshape(manifest)
expected = b"Hello world !\n More conte"
assert storage.read(manifest, 25, 0) == expected
assert manifest.is_reshaped()
(chunk10,), (chunk11,) = manifest.blocks
assert storage[chunk10.id] == b"Hello world !\n M"
assert storage[chunk11.id] == b"ore conte"
assert manifest == base.evolve(size=25, blocks=((chunk10,), (chunk11,)), updated=t7)