async def test_realm_checkpoint(alice_workspace_storage):
aws = alice_workspace_storage
manifest = create_manifest(aws.device, LocalFileManifest)
assert await aws.get_realm_checkpoint() == 0
assert await aws.get_need_sync_entries() == (set(), set())
await aws.update_realm_checkpoint(11, {manifest.id: 22, EntryID.new(): 33})
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set(), set())
await aws.set_manifest(manifest.id, manifest, check_lock_status=False)
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set([manifest.id]), set())
await aws.set_manifest(manifest.id,
manifest.evolve(need_sync=False),
check_lock_status=False)
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set(), set())
await aws.update_realm_checkpoint(44, {manifest.id: 55, EntryID.new(): 66})
assert await aws.get_realm_checkpoint() == 44
assert await aws.get_need_sync_entries() == (set(), set([manifest.id]))
def test_merge_manifests_with_a_placeholder(alice, bob):
timestamp = alice.timestamp()
my_device = alice.device_id
other_device = bob.device_id
parent = EntryID.new()
m1 = LocalFolderManifest.new_placeholder(my_device,
parent=parent,
timestamp=timestamp)
m2 = merge_manifests(my_device, timestamp, empty_pattern, m1)
assert m2 == m1
v1 = m1.to_remote(author=my_device, timestamp=timestamp)
m2a = merge_manifests(my_device, timestamp, empty_pattern, m1, v1)
assert m2a == LocalFolderManifest.from_remote(v1, empty_pattern)
m2b = m1.evolve_children_and_mark_updated({EntryName("a"): EntryID.new()},
empty_pattern,
timestamp=timestamp)
m3b = merge_manifests(my_device, timestamp, empty_pattern, m2b, v1)
assert m3b == m2b.evolve(base=v1)
v2 = v1.evolve(version=2,
author=other_device,
children={EntryName("b"): EntryID.new()})
m2c = m1.evolve_children_and_mark_updated({EntryName("a"): EntryID.new()},
empty_pattern,
timestamp=timestamp)
m3c = merge_manifests(my_device, timestamp, empty_pattern, m2c, v2)
children = {**v2.children, **m2c.children}
assert m3c == m2c.evolve(base=v2, children=children, updated=m3c.updated)
async def init(self):
nonlocal tentative
tentative += 1
await self.reset_all()
await self.start_backend()
self.device = alice
await self.start_transactions()
self.file_transactions = self.transactions_controller.file_transactions
self.local_storage = self.file_transactions.local_storage
self.fresh_manifest = LocalFileManifest.new_placeholder(
alice.device_id,
parent=EntryID.new(),
timestamp=alice.timestamp())
self.entry_id = self.fresh_manifest.id
async with self.local_storage.lock_entry_id(self.entry_id):
await self.local_storage.set_manifest(self.entry_id,
self.fresh_manifest)
self.fd = self.local_storage.create_file_descriptor(
self.fresh_manifest)
self.file_oracle_path = tmpdir / f"oracle-test-{tentative}.txt"
self.file_oracle_fd = os.open(self.file_oracle_path,
os.O_RDWR | os.O_CREAT)
async def test_link_file_unknown_org(
aqtbot, core_config, gui_factory, autoclose_dialog, running_backend, alice
):
password = "******"
save_device_with_password_in_config(core_config.config_dir, alice, password)
# Cheating a bit but it does not matter, we just want a link that appears valid with
# an unknown organization
org_addr = BackendOrganizationAddr.build(
running_backend.addr, OrganizationID("UnknownOrg"), alice.organization_addr.root_verify_key
)
file_link = BackendOrganizationFileLinkAddr.build(
organization_addr=org_addr, workspace_id=EntryID.new(), encrypted_path=b"<whatever>"
)
gui = await gui_factory(core_config=core_config, start_arg=file_link.to_url())
lw = gui.test_get_login_widget()
assert len(autoclose_dialog.dialogs) == 1
assert autoclose_dialog.dialogs[0][0] == "Error"
assert autoclose_dialog.dialogs[0][1] == translate(
"TEXT_FILE_LINK_NOT_IN_ORG_organization"
).format(organization="UnknownOrg")
def _devices_listed():
assert lw.widget.layout().count() > 0
await aqtbot.wait_until(_devices_listed)
accounts_w = lw.widget.layout().itemAt(0).widget()
assert accounts_w
assert isinstance(accounts_w, LoginPasswordInputWidget)
async def test_path_info_remote_loader_exceptions(monkeypatch, alice_workspace,
alice):
manifest, _ = await alice_workspace.transactions._get_manifest_from_path(
FsPath("/foo/bar"))
async with alice_workspace.local_storage.lock_entry_id(manifest.id):
await alice_workspace.local_storage.clear_manifest(manifest.id)
vanilla_file_manifest_deserialize = BaseRemoteManifest._deserialize
def mocked_file_manifest_deserialize(*args, **kwargs):
return vanilla_file_manifest_deserialize(
*args, **kwargs).evolve(**manifest_modifiers)
monkeypatch.setattr(BaseRemoteManifest, "_deserialize",
mocked_file_manifest_deserialize)
manifest_modifiers = {"id": EntryID.new()}
with pytest.raises(FSError) as exc:
await alice_workspace.path_info(FsPath("/foo/bar"))
assert f"Invalid entry ID: expected `{manifest.id}`, got `{manifest_modifiers['id']}`" in str(
exc.value)
manifest_modifiers = {"version": 4}
with pytest.raises(FSError) as exc:
await alice_workspace.path_info(FsPath("/foo/bar"))
assert "Invalid version: expected `1`, got `4`" in str(exc.value)
manifest_modifiers = {"author": DeviceID("mallory@pc1")}
with pytest.raises(FSError) as exc:
await alice_workspace.path_info(FsPath("/foo/bar"))
assert "Invalid author: expected `alice@dev1`, got `mallory@pc1`" in str(
exc.value)
async def test_synchronization_step_transaction(alice_sync_transactions, type):
sync_transactions = alice_sync_transactions
synchronization_step = sync_transactions.synchronization_step
entry_id = sync_transactions.get_workspace_entry().id
# Sync a placeholder
manifest = await synchronization_step(entry_id)
# Acknowledge a successful synchronization
assert await synchronization_step(entry_id, manifest) is None
# Local change
if type == "file":
a_id, fd = await sync_transactions.file_create(FsPath("/a"))
await sync_transactions.fd_write(fd, b"abc", 0)
await sync_transactions.fd_close(fd)
else:
a_id = await sync_transactions.folder_create(FsPath("/a"))
# Sync parent with a placeholder child
manifest = await synchronization_step(entry_id)
children = []
async for child in sync_transactions.get_placeholder_children(manifest):
children.append(child)
a_entry_id, = children
assert a_entry_id == a_id
# Sync child
if type == "file":
await synchronization_step(a_entry_id)
a_manifest = await synchronization_step(a_entry_id)
assert await synchronization_step(a_entry_id, a_manifest) is None
# Acknowledge the manifest
assert sorted(manifest.children) == ["a"]
assert await synchronization_step(entry_id, manifest) is None
# Local change
b_id = await sync_transactions.folder_create(FsPath("/b"))
# Remote change
children = {**manifest.children, "c": EntryID.new()}
manifest = manifest.evolve(version=5, children=children, author="b@b")
# Sync parent with a placeholder child
manifest = await synchronization_step(entry_id, manifest)
children = []
async for child in sync_transactions.get_placeholder_children(manifest):
children.append(child)
b_entry_id, = children
assert b_entry_id == b_id
# Sync child
b_manifest = await synchronization_step(b_entry_id)
assert await synchronization_step(b_entry_id, b_manifest) is None
# Acknowledge the manifest
assert sorted(manifest.children) == ["a", "b", "c"]
assert await synchronization_step(entry_id, manifest) is None
async def _transactions_controlled_cb(started_cb):
async with WorkspaceStorage.run(
tmp_path / f"file_operations-{tentative}", alice,
EntryID.new()) as local_storage:
async with file_transactions_factory(
self.device,
local_storage=local_storage) as file_transactions:
await started_cb(file_transactions=file_transactions)
async def _transactions_controlled_cb(started_cb):
async with WorkspaceStorage.run(
alice, Path("/dummy"), EntryID.new()) as local_storage:
file_transactions = await file_transactions_factory(
self.device,
alice_backend_cmds,
local_storage=local_storage)
await started_cb(file_transactions=file_transactions)
async def test_unknown_workspace(alice_user_fs):
bad_wid = EntryID.new()
with pytest.raises(FSWorkspaceNotFoundError):
await alice_user_fs.workspace_start_reencryption(bad_wid)
with pytest.raises(FSWorkspaceNotFoundError):
await alice_user_fs.workspace_continue_reencryption(bad_wid)
def __init__(self) -> None:
super().__init__()
self.oracle = open(tmpdir / "oracle.txt", "w+b")
self.manifest = LocalFileManifest.new_placeholder(
alice.device_id,
parent=EntryID.new(),
blocksize=8,
timestamp=alice.timestamp())
self.storage = Storage()
async def foo_txt(alice, alice_file_transactions):
local_storage = alice_file_transactions.local_storage
now = datetime(2000, 1, 2)
placeholder = LocalFileManifest.new_placeholder(alice.device_id,
parent=EntryID.new(),
timestamp=now)
remote_v1 = placeholder.to_remote(author=alice.device_id, timestamp=now)
manifest = LocalFileManifest.from_remote(remote_v1)
async with local_storage.lock_entry_id(manifest.id):
await local_storage.set_manifest(manifest.id, manifest)
return File(local_storage, manifest)
async def test_realm_checkpoint(alice_workspace_storage):
aws = alice_workspace_storage
manifest = create_manifest(aws.device, LocalFileManifest)
assert await aws.get_realm_checkpoint() == 0
# Workspace storage starts with a speculative workspace manifest placeholder
assert await aws.get_need_sync_entries() == ({aws.workspace_id}, set())
workspace_manifest = create_manifest(aws.device, LocalWorkspaceManifest)
base = workspace_manifest.to_remote(aws.device.device_id,
timestamp=aws.device.timestamp())
workspace_manifest = workspace_manifest.evolve(base=base, need_sync=False)
await aws.set_manifest(aws.workspace_id,
workspace_manifest,
check_lock_status=False)
assert await aws.get_realm_checkpoint() == 0
assert await aws.get_need_sync_entries() == (set(), set())
await aws.update_realm_checkpoint(11, {manifest.id: 22, EntryID.new(): 33})
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set(), set())
await aws.set_manifest(manifest.id, manifest, check_lock_status=False)
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set([manifest.id]), set())
await aws.set_manifest(manifest.id,
manifest.evolve(need_sync=False),
check_lock_status=False)
assert await aws.get_realm_checkpoint() == 11
assert await aws.get_need_sync_entries() == (set(), set())
await aws.update_realm_checkpoint(44, {manifest.id: 55, EntryID.new(): 66})
assert await aws.get_realm_checkpoint() == 44
assert await aws.get_need_sync_entries() == (set(), set([manifest.id]))
def test_merge_folder_children():
m1 = EntryID.new()
m2 = EntryID.new()
a1 = {"a": m1}
a2 = {"a": m2}
b1 = {"b.txt": m1}
b2 = {"b.txt": m2}
c1 = {"c.tar.gz": m1}
c2 = {"c.tar.gz": m2}
# Empty folder
assert merge_folder_children({}, {}, {}, "a@a") == {}
# Adding children
assert merge_folder_children({}, a1, {}, "a@a") == a1
assert merge_folder_children({}, {}, a1, "a@a") == a1
assert merge_folder_children({}, a1, a1, "a@a") == a1
# Removing children
assert merge_folder_children(a1, {}, a1, "a@a") == {}
assert merge_folder_children(a1, a1, {}, "a@a") == {}
assert merge_folder_children(a1, {}, {}, "a@a") == {}
# Renaming children
assert merge_folder_children(a1, a1, b1, "a@a") == b1
assert merge_folder_children(a1, b1, a1, "a@a") == b1
assert merge_folder_children(a1, b1, b1, "a@a") == b1
# Conflicting renaming
result = merge_folder_children(a1, b1, c1, "a@a")
assert result == {"c (renamed by a@a).tar.gz": m1}
# Conflicting names
result = merge_folder_children({}, a1, a2, "a@a")
assert result == {"a": m2, "a (conflicting with a@a)": m1}
result = merge_folder_children({}, b1, b2, "a@a")
assert result == {"b.txt": m2, "b (conflicting with a@a).txt": m1}
result = merge_folder_children({}, c1, c2, "a@a")
assert result == {"c.tar.gz": m2, "c (conflicting with a@a).tar.gz": m1}
def test_merge_manifests_with_a_placeholder():
my_device = DeviceID("b@b")
other_device = DeviceID("a@a")
parent = EntryID.new()
m1 = LocalFolderManifest.new_placeholder(my_device, parent=parent)
m2 = merge_manifests(my_device, empty_pattern, m1)
assert m2 == m1
v1 = m1.to_remote(author=my_device)
m2a = merge_manifests(my_device, empty_pattern, m1, v1)
assert m2a == LocalFolderManifest.from_remote(v1, empty_pattern)
m2b = m1.evolve_children_and_mark_updated({"a": EntryID.new()}, empty_pattern)
m3b = merge_manifests(my_device, empty_pattern, m2b, v1)
assert m3b == m2b.evolve(base=v1)
v2 = v1.evolve(version=2, author=other_device, children={"b": EntryID.new()})
m2c = m1.evolve_children_and_mark_updated({"a": EntryID.new()}, empty_pattern)
m3c = merge_manifests(my_device, empty_pattern, m2c, v2)
children = {**v2.children, **m2c.children}
assert m3c == m2c.evolve(base=v2, children=children, updated=m3c.updated)
def create_manifest(device,
type=LocalWorkspaceManifest,
use_legacy_none_author=False):
author = device.device_id
if type is LocalUserManifest:
manifest = LocalUserManifest.new_placeholder(author)
elif type is LocalWorkspaceManifest:
manifest = type.new_placeholder(author)
else:
manifest = type.new_placeholder(author, parent=EntryID.new())
if use_legacy_none_author:
base = manifest.base.evolve(author=None)
manifest = manifest.evolve(base=base)
return manifest
async def test_timestamped_storage(alice_workspace_storage):
timestamp = now()
aws = alice_workspace_storage
taws = aws.to_timestamped(timestamp)
assert taws.timestamp == timestamp
assert taws.device == aws.device
assert taws.path == aws.path
assert taws.workspace_id == aws.workspace_id
assert taws.manifest_storage is None
assert taws.block_storage == aws.block_storage
assert taws.chunk_storage == aws.chunk_storage
with pytest.raises(FSError):
await taws.set_chunk("chunk id", "data")
with pytest.raises(FSError):
await taws.clear_chunk("chunk id")
with pytest.raises(FSError):
await taws.clear_manifest("manifest id")
with pytest.raises(FSError):
await taws.run_vacuum()
with pytest.raises(FSError):
await taws.get_need_sync_entries()
with pytest.raises(FSError):
await taws.get_realm_checkpoint()
with pytest.raises(FSError):
await taws.clear_memory_cache("flush")
with pytest.raises(FSLocalMissError):
await taws.get_manifest(EntryID.new())
manifest = create_manifest(aws.device)
with pytest.raises(FSError):
await taws.set_manifest(manifest.id, manifest)
manifest = manifest.evolve(need_sync=False)
async with taws.lock_entry_id(manifest.id):
await taws.set_manifest(manifest.id, manifest)
assert await taws.get_manifest(manifest.id) == manifest
# No-op
await taws.ensure_manifest_persistent(manifest.id)
def test_merge_file_manifests(alice, bob):
timestamp = alice.timestamp()
my_device = alice.device_id
other_device = bob.device_id
parent = EntryID.new()
v1 = LocalFileManifest.new_placeholder(my_device,
parent=parent,
timestamp=timestamp).to_remote(
author=other_device,
timestamp=timestamp)
def evolve(m, n):
chunk = Chunk.new(0, n).evolve_as_block(b"a" * n)
blocks = ((chunk, ), )
return m1.evolve_and_mark_updated(size=n,
blocks=blocks,
timestamp=timestamp)
# Initial base manifest
m1 = LocalFileManifest.from_remote(v1)
assert merge_manifests(my_device, timestamp, empty_pattern, m1) == m1
# Local change
m2 = evolve(m1, 1)
assert merge_manifests(my_device, timestamp, empty_pattern, m2) == m2
# Successful upload
v2 = m2.to_remote(author=my_device, timestamp=timestamp)
m3 = merge_manifests(my_device, timestamp, empty_pattern, m2, v2)
assert m3 == LocalFileManifest.from_remote(v2)
# Two local changes
m4 = evolve(m3, 2)
assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4
m5 = evolve(m4, 3)
assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4
# M4 has been successfully uploaded
v3 = m4.to_remote(author=my_device, timestamp=timestamp)
m6 = merge_manifests(my_device, timestamp, empty_pattern, m5, v3)
assert m6 == m5.evolve(base=v3)
# The remote has changed
v4 = v3.evolve(version=4, size=0, author=other_device)
with pytest.raises(FSFileConflictError):
merge_manifests(my_device, timestamp, empty_pattern, m6, v4)
async def test_realm_checkpoint(alice, alice_user_storage,
initial_user_manifest_state):
aws = alice_user_storage
user_manifest_id = alice.user_manifest_id
# Brand new user storage contains user manifest placeholder
assert await aws.get_realm_checkpoint() == 0
assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
# Modified entries not in storage should be ignored
await aws.update_realm_checkpoint(1, {EntryID.new(): 2})
assert await aws.get_realm_checkpoint() == 1
assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
# Another device updated the user manifest remotly
await aws.update_realm_checkpoint(2, {user_manifest_id: 1})
assert await aws.get_realm_checkpoint() == 2
assert await aws.get_need_sync_entries() == ({user_manifest_id},
{user_manifest_id})
# Load the up to date version of the user manifest
manifest_v1 = initial_user_manifest_state.get_user_manifest_v1_for_device(
alice)
await aws.set_user_manifest(manifest_v1)
assert await aws.get_realm_checkpoint() == 2
assert await aws.get_need_sync_entries() == (set(), set())
# Provide new remote changes
await aws.update_realm_checkpoint(3, {user_manifest_id: 2})
assert await aws.get_realm_checkpoint() == 3
assert await aws.get_need_sync_entries() == (set(), {user_manifest_id})
# Provide new local changes too
manifest_v1_modified = manifest_v1.evolve(need_sync=True)
await aws.set_user_manifest(manifest_v1_modified)
assert await aws.get_realm_checkpoint() == 3
assert await aws.get_need_sync_entries() == ({user_manifest_id},
{user_manifest_id})
# Checkpoint's remote version should be ignored if manifest base version is greater
manifest_v2 = manifest_v1_modified.evolve(base=manifest_v1.base.evolve(
version=4))
await aws.set_user_manifest(manifest_v2)
assert await aws.get_realm_checkpoint() == 3
assert await aws.get_need_sync_entries() == ({user_manifest_id}, set())
def test_merge_folder_manifests():
my_device = DeviceID("b@b")
other_device = DeviceID("a@a")
parent = EntryID.new()
v1 = LocalFolderManifest.new_placeholder(my_device, parent=parent).to_remote(
author=other_device
)
# Initial base manifest
m1 = LocalFolderManifest.from_remote(v1, empty_pattern)
assert merge_manifests(my_device, empty_pattern, m1) == m1
# Local change
m2 = m1.evolve_children_and_mark_updated({"a": EntryID.new()}, empty_pattern)
assert merge_manifests(my_device, empty_pattern, m2) == m2
# Successful upload
v2 = m2.to_remote(author=my_device)
m3 = merge_manifests(my_device, empty_pattern, m2, v2)
assert m3 == LocalFolderManifest.from_remote(v2, empty_pattern)
# Two local changes
m4 = m3.evolve_children_and_mark_updated({"b": EntryID.new()}, empty_pattern)
assert merge_manifests(my_device, empty_pattern, m4) == m4
m5 = m4.evolve_children_and_mark_updated({"c": EntryID.new()}, empty_pattern)
assert merge_manifests(my_device, empty_pattern, m4) == m4
# M4 has been successfully uploaded
v3 = m4.to_remote(author=my_device)
m6 = merge_manifests(my_device, empty_pattern, m5, v3)
assert m6 == m5.evolve(base=v3)
# The remote has changed
v4 = v3.evolve(version=4, children={"d": EntryID.new(), **v3.children}, author=other_device)
m7 = merge_manifests(my_device, empty_pattern, m6, v4)
assert m7.base_version == 4
assert sorted(m7.children) == ["a", "b", "c", "d"]
assert m7.need_sync
# Successful upload
v5 = m7.to_remote(author=my_device)
m8 = merge_manifests(my_device, empty_pattern, m7, v5)
assert m8 == LocalFolderManifest.from_remote(v5, empty_pattern)
# The remote has changed
v6 = v5.evolve(version=6, children={"e": EntryID.new(), **v5.children}, author=other_device)
m9 = merge_manifests(my_device, empty_pattern, m8, v6)
assert m9 == LocalFolderManifest.from_remote(v6, empty_pattern)
async def _do_folder_stat(workspace_fs, path, default_selection):
stats = {}
dir_stat = await workspace_fs.path_info(path)
# Retrieve children info, this is not an atomic operation so our view
# on the folder might be slightly non-causal (typically if a file is
# created during while we do the loop it won't appear in the final result,
# but later update on existing files will appear).
# We consider this fine enough given a change in the folder should lead
# to an event from the corefs which in turn will re-trigger this stat code
for child in dir_stat["children"]:
try:
child_stat = await workspace_fs.path_info(path / child)
except FSFileNotFoundError:
# The child entry as been concurrently removed, just ignore it
continue
except FSRemoteManifestNotFound:
# Cannot get informations about this child entry, this can occur if
# if the manifest is inconsistent (broken data or signature).
child_stat = {"type": "inconsistency", "id": EntryID.new()}
stats[child] = child_stat
return path, dir_stat["id"], stats, default_selection
def generate_new_device(
organization_addr: BackendOrganizationAddr,
device_id: Optional[DeviceID] = None,
profile: UserProfile = UserProfile.STANDARD,
human_handle: Optional[HumanHandle] = None,
device_label: Optional[DeviceLabel] = None,
signing_key: Optional[SigningKey] = None,
private_key: Optional[PrivateKey] = None,
) -> LocalDevice:
return LocalDevice(
organization_addr=organization_addr,
device_id=device_id or DeviceID.new(),
device_label=device_label,
human_handle=human_handle,
signing_key=signing_key or SigningKey.generate(),
private_key=private_key or PrivateKey.generate(),
profile=profile,
user_manifest_id=EntryID.new(),
user_manifest_key=SecretKey.generate(),
local_symkey=SecretKey.generate(),
)
async def test_link_file_unknown_workspace(
aqtbot, core_config, gui_factory, autoclose_dialog, running_backend, alice
):
password = "******"
save_device_with_password_in_config(core_config.config_dir, alice, password)
file_link = BackendOrganizationFileLinkAddr.build(
organization_addr=alice.organization_addr,
workspace_id=EntryID.new(),
encrypted_path=b"<whatever>",
)
gui = await gui_factory(core_config=core_config, start_arg=file_link.to_url())
lw = gui.test_get_login_widget()
def _password_prompt():
assert len(autoclose_dialog.dialogs) == 0
lpi_w = lw.widget.layout().itemAt(0).widget()
assert isinstance(lpi_w, LoginPasswordInputWidget)
await aqtbot.wait_until(_password_prompt)
lpi_w = lw.widget.layout().itemAt(0).widget()
await aqtbot.key_clicks(lpi_w.line_edit_password, password)
await aqtbot.wait_until(lambda: lpi_w.line_edit_password.text() == password)
tabw = gui.test_get_tab()
async with aqtbot.wait_signals([lw.login_with_password_clicked, tabw.logged_in]):
aqtbot.mouse_click(lpi_w.button_login, QtCore.Qt.LeftButton)
def _error_shown():
assert len(autoclose_dialog.dialogs) == 1
print(autoclose_dialog.dialogs)
assert autoclose_dialog.dialogs[0] == (
"Error",
"You do not have access to the workspace containing the file. It may not have been shared with you.",
)
await aqtbot.wait_until(_error_shown)
async def alice_transaction_local_storage(alice, persistent_mockup):
async with WorkspaceStorage.run(alice, Path("/dummy"),
EntryID.new()) as storage:
yield storage
def test_complete_scenario():
storage = Storage()
with freeze_time("2000-01-01"):
base = manifest = LocalFileManifest.new_placeholder(
DeviceID.new(), parent=EntryID.new(), blocksize=16)
assert manifest == base.evolve(size=0)
with freeze_time("2000-01-02") as t2:
manifest = storage.write(manifest, b"Hello ", 0)
assert storage.read(manifest, 6, 0) == b"Hello "
(chunk0, ), = manifest.blocks
assert manifest == base.evolve(size=6, blocks=((chunk0, ), ), updated=t2)
assert chunk0 == Chunk(id=chunk0.id,
start=0,
stop=6,
raw_offset=0,
raw_size=6,
access=None)
assert storage[chunk0.id] == b"Hello "
with freeze_time("2000-01-03") as t3:
manifest = storage.write(manifest, b"world !", 6)
assert storage.read(manifest, 13, 0) == b"Hello world !"
(_, chunk1), = manifest.blocks
assert manifest == base.evolve(size=13,
blocks=((chunk0, chunk1), ),
updated=t3)
assert chunk1 == Chunk(id=chunk1.id,
start=6,
stop=13,
raw_offset=6,
raw_size=7,
access=None)
assert storage[chunk1.id] == b"world !"
with freeze_time("2000-01-04") as t4:
manifest = storage.write(manifest, b"\n More kontent", 13)
assert storage.read(manifest, 27, 0) == b"Hello world !\n More kontent"
(_, _, chunk2), (chunk3, ) = manifest.blocks
assert storage[chunk2.id] == b"\n M"
assert storage[chunk3.id] == b"ore kontent"
assert manifest == base.evolve(size=27,
blocks=((chunk0, chunk1, chunk2),
(chunk3, )),
updated=t4)
with freeze_time("2000-01-05") as t5:
manifest = storage.write(manifest, b"c", 20)
assert storage.read(manifest, 27, 0) == b"Hello world !\n More content"
chunk4, chunk5, chunk6 = manifest.blocks[1]
assert chunk3.id == chunk4.id == chunk6.id
assert storage[chunk5.id] == b"c"
assert manifest == base.evolve(size=27,
blocks=((chunk0, chunk1, chunk2),
(chunk4, chunk5, chunk6)),
updated=t5)
with freeze_time("2000-01-06") as t6:
manifest = storage.resize(manifest, 40)
expected = b"Hello world !\n More content" + b"\x00" * 13
assert storage.read(manifest, 40, 0) == expected
(_, _, _, chunk7), (chunk8, ) = manifest.blocks[1:]
assert storage[chunk7.id] == b"\x00" * 5
assert storage[chunk8.id] == b"\x00" * 8
assert manifest == base.evolve(
size=40,
blocks=((chunk0, chunk1, chunk2), (chunk4, chunk5, chunk6, chunk7),
(chunk8, )),
updated=t6,
)
with freeze_time("2000-01-07") as t7:
manifest = storage.resize(manifest, 25)
expected = b"Hello world !\n More conte"
assert storage.read(manifest, 25, 0) == expected
(_, _, chunk9), = manifest.blocks[1:]
assert chunk9.id == chunk6.id
assert manifest == base.evolve(size=25,
blocks=((chunk0, chunk1, chunk2),
(chunk4, chunk5, chunk9)),
updated=t7)
with freeze_time("2000-01-08"):
assert not manifest.is_reshaped()
manifest = storage.reshape(manifest)
expected = b"Hello world !\n More conte"
assert storage.read(manifest, 25, 0) == expected
assert manifest.is_reshaped()
(chunk10, ), (chunk11, ) = manifest.blocks
assert storage[chunk10.id] == b"Hello world !\n M"
assert storage[chunk11.id] == b"ore conte"
assert manifest == base.evolve(size=25,
blocks=((chunk10, ), (chunk11, )),
updated=t7)
def test_merge_folder_manifests(alice, bob):
timestamp = alice.timestamp()
my_device = alice.device_id
other_device = bob.device_id
parent = EntryID.new()
v1 = LocalFolderManifest.new_placeholder(my_device,
parent=parent,
timestamp=timestamp).to_remote(
author=other_device,
timestamp=timestamp)
# Initial base manifest
m1 = LocalFolderManifest.from_remote(v1, empty_pattern)
assert merge_manifests(my_device, timestamp, empty_pattern, m1) == m1
# Local change
m2 = m1.evolve_children_and_mark_updated({EntryName("a"): EntryID.new()},
empty_pattern,
timestamp=timestamp)
assert merge_manifests(my_device, timestamp, empty_pattern, m2) == m2
# Successful upload
v2 = m2.to_remote(author=my_device, timestamp=timestamp)
m3 = merge_manifests(my_device, timestamp, empty_pattern, m2, v2)
assert m3 == LocalFolderManifest.from_remote(v2, empty_pattern)
# Two local changes
m4 = m3.evolve_children_and_mark_updated({EntryName("b"): EntryID.new()},
empty_pattern,
timestamp=timestamp)
assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4
m5 = m4.evolve_children_and_mark_updated({EntryName("c"): EntryID.new()},
empty_pattern,
timestamp=timestamp)
assert merge_manifests(my_device, timestamp, empty_pattern, m4) == m4
# M4 has been successfully uploaded
v3 = m4.to_remote(author=my_device, timestamp=timestamp)
m6 = merge_manifests(my_device, timestamp, empty_pattern, m5, v3)
assert m6 == m5.evolve(base=v3)
# The remote has changed
v4 = v3.evolve(version=4,
children={
EntryName("d"): EntryID.new(),
**v3.children
},
author=other_device)
m7 = merge_manifests(my_device, timestamp, empty_pattern, m6, v4)
assert m7.base_version == 4
assert sorted(m7.children) == [
EntryName("a"),
EntryName("b"),
EntryName("c"),
EntryName("d")
]
assert m7.need_sync
# Successful upload
v5 = m7.to_remote(author=my_device, timestamp=timestamp)
m8 = merge_manifests(my_device, timestamp, empty_pattern, m7, v5)
assert m8 == LocalFolderManifest.from_remote(v5, empty_pattern)
# The remote has changed
v6 = v5.evolve(version=6,
children={
EntryName("e"): EntryID.new(),
**v5.children
},
author=other_device)
m9 = merge_manifests(my_device, timestamp, empty_pattern, m8, v6)
assert m9 == LocalFolderManifest.from_remote(v6, empty_pattern)
def test_merge_folder_manifests_with_concurrent_remote_change(
local_change, remote_change, alice, bob):
timestamp = alice.timestamp()
my_device = alice.device_id
other_device = bob.device_id
parent = EntryID.new()
foo_txt = EntryID.new()
remote_manifest_v1 = (LocalFolderManifest.new_placeholder(
my_device, parent=parent,
timestamp=timestamp).evolve(children={
EntryName("foo.txt"): foo_txt
}).to_remote(author=my_device, timestamp=timestamp))
prevent_sync_pattern = re.compile(r".*\.tmp\Z")
# Load the manifest in local
local_manifest = LocalFolderManifest.from_remote(
remote_manifest_v1, prevent_sync_pattern=prevent_sync_pattern)
# In local, `foo.txt` is renamed
if local_change == "rename":
foo_txt_new_name = EntryName("foo2.txt")
else:
assert local_change == "prevent_sync_rename"
foo_txt_new_name = EntryName("foo.txt.tmp")
local_manifest = local_manifest.evolve_children_and_mark_updated(
data={
EntryName("foo.txt"): None,
foo_txt_new_name: foo_txt
},
prevent_sync_pattern=prevent_sync_pattern,
timestamp=timestamp,
)
# In remote, a change also occurs
if remote_change == "same_entry_moved":
remote_manifest_v2_children = {
EntryName("bar.txt"):
remote_manifest_v1.children[EntryName("foo.txt")]
}
else:
assert remote_change == "new_entry_added"
remote_manifest_v2_children = {
**remote_manifest_v1.children,
EntryName("bar.txt"): EntryID.new(),
}
remote_manifest_v2 = remote_manifest_v1.evolve(
author=other_device,
version=remote_manifest_v1.version + 1,
children=remote_manifest_v2_children,
)
# Now merging should detect the duplication
merged_manifest = merge_manifests(
local_author=my_device,
timestamp=timestamp,
prevent_sync_pattern=prevent_sync_pattern,
local_manifest=local_manifest,
remote_manifest=remote_manifest_v2,
force_apply_pattern=False,
)
if remote_change == "same_entry_moved":
assert list(merged_manifest.children) == [EntryName("bar.txt")]
else:
assert remote_change == "new_entry_added"
if local_change == "rename":
assert sorted(merged_manifest.children) == [
EntryName("bar.txt"),
EntryName("foo2.txt")
]
else:
assert local_change == "prevent_sync_rename"
assert sorted(merged_manifest.children) == [
EntryName("bar.txt"),
EntryName("foo.txt.tmp"),
]
async def test_share_unknown(running_backend, alice_user_fs, bob):
wid = EntryID.new()
with pytest.raises(FSWorkspaceNotFoundError):
await alice_user_fs.workspace_share(wid, bob.user_id, WorkspaceRole.MANAGER)
def workspace_id():
return EntryID.new()
def test_merge_folder_children(preferred_language, suffix):
m1 = EntryID.new()
m2 = EntryID.new()
m3 = EntryID.new()
a1 = {EntryName("a"): m1}
a2 = {EntryName("a"): m2}
b1 = {EntryName("b.txt"): m1}
b2 = {EntryName("b.txt"): m2}
c1 = {EntryName("c.tar.gz"): m1}
c2 = {EntryName("c.tar.gz"): m2}
# Empty folder
assert merge_folder_children({}, {}, {}, preferred_language) == {}
# Adding children
assert merge_folder_children({}, a1, {}, preferred_language) == a1
assert merge_folder_children({}, {}, a1, preferred_language) == a1
assert merge_folder_children({}, a1, a1, preferred_language) == a1
# Removing children
assert merge_folder_children(a1, {}, a1, preferred_language) == {}
assert merge_folder_children(a1, a1, {}, preferred_language) == {}
assert merge_folder_children(a1, {}, {}, preferred_language) == {}
# Renaming children
assert merge_folder_children(a1, a1, b1, preferred_language) == b1
assert merge_folder_children(a1, b1, a1, preferred_language) == b1
assert merge_folder_children(a1, b1, b1, preferred_language) == b1
# Conflicting renaming
result = merge_folder_children(a1, b1, c1, preferred_language)
assert result == {EntryName("c.tar.gz"): m1}
# Conflicting names
result = merge_folder_children({}, a1, a2, preferred_language)
assert result == {
EntryName("a"): m2,
EntryName(f"a (Parsec - {suffix})"): m1
}
result = merge_folder_children({}, b1, b2, preferred_language)
assert result == {
EntryName("b.txt"): m2,
EntryName(f"b (Parsec - {suffix}).txt"): m1
}
result = merge_folder_children({}, c1, c2, preferred_language)
assert result == {
EntryName("c.tar.gz"): m2,
EntryName(f"c (Parsec - {suffix}).tar.gz"): m1
}
# Conflicting name with special pattern filename
base = {EntryName(f"a (Parsec - {suffix})"): m3}
a3 = {**base, **a1}
b3 = {**base, **a2}
result = merge_folder_children(base, a3, b3, preferred_language)
assert result == {
EntryName("a"): m2,
EntryName(f"a (Parsec - {suffix})"): m3,
EntryName(f"a (Parsec - {suffix} (2))"): m1,
}
m4 = EntryID.new()
base = {**base, EntryName(f"a (Parsec - {suffix} (2))"): m4}
a3 = {**base, **a1}
b3 = {**base, **a2}
result = merge_folder_children(base, a3, b3, preferred_language)
assert result == {
EntryName("a"): m2,
EntryName(f"a (Parsec - {suffix})"): m3,
EntryName(f"a (Parsec - {suffix} (2))"): m4,
EntryName(f"a (Parsec - {suffix} (3))"): m1,
}
def test_merge_speculative_with_it_unsuspected_former_self(
local_changes, core_config):
d1 = datetime(2000, 1, 1)
d2 = datetime(2000, 1, 2)
d3 = datetime(2000, 1, 3)
d4 = datetime(2000, 1, 4)
d5 = datetime(2000, 1, 5)
my_device = DeviceID("a@a")
# 1) Workspace manifest is originally created by our device
local = LocalWorkspaceManifest.new_placeholder(author=my_device,
timestamp=d1)
foo_id = EntryID.new()
local = local.evolve(updated=d2,
children=FrozenDict({EntryName("foo"): foo_id}))
# 2) We sync the workspace manifest
v1 = local.to_remote(author=my_device, timestamp=d3)
# 3) Now let's pretend we lost local storage, hence creating a new speculative manifest
new_local = LocalWorkspaceManifest.new_placeholder(author=my_device,
id=local.id,
timestamp=d3,
speculative=True)
if local_changes:
bar_id = EntryID.new()
new_local = new_local.evolve(updated=d4,
children=FrozenDict(
{EntryName("bar"): bar_id}))
# 4) When syncing the manifest, we shouldn't remove any data from the remote
merged = merge_manifests(
local_author=my_device,
timestamp=d5,
prevent_sync_pattern=empty_pattern,
local_manifest=new_local,
remote_manifest=v1,
)
if local_changes:
assert merged == LocalWorkspaceManifest(
base=v1,
need_sync=True,
updated=d5,
children=FrozenDict({
**v1.children,
**new_local.children
}),
local_confinement_points=frozenset(),
remote_confinement_points=frozenset(),
speculative=False,
)
else:
assert merged == LocalWorkspaceManifest(
base=v1,
need_sync=False,
updated=v1.updated,
children=v1.children,
local_confinement_points=frozenset(),
remote_confinement_points=frozenset(),
speculative=False,
)
