def test_workspace_reencryption_need(
hypothesis_settings,
reset_testbed,
backend_factory,
backend_data_binder_factory,
server_factory,
user_fs_factory,
local_device_factory,
alice,
):
class WorkspaceFSReencrytionNeed(TrioAsyncioRuleBasedStateMachine):
Users = Bundle("user")
async def start_user_fs(self):
try:
await self.user_fs_controller.stop()
except AttributeError:
pass
async def _user_fs_controlled_cb(started_cb):
async with user_fs_factory(device=self.device) as user_fs:
await started_cb(user_fs=user_fs)
self.user_fs_controller = await self.get_root_nursery().start(
call_with_control, _user_fs_controlled_cb)
async def start_backend(self):
async def _backend_controlled_cb(started_cb):
async with backend_factory() as backend:
async with server_factory(backend.handle_client) as server:
await started_cb(backend=backend, server=server)
self.backend_controller = await self.get_root_nursery().start(
call_with_control, _backend_controlled_cb)
def _oracle_give_or_change_role(self, user):
current_role = self.user_roles.get(user.user_id)
new_role = RealmRole.MANAGER if current_role != RealmRole.MANAGER else RealmRole.READER
self.since_reencryption_role_revoked.discard(user.user_id)
self.user_roles[user.user_id] = new_role
return new_role
def _oracle_revoke_role(self, user):
if user.user_id in self.user_roles:
self.since_reencryption_role_revoked.add(user.user_id)
self.user_roles.pop(user.user_id, None)
return True
else:
return False
def _oracle_revoke_user(self, user):
if user.user_id in self.user_roles:
self.since_reencryption_user_revoked.add(user.user_id)
async def _update_role(self, author, user, role=RealmRole.MANAGER):
now = pendulum_now()
certif = RealmRoleCertificateContent(
author=author.device_id,
timestamp=now,
realm_id=RealmID(self.wid.uuid),
user_id=user.user_id,
role=role,
).dump_and_sign(author.signing_key)
await self.backend.realm.update_roles(
author.organization_id,
RealmGrantedRole(
certificate=certif,
realm_id=RealmID(self.wid.uuid),
user_id=user.user_id,
role=role,
granted_by=author.device_id,
granted_on=now,
),
)
return certif
@property
def user_fs(self):
return self.user_fs_controller.user_fs
@property
def backend(self):
return self.backend_controller.backend
@initialize()
async def init(self):
await reset_testbed()
self.user_roles = {}
self.since_reencryption_user_revoked = set()
self.since_reencryption_role_revoked = set()
await self.start_backend()
self.device = self.backend_controller.server.correct_addr(alice)
self.backend_data_binder = backend_data_binder_factory(
self.backend)
await self.start_user_fs()
self.wid = await self.user_fs.workspace_create(EntryName("w"))
await self.user_fs.sync()
self.workspacefs = self.user_fs.get_workspace(self.wid)
@rule(target=Users)
async def give_role(self):
new_user = local_device_factory()
await self.backend_data_binder.bind_device(new_user)
new_role = self._oracle_give_or_change_role(new_user)
await self._update_role(alice, new_user, role=new_role)
return new_user
@rule(user=Users)
async def change_role(self, user):
new_role = self._oracle_give_or_change_role(user)
await self._update_role(alice, user, role=new_role)
@rule(user=Users)
async def revoke_role(self, user):
if self._oracle_revoke_role(user):
await self._update_role(alice, user, role=None)
@rule(user=consumes(Users))
async def revoke_user(self, user):
await self.backend_data_binder.bind_revocation(user.user_id, alice)
self._oracle_revoke_user(user)
@rule()
async def reencrypt(self):
job = await self.user_fs.workspace_start_reencryption(self.wid)
while True:
total, done = await job.do_one_batch()
if total <= done:
break
self.since_reencryption_user_revoked.clear()
self.since_reencryption_role_revoked.clear()
# Needed to keep encryption revision up to date
await self.user_fs.process_last_messages()
@invariant()
async def check_reencryption_need(self):
if not hasattr(self, "workspacefs"):
return
need = await self.workspacefs.get_reencryption_need()
assert set(
need.role_revoked) == self.since_reencryption_role_revoked
assert (set(
need.user_revoked) == self.since_reencryption_user_revoked -
self.since_reencryption_role_revoked)
run_state_machine_as_test(WorkspaceFSReencrytionNeed,
settings=hypothesis_settings)
def run_as_test(cls):
run_state_machine_as_test(cls, settings=hypothesis_settings)
def test_file_operations(tmpdir, hypothesis_settings, reset_testbed,
file_transactions_factory, alice, alice_backend_cmds):
tentative = 0
class FileOperationsStateMachine(TrioAsyncioRuleBasedStateMachine):
async def start_transactions(self):
async def _transactions_controlled_cb(started_cb):
async with WorkspaceStorage.run(alice, Path("/dummy"),
EntryID()) as local_storage:
file_transactions = await file_transactions_factory(
self.device,
alice_backend_cmds,
local_storage=local_storage)
await started_cb(file_transactions=file_transactions)
self.transactions_controller = await self.get_root_nursery().start(
call_with_control, _transactions_controlled_cb)
@initialize()
async def init(self):
nonlocal tentative
tentative += 1
await reset_testbed()
self.device = alice
await self.start_transactions()
self.file_transactions = self.transactions_controller.file_transactions
self.local_storage = self.file_transactions.local_storage
self.fresh_manifest = LocalFileManifest.new_placeholder(
alice.device_id, parent=EntryID())
self.entry_id = self.fresh_manifest.id
async with self.local_storage.lock_entry_id(self.entry_id):
await self.local_storage.set_manifest(self.entry_id,
self.fresh_manifest)
self.fd = self.local_storage.create_file_descriptor(
self.fresh_manifest)
self.file_oracle_path = tmpdir / f"oracle-test-{tentative}.txt"
self.file_oracle_fd = os.open(self.file_oracle_path,
os.O_RDWR | os.O_CREAT)
async def teardown(self):
if not hasattr(self, "fd"):
return
await self.file_transactions.fd_close(self.fd)
os.close(self.file_oracle_fd)
@rule(size=size, offset=size)
async def read(self, size, offset):
data = await self.file_transactions.fd_read(self.fd, size, offset)
os.lseek(self.file_oracle_fd, offset, os.SEEK_SET)
expected = os.read(self.file_oracle_fd, size)
assert data == expected
@rule(content=st.binary(), offset=size)
async def write(self, content, offset):
await self.file_transactions.fd_write(self.fd, content, offset)
os.lseek(self.file_oracle_fd, offset, os.SEEK_SET)
os.write(self.file_oracle_fd, content)
@rule(length=size)
async def resize(self, length):
await self.file_transactions.fd_resize(self.fd, length)
os.ftruncate(self.file_oracle_fd, length)
@rule()
async def reopen(self):
await self.file_transactions.fd_close(self.fd)
self.fd = self.local_storage.create_file_descriptor(
self.fresh_manifest)
os.close(self.file_oracle_fd)
self.file_oracle_fd = os.open(self.file_oracle_path, os.O_RDWR)
run_state_machine_as_test(FileOperationsStateMachine,
settings=hypothesis_settings)
def test_fs_online_concurrent_tree_and_sync(
hypothesis_settings,
reset_testbed,
backend_addr,
backend_factory,
server_factory,
user_fs_factory,
alice,
alice2,
):
class FSOnlineConcurrentTreeAndSync(TrioAsyncioRuleBasedStateMachine):
Files = Bundle("file")
Folders = Bundle("folder")
FSs = Bundle("fs")
async def start_fs(self, device):
async def _user_fs_controlled_cb(started_cb):
async with user_fs_factory(device=device) as fs:
await started_cb(fs=fs)
return await self.get_root_nursery().start(call_with_control,
_user_fs_controlled_cb)
async def start_backend(self, devices):
async def _backend_controlled_cb(started_cb):
async with backend_factory() as backend:
async with server_factory(backend.handle_client,
backend_addr) as server:
await started_cb(backend=backend, server=server)
return await self.get_root_nursery().start(call_with_control,
_backend_controlled_cb)
@property
def user_fs1(self):
return self.user_fs1_controller.fs
@property
def user_fs2(self):
return self.user_fs2_controller.fs
@initialize(target=Folders)
async def init(self):
await reset_testbed()
self.device1 = alice
self.device2 = alice2
self.backend_controller = await self.start_backend(
[self.device1, self.device2])
self.user_fs1_controller = await self.start_fs(self.device1)
self.user_fs2_controller = await self.start_fs(self.device2)
self.wid = await self.user_fs1.workspace_create("w")
workspace = self.user_fs1.get_workspace(self.wid)
await workspace.sync()
await self.user_fs1.sync()
await self.user_fs2.sync()
return "/"
@initialize(target=FSs, force_after_init=Folders)
async def register_user_fs1(self, force_after_init):
return self.user_fs1
@initialize(target=FSs, force_after_init=Folders)
async def register_user_fs2(self, force_after_init):
return self.user_fs2
@rule(target=Files, fs=FSs, parent=Folders, name=st_entry_name)
async def create_file(self, fs, parent, name):
path = f"{parent}/{name}"
workspace = fs.get_workspace(self.wid)
try:
await workspace.touch(path=path)
except OSError:
pass
return path
@rule(target=Folders, fs=FSs, parent=Folders, name=st_entry_name)
async def create_folder(self, fs, parent, name):
path = f"{parent}/{name}"
workspace = fs.get_workspace(self.wid)
try:
await workspace.mkdir(path=path)
except OSError:
pass
return path
@rule(fs=FSs, path=Files)
async def update_file(self, fs, path):
workspace = fs.get_workspace(self.wid)
try:
await workspace.write_bytes(path, data=b"a")
except OSError:
pass
@rule(fs=FSs, path=Files)
async def delete_file(self, fs, path):
workspace = fs.get_workspace(self.wid)
try:
await workspace.unlink(path=path)
except OSError:
pass
return path
@rule(fs=FSs, path=Folders)
async def delete_folder(self, fs, path):
workspace = fs.get_workspace(self.wid)
try:
await workspace.rmdir(path=path)
except OSError:
pass
return path
@rule(target=Files,
fs=FSs,
src=Files,
dst_parent=Folders,
dst_name=st_entry_name)
async def move_file(self, fs, src, dst_parent, dst_name):
dst = f"{dst_parent}/{dst_name}"
workspace = fs.get_workspace(self.wid)
try:
await workspace.move(src, dst)
except OSError:
pass
return dst
@rule(target=Folders,
fs=FSs,
src=Folders,
dst_parent=Folders,
dst_name=st_entry_name)
async def move_folder(self, fs, src, dst_parent, dst_name):
dst = f"{dst_parent}/{dst_name}"
workspace = fs.get_workspace(self.wid)
try:
await workspace.move(src, dst)
except OSError:
pass
return dst
@rule()
async def sync_all_the_files(self):
# Less than 4 retries causes the following test case to fail:
# ```python
# state = FSOnlineConcurrentTreeAndSync()
# async def steps():
# v1 = await state.init()
# v2 = await state.register_user_fs1(force_after_init=v1)
# v3 = await state.register_user_fs2(force_after_init=v1)
# v4 = await state.create_file(fs=v3, name='b', parent=v1)
# await state.sync_all_the_files()
# await state.update_file(fs=v3, path=v4)
# await state.update_file(fs=v2, path=v4)
# v5 = await state.create_file(fs=v3, name='a', parent=v1)
# v6 = await state.create_file(fs=v3, name='a', parent=v1)
# v7 = await state.move_file(dst_name='a', dst_parent=v1, fs=v2, src=v4)
# await state.sync_all_the_files()
# await state.teardown()
# state.trio_run(steps)
# ```
# for fs in [self.user_fs1, self.user_fs2]:
# workspace = fs.get_workspace(self.wid)
retries = 4
workspace1 = self.user_fs1.get_workspace(self.wid)
workspace2 = self.user_fs2.get_workspace(self.wid)
for _ in range(retries):
await workspace1.sync()
await workspace2.sync()
await self.user_fs1.sync()
await self.user_fs2.sync()
user_fs_dump_1 = await workspace1.dump()
user_fs_dump_2 = await workspace2.dump()
compare_fs_dumps(user_fs_dump_1, user_fs_dump_2)
run_state_machine_as_test(FSOnlineConcurrentTreeAndSync,
settings=hypothesis_settings)
def test_shuffle_roles(
hypothesis_settings,
reset_testbed,
backend_addr,
backend_factory,
server_factory,
backend_data_binder_factory,
backend_sock_factory,
local_device_factory,
realm_factory,
coolorg,
alice,
):
class ShuffleRoles(TrioAsyncioRuleBasedStateMachine):
realm_role_strategy = st.one_of(st.just(x) for x in RealmRole)
User = Bundle("user")
async def start_backend(self):
async def _backend_controlled_cb(started_cb):
async with backend_factory(populated=False) as backend:
async with server_factory(backend.handle_client, backend_addr) as server:
await started_cb(backend=backend, server=server)
return await self.get_root_nursery().start(call_with_control, _backend_controlled_cb)
@property
def backend(self):
return self.backend_controller.backend
@initialize(target=User)
async def init(self):
await reset_testbed()
self.backend_controller = await self.start_backend()
# Create organization and first user
self.backend_data_binder = backend_data_binder_factory(self.backend)
await self.backend_data_binder.bind_organization(coolorg, alice)
# Create realm
self.realm_id = await realm_factory(self.backend, alice)
self.current_roles = {alice.user_id: RealmRole.OWNER}
self.certifs = [ANY]
self.socks = {}
return alice
async def get_sock(self, device):
try:
return self.socks[device.user_id]
except KeyError:
pass
async def _start_sock(device, *, task_status=trio.TASK_STATUS_IGNORED):
async with backend_sock_factory(self.backend, device) as sock:
task_status.started(sock)
await trio.sleep_forever()
sock = await self.get_root_nursery().start(_start_sock, device)
self.socks[device.user_id] = sock
return sock
@rule(target=User, author=User, role=realm_role_strategy)
async def give_role_to_new_user(self, author, role):
# Create new user/device
new_device = local_device_factory()
await self.backend_data_binder.bind_device(new_device)
self.current_roles[new_device.user_id] = None
# Assign role
author_sock = await self.get_sock(author)
if await self._give_role(author_sock, author, new_device, role):
return new_device
else:
return multiple()
@rule(author=User, recipient=User, role=realm_role_strategy)
async def change_role_for_existing_user(self, author, recipient, role):
author_sock = await self.get_sock(author)
await self._give_role(author_sock, author, recipient, role)
async def _give_role(self, author_sock, author, recipient, role):
author_sock = await self.get_sock(author)
certif = RealmRoleCertificateContent(
author=author.device_id,
timestamp=pendulum_now(),
realm_id=self.realm_id,
user_id=recipient.user_id,
role=role,
).dump_and_sign(author.signing_key)
rep = await realm_update_roles(author_sock, certif, check_rep=False)
if author.user_id == recipient.user_id:
assert rep == {
"status": "invalid_data",
"reason": "Realm role certificate cannot be self-signed.",
}
else:
owner_only = (RealmRole.OWNER,)
owner_or_manager = (RealmRole.OWNER, RealmRole.MANAGER)
existing_recipient_role = self.current_roles[recipient.user_id]
if existing_recipient_role in owner_or_manager or role in owner_or_manager:
allowed_roles = owner_only
else:
allowed_roles = owner_or_manager
if self.current_roles[author.user_id] in allowed_roles:
# print(f"+ {author.user_id} -{role.value}-> {recipient.user_id}")
if existing_recipient_role != role:
assert rep == {"status": "ok"}
self.current_roles[recipient.user_id] = role
self.certifs.append(certif)
else:
assert rep == {"status": "already_granted"}
else:
# print(f"- {author.user_id} -{role.value}-> {recipient.user_id}")
assert rep == {"status": "not_allowed"}
return rep["status"] == "ok"
@rule(author=User)
async def get_role_certificates(self, author):
sock = await self.get_sock(author)
rep = await realm_get_role_certificates(sock, self.realm_id)
if self.current_roles.get(author.user_id) is not None:
assert rep["status"] == "ok"
assert rep["certificates"] == self.certifs
else:
assert rep == {}
@invariant()
async def check_current_roles(self):
try:
backend = self.backend
except AttributeError:
return
roles = await backend.realm.get_current_roles(alice.organization_id, self.realm_id)
assert roles == {k: v for k, v in self.current_roles.items() if v is not None}
run_state_machine_as_test(ShuffleRoles, settings=hypothesis_settings)
def test_sync_monitor_stateful(
hypothesis_settings,
reset_testbed,
backend_addr,
backend_factory,
server_factory,
client_factory,
user_fs_factory,
alice,
bob,
monkeypatch,
):
# Force a sleep in the monitors mockpoints will freeze them until we reach
# the `let_client_monitors_process_changes` rule
async def mockpoint_sleep():
await trio.sleep(0.01)
monkeypatch.setattr(
"guardata.client.sync_monitor.freeze_sync_monitor_mockpoint",
mockpoint_sleep)
monkeypatch.setattr(
"guardata.client.messages_monitor.freeze_messages_monitor_mockpoint",
mockpoint_sleep)
class SyncMonitorStateful(TrioAsyncioRuleBasedStateMachine):
SharedWorkspaces = Bundle("shared_workspace")
SyncedFiles = Bundle("synced_files")
def __init__(self):
super().__init__()
# Core's sync and message monitors must be kept frozen
mock_clock = trio.testing.MockClock(rate=0, autojump_threshold=0)
self.set_clock(mock_clock)
self.file_count = 0
self.data_count = 0
self.workspace_count = 0
def get_next_file_path(self):
self.file_count = self.file_count + 1
return f"/file-{self.file_count}.txt"
def get_next_data(self):
self.data_count = self.data_count + 1
return f"data {self.data_count}".encode()
def get_next_workspace_name(self):
self.workspace_count = self.workspace_count + 1
return f"w{self.workspace_count}"
def get_workspace(self, device_id, wid):
return self.user_fs_per_device[device_id].get_workspace(wid)
async def start_alice_client(self):
async def _client_controlled_cb(started_cb):
async with client_factory(alice) as client:
await started_cb(client=client)
self.alice_client_controller = await self.get_root_nursery().start(
call_with_control, _client_controlled_cb)
return self.alice_client_controller.client
async def start_bob_user_fs(self):
async def _user_fs_controlled_cb(started_cb):
async with user_fs_factory(device=bob) as user_fs:
await started_cb(user_fs=user_fs)
self.bob_user_fs_controller = await self.get_root_nursery().start(
call_with_control, _user_fs_controlled_cb)
return self.bob_user_fs_controller.user_fs
async def start_backend(self):
async def _backend_controlled_cb(started_cb):
async with backend_factory() as backend:
async with server_factory(backend.handle_client,
backend_addr) as server:
await started_cb(backend=backend, server=server)
self.backend_controller = await self.get_root_nursery().start(
call_with_control, _backend_controlled_cb)
@initialize()
async def init(self):
await reset_testbed()
await self.start_backend()
self.bob_user_fs = await self.start_bob_user_fs()
self.alice_client = await self.start_alice_client()
self.user_fs_per_device = {
alice.device_id: self.alice_client.user_fs,
bob.device_id: self.bob_user_fs,
}
self.synced_files = set()
self.alice_workspaces_role = {}
@rule(
target=SharedWorkspaces,
role=st.one_of(st.just(WorkspaceRole.CONTRIBUTOR),
st.just(WorkspaceRole.READER)),
)
async def create_sharing(self, role):
wname = self.get_next_workspace_name()
wid = await self.bob_user_fs.workspace_create(wname)
await self.bob_user_fs.workspace_share(wid, alice.user_id, role)
self.alice_workspaces_role[wid] = role
return wid
@rule(
wid=SharedWorkspaces,
new_role=st.one_of(st.just(WorkspaceRole.CONTRIBUTOR),
st.just(WorkspaceRole.READER), st.just(None)),
)
async def update_sharing(self, wid, new_role):
await self.bob_user_fs.workspace_share(wid, alice.user_id,
new_role)
self.alice_workspaces_role[wid] = new_role
@rule(author=st.one_of(st.just(alice.device_id),
st.just(bob.device_id)),
wid=SharedWorkspaces)
async def create_file(self, author, wid):
file_path = self.get_next_file_path()
if author == bob.device_id:
await self._bob_update_file(wid, file_path, create_file=True)
else:
await self._alice_update_file(wid, file_path, create_file=True)
@rule(author=st.one_of(st.just(alice.device_id),
st.just(bob.device_id)),
file=SyncedFiles)
async def update_file(self, author, file):
wid, file_path = file
if author == bob.device_id:
await self._bob_update_file(wid, file_path)
else:
await self._alice_update_file(wid, file_path)
async def _bob_update_file(self, wid, file_path, create_file=False):
wfs = self.get_workspace(bob.device_id, wid)
if create_file:
await wfs.touch(file_path)
else:
data = self.get_next_data()
await wfs.write_bytes(file_path, data)
await wfs.sync()
async def _alice_update_file(self, wid, file_path, create_file=False):
try:
wfs = self.get_workspace(alice.device_id, wid)
except FSWorkspaceNotFoundError:
return
if create_file:
try:
await wfs.touch(file_path)
except (FSWorkspaceNoAccess, OSError):
return
else:
data = self.get_next_data()
try:
await wfs.write_bytes(file_path, data)
except (FSWorkspaceNoAccess, OSError):
return
@rule(target=SyncedFiles)
async def let_client_monitors_process_changes(self):
# Wait for alice client to settle down
await trio.sleep(300)
# Bob get back alice's changes
await self.bob_user_fs.sync()
for bob_workspace_entry in self.bob_user_fs.get_user_manifest(
).workspaces:
bob_w = self.bob_user_fs.get_workspace(bob_workspace_entry.id)
await bob_w.sync()
# Alice get back possible changes from bob's sync
await trio.sleep(300)
# Now alice and bob should have agreed on the data
new_synced_files = []
for alice_workspace_entry in self.alice_client.user_fs.get_user_manifest(
).workspaces:
alice_w = self.alice_client.user_fs.get_workspace(
alice_workspace_entry.id)
bob_w = self.bob_user_fs.get_workspace(
alice_workspace_entry.id)
if alice_workspace_entry.role is None:
# No access, workspace can only diverge from bob's
continue
bob_dump = await bob_w.dump()
alice_dump = await alice_w.dump()
if self.alice_workspaces_role[
alice_workspace_entry.id] == WorkspaceRole.READER:
# Synced with bob, but we can have local changes that cannot be synced
recursive_compare_fs_dumps(alice_dump,
bob_dump,
ignore_need_sync=True)
else:
# Fully synced with bob
recursive_compare_fs_dumps(alice_dump,
bob_dump,
ignore_need_sync=False)
for child_name in bob_dump["children"].keys():
key = (alice_workspace_entry.id, f"/{child_name}")
if key not in self.synced_files:
new_synced_files.append(key)
self.synced_files.update(new_synced_files)
return multiple(*(sorted(new_synced_files)))
run_state_machine_as_test(SyncMonitorStateful,
settings=hypothesis_settings)
def test_fs_online_idempotent_sync(
hypothesis_settings,
reset_testbed,
backend_addr,
backend_factory,
server_factory,
user_fs_factory,
alice,
):
class FSOnlineIdempotentSync(TrioAsyncioRuleBasedStateMachine):
BadPath = Bundle("bad_path")
GoodFilePath = Bundle("good_file_path")
GoodFolderPath = Bundle("good_folder_path")
GoodPath = st.one_of(GoodFilePath, GoodFolderPath)
async def start_user_fs(self, device):
async def _user_fs_controlled_cb(started_cb):
async with user_fs_factory(device=device) as user_fs:
await started_cb(user_fs=user_fs)
return await self.get_root_nursery().start(call_with_control, _user_fs_controlled_cb)
async def start_backend(self):
async def _backend_controlled_cb(started_cb):
async with backend_factory() as backend:
async with server_factory(backend.handle_client, backend_addr) as server:
await started_cb(backend=backend, server=server)
return await self.get_root_nursery().start(call_with_control, _backend_controlled_cb)
@property
def user_fs(self):
return self.user_fs_controller.user_fs
@initialize(target=BadPath)
async def init(self):
await reset_testbed()
self.backend_controller = await self.start_backend()
self.device = alice
self.user_fs_controller = await self.start_user_fs(alice)
wid = await self.user_fs.workspace_create("w")
self.workspace = self.user_fs.get_workspace(wid)
await self.workspace.touch("/good_file.txt")
await self.workspace.mkdir("/good_folder")
await self.workspace.touch("/good_folder/good_sub_file.txt")
await self.workspace.sync()
await self.user_fs.sync()
self.initial_fs_dump = await self.workspace.dump()
check_fs_dump(self.initial_fs_dump)
return "/dummy"
@initialize(target=GoodFolderPath)
async def init_good_folder_pathes(self):
return multiple("/", "/good_folder/")
@initialize(target=GoodFilePath)
async def init_good_file_pathes(self):
return multiple("/good_file.txt", "/good_folder/good_sub_file.txt")
@rule(target=BadPath, type=st_entry_type, bad_parent=BadPath, name=st_entry_name)
async def try_to_create_bad_path(self, type, bad_parent, name):
path = f"{bad_parent}/{name}"
with pytest.raises(FileNotFoundError):
if type == "file":
await self.workspace.touch(path)
else:
await self.workspace.mkdir(path)
return path
@rule(type=st_entry_type, path=GoodPath)
async def try_to_create_already_exists(self, type, path):
if type == "file":
if str(path) == "/":
with pytest.raises(PermissionError):
await self.workspace.mkdir(path)
else:
with pytest.raises(FileExistsError):
await self.workspace.mkdir(path)
@rule(path=BadPath)
async def try_to_update_file(self, path):
with pytest.raises(OSError):
async with await self.workspace.open_file(path, "r+") as f:
await f.write(b"a")
@rule(path=BadPath)
async def try_to_delete(self, path):
with pytest.raises(FileNotFoundError):
await self.workspace.unlink(path)
with pytest.raises(FileNotFoundError):
await self.workspace.rmdir(path)
@rule(src=BadPath, dst_name=st_entry_name)
async def try_to_move_bad_src(self, src, dst_name):
dst = "/%s" % dst_name
with pytest.raises(OSError):
await self.workspace.rename(src, dst)
@rule(src=GoodPath, dst=GoodFolderPath)
async def try_to_move_bad_dst(self, src, dst):
# TODO: why so much special cases ?
if src == dst and src != "/":
await self.workspace.rename(src, dst)
else:
with pytest.raises(OSError):
await self.workspace.rename(src, dst)
@rule(path=GoodPath)
async def sync(self, path):
entry_id = await self.workspace.path_id(path)
await self.workspace.sync_by_id(entry_id)
@invariant()
async def check_fs(self):
try:
fs_dump = await self.workspace.dump()
except AttributeError:
# FS not yet initialized
pass
else:
assert fs_dump == self.initial_fs_dump
run_state_machine_as_test(FSOnlineIdempotentSync, settings=hypothesis_settings)
def test_entry_transactions(
tmpdir,
hypothesis_settings,
reset_testbed,
entry_transactions_factory,
file_transactions_factory,
alice,
alice_backend_cmds,
):
tentative = 0
# The point is not to find breaking filenames here, so keep it simple
st_entry_name = st.text(alphabet=ascii_lowercase, min_size=4, max_size=6)
class EntryTransactionsStateMachine(TrioAsyncioRuleBasedStateMachine):
Files = Bundle("file")
Folders = Bundle("folder")
async def start_transactions(self):
async def _transactions_controlled_cb(started_cb):
async with WorkspaceStorage.run(alice, Path("/dummy"), EntryID()) as local_storage:
entry_transactions = await entry_transactions_factory(
self.device, alice_backend_cmds, local_storage=local_storage
)
file_transactions = await file_transactions_factory(
self.device, alice_backend_cmds, local_storage=local_storage
)
await started_cb(
entry_transactions=entry_transactions, file_transactions=file_transactions
)
self.transactions_controller = await self.get_root_nursery().start(
call_with_control, _transactions_controlled_cb
)
@initialize(target=Folders)
async def init_root(self):
nonlocal tentative
tentative += 1
await reset_testbed()
self.last_step_id_to_path = set()
self.device = alice
await self.start_transactions()
self.entry_transactions = self.transactions_controller.entry_transactions
self.file_transactions = self.transactions_controller.file_transactions
self.folder_oracle = Path(tmpdir / f"oracle-test-{tentative}")
self.folder_oracle.mkdir()
oracle_root = self.folder_oracle / "root"
oracle_root.mkdir()
self.folder_oracle.chmod(0o500) # Root oracle can no longer be removed this way
return PathElement("/", oracle_root)
@rule(target=Files, parent=Folders, name=st_entry_name)
async def touch(self, parent, name):
path = parent / name
expected_exc = None
try:
path.to_oracle().touch(exist_ok=False)
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
_, fd = await self.entry_transactions.file_create(path.to_guardata())
await self.file_transactions.fd_close(fd)
return path
@rule(target=Folders, parent=Folders, name=st_entry_name)
async def mkdir(self, parent, name):
path = parent / name
expected_exc = None
try:
path.to_oracle().mkdir(exist_ok=False)
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
await self.entry_transactions.folder_create(path.to_guardata())
return path
@rule(path=Files)
async def unlink(self, path):
expected_exc = None
try:
path.to_oracle().unlink()
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
await self.entry_transactions.file_delete(path.to_guardata())
@rule(path=Files, length=st.integers(min_value=0, max_value=32))
async def resize(self, path, length):
expected_exc = None
try:
os.truncate(path.to_oracle(), length)
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
await self.entry_transactions.file_resize(path.to_guardata(), length)
@rule(path=Folders)
async def rmdir(self, path):
expected_exc = None
try:
path.to_oracle().rmdir()
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
await self.entry_transactions.folder_delete(path.to_guardata())
async def _rename(self, src, dst_parent, dst_name):
dst = dst_parent / dst_name
expected_exc = None
try:
oracle_rename(src.to_oracle(), dst.to_oracle())
except OSError as exc:
expected_exc = exc
with expect_raises(expected_exc):
await self.entry_transactions.entry_rename(src.to_guardata(), dst.to_guardata())
return dst
@rule(target=Files, src=Files, dst_parent=Folders, dst_name=st_entry_name)
async def rename_file(self, src, dst_parent, dst_name):
return await self._rename(src, dst_parent, dst_name)
@rule(target=Folders, src=Folders, dst_parent=Folders, dst_name=st_entry_name)
async def rename_folder(self, src, dst_parent, dst_name):
return await self._rename(src, dst_parent, dst_name)
@invariant()
async def check_access_to_path_unicity(self):
try:
self.entry_transactions
except AttributeError:
return
local_storage = self.entry_transactions.local_storage
root_entry_id = self.entry_transactions.get_workspace_entry().id
new_id_to_path = set()
async def _recursive_build_id_to_path(entry_id, parent_id):
new_id_to_path.add((entry_id, parent_id))
manifest = await local_storage.get_manifest(entry_id)
if is_folder_manifest(manifest):
for child_name, child_entry_id in manifest.children.items():
await _recursive_build_id_to_path(child_entry_id, entry_id)
await _recursive_build_id_to_path(root_entry_id, None)
added_items = new_id_to_path - self.last_step_id_to_path
for added_id, added_parent in added_items:
for old_id, old_parent in self.last_step_id_to_path:
if old_id == added_id and added_parent != old_parent.parent:
raise AssertionError(
f"Same id ({old_id}) but different parent: {old_parent} -> {added_parent}"
)
self.last_step_id_to_path = new_id_to_path
run_state_machine_as_test(EntryTransactionsStateMachine, settings=hypothesis_settings)
def test_fs_online_concurrent_user(
hypothesis_settings,
reset_testbed,
backend_addr,
backend_factory,
server_factory,
user_fs_factory,
alice,
alice2,
):
class FSOnlineConcurrentUser(TrioAsyncioRuleBasedStateMachine):
Workspaces = Bundle("workspace")
FSs = Bundle("fs")
async def start_user_fs(self, device):
async def _user_fs_controlled_cb(started_cb):
async with user_fs_factory(device=device) as fs:
await started_cb(fs=fs)
return await self.get_root_nursery().start(call_with_control,
_user_fs_controlled_cb)
async def start_backend(self):
async def _backend_controlled_cb(started_cb):
async with backend_factory() as backend:
async with server_factory(backend.handle_client) as server:
await started_cb(backend=backend, server=server)
return await self.get_root_nursery().start(call_with_control,
_backend_controlled_cb)
@property
def user_fs1(self):
return self.user_fs1_controller.fs
@property
def user_fs2(self):
return self.user_fs2_controller.fs
@initialize(target=Workspaces)
async def init(self):
await reset_testbed()
self.backend_controller = await self.start_backend()
self.device1 = self.backend_controller.server.correct_addr(alice)
self.device2 = self.backend_controller.server.correct_addr(alice2)
self.user_fs1_controller = await self.start_user_fs(self.device1)
self.user_fs2_controller = await self.start_user_fs(self.device2)
self.wid = await self.user_fs1.workspace_create(EntryName("w"))
workspace = self.user_fs1.get_workspace(self.wid)
await workspace.sync()
await self.user_fs1.sync()
await self.user_fs2.sync()
return self.wid, "w"
@initialize(target=FSs, force_after_init=Workspaces)
async def register_user_fs1(self, force_after_init):
return self.user_fs1
@initialize(target=FSs, force_after_init=Workspaces)
async def register_user_fs2(self, force_after_init):
return self.user_fs2
@rule(target=Workspaces, fs=FSs, name=st_entry_name)
async def create_workspace(self, fs, name):
try:
wid = await fs.workspace_create(EntryName(name))
workspace = fs.get_workspace(wid)
await workspace.sync()
except AssertionError:
return "wrong", name
return wid, name
@rule(target=Workspaces,
fs=FSs,
src=Workspaces,
dst_name=st_entry_name)
async def rename_workspace(self, fs, src, dst_name):
wid, _ = src
if wid == "wrong":
return src[0], src[1]
try:
await fs.workspace_rename(wid, EntryName(dst_name))
except FSWorkspaceNotFoundError:
pass
return wid, dst_name
@rule()
async def sync(self):
# Send two syncs in a row given file conflict results are not synced
# once created
workspace1 = self.user_fs1.get_workspace(self.wid)
workspace2 = self.user_fs2.get_workspace(self.wid)
# Sync 1
await workspace1.sync()
await self.user_fs1.sync()
await self.user_fs1.sync()
# Sync 2
await workspace2.sync()
await self.user_fs2.sync()
await self.user_fs2.sync()
# Sync 1
await workspace1.sync()
await self.user_fs1.sync()
fs_dump_1 = await workspace1.dump()
fs_dump_2 = await workspace2.dump()
compare_fs_dumps(fs_dump_1, fs_dump_2)
run_state_machine_as_test(FSOnlineConcurrentUser,
settings=hypothesis_settings)
