async def test_create_certif_too_old(alice, alice_backend_sock):
now = pendulum.now()
# Generate a certificate
realm_id = RealmID.from_hex("C0000000000000000000000000000000")
certif = RealmRoleCertificateContent.build_realm_root_certif(
author=alice.device_id, timestamp=now,
realm_id=realm_id).dump_and_sign(alice.signing_key)
# Create a realm a tiny bit too late
later = now.add(seconds=BALLPARK_CLIENT_LATE_OFFSET)
with freeze_time(later):
rep = await realm_create(alice_backend_sock, certif)
assert rep == {
"status": "bad_timestamp",
"backend_timestamp": later,
"ballpark_client_early_offset": BALLPARK_CLIENT_EARLY_OFFSET,
"ballpark_client_late_offset": BALLPARK_CLIENT_LATE_OFFSET,
"client_timestamp": now,
}
# Create a realm late but right before the deadline
later = now.add(seconds=BALLPARK_CLIENT_LATE_OFFSET, microseconds=-1)
with freeze_time(later):
rep = await realm_create(alice_backend_sock, certif)
assert rep["status"] == "ok"
# Generate a new certificate
realm_id = RealmID.from_hex("C0000000000000000000000000000001")
certif = RealmRoleCertificateContent.build_realm_root_certif(
author=alice.device_id, timestamp=now,
realm_id=realm_id).dump_and_sign(alice.signing_key)
# Create a realm a tiny bit too soon
sooner = now.subtract(seconds=BALLPARK_CLIENT_EARLY_OFFSET)
with freeze_time(sooner):
rep = await realm_create(alice_backend_sock, certif)
assert rep == {
"status": "bad_timestamp",
"backend_timestamp": sooner,
"ballpark_client_early_offset": BALLPARK_CLIENT_EARLY_OFFSET,
"ballpark_client_late_offset": BALLPARK_CLIENT_LATE_OFFSET,
"client_timestamp": now,
}
# Create a realm soon but after the limit
sooner = now.subtract(seconds=BALLPARK_CLIENT_EARLY_OFFSET,
microseconds=-1)
with freeze_time(sooner):
rep = await realm_create(alice_backend_sock, certif)
assert rep["status"] == "ok"
async def test_get_reencryption_need(alice_workspace, running_backend,
monkeypatch):
expected = ReencryptionNeed(user_revoked=(), role_revoked=())
assert await alice_workspace.get_reencryption_need() == expected
with running_backend.offline():
with pytest.raises(FSBackendOfflineError):
await alice_workspace.get_reencryption_need()
# Reproduce a backend offline after the certificates have been retrieved (see issue #1335)
reply = await alice_workspace.remote_loader.backend_cmds.realm_get_role_certificates(
RealmID(alice_workspace.workspace_id.uuid))
original = alice_workspace.remote_loader.backend_cmds.realm_get_role_certificates
async def mockup(*args):
if args == (alice_workspace.workspace_id, ):
return reply
return await original(*args)
monkeypatch.setattr(alice_workspace.remote_loader.backend_cmds,
"realm_get_role_certificates", mockup)
with running_backend.offline():
with pytest.raises(FSBackendOfflineError):
await alice_workspace.get_reencryption_need()
async def read(
self, organization_id: OrganizationID, author: DeviceID, block_id: BlockID
) -> bytes:
async with self.dbh.pool.acquire() as conn, conn.transaction():
realm_id_uuid = await conn.fetchval(
*_q_get_realm_id_from_block_id(
organization_id=organization_id.str, block_id=block_id.uuid
)
)
if not realm_id_uuid:
raise BlockNotFoundError()
realm_id = RealmID(realm_id_uuid)
await _check_realm(conn, organization_id, realm_id, OperationKind.DATA_READ)
ret = await conn.fetchrow(
*_q_get_block_meta(
organization_id=organization_id.str,
block_id=block_id.uuid,
user_id=author.user_id.str,
)
)
if not ret or ret["deleted_on"]:
raise BlockNotFoundError()
elif not ret["has_access"]:
raise BlockAccessError()
# We can do the blockstore read outside of the transaction given the block
# are never modified/removed
return await self._blockstore_component.read(organization_id, block_id)
async def _get_realm_id_from_vlob_id(conn, organization_id: OrganizationID,
vlob_id: VlobID) -> RealmID:
realm_id_uuid = await conn.fetchval(*_q_get_realm_id_from_vlob_id(
organization_id=organization_id.str, vlob_id=vlob_id.uuid))
if not realm_id_uuid:
raise VlobNotFoundError(f"Vlob `{vlob_id}` doesn't exist")
return RealmID(realm_id_uuid)
async def get_reencryption_need(self) -> ReencryptionNeed:
"""
Raises:
FSError
FSBackendOfflineError
FSWorkspaceNoAccess
"""
wentry = self.get_workspace_entry()
try:
workspace_manifest = self.local_storage.get_workspace_manifest()
if workspace_manifest.is_placeholder and not workspace_manifest.speculative:
return ReencryptionNeed(user_revoked=(),
role_revoked=(),
reencryption_already_in_progress=False)
except FSLocalMissError:
pass
try:
rep = await self.backend_cmds.realm_status(
RealmID(self.workspace_id.uuid))
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(
f"Cannot retreive remote status for workspace {self.workspace_id}: {exc}"
) from exc
reencryption_already_in_progress = (rep["in_maintenance"]
and rep["maintenance_type"]
== MaintenanceType.REENCRYPTION)
certificates = await self.remote_loader.load_realm_role_certificates()
has_role = set()
role_revoked = set()
for certif in certificates:
if certif.role is None:
if certif.timestamp > wentry.encrypted_on:
role_revoked.add(certif.user_id)
has_role.discard(certif.user_id)
else:
role_revoked.discard(certif.user_id)
has_role.add(certif.user_id)
user_revoked = []
for user_id in has_role:
_, revoked_user = await self.remote_loader.get_user(user_id,
no_cache=True)
if revoked_user and revoked_user.timestamp > wentry.encrypted_on:
user_revoked.append(user_id)
return ReencryptionNeed(
user_revoked=tuple(user_revoked),
role_revoked=tuple(role_revoked),
reencryption_already_in_progress=reencryption_already_in_progress,
)
async def query_get_realms_for_user(
conn, organization_id: OrganizationID,
user: UserID) -> Dict[RealmID, Optional[RealmRole]]:
rep = await conn.fetch(*_q_get_realms_for_user(
organization_id=organization_id.str, user_id=user.str))
return {
RealmID(row["realm_id"]): RealmRole(row["role"])
for row in rep if row["role"] is not None
}
async def test_create_but_unknown_realm(alice_backend_sock):
bad_realm_id = RealmID.new()
blob = b"Initial commit."
rep = await vlob_create(alice_backend_sock,
bad_realm_id,
VLOB_ID,
blob,
check_rep=False)
assert rep["status"] == "not_allowed"
async def _test_create_ok(backend, device, device_backend_sock):
await events_subscribe(device_backend_sock)
realm_id = RealmID.from_hex("C0000000000000000000000000000000")
certif = RealmRoleCertificateContent.build_realm_root_certif(
author=device.device_id, timestamp=pendulum.now(),
realm_id=realm_id).dump_and_sign(device.signing_key)
with backend.event_bus.listen() as spy:
rep = await realm_create(device_backend_sock, certif)
assert rep == {"status": "ok"}
await spy.wait_with_timeout(BackendEvent.REALM_ROLES_UPDATED)
async def test_create_invalid_certif(bob, alice_backend_sock):
realm_id = RealmID.from_hex("C0000000000000000000000000000000")
certif = RealmRoleCertificateContent.build_realm_root_certif(
author=bob.device_id, timestamp=pendulum.now(),
realm_id=realm_id).dump_and_sign(bob.signing_key)
rep = await realm_create(alice_backend_sock, certif)
assert rep == {
"status": "invalid_certification",
"reason":
"Invalid certification data (Signature was forged or corrupt).",
}
async def _load_changes(self) -> bool:
if self._changes_loaded:
return True
# Initialize due_time so that if we cannot retrieve the changes, we
# will wait until an external event (most likely a `sharing.updated`)
# make it worth to retry
self.due_time = math.inf
# 1) Fetch new checkpoint and changes
realm_checkpoint = await self._get_local_storage().get_realm_checkpoint()
try:
rep = await self._get_backend_cmds().vlob_poll_changes(
RealmID(self.id.uuid), realm_checkpoint
)
except BackendNotAvailable:
raise
# Another backend error
except BackendConnectionError as exc:
logger.warning("Unexpected backend response during sync bootstrap", exc_info=exc)
return False
if rep["status"] == "not_found":
# Workspace not yet synchronized with backend
new_checkpoint = 0
changes = {}
elif rep["status"] in ("in_maintenance", "not_allowed"):
return False
elif rep["status"] != "ok":
return False
else:
new_checkpoint = rep["current_checkpoint"]
changes = rep["changes"]
# 2) Store new checkpoint and changes
await self._get_local_storage().update_realm_checkpoint(
new_checkpoint, {EntryID.from_hex(name.hex): val for name, val in changes.items()}
)
# 3) Compute local and remote changes that need to be synced
need_sync_local, need_sync_remote = await self._get_local_storage().get_need_sync_entries()
now = current_time()
# Ignore local changes in read only mode
if not self.read_only:
self._local_changes = {entry_id: LocalChange(now) for entry_id in need_sync_local}
self._remote_changes = need_sync_remote
# 4) Finally refresh due time according to the changes
self._compute_due_time()
self._changes_loaded = True
return True
async def _update_role(self, author, user, role=RealmRole.MANAGER):
now = pendulum_now()
certif = RealmRoleCertificateContent(
author=author.device_id,
timestamp=now,
realm_id=RealmID(self.wid.uuid),
user_id=user.user_id,
role=role,
).dump_and_sign(author.signing_key)
await self.backend.realm.update_roles(
author.organization_id,
RealmGrantedRole(
certificate=certif,
realm_id=RealmID(self.wid.uuid),
user_id=user.user_id,
role=role,
granted_by=author.device_id,
granted_on=now,
),
)
return certif
async def workspace_continue_reencryption(
self, workspace_id: EntryID) -> ReencryptionJob:
"""
Raises:
FSError
FSBackendOfflineError
FSWorkspaceNoAccess
FSWorkspaceNotFoundError
"""
user_manifest = self.get_user_manifest()
workspace_entry = user_manifest.get_workspace_entry(workspace_id)
if not workspace_entry:
raise FSWorkspaceNotFoundError(
f"Unknown workspace `{workspace_id}`")
# First make sure the workspace is under maintenance
try:
rep = await self.backend_cmds.realm_status(
RealmID(workspace_entry.id.uuid))
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(
f"Cannot continue maintenance on workspace {workspace_id}: {exc}"
) from exc
if rep["status"] == "not_allowed":
raise FSWorkspaceNoAccess(
f"Not allowed to access workspace {workspace_id}: {rep}")
elif rep["status"] != "ok":
raise FSError(
f"Error while getting status for workspace {workspace_id}: {rep}"
)
if not rep["in_maintenance"] or rep[
"maintenance_type"] != MaintenanceType.REENCRYPTION:
raise FSWorkspaceNotInMaintenance(
"Not in reencryption maintenance")
if rep["encryption_revision"] != workspace_entry.encryption_revision:
raise FSError("Bad encryption revision")
previous_workspace_entry = await self._get_previous_workspace_entry(
workspace_entry)
if not previous_workspace_entry:
raise FSError(
f"Never had access to encryption revision {workspace_entry.encryption_revision - 1}"
)
return ReencryptionJob(self.backend_cmds, workspace_entry,
previous_workspace_entry)
async def test_create_certif_role_not_owner(alice, alice_backend_sock):
realm_id = RealmID.from_hex("C0000000000000000000000000000000")
certif = RealmRoleCertificateContent(
author=alice.device_id,
timestamp=pendulum.now(),
realm_id=realm_id,
user_id=alice.user_id,
role=RealmRole.MANAGER,
).dump_and_sign(alice.signing_key)
rep = await realm_create(alice_backend_sock, certif)
assert rep == {
"status": "invalid_data",
"reason": "Initial realm role certificate must set OWNER role.",
}
async def test_share_no_manager_right(running_backend, alice_user_fs, alice,
bob):
with freeze_time("2000-01-02"):
wid = await alice_user_fs.workspace_create(EntryName("w1"))
await alice_user_fs.sync()
# Drop manager right (and give to Bob the ownership)
await running_backend.backend.realm.update_roles(
alice.organization_id,
RealmGrantedRole(
realm_id=RealmID(wid.uuid),
user_id=bob.user_id,
certificate=b"<dummy>",
role=RealmRole.OWNER,
granted_by=alice.device_id,
granted_on=datetime(2000, 1, 3),
),
)
await running_backend.backend.realm.update_roles(
alice.organization_id,
RealmGrantedRole(
realm_id=RealmID(wid.uuid),
user_id=alice.user_id,
certificate=b"<dummy>",
role=RealmRole.CONTRIBUTOR,
granted_by=bob.device_id,
granted_on=datetime(2000, 1, 4),
),
)
with pytest.raises(FSSharingNotAllowedError) as exc:
await alice_user_fs.workspace_share(wid, bob.user_id,
WorkspaceRole.MANAGER)
assert (
exc.value.message ==
"Must be Owner or Manager on the workspace is mandatory to share it: {'status': 'not_allowed'}"
)
async def test_do_reencryption(running_backend, workspace, alice, alice_user_fs):
with running_backend.backend.event_bus.listen() as spy:
job = await alice_user_fs.workspace_start_reencryption(workspace)
# Check events
await spy.wait_multiple_with_timeout(
[
(
BackendEvent.REALM_MAINTENANCE_STARTED,
{
"organization_id": alice.organization_id,
"author": alice.device_id,
"realm_id": RealmID(workspace.uuid),
"encryption_revision": 2,
},
),
(
BackendEvent.MESSAGE_RECEIVED,
{
"organization_id": alice.organization_id,
"author": alice.device_id,
"recipient": alice.user_id,
"index": 1,
},
),
]
)
total, done = await job.do_one_batch(size=1)
assert total == 4
assert done == 1
total, done = await job.do_one_batch(size=2)
assert total == 4
assert done == 3
total, done = await job.do_one_batch(size=2)
assert total == 4
assert done == 4
with pytest.raises(FSWorkspaceNotInMaintenance):
await job.do_one_batch()
async def _send_start_reencryption_cmd(
self,
workspace_id: EntryID,
encryption_revision: int,
timestamp: DateTime,
per_user_ciphered_msgs: Dict[UserID, bytes],
) -> bool:
"""
Raises:
FSError
FSBackendOfflineError
FSWorkspaceNoAccess
BackendCmdsParticipantsMismatchError
"""
# Finally send command to the backend
try:
rep = await self.backend_cmds.realm_start_reencryption_maintenance(
RealmID(workspace_id.uuid), encryption_revision, timestamp,
per_user_ciphered_msgs)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(
f"Cannot start maintenance on workspace {workspace_id}: {exc}"
) from exc
if rep["status"] == "participants_mismatch":
# Catched by caller
return False
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
f"Workspace {workspace_id} already in maintenance: {rep}")
elif rep["status"] == "not_allowed":
raise FSWorkspaceNoAccess(
f"Not allowed to start maintenance on workspace {workspace_id}: {rep}"
)
elif rep["status"] != "ok":
raise FSError(
f"Cannot start maintenance on workspace {workspace_id}: {rep}")
return True
async def _realm_factory(backend, author, realm_id=None, now=None):
realm_id = realm_id or RealmID.new()
now = now or next_timestamp()
certif = RealmRoleCertificateContent.build_realm_root_certif(
author=author.device_id, timestamp=now,
realm_id=realm_id).dump_and_sign(author.signing_key)
with backend.event_bus.listen() as spy:
await backend.realm.create(
organization_id=author.organization_id,
self_granted_role=RealmGrantedRole(
realm_id=realm_id,
user_id=author.user_id,
certificate=certif,
role=RealmRole.OWNER,
granted_by=author.device_id,
granted_on=now,
),
)
await spy.wait_with_timeout(BackendEvent.REALM_ROLES_UPDATED)
return realm_id
async def upload_block(self, access: BlockAccess, data: bytes) -> None:
"""
Raises:
FSError
FSBackendOfflineError
FSRemoteOperationError
FSWorkspaceInMaintenance
FSWorkspaceNoAccess
"""
# Encryption
try:
ciphered = access.key.encrypt(data)
# Encryption error
except CryptoError as exc:
raise FSError(f"Cannot encrypt block: {exc}") from exc
# Upload block
with translate_backend_cmds_errors():
rep = await self.backend_cmds.block_create(
access.id, RealmID(self.workspace_id.uuid), ciphered)
if rep["status"] == "already_exists":
# Ignore exception if the block has already been uploaded
# This might happen when a failure occurs before the local storage is updated
pass
elif rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoWriteAccess(
"Cannot upload block: no write access")
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
"Cannot upload block while the workspace in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot upload block: {rep}")
# Update local storage
await self.local_storage.set_clean_block(access.id, data)
await self.local_storage.clear_chunk(ChunkID(access.id.uuid),
miss_ok=True)
async def _vlob_create(self, encryption_revision: int, entry_id: EntryID,
ciphered: bytes, now: DateTime) -> None:
"""
Raises:
FSError
FSRemoteSyncError
FSBackendOfflineError
FSRemoteOperationError
FSWorkspaceInMaintenance
FSBadEncryptionRevision
FSWorkspaceNoAccess
"""
# Vlob upload
with translate_backend_cmds_errors():
rep = await self.backend_cmds.vlob_create(
RealmID(self.workspace_id.uuid),
encryption_revision,
VlobID(entry_id.uuid),
now,
ciphered,
)
if rep["status"] == "already_exists":
raise FSRemoteSyncError(entry_id)
elif rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoWriteAccess(
"Cannot upload manifest: no write access")
elif rep["status"] == "require_greater_timestamp":
raise VlobRequireGreaterTimestampError(
rep["strictly_greater_than"])
elif rep["status"] == "bad_encryption_revision":
raise FSBadEncryptionRevision(
f"Cannot create vlob {entry_id}: Bad encryption revision provided"
)
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
"Cannot create vlob while the workspace is in maintenance")
elif rep["status"] != "ok":
raise FSError(f"Cannot create vlob {entry_id}: `{rep['status']}`")
async def create_realm(self, realm_id: EntryID) -> None:
"""
Raises:
FSError
FSRemoteOperationError
FSBackendOfflineError
"""
timestamp = self.device.timestamp()
certif = RealmRoleCertificateContent.build_realm_root_certif(
author=self.device.device_id,
timestamp=timestamp,
realm_id=RealmID(realm_id.uuid)).dump_and_sign(
self.device.signing_key)
with translate_backend_cmds_errors():
rep = await self.backend_cmds.realm_create(certif)
if rep["status"] == "already_exists":
# It's possible a previous attempt to create this realm
# succeeded but we didn't receive the confirmation, hence
# we play idempotent here.
return
elif rep["status"] != "ok":
raise FSError(f"Cannot create realm {realm_id}: `{rep['status']}`")
async def test_vlobs_updated_event_realm_created_after_subscribe(
backend, alice_backend_sock, alice, alice2, realm_created_by_self):
realm_id = RealmID.from_hex("0000000000000000000000000000000A")
await events_subscribe(alice_backend_sock)
# New realm, should get events anyway
with backend.event_bus.listen() as spy:
realm_creator = alice if realm_created_by_self else alice2
# Create the realm
await backend.realm.create(
organization_id=realm_creator.organization_id,
self_granted_role=RealmGrantedRole(
realm_id=realm_id,
user_id=realm_creator.user_id,
certificate=b"<dummy>",
role=RealmRole.OWNER,
granted_by=realm_creator.device_id,
granted_on=datetime(2000, 1, 2),
),
)
# Create vlob in realm
await backend.vlob.create(
organization_id=realm_creator.organization_id,
author=realm_creator.device_id,
realm_id=realm_id,
encryption_revision=1,
vlob_id=VLOB_ID,
timestamp=NOW,
blob=b"v1",
)
# Update vlob in realm
await backend.vlob.update(
organization_id=alice2.organization_id,
author=alice2.device_id,
encryption_revision=1,
vlob_id=VLOB_ID,
version=2,
timestamp=NOW,
blob=b"v2",
)
# Wait for events to be processed by the backend
await spy.wait_multiple_with_timeout([
BackendEvent.REALM_ROLES_UPDATED,
BackendEvent.REALM_VLOBS_UPDATED,
BackendEvent.REALM_VLOBS_UPDATED,
])
# Realm access granted
rep = await events_listen_nowait(alice_backend_sock)
assert rep == {
"status": "ok",
"event": APIEvent.REALM_ROLES_UPDATED,
"realm_id": realm_id,
"role": RealmRole.OWNER,
}
# Create vlob in realm event
if not realm_created_by_self:
rep = await events_listen_nowait(alice_backend_sock)
assert rep == {
"status": "ok",
"event": APIEvent.REALM_VLOBS_UPDATED,
"realm_id": realm_id,
"checkpoint": 1,
"src_id": VLOB_ID,
"src_version": 1,
}
# Update vlob in realm event
rep = await events_listen_nowait(alice_backend_sock)
assert rep == {
"status": "ok",
"event": APIEvent.REALM_VLOBS_UPDATED,
"realm_id": realm_id,
"checkpoint": 2,
"src_id": VLOB_ID,
"src_version": 2,
}
rep = await events_listen_nowait(alice_backend_sock)
assert rep == {"status": "no_events"}
async def test_reconnect_with_remote_changes(frozen_clock, alice2,
running_backend, server_factory,
alice_core, user_fs_factory):
wid = await alice_core.user_fs.workspace_create(EntryName("w"))
alice_w = alice_core.user_fs.get_workspace(wid)
await alice_w.mkdir("/foo")
await alice_w.touch("/bar.txt")
# Wait for sync monitor to do it job
await frozen_clock.sleep_with_autojump(60)
async with frozen_clock.real_clock_timeout():
await alice_core.wait_idle_monitors()
# Alice2 connect to the backend through a different server so that we can
# switch alice offline while keeping alice2 connected
async with server_factory(running_backend.backend.handle_client) as server:
alice2 = server.correct_addr(alice2)
async with user_fs_factory(alice2) as alice2_user_fs:
# Switch backend offline for alice (but not alice2 !)
with running_backend.offline():
# Get back modifications from alice
await alice2_user_fs.sync()
alice2_w = alice2_user_fs.get_workspace(wid)
await alice2_w.sync()
# Modify the workspace while alice is offline
await alice2_w.mkdir("/foo/spam")
await alice2_w.write_bytes("/bar.txt", b"v2")
foo_id = await alice2_w.path_id("/foo")
spam_id = await alice2_w.path_id("/foo/spam")
bar_id = await alice2_w.path_id("/bar.txt")
with running_backend.backend.event_bus.listen() as spy:
await alice2_w.sync()
# Alice misses the vlob updated events before being back online
await spy.wait_multiple_with_timeout(
[
(
BackendEvent.REALM_VLOBS_UPDATED,
{
"organization_id": alice2.organization_id,
"author": alice2.device_id,
"realm_id": RealmID(wid.uuid),
"checkpoint": ANY,
"src_id": VlobID(spam_id.uuid),
"src_version": 1,
},
),
(
BackendEvent.REALM_VLOBS_UPDATED,
{
"organization_id": alice2.organization_id,
"author": alice2.device_id,
"realm_id": RealmID(wid.uuid),
"checkpoint": ANY,
"src_id": VlobID(foo_id.uuid),
"src_version": 2,
},
),
(
BackendEvent.REALM_VLOBS_UPDATED,
{
"organization_id": alice2.organization_id,
"author": alice2.device_id,
"realm_id": RealmID(wid.uuid),
"checkpoint": ANY,
"src_id": VlobID(bar_id.uuid),
"src_version": 2,
},
),
],
in_order=False,
)
with alice_core.event_bus.listen() as spy:
# Now alice should sync back the changes
await frozen_clock.sleep_with_autojump(60)
await spy.wait_multiple_with_timeout(
[
(
CoreEvent.BACKEND_CONNECTION_CHANGED,
{
"status": BackendConnStatus.READY,
"status_exc": spy.ANY
},
),
(CoreEvent.FS_ENTRY_DOWNSYNCED, {
"workspace_id": wid,
"id": foo_id
}),
(CoreEvent.FS_ENTRY_DOWNSYNCED, {
"workspace_id": wid,
"id": bar_id
}),
],
in_order=False,
)
async def _load_realm_role_certificates(
self,
realm_id: Optional[EntryID] = None
) -> Tuple[List[RealmRoleCertificateContent], Dict[UserID, RealmRole]]:
with translate_backend_cmds_errors():
rep = await self.backend_cmds.realm_get_role_certificates(
RealmID((realm_id or self.workspace_id).uuid))
if rep["status"] == "not_allowed":
# Seems we lost the access to the realm
raise FSWorkspaceNoReadAccess(
"Cannot get workspace roles: no read access")
elif rep["status"] != "ok":
raise FSError(
f"Cannot retrieve workspace roles: `{rep['status']}`")
try:
# Must read unverified certificates to access metadata
unsecure_certifs = sorted(
[(RealmRoleCertificateContent.unsecure_load(uv_role), uv_role)
for uv_role in rep["certificates"]],
key=lambda x: x[0].timestamp,
)
current_roles: Dict[UserID, RealmRole] = {}
owner_only = (RealmRole.OWNER, )
owner_or_manager = (RealmRole.OWNER, RealmRole.MANAGER)
# Now verify each certif
for unsecure_certif, raw_certif in unsecure_certifs:
with translate_remote_devices_manager_errors():
author = await self.remote_devices_manager.get_device(
unsecure_certif.author)
RealmRoleCertificateContent.verify_and_load(
raw_certif,
author_verify_key=author.verify_key,
expected_author=author.device_id,
)
# Make sure author had the right to do this
existing_user_role = current_roles.get(unsecure_certif.user_id)
if not current_roles and unsecure_certif.user_id == author.device_id.user_id:
# First user is autosigned
needed_roles: Tuple[Optional[RealmRole], ...] = (None, )
elif (existing_user_role in owner_or_manager
or unsecure_certif.role in owner_or_manager):
needed_roles = owner_only
else:
needed_roles = owner_or_manager
# TODO: typing, author is optional in base.py but it seems that manifests always have an author (no RVK)
if (current_roles.get(
cast(DeviceID, unsecure_certif.author).user_id)
not in needed_roles):
raise FSError(
f"Invalid realm role certificates: "
f"{unsecure_certif.author} has not right to give "
f"{unsecure_certif.role} role to {unsecure_certif.user_id} "
f"on {unsecure_certif.timestamp}")
if unsecure_certif.role is None:
current_roles.pop(unsecure_certif.user_id, None)
else:
current_roles[
unsecure_certif.user_id] = unsecure_certif.role
# Decryption error
except DataError as exc:
raise FSError(f"Invalid realm role certificates: {exc}") from exc
# Now unsecure_certifs is no longer unsecure given we have valided it items
return [c for c, _ in unsecure_certifs], current_roles
async def do_one_batch(self, size: int = 1000) -> Tuple[int, int]:
"""
Raises:
FSError
FSBackendOfflineError
FSWorkspaceInMaintenance
FSWorkspaceNoAccess
"""
workspace_id = RealmID(self.new_workspace_entry.id.uuid)
new_encryption_revision = self.new_workspace_entry.encryption_revision
# Get the batch
try:
rep = await self.backend_cmds.vlob_maintenance_get_reencryption_batch(
workspace_id, new_encryption_revision, size)
if rep["status"] in ("not_in_maintenance",
"bad_encryption_revision"):
raise FSWorkspaceNotInMaintenance(
f"Reencryption job already finished: {rep}")
elif rep["status"] == "not_allowed":
raise FSWorkspaceNoAccess(
f"Not allowed to do reencryption maintenance on workspace {workspace_id}: {rep}"
)
elif rep["status"] != "ok":
raise FSError(
f"Cannot do reencryption maintenance on workspace {workspace_id}: {rep}"
)
donebatch = []
for item in rep["batch"]:
cleartext = self.old_workspace_entry.key.decrypt(item["blob"])
newciphered = self.new_workspace_entry.key.encrypt(cleartext)
donebatch.append(
(item["vlob_id"], item["version"], newciphered))
rep = await self.backend_cmds.vlob_maintenance_save_reencryption_batch(
workspace_id, new_encryption_revision, donebatch)
if rep["status"] in ("not_in_maintenance",
"bad_encryption_revision"):
raise FSWorkspaceNotInMaintenance(
f"Reencryption job already finished: {rep}")
elif rep["status"] == "not_allowed":
raise FSWorkspaceNoAccess(
f"Not allowed to do reencryption maintenance on workspace {workspace_id}: {rep}"
)
elif rep["status"] != "ok":
raise FSError(
f"Cannot do reencryption maintenance on workspace {workspace_id}: {rep}"
)
total = rep["total"]
done = rep["done"]
if total == done:
# Finish the maintenance
rep = await self.backend_cmds.realm_finish_reencryption_maintenance(
workspace_id, new_encryption_revision)
if rep["status"] in ("not_in_maintenance",
"bad_encryption_revision"):
raise FSWorkspaceNotInMaintenance(
f"Reencryption job already finished: {rep}")
elif rep["status"] == "not_allowed":
raise FSWorkspaceNoAccess(
f"Not allowed to do reencryption maintenance on workspace {workspace_id}: {rep}"
)
elif rep["status"] != "ok":
raise FSError(
f"Cannot do reencryption maintenance on workspace {workspace_id}: {rep}"
)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(
f"Cannot do reencryption maintenance on workspace {workspace_id}: {exc}"
) from exc
return total, done
async def workspace_share(
self,
workspace_id: EntryID,
recipient: UserID,
role: Optional[WorkspaceRole],
timestamp_greater_than: Optional[DateTime] = None,
) -> None:
"""
Raises:
FSError
FSWorkspaceNotFoundError
FSBackendOfflineError
FSSharingNotAllowedError
"""
if self.device.user_id == recipient:
raise FSError("Cannot share to oneself")
user_manifest = self.get_user_manifest()
workspace_entry = user_manifest.get_workspace_entry(workspace_id)
if not workspace_entry:
raise FSWorkspaceNotFoundError(
f"Unknown workspace `{workspace_id}`")
# Make sure the workspace is not a placeholder
await self._workspace_minimal_sync(workspace_entry)
# Retrieve the user
recipient_user, revoked_recipient_user = await self.remote_loader.get_user(
recipient)
if revoked_recipient_user:
raise FSError(f"User {recipient} revoked")
# Note we don't bother to check workspace's access roles given they
# could be outdated (and backend will do the check anyway)
timestamp = self.device.timestamp()
if timestamp_greater_than is not None:
timestamp = max(
timestamp,
timestamp_greater_than.add(
microseconds=ROLE_CERTIFICATE_STAMP_AHEAD_US))
# Build the sharing message
try:
if role is not None:
recipient_message: Union[
SharingGrantedMessageContent,
SharingRevokedMessageContent] = SharingGrantedMessageContent(
author=self.device.device_id,
timestamp=timestamp,
name=workspace_entry.name,
id=workspace_entry.id,
encryption_revision=workspace_entry.
encryption_revision,
encrypted_on=workspace_entry.encrypted_on,
key=workspace_entry.key,
)
else:
recipient_message = SharingRevokedMessageContent(
author=self.device.device_id,
timestamp=timestamp,
id=workspace_entry.id)
ciphered_recipient_message = recipient_message.dump_sign_and_encrypt_for(
author_signkey=self.device.signing_key,
recipient_pubkey=recipient_user.public_key)
except DataError as exc:
raise FSError(
f"Cannot create sharing message for `{recipient}`: {exc}"
) from exc
# Build role certificate
role_certificate = RealmRoleCertificateContent(
author=self.device.device_id,
timestamp=timestamp,
realm_id=RealmID(workspace_id.uuid),
user_id=recipient,
role=role,
).dump_and_sign(self.device.signing_key)
# Actually send the command to the backend
try:
rep = await self.backend_cmds.realm_update_roles(
role_certificate, ciphered_recipient_message)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(
f"Error while trying to set vlob group roles in backend: {exc}"
) from exc
if rep["status"] == "not_allowed":
raise FSSharingNotAllowedError(
f"Must be Owner or Manager on the workspace is mandatory to share it: {rep}"
)
elif rep["status"] == "require_greater_timestamp":
return await self.workspace_share(workspace_id, recipient, role,
rep["strictly_greater_than"])
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
f"Cannot share workspace while it is in maintenance: {rep}")
elif rep["status"] == "already_granted":
# Stay idempotent
return
elif rep["status"] != "ok":
raise FSError(
f"Error while trying to set vlob group roles in backend: {rep}"
)
async def _outbound_sync_inner(
self, timestamp_greater_than: Optional[DateTime] = None) -> bool:
base_um = self.get_user_manifest()
if not base_um.need_sync:
return True
# Make sure the corresponding realm has been created in the backend
if base_um.is_placeholder:
certif = RealmRoleCertificateContent.build_realm_root_certif(
author=self.device.device_id,
timestamp=self.device.timestamp(),
realm_id=RealmID(self.device.user_manifest_id.uuid),
).dump_and_sign(self.device.signing_key)
try:
rep = await self.backend_cmds.realm_create(certif)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(
f"Cannot create user manifest's realm in backend: {exc}"
) from exc
if rep["status"] == "already_exists":
# It's possible a previous attempt to create this realm
# succeeded but we didn't receive the confirmation, hence
# we play idempotent here.
pass
elif rep["status"] != "ok":
raise FSError(
f"Cannot create user manifest's realm in backend: {rep}")
# Sync placeholders
for w in base_um.workspaces:
await self._workspace_minimal_sync(w)
# Build vlob
timestamp = self.device.timestamp()
if timestamp_greater_than is not None:
timestamp = max(
timestamp,
timestamp_greater_than.add(
microseconds=MANIFEST_STAMP_AHEAD_US))
to_sync_um = base_um.to_remote(author=self.device.device_id,
timestamp=timestamp)
ciphered = to_sync_um.dump_sign_and_encrypt(
author_signkey=self.device.signing_key,
key=self.device.user_manifest_key)
# Sync the vlob with backend
try:
# Note encryption_revision is always 1 given we never reencrypt
# the user manifest's realm
if to_sync_um.version == 1:
rep = await self.backend_cmds.vlob_create(
RealmID(self.user_manifest_id.uuid),
1,
VlobID(self.user_manifest_id.uuid),
timestamp,
ciphered,
)
else:
rep = await self.backend_cmds.vlob_update(
1, VlobID(self.user_manifest_id.uuid), to_sync_um.version,
timestamp, ciphered)
except BackendNotAvailable as exc:
raise FSBackendOfflineError(str(exc)) from exc
except BackendConnectionError as exc:
raise FSError(f"Cannot sync user manifest: {exc}") from exc
if rep["status"] in ("already_exists", "bad_version"):
# Concurrency error (handled by the caller)
return False
elif rep["status"] == "in_maintenance":
raise FSWorkspaceInMaintenance(
f"Cannot modify workspace data while it is in maintenance: {rep}"
)
elif rep["status"] == "require_greater_timestamp":
return await self._outbound_sync_inner(rep["strictly_greater_than"]
)
elif rep["status"] != "ok":
raise FSError(f"Cannot sync user manifest: {rep}")
# Merge back the manifest in local
async with self._update_user_manifest_lock:
diverged_um = self.get_user_manifest()
# Final merge could have been achieved by a concurrent operation
if to_sync_um.version > diverged_um.base_version:
merged_um = merge_local_user_manifests(diverged_um, to_sync_um)
await self.set_user_manifest(merged_um)
self.event_bus.send(CoreEvent.FS_ENTRY_SYNCED,
id=self.user_manifest_id)
return True
# Parsec Cloud (https://parsec.cloud) Copyright (c) AGPLv3 2016-2021 Scille SAS
import pytest
from parsec.api.protocol import RealmID, VlobID, BlockID
from tests.backend.common import realm_stats
from tests.backend.common import vlob_create, block_create
REALM_ID_FAKE = RealmID.from_hex("00000000-0000-0000-0000-000000000001")
@pytest.mark.trio
async def test_realm_stats_ok(alice_backend_sock, realm):
# Create new data
await block_create(alice_backend_sock,
realm_id=realm,
block_id=BlockID.new(),
block=b"1234")
rep = await realm_stats(alice_backend_sock, realm_id=realm)
assert rep == {"status": "ok", "blocks_size": 4, "vlobs_size": 0}
# Create new metadata
await vlob_create(alice_backend_sock,
realm_id=realm,
vlob_id=VlobID.new(),
blob=b"1234")
rep = await realm_stats(alice_backend_sock, realm_id=realm)
assert rep == {"status": "ok", "blocks_size": 4, "vlobs_size": 4}
async def _create_realm_and_first_vlob(self, device):
manifest = initial_user_manifest_state.get_user_manifest_v1_for_backend(device)
if manifest.author == device.device_id:
author = device
else:
author = self.get_device(device.organization_id, manifest.author)
realm_id = RealmID(author.user_manifest_id.uuid)
vlob_id = VlobID(author.user_manifest_id.uuid)
with self.backend.event_bus.listen() as spy:
# The realm needs to be created srictly before the manifest timestamp
realm_create_timestamp = manifest.timestamp.subtract(microseconds=1)
await self.backend.realm.create(
organization_id=author.organization_id,
self_granted_role=RealmGrantedRole(
realm_id=realm_id,
user_id=author.user_id,
certificate=RealmRoleCertificateContent(
author=author.device_id,
timestamp=realm_create_timestamp,
realm_id=realm_id,
user_id=author.user_id,
role=RealmRole.OWNER,
).dump_and_sign(author.signing_key),
role=RealmRole.OWNER,
granted_by=author.device_id,
granted_on=realm_create_timestamp,
),
)
await self.backend.vlob.create(
organization_id=author.organization_id,
author=author.device_id,
realm_id=realm_id,
encryption_revision=1,
vlob_id=vlob_id,
timestamp=manifest.timestamp,
blob=manifest.dump_sign_and_encrypt(
author_signkey=author.signing_key, key=author.user_manifest_key
),
)
# Avoid possible race condition in tests listening for events
await spy.wait_multiple(
[
(
BackendEvent.REALM_ROLES_UPDATED,
{
"organization_id": author.organization_id,
"author": author.device_id,
"realm_id": realm_id,
"user": author.user_id,
"role": RealmRole.OWNER,
},
),
(
BackendEvent.REALM_VLOBS_UPDATED,
{
"organization_id": author.organization_id,
"author": author.device_id,
"realm_id": realm_id,
"checkpoint": 1,
"src_id": vlob_id,
"src_version": 1,
},
),
]
)
# Parsec Cloud (https://parsec.cloud) Copyright (c) AGPLv3 2016-2021 Scille SAS
import pytest
from pendulum import datetime
from unittest.mock import ANY
from parsec.api.protocol import VlobID, RealmID, RealmRole
from parsec.api.data import RealmRoleCertificateContent, UserProfile
from parsec.backend.realm import RealmGrantedRole
from tests.common import freeze_time, customize_fixtures
from tests.backend.common import realm_update_roles, realm_get_role_certificates, vlob_create
NOW = datetime(2000, 1, 1)
VLOB_ID = VlobID.from_hex("00000000000000000000000000000001")
REALM_ID = RealmID.from_hex("0000000000000000000000000000000A")
@pytest.mark.trio
async def test_get_roles_not_found(alice_backend_sock):
rep = await realm_get_role_certificates(alice_backend_sock, REALM_ID)
assert rep == {
"status": "not_found",
"reason": "Realm `0000000000000000000000000000000a` doesn't exist",
}
async def _realm_get_clear_role_certifs(sock, realm_id):
rep = await realm_get_role_certificates(sock, realm_id)
assert rep["status"] == "ok"
cooked = [
async def bob_realm(backend, bob, realm_factory):
realm_id = RealmID.from_hex("C0000000000000000000000000000000")
return await realm_factory(backend, bob, realm_id, datetime(2000, 1, 2))
