async def test_create_certif_too_old(alice, alice_backend_sock): now = pendulum.now() # Generate a certificate realm_id = RealmID.from_hex("C0000000000000000000000000000000") certif = RealmRoleCertificateContent.build_realm_root_certif( author=alice.device_id, timestamp=now, realm_id=realm_id).dump_and_sign(alice.signing_key) # Create a realm a tiny bit too late later = now.add(seconds=BALLPARK_CLIENT_LATE_OFFSET) with freeze_time(later): rep = await realm_create(alice_backend_sock, certif) assert rep == { "status": "bad_timestamp", "backend_timestamp": later, "ballpark_client_early_offset": BALLPARK_CLIENT_EARLY_OFFSET, "ballpark_client_late_offset": BALLPARK_CLIENT_LATE_OFFSET, "client_timestamp": now, } # Create a realm late but right before the deadline later = now.add(seconds=BALLPARK_CLIENT_LATE_OFFSET, microseconds=-1) with freeze_time(later): rep = await realm_create(alice_backend_sock, certif) assert rep["status"] == "ok" # Generate a new certificate realm_id = RealmID.from_hex("C0000000000000000000000000000001") certif = RealmRoleCertificateContent.build_realm_root_certif( author=alice.device_id, timestamp=now, realm_id=realm_id).dump_and_sign(alice.signing_key) # Create a realm a tiny bit too soon sooner = now.subtract(seconds=BALLPARK_CLIENT_EARLY_OFFSET) with freeze_time(sooner): rep = await realm_create(alice_backend_sock, certif) assert rep == { "status": "bad_timestamp", "backend_timestamp": sooner, "ballpark_client_early_offset": BALLPARK_CLIENT_EARLY_OFFSET, "ballpark_client_late_offset": BALLPARK_CLIENT_LATE_OFFSET, "client_timestamp": now, } # Create a realm soon but after the limit sooner = now.subtract(seconds=BALLPARK_CLIENT_EARLY_OFFSET, microseconds=-1) with freeze_time(sooner): rep = await realm_create(alice_backend_sock, certif) assert rep["status"] == "ok"
async def test_get_reencryption_need(alice_workspace, running_backend, monkeypatch): expected = ReencryptionNeed(user_revoked=(), role_revoked=()) assert await alice_workspace.get_reencryption_need() == expected with running_backend.offline(): with pytest.raises(FSBackendOfflineError): await alice_workspace.get_reencryption_need() # Reproduce a backend offline after the certificates have been retrieved (see issue #1335) reply = await alice_workspace.remote_loader.backend_cmds.realm_get_role_certificates( RealmID(alice_workspace.workspace_id.uuid)) original = alice_workspace.remote_loader.backend_cmds.realm_get_role_certificates async def mockup(*args): if args == (alice_workspace.workspace_id, ): return reply return await original(*args) monkeypatch.setattr(alice_workspace.remote_loader.backend_cmds, "realm_get_role_certificates", mockup) with running_backend.offline(): with pytest.raises(FSBackendOfflineError): await alice_workspace.get_reencryption_need()
async def read( self, organization_id: OrganizationID, author: DeviceID, block_id: BlockID ) -> bytes: async with self.dbh.pool.acquire() as conn, conn.transaction(): realm_id_uuid = await conn.fetchval( *_q_get_realm_id_from_block_id( organization_id=organization_id.str, block_id=block_id.uuid ) ) if not realm_id_uuid: raise BlockNotFoundError() realm_id = RealmID(realm_id_uuid) await _check_realm(conn, organization_id, realm_id, OperationKind.DATA_READ) ret = await conn.fetchrow( *_q_get_block_meta( organization_id=organization_id.str, block_id=block_id.uuid, user_id=author.user_id.str, ) ) if not ret or ret["deleted_on"]: raise BlockNotFoundError() elif not ret["has_access"]: raise BlockAccessError() # We can do the blockstore read outside of the transaction given the block # are never modified/removed return await self._blockstore_component.read(organization_id, block_id)
async def _get_realm_id_from_vlob_id(conn, organization_id: OrganizationID, vlob_id: VlobID) -> RealmID: realm_id_uuid = await conn.fetchval(*_q_get_realm_id_from_vlob_id( organization_id=organization_id.str, vlob_id=vlob_id.uuid)) if not realm_id_uuid: raise VlobNotFoundError(f"Vlob `{vlob_id}` doesn't exist") return RealmID(realm_id_uuid)
async def get_reencryption_need(self) -> ReencryptionNeed: """ Raises: FSError FSBackendOfflineError FSWorkspaceNoAccess """ wentry = self.get_workspace_entry() try: workspace_manifest = self.local_storage.get_workspace_manifest() if workspace_manifest.is_placeholder and not workspace_manifest.speculative: return ReencryptionNeed(user_revoked=(), role_revoked=(), reencryption_already_in_progress=False) except FSLocalMissError: pass try: rep = await self.backend_cmds.realm_status( RealmID(self.workspace_id.uuid)) except BackendNotAvailable as exc: raise FSBackendOfflineError(str(exc)) from exc except BackendConnectionError as exc: raise FSError( f"Cannot retreive remote status for workspace {self.workspace_id}: {exc}" ) from exc reencryption_already_in_progress = (rep["in_maintenance"] and rep["maintenance_type"] == MaintenanceType.REENCRYPTION) certificates = await self.remote_loader.load_realm_role_certificates() has_role = set() role_revoked = set() for certif in certificates: if certif.role is None: if certif.timestamp > wentry.encrypted_on: role_revoked.add(certif.user_id) has_role.discard(certif.user_id) else: role_revoked.discard(certif.user_id) has_role.add(certif.user_id) user_revoked = [] for user_id in has_role: _, revoked_user = await self.remote_loader.get_user(user_id, no_cache=True) if revoked_user and revoked_user.timestamp > wentry.encrypted_on: user_revoked.append(user_id) return ReencryptionNeed( user_revoked=tuple(user_revoked), role_revoked=tuple(role_revoked), reencryption_already_in_progress=reencryption_already_in_progress, )
async def query_get_realms_for_user( conn, organization_id: OrganizationID, user: UserID) -> Dict[RealmID, Optional[RealmRole]]: rep = await conn.fetch(*_q_get_realms_for_user( organization_id=organization_id.str, user_id=user.str)) return { RealmID(row["realm_id"]): RealmRole(row["role"]) for row in rep if row["role"] is not None }
async def test_create_but_unknown_realm(alice_backend_sock): bad_realm_id = RealmID.new() blob = b"Initial commit." rep = await vlob_create(alice_backend_sock, bad_realm_id, VLOB_ID, blob, check_rep=False) assert rep["status"] == "not_allowed"
async def _test_create_ok(backend, device, device_backend_sock): await events_subscribe(device_backend_sock) realm_id = RealmID.from_hex("C0000000000000000000000000000000") certif = RealmRoleCertificateContent.build_realm_root_certif( author=device.device_id, timestamp=pendulum.now(), realm_id=realm_id).dump_and_sign(device.signing_key) with backend.event_bus.listen() as spy: rep = await realm_create(device_backend_sock, certif) assert rep == {"status": "ok"} await spy.wait_with_timeout(BackendEvent.REALM_ROLES_UPDATED)
async def test_create_invalid_certif(bob, alice_backend_sock): realm_id = RealmID.from_hex("C0000000000000000000000000000000") certif = RealmRoleCertificateContent.build_realm_root_certif( author=bob.device_id, timestamp=pendulum.now(), realm_id=realm_id).dump_and_sign(bob.signing_key) rep = await realm_create(alice_backend_sock, certif) assert rep == { "status": "invalid_certification", "reason": "Invalid certification data (Signature was forged or corrupt).", }
async def _load_changes(self) -> bool: if self._changes_loaded: return True # Initialize due_time so that if we cannot retrieve the changes, we # will wait until an external event (most likely a `sharing.updated`) # make it worth to retry self.due_time = math.inf # 1) Fetch new checkpoint and changes realm_checkpoint = await self._get_local_storage().get_realm_checkpoint() try: rep = await self._get_backend_cmds().vlob_poll_changes( RealmID(self.id.uuid), realm_checkpoint ) except BackendNotAvailable: raise # Another backend error except BackendConnectionError as exc: logger.warning("Unexpected backend response during sync bootstrap", exc_info=exc) return False if rep["status"] == "not_found": # Workspace not yet synchronized with backend new_checkpoint = 0 changes = {} elif rep["status"] in ("in_maintenance", "not_allowed"): return False elif rep["status"] != "ok": return False else: new_checkpoint = rep["current_checkpoint"] changes = rep["changes"] # 2) Store new checkpoint and changes await self._get_local_storage().update_realm_checkpoint( new_checkpoint, {EntryID.from_hex(name.hex): val for name, val in changes.items()} ) # 3) Compute local and remote changes that need to be synced need_sync_local, need_sync_remote = await self._get_local_storage().get_need_sync_entries() now = current_time() # Ignore local changes in read only mode if not self.read_only: self._local_changes = {entry_id: LocalChange(now) for entry_id in need_sync_local} self._remote_changes = need_sync_remote # 4) Finally refresh due time according to the changes self._compute_due_time() self._changes_loaded = True return True
async def _update_role(self, author, user, role=RealmRole.MANAGER): now = pendulum_now() certif = RealmRoleCertificateContent( author=author.device_id, timestamp=now, realm_id=RealmID(self.wid.uuid), user_id=user.user_id, role=role, ).dump_and_sign(author.signing_key) await self.backend.realm.update_roles( author.organization_id, RealmGrantedRole( certificate=certif, realm_id=RealmID(self.wid.uuid), user_id=user.user_id, role=role, granted_by=author.device_id, granted_on=now, ), ) return certif
async def workspace_continue_reencryption( self, workspace_id: EntryID) -> ReencryptionJob: """ Raises: FSError FSBackendOfflineError FSWorkspaceNoAccess FSWorkspaceNotFoundError """ user_manifest = self.get_user_manifest() workspace_entry = user_manifest.get_workspace_entry(workspace_id) if not workspace_entry: raise FSWorkspaceNotFoundError( f"Unknown workspace `{workspace_id}`") # First make sure the workspace is under maintenance try: rep = await self.backend_cmds.realm_status( RealmID(workspace_entry.id.uuid)) except BackendNotAvailable as exc: raise FSBackendOfflineError(str(exc)) from exc except BackendConnectionError as exc: raise FSError( f"Cannot continue maintenance on workspace {workspace_id}: {exc}" ) from exc if rep["status"] == "not_allowed": raise FSWorkspaceNoAccess( f"Not allowed to access workspace {workspace_id}: {rep}") elif rep["status"] != "ok": raise FSError( f"Error while getting status for workspace {workspace_id}: {rep}" ) if not rep["in_maintenance"] or rep[ "maintenance_type"] != MaintenanceType.REENCRYPTION: raise FSWorkspaceNotInMaintenance( "Not in reencryption maintenance") if rep["encryption_revision"] != workspace_entry.encryption_revision: raise FSError("Bad encryption revision") previous_workspace_entry = await self._get_previous_workspace_entry( workspace_entry) if not previous_workspace_entry: raise FSError( f"Never had access to encryption revision {workspace_entry.encryption_revision - 1}" ) return ReencryptionJob(self.backend_cmds, workspace_entry, previous_workspace_entry)
async def test_create_certif_role_not_owner(alice, alice_backend_sock): realm_id = RealmID.from_hex("C0000000000000000000000000000000") certif = RealmRoleCertificateContent( author=alice.device_id, timestamp=pendulum.now(), realm_id=realm_id, user_id=alice.user_id, role=RealmRole.MANAGER, ).dump_and_sign(alice.signing_key) rep = await realm_create(alice_backend_sock, certif) assert rep == { "status": "invalid_data", "reason": "Initial realm role certificate must set OWNER role.", }
async def test_share_no_manager_right(running_backend, alice_user_fs, alice, bob): with freeze_time("2000-01-02"): wid = await alice_user_fs.workspace_create(EntryName("w1")) await alice_user_fs.sync() # Drop manager right (and give to Bob the ownership) await running_backend.backend.realm.update_roles( alice.organization_id, RealmGrantedRole( realm_id=RealmID(wid.uuid), user_id=bob.user_id, certificate=b"<dummy>", role=RealmRole.OWNER, granted_by=alice.device_id, granted_on=datetime(2000, 1, 3), ), ) await running_backend.backend.realm.update_roles( alice.organization_id, RealmGrantedRole( realm_id=RealmID(wid.uuid), user_id=alice.user_id, certificate=b"<dummy>", role=RealmRole.CONTRIBUTOR, granted_by=bob.device_id, granted_on=datetime(2000, 1, 4), ), ) with pytest.raises(FSSharingNotAllowedError) as exc: await alice_user_fs.workspace_share(wid, bob.user_id, WorkspaceRole.MANAGER) assert ( exc.value.message == "Must be Owner or Manager on the workspace is mandatory to share it: {'status': 'not_allowed'}" )
async def test_do_reencryption(running_backend, workspace, alice, alice_user_fs): with running_backend.backend.event_bus.listen() as spy: job = await alice_user_fs.workspace_start_reencryption(workspace) # Check events await spy.wait_multiple_with_timeout( [ ( BackendEvent.REALM_MAINTENANCE_STARTED, { "organization_id": alice.organization_id, "author": alice.device_id, "realm_id": RealmID(workspace.uuid), "encryption_revision": 2, }, ), ( BackendEvent.MESSAGE_RECEIVED, { "organization_id": alice.organization_id, "author": alice.device_id, "recipient": alice.user_id, "index": 1, }, ), ] ) total, done = await job.do_one_batch(size=1) assert total == 4 assert done == 1 total, done = await job.do_one_batch(size=2) assert total == 4 assert done == 3 total, done = await job.do_one_batch(size=2) assert total == 4 assert done == 4 with pytest.raises(FSWorkspaceNotInMaintenance): await job.do_one_batch()
async def _send_start_reencryption_cmd( self, workspace_id: EntryID, encryption_revision: int, timestamp: DateTime, per_user_ciphered_msgs: Dict[UserID, bytes], ) -> bool: """ Raises: FSError FSBackendOfflineError FSWorkspaceNoAccess BackendCmdsParticipantsMismatchError """ # Finally send command to the backend try: rep = await self.backend_cmds.realm_start_reencryption_maintenance( RealmID(workspace_id.uuid), encryption_revision, timestamp, per_user_ciphered_msgs) except BackendNotAvailable as exc: raise FSBackendOfflineError(str(exc)) from exc except BackendConnectionError as exc: raise FSError( f"Cannot start maintenance on workspace {workspace_id}: {exc}" ) from exc if rep["status"] == "participants_mismatch": # Catched by caller return False elif rep["status"] == "in_maintenance": raise FSWorkspaceInMaintenance( f"Workspace {workspace_id} already in maintenance: {rep}") elif rep["status"] == "not_allowed": raise FSWorkspaceNoAccess( f"Not allowed to start maintenance on workspace {workspace_id}: {rep}" ) elif rep["status"] != "ok": raise FSError( f"Cannot start maintenance on workspace {workspace_id}: {rep}") return True
async def _realm_factory(backend, author, realm_id=None, now=None): realm_id = realm_id or RealmID.new() now = now or next_timestamp() certif = RealmRoleCertificateContent.build_realm_root_certif( author=author.device_id, timestamp=now, realm_id=realm_id).dump_and_sign(author.signing_key) with backend.event_bus.listen() as spy: await backend.realm.create( organization_id=author.organization_id, self_granted_role=RealmGrantedRole( realm_id=realm_id, user_id=author.user_id, certificate=certif, role=RealmRole.OWNER, granted_by=author.device_id, granted_on=now, ), ) await spy.wait_with_timeout(BackendEvent.REALM_ROLES_UPDATED) return realm_id
async def upload_block(self, access: BlockAccess, data: bytes) -> None: """ Raises: FSError FSBackendOfflineError FSRemoteOperationError FSWorkspaceInMaintenance FSWorkspaceNoAccess """ # Encryption try: ciphered = access.key.encrypt(data) # Encryption error except CryptoError as exc: raise FSError(f"Cannot encrypt block: {exc}") from exc # Upload block with translate_backend_cmds_errors(): rep = await self.backend_cmds.block_create( access.id, RealmID(self.workspace_id.uuid), ciphered) if rep["status"] == "already_exists": # Ignore exception if the block has already been uploaded # This might happen when a failure occurs before the local storage is updated pass elif rep["status"] == "not_allowed": # Seems we lost the access to the realm raise FSWorkspaceNoWriteAccess( "Cannot upload block: no write access") elif rep["status"] == "in_maintenance": raise FSWorkspaceInMaintenance( "Cannot upload block while the workspace in maintenance") elif rep["status"] != "ok": raise FSError(f"Cannot upload block: {rep}") # Update local storage await self.local_storage.set_clean_block(access.id, data) await self.local_storage.clear_chunk(ChunkID(access.id.uuid), miss_ok=True)
async def _vlob_create(self, encryption_revision: int, entry_id: EntryID, ciphered: bytes, now: DateTime) -> None: """ Raises: FSError FSRemoteSyncError FSBackendOfflineError FSRemoteOperationError FSWorkspaceInMaintenance FSBadEncryptionRevision FSWorkspaceNoAccess """ # Vlob upload with translate_backend_cmds_errors(): rep = await self.backend_cmds.vlob_create( RealmID(self.workspace_id.uuid), encryption_revision, VlobID(entry_id.uuid), now, ciphered, ) if rep["status"] == "already_exists": raise FSRemoteSyncError(entry_id) elif rep["status"] == "not_allowed": # Seems we lost the access to the realm raise FSWorkspaceNoWriteAccess( "Cannot upload manifest: no write access") elif rep["status"] == "require_greater_timestamp": raise VlobRequireGreaterTimestampError( rep["strictly_greater_than"]) elif rep["status"] == "bad_encryption_revision": raise FSBadEncryptionRevision( f"Cannot create vlob {entry_id}: Bad encryption revision provided" ) elif rep["status"] == "in_maintenance": raise FSWorkspaceInMaintenance( "Cannot create vlob while the workspace is in maintenance") elif rep["status"] != "ok": raise FSError(f"Cannot create vlob {entry_id}: `{rep['status']}`")
async def create_realm(self, realm_id: EntryID) -> None: """ Raises: FSError FSRemoteOperationError FSBackendOfflineError """ timestamp = self.device.timestamp() certif = RealmRoleCertificateContent.build_realm_root_certif( author=self.device.device_id, timestamp=timestamp, realm_id=RealmID(realm_id.uuid)).dump_and_sign( self.device.signing_key) with translate_backend_cmds_errors(): rep = await self.backend_cmds.realm_create(certif) if rep["status"] == "already_exists": # It's possible a previous attempt to create this realm # succeeded but we didn't receive the confirmation, hence # we play idempotent here. return elif rep["status"] != "ok": raise FSError(f"Cannot create realm {realm_id}: `{rep['status']}`")
async def test_vlobs_updated_event_realm_created_after_subscribe( backend, alice_backend_sock, alice, alice2, realm_created_by_self): realm_id = RealmID.from_hex("0000000000000000000000000000000A") await events_subscribe(alice_backend_sock) # New realm, should get events anyway with backend.event_bus.listen() as spy: realm_creator = alice if realm_created_by_self else alice2 # Create the realm await backend.realm.create( organization_id=realm_creator.organization_id, self_granted_role=RealmGrantedRole( realm_id=realm_id, user_id=realm_creator.user_id, certificate=b"<dummy>", role=RealmRole.OWNER, granted_by=realm_creator.device_id, granted_on=datetime(2000, 1, 2), ), ) # Create vlob in realm await backend.vlob.create( organization_id=realm_creator.organization_id, author=realm_creator.device_id, realm_id=realm_id, encryption_revision=1, vlob_id=VLOB_ID, timestamp=NOW, blob=b"v1", ) # Update vlob in realm await backend.vlob.update( organization_id=alice2.organization_id, author=alice2.device_id, encryption_revision=1, vlob_id=VLOB_ID, version=2, timestamp=NOW, blob=b"v2", ) # Wait for events to be processed by the backend await spy.wait_multiple_with_timeout([ BackendEvent.REALM_ROLES_UPDATED, BackendEvent.REALM_VLOBS_UPDATED, BackendEvent.REALM_VLOBS_UPDATED, ]) # Realm access granted rep = await events_listen_nowait(alice_backend_sock) assert rep == { "status": "ok", "event": APIEvent.REALM_ROLES_UPDATED, "realm_id": realm_id, "role": RealmRole.OWNER, } # Create vlob in realm event if not realm_created_by_self: rep = await events_listen_nowait(alice_backend_sock) assert rep == { "status": "ok", "event": APIEvent.REALM_VLOBS_UPDATED, "realm_id": realm_id, "checkpoint": 1, "src_id": VLOB_ID, "src_version": 1, } # Update vlob in realm event rep = await events_listen_nowait(alice_backend_sock) assert rep == { "status": "ok", "event": APIEvent.REALM_VLOBS_UPDATED, "realm_id": realm_id, "checkpoint": 2, "src_id": VLOB_ID, "src_version": 2, } rep = await events_listen_nowait(alice_backend_sock) assert rep == {"status": "no_events"}
async def test_reconnect_with_remote_changes(frozen_clock, alice2, running_backend, server_factory, alice_core, user_fs_factory): wid = await alice_core.user_fs.workspace_create(EntryName("w")) alice_w = alice_core.user_fs.get_workspace(wid) await alice_w.mkdir("/foo") await alice_w.touch("/bar.txt") # Wait for sync monitor to do it job await frozen_clock.sleep_with_autojump(60) async with frozen_clock.real_clock_timeout(): await alice_core.wait_idle_monitors() # Alice2 connect to the backend through a different server so that we can # switch alice offline while keeping alice2 connected async with server_factory(running_backend.backend.handle_client) as server: alice2 = server.correct_addr(alice2) async with user_fs_factory(alice2) as alice2_user_fs: # Switch backend offline for alice (but not alice2 !) with running_backend.offline(): # Get back modifications from alice await alice2_user_fs.sync() alice2_w = alice2_user_fs.get_workspace(wid) await alice2_w.sync() # Modify the workspace while alice is offline await alice2_w.mkdir("/foo/spam") await alice2_w.write_bytes("/bar.txt", b"v2") foo_id = await alice2_w.path_id("/foo") spam_id = await alice2_w.path_id("/foo/spam") bar_id = await alice2_w.path_id("/bar.txt") with running_backend.backend.event_bus.listen() as spy: await alice2_w.sync() # Alice misses the vlob updated events before being back online await spy.wait_multiple_with_timeout( [ ( BackendEvent.REALM_VLOBS_UPDATED, { "organization_id": alice2.organization_id, "author": alice2.device_id, "realm_id": RealmID(wid.uuid), "checkpoint": ANY, "src_id": VlobID(spam_id.uuid), "src_version": 1, }, ), ( BackendEvent.REALM_VLOBS_UPDATED, { "organization_id": alice2.organization_id, "author": alice2.device_id, "realm_id": RealmID(wid.uuid), "checkpoint": ANY, "src_id": VlobID(foo_id.uuid), "src_version": 2, }, ), ( BackendEvent.REALM_VLOBS_UPDATED, { "organization_id": alice2.organization_id, "author": alice2.device_id, "realm_id": RealmID(wid.uuid), "checkpoint": ANY, "src_id": VlobID(bar_id.uuid), "src_version": 2, }, ), ], in_order=False, ) with alice_core.event_bus.listen() as spy: # Now alice should sync back the changes await frozen_clock.sleep_with_autojump(60) await spy.wait_multiple_with_timeout( [ ( CoreEvent.BACKEND_CONNECTION_CHANGED, { "status": BackendConnStatus.READY, "status_exc": spy.ANY }, ), (CoreEvent.FS_ENTRY_DOWNSYNCED, { "workspace_id": wid, "id": foo_id }), (CoreEvent.FS_ENTRY_DOWNSYNCED, { "workspace_id": wid, "id": bar_id }), ], in_order=False, )
async def _load_realm_role_certificates( self, realm_id: Optional[EntryID] = None ) -> Tuple[List[RealmRoleCertificateContent], Dict[UserID, RealmRole]]: with translate_backend_cmds_errors(): rep = await self.backend_cmds.realm_get_role_certificates( RealmID((realm_id or self.workspace_id).uuid)) if rep["status"] == "not_allowed": # Seems we lost the access to the realm raise FSWorkspaceNoReadAccess( "Cannot get workspace roles: no read access") elif rep["status"] != "ok": raise FSError( f"Cannot retrieve workspace roles: `{rep['status']}`") try: # Must read unverified certificates to access metadata unsecure_certifs = sorted( [(RealmRoleCertificateContent.unsecure_load(uv_role), uv_role) for uv_role in rep["certificates"]], key=lambda x: x[0].timestamp, ) current_roles: Dict[UserID, RealmRole] = {} owner_only = (RealmRole.OWNER, ) owner_or_manager = (RealmRole.OWNER, RealmRole.MANAGER) # Now verify each certif for unsecure_certif, raw_certif in unsecure_certifs: with translate_remote_devices_manager_errors(): author = await self.remote_devices_manager.get_device( unsecure_certif.author) RealmRoleCertificateContent.verify_and_load( raw_certif, author_verify_key=author.verify_key, expected_author=author.device_id, ) # Make sure author had the right to do this existing_user_role = current_roles.get(unsecure_certif.user_id) if not current_roles and unsecure_certif.user_id == author.device_id.user_id: # First user is autosigned needed_roles: Tuple[Optional[RealmRole], ...] = (None, ) elif (existing_user_role in owner_or_manager or unsecure_certif.role in owner_or_manager): needed_roles = owner_only else: needed_roles = owner_or_manager # TODO: typing, author is optional in base.py but it seems that manifests always have an author (no RVK) if (current_roles.get( cast(DeviceID, unsecure_certif.author).user_id) not in needed_roles): raise FSError( f"Invalid realm role certificates: " f"{unsecure_certif.author} has not right to give " f"{unsecure_certif.role} role to {unsecure_certif.user_id} " f"on {unsecure_certif.timestamp}") if unsecure_certif.role is None: current_roles.pop(unsecure_certif.user_id, None) else: current_roles[ unsecure_certif.user_id] = unsecure_certif.role # Decryption error except DataError as exc: raise FSError(f"Invalid realm role certificates: {exc}") from exc # Now unsecure_certifs is no longer unsecure given we have valided it items return [c for c, _ in unsecure_certifs], current_roles
async def do_one_batch(self, size: int = 1000) -> Tuple[int, int]: """ Raises: FSError FSBackendOfflineError FSWorkspaceInMaintenance FSWorkspaceNoAccess """ workspace_id = RealmID(self.new_workspace_entry.id.uuid) new_encryption_revision = self.new_workspace_entry.encryption_revision # Get the batch try: rep = await self.backend_cmds.vlob_maintenance_get_reencryption_batch( workspace_id, new_encryption_revision, size) if rep["status"] in ("not_in_maintenance", "bad_encryption_revision"): raise FSWorkspaceNotInMaintenance( f"Reencryption job already finished: {rep}") elif rep["status"] == "not_allowed": raise FSWorkspaceNoAccess( f"Not allowed to do reencryption maintenance on workspace {workspace_id}: {rep}" ) elif rep["status"] != "ok": raise FSError( f"Cannot do reencryption maintenance on workspace {workspace_id}: {rep}" ) donebatch = [] for item in rep["batch"]: cleartext = self.old_workspace_entry.key.decrypt(item["blob"]) newciphered = self.new_workspace_entry.key.encrypt(cleartext) donebatch.append( (item["vlob_id"], item["version"], newciphered)) rep = await self.backend_cmds.vlob_maintenance_save_reencryption_batch( workspace_id, new_encryption_revision, donebatch) if rep["status"] in ("not_in_maintenance", "bad_encryption_revision"): raise FSWorkspaceNotInMaintenance( f"Reencryption job already finished: {rep}") elif rep["status"] == "not_allowed": raise FSWorkspaceNoAccess( f"Not allowed to do reencryption maintenance on workspace {workspace_id}: {rep}" ) elif rep["status"] != "ok": raise FSError( f"Cannot do reencryption maintenance on workspace {workspace_id}: {rep}" ) total = rep["total"] done = rep["done"] if total == done: # Finish the maintenance rep = await self.backend_cmds.realm_finish_reencryption_maintenance( workspace_id, new_encryption_revision) if rep["status"] in ("not_in_maintenance", "bad_encryption_revision"): raise FSWorkspaceNotInMaintenance( f"Reencryption job already finished: {rep}") elif rep["status"] == "not_allowed": raise FSWorkspaceNoAccess( f"Not allowed to do reencryption maintenance on workspace {workspace_id}: {rep}" ) elif rep["status"] != "ok": raise FSError( f"Cannot do reencryption maintenance on workspace {workspace_id}: {rep}" ) except BackendNotAvailable as exc: raise FSBackendOfflineError(str(exc)) from exc except BackendConnectionError as exc: raise FSError( f"Cannot do reencryption maintenance on workspace {workspace_id}: {exc}" ) from exc return total, done
async def workspace_share( self, workspace_id: EntryID, recipient: UserID, role: Optional[WorkspaceRole], timestamp_greater_than: Optional[DateTime] = None, ) -> None: """ Raises: FSError FSWorkspaceNotFoundError FSBackendOfflineError FSSharingNotAllowedError """ if self.device.user_id == recipient: raise FSError("Cannot share to oneself") user_manifest = self.get_user_manifest() workspace_entry = user_manifest.get_workspace_entry(workspace_id) if not workspace_entry: raise FSWorkspaceNotFoundError( f"Unknown workspace `{workspace_id}`") # Make sure the workspace is not a placeholder await self._workspace_minimal_sync(workspace_entry) # Retrieve the user recipient_user, revoked_recipient_user = await self.remote_loader.get_user( recipient) if revoked_recipient_user: raise FSError(f"User {recipient} revoked") # Note we don't bother to check workspace's access roles given they # could be outdated (and backend will do the check anyway) timestamp = self.device.timestamp() if timestamp_greater_than is not None: timestamp = max( timestamp, timestamp_greater_than.add( microseconds=ROLE_CERTIFICATE_STAMP_AHEAD_US)) # Build the sharing message try: if role is not None: recipient_message: Union[ SharingGrantedMessageContent, SharingRevokedMessageContent] = SharingGrantedMessageContent( author=self.device.device_id, timestamp=timestamp, name=workspace_entry.name, id=workspace_entry.id, encryption_revision=workspace_entry. encryption_revision, encrypted_on=workspace_entry.encrypted_on, key=workspace_entry.key, ) else: recipient_message = SharingRevokedMessageContent( author=self.device.device_id, timestamp=timestamp, id=workspace_entry.id) ciphered_recipient_message = recipient_message.dump_sign_and_encrypt_for( author_signkey=self.device.signing_key, recipient_pubkey=recipient_user.public_key) except DataError as exc: raise FSError( f"Cannot create sharing message for `{recipient}`: {exc}" ) from exc # Build role certificate role_certificate = RealmRoleCertificateContent( author=self.device.device_id, timestamp=timestamp, realm_id=RealmID(workspace_id.uuid), user_id=recipient, role=role, ).dump_and_sign(self.device.signing_key) # Actually send the command to the backend try: rep = await self.backend_cmds.realm_update_roles( role_certificate, ciphered_recipient_message) except BackendNotAvailable as exc: raise FSBackendOfflineError(str(exc)) from exc except BackendConnectionError as exc: raise FSError( f"Error while trying to set vlob group roles in backend: {exc}" ) from exc if rep["status"] == "not_allowed": raise FSSharingNotAllowedError( f"Must be Owner or Manager on the workspace is mandatory to share it: {rep}" ) elif rep["status"] == "require_greater_timestamp": return await self.workspace_share(workspace_id, recipient, role, rep["strictly_greater_than"]) elif rep["status"] == "in_maintenance": raise FSWorkspaceInMaintenance( f"Cannot share workspace while it is in maintenance: {rep}") elif rep["status"] == "already_granted": # Stay idempotent return elif rep["status"] != "ok": raise FSError( f"Error while trying to set vlob group roles in backend: {rep}" )
async def _outbound_sync_inner( self, timestamp_greater_than: Optional[DateTime] = None) -> bool: base_um = self.get_user_manifest() if not base_um.need_sync: return True # Make sure the corresponding realm has been created in the backend if base_um.is_placeholder: certif = RealmRoleCertificateContent.build_realm_root_certif( author=self.device.device_id, timestamp=self.device.timestamp(), realm_id=RealmID(self.device.user_manifest_id.uuid), ).dump_and_sign(self.device.signing_key) try: rep = await self.backend_cmds.realm_create(certif) except BackendNotAvailable as exc: raise FSBackendOfflineError(str(exc)) from exc except BackendConnectionError as exc: raise FSError( f"Cannot create user manifest's realm in backend: {exc}" ) from exc if rep["status"] == "already_exists": # It's possible a previous attempt to create this realm # succeeded but we didn't receive the confirmation, hence # we play idempotent here. pass elif rep["status"] != "ok": raise FSError( f"Cannot create user manifest's realm in backend: {rep}") # Sync placeholders for w in base_um.workspaces: await self._workspace_minimal_sync(w) # Build vlob timestamp = self.device.timestamp() if timestamp_greater_than is not None: timestamp = max( timestamp, timestamp_greater_than.add( microseconds=MANIFEST_STAMP_AHEAD_US)) to_sync_um = base_um.to_remote(author=self.device.device_id, timestamp=timestamp) ciphered = to_sync_um.dump_sign_and_encrypt( author_signkey=self.device.signing_key, key=self.device.user_manifest_key) # Sync the vlob with backend try: # Note encryption_revision is always 1 given we never reencrypt # the user manifest's realm if to_sync_um.version == 1: rep = await self.backend_cmds.vlob_create( RealmID(self.user_manifest_id.uuid), 1, VlobID(self.user_manifest_id.uuid), timestamp, ciphered, ) else: rep = await self.backend_cmds.vlob_update( 1, VlobID(self.user_manifest_id.uuid), to_sync_um.version, timestamp, ciphered) except BackendNotAvailable as exc: raise FSBackendOfflineError(str(exc)) from exc except BackendConnectionError as exc: raise FSError(f"Cannot sync user manifest: {exc}") from exc if rep["status"] in ("already_exists", "bad_version"): # Concurrency error (handled by the caller) return False elif rep["status"] == "in_maintenance": raise FSWorkspaceInMaintenance( f"Cannot modify workspace data while it is in maintenance: {rep}" ) elif rep["status"] == "require_greater_timestamp": return await self._outbound_sync_inner(rep["strictly_greater_than"] ) elif rep["status"] != "ok": raise FSError(f"Cannot sync user manifest: {rep}") # Merge back the manifest in local async with self._update_user_manifest_lock: diverged_um = self.get_user_manifest() # Final merge could have been achieved by a concurrent operation if to_sync_um.version > diverged_um.base_version: merged_um = merge_local_user_manifests(diverged_um, to_sync_um) await self.set_user_manifest(merged_um) self.event_bus.send(CoreEvent.FS_ENTRY_SYNCED, id=self.user_manifest_id) return True
# Parsec Cloud (https://parsec.cloud) Copyright (c) AGPLv3 2016-2021 Scille SAS import pytest from parsec.api.protocol import RealmID, VlobID, BlockID from tests.backend.common import realm_stats from tests.backend.common import vlob_create, block_create REALM_ID_FAKE = RealmID.from_hex("00000000-0000-0000-0000-000000000001") @pytest.mark.trio async def test_realm_stats_ok(alice_backend_sock, realm): # Create new data await block_create(alice_backend_sock, realm_id=realm, block_id=BlockID.new(), block=b"1234") rep = await realm_stats(alice_backend_sock, realm_id=realm) assert rep == {"status": "ok", "blocks_size": 4, "vlobs_size": 0} # Create new metadata await vlob_create(alice_backend_sock, realm_id=realm, vlob_id=VlobID.new(), blob=b"1234") rep = await realm_stats(alice_backend_sock, realm_id=realm) assert rep == {"status": "ok", "blocks_size": 4, "vlobs_size": 4}
async def _create_realm_and_first_vlob(self, device): manifest = initial_user_manifest_state.get_user_manifest_v1_for_backend(device) if manifest.author == device.device_id: author = device else: author = self.get_device(device.organization_id, manifest.author) realm_id = RealmID(author.user_manifest_id.uuid) vlob_id = VlobID(author.user_manifest_id.uuid) with self.backend.event_bus.listen() as spy: # The realm needs to be created srictly before the manifest timestamp realm_create_timestamp = manifest.timestamp.subtract(microseconds=1) await self.backend.realm.create( organization_id=author.organization_id, self_granted_role=RealmGrantedRole( realm_id=realm_id, user_id=author.user_id, certificate=RealmRoleCertificateContent( author=author.device_id, timestamp=realm_create_timestamp, realm_id=realm_id, user_id=author.user_id, role=RealmRole.OWNER, ).dump_and_sign(author.signing_key), role=RealmRole.OWNER, granted_by=author.device_id, granted_on=realm_create_timestamp, ), ) await self.backend.vlob.create( organization_id=author.organization_id, author=author.device_id, realm_id=realm_id, encryption_revision=1, vlob_id=vlob_id, timestamp=manifest.timestamp, blob=manifest.dump_sign_and_encrypt( author_signkey=author.signing_key, key=author.user_manifest_key ), ) # Avoid possible race condition in tests listening for events await spy.wait_multiple( [ ( BackendEvent.REALM_ROLES_UPDATED, { "organization_id": author.organization_id, "author": author.device_id, "realm_id": realm_id, "user": author.user_id, "role": RealmRole.OWNER, }, ), ( BackendEvent.REALM_VLOBS_UPDATED, { "organization_id": author.organization_id, "author": author.device_id, "realm_id": realm_id, "checkpoint": 1, "src_id": vlob_id, "src_version": 1, }, ), ] )
# Parsec Cloud (https://parsec.cloud) Copyright (c) AGPLv3 2016-2021 Scille SAS import pytest from pendulum import datetime from unittest.mock import ANY from parsec.api.protocol import VlobID, RealmID, RealmRole from parsec.api.data import RealmRoleCertificateContent, UserProfile from parsec.backend.realm import RealmGrantedRole from tests.common import freeze_time, customize_fixtures from tests.backend.common import realm_update_roles, realm_get_role_certificates, vlob_create NOW = datetime(2000, 1, 1) VLOB_ID = VlobID.from_hex("00000000000000000000000000000001") REALM_ID = RealmID.from_hex("0000000000000000000000000000000A") @pytest.mark.trio async def test_get_roles_not_found(alice_backend_sock): rep = await realm_get_role_certificates(alice_backend_sock, REALM_ID) assert rep == { "status": "not_found", "reason": "Realm `0000000000000000000000000000000a` doesn't exist", } async def _realm_get_clear_role_certifs(sock, realm_id): rep = await realm_get_role_certificates(sock, realm_id) assert rep["status"] == "ok" cooked = [
async def bob_realm(backend, bob, realm_factory): realm_id = RealmID.from_hex("C0000000000000000000000000000000") return await realm_factory(backend, bob, realm_id, datetime(2000, 1, 2))