async def test_init_online_backend_late_reply(server_factory, core_config, alice, event_bus, backend): can_serve_client = trio.Event() async def _handle_client(stream): await can_serve_client.wait() return await backend.handle_client(stream) async with server_factory(_handle_client) as server: alice = server.correct_addr(alice) async with real_clock_timeout(): async with logged_core_factory(config=core_config, device=alice, event_bus=event_bus) as core: # We don't want for backend to reply to finish core init with core.event_bus.listen() as spy: can_serve_client.set() # Now backend reply, monitor should send events accordingly await spy.wait( CoreEvent.BACKEND_CONNECTION_CHANGED, kwargs={ "status": BackendConnStatus.READY, "status_exc": None }, )
async def _share_workspace( config: CoreConfig, device: LocalDevice, name: str, user_id: Optional[UserID], recipiant: Optional[str], user_role: RealmRole, ) -> None: if recipiant and user_id or not recipiant and not user_id: raise click.ClickException( "Either --recipiant or --user-id should be used, but not both") async with logged_core_factory(config, device) as core: workspace = core.find_workspace_from_name(name) if recipiant: user_info_tab, nb = await core.find_humans(recipiant, page=1, per_page=100, omit_revoked=True, omit_non_human=False) if nb == 0: raise RuntimeError("Unknown recipiant") if nb != 1: for user in user_info_tab: click.echo(f"{user.human_handle} - UserID: {user.user_id}") raise RuntimeError( "Specify the user more precisely or use the --user-id option" ) user_id = user_info_tab[0].user_id await core.user_fs.workspace_share(workspace.id, user_id, user_role)
async def _run_mountpoint(config, device, mountpoint): config = config.evolve(mountpoint_enabled=True) async with logged_core_factory(config, device, mountpoint=mountpoint) as core: display_device = click.style(device.device_id, fg="yellow") mountpoint_display = click.style(str(core.mountpoint.absolute()), fg="yellow") click.echo(f"{display_device}'s drive mounted at {mountpoint_display}") await core.mountpoint_manager.join()
async def _run_mountpoint(config, device, timestamp: Pendulum = None): config = config.evolve(mountpoint_enabled=True) async with logged_core_factory(config, device) as core: await core.mountpoint_manager.mount_all(timestamp) display_device = click.style(device.device_id, fg="yellow") mountpoint_display = click.style(str(config.mountpoint_base_dir.absolute()), fg="yellow") click.echo(f"{display_device}'s drive mounted at {mountpoint_display}") await trio.sleep_forever()
async def _do_run_core(config, device, qt_on_ready): # Quick fix to avoid MultiError<Cancelled, ...> exception bubbling up # TODO: replace this by a proper generic MultiError handling with trio.MultiError.catch(lambda exc: None if isinstance(exc, trio.Cancelled) else exc): async with logged_core_factory(config=config, device=device, event_bus=None) as core: # Create our own job scheduler allows us to cancel all pending # jobs depending on us when we logout async with run_trio_job_scheduler() as core_jobs_ctx: qt_on_ready.emit(core, core_jobs_ctx) await trio.sleep_forever()
async def _run_mountpoint(config: CoreConfig, device: LocalDevice, timestamp: Optional[DateTime] = None) -> None: config = config.evolve(mountpoint_enabled=True) async with logged_core_factory(config, device): display_device = click.style(device.device_id, fg="yellow") mountpoint_display = click.style(str( config.mountpoint_base_dir.absolute()), fg="yellow") click.echo(f"{display_device}'s drive mounted at {mountpoint_display}") await trio.sleep_forever()
async def _do_run_core(config, device, qt_on_ready): # Quick fix to avoid MultiError<Cancelled, ...> exception bubbling up # TODO: replace this by a proper generic MultiError handling with trio.MultiError.catch(lambda exc: None if isinstance(exc, trio.Cancelled) else exc): async with logged_core_factory(config=config, device=device, event_bus=None) as core: if config.mountpoint_enabled: await core.mountpoint_manager.mount_all() # Create our own job scheduler allows us to cancel all pending # jobs depending on us when we logout core_jobs_ctx = QtToTrioJobScheduler() async with trio.open_service_nursery() as nursery: await nursery.start(core_jobs_ctx._start) qt_on_ready.emit(core, core_jobs_ctx)
async def _run(): try: portal = trio.BlockingTrioPortal() self.core_queue.put(portal) with trio.open_cancel_scope() as cancel_scope: self.core_queue.put(cancel_scope) async with logged_core_factory( self.core_config, self.current_device ) as core: self.core_queue.put(core) await trio.sleep_forever() # If we have an exception, we never put the core object in the queue. Since the # main thread except something to be there, we put the exception. except Exception as exc: self.core_queue.put(exc)
async def _rsync(config: CoreConfig, device: local_device.LocalDevice, source: str, destination: str): async with logged_core_factory(config, device) as core: workspace, destination_path = _parse_destination(core, destination) workspace_fs = core.user_fs.get_workspace(workspace.id) workspace_manifest = await workspace_fs.remote_loader.load_manifest( workspace.id) local_path = trio.Path(source) root_manifest, workspace_path = await _root_manifest_parent( destination_path, workspace_fs, workspace_manifest) await _sync_directory_content(workspace_path, local_path, workspace_fs, root_manifest) await _clear_directory(workspace_path, local_path, workspace_fs, root_manifest)
async def test_work_within_logged_core(base_mountpoint, core_config, alice, tmpdir): core_config = core_config.evolve(mountpoint_base_dir=base_mountpoint) async with logged_core_factory(core_config, alice) as alice_core: manager = alice_core.mountpoint_manager wid = await alice_core.user_fs.workspace_create("w") workspace = alice_core.user_fs.get_workspace(wid) await workspace.touch("/bar.txt") with pytest.raises(MountpointNotMounted): get_path_in_mountpoint(manager, wid, "/bar.txt") await manager.mount_workspace(wid) bar_txt = get_path_in_mountpoint(manager, wid, "/bar.txt") assert await bar_txt.exists() assert not await bar_txt.exists()
async def _human_find( config: CoreConfig, device: LocalDevice, query: str, omit_revoked: bool, omit_non_human: bool = False, page: int = 1, per_page: int = 100, ) -> None: async with logged_core_factory(config, device) as core: user_info_tab, nb = await core.find_humans( query, page=1, per_page=100, omit_revoked=omit_revoked, omit_non_human=False ) for user in user_info_tab: is_revoked = " (revoked)" if user.revoked_on is not None else "" click.echo(f"{user.human_handle} - UserID: {user.user_id}{is_revoked}") if not nb: click.echo("No human found!")
async def test_work_within_logged_core(base_mountpoint, core_config, alice, tmpdir): core_config = core_config.evolve(mountpoint_enabled=True, mountpoint_base_dir=base_mountpoint) mountpoint_path = base_mountpoint / "w" bar_txt = trio.Path(f"{mountpoint_path}/bar.txt") async with logged_core_factory(core_config, alice) as alice_core: wid = await alice_core.user_fs.workspace_create("w") workspace = alice_core.user_fs.get_workspace(wid) await workspace.touch("/bar.txt") assert not await bar_txt.exists() await alice_core.mountpoint_manager.mount_workspace(wid) assert await bar_txt.exists() assert not await bar_txt.exists()
async def test_init_offline_backend_late_reply(server_factory, core_config, alice, event_bus): can_serve_client = trio.Event() async def _handle_client(stream): await can_serve_client.wait() await stream.aclose() async with server_factory(_handle_client, alice.organization_addr): with trio.fail_after(1): async with logged_core_factory(config=core_config, device=alice, event_bus=event_bus) as core: with core.event_bus.listen() as spy: can_serve_client.set() await spy.wait( CoreEvent.BACKEND_CONNECTION_CHANGED, kwargs={ "status": BackendConnStatus.LOST, "status_exc": ANY }, )
async def initialize_test_organization( config_dir: Path, backend_address: BackendAddr, password: str, administration_token: str, force: bool, ) -> Tuple[LocalDevice, LocalDevice, LocalDevice]: configure_logging("WARNING") organization_id = OrganizationID("Org") # Create organization async with apiv1_backend_administration_cmds_factory( backend_address, administration_token ) as administration_cmds: rep = await administration_cmds.organization_create(organization_id) assert rep["status"] == "ok" bootstrap_token = rep["bootstrap_token"] organization_bootstrap_addr = BackendOrganizationBootstrapAddr.build( backend_address, organization_id, bootstrap_token ) # Bootstrap organization and Alice user async with apiv1_backend_anonymous_cmds_factory(organization_bootstrap_addr) as anonymous_cmds: alice_device = await bootstrap_organization( cmds=anonymous_cmds, human_handle=HumanHandle(label="Alice", email="*****@*****.**"), device_label="laptop", ) save_device_with_password(config_dir, alice_device, password, force=force) # Create a workspace for Alice config = load_config(config_dir, debug="DEBUG" in os.environ) async with logged_core_factory(config, alice_device) as core: alice_ws_id = await core.user_fs.workspace_create("alice_workspace") await core.user_fs.sync() # Register a new device for Alice other_alice_device = None async with backend_authenticated_cmds_factory( addr=alice_device.organization_addr, device_id=alice_device.device_id, signing_key=alice_device.signing_key, ) as alice_cmds: rep = await alice_cmds.invite_new(type=InvitationType.DEVICE) assert rep["status"] == "ok" invitation_addr = BackendInvitationAddr.build( backend_addr=alice_device.organization_addr, organization_id=alice_device.organization_id, invitation_type=InvitationType.DEVICE, token=rep["token"], ) async with backend_invited_cmds_factory(addr=invitation_addr) as invited_cmds: async def invite_task(): initial_ctx = DeviceGreetInitialCtx(cmds=alice_cmds, token=invitation_addr.token) in_progress_ctx = await initial_ctx.do_wait_peer() in_progress_ctx = await in_progress_ctx.do_wait_peer_trust() in_progress_ctx = await in_progress_ctx.do_signify_trust() in_progress_ctx = await in_progress_ctx.do_get_claim_requests() await in_progress_ctx.do_create_new_device( author=alice_device, device_label=in_progress_ctx.requested_device_label ) async def claim_task(): nonlocal other_alice_device initial_ctx = await claimer_retrieve_info(cmds=invited_cmds) in_progress_ctx = await initial_ctx.do_wait_peer() in_progress_ctx = await in_progress_ctx.do_signify_trust() in_progress_ctx = await in_progress_ctx.do_wait_peer_trust() other_alice_device = await in_progress_ctx.do_claim_device( requested_device_label="pc" ) async with trio.open_service_nursery() as nursery: nursery.start_soon(invite_task) nursery.start_soon(claim_task) save_device_with_password(config_dir, other_alice_device, password, force=force) # Invite Bob in bob_device = None rep = await alice_cmds.invite_new(type=InvitationType.USER, claimer_email="*****@*****.**") assert rep["status"] == "ok" invitation_addr = BackendInvitationAddr.build( backend_addr=alice_device.organization_addr, organization_id=alice_device.organization_id, invitation_type=InvitationType.USER, token=rep["token"], ) async with backend_invited_cmds_factory(addr=invitation_addr) as invited_cmds: async def invite_task(): initial_ctx = UserGreetInitialCtx(cmds=alice_cmds, token=invitation_addr.token) in_progress_ctx = await initial_ctx.do_wait_peer() in_progress_ctx = await in_progress_ctx.do_wait_peer_trust() in_progress_ctx = await in_progress_ctx.do_signify_trust() in_progress_ctx = await in_progress_ctx.do_get_claim_requests() await in_progress_ctx.do_create_new_user( author=alice_device, human_handle=in_progress_ctx.requested_human_handle, device_label=in_progress_ctx.requested_device_label, profile=UserProfile.STANDARD, ) async def claim_task(): nonlocal bob_device initial_ctx = await claimer_retrieve_info(cmds=invited_cmds) in_progress_ctx = await initial_ctx.do_wait_peer() in_progress_ctx = await in_progress_ctx.do_signify_trust() in_progress_ctx = await in_progress_ctx.do_wait_peer_trust() bob_device = await in_progress_ctx.do_claim_user( requested_human_handle=HumanHandle(label="Bob", email="*****@*****.**"), requested_device_label="laptop", ) async with trio.open_service_nursery() as nursery: nursery.start_soon(invite_task) nursery.start_soon(claim_task) save_device_with_password(config_dir, bob_device, password, force=force) # Create bob workspace and share with Alice async with logged_core_factory(config, bob_device) as core: bob_ws_id = await core.user_fs.workspace_create("bob_workspace") await core.user_fs.workspace_share(bob_ws_id, alice_device.user_id, WorkspaceRole.MANAGER) # Share Alice workspace with bob async with logged_core_factory(config, alice_device) as core: await core.user_fs.workspace_share(alice_ws_id, bob_device.user_id, WorkspaceRole.MANAGER) # Synchronize every device for device in (alice_device, other_alice_device, bob_device): async with logged_core_factory(config, device) as core: await core.user_fs.process_last_messages() await core.user_fs.sync() return (alice_device, other_alice_device, bob_device)
async def initialize_test_organization( backend_address, organization_id, alice_device_id, bob_device_id, other_device_name, alice_workspace, bob_workspace, password, administration_token, force, ): configure_logging("WARNING") config_dir = get_default_config_dir(os.environ) alice_slugid = f"{organization_id}:{alice_device_id}" bob_slugid = f"{organization_id}:{bob_device_id}" # Create organization async with backend_administration_cmds_factory( backend_address, administration_token) as cmds: rep = await cmds.organization_create(organization_id) assert rep["status"] == "ok" bootstrap_token = rep["bootstrap_token"] organization_bootstrap_addr = BackendOrganizationBootstrapAddr.build( backend_address, organization_id, bootstrap_token) # Bootstrap organization and Alice user async with backend_anonymous_cmds_factory( organization_bootstrap_addr) as cmds: root_signing_key = SigningKey.generate() root_verify_key = root_signing_key.verify_key organization_addr = organization_bootstrap_addr.generate_organization_addr( root_verify_key) alice_device = generate_new_device(alice_device_id, organization_addr, True) save_device_with_password(config_dir, alice_device, password, force=force) now = pendulum.now() user_certificate = UserCertificateContent( author=None, timestamp=now, user_id=alice_device.user_id, public_key=alice_device.public_key, is_admin=True, ).dump_and_sign(author_signkey=root_signing_key) device_certificate = DeviceCertificateContent( author=None, timestamp=now, device_id=alice_device.device_id, verify_key=alice_device.verify_key, ).dump_and_sign(author_signkey=root_signing_key) rep = await cmds.organization_bootstrap( organization_bootstrap_addr.organization_id, organization_bootstrap_addr.token, root_verify_key, user_certificate, device_certificate, ) assert rep["status"] == "ok" # Create a workspace for Alice config = load_config(config_dir, debug="DEBUG" in os.environ) async with logged_core_factory(config, alice_device) as core: alice_ws_id = await core.user_fs.workspace_create(f"{alice_workspace}") await core.user_fs.sync() # Register a new device for Alice token = generate_invitation_token() other_alice_device_id = DeviceID( f"{alice_device.user_id}@{other_device_name}") other_alice_slugid = f"{organization_id}:{other_alice_device_id}" async def invite_task(): await invite_and_create_device(alice_device, other_device_name, token) other_alice_device = None async def claim_task(): nonlocal other_alice_device other_alice_device = await retry_claim(claim_device, alice_device.organization_addr, other_alice_device_id, token) save_device_with_password(config_dir, other_alice_device, password, force=force) async with trio.open_service_nursery() as nursery: nursery.start_soon(invite_task) nursery.start_soon(claim_task) # Invite Bob in token = generate_invitation_token() bob_device = None async def invite_task(): await invite_and_create_user(alice_device, bob_device_id.user_id, token, is_admin=False) async def claim_task(): nonlocal bob_device bob_device = await retry_claim(claim_user, alice_device.organization_addr, bob_device_id, token) save_device_with_password(config_dir, bob_device, password, force=force) async with trio.open_service_nursery() as nursery: nursery.start_soon(invite_task) nursery.start_soon(claim_task) # Create bob workspace and share with Alice async with logged_core_factory(config, bob_device) as core: bob_ws_id = await core.user_fs.workspace_create(f"{bob_workspace}") await core.user_fs.workspace_share(bob_ws_id, alice_device_id.user_id, WorkspaceRole.MANAGER) # Share Alice workspace with bob async with logged_core_factory(config, alice_device) as core: await core.user_fs.workspace_share(alice_ws_id, bob_device_id.user_id, WorkspaceRole.MANAGER) # Synchronize every device for device in (alice_device, other_alice_device, bob_device): async with logged_core_factory(config, device) as core: await core.user_fs.process_last_messages() await core.user_fs.sync() return alice_slugid, other_alice_slugid, bob_slugid
async def initialize_test_organization( config_dir: Path, backend_address: BackendAddr, password: str, administration_token: str, additional_users_number: int, additional_devices_number: int, ) -> Tuple[LocalDevice, LocalDevice, LocalDevice]: organization_id = OrganizationID("Org") config = load_config(config_dir, debug="DEBUG" in os.environ) # Create organization bootstrap_token = await create_organization_req( organization_id, backend_address, administration_token ) organization_bootstrap_addr = BackendOrganizationBootstrapAddr.build( backend_address, organization_id, bootstrap_token ) # Bootstrap organization and Alice user and create device "laptop" for Alice alice_device = await bootstrap_organization( organization_bootstrap_addr, human_handle=HumanHandle(label="Alice", email="*****@*****.**"), device_label=DeviceLabel("laptop"), ) await user_storage_non_speculative_init(data_base_dir=config.data_base_dir, device=alice_device) save_device_with_password_in_config( config_dir=config_dir, device=alice_device, password=password ) # Create context manager, alice_core will be needed for the rest of the script async with logged_core_factory(config, alice_device) as alice_core: async with backend_authenticated_cmds_factory( addr=alice_device.organization_addr, device_id=alice_device.device_id, signing_key=alice_device.signing_key, ) as alice_cmds: # Create new device "pc" for Alice other_alice_device = await _register_new_device( cmds=alice_cmds, author=alice_device, device_label=DeviceLabel("pc") ) save_device_with_password_in_config( config_dir=config_dir, device=other_alice_device, password=password ) # Invite Bob in organization bob_device = await _register_new_user( cmds=alice_cmds, author=alice_device, device_label=DeviceLabel("laptop"), human_handle=HumanHandle(email="*****@*****.**", label="Bob"), profile=UserProfile.STANDARD, ) await user_storage_non_speculative_init( data_base_dir=config.data_base_dir, device=bob_device ) save_device_with_password_in_config( config_dir=config_dir, device=bob_device, password=password ) # Invite Toto in organization toto_device = await _register_new_user( cmds=alice_cmds, author=alice_device, device_label=DeviceLabel("laptop"), human_handle=HumanHandle(email="*****@*****.**", label="Toto"), profile=UserProfile.OUTSIDER, ) await user_storage_non_speculative_init( data_base_dir=config.data_base_dir, device=toto_device ) save_device_with_password_in_config( config_dir=config_dir, device=toto_device, password=password ) # Create Alice workspace alice_ws_id = await alice_core.user_fs.workspace_create(EntryName("alice_workspace")) # Create context manager async with logged_core_factory(config, bob_device) as bob_core: # Create Bob workspace bob_ws_id = await bob_core.user_fs.workspace_create(EntryName("bob_workspace")) # Bob share workspace with Alice await bob_core.user_fs.workspace_share( bob_ws_id, alice_device.user_id, WorkspaceRole.MANAGER ) # Alice share workspace with Bob await alice_core.user_fs.workspace_share( alice_ws_id, bob_device.user_id, WorkspaceRole.MANAGER ) # Add additional random users await _add_random_users( cmds=alice_cmds, author=alice_device, alice_core=alice_core, bob_core=bob_core, alice_ws_id=alice_ws_id, bob_ws_id=bob_ws_id, additional_users_number=additional_users_number, ) # Add additional random device for alice await _add_random_device( cmds=alice_cmds, device=alice_device, additional_devices_number=additional_devices_number, ) # Synchronize every device for device in (alice_device, other_alice_device, bob_device): async with logged_core_factory(config, device) as core: await core.user_fs.process_last_messages() await core.user_fs.sync() return (alice_device, other_alice_device, bob_device)
async def _create_workspace(config, device, name): async with logged_core_factory(config, device) as core: await core.user_fs.workspace_create(f"{name}")
async def _create_workspace(config: CoreConfig, device: LocalDevice, name: EntryName) -> None: async with logged_core_factory(config, device) as core: await core.user_fs.workspace_create(name)
async def _share_workspace(config, device, name, user_id): async with logged_core_factory(config, device) as core: await core.user_fs.workspace_share(f"/{name}", user_id)
async def amain( backend_address="ws://*****:*****@laptop", bob_device_id="bob@laptop", other_device_name="pc", alice_workspace="alicews", bob_workspace="bobws", password="******", administrator_token=DEFAULT_ADMINISTRATOR_TOKEN, force=False, ): configure_logging("WARNING") config_dir = get_default_config_dir(os.environ) organization_id = OrganizationID(organization_id) backend_address = BackendAddr(backend_address) alice_device_id = DeviceID(alice_device_id) bob_device_id = DeviceID(bob_device_id) alice_slugid = f"{organization_id}:{alice_device_id}" bob_slugid = f"{organization_id}:{bob_device_id}" # Create organization async with backend_administrator_cmds_factory(backend_address, administrator_token) as cmds: bootstrap_token = await cmds.organization_create(organization_id) organization_bootstrap_addr = BackendOrganizationBootstrapAddr.build( backend_address, organization_id, bootstrap_token ) # Bootstrap organization and Alice user async with backend_anonymous_cmds_factory(organization_bootstrap_addr) as cmds: root_signing_key = SigningKey.generate() root_verify_key = root_signing_key.verify_key organization_addr = organization_bootstrap_addr.generate_organization_addr(root_verify_key) alice_device = generate_new_device(alice_device_id, organization_addr) save_device_with_password(config_dir, alice_device, password, force=force) now = pendulum.now() certified_user = certify_user( None, root_signing_key, alice_device.user_id, alice_device.public_key, now ) certified_device = certify_device( None, root_signing_key, alice_device_id, alice_device.verify_key, now ) await cmds.organization_bootstrap( organization_bootstrap_addr.organization_id, organization_bootstrap_addr.bootstrap_token, root_verify_key, certified_user, certified_device, ) # Create a workspace for Alice config = load_config(config_dir, debug="DEBUG" in os.environ) async with logged_core_factory(config, alice_device) as core: await core.fs.workspace_create(f"/{alice_workspace}") # Register a new device for Alice token = generate_invitation_token() other_alice_device_id = DeviceID("@".join((alice_device.user_id, other_device_name))) other_alice_slugid = f"{organization_id}:{other_alice_device_id}" async def invite_task(): async with backend_cmds_factory( alice_device.organization_addr, alice_device.device_id, alice_device.signing_key ) as cmds: await invite_and_create_device(alice_device, cmds, other_device_name, token) async def claim_task(): async with backend_anonymous_cmds_factory(alice_device.organization_addr) as cmds: other_alice_device = await retry(claim_device, cmds, other_alice_device_id, token) save_device_with_password(config_dir, other_alice_device, password, force=force) async with trio.open_nursery() as nursery: nursery.start_soon(invite_task) nursery.start_soon(claim_task) # Invite Bob in token = generate_invitation_token() async def invite_task(): async with backend_cmds_factory( alice_device.organization_addr, alice_device.device_id, alice_device.signing_key ) as cmds: await invite_and_create_user( alice_device, cmds, bob_device_id.user_id, token, is_admin=True ) async def claim_task(): async with backend_anonymous_cmds_factory(alice_device.organization_addr) as cmds: bob_device = await retry(claim_user, cmds, bob_device_id, token) save_device_with_password(config_dir, bob_device, password, force=force) async with trio.open_nursery() as nursery: nursery.start_soon(invite_task) nursery.start_soon(claim_task) # Create bob workspace and share with Alice bob_device = load_device_with_password( config.config_dir, organization_id, bob_device_id, password ) async with logged_core_factory(config, bob_device) as core: await core.fs.workspace_create(f"/{bob_workspace}") await core.fs.share(f"/{bob_workspace}", alice_device_id.user_id) # Share Alice workspace with bob async with logged_core_factory(config, alice_device) as core: await core.fs.share(f"/{alice_workspace}", bob_device_id.user_id) # Print out click.echo( f""" Mount alice and bob drives using: $ parsec core run -P {password} -D {alice_slugid} $ parsec core run -P {password} -D {other_alice_slugid} $ parsec core run -P {password} -D {bob_slugid} """ )
async def _share_workspace(config, device, name, user_id, user_role): async with logged_core_factory(config, device) as core: workspace = core.find_workspace_from_name(name) await core.user_fs.workspace_share(workspace.id, user_id, user_role)