async def http_request_handler(server,
request,
*,
title=None,
http_port=None,
ws_port=None):
target = request.target.decode('ascii')
request_body = ''
async for part in _http_server.receive_body(server):
request_body += part # pylint: disable=consider-using-join
if target == '/':
path = trio.Path(HTML_PATH / 'index.html')
template = Template(await path.read_text())
body = template.substitute(title=title,
ws_url_path='/ws/',
http_port=http_port,
ws_port=ws_port)
await _http_server.send_simple_response(server, int(HTTPStatus.OK),
'text/html; charset=utf-8',
body.encode('utf-8'))
elif target == '/js/pura.js':
path = trio.Path(HTML_PATH / target.lstrip('/'))
body = await path.read_text()
await _http_server.send_simple_response(
server, int(HTTPStatus.OK),
'application/javascript; charset=utf-8', body.encode('utf-8'))
else:
raise h11.RemoteProtocolError(f'{target} not found',
int(HTTPStatus.NOT_FOUND))
def __init__(self, image_path: pathlib.Path, thumbnail_path: pathlib.Path, json_path: pathlib.Path):
self.image_path = trio.Path(image_path)
self.thumbnail_path = trio.Path(thumbnail_path)
self.json_path = json_path
with open(json_path) as fp:
data = json.loads(fp.read())
self.dimension = data["clipPoints"][2]
async def rename(self,
key1: datastore.Key,
key2: datastore.Key,
*,
replace: bool = True) -> None:
"""Moves key *key1* to *key2*
Arguments
---------
key1
The key to rename, must exist
key2
The new name of the key; if *replace* is ``False``, a key of the
same name may not already exist
replace
Should an existing key at name *key2* be replaced?
Raises
------
KeyError
Key *key1* did not exist in this datastore
KeyError
Key *key2* already exists in this datastore, but *replace* was not ``True``
"""
# Validate that the keys are well-formed
assert self.verify_key_valid(key1)
assert self.verify_key_valid(key2, False)
path1 = trio.Path(self.object_path(key1))
path2 = trio.Path(self.object_path(key2))
if not replace:
# Just do the replace and fail if it didn't succeed
#
# No weird accounting stuff here since this operation never changes the
# size of datastore.
try:
await run_blocking_nointr(rename_noreplace.rename_noreplace,
path1, path2)
except FileNotFoundError as exc:
raise KeyError(key1) from exc
except FileExistsError as exc:
raise KeyError(key2) from exc
else:
try:
async with self._stats_lock: # type: ignore[union-attr]
await run_blocking_nointr(self._rename_replace_sync, path1,
path2)
except FileNotFoundError as exc:
raise KeyError(key1) from exc
async def test_idempotent_mount(base_mountpoint, alice_user_fs, event_bus,
manual_unmount):
mountpoint_path = base_mountpoint / "w"
# Populate a bit the fs first...
wid = await alice_user_fs.workspace_create("w")
workspace = alice_user_fs.get_workspace(wid)
await workspace.touch("/bar.txt")
bar_txt = trio.Path(f"{mountpoint_path}/bar.txt")
# Now we can start fuse
async with mountpoint_manager_factory(
alice_user_fs, event_bus, base_mountpoint) as mountpoint_manager:
await mountpoint_manager.mount_workspace(wid)
assert await bar_txt.exists()
with pytest.raises(MountpointAlreadyMounted):
await mountpoint_manager.mount_workspace(wid)
assert await bar_txt.exists()
await mountpoint_manager.unmount_workspace(wid)
assert not await bar_txt.exists()
with pytest.raises(MountpointNotMounted):
await mountpoint_manager.unmount_workspace(wid)
assert not await bar_txt.exists()
await mountpoint_manager.mount_workspace(wid)
assert await bar_txt.exists()
async def test_cancel_mount_workspace(base_mountpoint, alice_user_fs,
event_bus, timeout):
"""
This function tests the race conditions between the mounting of a workspace
and trio cancellation. In particular, it produces interesting results when trying to
unmount a workspace while it's still initializing.
The following timeout values are useful for more thorough testing:
[x * 0.00001 for x in range(2000, 2500)]
"""
wid = await alice_user_fs.workspace_create("w")
# The timeout for `_stop_fuse_thread` is 1 second (3 seconds for macOS),
# so let's use a slightly lower timeout to make sure a potential failure
# doesn't go undetected.
timeout = 3.0 if sys.platform == "darwin" else 1.0
with trio.fail_after(timeout * 0.9):
async with mountpoint_manager_factory(
alice_user_fs, event_bus,
base_mountpoint) as mountpoint_manager:
with trio.move_on_after(timeout) as cancel_scope:
await mountpoint_manager.mount_workspace(wid)
if cancel_scope.cancelled_caught:
with pytest.raises(MountpointNotMounted):
mountpoint_manager.get_path_in_mountpoint(wid, FsPath("/"))
else:
path = trio.Path(
mountpoint_manager.get_path_in_mountpoint(
wid, FsPath("/")))
await path.exists()
assert not await (path / "foo").exists()
async def configure_mime_types():
if sys.platform == "win32" or sys.platform == "darwin":
return
XDG_DATA_HOME = os.environ["XDG_DATA_HOME"]
desktop_file = trio.Path(f"{XDG_DATA_HOME}/applications/parsec.desktop")
await desktop_file.parent.mkdir(exist_ok=True, parents=True)
await desktop_file.write_text("""\
[Desktop Entry]
Name=Parsec
Exec=parsec core gui %u
Type=Application
Terminal=false
StartupNotify=false
StartupWMClass=Parsec
MimeType=x-scheme-handler/parsec;
""")
try:
await trio.run_process("update-desktop-database -q".split(),
check=False)
except FileNotFoundError:
# Ignore if command is not available
pass
try:
await trio.run_process(
"xdg-mime default parsec.desktop x-scheme-handler/parsec".split())
except FileNotFoundError:
# Ignore if command is not available
pass
async def _process_connections(
self,
channel: trio.abc.ReceiveChannel[ConnectionConfig],
nursery: trio_typing.Nursery,
) -> None:
"""
Long running process that establishes connections to endpoint servers
and runs the handler for receiving events sent by the server over that
connection.
"""
self.logger.debug("%s: starting new connection channel", self)
self._connection_loop_running.set()
async for config in channel:
# Allow some time for for the IPC socket to appear
with trio.fail_after(constants.IPC_WAIT_SECONDS):
await _wait_for_path(trio.Path(config.path))
await trio.sleep(0.001)
# Establish the socket connection
connection = await TrioConnection.connect_to(config.path)
# Create the remote
remote = TrioRemoteEndpoint(
self.name,
connection,
self._remote_subscriptions_changed,
self._inbound_send_channel.send,
)
nursery.start_soon(self._run_remote_endpoint, remote)
def _get_robots_analyzer(self) -> RobotsAnalyzer:
logger.debug('getting a default robots analyzer')
return RobotsAnalyzer(
http_client=self._http_client,
robots_cache=trio.Path(self.config.robots_cache_folder),
user_agent=self.config.user_agent
)
async def _init_stats(self) -> None:
# Start fresh
self._stats = Stats()
# Try to read existing stats file
path = trio.Path(self.object_path(self.stats_key))
try:
async with await path.open() as stats_file:
self._stats = Stats.from_json(
json.loads(await stats_file.read()))
self._stats.mtime_ns = (await run_blocking_intr(
os.fstat, stats_file.fileno())).st_mtime_ns
except FileNotFoundError:
# At least set an appropriate accuracy value if there are no files yet
is_empty = await run_blocking_intr(check_dir_empty_sync,
self.root_path)
if is_empty:
self._stats.disk_usage = 0
self._stats.accuracy = "initial-exact"
else:
pass #XXX: We could call `os.walk` here to get the initial size estimate…
self._stats_prev = self._stats.copy()
# This will synchronize our current state with the one on the disk
await self._flush_stats(expect_file=False)
async def _mocked_bootstrap_mountpoint(*args):
trio_mountpoint_path = trio.Path(f"{mountpoint_path}")
await trio_mountpoint_path.mkdir(parents=True)
file_path = trio_mountpoint_path / "bar.txt"
await file_path.touch()
st_dev = (await trio_mountpoint_path.stat()).st_dev
return mountpoint_path, st_dev
def __init__(self, path, vacuum_threshold=None):
self._conn = None
self._lock = trio.Lock()
self._run_in_thread = None
self.path = trio.Path(path)
self.vacuum_threshold = vacuum_threshold
async def test_file_save_no_defaults(qtbot: pytestqt.qtbot.QtBot,
tmp_path: pathlib.Path) -> None:
path_to_select = trio.Path(tmp_path) / "another.thing"
dialog = qtrio.dialogs.create_file_save_dialog()
async def user(task_status):
async with qtrio._core.wait_signal_context(dialog.shown):
task_status.started()
# allow cancellation to occur even if the signal was received before the
# cancellation was requested.
await trio.sleep(0)
assert dialog.dialog is not None
dialog.dialog.setDirectory(os.fspath(path_to_select.parent))
[text_edit] = dialog.dialog.findChildren(QtWidgets.QLineEdit)
text_edit.setText(path_to_select.name)
dialog.dialog.accept()
async with trio.open_nursery() as nursery:
await nursery.start(user)
with qtrio._qt.connection(signal=dialog.shown, slot=qtbot.addWidget):
selected_path = await dialog.wait()
assert selected_path == path_to_select
async def test_get_dialog_canceled(
chunked_data: typing.List[bytes],
http_application: quart_trio.QuartTrio,
url: hyperlink.URL,
tmp_path: pathlib.Path,
) -> None:
temporary_directory = trio.Path(tmp_path)
destination = temporary_directory.joinpath("file")
with pytest.raises(qtrio.UserCancelledError):
async with trio.open_nursery() as nursery:
start = functools.partial(
qtrio.examples.download.start_get_dialog,
url=url,
destination=destination,
fps=0.1,
http_application=http_application,
)
widget: qtrio.examples.download.GetDialog = await nursery.start(start)
await widget.progress_dialog_shown_event.wait()
assert widget.progress_dialog is not None
assert widget.progress_dialog.dialog is not None
assert widget.progress_dialog.dialog.isVisible()
assert widget.message_box is None
assert widget.progress_dialog.cancel_button is not None
widget.progress_dialog.cancel_button.click()
async def test_get(
chunked_data: typing.List[bytes],
content_length: typing.Optional[int],
http_application: quart_trio.QuartTrio,
url: hyperlink.URL,
tmp_path: pathlib.Path,
) -> None:
temporary_directory = trio.Path(tmp_path)
data = b"".join(chunked_data)
destination = temporary_directory.joinpath("file")
progresses = []
async for progress in qtrio.examples.download.get(
url=url,
destination=destination,
update_period=0,
http_application=http_application,
):
progresses.append(progress)
async with await destination.open("rb") as written_file:
written = await written_file.read() # type: ignore[attr-defined]
assert written == data
assert all(progress.total == content_length for progress in progresses)
assert [progresses[0].downloaded, progresses[-1].downloaded] == [0, len(data)]
async def test_stats_restore(temp_path):
async with FileSystemDatastore.create(temp_path, stats=True) as fs:
assert fs.datastore_stats().size == 0
assert fs.datastore_stats().size_accuracy == "exact"
await fs.put(datastore.Key("/a"), b"1234")
assert fs.datastore_stats().size == 4
# Re-open datastore and check that the stats are still there
async with FileSystemDatastore.create(temp_path, stats=True) as fs:
assert fs.datastore_stats().size == 4
assert fs.datastore_stats().size_accuracy == "exact"
# Replace content with stuff written by a non-cooperating datastore user
await (trio.Path(temp_path) / "diskUsage.data").write_text(
json.dumps({
"diskUsage": 3,
"accuracy": "initial-exact"
}),
encoding="utf-8")
# Re-open datastore and check that the non-cooperating stats data is
# properly merged
async with FileSystemDatastore.create(temp_path, stats=True) as fs:
assert fs.datastore_stats().size == 7
assert fs.datastore_stats().size_accuracy == "exact"
async def get_all(self, key: datastore.Key) -> bytes:
"""Returns all the data named by `key` at once or raises `KeyError`
otherwise
This is an optimization over :meth:`get` for smaller files as it entails
only one context switch to open, read and close the file, rather then
several.
Arguments
---------
key
Key naming the data to retrieve
Raises
------
KeyError
The given object was not present in this datastore
RuntimeError
The given ``key`` names a subtree, not a value
"""
path = trio.Path(self.object_path(key))
try:
return await path.read_bytes()
except FileNotFoundError as exc:
raise KeyError(key) from exc
except IsADirectoryError as exc:
# Should hopefully only happen if `object_extension` is `""`
raise RuntimeError(
f"Key '{key}' names a subtree, not a value") from exc
async def _bootstrap_mountpoint(base_mountpoint_path: PurePath,
workspace_fs) -> PurePath:
# Find a suitable path where to mount the workspace. The check we are doing
# here are not atomic (and the mount operation is not itself atomic anyway),
# hence there is still edgecases where the mount can crash due to concurrent
# changes on the mountpoint path
workspace_name = workspace_fs.get_workspace_name()
for tentative in count(1):
if tentative == 1:
dirname = workspace_name
else:
dirname = f"{workspace_name} ({tentative})"
mountpoint_path = base_mountpoint_path / dirname
try:
# On POSIX systems, mounting target must exists
trio_mountpoint_path = trio.Path(mountpoint_path)
await trio_mountpoint_path.mkdir(exist_ok=True, parents=True)
base_st_dev = (await trio.Path(base_mountpoint_path).stat()).st_dev
initial_st_dev = (await trio_mountpoint_path.stat()).st_dev
if initial_st_dev != base_st_dev:
# mountpoint_path seems to already have a mountpoint on it,
# hence find another place to setup our own mountpoint
continue
if list(await trio_mountpoint_path.iterdir()):
# mountpoint_path not empty, cannot mount there
continue
return mountpoint_path, initial_st_dev
except OSError:
# In case of hard crash, it's possible the FUSE mountpoint is still
# mounted (but points to nothing). In such case just mount in
# another place
continue
async def main(i):
# Open connection
async with open_cdp_connection(uri_pwa) as conn:
logger.info('Connecting')
targets = await conn.execute(target.get_targets())
# Filter the targets to find the PWA one
targets = list(filter(findPWA, targets))
logger.info(targets[0])
target_id = targets[0].target_id
logger.info('Attaching to target id=%s', target_id)
session = await conn.open_session(target_id)
final_data = []
def addData(data):
final_data += data
outfile_path = trio.Path(name + '_' + str(i) + '.json')
async with await outfile_path.open('a+') as outfile:
logger.info('Tracing...')
# write things to file to be readable as a json
await outfile.write('[')
# data = '['
await generateTrace(session, outfile, final_data)
logger.info('Tracing n° {} ended'.format(i))
await outfile.write(',\n'.join(final_data))
await outfile.write("]")
async def cleanup_macos_mountpoint_folder(base_mountpoint_path):
# In case of a crash on macOS, workspaces don't unmount correctly and leave empty
# mountpoints or directories in the default mount folder. This function is used to
# unmount and/or delete these anytime a login occurs. In some rare and so far unknown
# conditions, the unmount call can fail, raising a CalledProcessError, hence the
# exception below. In such cases, the issue stops occurring after a system reboot,
# making the exception catching a temporary solution.
base_mountpoint_path = trio.Path(base_mountpoint_path)
try:
mountpoint_names = await base_mountpoint_path.iterdir()
except FileNotFoundError:
# Unlike with `pathlib.Path.iterdir` which returns a lazy itertor,
# `trio.Path.iterdir` does FS access and may raise FileNotFoundError
return
for mountpoint_name in mountpoint_names:
mountpoint_path = base_mountpoint_path / mountpoint_name
stats = await trio.Path(mountpoint_path).stat()
if (stats.st_size == 0 and stats.st_blocks == 0 and stats.st_atime == 0
and stats.st_mtime == 0 and stats.st_ctime == 0):
try:
await trio.run_process(
["diskutil", "unmount", "force", mountpoint_path])
except CalledProcessError as exc:
logger.warning(
"Error during mountpoint cleanup: diskutil unmount failed",
exc_info=exc,
mountpoint_path=mountpoint_path,
)
try:
await trio.Path(mountpoint_path).rmdir()
except FileNotFoundError:
pass
async def test_file_save(qtbot: pytestqt.qtbot.QtBot,
tmp_path: pathlib.Path) -> None:
path_to_select = trio.Path(tmp_path) / "something.new"
dialog = qtrio.dialogs.create_file_save_dialog(
default_directory=path_to_select.parent,
default_file=path_to_select,
)
async def user(task_status):
async with qtrio._core.wait_signal_context(dialog.shown):
task_status.started()
# allow cancellation to occur even if the signal was received before the
# cancellation was requested.
await trio.sleep(0)
assert dialog.dialog is not None
dialog.dialog.accept()
async with trio.open_nursery() as nursery:
await nursery.start(user)
with qtrio._qt.connection(signal=dialog.shown, slot=qtbot.addWidget):
selected_path = await dialog.wait()
assert selected_path == path_to_select
async def test_remote_error_event(tmpdir, monkeypatch, running_backend,
alice_user_fs, bob_user_fs, monitor):
wid = await create_shared_workspace("w1", bob_user_fs, alice_user_fs)
base_mountpoint = Path(tmpdir / "alice_mountpoint")
async with mountpoint_manager_factory(alice_user_fs,
alice_user_fs.event_bus,
base_mountpoint,
debug=False) as mountpoint_manager:
await mountpoint_manager.mount_workspace(wid)
# Create shared data
bob_w = bob_user_fs.get_workspace(wid)
await bob_w.touch("/foo.txt")
await bob_w.write_bytes("/foo.txt", b"hello")
await bob_w.sync()
alice_w = alice_user_fs.get_workspace(wid)
await alice_w.sync()
# Force manifest cache
await alice_w.path_id("/")
await alice_w.path_id("/foo.txt")
trio_w = trio.Path(
mountpoint_manager.get_path_in_mountpoint(wid, FsPath("/")))
# Switch the mountpoint in maintenance...
await bob_user_fs.workspace_start_reencryption(wid)
def _testbed():
# ...accessing workspace data in the backend should endup in remote error
with alice_user_fs.event_bus.listen() as spy:
fd = os.open(str(trio_w / "foo.txt"), os.O_RDONLY)
with pytest.raises(OSError):
os.read(fd, 10)
spy.assert_event_occured("mountpoint.remote_error")
# But should still be able to do local stuff though without remote errors
with alice_user_fs.event_bus.listen() as spy:
os.open(str(trio_w / "bar.txt"), os.O_RDWR | os.O_CREAT)
assert os.listdir(str(trio_w)) == ["bar.txt", "foo.txt"]
assert "mountpoint.remote_error" not in [
e.event for e in spy.events
]
# Finally test unhandled error
def _crash(*args, **kwargs):
raise RuntimeError("D'Oh !")
monkeypatch.setattr(
"parsec.core.fs.workspacefs.entry_transactions.EntryTransactions.folder_create",
_crash,
)
with alice_user_fs.event_bus.listen() as spy:
with pytest.raises(OSError):
os.mkdir(str(trio_w / "dummy"))
spy.assert_event_occured("mountpoint.unhandled_error")
await trio.to_thread.run_sync(_testbed)
async def test_should_return_python_objects_when_reading_file_without_custom_decoder(
self, tmp_path, create_msgpack_file):
given_data = [[1, 2], 'hello', {'fruit': 'apple'}]
mp_file = tmp_path / 'data.mp'
create_msgpack_file(mp_file, given_data)
for file in [f'{mp_file}', trio.Path(mp_file)]:
assert [item async for item in read_mp(file)] == given_data
async def generate_gui_config():
config_dir = None
if os.name == "nt":
config_dir = trio.Path(os.environ["APPDATA"]) / "parsec/config"
else:
config_dir = trio.Path(os.environ["XDG_CONFIG_HOME"]) / "parsec"
await config_dir.mkdir(parents=True, exist_ok=True)
config_file = config_dir / "config.json"
config = {
"gui_first_launch": False,
"gui_check_version_at_startup": False,
"gui_tray_enabled": False,
"gui_last_version": PARSEC_VERSION,
}
await config_file.write_text(json.dumps(config, indent=4))
def __init__(self, path):
path = trio.Path(path)
super().__init__(path)
self._file_image_names.add("Demo")
self._default_heightmap_names = ("Demo", )
self.scansize = 100
self.k = 1
self.defl_sens = 1
self.scandown = True
async def _fetch_pkg(self, url):
whl_resp = await asks.get(url, stream=True)
fname = url.split("/")[-1]
output_name = trio.Path(self.target_dir / fname)
async with await output_name.open("wb") as wheel:
async with whl_resp.body:
async for chunk in whl_resp.body:
await wheel.write(chunk)
return output_name
async def convert_ardf(ardf_path, conv_path="ARDFtoHDF5.exe", pbar=None):
"""Turn an ARDF into a corresponding ARH5, returning the path.
Requires converter executable available from Asylum Research"""
ardf_path = trio.Path(ardf_path)
h5file_path = ardf_path.with_suffix(".h5")
if pbar is None:
pipe = None
else:
pbar.set_description_str("Converting " + ardf_path.name)
pipe = PIPE
async def reading_stdout():
stdout = bytearray()
async for bytes_ in proc.stdout:
stdout.extend(bytes_)
stdout = stdout.decode()
if "Failed" in stdout:
raise RuntimeError(stdout)
else:
print(stdout)
async def reading_stderr():
async for bytes_ in proc.stderr:
i = bytes_.rfind(
b"\x08") + 1 # first thing on right not a backspace
most_recent_numeric_output = bytes_[i:-1] # crop % sign
if most_recent_numeric_output:
pbar.update(
float(most_recent_numeric_output.decode()) - pbar.n)
try:
async with await trio.open_process(
[str(conv_path), str(ardf_path),
str(h5file_path)],
stderr=pipe,
stdout=pipe,
startupinfo=STARTUPINFO(dwFlags=STARTF_USESHOWWINDOW),
) as proc:
if pbar is not None:
async with trio.open_nursery() as nursery:
nursery.start_soon(reading_stdout)
nursery.start_soon(reading_stderr)
except FileNotFoundError:
raise FileNotFoundError(
"Please acquire ARDFtoHDF5.exe and place it in the application's root folder."
)
except:
with trio.CancelScope(shield=True):
await h5file_path.unlink(missing_ok=True)
raise
finally:
if pbar is not None:
pbar.close()
return h5file_path
async def serve(cls,
config: ConnectionConfig) -> AsyncIterator["TrioEndpoint"]:
endpoint = cls(config.name)
async with endpoint.run():
async with trio.open_nursery() as nursery:
await endpoint._start_serving(nursery, trio.Path(config.path))
try:
yield endpoint
finally:
await endpoint._stop_serving()
async def generate_gui_config(backend_address):
config_dir = None
if sys.platform == "win32":
config_dir = trio.Path(os.environ["APPDATA"]) / "parsec/config"
else:
config_dir = trio.Path(os.environ["XDG_CONFIG_HOME"]) / "parsec"
await config_dir.mkdir(parents=True, exist_ok=True)
config_file = config_dir / "config.json"
config = {
"gui_first_launch": False,
"gui_check_version_at_startup": False,
"gui_tray_enabled": False,
"gui_last_version": PARSEC_VERSION,
"preferred_org_creation_backend_addr": backend_address.to_url(),
"gui_show_confined": True,
}
await config_file.write_text(json.dumps(config, indent=4))
async def test_should_return_python_objects_when_reading_file_with_custom_decoder(
self, tmp_path, decode_datetime, create_msgpack_file):
given_data = ['hello', datetime.now()]
mp_file = tmp_path / 'data.mp'
create_msgpack_file(mp_file, given_data)
for file in [str(mp_file), trio.Path(mp_file)]:
assert [
item async for item in read_mp(file, decoder=decode_datetime)
] == given_data
def __init__(self,
path: Union[str, Path, trio.Path],
vacuum_threshold: Optional[int] = None):
# Make sure only a single task access the connection object at a time
self._lock = trio.Lock()
# Those attributes are set by the `run` async context manager
self._conn: Connection
self.path = trio.Path(path)
self.vacuum_threshold = vacuum_threshold