async def sync_apply(
sync_function: t.Callable[[t.Iterable[T]], t.Any], data: t.AsyncIterable[T]
):
"""Apply a sync Callable[Iterable] to an async iterable by pulling values in a
thread.
"""
in_trio = trio.BlockingTrioPortal()
send_to_trio, receive_from_thread = trio.open_memory_channel[t.Tuple[T, t.Any]](0)
async with trio.open_nursery() as n:
iterator = await sync_function(
_pull_values_from_async_iterator(in_trio, data, send_to_trio)
)
n.start_soon(
trio.run_sync_in_worker_thread,
functools.partial(
_threaded_sync_apply,
function=sync_function,
iterator=iterator,
ait=data,
in_trio=in_trio,
send_to_trio=send_to_trio,
receive_from_thread=receive_from_thread,
),
)
async for x in receive_from_thread:
yield x
async def _trio_claim_user(
queue,
qt_on_done,
qt_on_error,
config,
addr,
device_id,
token,
use_pkcs11,
password=None,
pkcs11_token=None,
pkcs11_key=None,
):
portal = trio.BlockingTrioPortal()
queue.put(portal)
with trio.open_cancel_scope() as cancel_scope:
queue.put(cancel_scope)
try:
async with backend_anonymous_cmds_factory(addr) as cmds:
device = await core_claim_user(cmds, device_id, token)
if use_pkcs11:
devices_manager.save_device_with_pkcs11(
config.config_dir, device, pkcs11_token, pkcs11_key)
else:
devices_manager.save_device_with_password(
config.config_dir, device, password)
qt_on_done.emit()
except BackendCmdsBadResponse as e:
qt_on_error.emit(e.status)
def init(self, threads=1):
'''Start worker threads'''
self.portal = trio.BlockingTrioPortal()
self.to_upload = trio.open_memory_channel(0)
for _ in range(threads):
t = threading.Thread(target=self._upload_loop)
t.start()
self.upload_threads.append(t)
self.to_remove = Queue(1000)
with self.backend_pool() as backend:
has_delete_multi = backend.has_delete_multi
if has_delete_multi:
t = threading.Thread(target=self._removal_loop_multi)
t.daemon = True # interruption will do no permanent harm
t.start()
self.removal_threads.append(t)
else:
for _ in range(20):
t = threading.Thread(target=self._removal_loop_simple)
t.daemon = True # interruption will do no permanent harm
t.start()
self.removal_threads.append(t)
def __init__(self):
self._portal = trio.BlockingTrioPortal()
self._request_queue = trio.Queue(1)
self._response_queue = trio.Queue(1)
self._thread = threading.Thread(target=self.thread_fn, daemon=True)
self._thread.start()
self._has_quit = False
def __init__(self, awaitable):
self.awaitable = awaitable
try:
self.trio_portal = trio.BlockingTrioPortal()
except RuntimeError:
# There's no event loop in this thread. Look for the threadlocal if
# we're inside SyncToAsync
self.trio_portal = getattr(SyncToAsync.threadlocal, "trio_portal",
None)
async def wait_to_finish(app):
'''This method is also run by trio and periodically prints something.'''
async with trio.open_nursery() as nursery:
nursery.start_soon(watch_button_closely, app)
nursery.start_soon(waste_time_freely)
async for _ in app.async_bind(
'on_stop', thread_fn=trio.BlockingTrioPortal().run_sync):
break
nursery.cancel_scope.cancel()
print('completed waiting on app stop')
async def init_async_listener():
'''This method is also run by trio and periodically prints something.'''
app = App.get_running_app()
async with trio.open_nursery() as nursery:
nursery.start_soon(watch_ondrop_closely, app)
nursery.start_soon(watch_button_closely, app)
async for _ in app.async_bind(
'on_stop', thread_fn=trio.BlockingTrioPortal().run_sync):
break
nursery.cancel_scope.cancel()
def __init__(self, url: str, nursery):
"""
:param url: The gateway URL.
:param nursery: The nursery to use.
"""
super().__init__(url)
self.nursery = nursery
self._portal = trio.BlockingTrioPortal()
self._queue = trio.Queue(capacity=5)
self._cancelled = threading.Event()
self._ws = None # type: WebSocket
async def ctx():
ctx = Namespace()
ctx.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-')
ctx.backend_pool = BackendPool(lambda: local.Backend(
Namespace(storage_url='local://' + ctx.backend_dir)))
ctx.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-')
ctx.max_obj_size = 1024
# Destructors are not guaranteed to run, and we can't unlink
# the file immediately because apsw refers to it by name.
# Therefore, we unlink the file manually in tearDown()
ctx.dbfile = tempfile.NamedTemporaryFile(delete=False)
ctx.db = Connection(ctx.dbfile.name)
create_tables(ctx.db)
init_tables(ctx.db)
# Create an inode we can work with
ctx.inode = 42
now_ns = time_ns()
ctx.db.execute(
"INSERT INTO inodes (id,mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?,?)",
(ctx.inode, stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
| stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH,
os.getuid(), os.getgid(), now_ns, now_ns, now_ns, 1, 32))
cache = BlockCache(ctx.backend_pool, ctx.db, ctx.cachedir + "/cache",
ctx.max_obj_size * 100)
cache.portal = trio.BlockingTrioPortal()
ctx.cache = cache
# Monkeypatch around the need for removal and upload threads
cache.to_remove = DummyQueue(cache)
class DummyChannel:
async def send(self, arg):
await trio.run_sync_in_worker_thread(cache._do_upload, *arg)
cache.to_upload = (DummyChannel(), None)
try:
yield ctx
finally:
ctx.cache.backend_pool = ctx.backend_pool
if ctx.cache.destroy is not None:
await ctx.cache.destroy()
shutil.rmtree(ctx.cachedir)
shutil.rmtree(ctx.backend_dir)
ctx.dbfile.close()
os.unlink(ctx.dbfile.name)
async def __call__(self, *args, **kwargs):
if contextvars is not None:
context = contextvars.copy_context()
child = functools.partial(self.func, *args, **kwargs)
func = context.run
args = (child, )
kwargs = {}
else:
func = self.func
portal = trio.BlockingTrioPortal()
return await SyncToAsync.executor.run_sync(
functools.partial(self.thread_handler, portal, func, *args,
**kwargs))
def __init__(self, *, branch_from=None):
if branch_from is None:
self._portal = trio.BlockingTrioPortal()
self._request_queue = trio.Queue(1)
self._response_queue = trio.Queue(1)
self._thread = threading.Thread(target=self.thread_fn, daemon=True)
self._thread.start()
else:
self._portal = branch_from._portal
self._request_queue = branch_from._request_queue
self._response_queue = branch_from._response_queue
self._thread = branch_from._thread
self._branched = branch_from is not None
self._has_quit = False
async def _run():
try:
portal = trio.BlockingTrioPortal()
self.core_queue.put(portal)
with trio.open_cancel_scope() as cancel_scope:
self.core_queue.put(cancel_scope)
async with logged_core_factory(
self.core_config, self.current_device
) as core:
self.core_queue.put(core)
await trio.sleep_forever()
# If we have an exception, we never put the core object in the queue. Since the
# main thread except something to be there, we put the exception.
except Exception as exc:
self.core_queue.put(exc)
async def ctx():
ctx = Namespace()
ctx.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-')
plain_backend = local.Backend(
Namespace(storage_url='local://' + ctx.backend_dir))
ctx.backend_pool = BackendPool(
lambda: ComprencBackend(b'schwubl', ('zlib', 6), plain_backend))
ctx.backend = ctx.backend_pool.pop_conn()
ctx.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-')
ctx.max_obj_size = 1024
# Destructors are not guaranteed to run, and we can't unlink
# the file immediately because apsw refers to it by name.
# Therefore, we unlink the file manually in tearDown()
ctx.dbfile = tempfile.NamedTemporaryFile(delete=False)
ctx.db = Connection(ctx.dbfile.name)
create_tables(ctx.db)
init_tables(ctx.db)
cache = BlockCache(ctx.backend_pool, ctx.db, ctx.cachedir + "/cache",
ctx.max_obj_size * 5)
cache.portal = trio.BlockingTrioPortal()
ctx.cache = cache
ctx.server = fs.Operations(cache, ctx.db, ctx.max_obj_size,
InodeCache(ctx.db, 0))
ctx.server.init()
# Monkeypatch around the need for removal and upload threads
cache.to_remove = DummyQueue(cache)
class DummyChannel:
async def send(self, arg):
await trio.run_sync_in_worker_thread(cache._do_upload, *arg)
cache.to_upload = (DummyChannel(), None)
# Keep track of unused filenames
ctx.name_cnt = 0
yield ctx
ctx.server.inodes.destroy()
await ctx.cache.destroy()
shutil.rmtree(ctx.cachedir)
shutil.rmtree(ctx.backend_dir)
os.unlink(ctx.dbfile.name)
ctx.dbfile.close()
async def run_fuse_in_thread(
fs, mountpoint: Path, fuse_config: dict, event_bus, *, task_status=trio.TASK_STATUS_IGNORED
):
"""
Raises:
MountpointConfigurationError
"""
fuse_thread_started = threading.Event()
fuse_thread_stopped = threading.Event()
portal = trio.BlockingTrioPortal()
abs_mountpoint = str(mountpoint.absolute())
fs_access = ThreadFSAccess(portal, fs)
fuse_operations = FuseOperations(fs_access, abs_mountpoint)
async def _join_fuse_runner(stop=False):
await _join_fuse_thread(mountpoint, fuse_operations, fuse_thread_stopped, stop=stop)
initial_st_dev = _bootstrap_mountpoint(mountpoint)
try:
event_bus.send("mountpoint.starting", mountpoint=abs_mountpoint)
async with trio.open_nursery() as nursery:
def _run_fuse_thread():
try:
fuse_thread_started.set()
FUSE(fuse_operations, abs_mountpoint, foreground=True, **fuse_config)
finally:
fuse_thread_stopped.set()
nursery.start_soon(
lambda: trio.run_sync_in_worker_thread(_run_fuse_thread, cancellable=True)
)
await _wait_for_fuse_ready(mountpoint, fuse_thread_started, initial_st_dev)
task_status.started(_join_fuse_runner),
event_bus.send("mountpoint.started", mountpoint=abs_mountpoint)
finally:
await _join_fuse_runner(stop=True)
event_bus.send("mountpoint.stopped", mountpoint=abs_mountpoint)
_teardown_mountpoint(mountpoint)
async def watch_button_closely(app):
'''This method is also run by trio and watches and reacts to the button
shown in kivy.'''
root = app.root
if root is None:
return
# watch the on_release event of the button and react to every release
async for _ in root.ids.button_execute.async_bind(
'on_release', thread_fn=trio.BlockingTrioPortal().run_sync):
app.root.ids.slider_difficulty.disabled = True
app.root.ids.button_input_file.disabled = True
app.root.ids.button_execute.disabled = True
input_path = app.root.input_path
app.root.dmatcher_runs = 0
execute_algorithm(app, input_path)
app.root.ids.slider_difficulty.disabled = False
app.root.ids.button_execute.disabled = False
app.root.ids.button_input_file.disabled = False
async def watch_button_closely(app):
'''This method is also run by trio and watches and reacts to the button
shown in kivy.'''
root = app.root
print('app started')
label = root.ids.label
i = 0
# watch the on_release event of the button and react to every release
async for _ in root.ids.btn.async_bind(
'on_release', thread_fn=trio.BlockingTrioPortal().run_sync):
await trio_run_in_kivy_thread(setattr, label, 'text',
'Pressed x{}'.format(i))
if i == 7:
break
i += 1
await trio_run_in_kivy_thread(setattr, label, 'text', 'Goodbye :(')
print('Done with update_label1')
async def main():
portal = trio.BlockingTrioPortal()
send_to_thread, receive_from_trio = trio.open_memory_channel(0)
send_to_trio, receive_from_thread = trio.open_memory_channel(0)
async with trio.open_nursery() as nursery:
# In a background thread, run:
# thread_fn(portal, receive_from_trio, send_to_trio)
nursery.start_soon(
trio.run_sync_in_worker_thread,
thread_fn, portal, receive_from_trio, send_to_trio
)
# prints "1"
await send_to_thread.send(0)
print(await receive_from_thread.receive())
# prints "2"
await send_to_thread.send(1)
print(await receive_from_thread.receive())
# When we close the channel, it signals the thread to exit.
await send_to_thread.aclose()
def __init__(self, *, branch_from=None):
if branch_from is None:
self._portal = trio.BlockingTrioPortal()
send_to_thread, receive_from_trio = trio.open_memory_channel(1)
send_to_trio, receive_from_thread = trio.open_memory_channel(1)
self._send_to_thread = send_to_thread
self._send_to_trio = send_to_trio
self._receive_from_trio = receive_from_trio
self._receive_from_thread = receive_from_thread
self._thread = threading.Thread(target=self.thread_fn, daemon=True)
self._thread.start()
else:
self._portal = branch_from._portal
self._send_to_thread = branch_from._send_to_thread
self._send_to_trio = branch_from._send_to_trio
self._receive_from_trio = branch_from._receive_from_trio
self._receive_from_thread = branch_from._receive_from_thread
self._thread = branch_from._thread
self._branched = branch_from is not None
self._has_quit = False
def __init__(self):
self.portal = trio.BlockingTrioPortal()
self.ThreadsafeQueue = _universal_queue(self.portal)
async def watch_ondrop_closely(app):
async for args in Window.async_bind(
'on_dropfile', thread_fn=trio.BlockingTrioPortal().run_sync):
app.handledrops(*args)
