def spawnv_passfds(path: str, args: List[str], passfds: List[int]) -> Any:
if '-c' in args:
idx = args.index('-c')
patched_args = [spawn.get_executable(), '+runpy'] + args[idx + 1:]
else:
idx = args.index('--multiprocessing-fork')
prog = 'from multiprocessing.spawn import spawn_main; spawn_main(%s)'
prog %= ', '.join(item for item in args[idx + 1:])
patched_args = [spawn.get_executable(), '+runpy', prog]
return orig_spawn_passfds(kitty_exe(), patched_args, passfds)
def ensure_running(self):
'''Make sure that a fork server is running.
This can be called from any process. Note that usually a child
process will just reuse the forkserver started by its parent, so
ensure_running() will do nothing.
'''
with self._lock:
resource_tracker.ensure_running()
if self._forkserver_pid is not None:
# forkserver was launched before, is it still running?
pid, status = os.waitpid(self._forkserver_pid, os.WNOHANG)
if not pid:
# still alive
return
# dead, launch it again
os.close(self._forkserver_alive_fd)
self._forkserver_address = None
self._forkserver_alive_fd = None
self._forkserver_pid = None
# XXX only thing that changed!
cmd = ('from tractor._forkserver_override import main; ' +
'main(%d, %d, %r, **%r)')
if self._preload_modules:
desired_keys = {'main_path', 'sys_path'}
data = spawn.get_preparation_data('ignore')
data = {x: y for x, y in data.items() if x in desired_keys}
else:
data = {}
with socket.socket(socket.AF_UNIX) as listener:
address = connection.arbitrary_address('AF_UNIX')
listener.bind(address)
if not util.is_abstract_socket_namespace(address):
os.chmod(address, 0o600)
listener.listen()
# all client processes own the write end of the "alive" pipe;
# when they all terminate the read end becomes ready.
alive_r, alive_w = os.pipe()
try:
fds_to_pass = [listener.fileno(), alive_r]
cmd %= (listener.fileno(), alive_r, self._preload_modules,
data)
exe = spawn.get_executable()
args = [exe] + util._args_from_interpreter_flags()
args += ['-c', cmd]
pid = util.spawnv_passfds(exe, args, fds_to_pass)
except:
os.close(alive_w)
raise
finally:
os.close(alive_r)
self._forkserver_address = address
self._forkserver_alive_fd = alive_w
self._forkserver_pid = pid
def spawnv_passfds(path: str, args: List[str], passfds: List[int]) -> Any:
idx = args.index('-c')
patched_args = [spawn.get_executable(), '+runpy'] + args[idx + 1:]
return orig_spawn_passfds(kitty_exe(), patched_args, passfds)
# vim:fileencoding=utf-8
# License: GPLv3 Copyright: 2020, Kovid Goyal <kovid at kovidgoyal.net>
# Monkeypatch the stdlib multiprocessing module to work with the embedded python
# in kitty, when using the spawn launcher.
from concurrent.futures import ProcessPoolExecutor
from multiprocessing import util # type: ignore
from multiprocessing import context, get_all_start_methods, get_context, spawn
from typing import Any, Callable, List, Optional, Tuple, Union
from .constants import kitty_exe
orig_spawn_passfds = util.spawnv_passfds
orig_executable = spawn.get_executable()
def spawnv_passfds(path: str, args: List[str], passfds: List[int]) -> Any:
idx = args.index('-c')
patched_args = [spawn.get_executable(), '+runpy'] + args[idx + 1:]
return orig_spawn_passfds(kitty_exe(), patched_args, passfds)
def monkey_patch_multiprocessing() -> None:
# Use kitty to run the worker process used by multiprocessing
spawn.set_executable(kitty_exe())
util.spawnv_passfds = spawnv_passfds
def unmonkey_patch_multiprocessing() -> None:
def ensure_running(self):
'''Make sure that resource tracker process is running.
This can be run from any process. Usually a child process will use
the resource created by its parent.'''
with self._lock:
if self._fd is not None:
# resource tracker was launched before, is it still running?
if self._check_alive():
# => still alive
return
# => dead, launch it again
os.close(self._fd)
# Clean-up to avoid dangling processes.
try:
# _pid can be None if this process is a child from another
# python process, which has started the resource_tracker.
if self._pid is not None:
os.waitpid(self._pid, 0)
except ChildProcessError:
# The resource_tracker has already been terminated.
pass
self._fd = None
self._pid = None
warnings.warn('resource_tracker: process died unexpectedly, '
'relaunching. Some resources might leak.')
fds_to_pass = []
try:
fds_to_pass.append(sys.stderr.fileno())
except Exception:
pass
cmd = 'from shared_memory.resource_tracker import main;main(%d)'
r, w = os.pipe()
try:
fds_to_pass.append(r)
# process will out live us, so no need to wait on pid
exe = spawn.get_executable()
args = [exe] + util._args_from_interpreter_flags()
args += ['-c', cmd % r]
# bpo-33613: Register a signal mask that will block the signals.
# This signal mask will be inherited by the child that is going
# to be spawned and will protect the child from a race condition
# that can make the child die before it registers signal handlers
# for SIGINT and SIGTERM. The mask is unregistered after spawning
# the child.
try:
if _HAVE_SIGMASK:
signal.pthread_sigmask(signal.SIG_BLOCK,
_IGNORED_SIGNALS)
pid = util.spawnv_passfds(exe, args, fds_to_pass)
finally:
if _HAVE_SIGMASK:
signal.pthread_sigmask(signal.SIG_UNBLOCK,
_IGNORED_SIGNALS)
except:
os.close(w)
raise
else:
self._fd = w
self._pid = pid
finally:
os.close(r)