def test_check_process_leak_post_cleanup(ignore_sigterm):
barrier = mp_context.Barrier(parties=2)
with check_process_leak(check=False, term_timeout=0.2):
p = mp_context.Process(target=garbage_process,
args=(barrier, ignore_sigterm))
p.start()
barrier.wait()
assert not p.is_alive()
def test_check_process_leak():
barrier = mp_context.Barrier(parties=2)
with pytest.raises(AssertionError):
with check_process_leak(check=True, check_timeout=0.01):
p = mp_context.Process(target=garbage_process, args=(barrier, ))
p.start()
barrier.wait()
assert not p.is_alive()
def _test_workspace_concurrency(tmpdir, timeout, max_procs):
"""
WorkSpace concurrency test. We merely check that no exception or
deadlock happens.
"""
base_dir = str(tmpdir)
err_q = mp_context.Queue()
purged_q = mp_context.Queue()
stop_evt = mp_context.Event()
ws = WorkSpace(base_dir)
# Make sure purging only happens in the child processes
ws._purge_leftovers = lambda: None
# Run a bunch of child processes that will try to purge concurrently
NPROCS = 2 if sys.platform == "win32" else max_procs
processes = [
mp_context.Process(
target=_workspace_concurrency, args=(base_dir, purged_q, err_q, stop_evt)
)
for i in range(NPROCS)
]
for p in processes:
p.start()
n_created = 0
n_purged = 0
try:
t1 = time()
while time() - t1 < timeout:
# Add a bunch of locks, and simulate forgetting them.
# The concurrent processes should try to purge them.
for i in range(50):
d = ws.new_work_dir(prefix="workspace-concurrency-")
d._finalizer.detach()
n_created += 1
sleep(1e-2)
finally:
stop_evt.set()
for p in processes:
p.join()
# Any errors?
try:
err = err_q.get_nowait()
except Empty:
pass
else:
raise err
try:
while True:
n_purged += purged_q.get_nowait()
except Empty:
pass
# We attempted to purge most directories at some point
assert n_purged >= 0.5 * n_created > 0
return n_created, n_purged
def test_check_process_leak_slow_cleanup():
"""check_process_leak waits a bit for processes to terminate themselves"""
barrier = mp_context.Barrier(parties=2)
with check_process_leak(check=True):
p = mp_context.Process(target=garbage_process,
args=(barrier, False, 0.2))
p.start()
barrier.wait()
assert not p.is_alive()
def _test_workspace_concurrency(tmpdir, timeout, max_procs):
base_dir = str(tmpdir)
err_q = mp_context.Queue()
purged_q = mp_context.Queue()
stop_evt = mp_context.Event()
ws = WorkSpace(base_dir)
# Make sure purging only happens in the child processes
ws._purge_leftovers = lambda: None
NPROCS = 2 if sys.platform == 'win32' else max_procs
processes = [
mp_context.Process(target=_workspace_concurrency,
args=(base_dir, purged_q, err_q, stop_evt))
for i in range(NPROCS)
]
for p in processes:
p.start()
n_created = 0
n_purged = 0
try:
t1 = time()
while time() - t1 < timeout:
# Add a bunch of locks, and simulate forgetting them
for i in range(50):
d = ws.new_work_dir(prefix='workspace-concurrency-')
d._finalizer.detach()
n_created += 1
sleep(1e-2)
finally:
stop_evt.set()
for p in processes:
p.join()
# Any errors?
try:
err = err_q.get_nowait()
except Empty:
pass
else:
raise err
try:
while True:
n_purged += purged_q.get_nowait()
except Empty:
pass
return n_created, n_purged
def __init__(self, loop=None, target=None, name=None, args=(), kwargs={}):
if not callable(target):
raise TypeError(
f"`target` needs to be callable, not {type(target)!r}")
self._state = _ProcessState()
self._loop = loop or IOLoop.current(instance=False)
# _keep_child_alive is the write side of a pipe, which, when it is
# closed, causes the read side of the pipe to unblock for reading. Note
# that it is never closed directly. The write side is closed by the
# kernel when our process exits, or possibly by the garbage collector
# closing the file descriptor when the last reference to
# _keep_child_alive goes away. We can take advantage of this fact to
# monitor from the child and exit when the parent goes away unexpectedly
# (for example due to SIGKILL). This variable is otherwise unused except
# for the assignment here.
parent_alive_pipe, self._keep_child_alive = mp_context.Pipe(
duplex=False)
self._process = mp_context.Process(
target=self._run,
name=name,
args=(
target,
args,
kwargs,
parent_alive_pipe,
self._keep_child_alive,
dask.config.global_config,
),
)
self._name = self._process.name
self._proc_finalizer = weakref.finalize(self, _asyncprocess_finalizer,
self._process)
self._watch_q = PyQueue()
self._exit_future = Future()
self._exit_callback = None
self._closed = False
self._start_threads()
def test_asyncprocess_child_teardown_on_parent_exit():
r"""Check that a child process started by AsyncProcess exits if its parent
exits.
The motivation is to ensure that if an AsyncProcess is created and the
creator process dies unexpectedly (e.g, via Out-of-memory SIGKILL), the
child process and resources held by it should not be leaked.
The child should monitor its parent and exit promptly if the parent exits.
[test process] -> [parent using AsyncProcess (dies)] -> [worker process]
\ /
\________ <-- child_pipe <-- ________/
"""
# When child_pipe is closed, the children_alive pipe unblocks.
children_alive, child_pipe = mp_context.Pipe(duplex=False)
try:
parent = mp_context.Process(target=_parent_process,
args=(child_pipe, ))
parent.start()
# Close our reference to child_pipe so that the child has the only one.
child_pipe.close()
# Wait for the parent to exit. By the time join returns, the child
# process is orphaned, and should be in the process of exiting by
# itself.
parent.join()
# By the time we reach here,the parent has exited. The parent only exits
# when the child is ready to enter the sleep, so all of the slow things
# (process startup, etc) should have happened by now, even on a busy
# system. A short timeout should therefore be appropriate.
short_timeout = 5.0
# Poll is used to allow other tests to proceed after this one in case of
# test failure.
try:
readable = children_alive.poll(short_timeout)
except BrokenPipeError:
assert WINDOWS, "should only raise on windows"
# Broken pipe implies closed, which is readable.
readable = True
# If this assert fires, then something went wrong. Either the child
# should write into the pipe, or it should exit and the pipe should be
# closed (which makes it become readable).
assert readable
try:
# This won't block due to the above 'assert readable'.
result = children_alive.recv()
except EOFError:
pass # Test passes.
except BrokenPipeError:
assert WINDOWS, "should only raise on windows"
# Test passes.
else:
# Oops, children_alive read something. It should be closed. If
# something was read, it's a message from the child telling us they
# are still alive!
raise RuntimeError(f"unreachable: {result}")
finally:
# Cleanup.
children_alive.close()
def test_workspace_concurrency(tmpdir):
"""WorkSpace concurrency test. We merely check that no exception or
deadlock happens.
"""
base_dir = str(tmpdir)
err_q = mp_context.Queue()
purged_q = mp_context.Queue()
stop_evt = mp_context.Event()
ws = WorkSpace(base_dir)
# Make sure purging only happens in the child processes
ws._purge_leftovers = lambda: None
# Windows (or at least Windows GitHub CI) has been observed to be exceptionally
# slow. Don't stress it too much.
max_procs = 2 if WINDOWS else 16
# Run a bunch of child processes that will try to purge concurrently
barrier = mp_context.Barrier(parties=max_procs + 1)
processes = [
mp_context.Process(
target=_workspace_concurrency,
args=(base_dir, purged_q, err_q, stop_evt, barrier),
)
for _ in range(max_procs)
]
for p in processes:
p.start()
barrier.wait()
n_created = 0
n_purged = 0
t1 = time()
try:
# On Linux, you will typically end with n_created > 10.000
# On Windows, it can take 60 seconds to create 50 locks!
while time() - t1 < 10:
# Add a bunch of locks and simulate forgetting them.
# The concurrent processes should try to purge them.
for _ in range(100):
d = ws.new_work_dir(prefix="workspace-concurrency-")
d._finalizer.detach()
n_created += 1
finally:
stop_evt.set()
for p in processes:
p.join()
# Any errors?
try:
err = err_q.get_nowait()
except queue.Empty:
pass
else:
raise err
try:
while True:
n_purged += purged_q.get_nowait()
except queue.Empty:
pass
# We attempted to purge most directories at some point
assert n_purged >= 0.5 * n_created > 0
