def test_loky_get_context(self):
# check the behavior of get_context
ctx_default = get_context()
assert ctx_default.get_start_method() == "loky"
ctx_loky = get_context("loky")
assert ctx_loky.get_start_method() == "loky"
ctx_loky_init_main = get_context("loky_init_main")
assert ctx_loky_init_main.get_start_method() == "loky_init_main"
with pytest.raises(ValueError):
get_context("not_available")
def setup_method(self, method):
default_start_method = get_context().get_start_method()
assert default_start_method == "loky", default_start_method
executor = get_reusable_executor(max_workers=2)
_check_executor_started(executor)
# There can be less than 2 workers because of the worker timeout
_check_subprocesses_number(executor, expected_max_process_number=2)
def test_process(self, capsys, context_name):
"""behavior of Process variables and functional connection objects
"""
import contextlib
@contextlib.contextmanager
def no_mgr():
yield None
with capsys.disabled() if sys.version_info[:2] == (3, 3) else no_mgr():
q = self.Queue()
sq = self.SimpleQueue()
args = (q, sq, 1, 2)
kwargs = {'hello': 23, 'bye': 2.54}
name = 'TestLokyProcess'
ctx = get_context(context_name)
p = ctx.Process(target=self._test_process,
args=args,
kwargs=kwargs,
name=name)
p.daemon = True
current = self.current_process()
assert p.authkey == current.authkey
assert not p.is_alive()
assert p.daemon
assert p not in self.active_children()
assert type(self.active_children()) is list
assert p.exitcode is None
# Make sure we do not break security
with pytest.raises(TypeError):
pickle.dumps(p.authkey)
# Make sure we detect bad pickling
with pytest.raises(RuntimeError):
pickle.dumps(q)
p.start()
assert p.exitcode is None
assert p.is_alive()
assert p in self.active_children()
assert q.get() == args[2:]
assert sq.get() == args[2:]
assert q.get() == kwargs
assert q.get() == p.name
assert q.get() == current.authkey
assert q.get() == p.pid
p.join()
assert p.exitcode == 0
assert not p.is_alive()
assert p not in self.active_children()
def test_worker_timeout_mock(self):
timeout = .001
context = get_context()
executor = ProcessPoolExecutor(
max_workers=4, context=context, timeout=timeout)
result_queue = DelayedSimpleQueue(ctx=context, delay=.001)
executor._result_queue = result_queue
with pytest.warns(UserWarning,
match=r'^A worker stopped while some jobs'):
for i in range(5):
# Trigger worker spawn for lazy executor implementations
for result in executor.map(id, range(8)):
pass
executor.shutdown()
result_queue.close()
def test_default_subcontext(method):
code = """if True:
import sys
from loky.backend.context import get_context, set_start_method
from tests.test_loky_backend import _test_default_subcontext
set_start_method('{method}')
ctx = get_context()
assert ctx.get_start_method() == '{method}'
queue = ctx.SimpleQueue()
p = ctx.Process(target=_test_default_subcontext, args=(queue,))
p.start()
p.join()
start_method = queue.get()
assert start_method == '{method}', start_method
try:
set_start_method('loky')
except RuntimeError:
pass
else:
raise AssertionError("Did not raise RuntimeError when resetting"
"start_method without force")
set_start_method(None, force=True)
ctx = get_context()
assert ctx.get_start_method() == 'loky'
""".format(method=method)
cmd = [sys.executable, "-c", code]
check_subprocess_call(cmd, timeout=10)
ctx_default = get_context()
assert ctx_default.get_start_method() == "loky"
if not hasattr(socket, "socketpair"):
def socketpair():
s1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s1.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s1.bind((socket.gethostname(), 8080))
s1.listen(1)
s2.connect((socket.gethostname(), 8080))
conn, addr = s1.accept()
return conn, s2
socket.socketpair = socketpair
DELTA = 0.1
ctx_loky = get_context("loky")
HAVE_SEND_HANDLE = (sys.platform == "win32"
or (hasattr(socket, 'CMSG_LEN')
and hasattr(socket, 'SCM_RIGHTS')
and hasattr(socket.socket, 'sendmsg')))
HAVE_FROM_FD = hasattr(socket, "fromfd")
class TestLokyBackend:
# loky processes
Process = staticmethod(ctx_loky.Process)
current_process = staticmethod(mp.current_process)
active_children = staticmethod(mp.active_children)
# interprocess communication objects
Pipe = staticmethod(ctx_loky.Pipe)
import os
import sys
import time
import pytest
import signal
import threading
from loky.backend import get_context
from .utils import TimingWrapper
loky_context = get_context("loky")
DELTA = 0.1
TIMEOUT1 = .1
TIMEOUT2 = .3
@pytest.mark.skipif(sys.platform == "win32", reason="UNIX test")
def test_semlock_failure():
from loky.backend.synchronize import SemLock, sem_unlink
name = "loky-test-semlock"
try:
sem_unlink(name)
except FileNotFoundError:
pass
try:
sl = SemLock(0, 1, 1, name=name)
assert sl.name == name
class ProcessPoolLokyMixin(ExecutorMixin):
# Makes sure that the context is defined
executor_type = process_executor.ProcessPoolExecutor
context = get_context("loky")
class ProcessPoolForkserverMixin(ExecutorMixin):
executor_type = process_executor.ProcessPoolExecutor
context = get_context('forkserver')
def test_invalid_context():
"""Raise error on invalid context"""
with pytest.warns(UserWarning):
with pytest.raises(ValueError):
get_reusable_executor(max_workers=2, context=get_context("fork"))
class ProcessPoolSpawnMixin(ExecutorMixin):
executor_type = process_executor.ProcessPoolExecutor
context = get_context('spawn')
"""Deadlock with pickling errors
================================
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to pickling error (at least in versions
3.6 and lower).
"""
import time
from loky import ProcessPoolExecutor
from loky.backend import get_context
class SlowPickle:
def __init__(self, delay=.1):
self.delay = delay
def __reduce__(self):
time.sleep(self.delay)
return SlowPickle, (self.delay,)
if __name__ == "__main__":
ctx = get_context("spawn")
o = SlowPickle()
with ProcessPoolExecutor(max_workers=2, timeout=.01, context=ctx) as e:
f = e.submit(id, SlowPickle())
f.result()