def test_set_loky_pickler_failures(self):
# Check that `set_loky_pickler` fails when a non-existing module is
# required.
with pytest.raises(ImportError, match=r"loky_pickler to 'no_module'"):
set_loky_pickler("no_module")
# The module passed to `set_loky_pickler` should have a Pickler
# attribute.
with pytest.raises(AttributeError, match=r"loky_pickler to 'os'"):
set_loky_pickler("os")
def set_parallelization(
use_loky=False,
start_method=None,
loky_pickler=None,
use_threadpool_limits=True,
): # pragma: nocover
"""Configure multi-process parallelization.
Args:
use_loky (bool): Value for :obj:`USE_LOKY`.
start_method (None or str): One of 'fork', 'spawn', and 'forkserver',
see :func:`multiprocessing.set_start_method`. If ``use_loky=True``,
also 'loky' and 'loky_int_main', see :mod:`loky`. If None, a
platform and version-dependent default will be chosen automatically
(e.g., 'fork' on Linux, 'spawn' on Windows, 'loky' if
``use_loky=True``)
loky_pickler (None or str): Serialization module to use for
:mod:`loky`. One of 'cloudpickle', 'pickle'. This forces the
serialization for *all* objects. The default value None chooses the
serialization automatically depending of the type of object. Using
'cloudpickle' is signficiantly slower than 'pickle' (but 'pickle'
cannot serialize all objects, such as lambda functions or functions
defined in a Jupyter notebook).
use_threadpool_limits (bool): Value for :obj:`USE_THREADPOOL_LIMITS`.
Raises:
ImportError: if ``use_loky=True`` but :mod:`loky` is not installed.
Note:
When working in Jupyter notebooks on systems that use the 'spawn'
`start_method` (Windows, or macOS with Python >= 3.8), you may have to
use :mod:`loky` (``use_loky=True``). This will incur a signficiant
increase in multi-processing overhead. Use Linux if you can.
Warning:
This function should only be called once per script/notebook, at its
very beginning. The :obj:`USE_LOKY` and :obj:`USE_THREADPOOL_LIMITS`
variables may be set at any time.
"""
global USE_LOKY
start_methods = ['fork', 'spawn', 'forkserver']
if use_loky:
start_methods.extend(['loky', 'loky_int_main'])
if start_method is not None:
if start_method not in start_methods:
raise ValueError("start_method not in %s" % str(start_methods))
if use_loky:
if not _HAS_LOKY:
raise ImportError("The loky library is not installed.")
USE_LOKY = True
loky.backend.context.set_start_method(start_method)
if loky_pickler is not None:
loky.set_loky_pickler(loky_pickler)
else:
multiprocessing.set_start_method(start_method)
executor = get_reusable_executor(max_workers=1)
executor.submit(func_async, 21, large_list).result()
print("With cloudpickle serialization: {:.3f}s".format(time.time() - t_start))
###############################################################################
# To mitigate this, it is possible to fully rely on ``pickle`` to serialize
# all communications between the main process and the workers. This can be done
# with an environment variable ``LOKY_PICKLER=pickle`` set before the
# script is launched, or with the switch ``set_loky_pickler`` provided in the
# ``loky`` API.
#
# Now set the `loky_pickler` to use the pickle serialization from stdlib. Here,
# we do not pass the desired function ``call_function`` as it is not picklable
# but it is replaced by ``id`` for demonstration purposes.
set_loky_pickler('pickle')
t_start = time.time()
executor = get_reusable_executor(max_workers=1)
executor.submit(id, large_list).result()
print("With pickle serialization: {:.3f}s".format(time.time() - t_start))
###############################################################################
# However, the function and objects defined in ``__main__`` are not
# serializable anymore using ``pickle`` and it is not possible to call
# ``func_async`` using this pickler.
#
try:
executor = get_reusable_executor(max_workers=1)
executor.submit(func_async, 21, large_list).result()
except Exception: