def test_worker_timeout_mock(self):
timeout = .001
context = get_context()
executor = ProcessPoolExecutor(
max_workers=4, context=context, timeout=timeout)
result_queue = DelayedSimpleQueue(ctx=context, delay=.001)
executor._result_queue = result_queue
with pytest.warns(UserWarning,
match=r'^A worker stopped while some jobs'):
for i in range(5):
# Trigger worker spawn for lazy executor implementations
for result in executor.map(id, range(8)):
pass
executor.shutdown()
result_queue.close()
import multiprocessing as mp
from loky import ProcessPoolExecutor
# normal code
res = map(greet, ['Sipho', 'Sindi'])
# use worker pools
pool = mp.Pool(2)
res = pool.map(greet, ['Sipho', 'Sindi'])
# use loky for cross platform consistancy
executor = ProcessPoolExecutor(max_workers=2)
res = executor.map(greet, ('Sipho', 'Sindi'))
#TODO: Research joblib
def run_in_parallel():
eight = len(ids_to_do) // 8
quater = len(ids_to_do) // 4
half = len(ids_to_do) // 2
args = [
ids_to_do[:eight], ids_to_do[eight:quater],
ids_to_do[quater:(quater + eight)], ids_to_do[(quater + eight):half],
ids_to_do[half:-(quater + eight)],
ids_to_do[-(quater + eight):-quater], ids_to_do[-quater:-eight],
ids_to_do[-eight:]
]
pool = multiprocessing.Pool(processes=len(args))
pool.map(export_images2, args)
# %% {"slideshow": {"slide_type": "slide"}}
%%time
import os
if os.name == "nt":
from loky import ProcessPoolExecutor # for Windows users
else:
from concurrent.futures import ProcessPoolExecutor
def delayed_square(x):
sleep(1)
return x*x
e = ProcessPoolExecutor()
results = list(e.map(delayed_square, range(8)))
# %% {"slideshow": {"slide_type": "slide"}}
%%time
from concurrent.futures import ThreadPoolExecutor
e = ThreadPoolExecutor()
results = list(e.map(delayed_square, range(8)))
# %% [markdown] {"slideshow": {"slide_type": "slide"}}
# # Asynchronous Future
# While many parallel applications can be described as maps, some can be more complex. In this section we look at the asynchronous Future interface, which provides a simple API for ad-hoc parallelism. This is useful for when your computations don't fit a regular pattern.
#
# %% [markdown] {"slideshow": {"slide_type": "fragment"}}