def test_child_retrieves_resource_tracker(self):
parent_rtracker_pid = get_rtracker_pid()
executor = ProcessPoolExecutor(max_workers=2)
child_rtracker_pid = executor.submit(get_rtracker_pid).result()
# First simple pid retrieval check (see #200)
assert child_rtracker_pid == parent_rtracker_pid
# Register a resource in the parent process, and un-register it in the
# child process. If the two processes do not share the same
# resource_tracker, a cache KeyError should be printed in stderr.
import subprocess
cmd = '''if 1:
import os, sys
from loky import ProcessPoolExecutor
from loky.backend import resource_tracker
from loky.backend.semlock import SemLock
from tempfile import NamedTemporaryFile
tmpfile = NamedTemporaryFile(delete=False)
tmpfile.close()
filename = tmpfile.name
resource_tracker.VERBOSE = True
resource_tracker.register(filename, "file")
def maybe_unlink(name, rtype):
# resource_tracker.maybe_unlink is actually a bound method of the
# ResourceTracker. We need a custom wrapper to avoid object
# serialization.
from loky.backend import resource_tracker
resource_tracker.maybe_unlink(name, rtype)
print(filename)
e = ProcessPoolExecutor(1)
e.submit(maybe_unlink, filename, "file").result()
e.shutdown()
'''
try:
p = subprocess.Popen([sys.executable, '-E', '-c', cmd],
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
p.wait()
filename = p.stdout.readline().decode('utf-8').strip()
err = p.stderr.read().decode('utf-8')
p.stderr.close()
p.stdout.close()
pattern = "decremented refcount of file %s" % filename
assert pattern in err
assert "leaked" not in err
pattern = "KeyError: '%s'" % filename
assert pattern not in err
finally:
executor.shutdown()
def test_child_retrieves_resource_tracker(self):
parent_rtracker_pid = get_rtracker_pid()
executor = ProcessPoolExecutor(max_workers=2)
child_rtracker_pid = executor.submit(get_rtracker_pid).result()
# First simple pid retrieval check (see #200)
assert child_rtracker_pid == parent_rtracker_pid
# Register a resource in the parent process, and un-register it in the
# child process. If the two processes do not share the same
# resource_tracker, a cache KeyError should be printed in stderr.
import subprocess
folder_name = 'loky_tempfolder'
cmd = '''if 1:
import os, sys
from loky import ProcessPoolExecutor
from loky.backend import resource_tracker
from loky.backend.semlock import SemLock
resource_tracker.VERBOSE=True
folder_name = "{}"
# We don't need to create the semaphore as registering / unregistering
# operations simply add / remove entries from a cache, but do not
# manipulate the actual semaphores.
resource_tracker.register(folder_name, "folder")
def unregister(name, rtype):
# resource_tracker.unregister is actually a bound method of the
# ResourceTracker. We need a custom wrapper to avoid object
# serialization.
from loky.backend import resource_tracker
resource_tracker.unregister(folder_name, rtype)
e = ProcessPoolExecutor(1)
e.submit(unregister, folder_name, "folder").result()
e.shutdown()
'''
try:
p = subprocess.Popen(
[sys.executable, '-E', '-c', cmd.format(folder_name)],
stderr=subprocess.PIPE)
p.wait()
err = p.stderr.read().decode('utf-8')
p.stderr.close()
assert re.search("unregister %s" % folder_name, err) is not None
assert re.search("leaked", err) is None
assert re.search("KeyError: '%s'" % folder_name, err) is None
finally:
executor.shutdown()
def scanner(self, queue, parent, npes, track, sensor, dtype, dt_tol,
dL_tol):
timeout = 900
self.logger.debug("Scanner, pid: {}".format(os.getpid()))
filenames = bloom_filter(track, sensor, dtype, dt_tol, dL_tol)
self.logger.debug("Finished bloom filter")
with ProcessPoolExecutor(max_workers=npes,
timeout=timeout) as executor:
results = []
for f in filenames:
self.logger.info("Scanning: {}".format(f))
if (len(results) >= npes) and parent.is_alive():
idx = [r.done() for r in results]
while not np.any(idx):
time.sleep(1)
idx = [r.done() for r in results]
tmp = results.pop(idx.index(True)).result()
self.logger.debug("Finished reading another file")
if not tmp.empty:
self.logger.warning("Found {} matchs".format(len(tmp)))
queue.put(tmp)
self.logger.debug("Getting {}".format(f))
ds = self.db[f].compute()
if not parent.is_alive():
return
self.logger.debug("Submitting a new inrange process")
results.append(
executor.submit(matchup, track, ds, dL_tol, dt_tol))
for tmp in (r.result(timeout) for r in results):
if not parent.is_alive():
return
self.logger.debug("Finished reading another file")
if not tmp.empty:
self.logger.warning("Found {} matchs".format(len(tmp)))
queue.put(tmp)
self.logger.debug("Finished scanning all potential matchups.")
queue.put("END")
def test_worker_timeout_mock(self):
timeout = .001
context = get_context()
executor = ProcessPoolExecutor(
max_workers=4, context=context, timeout=timeout)
result_queue = DelayedSimpleQueue(ctx=context, delay=.001)
executor._result_queue = result_queue
with pytest.warns(UserWarning,
match=r'^A worker stopped while some jobs'):
for i in range(5):
# Trigger worker spawn for lazy executor implementations
for result in executor.map(id, range(8)):
pass
executor.shutdown()
result_queue.close()
def target():
def done(future: Future):
try:
q_out.put(future.result())
q_in.task_done()
except BaseException:
stop_event.set()
raise
finally:
counter.release()
# start worker
executor = ProcessPoolExecutor(n_workers)
counter = Semaphore(n_workers)
wait = True
try:
for value in iter(q_in.get, SourceExhausted()):
counter.acquire()
executor.submit(transform, value, *args,
**kwargs).add_done_callback(done)
# wait for other processes
q_in.task_done()
q_in.join()
q_out.put(SourceExhausted())
except StopEvent:
pass
except BaseException:
wait = False
stop_event.set()
raise
finally:
executor.shutdown(wait=wait)
import multiprocessing as mp
from loky import ProcessPoolExecutor
# normal code
res = map(greet, ['Sipho', 'Sindi'])
# use worker pools
pool = mp.Pool(2)
res = pool.map(greet, ['Sipho', 'Sindi'])
# use loky for cross platform consistancy
executor = ProcessPoolExecutor(max_workers=2)
res = executor.map(greet, ('Sipho', 'Sindi'))
#TODO: Research joblib
def run_in_parallel():
eight = len(ids_to_do) // 8
quater = len(ids_to_do) // 4
half = len(ids_to_do) // 2
args = [
ids_to_do[:eight], ids_to_do[eight:quater],
ids_to_do[quater:(quater + eight)], ids_to_do[(quater + eight):half],
ids_to_do[half:-(quater + eight)],
ids_to_do[-(quater + eight):-quater], ids_to_do[-quater:-eight],
ids_to_do[-eight:]
]
pool = multiprocessing.Pool(processes=len(args))
pool.map(export_images2, args)
#
# The asynchronous execution can be performed with threads, using ThreadPoolExecutor, or separate processes, using ProcessPoolExecutor. Both implement the same interface, which is defined by the abstract Executor class.
# %% {"slideshow": {"slide_type": "slide"}}
%%time
import os
if os.name == "nt":
from loky import ProcessPoolExecutor # for Windows users
else:
from concurrent.futures import ProcessPoolExecutor
def delayed_square(x):
sleep(1)
return x*x
e = ProcessPoolExecutor()
results = list(e.map(delayed_square, range(8)))
# %% {"slideshow": {"slide_type": "slide"}}
%%time
from concurrent.futures import ThreadPoolExecutor
e = ThreadPoolExecutor()
results = list(e.map(delayed_square, range(8)))
# %% [markdown] {"slideshow": {"slide_type": "slide"}}
# # Asynchronous Future
# While many parallel applications can be described as maps, some can be more complex. In this section we look at the asynchronous Future interface, which provides a simple API for ad-hoc parallelism. This is useful for when your computations don't fit a regular pattern.
#
"""Deadlock with pickling errors
================================
This example highlights the fact that the ProcessPoolExecutor implementation
from concurrent.futures is not robust to pickling error (at least in versions
3.6 and lower).
"""
import time
from loky import ProcessPoolExecutor
from loky.backend import get_context
class SlowPickle:
def __init__(self, delay=.1):
self.delay = delay
def __reduce__(self):
time.sleep(self.delay)
return SlowPickle, (self.delay,)
if __name__ == "__main__":
ctx = get_context("spawn")
o = SlowPickle()
with ProcessPoolExecutor(max_workers=2, timeout=.01, context=ctx) as e:
f = e.submit(id, SlowPickle())
f.result()
from time import time, sleep
if os.name == "nt":
from loky import ProcessPoolExecutor # for Windows users
else:
from concurrent.futures import ProcessPoolExecutor
from time import time, sleep
def delayed_square(x):
sleep(1)
return x*x
if __name__ == "__main__":
start = time()
data = list(range(8))
with ProcessPoolExecutor() as pool:
result = sum(pool.map(delayed_square, data))
stop = time()
print(f" result : {result} - elapsed time {stop - start}")
# + {"slideshow": {"slide_type": "slide"}, "execution": {"iopub.status.busy": "2020-09-12T14:00:37.368389Z", "iopub.execute_input": "2020-09-12T14:00:37.369466Z", "iopub.status.idle": "2020-09-12T14:00:39.614789Z", "shell.execute_reply": "2020-09-12T14:00:39.615379Z"}}
!{sys.executable} process_pool.py
# + {"slideshow": {"slide_type": "slide"}, "execution": {"iopub.status.busy": "2020-09-12T14:00:39.620798Z", "iopub.execute_input": "2020-09-12T14:00:39.621801Z", "iopub.status.idle": "2020-09-12T14:00:40.633249Z", "shell.execute_reply": "2020-09-12T14:00:40.634198Z"}}
%%time
from concurrent.futures import ThreadPoolExecutor
e = ThreadPoolExecutor()
results = list(e.map(delayed_square, range(8)))