def test_child_retrieves_resource_tracker(self): parent_rtracker_pid = get_rtracker_pid() executor = ProcessPoolExecutor(max_workers=2) child_rtracker_pid = executor.submit(get_rtracker_pid).result() # First simple pid retrieval check (see #200) assert child_rtracker_pid == parent_rtracker_pid # Register a resource in the parent process, and un-register it in the # child process. If the two processes do not share the same # resource_tracker, a cache KeyError should be printed in stderr. import subprocess cmd = '''if 1: import os, sys from loky import ProcessPoolExecutor from loky.backend import resource_tracker from loky.backend.semlock import SemLock from tempfile import NamedTemporaryFile tmpfile = NamedTemporaryFile(delete=False) tmpfile.close() filename = tmpfile.name resource_tracker.VERBOSE = True resource_tracker.register(filename, "file") def maybe_unlink(name, rtype): # resource_tracker.maybe_unlink is actually a bound method of the # ResourceTracker. We need a custom wrapper to avoid object # serialization. from loky.backend import resource_tracker resource_tracker.maybe_unlink(name, rtype) print(filename) e = ProcessPoolExecutor(1) e.submit(maybe_unlink, filename, "file").result() e.shutdown() ''' try: p = subprocess.Popen([sys.executable, '-E', '-c', cmd], stderr=subprocess.PIPE, stdout=subprocess.PIPE) p.wait() filename = p.stdout.readline().decode('utf-8').strip() err = p.stderr.read().decode('utf-8') p.stderr.close() p.stdout.close() pattern = "decremented refcount of file %s" % filename assert pattern in err assert "leaked" not in err pattern = "KeyError: '%s'" % filename assert pattern not in err finally: executor.shutdown()
def test_child_retrieves_resource_tracker(self): parent_rtracker_pid = get_rtracker_pid() executor = ProcessPoolExecutor(max_workers=2) child_rtracker_pid = executor.submit(get_rtracker_pid).result() # First simple pid retrieval check (see #200) assert child_rtracker_pid == parent_rtracker_pid # Register a resource in the parent process, and un-register it in the # child process. If the two processes do not share the same # resource_tracker, a cache KeyError should be printed in stderr. import subprocess folder_name = 'loky_tempfolder' cmd = '''if 1: import os, sys from loky import ProcessPoolExecutor from loky.backend import resource_tracker from loky.backend.semlock import SemLock resource_tracker.VERBOSE=True folder_name = "{}" # We don't need to create the semaphore as registering / unregistering # operations simply add / remove entries from a cache, but do not # manipulate the actual semaphores. resource_tracker.register(folder_name, "folder") def unregister(name, rtype): # resource_tracker.unregister is actually a bound method of the # ResourceTracker. We need a custom wrapper to avoid object # serialization. from loky.backend import resource_tracker resource_tracker.unregister(folder_name, rtype) e = ProcessPoolExecutor(1) e.submit(unregister, folder_name, "folder").result() e.shutdown() ''' try: p = subprocess.Popen( [sys.executable, '-E', '-c', cmd.format(folder_name)], stderr=subprocess.PIPE) p.wait() err = p.stderr.read().decode('utf-8') p.stderr.close() assert re.search("unregister %s" % folder_name, err) is not None assert re.search("leaked", err) is None assert re.search("KeyError: '%s'" % folder_name, err) is None finally: executor.shutdown()
def scanner(self, queue, parent, npes, track, sensor, dtype, dt_tol, dL_tol): timeout = 900 self.logger.debug("Scanner, pid: {}".format(os.getpid())) filenames = bloom_filter(track, sensor, dtype, dt_tol, dL_tol) self.logger.debug("Finished bloom filter") with ProcessPoolExecutor(max_workers=npes, timeout=timeout) as executor: results = [] for f in filenames: self.logger.info("Scanning: {}".format(f)) if (len(results) >= npes) and parent.is_alive(): idx = [r.done() for r in results] while not np.any(idx): time.sleep(1) idx = [r.done() for r in results] tmp = results.pop(idx.index(True)).result() self.logger.debug("Finished reading another file") if not tmp.empty: self.logger.warning("Found {} matchs".format(len(tmp))) queue.put(tmp) self.logger.debug("Getting {}".format(f)) ds = self.db[f].compute() if not parent.is_alive(): return self.logger.debug("Submitting a new inrange process") results.append( executor.submit(matchup, track, ds, dL_tol, dt_tol)) for tmp in (r.result(timeout) for r in results): if not parent.is_alive(): return self.logger.debug("Finished reading another file") if not tmp.empty: self.logger.warning("Found {} matchs".format(len(tmp))) queue.put(tmp) self.logger.debug("Finished scanning all potential matchups.") queue.put("END")
def test_worker_timeout_mock(self): timeout = .001 context = get_context() executor = ProcessPoolExecutor( max_workers=4, context=context, timeout=timeout) result_queue = DelayedSimpleQueue(ctx=context, delay=.001) executor._result_queue = result_queue with pytest.warns(UserWarning, match=r'^A worker stopped while some jobs'): for i in range(5): # Trigger worker spawn for lazy executor implementations for result in executor.map(id, range(8)): pass executor.shutdown() result_queue.close()
def target(): def done(future: Future): try: q_out.put(future.result()) q_in.task_done() except BaseException: stop_event.set() raise finally: counter.release() # start worker executor = ProcessPoolExecutor(n_workers) counter = Semaphore(n_workers) wait = True try: for value in iter(q_in.get, SourceExhausted()): counter.acquire() executor.submit(transform, value, *args, **kwargs).add_done_callback(done) # wait for other processes q_in.task_done() q_in.join() q_out.put(SourceExhausted()) except StopEvent: pass except BaseException: wait = False stop_event.set() raise finally: executor.shutdown(wait=wait)
import multiprocessing as mp from loky import ProcessPoolExecutor # normal code res = map(greet, ['Sipho', 'Sindi']) # use worker pools pool = mp.Pool(2) res = pool.map(greet, ['Sipho', 'Sindi']) # use loky for cross platform consistancy executor = ProcessPoolExecutor(max_workers=2) res = executor.map(greet, ('Sipho', 'Sindi')) #TODO: Research joblib def run_in_parallel(): eight = len(ids_to_do) // 8 quater = len(ids_to_do) // 4 half = len(ids_to_do) // 2 args = [ ids_to_do[:eight], ids_to_do[eight:quater], ids_to_do[quater:(quater + eight)], ids_to_do[(quater + eight):half], ids_to_do[half:-(quater + eight)], ids_to_do[-(quater + eight):-quater], ids_to_do[-quater:-eight], ids_to_do[-eight:] ] pool = multiprocessing.Pool(processes=len(args)) pool.map(export_images2, args)
# # The asynchronous execution can be performed with threads, using ThreadPoolExecutor, or separate processes, using ProcessPoolExecutor. Both implement the same interface, which is defined by the abstract Executor class. # %% {"slideshow": {"slide_type": "slide"}} %%time import os if os.name == "nt": from loky import ProcessPoolExecutor # for Windows users else: from concurrent.futures import ProcessPoolExecutor def delayed_square(x): sleep(1) return x*x e = ProcessPoolExecutor() results = list(e.map(delayed_square, range(8))) # %% {"slideshow": {"slide_type": "slide"}} %%time from concurrent.futures import ThreadPoolExecutor e = ThreadPoolExecutor() results = list(e.map(delayed_square, range(8))) # %% [markdown] {"slideshow": {"slide_type": "slide"}} # # Asynchronous Future # While many parallel applications can be described as maps, some can be more complex. In this section we look at the asynchronous Future interface, which provides a simple API for ad-hoc parallelism. This is useful for when your computations don't fit a regular pattern. #
"""Deadlock with pickling errors ================================ This example highlights the fact that the ProcessPoolExecutor implementation from concurrent.futures is not robust to pickling error (at least in versions 3.6 and lower). """ import time from loky import ProcessPoolExecutor from loky.backend import get_context class SlowPickle: def __init__(self, delay=.1): self.delay = delay def __reduce__(self): time.sleep(self.delay) return SlowPickle, (self.delay,) if __name__ == "__main__": ctx = get_context("spawn") o = SlowPickle() with ProcessPoolExecutor(max_workers=2, timeout=.01, context=ctx) as e: f = e.submit(id, SlowPickle()) f.result()
from time import time, sleep if os.name == "nt": from loky import ProcessPoolExecutor # for Windows users else: from concurrent.futures import ProcessPoolExecutor from time import time, sleep def delayed_square(x): sleep(1) return x*x if __name__ == "__main__": start = time() data = list(range(8)) with ProcessPoolExecutor() as pool: result = sum(pool.map(delayed_square, data)) stop = time() print(f" result : {result} - elapsed time {stop - start}") # + {"slideshow": {"slide_type": "slide"}, "execution": {"iopub.status.busy": "2020-09-12T14:00:37.368389Z", "iopub.execute_input": "2020-09-12T14:00:37.369466Z", "iopub.status.idle": "2020-09-12T14:00:39.614789Z", "shell.execute_reply": "2020-09-12T14:00:39.615379Z"}} !{sys.executable} process_pool.py # + {"slideshow": {"slide_type": "slide"}, "execution": {"iopub.status.busy": "2020-09-12T14:00:39.620798Z", "iopub.execute_input": "2020-09-12T14:00:39.621801Z", "iopub.status.idle": "2020-09-12T14:00:40.633249Z", "shell.execute_reply": "2020-09-12T14:00:40.634198Z"}} %%time from concurrent.futures import ThreadPoolExecutor e = ThreadPoolExecutor() results = list(e.map(delayed_square, range(8)))