def _register_autoshutdown(self): if sys.version_info < (3, 9): atexit.register(self.shutdown) # atexit registered callback is called after python stops all non-daemon threads # Python 2.7, 3.6, 3.7, 3.8 - ThreadPoolExecutor creates daemon threads # so, we can reach atexit callback else: # Python 3.9 - ThreadPoolExecutor creates non-daemon threads # we use same "private" machinery of threading as concurrent.futures of Python 3.9 # Register for `_python_exit()` to be called just before joining all # non-daemon threads. This is used instead of `atexit.register()` for # compatibility with subinterpreters, which no longer support daemon threads. # See bpo-39812 for context. threading._register_atexit(self.shutdown)
def _start_executor_manager_thread(self): if self._executor_manager_thread is None: mp.util.debug('_start_executor_manager_thread called') # Start the processes so that their sentinels are known. self._executor_manager_thread = _ExecutorManagerThread(self) self._executor_manager_thread.start() # register this executor in a mechanism that ensures it will wakeup # when the interpreter is exiting. _threads_wakeups[self._executor_manager_thread] = \ (self._shutdown_lock, self._executor_manager_thread_wakeup) global process_pool_executor_at_exit if process_pool_executor_at_exit is None: # Ensure that the _python_exit function will be called before # the multiprocessing.Queue._close finalizers which have an # exitpriority of 10. if sys.version_info < (3, 9): process_pool_executor_at_exit = mp.util.Finalize( None, _python_exit, exitpriority=20) else: process_pool_executor_at_exit = threading._register_atexit( _python_exit)
def _start_executor_manager_thread(self): if self._executor_manager_thread is None: mp.util.debug('_start_executor_manager_thread called') # When the executor gets garbarge collected, the weakref callback # will wake up the queue management thread so that it can terminate # if there is no pending work item. def weakref_cb(_, thread_wakeup=self._executor_manager_thread_wakeup, shutdown_lock=self._shutdown_lock): mp.util.debug('Executor collected: triggering callback for' ' QueueManager wakeup') with self._shutdown_lock: thread_wakeup.wakeup() # Start the processes so that their sentinels are known. self._executor_manager_thread = _ExecutorManagerThread(self) self._executor_manager_thread.start() # register this executor in a mechanism that ensures it will wakeup # when the interpreter is exiting. _threads_wakeups[self._executor_manager_thread] = \ (self._shutdown_lock, self._executor_manager_thread_wakeup) global process_pool_executor_at_exit if process_pool_executor_at_exit is None: # Ensure that the _python_exit function will be called before # the multiprocessing.Queue._close finalizers which have an # exitpriority of 10. if sys.version_info < (3, 9): process_pool_executor_at_exit = mp.util.Finalize( None, _python_exit, exitpriority=20) else: process_pool_executor_at_exit = threading._register_atexit( _python_exit)
def _python_exit(): global _shutdown with _global_shutdown_lock: _shutdown = True items = list(_threads_queues.items()) for t, q in items: q.put(None) for t, q in items: t.join() # Register for `_python_exit()` to be called just before joining all # non-daemon threads. This is used instead of `atexit.register()` for # compatibility with subinterpreters, which no longer support daemon threads. # See bpo-39812 for context. threading._register_atexit(_python_exit) class _WorkItem(object): def __init__(self, future, fn, args, kwargs): self.future = future self.fn = fn self.args = args self.kwargs = kwargs def run(self): if not self.future.set_running_or_notify_cancel(): return try: result = self.fn(*self.args, **self.kwargs)
def update_event(self, inp=-1): self.set_output_val(0, threading._register_atexit(self.input(0)))
pop = collections.deque.pop THREADS_QUEUES = weakref.WeakKeyDictionary() # type: weakref.WeakKeyDictionary def join_threads(threads_queues=THREADS_QUEUES): items = list(threads_queues.items()) for _, queue in items: # pragma: no cover queue.put(None) for thread, _ in items: # pragma: no cover thread.join() try: threading._register_atexit(join_threads) # type: ignore[attr-defined] except AttributeError: # pragma: no cover atexit.register(join_threads) class Pool: def __init__(self, executor, manager, *args): self.size = None self.event = threading.Event() self.queue = queue = Queue() self.exref = weakref.ref(executor, lambda _, q=queue: q.put(None)) args = (self,) + args kwargs = executor._options thread = threading.Thread(target=manager, args=args, kwargs=kwargs)
pop = collections.deque.pop THREADS_QUEUES = weakref.WeakKeyDictionary() def join_threads(threads_queues=THREADS_QUEUES): items = list(threads_queues.items()) for _, queue in items: # pragma: no cover queue.put(None) for thread, _ in items: # pragma: no cover thread.join() try: threading._register_atexit(join_threads) except AttributeError: # pragma: no cover atexit.register(join_threads) class Pool(object): def __init__(self, executor, manager, *args): self.size = None self.event = threading.Event() self.queue = queue = Queue() self.exref = weakref.ref(executor, lambda _, q=queue: q.put(None)) args = (self,) + args kwargs = executor._options thread = threading.Thread(target=manager, args=args, kwargs=kwargs)