def _register_autoshutdown(self):
if sys.version_info < (3, 9):
atexit.register(self.shutdown)
# atexit registered callback is called after python stops all non-daemon threads
# Python 2.7, 3.6, 3.7, 3.8 - ThreadPoolExecutor creates daemon threads
# so, we can reach atexit callback
else:
# Python 3.9 - ThreadPoolExecutor creates non-daemon threads
# we use same "private" machinery of threading as concurrent.futures of Python 3.9
# Register for `_python_exit()` to be called just before joining all
# non-daemon threads. This is used instead of `atexit.register()` for
# compatibility with subinterpreters, which no longer support daemon threads.
# See bpo-39812 for context.
threading._register_atexit(self.shutdown)
def _start_executor_manager_thread(self):
if self._executor_manager_thread is None:
mp.util.debug('_start_executor_manager_thread called')
# Start the processes so that their sentinels are known.
self._executor_manager_thread = _ExecutorManagerThread(self)
self._executor_manager_thread.start()
# register this executor in a mechanism that ensures it will wakeup
# when the interpreter is exiting.
_threads_wakeups[self._executor_manager_thread] = \
(self._shutdown_lock,
self._executor_manager_thread_wakeup)
global process_pool_executor_at_exit
if process_pool_executor_at_exit is None:
# Ensure that the _python_exit function will be called before
# the multiprocessing.Queue._close finalizers which have an
# exitpriority of 10.
if sys.version_info < (3, 9):
process_pool_executor_at_exit = mp.util.Finalize(
None, _python_exit, exitpriority=20)
else:
process_pool_executor_at_exit = threading._register_atexit(
_python_exit)
def _start_executor_manager_thread(self):
if self._executor_manager_thread is None:
mp.util.debug('_start_executor_manager_thread called')
# When the executor gets garbarge collected, the weakref callback
# will wake up the queue management thread so that it can terminate
# if there is no pending work item.
def weakref_cb(_,
thread_wakeup=self._executor_manager_thread_wakeup,
shutdown_lock=self._shutdown_lock):
mp.util.debug('Executor collected: triggering callback for'
' QueueManager wakeup')
with self._shutdown_lock:
thread_wakeup.wakeup()
# Start the processes so that their sentinels are known.
self._executor_manager_thread = _ExecutorManagerThread(self)
self._executor_manager_thread.start()
# register this executor in a mechanism that ensures it will wakeup
# when the interpreter is exiting.
_threads_wakeups[self._executor_manager_thread] = \
(self._shutdown_lock,
self._executor_manager_thread_wakeup)
global process_pool_executor_at_exit
if process_pool_executor_at_exit is None:
# Ensure that the _python_exit function will be called before
# the multiprocessing.Queue._close finalizers which have an
# exitpriority of 10.
if sys.version_info < (3, 9):
process_pool_executor_at_exit = mp.util.Finalize(
None, _python_exit, exitpriority=20)
else:
process_pool_executor_at_exit = threading._register_atexit(
_python_exit)
def _python_exit():
global _shutdown
with _global_shutdown_lock:
_shutdown = True
items = list(_threads_queues.items())
for t, q in items:
q.put(None)
for t, q in items:
t.join()
# Register for `_python_exit()` to be called just before joining all
# non-daemon threads. This is used instead of `atexit.register()` for
# compatibility with subinterpreters, which no longer support daemon threads.
# See bpo-39812 for context.
threading._register_atexit(_python_exit)
class _WorkItem(object):
def __init__(self, future, fn, args, kwargs):
self.future = future
self.fn = fn
self.args = args
self.kwargs = kwargs
def run(self):
if not self.future.set_running_or_notify_cancel():
return
try:
result = self.fn(*self.args, **self.kwargs)
def update_event(self, inp=-1):
self.set_output_val(0, threading._register_atexit(self.input(0)))
pop = collections.deque.pop
THREADS_QUEUES = weakref.WeakKeyDictionary() # type: weakref.WeakKeyDictionary
def join_threads(threads_queues=THREADS_QUEUES):
items = list(threads_queues.items())
for _, queue in items: # pragma: no cover
queue.put(None)
for thread, _ in items: # pragma: no cover
thread.join()
try:
threading._register_atexit(join_threads) # type: ignore[attr-defined]
except AttributeError: # pragma: no cover
atexit.register(join_threads)
class Pool:
def __init__(self, executor, manager, *args):
self.size = None
self.event = threading.Event()
self.queue = queue = Queue()
self.exref = weakref.ref(executor, lambda _, q=queue: q.put(None))
args = (self,) + args
kwargs = executor._options
thread = threading.Thread(target=manager, args=args, kwargs=kwargs)
pop = collections.deque.pop
THREADS_QUEUES = weakref.WeakKeyDictionary()
def join_threads(threads_queues=THREADS_QUEUES):
items = list(threads_queues.items())
for _, queue in items: # pragma: no cover
queue.put(None)
for thread, _ in items: # pragma: no cover
thread.join()
try:
threading._register_atexit(join_threads)
except AttributeError: # pragma: no cover
atexit.register(join_threads)
class Pool(object):
def __init__(self, executor, manager, *args):
self.size = None
self.event = threading.Event()
self.queue = queue = Queue()
self.exref = weakref.ref(executor, lambda _, q=queue: q.put(None))
args = (self,) + args
kwargs = executor._options
thread = threading.Thread(target=manager, args=args, kwargs=kwargs)
