def submit(self, fn, *args, **kwargs):
"""
Sending the function to the execution queue
:param fn: function being executed
:type fn: callable
:param args: function's positional arguments
:param kwargs: function's keywords arguments
:return: future instance
:rtype: _base.Future
Added keyword:
- priority (integer later sys.maxsize)
"""
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
priority = kwargs.get('priority', random.randint(0, sys.maxsize-1))
if 'priority' in kwargs:
del kwargs['priority']
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put((priority, w))
self._adjust_thread_count()
return f
def submit(fn, *args, **kwargs):
'''Submit a function to be run on the executor (internal)
Args:
fn (callable): function to call
args (tuple): args to pass to function
kwargs (dict): kwargs to pass to function
'''
if _EXECUTOR is None:
raise RuntimeError('Already stopped!')
self = _EXECUTOR
with self._shutdown_lock:
if hasattr(self, '_broken') and self._broken:
raise BrokenThreadPool(self._broken)
if hasattr(self, '_shutdown') and self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
if cft._shutdown:
raise RuntimeError('cannot schedule new futures after'
'interpreter shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(self, fn, *args, **kwargs):
"""
This is an exact reimplementation of the `submit()` method on the parent class, except
with an added `try/except` around `self._adjust_thread_count()`. So long as there is at
least one living thread, this thread pool will not throw an exception if threads cannot
be expanded to `max_workers`.
In the implementation, we use "protected" attributes from concurrent.futures (`_base`
and `_WorkItem`). Consider vendoring the whole concurrent.futures library
as an alternative to these protected imports.
https://github.com/agronholm/pythonfutures/blob/3.2.0/concurrent/futures/thread.py#L121-L131 # NOQA
https://github.com/python/cpython/blob/v3.6.4/Lib/concurrent/futures/thread.py#L114-L124
"""
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError(
'cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
try:
self._adjust_thread_count()
except RuntimeError:
# RuntimeError: can't start new thread
# See https://github.com/conda/conda/issues/6624
if len(self._threads) > 0:
# It's ok to not be able to start new threads if we already have at least
# one thread alive.
pass
else:
raise
return f
def submit(self, fn, *args, **kwargs):
"""
This is an exact reimplementation of the `submit()` method on the parent class, except
with an added `try/except` around `self._adjust_thread_count()`. So long as there is at
least one living thread, this thread pool will not throw an exception if threads cannot
be expanded to `max_workers`.
In the implementation, we use "protected" attributes from concurrent.futures (`_base`
and `_WorkItem`). Consider vendoring the whole concurrent.futures library
as an alternative to these protected imports.
https://github.com/agronholm/pythonfutures/blob/3.2.0/concurrent/futures/thread.py#L121-L131 # NOQA
https://github.com/python/cpython/blob/v3.6.4/Lib/concurrent/futures/thread.py#L114-L124
"""
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
try:
self._adjust_thread_count()
except RuntimeError:
# RuntimeError: can't start new thread
# See https://github.com/conda/conda/issues/6624
if len(self._threads) > 0:
# It's ok to not be able to start new threads if we already have at least
# one thread alive.
pass
else:
raise
return f
def submit(self, func, *args, **kwargs):
"""Submit a function to the pool, `self.submit(function,arg1,arg2,arg3=3)`"""
with self._shutdown_lock:
if PY3 and self._broken:
raise BrokenProcessPool(
"A child process terminated "
"abruptly, the process pool is not usable anymore")
if self._shutdown_thread:
raise RuntimeError(
"cannot schedule new futures after shutdown")
callback = kwargs.pop("callback", self.default_callback)
future = NewFuture(
self._timeout,
args,
kwargs,
callback=callback,
catch_exception=self.catch_exception,
)
w = _WorkItem(future, func, args, kwargs)
self._pending_work_items[self._queue_count] = w
self._work_ids.put(self._queue_count)
self._queue_count += 1
self._result_queue.put(None)
self._start_queue_management_thread()
if PY2:
self._adjust_process_count()
self._all_futures.add(future)
return future
def submitf(self, dependences, f, fn, *args, **kwargs):
"""
submit a task, handled by the given fresh Future
dependences (set): set of dependences. The task will be scheduled only
when no running task has a dependence intersection.
f (Future): a newly created Future
fn (callable): the function to call
args, kwargs -> passed to fn
returns (Future): f
"""
# logging.debug("queuing action %s with future %s", fn, f.__class__.__name__)
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError(
'cannot schedule new futures after shutdown')
# add to the queue and track the task
self._queue.append(f)
f.add_done_callback(self._on_done)
w = _WorkItem(f, fn, args, kwargs)
with self._set_remove:
self._waiting_work.appendleft((w, f, dependences))
self._schedule_work()
return f
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
f = NewFuture(self.timeout, self.timeout_return)
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
f = Future()
fn_deco = _deco(fn)
w = _WorkItem(f, fn_deco, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError(
'cannot schedule new futures after shutdown')
future = Future()
work_item = _WorkItem(future, self._monitored_fn, (fn, ) + args,
kwargs)
# Blocks until item is put in queue.
self._queue.put(work_item)
return future
def submit(self, runner, *args, **kw):
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
w = _WorkItem(runner.future, runner.task, args, kw)
def on_done(f):
self._work_queue.task_done()
runner.future.add_done_callback(on_done)
self._work_queue.put(w)
self._adjust_thread_count()
return runner.future
def submit(self, fn, *args, **kwargs):
if self._shutdown_lock.acquire():
if self._shutdown:
self._shutdown_lock.release()
raise RuntimeError('Cannot schedule new futures after shutdown')
f = WSGIFuture(self.futures)
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
self._shutdown_lock.release()
return f
else:
return False
def submit(self, fn, *args, **kwargs):
"""Submits a callable to be executed with the given arguments.
Schedules the callable to be executed as fn(*args, **kwargs) and returns
a ImplicitFuture instance representing the execution of the callable.
Returns:
An ImplicitFuture representing the given call.
"""
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
f = ImplicitFuture()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(fn, *args, **kwargs):
if _EXECUTOR is None:
raise RuntimeError('Already stopped!')
self = _EXECUTOR
with self._shutdown_lock:
if self._broken:
raise BrokenThreadPool(self._broken)
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
if cft._shutdown:
raise RuntimeError('cannot schedule new futures after'
'interpreter shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(self, func, *args, **kwargs):
"""Submit a function to the pool, `self.submit(function,arg1,arg2,arg3=3)`"""
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError(
"cannot schedule new futures after shutdown")
callback = kwargs.pop("callback", self.default_callback)
future = NewFuture(
self._timeout,
args,
kwargs,
callback=callback,
catch_exception=self.catch_exception,
)
w = _WorkItem(future, func, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
self._all_futures.add(future)
return future
def submitf(self, f, fn, *args, **kwargs):
"""
submit a task, handled by the given fresh Future
f (Future): a newly created Future
fn (callable): the function to call
args, kwargs -> passed to fn
returns (Future): f
"""
# logging.debug("queuing action %s with future %s", fn, f.__class__.__name__)
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
# add to the queue and track the task
self._queue.append(f)
f.add_done_callback(self._on_done)
return f
def submitf(self, f, fn, *args, **kwargs):
"""
submit a task, handled by the given fresh Future
f (Future): a newly created Future
fn (callable): the function to call
args, kwargs -> passed to fn
returns (Future): f
"""
# logging.debug("queuing action %s with future %s", fn, f.__class__.__name__)
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
# add to the queue and track the task
self._queue.append(f)
f.add_done_callback(self._on_done)
return f
def submitf(self, dependences, f, fn, *args, **kwargs):
"""
submit a task, handled by the given fresh Future
dependences (set): set of dependences. The task will be scheduled only
when no running task has a dependence intersection.
f (Future): a newly created Future
fn (callable): the function to call
args, kwargs -> passed to fn
returns (Future): f
"""
# logging.debug("queuing action %s with future %s", fn, f.__class__.__name__)
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
# add to the queue and track the task
self._queue.append(f)
f.add_done_callback(self._on_done)
w = _WorkItem(f, fn, args, kwargs)
with self._set_remove:
self._waiting_work.appendleft((w, f, dependences))
self._schedule_work()
return f
def run(self, callable, *args, **kwargs):
"""Call the callable and stores the result in this future."""
if self.done():
raise Exception('run() called multiple times on the same future.')
_WorkItem(self, callable, args, kwargs).run()
import sys
import queue
import random
import atexit
import weakref
import threading
from concurrent.futures.thread import ThreadPoolExecutor, _base, _WorkItem, _python_exit, _threads_queues
########################################################################################################################
# Global variables #
########################################################################################################################
NULL_ENTRY = (sys.maxsize, _WorkItem(None, None, (), {}))
_shutdown = False
########################################################################################################################
# Before system exit procedure #
########################################################################################################################
def python_exit():
"""
Cleanup before system exit
"""
global _shutdown
_shutdown = True
items = list(_threads_queues.items())
for t, q in items:
q.put(NULL_ENTRY)
