class ThreadPoolExecutor(_base.Executor):
# Used to assign unique thread names when thread_name_prefix is not supplied.
_counter = itertools.count().__next__
def __init__(self, max_workers=None, thread_name_prefix='',
initializer=None, initargs=()):
"""Initializes a new ThreadPoolExecutor instance.
Args:
max_workers: The maximum number of threads that can be used to
execute the given calls.
thread_name_prefix: An optional name prefix to give our threads.
initializer: A callable used to initialize worker threads.
initargs: A tuple of arguments to pass to the initializer.
"""
if max_workers is None:
# ThreadPoolExecutor is often used to:
# * CPU bound task which releases GIL
# * I/O bound task (which releases GIL, of course)
#
# We use cpu_count + 4 for both types of tasks.
# But we limit it to 32 to avoid consuming surprisingly large resource
# on many core machine.
max_workers = min(32, (os.cpu_count() or 1) + 4)
if max_workers <= 0:
raise ValueError("max_workers must be greater than 0")
if initializer is not None and not callable(initializer):
raise TypeError("initializer must be a callable")
self._max_workers = max_workers
self._work_queue = queue.SimpleQueue()
self._idle_semaphore = threading.Semaphore(0)
self._threads = set()
self._broken = False
self._shutdown = False
self._shutdown_lock = threading.Lock()
self._thread_name_prefix = (thread_name_prefix or
("ThreadPoolExecutor-%d" % self._counter()))
self._initializer = initializer
self._initargs = initargs
def submit(self, fn, /, *args, **kwargs):
with self._shutdown_lock, _global_shutdown_lock:
if self._broken:
raise BrokenThreadPool(self._broken)
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
if _shutdown:
raise RuntimeError('cannot schedule new futures after '
'interpreter shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(self, fn, *args, **kwargs):
"""Attempts to submit the work item.
A runtime error is raised if the pool has been shutdown.
"""
future = _base.Future()
work_item = _WorkItem(future, fn, args, kwargs)
try:
# Keep trying to get an idle worker from the queue until we find one
# that accepts the work.
while not self._idle_worker_queue.get(
block=False).accepted_work(work_item):
pass
return future
except queue.Empty:
with self._lock:
if self._shutdown:
raise RuntimeError(
'Cannot schedule new tasks after thread pool '
'has been shutdown.')
worker = _Worker(self._idle_worker_queue,
self._permitted_thread_age_in_seconds,
work_item)
worker.daemon = True
worker.start()
self._workers.add(worker)
return future
def __init__(self, fn, sensor_id):
self.fn = fn
if isinstance(sensor_id, Sensor):
self.sensor_id = sensor_id.id
else:
self.sensor_id = sensor_id
self.future = _base.Future()
def submit(self, fn, *args, **kwargs):
"""Attempts to submit the work item.
A runtime error is raised if the pool has been shutdown.
"""
future = _base.Future()
work_item = _WorkItem(future, fn, args, kwargs)
with self._lock:
if self._shutdown:
raise RuntimeError(
'Cannot schedule new tasks after thread pool has been shutdown.'
)
try:
self._idle_worker_queue.get(block=False).assign_work(work_item)
# If we have more idle threads then the max allowed, shutdown a thread.
if self._idle_worker_queue.qsize() > self._max_idle_threads:
try:
self._idle_worker_queue.get(block=False).shutdown()
except queue.Empty:
pass
except queue.Empty:
worker = _Worker(self._idle_worker_queue, work_item)
worker.daemon = True
worker.start()
self._workers.add(worker)
return future
def submit(self, cmd, *args, **kwargs):
"""Submits a command to be executed as Popen(cmd.format(*args), **kwargs)
Schedules the command to delegate for `subprocess.Popen()` and return a
Future instance representing the execution of the callable.
there have some arguments different with Popen are:
- stdout, stderr always `PIPE`. (not allowed re-assign)
- shell be changed to True for default.
- universal_newlines be changed to True for default.
- position arguments need to use keyword arguments for propagate.
Returns:
A Future representing the given call.
"""
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError(
"cannot schedule new futures after shutdown.")
f = _base.Future()
w = _WorkItem(f, cmd.format(*args), kwargs)
self._work_queue.put(w)
self._start_management_thread()
return f
def submit(*args, **kwargs):
if len(args) >= 2:
self, fn, *args = args
elif not args:
raise TypeError(
"descriptor 'submit' of 'ThreadPoolExecutor' object "
"needs an argument")
elif 'fn' in kwargs:
fn = kwargs.pop('fn')
self, *args = args
else:
raise TypeError('submit expected at least 1 positional argument, '
'got %d' % (len(args) - 1))
with self._shutdown_lock:
if self._broken:
raise BrokenThreadPool(self._broken)
if self._shutdown:
raise RuntimeError(
'cannot schedule new futures after shutdown')
if _shutdown:
raise RuntimeError('cannot schedule new futures after '
'interpreter shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(self, fn, *args, **kwargs):
"""
This is an exact reimplementation of the `submit()` method on the parent class, except
with an added `try/except` around `self._adjust_thread_count()`. So long as there is at
least one living thread, this thread pool will not throw an exception if threads cannot
be expanded to `max_workers`.
In the implementation, we use "protected" attributes from concurrent.futures (`_base`
and `_WorkItem`). Consider vendoring the whole concurrent.futures library
as an alternative to these protected imports.
https://github.com/agronholm/pythonfutures/blob/3.2.0/concurrent/futures/thread.py#L121-L131 # NOQA
https://github.com/python/cpython/blob/v3.6.4/Lib/concurrent/futures/thread.py#L114-L124
"""
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError(
'cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
try:
self._adjust_thread_count()
except RuntimeError:
# RuntimeError: can't start new thread
# See https://github.com/conda/conda/issues/6624
if len(self._threads) > 0:
# It's ok to not be able to start new threads if we already have at least
# one thread alive.
pass
else:
raise
return f
def submit(fn, *args, **kwargs):
'''Submit a function to be run on the executor (internal)
Args:
fn (callable): function to call
args (tuple): args to pass to function
kwargs (dict): kwargs to pass to function
'''
if _EXECUTOR is None:
raise RuntimeError('Already stopped!')
self = _EXECUTOR
with self._shutdown_lock:
if hasattr(self, '_broken') and self._broken:
raise BrokenThreadPool(self._broken)
if hasattr(self, '_shutdown') and self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
if cft._shutdown:
raise RuntimeError('cannot schedule new futures after'
'interpreter shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(*args, **kwargs):
if len(args) >= 2:
self, fn, *args = args
elif not args:
raise TypeError(
"descriptor 'submit' of 'ProcessPoolExecutor' object "
"needs an argument")
elif 'fn' in kwargs:
fn = kwargs.pop('fn')
self, *args = args
else:
raise TypeError('submit expected at least 1 positional argument, '
'got %d' % (len(args) - 1))
with self._shutdown_lock:
if self._broken:
raise BrokenProcessPool(self._broken)
if self._shutdown_thread:
raise RuntimeError(
'cannot schedule new futures after shutdown')
if _global_shutdown:
raise RuntimeError('cannot schedule new futures after '
'interpreter shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._pending_work_items[self._queue_count] = w
self._work_ids.put(self._queue_count)
self._queue_count += 1
# Wake up queue management thread
self._queue_management_thread_wakeup.wakeup()
self._start_queue_management_thread()
return f
def submit(*args, **kwargs):
if len(args) >= 2:
self, fn, *args = args
elif not args:
raise TypeError("descriptor 'submit' of 'ErDummyThreadPool' object "
"needs an argument")
elif 'fn' in kwargs:
fn = kwargs.pop('fn')
self, *args = args
else:
raise TypeError('submit expected at least 1 positional argument, '
'got %d' % (len(args)-1))
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
if _shutdown:
raise RuntimeError('cannot schedule new futures after '
'interpreter shutdown')
self.increase_thread_count()
thread_name = f'{self._thread_name_prefix or self}_{self._num_threads}'
f = _base.Future()
w = _ErWorkItem(f, fn, args, kwargs)
t = threading.Thread(name=thread_name, target=w.run, kwargs={"on_join": self.decrease_thread_count})
t.start()
return f
def __init__(self, fn, sensor_id):
self.fn = fn
if isinstance(sensor_id, Device):
self.sensor_id = sensor_id.deviceId
else:
self.sensor_id = int(sensor_id)
self.future = _base.Future()
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError(
'cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(self, fn, *args, **kwargs):
with self._lock:
if not self._running:
raise RuntimeError('task pool is not running')
future = futurebase.Future()
task = Task(future, fn, args, kwargs)
self._queue.put(task)
return future
def post(self, message):
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError(
'cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, message)
self._work_queue.put(w)
self._initialize_actor()
return f
def __init__(self, fn, sensor_id):
"""
Initialize the WorkItem.
Args:
fn (func): The function to be called to do the actual work.
sensor_id (object): The sensor ID or Sensor object the work item is directed for.
"""
self.fn = fn
if isinstance(sensor_id, Sensor):
self.sensor_id = sensor_id.id
else:
self.sensor_id = int(sensor_id)
self.future = _base.Future()
def submit(self, fn, *args, **kwargs):
#submit函数会立刻返回
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError(
'cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
#work queue被所有线程共享, 里面放入 _WorkItem对象
#_WotkItem对象有一个 属性是Future的实例, 用它来储存返回结果, 并且该实例会被submmit返回
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
if self._shutdown_thread:
raise RuntimeError('cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._pending_work_items[self._queue_count] = w
self._work_ids.put(self._queue_count)
self._queue_count += 1
self._start_queue_management_thread()
self._adjust_process_count()
return f
def __init__(self, fn, device_id):
"""
Initialize the WorkItem.
Args:
fn (func): The function to be called to do the actual work.
device_id (object): The device ID or Device object the work item is directed for.
"""
self.fn = fn
if isinstance(device_id, Device):
self.device_id = device_id.id
else:
self.device_id = int(device_id)
self.future = _base.Future()
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
if self._broken:
raise BrokenProcessPool(
'A child process terminated abruptly, the process pool is not usable anymore'
)
if self._shutdown_thread:
raise RuntimeError(
'cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._pending_work_items[self._queue_count] = w
self._work_ids.put(self._queue_count)
self._result_queue.put(None)
self._start_queue_management_thread()
return f
def submit(self, fn, *args, **kwargs):
future = _base.Future()
work_item = WorkItem(future, fn, args, kwargs)
self.work_queue.put(work_item)
start_worker = False
with self.workers_busy_lock:
if not any([not worker.busy for worker in self.workers]):
logger.trace("Starting new worker in namespace %r because there are no free workers",
self.thread_name_prefix)
start_worker = True
if start_worker:
self._start_worker()
return future
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
if self._broken:
raise BrokenProcessPool(self._broken)
if self._shutdown_thread:
raise RuntimeError('cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._pending_work_items[self._queue_count] = w
self._work_ids.put(self._queue_count)
self._queue_count += 1
# Wake up queue management thread
self._result_queue.put(None)
self._start_queue_management_thread()
return f
def submit(self, fn, *args, **kwargs): # 每创建一个submit，都是会创建一个新的线程
with self._shutdown_lock: # 确实是每一个都上锁操作
if self._broken:
raise BrokenThreadPool(self._broken)
if self._shutdown:
raise RuntimeError(
'cannot schedule new futures after shutdown')
if _shutdown:
raise RuntimeError('cannot schedule new futures after '
'interpreter shutdown')
f = _base.Future() # 创建一个未来对象
w = _WorkItem(f, fn, args, kwargs) # 实际工作对象
# 所谓的submit其实就是用目标函数注册一个未来对象，用此未来对象申请一个工作调度对象，然后将此工作对象推到队列中就完事了
self._work_queue.put(w) # 每一次submit都会将实际工作对象推入队列中
self._adjust_thread_count()
return f # 返回注册函数申请的未来对象
def submit(fn, *args, **kwargs):
if _EXECUTOR is None:
raise RuntimeError('Already stopped!')
self = _EXECUTOR
with self._shutdown_lock:
if self._broken:
raise BrokenThreadPool(self._broken)
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
if cft._shutdown:
raise RuntimeError('cannot schedule new futures after'
'interpreter shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
self._adjust_thread_count()
return f
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
if self._broken:
raise BrokenProcessPool('A child process terminated '
'abruptly, the process pool is not usable anymore')
if self._shutdown_thread:
raise RuntimeError('cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._pending_work_items[self._queue_count] = w
self._work_ids.put(self._queue_count)
self._queue_count += 1
# Wake up queue management thread
self._result_queue.put(None)
# 这个None其实只是为了让workder thread的ready = wait([reader] + sentinels)
# 的reader有东西,可以往下运行,执行到下一次循环，调用_add_call_item_to_queue
# 不会做其它任何事
self._start_queue_management_thread()
return f
def post(self, message):
with self._shutdown_lock:
if self._broken:
raise process.BrokenProcessPool(
'A child process terminated '
'abruptly, the process pool is not usable anymore')
if self._shutdown_thread:
raise RuntimeError(
'cannot schedule new futures after shutdown')
f = _base.Future()
w = _WorkItem(f, message)
self._pending_work_items[self._queue_count] = w
self._work_ids.put(self._queue_count)
self._queue_count += 1
# Wake up queue management thread
self._result_queue.put(None)
self._start_queue_management_thread()
return f
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
if self._broken:
raise BrokenThreadPool(self._broken)
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
if _shutdown:
raise RuntimeError('cannot schedule new futures after '
'interpreter shutdown')
priority = kwargs.get('priority', random.randint(0, sys.maxsize-1))
if 'priority' in kwargs:
del kwargs['priority']
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put((priority, w), block=False)
self._adjust_thread_count()
return f
def submit(*args, **kwargs):
if len(args) >= 2:
self, fn, *args = args
elif not args:
raise TypeError(
"descriptor 'submit' of 'ThreadPoolExecutor' object "
"needs an argument")
elif 'fn' in kwargs:
fn = kwargs.pop('fn')
self, *args = args
import warnings
warnings.warn("Passing 'fn' as keyword argument is deprecated",
DeprecationWarning,
stacklevel=2)
else:
raise TypeError('submit expected at least 1 positional argument, '
'got %d' % (len(args) - 1))
with self._shutdown_lock, _global_shutdown_lock:
if self._broken:
raise thread.BrokenThreadPool(self._broken)
if self._shutdown:
raise RuntimeError(
'cannot schedule new futures after shutdown')
if _shutdown:
raise RuntimeError('cannot schedule new futures after '
'interpreter shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put(w)
with self._job_counter_lock:
self._job_counter += 1
self._adjust_thread_count()
return f
def submit(self, fn, *args, **kwargs):
with self._shutdown_lock:
if self._broken:
raise BrokenThreadPool(self._broken)
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
if _shutdown:
raise RuntimeError('cannot schedule new futures after '
'interpreter shutdown')
priority = kwargs.get('priority', random.randint(0, 1000000))
if priority == 0:
priority = random.randint(1, 100)
eplison = random.uniform(0,0.01) * priority
if 'priority' in kwargs:
del kwargs['priority']
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._work_queue.put((-float(priority + eplison), w), block=False)
self._adjust_thread_count()
return f
def _shutdown_trigger(self):
return base.Future()
class ProcessPoolExecutor(_base.Executor):
def __init__(self,
max_workers=None,
mp_context=None,
initializer=None,
initargs=()):
"""Initializes a new ProcessPoolExecutor instance.
Args:
max_workers: The maximum number of processes that can be used to
execute the given calls. If None or not given then as many
worker processes will be created as the machine has processors.
mp_context: A multiprocessing context to launch the workers. This
object should provide SimpleQueue, Queue and Process.
initializer: A callable used to initialize worker processes.
initargs: A tuple of arguments to pass to the initializer.
"""
_check_system_limits()
if max_workers is None:
self._max_workers = os.cpu_count() or 1
if sys.platform == 'win32':
self._max_workers = min(_MAX_WINDOWS_WORKERS,
self._max_workers)
else:
if max_workers <= 0:
raise ValueError("max_workers must be greater than 0")
elif (sys.platform == 'win32'
and max_workers > _MAX_WINDOWS_WORKERS):
raise ValueError(
f"max_workers must be <= {_MAX_WINDOWS_WORKERS}")
self._max_workers = max_workers
if mp_context is None:
mp_context = mp.get_context()
self._mp_context = mp_context
if initializer is not None and not callable(initializer):
raise TypeError("initializer must be a callable")
self._initializer = initializer
self._initargs = initargs
# Management thread
self._executor_manager_thread = None
# Map of pids to processes
self._processes = {}
# Shutdown is a two-step process.
self._shutdown_thread = False
self._shutdown_lock = threading.Lock()
self._idle_worker_semaphore = threading.Semaphore(0)
self._broken = False
self._queue_count = 0
self._pending_work_items = {}
self._cancel_pending_futures = False
# _ThreadWakeup is a communication channel used to interrupt the wait
# of the main loop of executor_manager_thread from another thread (e.g.
# when calling executor.submit or executor.shutdown). We do not use the
# _result_queue to send wakeup signals to the executor_manager_thread
# as it could result in a deadlock if a worker process dies with the
# _result_queue write lock still acquired.
#
# _shutdown_lock must be locked to access _ThreadWakeup.
self._executor_manager_thread_wakeup = _ThreadWakeup()
# Create communication channels for the executor
# Make the call queue slightly larger than the number of processes to
# prevent the worker processes from idling. But don't make it too big
# because futures in the call queue cannot be cancelled.
queue_size = self._max_workers + EXTRA_QUEUED_CALLS
self._call_queue = _SafeQueue(
max_size=queue_size,
ctx=self._mp_context,
pending_work_items=self._pending_work_items,
shutdown_lock=self._shutdown_lock,
thread_wakeup=self._executor_manager_thread_wakeup)
# Killed worker processes can produce spurious "broken pipe"
# tracebacks in the queue's own worker thread. But we detect killed
# processes anyway, so silence the tracebacks.
self._call_queue._ignore_epipe = True
self._result_queue = mp_context.SimpleQueue()
self._work_ids = queue.Queue()
def _start_executor_manager_thread(self):
if self._executor_manager_thread is None:
# Start the processes so that their sentinels are known.
self._executor_manager_thread = _ExecutorManagerThread(self)
self._executor_manager_thread.start()
_threads_wakeups[self._executor_manager_thread] = \
self._executor_manager_thread_wakeup
def _adjust_process_count(self):
# if there's an idle process, we don't need to spawn a new one.
if self._idle_worker_semaphore.acquire(blocking=False):
return
process_count = len(self._processes)
if process_count < self._max_workers:
p = self._mp_context.Process(
target=_process_worker,
args=(self._call_queue, self._result_queue, self._initializer,
self._initargs))
p.start()
self._processes[p.pid] = p
def submit(self, fn, /, *args, **kwargs):
with self._shutdown_lock:
if self._broken:
raise BrokenProcessPool(self._broken)
if self._shutdown_thread:
raise RuntimeError(
'cannot schedule new futures after shutdown')
if _global_shutdown:
raise RuntimeError('cannot schedule new futures after '
'interpreter shutdown')
f = _base.Future()
w = _WorkItem(f, fn, args, kwargs)
self._pending_work_items[self._queue_count] = w
self._work_ids.put(self._queue_count)
self._queue_count += 1
# Wake up queue management thread
self._executor_manager_thread_wakeup.wakeup()
self._adjust_process_count()
self._start_executor_manager_thread()
return f