def submit(self, fn, *args, **kwargs): """ Sending the function to the execution queue :param fn: function being executed :type fn: callable :param args: function's positional arguments :param kwargs: function's keywords arguments :return: future instance :rtype: _base.Future Added keyword: - priority (integer later sys.maxsize) """ with self._shutdown_lock: if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') priority = kwargs.get('priority', random.randint(0, sys.maxsize-1)) if 'priority' in kwargs: del kwargs['priority'] f = _base.Future() w = _WorkItem(f, fn, args, kwargs) self._work_queue.put((priority, w)) self._adjust_thread_count() return f
def submit(fn, *args, **kwargs): '''Submit a function to be run on the executor (internal) Args: fn (callable): function to call args (tuple): args to pass to function kwargs (dict): kwargs to pass to function ''' if _EXECUTOR is None: raise RuntimeError('Already stopped!') self = _EXECUTOR with self._shutdown_lock: if hasattr(self, '_broken') and self._broken: raise BrokenThreadPool(self._broken) if hasattr(self, '_shutdown') and self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') if cft._shutdown: raise RuntimeError('cannot schedule new futures after' 'interpreter shutdown') f = _base.Future() w = _WorkItem(f, fn, args, kwargs) self._work_queue.put(w) self._adjust_thread_count() return f
def submit(self, fn, *args, **kwargs): """ This is an exact reimplementation of the `submit()` method on the parent class, except with an added `try/except` around `self._adjust_thread_count()`. So long as there is at least one living thread, this thread pool will not throw an exception if threads cannot be expanded to `max_workers`. In the implementation, we use "protected" attributes from concurrent.futures (`_base` and `_WorkItem`). Consider vendoring the whole concurrent.futures library as an alternative to these protected imports. https://github.com/agronholm/pythonfutures/blob/3.2.0/concurrent/futures/thread.py#L121-L131 # NOQA https://github.com/python/cpython/blob/v3.6.4/Lib/concurrent/futures/thread.py#L114-L124 """ with self._shutdown_lock: if self._shutdown: raise RuntimeError( 'cannot schedule new futures after shutdown') f = _base.Future() w = _WorkItem(f, fn, args, kwargs) self._work_queue.put(w) try: self._adjust_thread_count() except RuntimeError: # RuntimeError: can't start new thread # See https://github.com/conda/conda/issues/6624 if len(self._threads) > 0: # It's ok to not be able to start new threads if we already have at least # one thread alive. pass else: raise return f
def submit(self, fn, *args, **kwargs): """ This is an exact reimplementation of the `submit()` method on the parent class, except with an added `try/except` around `self._adjust_thread_count()`. So long as there is at least one living thread, this thread pool will not throw an exception if threads cannot be expanded to `max_workers`. In the implementation, we use "protected" attributes from concurrent.futures (`_base` and `_WorkItem`). Consider vendoring the whole concurrent.futures library as an alternative to these protected imports. https://github.com/agronholm/pythonfutures/blob/3.2.0/concurrent/futures/thread.py#L121-L131 # NOQA https://github.com/python/cpython/blob/v3.6.4/Lib/concurrent/futures/thread.py#L114-L124 """ with self._shutdown_lock: if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') f = _base.Future() w = _WorkItem(f, fn, args, kwargs) self._work_queue.put(w) try: self._adjust_thread_count() except RuntimeError: # RuntimeError: can't start new thread # See https://github.com/conda/conda/issues/6624 if len(self._threads) > 0: # It's ok to not be able to start new threads if we already have at least # one thread alive. pass else: raise return f
def submit(self, func, *args, **kwargs): """Submit a function to the pool, `self.submit(function,arg1,arg2,arg3=3)`""" with self._shutdown_lock: if PY3 and self._broken: raise BrokenProcessPool( "A child process terminated " "abruptly, the process pool is not usable anymore") if self._shutdown_thread: raise RuntimeError( "cannot schedule new futures after shutdown") callback = kwargs.pop("callback", self.default_callback) future = NewFuture( self._timeout, args, kwargs, callback=callback, catch_exception=self.catch_exception, ) w = _WorkItem(future, func, args, kwargs) self._pending_work_items[self._queue_count] = w self._work_ids.put(self._queue_count) self._queue_count += 1 self._result_queue.put(None) self._start_queue_management_thread() if PY2: self._adjust_process_count() self._all_futures.add(future) return future
def submitf(self, dependences, f, fn, *args, **kwargs): """ submit a task, handled by the given fresh Future dependences (set): set of dependences. The task will be scheduled only when no running task has a dependence intersection. f (Future): a newly created Future fn (callable): the function to call args, kwargs -> passed to fn returns (Future): f """ # logging.debug("queuing action %s with future %s", fn, f.__class__.__name__) with self._shutdown_lock: if self._shutdown: raise RuntimeError( 'cannot schedule new futures after shutdown') # add to the queue and track the task self._queue.append(f) f.add_done_callback(self._on_done) w = _WorkItem(f, fn, args, kwargs) with self._set_remove: self._waiting_work.appendleft((w, f, dependences)) self._schedule_work() return f
def submit(self, fn, *args, **kwargs): with self._shutdown_lock: if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') f = NewFuture(self.timeout, self.timeout_return) w = _WorkItem(f, fn, args, kwargs) self._work_queue.put(w) self._adjust_thread_count() return f
def submit(self, fn, *args, **kwargs): with self._shutdown_lock: if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') f = Future() fn_deco = _deco(fn) w = _WorkItem(f, fn_deco, args, kwargs) self._work_queue.put(w) self._adjust_thread_count() return f
def submit(self, fn, *args, **kwargs): with self._shutdown_lock: if self._shutdown: raise RuntimeError( 'cannot schedule new futures after shutdown') future = Future() work_item = _WorkItem(future, self._monitored_fn, (fn, ) + args, kwargs) # Blocks until item is put in queue. self._queue.put(work_item) return future
def submit(self, runner, *args, **kw): with self._shutdown_lock: if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') w = _WorkItem(runner.future, runner.task, args, kw) def on_done(f): self._work_queue.task_done() runner.future.add_done_callback(on_done) self._work_queue.put(w) self._adjust_thread_count() return runner.future
def submit(self, fn, *args, **kwargs): if self._shutdown_lock.acquire(): if self._shutdown: self._shutdown_lock.release() raise RuntimeError('Cannot schedule new futures after shutdown') f = WSGIFuture(self.futures) w = _WorkItem(f, fn, args, kwargs) self._work_queue.put(w) self._adjust_thread_count() self._shutdown_lock.release() return f else: return False
def submit(self, fn, *args, **kwargs): """Submits a callable to be executed with the given arguments. Schedules the callable to be executed as fn(*args, **kwargs) and returns a ImplicitFuture instance representing the execution of the callable. Returns: An ImplicitFuture representing the given call. """ with self._shutdown_lock: if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') f = ImplicitFuture() w = _WorkItem(f, fn, args, kwargs) self._work_queue.put(w) self._adjust_thread_count() return f
def submit(fn, *args, **kwargs): if _EXECUTOR is None: raise RuntimeError('Already stopped!') self = _EXECUTOR with self._shutdown_lock: if self._broken: raise BrokenThreadPool(self._broken) if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') if cft._shutdown: raise RuntimeError('cannot schedule new futures after' 'interpreter shutdown') f = _base.Future() w = _WorkItem(f, fn, args, kwargs) self._work_queue.put(w) self._adjust_thread_count() return f
def submit(self, func, *args, **kwargs): """Submit a function to the pool, `self.submit(function,arg1,arg2,arg3=3)`""" with self._shutdown_lock: if self._shutdown: raise RuntimeError( "cannot schedule new futures after shutdown") callback = kwargs.pop("callback", self.default_callback) future = NewFuture( self._timeout, args, kwargs, callback=callback, catch_exception=self.catch_exception, ) w = _WorkItem(future, func, args, kwargs) self._work_queue.put(w) self._adjust_thread_count() self._all_futures.add(future) return future
def submitf(self, f, fn, *args, **kwargs): """ submit a task, handled by the given fresh Future f (Future): a newly created Future fn (callable): the function to call args, kwargs -> passed to fn returns (Future): f """ # logging.debug("queuing action %s with future %s", fn, f.__class__.__name__) with self._shutdown_lock: if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') w = _WorkItem(f, fn, args, kwargs) self._work_queue.put(w) self._adjust_thread_count() # add to the queue and track the task self._queue.append(f) f.add_done_callback(self._on_done) return f
def submitf(self, dependences, f, fn, *args, **kwargs): """ submit a task, handled by the given fresh Future dependences (set): set of dependences. The task will be scheduled only when no running task has a dependence intersection. f (Future): a newly created Future fn (callable): the function to call args, kwargs -> passed to fn returns (Future): f """ # logging.debug("queuing action %s with future %s", fn, f.__class__.__name__) with self._shutdown_lock: if self._shutdown: raise RuntimeError('cannot schedule new futures after shutdown') # add to the queue and track the task self._queue.append(f) f.add_done_callback(self._on_done) w = _WorkItem(f, fn, args, kwargs) with self._set_remove: self._waiting_work.appendleft((w, f, dependences)) self._schedule_work() return f
def run(self, callable, *args, **kwargs): """Call the callable and stores the result in this future.""" if self.done(): raise Exception('run() called multiple times on the same future.') _WorkItem(self, callable, args, kwargs).run()
import sys import queue import random import atexit import weakref import threading from concurrent.futures.thread import ThreadPoolExecutor, _base, _WorkItem, _python_exit, _threads_queues ######################################################################################################################## # Global variables # ######################################################################################################################## NULL_ENTRY = (sys.maxsize, _WorkItem(None, None, (), {})) _shutdown = False ######################################################################################################################## # Before system exit procedure # ######################################################################################################################## def python_exit(): """ Cleanup before system exit """ global _shutdown _shutdown = True items = list(_threads_queues.items()) for t, q in items: q.put(NULL_ENTRY)