def __init__(self, app=None, name=''):
self.app = app
self._executor = None
self.futures = FutureCollection()
if re.match(r'^(\w+)?$', name) is None:
raise ValueError(
"Executor names may only contain letters, numbers or underscores"
)
self.name = name
if len(name) > 0:
name = name.upper() + '_'
self.EXECUTOR_TYPE = name + 'EXECUTOR_TYPE'
self.EXECUTOR_MAX_WORKERS = name + 'EXECUTOR_MAX_WORKERS'
self.EXECUTOR_FUTURES_MAX_LENGTH = name + 'EXECUTOR_FUTURES_MAX_LENGTH'
if app is not None:
self.init_app(app)
def __init__(self, app=None, name=''):
self.app = app
self._default_done_callbacks = []
self.futures = FutureCollection()
if re.match(r'^(\w+)?$', name) is None:
raise ValueError(
"Executor names may only contain letters, numbers or underscores"
)
self.name = name
prefix = name.upper() + '_' if name else ''
self.EXECUTOR_TYPE = prefix + 'EXECUTOR_TYPE'
self.EXECUTOR_MAX_WORKERS = prefix + 'EXECUTOR_MAX_WORKERS'
self.EXECUTOR_FUTURES_MAX_LENGTH = prefix + 'EXECUTOR_FUTURES_MAX_LENGTH'
self.EXECUTOR_PROPAGATE_EXCEPTIONS = prefix + 'EXECUTOR_PROPAGATE_EXCEPTIONS'
if app is not None:
self.init_app(app)
def test_plain_future():
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
futures = FutureCollection()
future = executor.submit(fib, 33)
futures.add('fibonacci', future)
assert futures.done('fibonacci') is False
assert futures._state('fibonacci') is not None
assert future in futures
futures.pop('fibonacci')
assert future not in futures
def test_futures_max_length():
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
futures = FutureCollection(max_length=10)
future = executor.submit(pow, 2, 4)
futures.add(0, future)
assert future in futures
assert len(futures) == 1
for i in range(1, 11):
futures.add(i, executor.submit(pow, 2, 4))
assert len(futures) == 10
assert future not in futures
def test_duplicate_add_future():
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
futures = FutureCollection()
future = executor.submit(fib, 33)
futures.add('fibonacci', future)
try:
futures.add('fibonacci', future)
except ValueError:
assert True
else:
assert False
class Executor(InstanceProxy, concurrent.futures._base.Executor):
"""An executor interface for :py:mod:`concurrent.futures` designed for
working with Flask applications.
:param app: A Flask application instance.
:param name: An optional name for the executor. This can be used to
configure multiple executors. Named executors will look for
environment variables prefixed with the name in uppercase,
e.g. ``CUSTOM_EXECUTOR_TYPE``.
"""
def __init__(self, app=None, name=''):
self.app = app
self._default_done_callbacks = []
self.futures = FutureCollection()
if re.match(r'^(\w+)?$', name) is None:
raise ValueError(
"Executor names may only contain letters, numbers or underscores"
)
self.name = name
prefix = name.upper() + '_' if name else ''
self.EXECUTOR_TYPE = prefix + 'EXECUTOR_TYPE'
self.EXECUTOR_MAX_WORKERS = prefix + 'EXECUTOR_MAX_WORKERS'
self.EXECUTOR_FUTURES_MAX_LENGTH = prefix + 'EXECUTOR_FUTURES_MAX_LENGTH'
self.EXECUTOR_PROPAGATE_EXCEPTIONS = prefix + 'EXECUTOR_PROPAGATE_EXCEPTIONS'
self.EXECUTOR_PUSH_APP_CONTEXT = prefix + 'EXECUTOR_PUSH_APP_CONTEXT'
if app is not None:
self.init_app(app)
def init_app(self, app):
"""Initialise application. This will also intialise the configured
executor type:
* :class:`concurrent.futures.ThreadPoolExecutor`
* :class:`concurrent.futures.ProcessPoolExecutor`
"""
app.config.setdefault(self.EXECUTOR_TYPE, 'thread')
app.config.setdefault(self.EXECUTOR_PUSH_APP_CONTEXT, False)
futures_max_length = app.config.setdefault(
self.EXECUTOR_FUTURES_MAX_LENGTH, None)
propagate_exceptions = app.config.setdefault(
self.EXECUTOR_PROPAGATE_EXCEPTIONS, False)
if futures_max_length is not None:
self.futures.max_length = int(futures_max_length)
if str2bool(propagate_exceptions):
self.add_default_done_callback(propagate_exceptions_callback)
self._self = self._make_executor(app)
app.extensions[self.name + 'executor'] = self
def _make_executor(self, app):
executor_max_workers = app.config.setdefault(self.EXECUTOR_MAX_WORKERS,
None)
if executor_max_workers is not None:
executor_max_workers = int(executor_max_workers)
executor_type = app.config[self.EXECUTOR_TYPE]
if executor_type == 'thread':
_executor = concurrent.futures.ThreadPoolExecutor
elif executor_type == 'process':
_executor = concurrent.futures.ProcessPoolExecutor
else:
raise ValueError(
"{} is not a valid executor type.".format(executor_type))
return _executor(max_workers=executor_max_workers)
def _prepare_fn(self, fn, force_copy=False):
if isinstance(self._self, concurrent.futures.ThreadPoolExecutor) \
or force_copy:
fn = copy_current_request_context(fn)
if current_app.config[self.EXECUTOR_PUSH_APP_CONTEXT]:
fn = push_app_context(fn)
else:
fn = copy_current_app_context(fn)
return fn
def submit(self, fn, *args, **kwargs):
r"""Schedules the callable, fn, to be executed as fn(\*args \**kwargs)
and returns a :class:`~flask_executor.futures.FutureProxy` object, a
:class:`~concurrent.futures.Future` subclass representing
the execution of the callable.
See also :meth:`concurrent.futures.Executor.submit`.
Callables are wrapped a copy of the current application context and the
current request context. Code that depends on information or
configuration stored in :data:`flask.current_app`,
:data:`flask.request` or :data:`flask.g` can be run without
modification.
Note: Because callables only have access to *copies* of the application
or request contexts any changes made to these copies will not be
reflected in the original view. Further, changes in the original app or
request context that occur after the callable is submitted will not be
available to the callable.
Example::
future = executor.submit(pow, 323, 1235)
print(future.result())
:param fn: The callable to be executed.
:param \*args: A list of positional parameters used with
the callable.
:param \**kwargs: A dict of named parameters used with
the callable.
:rtype: flask_executor.FutureProxy
"""
fn = self._prepare_fn(fn)
future = self._self.submit(fn, *args, **kwargs)
for callback in self._default_done_callbacks:
future.add_done_callback(callback)
return FutureProxy(future, self)
def submit_stored(self, future_key, fn, *args, **kwargs):
r"""Submits the callable using :meth:`Executor.submit` and stores the
Future in the executor via a
:class:`~flask_executor.futures.FutureCollection` object available at
:data:`Executor.futures`. These futures can be retrieved anywhere
inside your application and queried for status or popped from the
collection. Due to memory concerns, the maximum length of the
FutureCollection is limited, and the oldest Futures will be dropped
when the limit is exceeded.
See :class:`flask_executor.futures.FutureCollection` for more
information on how to query futures in a collection.
Example::
@app.route('/start-task')
def start_task():
executor.submit_stored('calc_power', pow, 323, 1235)
return jsonify({'result':'success'})
@app.route('/get-result')
def get_result():
if not executor.futures.done('calc_power'):
future_status = executor.futures._state('calc_power')
return jsonify({'status': future_status})
future = executor.futures.pop('calc_power')
return jsonify({'status': done, 'result': future.result()})
:param future_key: Stores the Future for the submitted task inside the
executor's ``futures`` object with the specified
key.
:param fn: The callable to be executed.
:param \*args: A list of positional parameters used with
the callable.
:param \**kwargs: A dict of named parameters used with
the callable.
:rtype: concurrent.futures.Future
"""
future = self.submit(fn, *args, **kwargs)
self.futures.add(future_key, future)
return future
def map(self, fn, *iterables, **kwargs):
r"""Submits the callable, fn, and an iterable of arguments to the
executor and returns the results inside a generator.
See also :meth:`concurrent.futures.Executor.map`.
Callables are wrapped a copy of the current application context and the
current request context. Code that depends on information or
configuration stored in :data:`flask.current_app`,
:data:`flask.request` or :data:`flask.g` can be run without
modification.
Note: Because callables only have access to *copies* of the application
or request contexts
any changes made to these copies will not be reflected in the original
view. Further, changes in the original app or request context that
occur after the callable is submitted will not be available to the
callable.
:param fn: The callable to be executed.
:param \*iterables: An iterable of arguments the callable will apply to.
:param \**kwargs: A dict of named parameters to pass to the underlying
executor's :meth:`~concurrent.futures.Executor.map`
method.
"""
fn = self._prepare_fn(fn)
return self._self.map(fn, *iterables, **kwargs)
def job(self, fn):
"""Decorator. Use this to transform functions into `ExecutorJob`
instances that can submit themselves directly to the executor.
Example::
@executor.job
def fib(n):
if n <= 2:
return 1
else:
return fib(n-1) + fib(n-2)
future = fib.submit(5)
results = fib.map(range(1, 6))
"""
if isinstance(self._self, concurrent.futures.ProcessPoolExecutor):
raise TypeError(
"Can't decorate {}: Executors that use multiprocessing "
"don't support decorators".format(fn))
return ExecutorJob(executor=self, fn=fn)
def add_default_done_callback(self, fn):
"""Registers callable to be attached to all newly created futures. When a
callable is submitted to the executor,
:meth:`concurrent.futures.Future.add_done_callback` is called for every default
callable that has been set."
:param fn: The callable to be added to the list of default done callbacks for new
Futures.
"""
self._default_done_callbacks.append(fn)
def test_missing_future():
futures = FutureCollection()
assert futures.running('test') is None
