class RequestManager:
"""
Wrapper class around Executor for accessing request states and results
"""
def __init__(self, app=None, name=''):
self._executor = Executor(app, name)
self._ticket_dispenser = _TicketDispenser()
def submit_ticketed(self, function, *args, **kwargs) -> str:
"""
Submits a function for execution and returns an id for later access to
the requests state and result.
:param function: the function
:param args: the functions arguments
:return: the id of the submitted execution request
"""
key = self._ticket_dispenser.get_ticket()
self._executor.submit_stored(key, function, *args, **kwargs)
return key
def request_processed(self, request_id: str) -> bool:
"""
Checks if a request has already been processed.
:param request_id: id of the request
:return: True if the request was processed else False
"""
return self._executor.futures.done(request_id)
def get_result(self, request_id: str):
"""
Returns the result of a finished request.
:param request_id: id of the request
:return: the result of the request
"""
return self._executor.futures.pop(request_id).result()
def test_submit_app_context(app):
test_value = random.randint(1, 101)
app.config['TEST_VALUE'] = test_value
executor = Executor(app)
with app.test_request_context(''):
future = executor.submit(app_context_test_value)
assert future.result() == test_value
def test_futures_as_completed(default_app):
executor = Executor(default_app)
with default_app.test_request_context(''):
futures = {executor.submit(fib, x): x for x in range(30)}
for future in concurrent.futures.as_completed(futures):
x = futures[future]
assert future.result() == fib(x)
def test_coerce_config_types(default_app):
default_app.config['EXECUTOR_MAX_WORKERS'] = '5'
default_app.config['EXECUTOR_FUTURES_MAX_LENGTH'] = '10'
default_app.config['EXECUTOR_PROPAGATE_EXCEPTIONS'] = 'true'
executor = Executor(default_app)
with default_app.test_request_context():
future = executor.submit_stored('fibonacci', fib, 35)
def test_submit_request_context(app):
test_value = random.randint(1, 101)
executor = Executor(app)
with app.test_request_context(''):
request.test_value = test_value
future = executor.submit(request_context_test_value)
assert future.result() == test_value
def test_process_executor():
app = Flask(__name__)
app.config['EXECUTOR_TYPE'] = 'process'
executor = Executor(app)
with app.app_context():
executor.submit(fib, 5)
assert type(executor._executor) == concurrent.futures.ProcessPoolExecutor
def test_submit_result():
app = Flask(__name__)
executor = Executor(app)
with app.app_context():
future = executor.submit(fib, 5)
assert type(future) == concurrent.futures.Future
assert future.result() == fib(5)
def test_submit_g_context_process(default_app):
test_value = random.randint(1, 101)
executor = Executor(default_app)
with default_app.test_request_context(''):
g.test_value = test_value
future = executor.submit(g_context_test_value)
assert future.result() == test_value
def test_process_workers(app):
app.config['EXECUTOR_TYPE'] = 'process'
app.config['EXECUTOR_MAX_WORKERS'] = EXECUTOR_MAX_WORKERS
executor = Executor(app)
with app.app_context():
executor.submit(fib, 5)
assert executor._executor._max_workers == EXECUTOR_MAX_WORKERS
def test_thread_workers():
app = Flask(__name__)
app.config['EXECUTOR_TYPE'] = 'thread'
app.config['EXECUTOR_MAX_WORKERS'] = EXECUTOR_MAX_WORKERS
executor = Executor(app)
with app.app_context():
executor.submit(fib, 5)
assert executor._executor._max_workers == EXECUTOR_MAX_WORKERS
def test_executor_stored_future(default_app):
executor = Executor(default_app)
with default_app.test_request_context():
future = executor.submit_stored('fibonacci', fib, 35)
assert executor.futures.done('fibonacci') is False
assert future in executor.futures
executor.futures.pop('fibonacci')
assert future not in executor.futures
def test_map_g_context_process(default_app):
test_value = random.randint(1, 101)
iterator = list(range(5))
executor = Executor(default_app)
with default_app.test_request_context(''):
g.test_value = test_value
results = executor.map(g_context_test_value, iterator)
for r in results:
assert r == test_value
def test_map_app_context(app):
test_value = random.randint(1, 101)
iterator = list(range(5))
app.config['TEST_VALUE'] = test_value
executor = Executor(app)
with app.test_request_context(''):
results = executor.map(app_context_test_value, iterator)
for r in results:
assert r == test_value
def test_map_request_context(app):
test_value = random.randint(1, 101)
iterator = list(range(5))
executor = Executor(app)
with app.test_request_context('/'):
request.test_value = test_value
results = executor.map(request_context_test_value, iterator)
for r in results:
assert r == test_value
def test_propagate_exception_callback(app):
app.config['EXECUTOR_PROPAGATE_EXCEPTIONS'] = True
executor = Executor(app)
with pytest.raises(NameError):
with app.test_request_context('/'):
future = executor.submit(fail)
concurrent.futures.wait([future])
assert propagate_exceptions_callback in future._done_callbacks
propagate_exceptions_callback(future)
def test_default_done_callback(app):
executor = Executor(app)
def callback(future):
setattr(future, 'test', 'test')
executor.add_default_done_callback(callback)
with app.test_request_context('/'):
future = executor.submit(fib, 5)
concurrent.futures.wait([future])
assert hasattr(future, 'test')
def test_propagate_exception_callback(app, caplog):
caplog.set_level(logging.ERROR)
app.config['EXECUTOR_PROPAGATE_EXCEPTIONS'] = True
executor = Executor(app)
with pytest.raises(NameError):
with app.test_request_context('/'):
future = executor.submit(fail)
assert propagate_exceptions_callback in future._done_callbacks
concurrent.futures.wait([future])
future.result()
def test_teardown_appcontext_is_called(default_app):
default_app.config['EXECUTOR_MAX_WORKERS'] = 1
default_app.config['EXECUTOR_PUSH_APP_CONTEXT'] = True
default_app.teardown_appcontext(clear_thread_local)
executor = Executor(default_app)
with default_app.test_request_context():
futures = [executor.submit(set_thread_local) for _ in range(2)]
concurrent.futures.wait(futures)
[propagate_exceptions_callback(future) for future in futures]
def test_pre_init_executor(default_app):
executor = Executor()
@executor.job
def decorated(n):
return fib(n)
assert executor
executor.init_app(default_app)
with default_app.test_request_context(''):
future = decorated.submit(5)
assert future.result() == fib(5)
def test_named_executor(default_app):
name = 'custom'
EXECUTOR_MAX_WORKERS = 5
CUSTOM_EXECUTOR_MAX_WORKERS = 10
default_app.config['EXECUTOR_MAX_WORKERS'] = EXECUTOR_MAX_WORKERS
default_app.config['CUSTOM_EXECUTOR_MAX_WORKERS'] = CUSTOM_EXECUTOR_MAX_WORKERS
executor = Executor(default_app)
custom_executor = Executor(default_app, name=name)
assert 'executor' in default_app.extensions
assert name + 'executor' in default_app.extensions
assert executor._executor._max_workers == EXECUTOR_MAX_WORKERS
assert custom_executor._executor._max_workers == CUSTOM_EXECUTOR_MAX_WORKERS
def test_future_proxy(default_app):
executor = Executor(default_app)
with default_app.test_request_context(''):
future = executor.submit(pow, 2, 4)
# Test if we're returning a subclass of Future
assert isinstance(future, concurrent.futures.Future)
assert isinstance(future, FutureProxy)
concurrent.futures.wait([future])
# test standard Future methods and attributes
assert future._state == concurrent.futures._base.FINISHED
assert future.done()
assert future.exception(timeout=0) is None
def test_teardown_appcontext_is_not_called(default_app):
default_app.config['EXECUTOR_MAX_WORKERS'] = 1
default_app.config['EXECUTOR_PUSH_APP_CONTEXT'] = False
default_app.teardown_appcontext(clear_thread_local)
executor = Executor(default_app)
with pytest.raises(ValueError):
with default_app.test_request_context():
for i in range(2):
future = executor.submit(set_thread_local)
concurrent.futures.wait([future])
propagate_exceptions_callback(future)
def get_executor():
# Lazy instatiantion of the executor
global executor
global executor_lock
with executor_lock:
if not executor:
executor = Executor(current_app)
return executor
def buildpublish_start():
global executor
if executor == None:
executor = Executor(current_app)
# if build already in progress, dont start another one
if executor.futures._state('buildpublish') == 'RUNNING':
return jsonify(okay=False, error='Build already in progress')
# get server to publish to
server = request.form.get('server') or None
if server == None:
return jsonify(okay=False, error='No server specified to publish to')
# start a build-publish
executor.submit_stored('buildpublish', buildpublish, server)
return jsonify(okay=True)
def test_named_executor_name(default_app):
name = 'invalid name'
try:
executor = Executor(default_app, name=name)
except ValueError:
assert True
else:
assert False
def test_invalid_executor_init(default_app):
default_app.config['EXECUTOR_TYPE'] = 'invalid_value'
try:
executor = Executor(default_app)
except ValueError:
assert True
else:
assert False
def test_thread_decorator_submit(default_app):
default_app.config['EXECUTOR_TYPE'] = 'thread'
executor = Executor(default_app)
@executor.job
def decorated(n):
return fib(n)
with default_app.test_request_context(''):
future = decorated.submit(5)
assert future.result() == fib(5)
def test_thread_decorator(app):
app.config['EXECUTOR_TYPE'] = 'thread'
executor = Executor(app)
@executor.job
def decorated(n):
return fib(n)
assert type(decorated) == ExecutorJob
with app.app_context():
future = decorated.submit(5)
assert type(future) == concurrent.futures.Future
assert future.result() == fib(5)
def test_thread_decorator_map(default_app):
iterable = list(range(5))
default_app.config['EXECUTOR_TYPE'] = 'thread'
executor = Executor(default_app)
@executor.job
def decorated(n):
return fib(n)
with default_app.test_request_context(''):
results = decorated.map(iterable)
for i, r in zip(iterable, results):
assert fib(i) == r
def test_add_done_callback(default_app):
"""Exceptions thrown in callbacks can't be easily caught and make it hard
to test for callback failure. To combat this, a global variable is used to
store the value of an exception and test for its existence.
"""
executor = Executor(default_app)
global exception
exception = None
with default_app.test_request_context(''):
future = executor.submit(time.sleep, 0.5)
def callback(future):
global exception
try:
executor.submit(time.sleep, 0)
except RuntimeError as e:
exception = e
future.add_done_callback(callback)
concurrent.futures.wait([future])
assert exception is None
