class RequestManager: """ Wrapper class around Executor for accessing request states and results """ def __init__(self, app=None, name=''): self._executor = Executor(app, name) self._ticket_dispenser = _TicketDispenser() def submit_ticketed(self, function, *args, **kwargs) -> str: """ Submits a function for execution and returns an id for later access to the requests state and result. :param function: the function :param args: the functions arguments :return: the id of the submitted execution request """ key = self._ticket_dispenser.get_ticket() self._executor.submit_stored(key, function, *args, **kwargs) return key def request_processed(self, request_id: str) -> bool: """ Checks if a request has already been processed. :param request_id: id of the request :return: True if the request was processed else False """ return self._executor.futures.done(request_id) def get_result(self, request_id: str): """ Returns the result of a finished request. :param request_id: id of the request :return: the result of the request """ return self._executor.futures.pop(request_id).result()
def test_submit_app_context(app): test_value = random.randint(1, 101) app.config['TEST_VALUE'] = test_value executor = Executor(app) with app.test_request_context(''): future = executor.submit(app_context_test_value) assert future.result() == test_value
def test_futures_as_completed(default_app): executor = Executor(default_app) with default_app.test_request_context(''): futures = {executor.submit(fib, x): x for x in range(30)} for future in concurrent.futures.as_completed(futures): x = futures[future] assert future.result() == fib(x)
def test_coerce_config_types(default_app): default_app.config['EXECUTOR_MAX_WORKERS'] = '5' default_app.config['EXECUTOR_FUTURES_MAX_LENGTH'] = '10' default_app.config['EXECUTOR_PROPAGATE_EXCEPTIONS'] = 'true' executor = Executor(default_app) with default_app.test_request_context(): future = executor.submit_stored('fibonacci', fib, 35)
def test_submit_request_context(app): test_value = random.randint(1, 101) executor = Executor(app) with app.test_request_context(''): request.test_value = test_value future = executor.submit(request_context_test_value) assert future.result() == test_value
def test_process_executor(): app = Flask(__name__) app.config['EXECUTOR_TYPE'] = 'process' executor = Executor(app) with app.app_context(): executor.submit(fib, 5) assert type(executor._executor) == concurrent.futures.ProcessPoolExecutor
def test_submit_result(): app = Flask(__name__) executor = Executor(app) with app.app_context(): future = executor.submit(fib, 5) assert type(future) == concurrent.futures.Future assert future.result() == fib(5)
def test_submit_g_context_process(default_app): test_value = random.randint(1, 101) executor = Executor(default_app) with default_app.test_request_context(''): g.test_value = test_value future = executor.submit(g_context_test_value) assert future.result() == test_value
def test_process_workers(app): app.config['EXECUTOR_TYPE'] = 'process' app.config['EXECUTOR_MAX_WORKERS'] = EXECUTOR_MAX_WORKERS executor = Executor(app) with app.app_context(): executor.submit(fib, 5) assert executor._executor._max_workers == EXECUTOR_MAX_WORKERS
def test_thread_workers(): app = Flask(__name__) app.config['EXECUTOR_TYPE'] = 'thread' app.config['EXECUTOR_MAX_WORKERS'] = EXECUTOR_MAX_WORKERS executor = Executor(app) with app.app_context(): executor.submit(fib, 5) assert executor._executor._max_workers == EXECUTOR_MAX_WORKERS
def test_executor_stored_future(default_app): executor = Executor(default_app) with default_app.test_request_context(): future = executor.submit_stored('fibonacci', fib, 35) assert executor.futures.done('fibonacci') is False assert future in executor.futures executor.futures.pop('fibonacci') assert future not in executor.futures
def test_map_g_context_process(default_app): test_value = random.randint(1, 101) iterator = list(range(5)) executor = Executor(default_app) with default_app.test_request_context(''): g.test_value = test_value results = executor.map(g_context_test_value, iterator) for r in results: assert r == test_value
def test_map_app_context(app): test_value = random.randint(1, 101) iterator = list(range(5)) app.config['TEST_VALUE'] = test_value executor = Executor(app) with app.test_request_context(''): results = executor.map(app_context_test_value, iterator) for r in results: assert r == test_value
def test_map_request_context(app): test_value = random.randint(1, 101) iterator = list(range(5)) executor = Executor(app) with app.test_request_context('/'): request.test_value = test_value results = executor.map(request_context_test_value, iterator) for r in results: assert r == test_value
def test_propagate_exception_callback(app): app.config['EXECUTOR_PROPAGATE_EXCEPTIONS'] = True executor = Executor(app) with pytest.raises(NameError): with app.test_request_context('/'): future = executor.submit(fail) concurrent.futures.wait([future]) assert propagate_exceptions_callback in future._done_callbacks propagate_exceptions_callback(future)
def test_default_done_callback(app): executor = Executor(app) def callback(future): setattr(future, 'test', 'test') executor.add_default_done_callback(callback) with app.test_request_context('/'): future = executor.submit(fib, 5) concurrent.futures.wait([future]) assert hasattr(future, 'test')
def test_propagate_exception_callback(app, caplog): caplog.set_level(logging.ERROR) app.config['EXECUTOR_PROPAGATE_EXCEPTIONS'] = True executor = Executor(app) with pytest.raises(NameError): with app.test_request_context('/'): future = executor.submit(fail) assert propagate_exceptions_callback in future._done_callbacks concurrent.futures.wait([future]) future.result()
def test_teardown_appcontext_is_called(default_app): default_app.config['EXECUTOR_MAX_WORKERS'] = 1 default_app.config['EXECUTOR_PUSH_APP_CONTEXT'] = True default_app.teardown_appcontext(clear_thread_local) executor = Executor(default_app) with default_app.test_request_context(): futures = [executor.submit(set_thread_local) for _ in range(2)] concurrent.futures.wait(futures) [propagate_exceptions_callback(future) for future in futures]
def test_pre_init_executor(default_app): executor = Executor() @executor.job def decorated(n): return fib(n) assert executor executor.init_app(default_app) with default_app.test_request_context(''): future = decorated.submit(5) assert future.result() == fib(5)
def test_named_executor(default_app): name = 'custom' EXECUTOR_MAX_WORKERS = 5 CUSTOM_EXECUTOR_MAX_WORKERS = 10 default_app.config['EXECUTOR_MAX_WORKERS'] = EXECUTOR_MAX_WORKERS default_app.config['CUSTOM_EXECUTOR_MAX_WORKERS'] = CUSTOM_EXECUTOR_MAX_WORKERS executor = Executor(default_app) custom_executor = Executor(default_app, name=name) assert 'executor' in default_app.extensions assert name + 'executor' in default_app.extensions assert executor._executor._max_workers == EXECUTOR_MAX_WORKERS assert custom_executor._executor._max_workers == CUSTOM_EXECUTOR_MAX_WORKERS
def test_future_proxy(default_app): executor = Executor(default_app) with default_app.test_request_context(''): future = executor.submit(pow, 2, 4) # Test if we're returning a subclass of Future assert isinstance(future, concurrent.futures.Future) assert isinstance(future, FutureProxy) concurrent.futures.wait([future]) # test standard Future methods and attributes assert future._state == concurrent.futures._base.FINISHED assert future.done() assert future.exception(timeout=0) is None
def test_teardown_appcontext_is_not_called(default_app): default_app.config['EXECUTOR_MAX_WORKERS'] = 1 default_app.config['EXECUTOR_PUSH_APP_CONTEXT'] = False default_app.teardown_appcontext(clear_thread_local) executor = Executor(default_app) with pytest.raises(ValueError): with default_app.test_request_context(): for i in range(2): future = executor.submit(set_thread_local) concurrent.futures.wait([future]) propagate_exceptions_callback(future)
def get_executor(): # Lazy instatiantion of the executor global executor global executor_lock with executor_lock: if not executor: executor = Executor(current_app) return executor
def buildpublish_start(): global executor if executor == None: executor = Executor(current_app) # if build already in progress, dont start another one if executor.futures._state('buildpublish') == 'RUNNING': return jsonify(okay=False, error='Build already in progress') # get server to publish to server = request.form.get('server') or None if server == None: return jsonify(okay=False, error='No server specified to publish to') # start a build-publish executor.submit_stored('buildpublish', buildpublish, server) return jsonify(okay=True)
def test_named_executor_name(default_app): name = 'invalid name' try: executor = Executor(default_app, name=name) except ValueError: assert True else: assert False
def test_invalid_executor_init(default_app): default_app.config['EXECUTOR_TYPE'] = 'invalid_value' try: executor = Executor(default_app) except ValueError: assert True else: assert False
def test_thread_decorator_submit(default_app): default_app.config['EXECUTOR_TYPE'] = 'thread' executor = Executor(default_app) @executor.job def decorated(n): return fib(n) with default_app.test_request_context(''): future = decorated.submit(5) assert future.result() == fib(5)
def test_thread_decorator(app): app.config['EXECUTOR_TYPE'] = 'thread' executor = Executor(app) @executor.job def decorated(n): return fib(n) assert type(decorated) == ExecutorJob with app.app_context(): future = decorated.submit(5) assert type(future) == concurrent.futures.Future assert future.result() == fib(5)
def test_thread_decorator_map(default_app): iterable = list(range(5)) default_app.config['EXECUTOR_TYPE'] = 'thread' executor = Executor(default_app) @executor.job def decorated(n): return fib(n) with default_app.test_request_context(''): results = decorated.map(iterable) for i, r in zip(iterable, results): assert fib(i) == r
def test_add_done_callback(default_app): """Exceptions thrown in callbacks can't be easily caught and make it hard to test for callback failure. To combat this, a global variable is used to store the value of an exception and test for its existence. """ executor = Executor(default_app) global exception exception = None with default_app.test_request_context(''): future = executor.submit(time.sleep, 0.5) def callback(future): global exception try: executor.submit(time.sleep, 0) except RuntimeError as e: exception = e future.add_done_callback(callback) concurrent.futures.wait([future]) assert exception is None