def wrapper(*args, **kwargs):
future = Future()
callback, args, kwargs = replacer.replace(
lambda value=_NO_RESULT: future_set_result_unless_cancelled(future, value),
args, kwargs)
def handle_error(typ, value, tb):
future_set_exc_info(future, (typ, value, tb))
return True
exc_info = None
esc = ExceptionStackContext(handle_error, delay_warning=True)
with esc:
if not warn:
# HACK: In non-deprecated mode (only used in auth.py),
# suppress the warning entirely. Since this is added
# in a 5.1 patch release and already removed in 6.0
# I'm prioritizing a minimial change instead of a
# clean solution.
esc.delay_warning = False
try:
result = f(*args, **kwargs)
if result is not None:
raise ReturnValueIgnoredError(
"@return_future should not be used with functions "
"that return values")
except:
exc_info = sys.exc_info()
raise
if exc_info is not None:
# If the initial synchronous part of f() raised an exception,
# go ahead and raise it to the caller directly without waiting
# for them to inspect the Future.
future.result()
# If the caller passed in a callback, schedule it to be called
# when the future resolves. It is important that this happens
# just before we return the future, or else we risk confusing
# stack contexts with multiple exceptions (one here with the
# immediate exception, and again when the future resolves and
# the callback triggers its exception by calling future.result()).
if callback is not None:
warnings.warn("callback arguments are deprecated, use the returned Future instead",
DeprecationWarning)
def run_callback(future):
result = future.result()
if result is _NO_RESULT:
callback()
else:
callback(future.result())
future_add_done_callback(future, wrap(run_callback))
return future
def test_add_future_stack_context(self):
ready = threading.Event()
def task():
# we must wait for the ioloop callback to be scheduled before
# the task completes to ensure that add_future adds the callback
# asynchronously (which is the scenario in which capturing
# the stack_context matters)
ready.wait(1)
assert ready.isSet(), "timed out"
raise Exception("worker")
def callback(future):
self.future = future
raise Exception("callback")
def handle_exception(typ, value, traceback):
self.exception = value
self.stop()
return True
# stack_context propagates to the ioloop callback, but the worker
# task just has its exceptions caught and saved in the Future.
with futures.ThreadPoolExecutor(1) as pool:
with ExceptionStackContext(handle_exception):
self.io_loop.add_future(pool.submit(task), callback)
ready.set()
self.wait()
self.assertEqual(self.exception.args[0], "callback")
self.assertEqual(self.future.exception().args[0], "worker")
def run(self, result=None):
with ExceptionStackContext(self._handle_exception):
super(AsyncTestCase, self).run(result)
# As a last resort, if an exception escaped super.run() and wasn't
# re-raised in tearDown, raise it here. This will cause the
# unittest run to fail messily, but that's better than silently
# ignoring an error.
self.__rethrow()
def responseCallback(response):
def handle_exc(*args):
return True
if response.error is not None:
with ExceptionStackContext(handle_exc):
response.rethrow()
elif callback:
callback(eval(response._get_body()))
def CallWithRetryAsync(retry_policy, func, *args, **kwargs):
"""This is a higher-order function that wraps an arbitrary asynchronous
function (plus its arguments) in order to add retry functionality. If the
wrapped function completes with an error, then CallWithRetryAsync may call
it again. Pass a "retry_policy" argument that derives from RetryPolicy in
order to control the retry behavior. The retry policy determines whether
the function completed with a retry-able error, and then decides how many
times to retry, and how frequently to retry.
"""
# Validate presence of named "callback" argument.
inner_callback = kwargs.get('callback', None)
assert 'callback' in kwargs, 'CallWithRetryAsync requires a named "callback" argument that is not None.'
retry_manager = retry_policy.CreateManager()
# Called when "func" is complete; checks whether to retry the call.
def _OnCompletedCall(*callback_args, **callback_kwargs):
"""Called when the operation has completed. Determine whether to retry,
based on the arguments to the callback.
"""
retry_func = functools.partial(func, *args, **kwargs)
if not retry_manager.MaybeRetryOnResult(retry_func, *callback_args, **callback_kwargs):
# If the async operation completes successfully, don't want to retry if continuation code raises an exception.
exception_context.check_retry = False
inner_callback(*callback_args, **callback_kwargs)
def _OnException(type, value, tb):
"""Called if the operation raises an exception. Determine whether to retry
or re-raise the exception, based on the exception details.
"""
if exception_context.check_retry:
retry_func = functools.partial(func, *args, **kwargs)
return retry_manager.MaybeRetryOnException(retry_func, type, value, tb)
# Replace the callback argument with a callback to _OnCompletedCall which will check for retry.
kwargs['callback'] = _OnCompletedCall
# Catch any exceptions in order to possibly retry in that case.
exception_context = ExceptionStackContext(_OnException)
exception_context.check_retry = True
with exception_context:
future = func(*args, **kwargs)
return future
def test_bad_host(self):
def handler(exc_typ, exc_val, exc_tb):
self.stop(exc_val)
return True # Halt propagation.
with ExceptionStackContext(handler):
self.resolver.resolve('an invalid domain', 80, callback=self.stop)
result = self.wait()
self.assertIsInstance(result, Exception)
def wrapper(*args, **kwargs):
future = TracebackFuture()
callback, args, kwargs = replacer.replace(
lambda value=_NO_RESULT: future.set_result(value), args, kwargs)
def handle_error(typ, value, tb):
future.set_exc_info((typ, value, tb))
return True
exc_info = None
with ExceptionStackContext(handle_error):
try:
result = f(*args, **kwargs)
if result is not None:
raise ReturnValueIgnoredError(
"@return_future should not be used with functions "
"that return values")
except:
exc_info = sys.exc_info()
raise
# with 语句抛出的异常不会直接继续抛出, 因为 handle_error 一直返回 True,
# ExceptionStackContext 类里的 __exit__ 函数处理异常时, 调用的是 handle_error
if exc_info is not None:
# 这里才真正处理异常
# If the initial synchronous part of f() raised an exception,
# go ahead and raise it to the caller directly without waiting
# for them to inspect the Future.
# 4.1 继续执行 future.result()
future.result()
# If the caller passed in a callback, schedule it to be called
# when the future resolves. It is important that this happens
# just before we return the future, or else we risk confusing
# stack contexts with multiple exceptions (one here with the
# immediate exception, and again when the future resolves and
# the callback triggers its exception by calling future.result()).
"""
如果 caller 传递进一个 callback, 另它在 future 解析时被调用.
在我们返回 future 之前, 这是非常重要, 否则我们冒险地将多个 exception
混淆了 stack context(这里一个紧急的 exception, 而 future 解析时又一个, 且
callback 在调用 future.result() 时出发了它自己的 exception )
"""
if callback is not None:
def run_callback(future):
result = future.result()
if result is _NO_RESULT:
callback()
else:
callback(future.result())
future.add_done_callback(wrap(run_callback))
return future
def wrapper(*args, **kwargs):
future = Future()
if kwargs.get('callback') is not None:
future.add_done_callback(kwargs.pop('callback'))
kwargs['callback'] = future.set_result
def handle_error(typ, value, tb):
future.set_exception(value)
return True
with ExceptionStackContext(handle_error):
f(*args, **kwargs)
return future
def test_header_callback_stack_context(self):
exc_info = []
def error_handler(typ, value, tb):
exc_info.append((typ, value, tb))
return True
def header_callback(header_line):
if header_line.startswith('Content-Type:'):
1 / 0
with ExceptionStackContext(error_handler):
self.fetch('/chunk', header_callback=header_callback)
self.assertEqual(len(exc_info), 1)
self.assertIs(exc_info[0][0], ZeroDivisionError)
def wrapper(*args, **kwargs):
future = Future()
callback, args, kwargs = replacer.replace(
lambda value=_NO_RESULT: future_set_result_unless_cancelled(
future, value),
args,
kwargs)
def handle_error(typ, value, tb):
future_set_exc_info(future, (typ, value, tb))
return True
exc_info = None
with ExceptionStackContext(handle_error):
try:
result = f(*args, **kwargs)
if result is not None:
raise ReturnValueIgnoredError(
"@return_future should not be used with functions "
"that return values")
except:
exc_info = sys.exc_info()
raise
if exc_info is not None:
# If the initial synchronous part of f() raised an exception,
# go ahead and raise it to the caller directly without waiting
# for them to inspect the Future.
future.result()
# If the caller passed in a callback, schedule it to be called
# when the future resolves. It is important that this happens
# just before we return the future, or else we risk confusing
# stack contexts with multiple exceptions (one here with the
# immediate exception, and again when the future resolves and
# the callback triggers its exception by calling future.result()).
if callback is not None:
warnings.warn(
"callback arguments are deprecated, use the returned Future instead",
DeprecationWarning)
def run_callback(future):
result = future.result()
if result is _NO_RESULT:
callback()
else:
callback(future.result())
future_add_done_callback(future, wrap(run_callback))
return future
def wrapper(*args, **kwargs):
future = TracebackFuture()
callback, args, kwargs = replacer.replace(future, args, kwargs)
if callback is not None:
future.add_done_callback(
functools.partial(_auth_future_to_callback, callback))
def handle_exception(typ, value, tb):
if future.done():
return False
else:
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception):
f(*args, **kwargs)
return future
def test_prepare_curl_callback_stack_context(self):
exc_info = []
error_event = Event()
def error_handler(typ, value, tb):
exc_info.append((typ, value, tb))
error_event.set()
return True
with ExceptionStackContext(error_handler):
request = HTTPRequest(self.get_url('/custom_reason'),
prepare_curl_callback=lambda curl: 1 / 0)
yield [error_event.wait(), self.http_client.fetch(request)]
self.assertEqual(1, len(exc_info))
self.assertIs(exc_info[0][0], ZeroDivisionError)
def test_prepare_curl_callback_stack_context(self):
exc_info = []
def error_handler(typ, value, tb):
exc_info.append((typ, value, tb))
self.stop()
return True
with ExceptionStackContext(error_handler):
request = HTTPRequest(self.get_url('/'),
prepare_curl_callback=lambda curl: 1 / 0)
self.http_client.fetch(request, callback=self.stop)
self.wait()
self.assertEqual(1, len(exc_info))
self.assertIs(exc_info[0][0], ZeroDivisionError)
def connect(self, init_future, redis_tuple, active_trans, cmd_count):
"""
:param init_future: 第一个future对象
:param redis_tuple: (ip, port, db)
:param active_trans: 事务是否激活
:param cmd_count: 指令个数
"""
if self.__stream is not None:
return
#future, connect_count, transaction, cmd_count
self.__cmd_env.append((init_future, 1 + int(self.__haspass), active_trans, cmd_count))
with ExceptionStackContext(self.__handle_ex):
self.__stream = IOStream(socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0),
io_loop=self.__io_loop)
self.__stream.connect(redis_tuple[:2], self.__on_connect)
def wrapper(*args, **kwargs):
runner = None
def handle_exception(typ, value, tb):
# if the function throws an exception before its first "yield"
# (or is not a generator at all), the Runner won't exist yet.
# However, in that case we haven't reached anything asynchronous
# yet, so we can just let the exception propagate.
if runner is not None:
return runner.handle_exception(typ, value, tb)
return False
with ExceptionStackContext(handle_exception):
gen = func(*args, **kwargs)
if isinstance(gen, types.GeneratorType):
runner = Runner(gen)
runner.run()
return
assert gen is None, gen
def test_streaming_stack_context(self):
chunks = []
exc_info = []
def error_handler(typ, value, tb):
exc_info.append((typ, value, tb))
return True
def streaming_cb(chunk):
chunks.append(chunk)
if chunk == b('qwer'):
1 / 0
with ExceptionStackContext(error_handler):
self.fetch('/chunk', streaming_callback=streaming_cb)
self.assertEqual(chunks, [b('asdf'), b('qwer')])
self.assertEqual(1, len(exc_info))
self.assertIs(exc_info[0][0], ZeroDivisionError)
def wrapper(*args, **kwargs):
runner = None
future = TracebackFuture()
# 注册进去了......
if 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
def handle_exception(typ, value, tb):
try:
if runner is not None and runner.handle_exception(
typ, value, tb):
return True # 这样就吃掉exception了?
except Exception:
typ, value, tb = sys.exc_info()
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception) as deactivate:
try:
result = func(*args,
**kwargs) # result是func调用返回的一个结果，有可能是一个iterator?
except (Return, StopIteration) as e:
result = getattr(e, 'value', None) #TODO 这里也是
except Exception:
deactivate()
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
def final_callback(value):
deactivate()
future.set_result(value)
# 这里的result很可能是一个iterator，看看头部的调用例子
runner = Runner(result, final_callback) # 关键点
runner.run()
return future
deactivate()
future.set_result(result)
return future
def wrapper(*args, **kwargs):
runner = None
future = TracebackFuture()
if 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
def handle_exception(typ, value, tb):
try:
if runner is not None and runner.handle_exception(
typ, value, tb):
return True
except Exception:
typ, value, tb = sys.exc_info()
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception) as deactivate:
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
deactivate()
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
def final_callback(value):
deactivate()
future.set_result(value)
runner = Runner(result, final_callback)
runner.run()
return future
deactivate()
future.set_result(result)
return future
def wrapper(*args, **kwargs):
runner = None # 生成器的执行依赖于一个Runner。每一个engine在被调用的时候都会转换成一个Runner然后去执行。
def handle_exception(typ, value, tb): # 异常处理函数
# if the function throws an exception before its first "yield" (or is not a generator at all), the Runner won't exist yet.
# However, in that case we haven't reached anything asynchronous yet, so we can just let the exception propagate.
if runner is not None:
return runner.handle_exception(typ, value, tb)
return False
with ExceptionStackContext(handle_exception) as deactivate:
gen = func(*args,
**kwargs) # 调用func，得到gen。无论func是生成器函数，还是普通异步函数，这里都不会阻塞。
# 如果func是普通同步函数，则这里已经完成所有的工作了，只要func不返回非None的值，则该engine正常返回。
if isinstance(
gen, types.GeneratorType): # 如果gen是一个生成器，则构建一个Runner对象去运行它
runner = Runner(gen, deactivate)
runner.run() # 只要func中yield的Task包装的是异步方法，则该run()方法不会被阻塞
return
# 如果gen不是生成器（即func中没有yield），则它必须是None（即不func不能返回任何值），否则报错
# 且此时不用再做任何事情，因为func的调用已经结束了。
assert gen is None, gen
deactivate()
def wrapper(*args, **kwargs):
runner = None
def handle_exception(typ, value, tb):
# if the function throws an exception before its first "yield"
# (or is not a generator at all), the Runner won't exist yet.
# However, in that case we haven't reached anything asynchronous
# yet, so we can just let the exception propagate.
if runner is not None:
return runner.handle_exception(typ, value, tb)
return False
with ExceptionStackContext(handle_exception) as deactivate:
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
else:
if isinstance(result, types.GeneratorType):
def final_callback(value):
if value is not None:
raise ReturnValueIgnoredError(
"@gen.engine functions cannot return values: "
"%r" % (value, ))
assert value is None
deactivate()
runner = Runner(result, final_callback)
runner.run()
return
if result is not None:
raise ReturnValueIgnoredError(
"@gen.engine functions cannot return values: %r" %
(result, ))
deactivate()
def run(self, result=None):
with ExceptionStackContext(self._handle_exception):
super(AsyncTestCase, self).run(result)
# In case an exception escaped super.run or the StackContext caught
# an exception when there wasn't a wait() to re-raise it, do so here.
self.__rethrow()
def test_yield_outside_with_exception_stack_context(self):
cb = yield gen.Callback('k1')
with ExceptionStackContext(lambda t, v, tb: False):
self.io_loop.add_callback(cb)
yield gen.Wait('k1')
def f():
with ExceptionStackContext(lambda t, v, tb: False):
yield gen.Task(self.io_loop.add_callback)
def f():
try:
with ExceptionStackContext(lambda t, v, tb: False):
yield gen.Task(self.io_loop.add_callback)
except StackContextInconsistentError:
pass
