def open(self): future = TracebackFuture() try: self._stream.open(lambda :future.set_result(None)) except Exception: future.set_exc_info(sys.exc_info()) return future
def wrapper(*args, **kwargs): runner = None future = TracebackFuture() if 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) def handle_exception(typ, value, tb): try: if runner is not None and runner.handle_exception(typ, value, tb): return True except Exception: typ, value, tb = sys.exc_info() future.set_exc_info((typ, value, tb)) return True with ExceptionStackContext(handle_exception): try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = getattr(e, 'value', None) except Exception: future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): def final_callback(value): future.set_result(value) runner = Runner(result, final_callback) runner.run() return future future.set_result(result) return future
def wrapper(*args, **kwargs): future = TracebackFuture() try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = getattr(e, 'value', None) except Exception: future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: orig_stack_contexts = stack_context._state.contexts yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = TracebackFuture() yielded.set_exception( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: future.set_result(getattr(e, 'value', None)) except Exception: future.set_exc_info(sys.exc_info()) else: # post runner into Cocaine ioloop CocaineIO.instance().post(Runner, result, future, yielded) return future future.set_result(result) return future
def coroutine_run(): """ 执行协程 """ try: result = consume_mq() except Exception: future = TracebackFuture() future.set_exc_info(sys.exc_info()) else: assert isinstance(result, Future) future = result def consumer_finish(f): """ 重新启动ioloop回调 :param f: """ if f.exc_info() is not None: logger.warn(f.exc_info()) elif f.exception() is not None: logger.warn(f.exception()) IOLoop.current().add_callback(coroutine_run) IOLoop.current().add_future(future, consumer_finish)
def open(self): try: future = self._stream.open() except: exc_info = sys.exc_info() future = TracebackFuture() future.set_exc_info(exc_info) return future
def wrapper(*args, **kwargs): future = TracebackFuture() if replace_callback and 'callback' in kwargs: # 调用的参数有callback,不实用future使用异步callback callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) try: # 调用实际的处理函数如果返回GeneratorType,那么进入Runner类循环执行直到终止 result = func(*args, **kwargs) except (Return, StopIteration) as e: result = _value_from_stopiteration(e) except Exception: future.set_exc_info(sys.exc_info()) return future else: # 生成器 if isinstance(result, GeneratorType): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: orig_stack_contexts = stack_context._state.contexts # 返回第一个Future,异步调用返回的Future, # 在异步调用成功后会将这个Future给set_done yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = TracebackFuture() yielded.set_exception( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: future.set_result(_value_from_stopiteration(e)) except Exception: future.set_exc_info(sys.exc_info()) else: Runner(result, future, yielded) try: # 进入Runner,返回 return future finally: # Subtle memory optimization: if next() raised an exception, # the future's exc_info contains a traceback which # includes this stack frame. This creates a cycle, # which will be collected at the next full GC but has # been shown to greatly increase memory usage of # benchmarks (relative to the refcount-based scheme # used in the absence of cycles). We can avoid the # cycle by clearing the local variable after we return it. future = None future.set_result(result) return future
def wrapper(*args, **kwargs): future = TracebackFuture() if replace_callback and "callback" in kwargs: callback = kwargs.pop("callback") IOLoop.current().add_future(future, lambda future: callback(future.result())) try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = _value_from_stopiteration(e) except Exception: future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, GeneratorType): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: orig_stack_contexts = stack_context._state.contexts yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = TracebackFuture() yielded.set_exception( stack_context.StackContextInconsistentError( "stack_context inconsistency (probably caused " 'by yield within a "with StackContext" block)' ) ) except (StopIteration, Return) as e: future.set_result(_value_from_stopiteration(e)) except Exception: future.set_exc_info(sys.exc_info()) else: _futures_to_runners[future] = Runner(result, future, yielded) try: return future finally: # Subtle memory optimization: if next() raised an exception, # the future's exc_info contains a traceback which # includes this stack frame. This creates a cycle, # which will be collected at the next full GC but has # been shown to greatly increase memory usage of # benchmarks (relative to the refcount-based scheme # used in the absence of cycles). We can avoid the # cycle by clearing the local variable after we return it. future = None future.set_result(result) return future
def _defer_to_future(defer): future = TracebackFuture() defer.addCallbacks( future.set_result, lambda failure: future.set_exc_info( (failure.type, failure.value, failure.tb))) return future
def wrapper(*args, **kwargs): runner = None future = TracebackFuture() # 注册进去了...... if 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) def handle_exception(typ, value, tb): try: if runner is not None and runner.handle_exception( typ, value, tb): return True # 这样就吃掉exception了? except Exception: typ, value, tb = sys.exc_info() future.set_exc_info((typ, value, tb)) return True with ExceptionStackContext(handle_exception) as deactivate: try: result = func(*args, **kwargs) # result是func调用返回的一个结果,有可能是一个iterator? except (Return, StopIteration) as e: result = getattr(e, 'value', None) #TODO 这里也是 except Exception: deactivate() future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): def final_callback(value): deactivate() future.set_result(value) # 这里的result很可能是一个iterator,看看头部的调用例子 runner = Runner(result, final_callback) # 关键点 runner.run() return future deactivate() future.set_result(result) return future
def wrapper(*args, **kwargs): ipdb.set_trace() future = TracebackFuture() if replace_callback and 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future(future, lambda future: callback(future.result())) try: result = func(*args, **kwargs) except (Return, StopIteration) as e: # ipdb.set_trace() result = gen._value_from_stopiteration(e) except Exception: # ipdb.set_trace() future.set_exc_info(sys.exc_info()) return future else: # ipdb.set_trace() if isinstance(result, GeneratorType): try: orig_stack_contexts = stack_context._state.contexts yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = TracebackFuture() yielded.set_exception(stack_context.StackContextInconsistentError('stack_context inconsistency (probably caused')) except (StopIteration, Return) as e: future.set_result(gen._value_from_stopiteration(e)) except Exception: future.set_exc_info(sys.exc_info()) else: try: result.send(yielded.result()) except (StopIteration, Return) as e: ipdb.set_trace() future.set_result(gen._value_from_stopiteration(e)) # gen.Runner(result, future, yielded) try: return future finally: future = None future.set_result(result) return future
def wrapper(*args, **kwargs): runner = None future = TracebackFuture() # 注册进去了...... if 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) def handle_exception(typ, value, tb): try: if runner is not None and runner.handle_exception(typ, value, tb): return True # 这样就吃掉exception了? except Exception: typ, value, tb = sys.exc_info() future.set_exc_info((typ, value, tb)) return True with ExceptionStackContext(handle_exception) as deactivate: try: result = func(*args, **kwargs) # result是func调用返回的一个结果,有可能是一个iterator,比如我们的get、post方法里面塞几个yield的调用 except (Return, StopIteration) as e: result = getattr(e, 'value', None) except Exception: deactivate() future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): def final_callback(value): deactivate() future.set_result(value) # 这里的result很可能是一个iterator,看看头部的调用例子 runner = Runner(result, final_callback) # 关键点 runner.run() return future deactivate() future.set_result(result) return future
def wrapper(*args, **kwargs): runner = None future = TracebackFuture() if 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) def handle_exception(typ, value, tb): try: if runner is not None and runner.handle_exception(typ, value, tb): return True except Exception: typ, value, tb = sys.exc_info() if sys.getrefcount(future) == 2: # there are no references left to this future object # to prevent the error passing silently throw the exception right here # so it can be catched and handled by IOLoop.handle_callback_exception raise else: future.set_exc_info((typ, value, tb)) return True with ExceptionStackContext(handle_exception) as deactivate: try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = getattr(e, 'value', None) except Exception: deactivate() future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): def final_callback(value): deactivate() future.set_result(value) runner = Runner(result, final_callback) runner.run() return future deactivate() future.set_result(result) return future
def wrapper(*args, **kwargs): future = TracebackFuture() if replace_callback and 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) try: with Measure(func): result = func(*args, **kwargs) except (Return, StopIteration) as e: result = getattr(e, 'value', None) except Exception: future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): try: orig_stack_contexts = stack_context._state.contexts with Measure(result, first=True): yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = TracebackFuture() yielded.set_exception( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: future.set_result(getattr(e, 'value', None)) except Exception: future.set_exc_info(sys.exc_info()) else: Runner(result, future, yielded) try: return future finally: future = None future.set_result(result) return future
def wrapper(*args, **kwargs): runner = None future = TracebackFuture() if 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) def handle_exception(typ, value, tb): try: if runner is not None and runner.handle_exception( typ, value, tb): return True except Exception: typ, value, tb = sys.exc_info() future.set_exc_info((typ, value, tb)) return True with ExceptionStackContext(handle_exception) as deactivate: try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = getattr(e, 'value', None) except Exception: deactivate() future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): def final_callback(value): deactivate() future.set_result(value) runner = Runner(result, final_callback) runner.run() return future deactivate() future.set_result(result) return future
def wrapper(*args, **kwargs): future = TracebackFuture() if replace_callback and "callback" in kwargs: callback = kwargs.pop("callback") IOLoop.current().add_future(future, lambda future: callback(future.result())) try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = getattr(e, "value", None) except Exception: future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: orig_stack_contexts = stack_context._state.contexts yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = TracebackFuture() yielded.set_exception( stack_context.StackContextInconsistentError( "stack_context inconsistency (probably caused " 'by yield within a "with StackContext" block)' ) ) except (StopIteration, Return) as e: future.set_result(getattr(e, "value", None)) except Exception: future.set_exc_info(sys.exc_info()) else: Runner(result, future, yielded) return future future.set_result(result) return future
def wrapper(*args, **kwargs): future = TracebackFuture() if replace_callback and 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = getattr(e, 'value', None) except Exception: future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: orig_stack_contexts = stack_context._state.contexts yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = TracebackFuture() yielded.set_exception( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: future.set_result(getattr(e, 'value', None)) except Exception: future.set_exc_info(sys.exc_info()) else: Runner(result, future, yielded) return future future.set_result(result) return future
def wrapper(*args, **kwargs): future = TracebackFuture() if replace_callback and 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = getattr(e, 'value', None) except Exception: future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): try: orig_stack_contexts = stack_context._state.contexts yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = TracebackFuture() yielded.set_exception( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: future.set_result(getattr(e, 'value', None)) except Exception: future.set_exc_info(sys.exc_info()) else: Runner(result, future, yielded) try: return future finally: future = None future.set_result(result) return future
def wrapper(*args, **kwargs): runner = None future = TracebackFuture() if replace_callback and 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = getattr(e, 'value', None) except Exception: future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, types.GeneratorType): runner = Runner(result, future) runner.run() return future future.set_result(result) return future
class Runner(object): """Internal implementation of `tornado.gen.engine`. Maintains information about pending callbacks and their results. The results of the generator are stored in ``result_future`` (a `.TracebackFuture`) """ def __init__(self, gen, result_future, first_yielded): self.gen = gen self.result_future = result_future self.future = _null_future self.yield_point = None self.pending_callbacks = None self.results = None self.running = False self.finished = False self.had_exception = False self.io_loop = IOLoop.current() # For efficiency, we do not create a stack context until we # reach a YieldPoint (stack contexts are required for the historical # semantics of YieldPoints, but not for Futures). When we have # done so, this field will be set and must be called at the end # of the coroutine. self.stack_context_deactivate = None if self.handle_yield(first_yielded): self.run() def register_callback(self, key): """Adds ``key`` to the list of callbacks.""" if self.pending_callbacks is None: # Lazily initialize the old-style YieldPoint data structures. self.pending_callbacks = set() self.results = {} if key in self.pending_callbacks: raise KeyReuseError("key %r is already pending" % (key, )) self.pending_callbacks.add(key) def is_ready(self, key): """Returns true if a result is available for ``key``.""" if self.pending_callbacks is None or key not in self.pending_callbacks: raise UnknownKeyError("key %r is not pending" % (key, )) return key in self.results def set_result(self, key, result): """Sets the result for ``key`` and attempts to resume the generator.""" self.results[key] = result if self.yield_point is not None and self.yield_point.is_ready(): try: self.future.set_result(self.yield_point.get_result()) except: self.future.set_exc_info(sys.exc_info()) self.yield_point = None self.run() def pop_result(self, key): """Returns the result for ``key`` and unregisters it.""" self.pending_callbacks.remove(key) return self.results.pop(key) def run(self): """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: return try: self.running = True while True: future = self.future if not future.done(): return self.future = None try: orig_stack_contexts = stack_context._state.contexts try: value = future.result() except Exception: self.had_exception = True yielded = self.gen.throw(*sys.exc_info()) else: yielded = self.gen.send(value) if stack_context._state.contexts is not orig_stack_contexts: self.gen.throw( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: self.finished = True self.future = _null_future if self.pending_callbacks and not self.had_exception: # If we ran cleanly without waiting on all callbacks # raise an error (really more of a warning). If we # had an exception then some callbacks may have been # orphaned, so skip the check in that case. raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) self.result_future.set_result(getattr(e, 'value', None)) self.result_future = None self._deactivate_stack_context() return except Exception: self.finished = True self.future = _null_future self.result_future.set_exc_info(sys.exc_info()) self.result_future = None self._deactivate_stack_context() return if not self.handle_yield(yielded): return finally: self.running = False def handle_yield(self, yielded): if isinstance(yielded, list): if all(is_future(f) for f in yielded): yielded = multi_future(yielded) else: yielded = Multi(yielded) elif isinstance(yielded, dict): if all(is_future(f) for f in yielded.values()): yielded = multi_future(yielded) else: yielded = Multi(yielded) if isinstance(yielded, YieldPoint): self.future = TracebackFuture() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): self.future.set_result(yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() elif is_future(yielded): self.future = yielded if not self.future.done() or self.future is moment: self.io_loop.add_future(self.future, lambda f: self.run()) return False else: self.future = TracebackFuture() self.future.set_exception( BadYieldError("yielded unknown object %r" % (yielded, ))) return True def result_callback(self, key): return stack_context.wrap( _argument_adapter(functools.partial(self.set_result, key))) def handle_exception(self, typ, value, tb): if not self.running and not self.finished: self.future = TracebackFuture() self.future.set_exc_info((typ, value, tb)) self.run() return True else: return False def _deactivate_stack_context(self): if self.stack_context_deactivate is not None: self.stack_context_deactivate() self.stack_context_deactivate = None
class Runner(object): """Internal implementation of `tornado.gen.engine`. Maintains information about pending callbacks and their results. The results of the generator are stored in ``result_future`` (a `.TracebackFuture`) """ def __init__(self, gen, result_future, first_yielded): self.gen = gen self.result_future = result_future self.future = _null_future self.yield_point = None self.pending_callbacks = None self.results = None self.running = False self.finished = False self.had_exception = False self.io_loop = IOLoop.current() # For efficiency, we do not create a stack context until we # reach a YieldPoint (stack contexts are required for the historical # semantics of YieldPoints, but not for Futures). When we have # done so, this field will be set and must be called at the end # of the coroutine. self.stack_context_deactivate = None if self.handle_yield(first_yielded): self.run() def register_callback(self, key): """Adds ``key`` to the list of callbacks.""" if self.pending_callbacks is None: # Lazily initialize the old-style YieldPoint data structures. self.pending_callbacks = set() self.results = {} if key in self.pending_callbacks: raise KeyReuseError("key %r is already pending" % (key,)) self.pending_callbacks.add(key) def is_ready(self, key): """Returns true if a result is available for ``key``.""" if self.pending_callbacks is None or key not in self.pending_callbacks: raise UnknownKeyError("key %r is not pending" % (key,)) return key in self.results def set_result(self, key, result): """Sets the result for ``key`` and attempts to resume the generator.""" self.results[key] = result if self.yield_point is not None and self.yield_point.is_ready(): try: self.future.set_result(self.yield_point.get_result()) except: self.future.set_exc_info(sys.exc_info()) self.yield_point = None self.run() def pop_result(self, key): """Returns the result for ``key`` and unregisters it.""" self.pending_callbacks.remove(key) return self.results.pop(key) def run(self): """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: return try: self.running = True while True: future = self.future if not future.done(): return self.future = None try: orig_stack_contexts = stack_context._state.contexts exc_info = None try: value = future.result() except Exception: self.had_exception = True exc_info = sys.exc_info() if exc_info is not None: yielded = self.gen.throw(*exc_info) exc_info = None else: yielded = self.gen.send(value) if stack_context._state.contexts is not orig_stack_contexts: self.gen.throw( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: self.finished = True self.future = _null_future if self.pending_callbacks and not self.had_exception: # If we ran cleanly without waiting on all callbacks # raise an error (really more of a warning). If we # had an exception then some callbacks may have been # orphaned, so skip the check in that case. raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) self.result_future.set_result(_value_from_stopiteration(e)) self.result_future = None self._deactivate_stack_context() return except Exception: self.finished = True self.future = _null_future self.result_future.set_exc_info(sys.exc_info()) self.result_future = None self._deactivate_stack_context() return if not self.handle_yield(yielded): return finally: self.running = False def handle_yield(self, yielded): # Lists containing YieldPoints require stack contexts; # other lists are handled in convert_yielded. if _contains_yieldpoint(yielded): yielded = multi(yielded) if isinstance(yielded, YieldPoint): # YieldPoints are too closely coupled to the Runner to go # through the generic convert_yielded mechanism. self.future = TracebackFuture() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): self.future.set_result( yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() else: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if not self.future.done() or self.future is moment: self.io_loop.add_future( self.future, lambda f: self.run()) return False return True def result_callback(self, key): return stack_context.wrap(_argument_adapter( functools.partial(self.set_result, key))) def handle_exception(self, typ, value, tb): if not self.running and not self.finished: self.future = TracebackFuture() self.future.set_exc_info((typ, value, tb)) self.run() return True else: return False def _deactivate_stack_context(self): if self.stack_context_deactivate is not None: self.stack_context_deactivate() self.stack_context_deactivate = None
class Runner(object): def __init__(self, gen, result_future, first_yielded): self.gen = gen self.result_future = result_future self.future = _null_future self.yield_point = None self.pending_callbacks = None self.results = None self.running = False self.finished = False self.had_exception = False self.io_loop = IOLoop.current() self.stack_context_deactivate = None if self.handle_yield(first_yielded): self.run() def run(self): if self.running or self.finished: return try: self.running = True while True: future = self.future if not future.done(): return self.future = None try: orig_stack_contexts = stack_context._state.contexts exc_info = None try: value = future.result() except Exception: self.had_exception = True exc_info = sys.exc_info() if exc_info is not None: yielded = self.gen.throw(*exc_info) exc_info = None else: yielded = self.gen.send(value) if stack_context._state.contexts is not orig_stack_contexts: self.gen.throw( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: self.finished = True self.future = _null_future if self.pending_callbacks and not self.had_exception: raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) self.result_future.set_result(getattr(e, 'value', None)) self.result_future = None self._deactivate_stack_context() return except Exception: self.finished = True self.future = _null_future self.result_future.set_exc_info(sys.exc_info()) self.result_future = None self._deactivate_stack_context() return if not self.handle_yield(yielded): return finally: self.running = False def handle_yield(self, yielded): try: self.future = convert_yielded(yielded) except BadYieldError: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if not self.future.done() or self.future is moment: self.io_loop.add_future(self.future, lambda f: self.run()) return False return True def _deactivate_stack_context(self): if self.stack_context_deactivate is not None: self.stack_context_deactivate() self.stack_context_deactivate = None
class Runner(object): def __init__(self, gen, result_future, first_yielded): self.gen = gen self.result_future = result_future self.future = _null_future self.yield_point = None self.pending_callbacks = None self.results = None self.running = False self.finished = False self.had_exception = False self.io_loop = IOLoop.current() self.stack_context_deactivate = None if self.handle_yield(first_yielded): self.run() def run(self): if self.running or self.finished: return try: self.running = True while True: future = self.future if not future.done(): return self.future = None try: orig_stack_contexts = stack_context._state.contexts exc_info = None try: value = future.result() except Exception: self.had_exception = True exc_info = sys.exc_info() if exc_info is not None: yielded = self.gen.throw(*exc_info) exc_info = None else: yielded = self.gen.send(value) if stack_context._state.contexts is not orig_stack_contexts: self.gen.throw( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: self.finished = True self.future = _null_future if self.pending_callbacks and not self.had_exception: raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) self.result_future.set_result(getattr(e, 'value', None)) self.result_future = None self._deactivate_stack_context() return except Exception: self.finished = True self.future = _null_future self.result_future.set_exc_info(sys.exc_info()) self.result_future = None self._deactivate_stack_context() return if not self.handle_yield(yielded): return finally: self.running = False def handle_yield(self, yielded): try: self.future = convert_yielded(yielded) except BadYieldError: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if not self.future.done() or self.future is moment: self.io_loop.add_future( self.future, lambda f: self.run()) return False return True def _deactivate_stack_context(self): if self.stack_context_deactivate is not None: self.stack_context_deactivate() self.stack_context_deactivate = None
def wrapper(*args, **kwargs): # 这个 future 对象是用来存储 coroutine 执行完毕后的结果的。结果通常有 # 两种来源,一个是在 coroutine 中 yield Return 对象,另一种则是 # coroutine 作为 generator 执行完毕后 raise # StopIteration,并有结果数据存储在 StopIteration 这个异常对象中。 future = TracebackFuture() if replace_callback and 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = _value_from_stopiteration(e) except Exception: future.set_exc_info(sys.exc_info()) return future else: if isinstance(result, GeneratorType): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: orig_stack_contexts = stack_context._state.contexts yielded = next(result) if stack_context._state.contexts is \ not orig_stack_contexts: yielded = TracebackFuture() yielded.set_exception( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: future.set_result(_value_from_stopiteration(e)) except Exception: future.set_exc_info(sys.exc_info()) else: # Runner 负责执行一个 coroutine,如果 coroutine 中的异步 # 操作已经完成,则 future 会被填充该操作的结果,否则 future # 会继续处于等待结果的状态,并紧接着在下一行代码返回给 # coroutine 的调用者。 Runner(result, future, yielded) try: return future finally: # Subtle memory optimization: if next() raised an # exception, the future's exc_info contains a traceback # which includes this stack frame. This creates a cycle, # which will be collected at the next full GC but has # been shown to greatly increase memory usage of # benchmarks (relative to the refcount-based scheme # used in the absence of cycles). We can avoid the # cycle by clearing the local variable after we return it. future = None future.set_result(result) return future
class Runner(object): """Internal implementation of `tornado.gen.engine`. Maintains information about pending callbacks and their results. The results of the generator are stored in ``result_future`` (a `.TracebackFuture`) """ def __init__(self, gen, result_future, first_yielded): self.gen = gen self.result_future = result_future self.future = _null_future self.yield_point = None self.pending_callbacks = None self.results = None self.running = False self.finished = False self.had_exception = False self.io_loop = IOLoop.current() # For efficiency, we do not create a stack context until we # reach a YieldPoint (stack contexts are required for the historical # semantics of YieldPoints, but not for Futures). When we have # done so, this field will be set and must be called at the end # of the coroutine. self.stack_context_deactivate = None if self.handle_yield(first_yielded): # handle_yield 在 coroutine yield 的对象有了结果时返回 # True,这个时候就会马上执行 run 方法。 self.run() def register_callback(self, key): """Adds ``key`` to the list of callbacks.""" if self.pending_callbacks is None: # Lazily initialize the old-style YieldPoint data structures. self.pending_callbacks = set() self.results = {} if key in self.pending_callbacks: raise KeyReuseError("key %r is already pending" % (key, )) self.pending_callbacks.add(key) def is_ready(self, key): """Returns true if a result is available for ``key``.""" if self.pending_callbacks is None or key not in self.pending_callbacks: raise UnknownKeyError("key %r is not pending" % (key, )) return key in self.results def set_result(self, key, result): """Sets the result for ``key`` and attempts to resume the generator.""" self.results[key] = result if self.yield_point is not None and self.yield_point.is_ready(): try: self.future.set_result(self.yield_point.get_result()) except: self.future.set_exc_info(sys.exc_info()) self.yield_point = None self.run() def pop_result(self, key): """Returns the result for ``key`` and unregisters it.""" self.pending_callbacks.remove(key) return self.results.pop(key) def run(self): """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: # 如果 run 方法已经在被执行,则返回。 return try: self.running = True while True: future = self.future if not future.done(): # Runner 对应的 coroutine yield # 的对象还没有准备好,则返回。 return self.future = None try: orig_stack_contexts = stack_context._state.contexts exc_info = None try: value = future.result() except Exception: # 如果执行过程中有抛出异常,则捕获之,并让 coroutine # 也抛出该异常。Python 中的 generator 在 2.5 版本后才 # 提供了 throw 方法来提供实现 coroutine # 的可能性,详情可以参考: # https://www.python.org/dev/peps/pep-0342/ self.had_exception = True exc_info = sys.exc_info() if exc_info is not None: # 如果有异常,让 coroutine 抛出。 yielded = self.gen.throw(*exc_info) exc_info = None else: # 如果一切正常,则将结果反馈给 coroutine。 yielded = self.gen.send(value) if stack_context._state.contexts \ is not orig_stack_contexts: self.gen.throw( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: # coroutine 的执行绪执行完毕,Runner 的任务也完成了。 self.finished = True self.future = _null_future if self.pending_callbacks and not self.had_exception: # If we ran cleanly without waiting on all callbacks # raise an error (really more of a warning). If we # had an exception then some callbacks may have been # orphaned, so skip the check in that case. raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) # 从 Exception 对象中获得最终结果,并反馈给 # result_future,进而触发 result_future 的 done callbacks。 self.result_future.set_result(_value_from_stopiteration(e)) self.result_future = None self._deactivate_stack_context() return except Exception: self.finished = True self.future = _null_future self.result_future.set_exc_info(sys.exc_info()) self.result_future = None self._deactivate_stack_context() return if not self.handle_yield(yielded): return finally: self.running = False def handle_yield(self, yielded): # handle_yield 方法会对 coroutine yield 的结果进行转化,进而判断 yield # 的异步执行的代码是否已经执行完毕并且已经得到了结果,如果没有,则将相 # 应得 future 对象放到 Event Loop 中,等待下一次 Event Loop # 执行到时再判断执行结果 # Lists containing YieldPoints require stack contexts; # other lists are handled in convert_yielded. if _contains_yieldpoint(yielded): # 如果 coroutine yield 的结果中包含 YieldPoint,则用 stack context # 将其包装一次 yielded = multi(yielded) if isinstance(yielded, YieldPoint): # YieldPoints are too closely coupled to the Runner to go # through the generic convert_yielded mechanism. self.future = TracebackFuture() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): self.future.set_result(yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() else: # 经过前面的处理,YieldPoint 都已经被正确包装、转化,接下来就可以将 # coroutine yield 的内容转化成一个 future 对象。 try: self.future = convert_yielded(yielded) except BadYieldError: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if not self.future.done() or self.future is moment: # 如果 future 还没有收到结果,不是 done 的状态,则将其添加到 IOLoop # 实例中,在后续某一次 Event Loop 中,future 被赋予了结果后再去执行 # run 方法,从而让 coroutine 获得结果并继续执行下去。 self.io_loop.add_future(self.future, lambda f: self.run()) # 这时便不用去执行 run 方法去获得 future 中的结果并反馈给 coroutine # 了,一切都等到之后的 Event Loop 中见分晓。 return False # Future 对象这个时候已经有了结果,便可以去执行 run 方法,并将结果反馈给 # coroutine,让之继续后面的执行绪了。 return True def result_callback(self, key): return stack_context.wrap( _argument_adapter(functools.partial(self.set_result, key))) def handle_exception(self, typ, value, tb): if not self.running and not self.finished: self.future = TracebackFuture() self.future.set_exc_info((typ, value, tb)) self.run() return True else: return False def _deactivate_stack_context(self): if self.stack_context_deactivate is not None: self.stack_context_deactivate() self.stack_context_deactivate = None
class Runner(object): """Internal implementation of `tornado.gen.engine`. Maintains information about pending callbacks and their results. The results of the generator are stored in ``result_future`` (a `.TracebackFuture`) """ def __init__(self, gen, result_future, first_yielded): self.gen = gen self.result_future = result_future # 第一次在handle_yield指向first_yielded,之后指向gen() self.future = _null_future self.yield_point = None self.pending_callbacks = None self.results = None self.running = False self.finished = False self.had_exception = False self.io_loop = IOLoop.current() # For efficiency, we do not create a stack context until we # reach a YieldPoint (stack contexts are required for the historical # semantics of YieldPoints, but not for Futures). When we have # done so, this field will be set and must be called at the end # of the coroutine. self.stack_context_deactivate = None if self.handle_yield(first_yielded): self.run() def register_callback(self, key): """Adds ``key`` to the list of callbacks.""" if self.pending_callbacks is None: # Lazily initialize the old-style YieldPoint data structures. self.pending_callbacks = set() self.results = {} if key in self.pending_callbacks: raise KeyReuseError("key %r is already pending" % (key, )) self.pending_callbacks.add(key) def is_ready(self, key): """Returns true if a result is available for ``key``.""" if self.pending_callbacks is None or key not in self.pending_callbacks: raise UnknownKeyError("key %r is not pending" % (key, )) return key in self.results def set_result(self, key, result): """Sets the result for ``key`` and attempts to resume the generator.""" self.results[key] = result if self.yield_point is not None and self.yield_point.is_ready(): try: self.future.set_result(self.yield_point.get_result()) except: self.future.set_exc_info(sys.exc_info()) self.yield_point = None self.run() def pop_result(self, key): """Returns the result for ``key`` and unregisters it.""" self.pending_callbacks.remove(key) return self.results.pop(key) def run(self): """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: return try: self.running = True while True: future = self.future if not future.done(): return self.future = None try: orig_stack_contexts = stack_context._state.contexts exc_info = None try: value = future.result() except Exception: self.had_exception = True exc_info = sys.exc_info() if exc_info is not None: yielded = self.gen.throw(*exc_info) exc_info = None else: # 发送异步调用结果,通知任务完成 yielded = self.gen.send(value) # self.gen为生成器,即被@gen.coroutine修饰的函数, # self.gen.send发送结果到 yield语句进行赋值,即 # result = yield clien.fetch(url),这里的result # 即为value,生成器结束会得到StopIteration异常结束 # 调用,否则继续调用self.handle_yield if stack_context._state.contexts is not orig_stack_contexts: self.gen.throw( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: # 捕捉到StopIteration 或者 Return 异常,说明 self.gen生成器 # 结束,Return 异常一般由用户调用 raise Return(True) self.finished = True self.future = _null_future if self.pending_callbacks and not self.had_exception: # If we ran cleanly without waiting on all callbacks # raise an error (really more of a warning). If we # had an exception then some callbacks may have been # orphaned, so skip the check in that case. raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) # 如果是 Return 异常设置返回值 self.result_future.set_result(_value_from_stopiteration(e)) self.result_future = None self._deactivate_stack_context() return except Exception: self.finished = True self.future = _null_future self.result_future.set_exc_info(sys.exc_info()) self.result_future = None self._deactivate_stack_context() return # 返回True说明yielded已经完成done,继续运行返回结果给上层,如果yielded # 还在等待Future完成,那么直接返回,handle_yield已经将yielded放到ioloop # 进行处理,完成后回调run运行 if not self.handle_yield(yielded): return finally: self.running = False def handle_yield(self, yielded): # 返回True已经完成,False表示Future未完成,添加到ioloop中 # Lists containing YieldPoints require stack contexts; # other lists are handled in convert_yielded. # 处理 yield list or dict 情况 if _contains_yieldpoint(yielded): yielded = multi(yielded) if isinstance(yielded, YieldPoint): # YieldPoints are too closely coupled to the Runner to go # through the generic convert_yielded mechanism. self.future = TracebackFuture() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): self.future.set_result(yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() else: # Future情况,为r = yield fetch()之类返回的Future try: self.future = convert_yielded(yielded) except BadYieldError: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if not self.future.done() or self.future is moment: self.io_loop.add_future(self.future, lambda f: self.run()) return False return True def result_callback(self, key): return stack_context.wrap( _argument_adapter(functools.partial(self.set_result, key))) def handle_exception(self, typ, value, tb): if not self.running and not self.finished: self.future = TracebackFuture() self.future.set_exc_info((typ, value, tb)) self.run() return True else: return False def _deactivate_stack_context(self): if self.stack_context_deactivate is not None: self.stack_context_deactivate() self.stack_context_deactivate = None
def wrapper(*args, **kwargs): future = TracebackFuture() # 一个future的跟踪对象,这个是关键 # 原来的func不支持callback的参数,修饰之后是可以使用的 # 下面的这个future好好读懂,然后好好理解传说中的future功能 if replace_callback and 'callback' in kwargs: callback = kwargs.pop('callback') # 就是future成功了,执行lambda future: callback(future.result())函数,参数就是这个future # 也就是把这个可能执行的语句放到IOLoop里面去,这个就是这个add_future的功能 IOLoop.current().add_future( future, lambda future: callback(future.result())) # 异步的HTTP和异步的IOStream也许返回的类型不一样, # 感觉两个异步的东西才是tornado的关键 try: # 尝试运行函数,这里会yield出来,所以不会被阻塞的 # 1. 返回一个Return 2. 返回一个StopIteration(这个不是很清楚??) # 3. 返回一个Exception?就是运行错误了 result = func(*args, **kwargs) except (Return, StopIteration) as e: # 到这里说明结果已经出来了,future的任务结束了 # ?StopIteration无法理解 result = getattr(e, 'value', None) except Exception: # 如果出错的话 # 运行出错了,这个时候依旧会添加到IOLoop._callbacks # 下面这句话执行这个future全部的回调 future.set_exc_info(sys.exc_info()) return future # 一但结果确定,直接返回一个future else: # 如果内部有yield,直接返回一个types.GeneratorType if isinstance(result, types.GeneratorType): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: orig_stack_contexts = stack_context._state.contexts # 这里是获取yield的结果的地方,一般还是一个future的东西 yielded = next(result) # 获取yield的结果,这里一般是一个future类型yield出来的东西 # 先忽略下面的出错处理 if stack_context._state.contexts is not orig_stack_contexts: yielded = TracebackFuture() yielded.set_exception( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: # 直接出结果 future.set_result(getattr(e, 'value', None)) except Exception: # bug了 future.set_exc_info(sys.exc_info()) else: # 返回一个future Runner(result, future, yielded) # 如果是一个Exception,这里记录到future,然后返回了 return future future.set_result(result) # 这个时候添加回调到了IOLoop return future