def _io_callback(self, future, result, fd=None, events=None): try: state = self.connection.poll() except (psycopg2.Warning, psycopg2.Error): self.ioloop.remove_handler(self.fileno) future_set_exc_info(future, sys.exc_info()) else: try: if state == POLL_OK: self.ioloop.remove_handler(self.fileno) future.set_result(result) elif state == POLL_READ: self.ioloop.update_handler(self.fileno, IOLoop.READ) elif state == POLL_WRITE: self.ioloop.update_handler(self.fileno, IOLoop.WRITE) else: future.set_exception( psycopg2.OperationalError("poll() returned %s" % state)) except IOError: # Can happen when there are quite a lof of outstanding # requests. See https://github.com/FSX/momoko/issues/127 self.ioloop.remove_handler(self.fileno) future.set_exception( psycopg2.OperationalError("IOError on socket"))
def when_available(fut): try: conn = fut.result() except psycopg2.Error: future_set_exc_info(future, sys.exc_info()) if retry and not keep: self.putconn(retry[0]) return log.debug("Obtained connection: %s", conn.fileno) try: future_or_result = method(conn, *args, **kwargs) except Exception: log.debug("Method failed synchronously") return self._retry(retry, when_available, conn, keep, future) if not async: future.set_result(future_or_result) if not keep: self.putconn(conn) return def when_done(rfut): try: result = rfut.result() except psycopg2.Error: log.debug("Method failed Asynchronously") return self._retry(retry, when_available, conn, keep, future) future.set_result(result) if not keep: self.putconn(conn) self.ioloop.add_future(future_or_result, when_done)
def wrapper(*args, **kwargs): # type: (*Any, **Any) -> Future[_T] # This function is type-annotated with a comment to work around # https://bitbucket.org/pypy/pypy/issues/2868/segfault-with-args-type-annotation-in future = _create_future() if contextvars is not None: ctx_run = contextvars.copy_context().run # type: Callable else: ctx_run = _fake_ctx_run try: result = ctx_run(func, *args, **kwargs) except (Return, StopIteration) as e: result = _value_from_stopiteration(e) except Exception: future_set_exc_info(future, sys.exc_info()) try: return future finally: # Avoid circular references future = None # type: ignore else: if isinstance(result, Generator): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: yielded = ctx_run(next, result) except (StopIteration, Return) as e: future_set_result_unless_cancelled( future, _value_from_stopiteration(e)) except Exception: future_set_exc_info(future, sys.exc_info()) else: # Provide strong references to Runner objects as long # as their result future objects also have strong # references (typically from the parent coroutine's # Runner). This keeps the coroutine's Runner alive. # We do this by exploiting the public API # add_done_callback() instead of putting a private # attribute on the Future. # (GitHub issues #1769, #2229). runner = Runner(ctx_run, result, future, yielded) future.add_done_callback(lambda _: runner) yielded = None try: return future finally: # Subtle memory optimization: if next() raised an exception, # the future's exc_info contains a traceback which # includes this stack frame. This creates a cycle, # which will be collected at the next full GC but has # been shown to greatly increase memory usage of # benchmarks (relative to the refcount-based scheme # used in the absence of cycles). We can avoid the # cycle by clearing the local variable after we return it. future = None # type: ignore future_set_result_unless_cancelled(future, result) return future
def errback(failure): try: failure.raiseException() # Should never happen, but just in case raise Exception("errback called without error") except: future_set_exc_info(f, sys.exc_info())
def greenlet_base(): try: result = f(*args, **kwargs) except Exception: future_set_exc_info(future, sys.exc_info()) else: future.set_result(result)
def abort_pending(f): exc_info = sys.exc_info() if any(exc_info): # f.set_exc_info(exc_info) concurrent.future_set_exc_info(f, exc_info) else: f.set_exception(exception(self.host, self.port))
def run(self): """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: return try: self.running = True while True: future = self.future if not future.done(): return self.future = None try: exc_info = None try: value = future.result() except Exception: self.had_exception = True exc_info = sys.exc_info() future = None if exc_info is not None: try: yielded = self.gen.throw(*exc_info) finally: # Break up a reference to itself # for faster GC on CPython. exc_info = None else: yielded = self.gen.send(value) except (StopIteration, Return) as e: self.finished = True self.future = _null_future if self.pending_callbacks and not self.had_exception: # If we ran cleanly without waiting on all callbacks # raise an error (really more of a warning). If we # had an exception then some callbacks may have been # orphaned, so skip the check in that case. raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) future_set_result_unless_cancelled( self.result_future, _value_from_stopiteration(e)) self.result_future = None return except Exception: self.finished = True self.future = _null_future future_set_exc_info(self.result_future, sys.exc_info()) self.result_future = None return if not self.handle_yield(yielded): return yielded = None finally: self.running = False
def wrapper(*args, **kwargs): future = _create_future() if replace_callback and 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = _value_from_stopiteration(e) except Exception: future_set_exc_info(future, sys.exc_info()) try: return future finally: # Avoid circular references future = None else: if isinstance(result, GeneratorType): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: orig_stack_contexts = stack_context._state.contexts yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = _create_future() yielded.set_exception( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: future_set_result_unless_cancelled( future, _value_from_stopiteration(e)) except Exception: future_set_exc_info(future, sys.exc_info()) else: _futures_to_runners[future] = Runner( result, future, yielded) yielded = None try: return future finally: # Subtle memory optimization: if next() raised an exception, # the future's exc_info contains a traceback which # includes this stack frame. This creates a cycle, # which will be collected at the next full GC but has # been shown to greatly increase memory usage of # benchmarks (relative to the refcount-based scheme # used in the absence of cycles). We can avoid the # cycle by clearing the local variable after we return it. future = None future_set_result_unless_cancelled(future, result) return future
def handle_exception(self, typ, value, tb): if not self.running and not self.finished: self.future = Future() future_set_exc_info(self.future, (typ, value, tb)) self.run() return True else: return False
def handle_yield(self, yielded): # Lists containing YieldPoints require stack contexts; # other lists are handled in convert_yielded. if _contains_yieldpoint(yielded): yielded = multi(yielded) if isinstance(yielded, YieldPoint): # YieldPoints are too closely coupled to the Runner to go # through the generic convert_yielded mechanism. self.future = Future() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): future_set_result_unless_cancelled( self.future, yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() else: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.future is moment: self.io_loop.add_callback(self.run) return False elif not self.future.done(): def inner(f): # Break a reference cycle to speed GC. f = None # noqa self.run() self.io_loop.add_future(self.future, inner) return False return True
def wrapper(*args, **kwargs): # type: (*Any, **Any) -> Future[_T] # This function is type-annotated with a comment to work around # https://bitbucket.org/pypy/pypy/issues/2868/segfault-with-args-type-annotation-in future = _create_future() try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = _value_from_stopiteration(e) except Exception: future_set_exc_info(future, sys.exc_info()) try: return future finally: # Avoid circular references future = None # type: ignore else: if isinstance(result, Generator): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: yielded = next(result) except (StopIteration, Return) as e: future_set_result_unless_cancelled( future, _value_from_stopiteration(e) ) except Exception: future_set_exc_info(future, sys.exc_info()) else: # Provide strong references to Runner objects as long # as their result future objects also have strong # references (typically from the parent coroutine's # Runner). This keeps the coroutine's Runner alive. # We do this by exploiting the public API # add_done_callback() instead of putting a private # attribute on the Future. # (Github issues #1769, #2229). runner = Runner(result, future, yielded) future.add_done_callback(lambda _: runner) yielded = None try: return future finally: # Subtle memory optimization: if next() raised an exception, # the future's exc_info contains a traceback which # includes this stack frame. This creates a cycle, # which will be collected at the next full GC but has # been shown to greatly increase memory usage of # benchmarks (relative to the refcount-based scheme # used in the absence of cycles). We can avoid the # cycle by clearing the local variable after we return it. future = None # type: ignore future_set_result_unless_cancelled(future, result) return future
def handle_exception(self, typ: Type[Exception], value: Exception, tb: types.TracebackType) -> bool: if not self.running and not self.finished: self.future = Future() future_set_exc_info(self.future, (typ, value, tb)) self.run() return True else: return False
def wrapper(*args, **kwargs): future = _create_future() if replace_callback and 'callback' in kwargs: callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = _value_from_stopiteration(e) except Exception: future_set_exc_info(future, sys.exc_info()) try: return future finally: # Avoid circular references future = None else: if isinstance(result, GeneratorType): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: orig_stack_contexts = stack_context._state.contexts yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = _create_future() yielded.set_exception( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: future_set_result_unless_cancelled(future, _value_from_stopiteration(e)) except Exception: future_set_exc_info(future, sys.exc_info()) else: _futures_to_runners[future] = Runner(result, future, yielded) yielded = None try: return future finally: # Subtle memory optimization: if next() raised an exception, # the future's exc_info contains a traceback which # includes this stack frame. This creates a cycle, # which will be collected at the next full GC but has # been shown to greatly increase memory usage of # benchmarks (relative to the refcount-based scheme # used in the absence of cycles). We can avoid the # cycle by clearing the local variable after we return it. future = None future_set_result_unless_cancelled(future, result) return future
def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): future_set_result_unless_cancelled(self.future, yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = Future() future_set_exc_info(self.future, sys.exc_info())
def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): self.future.set_result(yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = Future() future_set_exc_info(self.future, sys.exc_info())
def set_result(self, key, result): """Sets the result for ``key`` and attempts to resume the generator.""" self.results[key] = result if self.yield_point is not None and self.yield_point.is_ready(): try: self.future.set_result(self.yield_point.get_result()) except: future_set_exc_info(self.future, sys.exc_info()) self.yield_point = None self.run()
def future_set_exc_info(fut, exc_info): """ Sets the exception information of a `Future`, including the traceback. """ try: # tornado >= 5 cc.future_set_exc_info(fut, exc_info) except AttributeError: # tornado < 5 fut.set_exc_info(exc_info)
def handle_exception( self, typ: Type[Exception], value: Exception, tb: types.TracebackType ) -> bool: if not self.running and not self.finished: self.future = Future() future_set_exc_info(self.future, (typ, value, tb)) self.run() return True else: return False
def _register(self, future, registrator, fut): try: cursor = fut.result() except Exception: future_set_exc_info(future, sys.exc_info()) return oid, array_oid = cursor.fetchone() registrator(oid, array_oid) future.set_result(None)
def handle_yield(self, yielded): # Lists containing YieldPoints require stack contexts; # other lists are handled in convert_yielded. if _contains_yieldpoint(yielded): yielded = multi(yielded) if isinstance(yielded, YieldPoint): # YieldPoints are too closely coupled to the Runner to go # through the generic convert_yielded mechanism. self.future = Future() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): future_set_result_unless_cancelled(self.future, yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() else: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.future is moment: self.io_loop.add_callback(self.run) return False elif not self.future.done(): def inner(f): # Break a reference cycle to speed GC. f = None # noqa self.run() self.io_loop.add_future( self.future, inner) return False return True
def set_result(self, key, result): """Sets the result for ``key`` and attempts to resume the generator.""" self.results[key] = result if self.yield_point is not None and self.yield_point.is_ready(): try: future_set_result_unless_cancelled(self.future, self.yield_point.get_result()) except: future_set_exc_info(self.future, sys.exc_info()) self.yield_point = None self.run()
def on_ping_done(ping_fut): try: ping_fut.result() except psycopg2.Error: if conn.closed: ping_future.set_exception(self._no_conn_available_error) else: future_set_exc_info(ping_future, sys.exc_info()) self.putconn(conn) else: ping_future.set_result(conn)
def run(self) -> None: """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: return try: self.running = True while True: future = self.future if future is None: raise Exception("No pending future") if not future.done(): return self.future = None try: exc_info = None try: value = future.result() except Exception: exc_info = sys.exc_info() future = None if exc_info is not None: try: yielded = self.gen.throw(*exc_info) # type: ignore finally: # Break up a reference to itself # for faster GC on CPython. exc_info = None else: yielded = self.gen.send(value) except (StopIteration, Return) as e: self.finished = True self.future = _null_future future_set_result_unless_cancelled( self.result_future, _value_from_stopiteration(e) ) self.result_future = None # type: ignore return except Exception: self.finished = True self.future = _null_future future_set_exc_info(self.result_future, sys.exc_info()) self.result_future = None # type: ignore return if not self.handle_yield(yielded): return yielded = None finally: self.running = False
def _retry(self, retry, what, conn, keep, future): if conn.closed: if not retry: retry.append(conn) self.ioloop.add_future(conn.connect(), what) return else: future.set_exception(self._no_conn_available_error) else: future_set_exc_info(future, sys.exc_info()) if not keep: self.putconn(conn) return
def run(): try: result = func() if result is not None: from tornado.gen import convert_yielded result = convert_yielded(result) except Exception: future_cell[0] = Future() future_set_exc_info(future_cell[0], sys.exc_info()) else: if is_future(result): future_cell[0] = result else: future_cell[0] = Future() future_cell[0].set_result(result) self.add_future(future_cell[0], lambda future: self.stop())
def run(): """execute dest func""" try: result = func(*args, **kwargs) if result is not None: result = convert_yielded(result) except Exception: future_cell[0] = Future() future_set_exc_info(future_cell[0], sys.exc_info()) else: if is_future(result): future_cell[0] = result else: future_cell[0] = Future() future_cell[0].set_result(result) IOLoop.current().add_future(future_cell[0], lambda future: check_stop())
def run() -> None: try: result = func() if result is not None: from tornado.gen import convert_yielded result = convert_yielded(result) except Exception: fut = Future() # type: Future[Any] future_cell[0] = fut future_set_exc_info(fut, sys.exc_info()) else: if is_future(result): future_cell[0] = result else: fut = Future() future_cell[0] = fut fut.set_result(result) assert future_cell[0] is not None self.add_future(future_cell[0], lambda future: self.stop())
def exec_statements(future): try: cursor = future.result() cursors.append(cursor) except Exception as error: if auto_rollback and not self.closed: self._rollback(transaction_future, error) else: future_set_exc_info(transaction_future, sys.exc_info()) return try: operation, parameters = next(queue) except StopIteration: transaction_future.set_result(cursors[1:-1]) return f = self.execute(operation, parameters, cursor_factory) self.ioloop.add_future(f, exec_statements)
def callback(f): unfinished_children.remove(f) if not unfinished_children: result_list = [] for f in children: try: result_list.append(f.result()) except Exception as e: if future.done(): if not isinstance(e, quiet_exceptions): app_log.error("Multiple exceptions in yield list", exc_info=True) else: future_set_exc_info(future, sys.exc_info()) if not future.done(): if keys is not None: future.set_result(dict(zip(keys, result_list))) else: future.set_result(result_list)
def handle_yield(self, yielded): try: self.future = convert_yielded(yielded) except BadYieldError: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.future is moment: self.io_loop.add_callback(self.run) return False elif not self.future.done(): def inner(f): # Break a reference cycle to speed GC. f = None # noqa self.run() self.io_loop.add_future(self.future, inner) return False return True
def callback(f): unfinished_children.remove(f) if not unfinished_children: result_list = [] for f in children: try: result_list.append(f.result()) except Exception as e: if future.done(): if not isinstance(e, quiet_exceptions): app_log.error("Multiple exceptions in yield list", exc_info=True) else: future_set_exc_info(future, sys.exc_info()) if not future.done(): if keys is not None: future_set_result_unless_cancelled(future, dict(zip(keys, result_list))) else: future_set_result_unless_cancelled(future, result_list)
def handle_yield(self, yielded: _Yieldable) -> bool: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.future is moment: self.io_loop.add_callback(self.run) return False elif self.future is None: raise Exception("no pending future") elif not self.future.done(): def inner(f: Any) -> None: # Break a reference cycle to speed GC. f = None # noqa self.run() self.io_loop.add_future( self.future, inner) return False return True
def connect(self): """ Initiate asynchronous connect. Returns future that resolves to this connection object. """ kwargs = {"async": True} if self.connection_factory: kwargs["connection_factory"] = self.connection_factory if self.cursor_factory: kwargs["cursor_factory"] = self.cursor_factory future = Future() self.connection = None try: self.connection = psycopg2.connect(self.dsn, **kwargs) except psycopg2.Error: self.connection = None future_set_exc_info(future, sys.exc_info()) return future self.fileno = self.connection.fileno() if self.setsession: on_connect_future = Future() def on_connect(on_connect_future): self.ioloop.add_future(self.transaction(self.setsession), lambda x: future.set_result(self)) self.ioloop.add_future(on_connect_future, on_connect) callback = partial(self._io_callback, on_connect_future, self) else: callback = partial(self._io_callback, future, self) self.ioloop.add_handler(self.fileno, callback, IOLoop.WRITE) self.ioloop.add_future(future, self._set_server_version) self.ioloop.add_future(future, self._close_on_fail) return future
def handle_yield(self, yielded: _Yieldable) -> bool: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = Future() future_set_exc_info(self.future, sys.exc_info()) if self.future is moment: self.io_loop.add_callback(self.run) return False elif self.future is None: raise Exception("no pending future") elif not self.future.done(): def inner(f: Any) -> None: # Break a reference cycle to speed GC. f = None # noqa: F841 self.run() self.io_loop.add_future(self.future, inner) return False return True
def handle_exception(typ, value, tb): if future.done(): return False future_set_exc_info(future, (typ, value, tb)) return True
def run(self): """Starts or resumes the generator, running until it reaches a yield point that is not ready. """ if self.running or self.finished: return try: self.running = True while True: future = self.future if not future.done(): return self.future = None try: orig_stack_contexts = stack_context._state.contexts exc_info = None try: value = future.result() except Exception: self.had_exception = True exc_info = sys.exc_info() future = None if exc_info is not None: try: yielded = self.gen.throw(*exc_info) finally: # Break up a reference to itself # for faster GC on CPython. exc_info = None else: yielded = self.gen.send(value) if stack_context._state.contexts is not orig_stack_contexts: self.gen.throw( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)')) except (StopIteration, Return) as e: self.finished = True self.future = _null_future if self.pending_callbacks and not self.had_exception: # If we ran cleanly without waiting on all callbacks # raise an error (really more of a warning). If we # had an exception then some callbacks may have been # orphaned, so skip the check in that case. raise LeakedCallbackError( "finished without waiting for callbacks %r" % self.pending_callbacks) future_set_result_unless_cancelled(self.result_future, _value_from_stopiteration(e)) self.result_future = None self._deactivate_stack_context() return except Exception: self.finished = True self.future = _null_future future_set_exc_info(self.result_future, sys.exc_info()) self.result_future = None self._deactivate_stack_context() return if not self.handle_yield(yielded): return yielded = None finally: self.running = False
def handle_exception(typ, value, tb): if future.done(): return False else: future_set_exc_info(future, (typ, value, tb)) return True
def wrapper(*args, **kwargs): future = _create_future() if replace_callback and 'callback' in kwargs: warnings.warn( "callback arguments are deprecated, use the returned Future instead", DeprecationWarning, stacklevel=2) callback = kwargs.pop('callback') IOLoop.current().add_future( future, lambda future: callback(future.result())) try: result = func(*args, **kwargs) except (Return, StopIteration) as e: result = _value_from_stopiteration(e) except Exception: future_set_exc_info(future, sys.exc_info()) try: return future finally: # Avoid circular references future = None else: if isinstance(result, GeneratorType): # Inline the first iteration of Runner.run. This lets us # avoid the cost of creating a Runner when the coroutine # never actually yields, which in turn allows us to # use "optional" coroutines in critical path code without # performance penalty for the synchronous case. try: orig_stack_contexts = stack_context._state.contexts yielded = next(result) if stack_context._state.contexts is not orig_stack_contexts: yielded = _create_future() yielded.set_exception( stack_context.StackContextInconsistentError( 'stack_context inconsistency (probably caused ' 'by yield within a "with StackContext" block)') ) except (StopIteration, Return) as e: future_set_result_unless_cancelled( future, _value_from_stopiteration(e)) except Exception: future_set_exc_info(future, sys.exc_info()) else: # Provide strong references to Runner objects as long # as their result future objects also have strong # references (typically from the parent coroutine's # Runner). This keeps the coroutine's Runner alive. # We do this by exploiting the public API # add_done_callback() instead of putting a private # attribute on the Future. # (Github issues #1769, #2229). runner = Runner(result, future, yielded) future.add_done_callback(lambda _: runner) yielded = None try: return future finally: # Subtle memory optimization: if next() raised an exception, # the future's exc_info contains a traceback which # includes this stack frame. This creates a cycle, # which will be collected at the next full GC but has # been shown to greatly increase memory usage of # benchmarks (relative to the refcount-based scheme # used in the absence of cycles). We can avoid the # cycle by clearing the local variable after we return it. future = None future_set_result_unless_cancelled(future, result) return future