def with_timeout(timeout, future, io_loop=None, quiet_exceptions=()):
result = Future()
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
def error_callback(future):
try:
future.result()
except Exception as e:
if not isinstance(e, quiet_exceptions):
print("Exception in Future %r after timeout" % future)
def timeout_callback():
result.set_exception(TimeoutError("Timeout"))
# In case the wrapped future goes on to fail, log it.
future.add_done_callback(error_callback)
timeout_handle = io_loop.add_timeout(
timeout, timeout_callback)
if isinstance(future, Future):
future.add_done_callback(
lambda future: io_loop.remove_timeout(timeout_handle))
else:
io_loop.add_future(
future, lambda future: io_loop.remove_timeout(timeout_handle))
return result
def with_timeout(timeout, future, io_loop=None, quiet_exceptions=()):
result = Future()
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
def error_callback(future):
try:
future.result()
except Exception as e:
if not isinstance(e, quiet_exceptions):
print("Exception in Future %r after timeout" % future)
def timeout_callback():
result.set_exception(TimeoutError("Timeout"))
# In case the wrapped future goes on to fail, log it.
future.add_done_callback(error_callback)
timeout_handle = io_loop.add_timeout(timeout, timeout_callback)
if isinstance(future, Future):
future.add_done_callback(
lambda future: io_loop.remove_timeout(timeout_handle))
else:
io_loop.add_future(
future, lambda future: io_loop.remove_timeout(timeout_handle))
return result
def on_reanimate_done(fut):
if self.conns.all_dead:
future.set_exception(self._no_conn_available_error)
return
f = self.conns.acquire()
assert isinstance(f, Future)
chain_future(f, future)
def __init__(self, future, io_loop, timeout_td, timeout_exception):
super(_Wait, self).__init__()
self._io_loop = io_loop
self._timeout_exception = timeout_exception
self._timeout_obj = io_loop.add_timeout(timeout_td, self._on_timeout)
concurrent.chain_future(future, self)
future.add_done_callback(clear_tb_log)
def watch(self, url_path, on_data, **kwargs):
local_data = dict(buffer="")
class WatchFuture(Future):
def cancel(self):
client.close()
logging.debug("AsyncHTTPClient closed")
def data_callback(data):
split_data = data.split("\n")
for index, fragment in enumerate(split_data):
if index + 1 < len(split_data):
on_data(json.loads(local_data["buffer"] + fragment))
local_data["buffer"] = ""
else:
local_data["buffer"] += fragment
params = self.build_params(url_path, **kwargs)
url = url_concat(self.build_url(url_path, **kwargs), params)
request = HTTPRequest(
url=url,
method="GET",
headers=self.build_headers(),
request_timeout=3600,
streaming_callback=data_callback)
client = AsyncHTTPClient(force_instance=True)
future = WatchFuture()
chain_future(client.fetch(request), future)
return future
def watch(self, url_path, on_data, **kwargs):
class WatchFuture(Future):
def cancel(self):
client.close()
logging.debug("AsyncHTTPClient closed")
def data_callback(data):
on_data(json.loads(data))
params = self.build_params(url_path, **kwargs)
url = url_concat(self.build_url(url_path, **kwargs), params)
request = HTTPRequest(
url=url,
method="GET",
headers=self.build_headers(),
request_timeout=3600,
streaming_callback=data_callback)
client = AsyncHTTPClient(force_instance=True)
future = WatchFuture()
chain_future(client.fetch(request), future)
return future
def with_timeout(timeout, future, io_loop=None):
"""Wraps a `.Future` in a timeout.
Raises `TimeoutError` if the input future does not complete before
``timeout``, which may be specified in any form allowed by
`.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time
relative to `.IOLoop.time`)
Currently only supports Futures, not other `YieldPoint` classes.
.. versionadded:: 4.0
"""
# TODO: allow yield points in addition to futures?
# Tricky to do with stack_context semantics.
#
# It's tempting to optimize this by cancelling the input future on timeout
# instead of creating a new one, but A) we can't know if we are the only
# one waiting on the input future, so cancelling it might disrupt other
# callers and B) concurrent futures can only be cancelled while they are
# in the queue, so cancellation cannot reliably bound our waiting time.
result = Future()
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
timeout_handle = io_loop.add_timeout(timeout, lambda: result.set_exception(TimeoutError("Timeout")))
if isinstance(future, Future):
# We know this future will resolve on the IOLoop, so we don't
# need the extra thread-safety of IOLoop.add_future (and we also
# don't care about StackContext here.
future.add_done_callback(lambda future: io_loop.remove_timeout(timeout_handle))
else:
# concurrent.futures.Futures may resolve on any thread, so we
# need to route them back to the IOLoop.
io_loop.add_future(future, lambda future: io_loop.remove_timeout(timeout_handle))
return result
def handle_connection(future):
conn = future.result()
if callback is not None:
def handle_result(future):
self._ioloop.add_callback(callback, future.result())
future1.add_done_callback(handle_result)
chain_future(conn.send_message(args), future1)
def with_timeout(timeout, future, io_loop=None, quiet_exceptions=()):
"""Wraps a `.Future` in a timeout.
Raises `TimeoutError` if the input future does not complete before
``timeout``, which may be specified in any form allowed by
`.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time
relative to `.IOLoop.time`)
If the wrapped `.Future` fails after it has timed out, the exception
will be logged unless it is of a type contained in ``quiet_exceptions``
(which may be an exception type or a sequence of types).
Currently only supports Futures, not other `YieldPoint` classes.
.. versionadded:: 4.0
.. versionchanged:: 4.1
Added the ``quiet_exceptions`` argument and the logging of unhandled
exceptions.
"""
# TODO: allow yield points in addition to futures?
# Tricky to do with stack_context semantics.
#
# It's tempting to optimize this by cancelling the input future on timeout
# instead of creating a new one, but A) we can't know if we are the only
# one waiting on the input future, so cancelling it might disrupt other
# callers and B) concurrent futures can only be cancelled while they are
# in the queue, so cancellation cannot reliably bound our waiting time.
result = Future()
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
def error_callback(future):
try:
future.result()
except Exception as e:
if not isinstance(e, quiet_exceptions):
app_log.error("Exception in Future %r after timeout",
future,
exc_info=True)
def timeout_callback():
result.set_exception(TimeoutError("Timeout"))
# In case the wrapped future goes on to fail, log it.
future.add_done_callback(error_callback)
timeout_handle = io_loop.add_timeout(timeout, timeout_callback)
if isinstance(future, Future):
# We know this future will resolve on the IOLoop, so we don't
# need the extra thread-safety of IOLoop.add_future (and we also
# don't care about StackContext here.
future.add_done_callback(
lambda future: io_loop.remove_timeout(timeout_handle))
else:
# concurrent.futures.Futures may resolve on any thread, so we
# need to route them back to the IOLoop.
io_loop.add_future(
future, lambda future: io_loop.remove_timeout(timeout_handle))
return result
def __init__(self, future, io_loop, timeout_td, timeout_exception):
super(_Wait, self).__init__()
self._io_loop = io_loop
self._timeout_exception = timeout_exception
self._timeout_obj = io_loop.add_timeout(timeout_td, self._on_timeout)
concurrent.chain_future(future, self)
future.add_done_callback(clear_tb_log)
def on_reanimate_done(fut):
if self.conns.all_dead:
future.set_exception(self._no_conn_available_error)
return
f = self.conns.acquire()
assert isinstance(f, Future)
chain_future(f, future)
def with_timeout(timeout, future, io_loop=None, quiet_exceptions=()):
"""Wraps a `.Future` (or other yieldable object) in a timeout.
Raises `TimeoutError` if the input future does not complete before
``timeout``, which may be specified in any form allowed by
`.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time
relative to `.IOLoop.time`)
If the wrapped `.Future` fails after it has timed out, the exception
will be logged unless it is of a type contained in ``quiet_exceptions``
(which may be an exception type or a sequence of types).
Does not support `YieldPoint` subclasses.
.. versionadded:: 4.0
.. versionchanged:: 4.1
Added the ``quiet_exceptions`` argument and the logging of unhandled
exceptions.
.. versionchanged:: 4.4
Added support for yieldable objects other than `.Future`.
"""
# TODO: allow YieldPoints in addition to other yieldables?
# Tricky to do with stack_context semantics.
#
# It's tempting to optimize this by cancelling the input future on timeout
# instead of creating a new one, but A) we can't know if we are the only
# one waiting on the input future, so cancelling it might disrupt other
# callers and B) concurrent futures can only be cancelled while they are
# in the queue, so cancellation cannot reliably bound our waiting time.
future = convert_yielded(future)
result = Future()
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
def error_callback(future):
try:
future.result()
except Exception as e:
if not isinstance(e, quiet_exceptions):
app_log.error("Exception in Future %r after timeout", future, exc_info=True)
def timeout_callback():
result.set_exception(TimeoutError("Timeout"))
# In case the wrapped future goes on to fail, log it.
future.add_done_callback(error_callback)
timeout_handle = io_loop.add_timeout(timeout, timeout_callback)
if isinstance(future, Future):
# We know this future will resolve on the IOLoop, so we don't
# need the extra thread-safety of IOLoop.add_future (and we also
# don't care about StackContext here.
future.add_done_callback(lambda future: io_loop.remove_timeout(timeout_handle))
else:
# concurrent.futures.Futures may resolve on any thread, so we
# need to route them back to the IOLoop.
io_loop.add_future(future, lambda future: io_loop.remove_timeout(timeout_handle))
return result
def when_available(fut):
try:
conn = fut.result()
except psycopg2.Error as error:
future.set_exc_info(sys.exc_info())
if retry:
self.putconn(retry[0])
return
log.debug("Obtained connection: %s", conn.fileno)
try:
future_or_result = method(conn, *args, **kwargs)
except psycopg2.Error as error:
if conn.closed:
if not retry:
retry.append(conn)
self.ioloop.add_future(conn.connect(), when_available)
return
else:
future.set_exception(self._no_conn_availble_error)
else:
future.set_exc_info(sys.exc_info())
log.debug(2)
self.putconn(conn)
return
if not async:
future.set_result(future_or_result)
log.debug(3)
self.putconn(conn)
return
chain_future(future_or_result, future)
if not keep:
future.add_done_callback(lambda f: self.putconn(conn))
def watch(self, url_path, on_data, **kwargs):
local_data = dict(buffer="")
class WatchFuture(Future):
def cancel(self):
client.close()
logging.debug("AsyncHTTPClient closed")
def data_callback(data):
split_data = data.split("\n")
for index, fragment in enumerate(split_data):
if index + 1 < len(split_data):
on_data(json.loads(local_data["buffer"] + fragment))
local_data["buffer"] = ""
else:
local_data["buffer"] += fragment
params = self.build_params(url_path, **kwargs)
url = url_concat(self.build_url(url_path, **kwargs), params)
request = HTTPRequest(
url=url,
method="GET",
headers=self.build_headers(),
request_timeout=3600,
streaming_callback=data_callback)
client = AsyncHTTPClient(force_instance=True)
future = WatchFuture()
chain_future(client.fetch(request), future)
return future
def _return_result(self, done):
"""Called set the returned future's state that of the future
we yielded, and set the current future for the iterator.
"""
chain_future(done, self._running_future)
self.current_future = done
self.current_index = self._unfinished.pop(done)
def on_reanimate_done(fut):
if self.conns.all_dead:
log.debug("all connections are still dead")
future.set_exception(self._no_conn_available_error)
return
f = self.conns.acquire()
assert isinstance(f, Future)
chain_future(f, future)
def _return_result(self, done):
"""Called set the returned future's state that of the future
we yielded, and set the current future for the iterator.
"""
chain_future(done, self._running_future)
self.current_future = done
self.current_index = self._unfinished.pop(done)
def func_wrapper(self, *args, **kwargs):
future = Future()
chain_future(self.main_future, future)
func(self, future, *args, **kwargs)
result = yield future
raise gen.Return(result)
def on_reanimate_done(fut):
if self.conns.all_dead:
log.debug("all connections are still dead")
future.set_exception(self._no_conn_available_error)
return
f = self.conns.acquire()
assert isinstance(f, Future)
chain_future(f, future)
def handle_connection(future):
conn = future.result()
if callback is not None:
def handle_result(future):
self._ioloop.add_callback(callback, future.result())
future1.add_done_callback(handle_result)
chain_future(conn.send_message(args), future1)
def facebook_request(self, path, callback, access_token=None,
post_args=None, **args):
"""Fetches the given relative API path, e.g., "/btaylor/picture"
If the request is a POST, ``post_args`` should be provided. Query
string arguments should be given as keyword arguments.
An introduction to the Facebook Graph API can be found at
http://developers.facebook.com/docs/api
Many methods require an OAuth access token which you can
obtain through `~OAuth2Mixin.authorize_redirect` and
`get_authenticated_user`. The user returned through that
process includes an ``access_token`` attribute that can be
used to make authenticated requests via this method.
Example usage:
..testcode::
class MainHandler(tornado.web.RequestHandler,
tornado.auth.FacebookGraphMixin):
@tornado.web.authenticated
@tornado.gen.coroutine
def get(self):
new_entry = yield self.facebook_request(
"/me/feed",
post_args={"message": "I am posting from my Tornado application!"},
access_token=self.current_user["access_token"])
if not new_entry:
# Call failed; perhaps missing permission?
yield self.authorize_redirect()
return
self.finish("Posted a message!")
.. testoutput::
:hide:
The given path is relative to ``self._FACEBOOK_BASE_URL``,
by default "https://graph.facebook.com".
This method is a wrapper around `OAuth2Mixin.oauth2_request`;
the only difference is that this method takes a relative path,
while ``oauth2_request`` takes a complete url.
.. versionchanged:: 3.1
Added the ability to override ``self._FACEBOOK_BASE_URL``.
"""
url = self._FACEBOOK_BASE_URL + path
# Thanks to the _auth_return_future decorator, our "callback"
# argument is a Future, which we cannot pass as a callback to
# oauth2_request. Instead, have oauth2_request return a
# future and chain them together.
oauth_future = self.oauth2_request(url, access_token=access_token,
post_args=post_args, **args)
chain_future(oauth_future, callback)
def facebook_request(self, path, callback, access_token=None,
post_args=None, **args):
"""Fetches the given relative API path, e.g., "/btaylor/picture"
If the request is a POST, ``post_args`` should be provided. Query
string arguments should be given as keyword arguments.
An introduction to the Facebook Graph API can be found at
http://developers.facebook.com/docs/api
Many methods require an OAuth access token which you can
obtain through `~OAuth2Mixin.authorize_redirect` and
`get_authenticated_user`. The user returned through that
process includes an ``access_token`` attribute that can be
used to make authenticated requests via this method.
Example usage:
..testcode::
class MainHandler(tornado.web.RequestHandler,
tornado.auth.FacebookGraphMixin):
@tornado.web.authenticated
@tornado.gen.coroutine
def get(self):
new_entry = yield self.facebook_request(
"/me/feed",
post_args={"message": "I am posting from my Tornado application!"},
access_token=self.current_user["access_token"])
if not new_entry:
# Call failed; perhaps missing permission?
yield self.authorize_redirect()
return
self.finish("Posted a message!")
.. testoutput::
:hide:
The given path is relative to ``self._FACEBOOK_BASE_URL``,
by default "https://graph.facebook.com".
This method is a wrapper around `OAuth2Mixin.oauth2_request`;
the only difference is that this method takes a relative path,
while ``oauth2_request`` takes a complete url.
.. versionchanged:: 3.1
Added the ability to override ``self._FACEBOOK_BASE_URL``.
"""
url = self._FACEBOOK_BASE_URL + path
# Thanks to the _auth_return_future decorator, our "callback"
# argument is a Future, which we cannot pass as a callback to
# oauth2_request. Instead, have oauth2_request return a
# future and chain them together.
oauth_future = self.oauth2_request(url, access_token=access_token,
post_args=post_args, **args)
chain_future(oauth_future, callback)
def _return_result(self, done: Future) -> None:
"""Called set the returned future's state that of the future
we yielded, and set the current future for the iterator.
"""
if self._running_future is None:
raise Exception("no future is running")
chain_future(done, self._running_future)
self.current_future = done
self.current_index = self._unfinished.pop(done)
def with_timeout(timeout, future, io_loop=None):
result = Future()
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
timeout_handle = io_loop.add_timeout(
timeout, lambda: result.set_exception(TimeoutError("Timeout")))
future.add_done_callback(
lambda future: io_loop.remove_timeout(timeout_handle))
return result
def enqueue(self, task):
if self._in_active(task):
future = concurrent.Future()
concurrent.chain_future(self._get_future_for_task(task), future)
return future
future = concurrent.Future()
self._add_to_active(task, future)
concurrent.chain_future(self._do(task), future)
return future
def run() -> None:
try:
result = func()
if result is not None:
from tornado.gen import convert_yielded
result = convert_yielded(result)
concurrent.chain_future(result, f)
except Exception as e:
f.set_exception(e)
def _return_result(self, done: Future) -> None:
"""Called set the returned future's state that of the future
we yielded, and set the current future for the iterator.
"""
if self._running_future is None:
raise Exception("no future is running")
chain_future(done, self._running_future)
self.current_future = done
self.current_index = self._unfinished.pop(done)
def with_timeout(timeout, future, io_loop=None):
result = Future()
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
timeout_handle = io_loop.add_timeout(
timeout,
lambda: result.set_exception(TimeoutError("Timeout")))
future.add_done_callback(
lambda future: io_loop.remove_timeout(timeout_handle))
return result
def start_request(self, auth_result, request, user_result):
if auth_result.exception():
concurrent.chain_future(auth_result, user_result)
return
try:
f = self.client.fetch(request)
except Exception:
user_result.set_exc_info(sys.exc_info())
else:
cb = functools.partial(self.on_request_done, user_result)
f.add_done_callback(cb)
def start_request(self, auth_result, request, user_result):
if auth_result.exception():
concurrent.chain_future(auth_result, user_result)
return
try:
f = self.client.fetch(request)
except Exception:
user_result.set_exc_info(sys.exc_info())
else:
cb = functools.partial(self.on_request_done, user_result)
f.add_done_callback(cb)
def on_request_done(self, user_result, fetch_result):
""" Finally parse the result and run the user's callback. """
if fetch_result.exception():
concurrent.chain_future(fetch_result, user_result)
else:
native_resp = fetch_result.result()
try:
content = self.serializer.decode(native_resp.body.decode())
except Exception:
user_result.set_exc_info(sys.exc_info())
else:
resp = base.Response(http_code=native_resp.code,
headers=native_resp.headers,
content=content, error=None,
extra=native_resp)
user_result.set_result(self.ingress_filter(resp))
def request_halt(self, req, msg):
"""Halts the server, logs to syslog and slack, and exits the program
Returns
-------
success : {'ok', 'fail'}
Whether scheduling the halt succeeded.
Examples
--------
::
?halt
!halt ok
TODO:
- Call halt method on superclass to avoid copy paste
Doing this caused an issue:
File "/Users/Eric/Berkeley/seti/packages/meerkat/lib/python2.7/site-packages/katcp/server.py", line 1102, in handle_request
assert (reply.mtype == Message.REPLY)
AttributeError: 'NoneType' object has no attribute 'mtype'
"""
f = Future()
@gen.coroutine
def _halt():
req.reply("ok")
yield gen.moment
self.stop(timeout=None)
raise AsyncReply
self.ioloop.add_callback(lambda: chain_future(_halt(), f))
log.critical("HALTING SERVER!!!")
# TODO: uncomment when you deploy
# notify_slack("KATCP server at MeerKAT has halted. Might want to check that!")
sys.exit(0)
def run_in_executor(
self,
executor: Optional[concurrent.futures.Executor],
func: Callable[..., _T],
*args: Any
) -> Awaitable[_T]:
"""Runs a function in a ``concurrent.futures.Executor``. If
``executor`` is ``None``, the IO loop's default executor will be used.
Use `functools.partial` to pass keyword arguments to ``func``.
.. versionadded:: 5.0
"""
if executor is None:
if not hasattr(self, "_executor"):
from tornado.process import cpu_count
self._executor = concurrent.futures.ThreadPoolExecutor(
max_workers=(cpu_count() * 5)
) # type: concurrent.futures.Executor
executor = self._executor
c_future = executor.submit(func, *args)
# Concurrent Futures are not usable with await. Wrap this in a
# Tornado Future instead, using self.add_future for thread-safety.
t_future = Future() # type: Future[_T]
self.add_future(c_future, lambda f: chain_future(f, t_future))
return t_future
def as_future(self, query):
"""Wrap a `sqlalchemy.orm.query.Query` object into a
`concurrent.futures.Future` so that it can be yielded.
Parameters
----------
query : sqlalchemy.orm.query.Query
SQLAlchemy query object to execute
Returns
-------
tornado.concurrent.Future
A `Future` object wrapping the given query so that tornado can
await/yield on it
"""
# concurrent.futures.Future is not compatible with the "new style"
# asyncio Future, and awaiting on such "old-style" futures does not
# work.
#
# tornado includes a `run_in_executor` function to help with this
# problem, but it's only included in version 5+. Hence, we copy a
# little bit of code here to handle this incompatibility.
if not self._pool:
self._pool = ThreadPoolExecutor(max_workers=self._max_workers)
old_future = self._pool.submit(query)
new_future = Future()
IOLoop.current().add_future(old_future,
lambda f: chain_future(f, new_future))
return new_future
def run_in_executor(
self,
executor: Optional[concurrent.futures.Executor],
func: Callable[..., _T],
*args: Any
) -> Awaitable[_T]:
"""Runs a function in a ``concurrent.futures.Executor``. If
``executor`` is ``None``, the IO loop's default executor will be used.
Use `functools.partial` to pass keyword arguments to ``func``.
.. versionadded:: 5.0
"""
if executor is None:
if not hasattr(self, "_executor"):
from tornado.process import cpu_count
self._executor = concurrent.futures.ThreadPoolExecutor(
max_workers=(cpu_count() * 5)
) # type: concurrent.futures.Executor
executor = self._executor
c_future = executor.submit(func, *args)
# Concurrent Futures are not usable with await. Wrap this in a
# Tornado Future instead, using self.add_future for thread-safety.
t_future = Future() # type: Future[_T]
self.add_future(c_future, lambda f: chain_future(f, t_future))
return t_future
def _execute_ioloop(self, func, *args):
"""Execute method ioloop."""
pool = ThreadPoolExecutor(max_workers=COUNT_CPU)
old_future = pool.submit(func, *args)
new_future = Future()
IOLoop.current().add_future(old_future,
lambda fut: chain_future(fut, new_future))
return new_future
def on_request_done(self, user_result, fetch_result):
""" Finally parse the result and run the user's callback. """
if fetch_result.exception():
concurrent.chain_future(fetch_result, user_result)
else:
native_resp = fetch_result.result()
try:
content = self.serializer.decode(native_resp.body.decode())
except Exception:
user_result.set_exc_info(sys.exc_info())
else:
resp = base.Response(http_code=native_resp.code,
headers=native_resp.headers,
content=content,
error=None,
extra=native_resp)
user_result.set_result(self.ingress_filter(resp))
def _as_future(self, func, *args, **kwargs):
c_future = self._pool.submit(func, *args, **kwargs)
# Concurrent Futures are not usable with await. Wrap this in a
# Tornado Future instead, using self.add_future for thread-safety.
t_future = Future()
IOLoop.current().add_future(c_future,
lambda f: chain_future(f, t_future))
return t_future
def get_authenticated_user(self, callback):
"""Fetches the authenticated user data upon redirect."""
# Look to see if we are doing combined OpenID/OAuth
oauth_ns = ""
for name, values in self.request.arguments.items():
if name.startswith("openid.ns.") and \
values[-1] == b"http://specs.openid.net/extensions/oauth/1.0":
oauth_ns = name[10:]
break
token = self.get_argument("openid." + oauth_ns + ".request_token", "")
if token:
http = self.get_auth_http_client()
token = dict(key=token, secret="")
http.fetch(self._oauth_access_token_url(token),
self.async_callback(self._on_access_token, callback))
else:
chain_future(OpenIdMixin.get_authenticated_user(self),
callback)
def with_timeout(timeout, future, io_loop=None):
"""Wraps a `.Future` in a timeout.
Raises `TimeoutError` if the input future does not complete before
``timeout``, which may be specified in any form allowed by
`.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or an absolute time
relative to `.IOLoop.time`)
Currently only supports Futures, not other `YieldPoint` classes.
.. versionadded:: 4.0
"""
# TODO: allow yield points in addition to futures?
# Tricky to do with stack_context semantics.
#
# It's tempting to optimize this by cancelling the input future on timeout
# instead of creating a new one, but A) we can't know if we are the only
# one waiting on the input future, so cancelling it might disrupt other
# callers and B) concurrent futures can only be cancelled while they are
# in the queue, so cancellation cannot reliably bound our waiting time.
result = Future()
# 把 future 和 result 串联起来，future 完成时若 result 没有完成或者被取消，则将
# future 的结果拷贝给 result。至于为什么不是超时取消 future，而是新建一个 result，
# 前面注释有解释。
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
timeout_handle = io_loop.add_timeout(
timeout, lambda: result.set_exception(TimeoutError("Timeout")))
# Future 是由 IOLoop 来负责解析，完成时由 IOLoop 触发回调，故不需要重复加入 IOLoop。
# 而 concurrent.futures.Futures 则可能由其他线程来解析，所以需要将其加入到 IOLoop
# 中，以便在其完成时由 IOLoop 触发回调。
if isinstance(future, Future):
# We know this future will resolve on the IOLoop, so we don't
# need the extra thread-safety of IOLoop.add_future (and we also
# don't care about StackContext here.
future.add_done_callback(
lambda future: io_loop.remove_timeout(timeout_handle))
else:
# concurrent.futures.Futures may resolve on any thread, so we
# need to route them back to the IOLoop.
io_loop.add_future(
future, lambda future: io_loop.remove_timeout(timeout_handle))
return result
def github_request(self, path, callback, access_token=None,
method="GET", body=None, **args):
"""Fetches the given relative API path, e.g., "/user/starred"
Example usage::
class MainHandler(tornado.web.RequestHandler, torngithub.GithubMixin):
@tornado.web.authenticated
@tornado.web.asynchronous
def get(self):
self.github_request(
"/user/starred",
callback=_on_get_user_starred,
access_token=self.current_user["access_token"])
def _on_get_user_starred(self, stars):
self.write(str(stars))
self.finish()
"""
chain_future(github_request(self.get_auth_http_client(),
path, None, access_token,
method, body, **args), callback)
def as_future(self, query):
if not self._pool:
self._pool = ThreadPoolExecutor(max_workers=self._max_workers)
old_future = self._pool.submit(query)
new_future = Future()
IOLoop.current().add_future(old_future,
lambda f: chain_future(f, new_future))
return new_future
def with_timeout(timeout, future, io_loop=None):
"""Wraps a `.Future` in a timeout.
"""
result = Future()
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
timeout_handle = io_loop.add_timeout(
timeout,
lambda: result.set_exception(HostConnectionTimeout("Timeout")))
if isinstance(future, Future):
# We know this future will resolve on the IOLoop, so we don't
# need the extra thread-safety of IOLoop.add_future (and we also
# don't care about StackContext here.
future.add_done_callback(
lambda future: io_loop.remove_timeout(timeout_handle))
else:
# concurrent.futures.Futures may resolve on any thread, so we
# need to route them back to the IOLoop.
io_loop.add_future(
future, lambda future: io_loop.remove_timeout(timeout_handle))
return result
def github_request(self, path, callback, access_token=None,
method="GET", body=None, **args):
"""Fetches the given relative API path, e.g., "/user/starred"
Example usage::
class MainHandler(tornado.web.RequestHandler, torngithub.GithubMixin):
@tornado.web.authenticated
@tornado.web.asynchronous
def get(self):
self.github_request(
"/user/starred",
callback=_on_get_user_starred,
access_token=self.current_user["access_token"])
def _on_get_user_starred(self, stars):
self.write(str(stars))
self.finish()
"""
chain_future(github_request(self.get_auth_http_client(),
path, None, access_token,
method, body, **args), callback)
def with_timeout(timeout, future, io_loop=None):
"""Wraps a `.Future` in a timeout.
"""
result = Future()
chain_future(future, result)
if io_loop is None:
io_loop = IOLoop.current()
timeout_handle = io_loop.add_timeout(
timeout,
lambda: result.set_exception(HostConnectionTimeout("Timeout")))
if isinstance(future, Future):
# We know this future will resolve on the IOLoop, so we don't
# need the extra thread-safety of IOLoop.add_future (and we also
# don't care about StackContext here.
future.add_done_callback(
lambda future: io_loop.remove_timeout(timeout_handle))
else:
# concurrent.futures.Futures may resolve on any thread, so we
# need to route them back to the IOLoop.
io_loop.add_future(
future, lambda future: io_loop.remove_timeout(timeout_handle))
return result
def watch(self, url_path, on_data, **kwargs):
class WatchFuture(Future):
def cancel(self):
client.close()
logging.debug("AsyncHTTPClient closed")
def data_callback(data):
on_data(json.loads(data))
params = self.build_params(url_path, **kwargs)
url = url_concat(self.build_url(url_path, **kwargs), params)
request = HTTPRequest(url=url,
method="GET",
headers=self.build_headers(),
request_timeout=3600,
streaming_callback=data_callback)
client = AsyncHTTPClient(force_instance=True)
future = WatchFuture()
chain_future(client.fetch(request), future)
return future
def as_future(self, query: Callable) -> Future:
# concurrent.futures.Future is not compatible with the "new style"
# asyncio Future, and awaiting on such "old-style" futures does not
# work.
#
# tornado includes a `run_in_executor` function to help with this
# problem, but it's only included in version 5+. Hence, we copy a
# little bit of code here to handle this incompatibility.
if not self._pool:
self._pool = ThreadPoolExecutor(max_workers=self._max_workers)
old_future = self._pool.submit(query)
new_future = Future() # type: Future
IOLoop.current().add_future(old_future,
lambda f: chain_future(f, new_future))
return new_future
def run_in_executor(self, executor, func, *args):
"""Runs a function in a ``concurrent.futures.Executor``. If
``executor`` is ``None``, the IO loop's default executor will be used.
Use `functools.partial` to pass keyword arguments to `func`.
"""
if ThreadPoolExecutor is None:
raise RuntimeError(
"concurrent.futures is required to use IOLoop.run_in_executor")
if executor is None:
if not hasattr(self, '_executor'):
from tornado.process import cpu_count
self._executor = ThreadPoolExecutor(max_workers=(cpu_count() * 5))
executor = self._executor
c_future = executor.submit(func, *args)
# Concurrent Futures are not usable with await. Wrap this in a
# Tornado Future instead, using self.add_future for thread-safety.
t_future = TracebackFuture()
self.add_future(c_future, lambda f: chain_future(f, t_future))
return t_future
def run_in_executor(self, executor, func, *args):
"""Runs a function in a ``concurrent.futures.Executor``. If
``executor`` is ``None``, the IO loop's default executor will be used.
Use `functools.partial` to pass keyword arguments to ``func``.
.. versionadded:: 5.0
"""
if ThreadPoolExecutor is None:
raise RuntimeError(
"concurrent.futures is required to use IOLoop.run_in_executor")
if executor is None:
if not hasattr(self, '_executor'):
from tornado.process import cpu_count
self._executor = ThreadPoolExecutor(max_workers=(cpu_count() * 5))
executor = self._executor
c_future = executor.submit(func, *args)
# Concurrent Futures are not usable with await. Wrap this in a
# Tornado Future instead, using self.add_future for thread-safety.
t_future = Future()
self.add_future(c_future, lambda f: chain_future(f, t_future))
return t_future
async def launch(self, kube):
"""Ask JupyterHub to launch the image."""
# check quota first
quota = self.settings.get('per_repo_quota')
# the image name (without tag) is unique per repo
# use this to count the number of pods running with a given repo
# if we added annotations/labels with the repo name via KubeSpawner
# we could do this better
image_no_tag = self.image_name.rsplit(':', 1)[0]
matching_pods = 0
total_pods = 0
# TODO: run a watch to keep this up to date in the background
pool = self.settings['build_pool']
f = pool.submit(
kube.list_namespaced_pod,
self.settings["build_namespace"],
label_selector='app=jupyterhub,component=singleuser-server',
)
# concurrent.futures.Future isn't awaitable
# wrap in tornado Future
# tornado 5 will have `.run_in_executor`
tf = Future()
chain_future(f, tf)
pods = await tf
for pod in pods.items:
total_pods += 1
for container in pod.spec.containers:
# is the container running the same image as us?
# if so, count one for the current repo.
image = container.image.rsplit(':', 1)[0]
if image == image_no_tag:
matching_pods += 1
break
# TODO: allow whitelist of repos to exceed quota
# TODO: put busy users in a queue rather than fail?
# That would be hard to do without in-memory state.
if quota and matching_pods >= quota:
app_log.error("%s has exceeded quota: %s/%s (%s total)", self.repo,
matching_pods, quota, total_pods)
await self.fail("Too many users running %s! Try again soon." %
self.repo)
return
if quota and matching_pods >= 0.5 * quota:
log = app_log.warning
else:
log = app_log.info
log("Launching pod for %s: %s other pods running this repo (%s total)",
self.repo, matching_pods, total_pods)
await self.emit({
'phase': 'launching',
'message': 'Launching server...\n',
})
launcher = self.settings['launcher']
username = launcher.username_from_repo(self.repo)
try:
launch_starttime = time.perf_counter()
server_info = await launcher.launch(image=self.image_name,
username=username)
LAUNCH_TIME.labels(
status='success',
**self.metric_labels).observe(time.perf_counter() -
launch_starttime)
except Exception:
LAUNCH_TIME.labels(
status='failure',
**self.metric_labels).observe(time.perf_counter() -
launch_starttime)
raise
event = {
'phase': 'ready',
'message': 'server running at %s\n' % server_info['url'],
}
event.update(server_info)
await self.emit(event)
def migrate_context():
connection_future = super(CocaineTCPClient, self).connect(host, port)
chain_future(connection_future, result_future)
def _cb():
chain_future(self._executor.submit(fn, *args, **kwargs), future)
async def launch(self, kube, provider):
"""Ask JupyterHub to launch the image."""
# Load the spec-specific configuration if it has been overridden
repo_config = provider.repo_config(self.settings)
# the image name (without tag) is unique per repo
# use this to count the number of pods running with a given repo
# if we added annotations/labels with the repo name via KubeSpawner
# we could do this better
image_no_tag = self.image_name.rsplit(':', 1)[0]
matching_pods = 0
total_pods = 0
# TODO: run a watch to keep this up to date in the background
pool = self.settings['executor']
f = pool.submit(kube.list_namespaced_pod,
self.settings["build_namespace"],
label_selector='app=jupyterhub,component=singleuser-server',
)
# concurrent.futures.Future isn't awaitable
# wrap in tornado Future
# tornado 5 will have `.run_in_executor`
tf = Future()
chain_future(f, tf)
pods = await tf
for pod in pods.items:
total_pods += 1
for container in pod.spec.containers:
# is the container running the same image as us?
# if so, count one for the current repo.
image = container.image.rsplit(':', 1)[0]
if image == image_no_tag:
matching_pods += 1
break
# TODO: put busy users in a queue rather than fail?
# That would be hard to do without in-memory state.
quota = repo_config.get('quota')
if quota and matching_pods >= quota:
app_log.error("%s has exceeded quota: %s/%s (%s total)",
self.repo_url, matching_pods, quota, total_pods)
await self.fail("Too many users running %s! Try again soon." % self.repo_url)
return
if quota and matching_pods >= 0.5 * quota:
log = app_log.warning
else:
log = app_log.info
log("Launching pod for %s: %s other pods running this repo (%s total)",
self.repo_url, matching_pods, total_pods)
await self.emit({
'phase': 'launching',
'message': 'Launching server...\n',
})
launcher = self.settings['launcher']
retry_delay = launcher.retry_delay
for i in range(launcher.retries):
launch_starttime = time.perf_counter()
if self.settings['auth_enabled']:
# get logged in user's name
user_model = self.hub_auth.get_user(self)
username = user_model['name']
if launcher.allow_named_servers:
# user can launch multiple servers, so create a unique server name
server_name = launcher.unique_name_from_repo(self.repo_url)
else:
server_name = ''
else:
# create a name for temporary user
username = launcher.unique_name_from_repo(self.repo_url)
server_name = ''
try:
extra_args = {
'binder_ref_url': self.ref_url,
'binder_launch_host': self.binder_launch_host,
'binder_request': self.binder_request,
'binder_persistent_request': self.binder_persistent_request,
}
server_info = await launcher.launch(image=self.image_name,
username=username,
server_name=server_name,
repo_url=self.repo_url,
extra_args=extra_args)
except Exception as e:
duration = time.perf_counter() - launch_starttime
if i + 1 == launcher.retries:
status = 'failure'
else:
status = 'retry'
# don't count retries in failure/retry
# retry count is only interesting in success
LAUNCH_TIME.labels(
status=status, retries=-1,
).observe(time.perf_counter() - launch_starttime)
if status == 'failure':
# don't count retries per repo
LAUNCH_COUNT.labels(
status=status, **self.repo_metric_labels,
).inc()
if i + 1 == launcher.retries:
# last attempt failed, let it raise
raise
# not the last attempt, try again
app_log.error(
"Retrying launch of %s after error (duration=%.0fs, attempt=%s): %r",
self.repo_url,
duration,
i + 1,
e,
)
await self.emit(
{
"phase": "launching",
"message": "Launch attempt {} failed, retrying...\n".format(
i + 1
),
}
)
await gen.sleep(retry_delay)
# exponential backoff for consecutive failures
retry_delay *= 2
continue
else:
# success
duration = time.perf_counter() - launch_starttime
LAUNCH_TIME.labels(status="success", retries=i).observe(duration)
LAUNCH_COUNT.labels(
status='success', **self.repo_metric_labels,
).inc()
app_log.info("Launched %s in %.0fs", self.repo_url, duration)
break
event = {
'phase': 'ready',
'message': 'server running at %s\n' % server_info['url'],
}
event.update(server_info)
await self.emit(event)
def with_timeout(
timeout: Union[float, datetime.timedelta],
future: _Yieldable,
quiet_exceptions: "Union[Type[Exception], Tuple[Type[Exception], ...]]" = (
),
) -> Future:
"""Wraps a `.Future` (or other yieldable object) in a timeout.
Raises `tornado.util.TimeoutError` if the input future does not
complete before ``timeout``, which may be specified in any form
allowed by `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or
an absolute time relative to `.IOLoop.time`)
If the wrapped `.Future` fails after it has timed out, the exception
will be logged unless it is either of a type contained in
``quiet_exceptions`` (which may be an exception type or a sequence of
types), or an ``asyncio.CancelledError``.
The wrapped `.Future` is not canceled when the timeout expires,
permitting it to be reused. `asyncio.wait_for` is similar to this
function but it does cancel the wrapped `.Future` on timeout.
.. versionadded:: 4.0
.. versionchanged:: 4.1
Added the ``quiet_exceptions`` argument and the logging of unhandled
exceptions.
.. versionchanged:: 4.4
Added support for yieldable objects other than `.Future`.
.. versionchanged:: 6.0.3
``asyncio.CancelledError`` is now always considered "quiet".
"""
# It's tempting to optimize this by cancelling the input future on timeout
# instead of creating a new one, but A) we can't know if we are the only
# one waiting on the input future, so cancelling it might disrupt other
# callers and B) concurrent futures can only be cancelled while they are
# in the queue, so cancellation cannot reliably bound our waiting time.
future_converted = convert_yielded(future)
result = _create_future()
chain_future(future_converted, result)
io_loop = IOLoop.current()
def error_callback(future: Future) -> None:
try:
future.result()
except asyncio.CancelledError:
pass
except Exception as e:
if not isinstance(e, quiet_exceptions):
app_log.error("Exception in Future %r after timeout",
future,
exc_info=True)
def timeout_callback() -> None:
if not result.done():
result.set_exception(TimeoutError("Timeout"))
# In case the wrapped future goes on to fail, log it.
future_add_done_callback(future_converted, error_callback)
timeout_handle = io_loop.add_timeout(timeout, timeout_callback)
if isinstance(future_converted, Future):
# We know this future will resolve on the IOLoop, so we don't
# need the extra thread-safety of IOLoop.add_future (and we also
# don't care about StackContext here.
future_add_done_callback(
future_converted,
lambda future: io_loop.remove_timeout(timeout_handle))
else:
# concurrent.futures.Futures may resolve on any thread, so we
# need to route them back to the IOLoop.
io_loop.add_future(
future_converted,
lambda future: io_loop.remove_timeout(timeout_handle))
return result
def with_timeout(
timeout: Union[float, datetime.timedelta],
future: _Yieldable,
quiet_exceptions: Union[Type[Exception], Tuple[Type[Exception], ...]] = (),
) -> Future:
"""Wraps a `.Future` (or other yieldable object) in a timeout.
Raises `tornado.util.TimeoutError` if the input future does not
complete before ``timeout``, which may be specified in any form
allowed by `.IOLoop.add_timeout` (i.e. a `datetime.timedelta` or
an absolute time relative to `.IOLoop.time`)
If the wrapped `.Future` fails after it has timed out, the exception
will be logged unless it is of a type contained in ``quiet_exceptions``
(which may be an exception type or a sequence of types).
The wrapped `.Future` is not canceled when the timeout expires,
permitting it to be reused. `asyncio.wait_for` is similar to this
function but it does cancel the wrapped `.Future` on timeout.
.. versionadded:: 4.0
.. versionchanged:: 4.1
Added the ``quiet_exceptions`` argument and the logging of unhandled
exceptions.
.. versionchanged:: 4.4
Added support for yieldable objects other than `.Future`.
"""
# It's tempting to optimize this by cancelling the input future on timeout
# instead of creating a new one, but A) we can't know if we are the only
# one waiting on the input future, so cancelling it might disrupt other
# callers and B) concurrent futures can only be cancelled while they are
# in the queue, so cancellation cannot reliably bound our waiting time.
future_converted = convert_yielded(future)
result = _create_future()
chain_future(future_converted, result)
io_loop = IOLoop.current()
def error_callback(future: Future) -> None:
try:
future.result()
except Exception as e:
if not isinstance(e, quiet_exceptions):
app_log.error(
"Exception in Future %r after timeout", future, exc_info=True
)
def timeout_callback() -> None:
if not result.done():
result.set_exception(TimeoutError("Timeout"))
# In case the wrapped future goes on to fail, log it.
future_add_done_callback(future_converted, error_callback)
timeout_handle = io_loop.add_timeout(timeout, timeout_callback)
if isinstance(future_converted, Future):
# We know this future will resolve on the IOLoop, so we don't
# need the extra thread-safety of IOLoop.add_future (and we also
# don't care about StackContext here.
future_add_done_callback(
future_converted, lambda future: io_loop.remove_timeout(timeout_handle)
)
else:
# concurrent.futures.Futures may resolve on any thread, so we
# need to route them back to the IOLoop.
io_loop.add_future(
future_converted, lambda future: io_loop.remove_timeout(timeout_handle)
)
return result
