def wrapper(*args, **kwargs):
future = TracebackFuture()
if replace_callback and 'callback' in kwargs:
# 调用的参数有callback，不实用future使用异步callback
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
try:
# 调用实际的处理函数如果返回GeneratorType，那么进入Runner类循环执行直到终止
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = _value_from_stopiteration(e)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
# 生成器
if isinstance(result, GeneratorType):
# Inline the first iteration of Runner.run. This lets us
# avoid the cost of creating a Runner when the coroutine
# never actually yields, which in turn allows us to
# use "optional" coroutines in critical path code without
# performance penalty for the synchronous case.
try:
orig_stack_contexts = stack_context._state.contexts
# 返回第一个Future，异步调用返回的Future，
# 在异步调用成功后会将这个Future给set_done
yielded = next(result)
if stack_context._state.contexts is not orig_stack_contexts:
yielded = TracebackFuture()
yielded.set_exception(
stack_context.StackContextInconsistentError(
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)')
)
except (StopIteration, Return) as e:
future.set_result(_value_from_stopiteration(e))
except Exception:
future.set_exc_info(sys.exc_info())
else:
Runner(result, future, yielded)
try:
# 进入Runner，返回
return future
finally:
# Subtle memory optimization: if next() raised an exception,
# the future's exc_info contains a traceback which
# includes this stack frame. This creates a cycle,
# which will be collected at the next full GC but has
# been shown to greatly increase memory usage of
# benchmarks (relative to the refcount-based scheme
# used in the absence of cycles). We can avoid the
# cycle by clearing the local variable after we return it.
future = None
future.set_result(result)
return future
def handle_yield(self, yielded):
# Lists containing YieldPoints require stack contexts;
# other lists are handled in convert_yielded.
if _contains_yieldpoint(yielded):
yielded = multi(yielded)
if isinstance(yielded, YieldPoint):
# YieldPoints are too closely coupled to the Runner to go
# through the generic convert_yielded mechanism.
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
else:
try:
self.future = convert_yielded(yielded)
except BadYieldError:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if not self.future.done() or self.future is moment:
def inner(f):
# Break a reference cycle to speed GC.
f = None # noqa
self.run()
self.io_loop.add_future(self.future, inner)
return False
return True
def handle_yield(self, yielded):
# Lists containing YieldPoints require stack contexts;
# other lists are handled via multi_future in convert_yielded.
if (isinstance(yielded, list) and
any(isinstance(f, YieldPoint) for f in yielded)):
yielded = Multi(yielded)
elif (isinstance(yielded, dict) and
any(isinstance(f, YieldPoint) for f in yielded.values())):
yielded = Multi(yielded)
if isinstance(yielded, YieldPoint):
# YieldPoints are too closely coupled to the Runner to go
# through the generic convert_yielded mechanism.
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(
yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
else:
try:
self.future = convert_yielded(yielded)
except BadYieldError:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if not self.future.done() or self.future is moment:
self.io_loop.add_future(
self.future, lambda f: self.run())
return False
return True
def run():
try:
result = func()
except Exception:
future_cell[0] = TracebackFuture()
future_cell[0].set_exc_info(sys.exc_info())
else:
if is_future(result):
future_cell[0] = result
else:
future_cell[0] = TracebackFuture()
future_cell[0].set_result(result)
self.add_future(future_cell[0], lambda future: self.stop())
def handle_yield(self, yielded):
# 返回True已经完成，False表示Future未完成，添加到ioloop中
# Lists containing YieldPoints require stack contexts;
# other lists are handled in convert_yielded.
# 处理 yield list or dict 情况
if _contains_yieldpoint(yielded):
yielded = multi(yielded)
if isinstance(yielded, YieldPoint):
# YieldPoints are too closely coupled to the Runner to go
# through the generic convert_yielded mechanism.
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
else:
# Future情况，为r = yield fetch()之类返回的Future
try:
self.future = convert_yielded(yielded)
except BadYieldError:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if not self.future.done() or self.future is moment:
self.io_loop.add_future(self.future, lambda f: self.run())
return False
return True
def handle_yield(self, yielded):
if isinstance(yielded, list):
if all(is_future(f) for f in yielded):
yielded = multi_future(yielded)
else:
yielded = Multi(yielded)
elif isinstance(yielded, dict):
if all(is_future(f) for f in yielded.values()):
yielded = multi_future(yielded)
else:
yielded = Multi(yielded)
if isinstance(yielded, YieldPoint):
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
elif is_future(yielded):
self.future = yielded
if not self.future.done() or self.future is moment:
self.io_loop.add_future(self.future, lambda f: self.run())
return False
else:
self.future = TracebackFuture()
self.future.set_exception(
BadYieldError("yielded unknown object %r" % (yielded, )))
return True
def mock_fetch(request, callback=None, raise_error=True, **kwargs):
"""mock 掉RESTfulAsyncClient的fetch函数，使之不再发起向外请求
并且响应始终是固定的
"""
future = TracebackFuture()
if callback is not None:
def handle_future(future):
"""future的回调，单元测试里一般会挂上self.top"""
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def immediately_done():
"""立即完成future"""
buf = StringIO()
buf.write(
json.dumps({ # 思索。。这里好像只能写死，要传参的话得在_request上再打一层
"ok": True,
"mock": True,
'user': {
"id": "1",
"username": "******",
}
}))
response = HTTPResponse(request, 200, buffer=buf)
future.set_result(response)
self.io_loop.add_callback(immediately_done)
return future
def method(self, *args, **kwargs):
#loop = self.get_io_loop()
loop = ioloop.IOLoop.current()
callback = kwargs.pop('callback', None)
if callback:
if not callable(callback):
raise callback_type_error
future = None
else:
future = TracebackFuture()
def call_method():
# Runs on child greenlet.
try:
result = sync_method(self, *args, **kwargs) # 用fun传对象调用方法
if callback:
# Schedule callback(result, None) on main greenlet.
loop.add_callback(functools.partial(
callback, result, None))
else:
# Schedule future to be resolved on main greenlet.
loop.add_callback(functools.partial(
future.set_result, result))
except Exception as e:
if callback:
loop.add_callback(functools.partial(
callback, None, e))
else:
loop.add_callback(functools.partial(
future.set_exc_info, sys.exc_info()))
# Start running the operation on a greenlet.
greenlet.greenlet(call_method).switch()
return future
def find_one(self, **kwargs):
"""Returns future.
Executes collection's find_one method based on keyword args
maps result ( dict to instance ) and return future
Example::
manager = EntityManager(Product)
product_saved = yield manager.find_one(_id=object_id)
"""
future = TracebackFuture()
def handle_response(result, error):
if error:
future.set_exception(error)
else:
instance = self.__entity()
instance.map_dict(result)
future.set_result(instance)
self.__collection.find_one(kwargs, callback=handle_response)
return future
def start_tls(self, server_side, ssl_options, server_hostname=None):
if not isinstance(ssl_options, SSL.Context):
raise ValueError("ssl_options is not SSL.Context")
_socket = self.detach()
_socket = SSL.Connection(ssl_options, _socket)
if server_side:
_socket.set_accept_state()
else:
_socket.set_connect_state()
if server_hostname:
_socket.set_tlsext_host_name(server_hostname.encode("idna"))
orig_close_callback = self._close_callback
self._close_callback = None
future = TracebackFuture()
ssl_stream = MicroProxySSLIOStream(_socket,
server_hostname=server_hostname,
ssl_options=ssl_options,
io_loop=self.io_loop)
def close_callback():
if not future.done():
future.set_exception(ssl_stream.error or StreamClosedError())
if orig_close_callback is not None:
orig_close_callback()
ssl_stream.set_close_callback(close_callback)
ssl_stream._ssl_connect_callback = lambda: future.set_result(ssl_stream)
ssl_stream.max_buffer_size = self.max_buffer_size
ssl_stream.read_chunk_size = self.read_chunk_size
return future
def coroutine_run():
"""
执行协程
"""
try:
result = consume_mq()
except Exception:
future = TracebackFuture()
future.set_exc_info(sys.exc_info())
else:
assert isinstance(result, Future)
future = result
def consumer_finish(f):
"""
重新启动ioloop回调
:param f:
"""
if f.exc_info() is not None:
logger.warn(f.exc_info())
elif f.exception() is not None:
logger.warn(f.exception())
IOLoop.current().add_callback(coroutine_run)
IOLoop.current().add_future(future, consumer_finish)
def run():
try:
result = func()
if result is not None:
from tornado.gen import convert_yielded
result = convert_yielded(result)
except Exception:
future_cell[0] = TracebackFuture()
future_cell[0].set_exc_info(sys.exc_info())
else:
if is_future(result):
future_cell[0] = result
else:
future_cell[0] = TracebackFuture()
future_cell[0].set_result(result)
self.add_future(future_cell[0], lambda future: self.stop())
def __init__(self, io_loop, request, on_message_callback=None,
compression_options=None):
self.compression_options = compression_options
self.connect_future = TracebackFuture()
self.protocol = None
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
self._on_message_callback = on_message_callback
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
if self.compression_options is not None:
# Always offer to let the server set our max_wbits (and even though
# we don't offer it, we will accept a client_no_context_takeover
# from the server).
# TODO: set server parameters for deflate extension
# if requested in self.compression_options.
request.headers['Sec-WebSocket-Extensions'] = (
'permessage-deflate; client_max_window_bits')
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection, self).__init__(
io_loop, None, request, lambda: None, self._on_http_response,
104857600, self.tcp_client, 65536)
def send_request(self, request):
method = request.method
data = request.body
headers = {'Content-Type': 'application/json'}
future = TracebackFuture()
def process_response_future(response):
if response.exc_info() is not None:
future.set_exc_info(response.exc_info())
elif response.exception() is not None:
future.set_exception(response.exception())
else:
result = response.result()
code = result.code
body = (result.body or b'').decode('utf8')
future.set_result(process_response(code, body))
request = tornado.httpclient.HTTPRequest(
request.url,
method=method,
body=data,
headers=headers,
request_timeout=self.client.timeout)
response_future = self.http.fetch(request, raise_error=False)
response_future.add_done_callback(process_response_future)
return future
def http_retry(client,
request,
raise_error=True,
attempts=5,
retry_wait=1,
**kwargs):
attempt = 1
future = TracebackFuture()
ioloop = IOLoop.current()
def _do_request(attempt):
http_future = client.fetch(request, raise_error=False, **kwargs)
http_future.add_done_callback(partial(handle_response, attempt))
def handle_response(attempt, future_response):
attempt += 1
result = future_response.result()
if result.error and\
attempt <= 5 and\
result.code >= 500 and\
result.code <= 599:
logging.error(result.error)
logging.error(result.body)
return ioloop.call_later(retry_wait, lambda: _do_request(attempt))
else:
if raise_error and result.error:
return future.set_exception(result.error)
future.set_result(result)
_do_request(attempt)
return future
def invoke(self, *args):
"""异步调用redis相关接口
:param args: 多条redis指令
"""
#默认开启multi
active_trans = True
write_buf = _chain_cmds(active_trans, *args)
future = TracebackFuture()
def handle_resp(resp):
f = resp.get(_RESP_FUTURE) or future
err = resp.get(RESP_ERR)
result = resp.get(RESP_RESULT)
if err:
f.set_exception(err)
else:
f.set_result(result)
with NullContext():
if self.__conn is None:
self.__conn = _RedisConnection(self.__io_loop, write_buf, handle_resp, self.__uri, self.__password)
self.__conn.connect(future, self.__uri, active_trans, len(args))
else:
self.__conn.write(write_buf, future, False, active_trans, len(args))
return future
def write(self, data, callback=None):
"""Write the given data to this stream.
If ``callback`` is given, we call it when all of the buffered write
data has been successfully written to the stream. If there was
previously buffered write data and an old write callback, that
callback is simply overwritten with this new callback.
"""
assert isinstance(data, bytes_type)
self._check_closed()
# We use bool(_write_buffer) as a proxy for write_buffer_size>0,
# so never put empty strings in the buffer.
if data:
# Break up large contiguous strings before inserting them in the
# write buffer, so we don't have to recopy the entire thing
# as we slice off pieces to send to the socket.
WRITE_BUFFER_CHUNK_SIZE = 128 * 1024
for i in range(0, len(data), WRITE_BUFFER_CHUNK_SIZE):
self._write_buffer.append(data[i:i + WRITE_BUFFER_CHUNK_SIZE])
if callback is not None:
self._write_callback = stack_context.wrap(callback)
future = None
else:
future = self._write_future = TracebackFuture()
if not self._connecting:
self._handle_write()
if self._write_buffer:
self._add_io_state(self.io_loop.WRITE)
self._maybe_add_error_listener()
return future
def open(self):
try:
future = self._stream.open()
except:
exc_info = sys.exc_info()
future = TracebackFuture()
future.set_exc_info(exc_info)
return future
def _set_read_callback(self, callback):
assert self._read_callback is None, "Already reading"
assert self._read_future is None, "Already reading"
if callback is not None:
self._read_callback = stack_context.wrap(callback)
else:
self._read_future = TracebackFuture()
return self._read_future
def handle_exception(self, typ, value, tb):
if not self.running and not self.finished:
self.future = TracebackFuture()
self.future.set_exc_info((typ, value, tb))
self.run()
return True
else:
return False
def submit(self, fn, *args, **kwargs):
future = TracebackFuture()
def _cb():
chain_future(self._executor.submit(fn, *args, **kwargs), future)
IOLoop.current().add_callback(_cb)
return future
def get_future(self):
"""
Get future and start timeout task when needed
"""
if not self.future or self.future.done():
self.future = TracebackFuture()
self.start_timeout()
return self.future
def fetch(self, request, callback=None, raise_error=True, **kwargs):
"""Executes a request, asynchronously returning an `HTTPResponse`.
The request may be either a string URL or an `HTTPRequest` object.
If it is a string, we construct an `HTTPRequest` using any additional
kwargs: ``HTTPRequest(request, **kwargs)``
This method returns a `.Future` whose result is an
`HTTPResponse`. By default, the ``Future`` will raise an
`HTTPError` if the request returned a non-200 response code
(other errors may also be raised if the server could not be
contacted). Instead, if ``raise_error`` is set to False, the
response will always be returned regardless of the response
code.
If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
In the callback interface, `HTTPError` is not automatically raised.
Instead, you must check the response's ``error`` attribute or
call its `~HTTPResponse.rethrow` method.
"""
if self._closed:
raise RuntimeError("fetch() called on closed AsyncHTTPClient")
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
# We may modify this (to add Host, Accept-Encoding, etc),
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
request.headers = httputil.HTTPHeaders(request.headers)
request = _RequestProxy(request, self.defaults)
future = TracebackFuture()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
exc = future.exception()
if isinstance(exc, HTTPError) and exc.response is not None:
response = exc.response
elif exc is not None:
response = HTTPResponse(request,
599,
error=exc,
request_time=time.time() -
request.start_time)
else:
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def handle_response(response):
if raise_error and response.error:
future.set_exception(response.error)
else:
future.set_result(response)
self.fetch_impl(request, handle_response)
return future
def async (task, *args, **kwargs):
future = TracebackFuture()
callback = kwargs.pop("callback", None)
if callback:
IOLoop.instance().add_future(future,
lambda future: callback(future.result()))
result = task.delay(*args, **kwargs)
IOLoop.instance().add_callback(_on_result, result, future)
return future
def fetch(self, request, callback=None, **kwargs):
"""Executes a request, asynchronously returning an `HTTPResponse`.
The request may be either a string URL or an `HTTPRequest` object.
If it is a string, we construct an `HTTPRequest` using any additional
kwargs: ``HTTPRequest(request, **kwargs)``
# 注意，任何状态码不是200的都会抛出异常，需要自己捕获或者添加一个callback函数
This method returns a `.Future` whose result is an
`HTTPResponse`. The ``Future`` will raise an `HTTPError` if
the request returned a non-200 response code.
如果添加了callback，那么不会有一个自动抛出的异常
If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
In the callback interface, `HTTPError` is not automatically raised.
Instead, you must check the response's ``error`` attribute or
call its `~HTTPResponse.rethrow` method.
"""
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
# We may modify this (to add Host, Accept-Encoding, etc),
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
# 构建headers，封装的一个类，处理起来比较方便
request.headers = httputil.HTTPHeaders(request.headers)
# request.request, request.defaults
request = _RequestProxy(request, self.defaults)
future = TracebackFuture()
# 这个callback先放过
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
exc = future.exception()
if isinstance(exc, HTTPError) and exc.response is not None:
response = exc.response
elif exc is not None:
response = HTTPResponse(
request, 599, error=exc,
request_time=time.time() - request.start_time)
else:
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def handle_response(response):
if response.error:
future.set_exception(response.error)
else:
future.set_result(response)
# rquest异步成功之后执行 handle_response(response)
self.fetch_impl(request, handle_response)
# 返回一个future类型
return future
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
def __init__(self,
io_loop,
request,
on_message_callback=None,
compression_options=None):
del request, on_message_callback, compression_options
self.connect_future = TracebackFuture()
self.written_messages = []
io_loop.add_timeout(self.TIMEOUT, self._complete)
def __init__(self, io_loop, request):
self.connect_future = TracebackFuture()
self.read_future = None
self.read_queue = collections.deque()
self.events = []
self.tcp_client = TCPClient(io_loop=io_loop)
super(EventSourceClient,
self).__init__(io_loop, None, request, lambda: None,
self._on_http_response, 104857600,
self.tcp_client, 65536)
def read(self, callback=None):
assert not self.reading(), 'Already reading'
self.bytes_to_read = None
if callback is not None:
self._read_callback = stack_context.wrap(callback)
future = None
else:
future = self._read_callback = TracebackFuture()
future.add_done_callback(lambda f: f.exception())
return future
def handle_yield(self, yielded):
try:
self.future = convert_yielded(yielded)
except BadYieldError:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if not self.future.done() or self.future is moment:
self.io_loop.add_future(self.future, lambda f: self.run())
return False
return True
