def append(self, param, req):
tableName = param['tableName'].name
val = param['val']
if not tableName in self.tables:
raise KeyError, 'Table Not Found!'
args = {
'tableName': tableName,
'key': self.tables[tableName]['len'],
'val': val
}
worker = self.workers[hash(self.tables[tableName]['len']) %
self.num_workers]
#print 'MASTER -> {0} Append: {1} to key {2}'.format(worker, val, self.tables[tableName]['len'])
self.client.fetch(formatQuery(worker, 'set', args))
fu = Future()
req.url = self.host
fu.set_result(
HTTPResponse(req,
200,
buffer=cStringIO.StringIO(
str(self.tables[tableName]['len']))))
self.tables[tableName]['len'] += 1
return fu
def wrapper(*args, **kwargs):
future = TracebackFuture()
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
# Inline the first iteration of Runner.run. This lets us
# avoid the cost of creating a Runner when the coroutine
# never actually yields, which in turn allows us to
# use "optional" coroutines in critical path code without
# performance penalty for the synchronous case.
try:
orig_stack_contexts = stack_context._state.contexts
yielded = next(result)
if stack_context._state.contexts is not orig_stack_contexts:
yielded = TracebackFuture()
yielded.set_exception(
stack_context.StackContextInconsistentError(
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)'))
except (StopIteration, Return) as e:
future.set_result(getattr(e, 'value', None))
except Exception:
future.set_exc_info(sys.exc_info())
else:
# post runner into Cocaine ioloop
CocaineIO.instance().post(Runner, result, future, yielded)
return future
future.set_result(result)
return future
class StubWebSocketClientConnection(object):
"""A stub version of tornado.websocket.WebSocketClientConnection for use in
unit tests.
Attributes:
TIMEOUT: class constant, which should be overridden in subclasses, in
order to vary the time from when the class is instantiated until the
returned future is resolved.
connect_future: the Future resolved when the client "connection" is
made.
written_messages: a list of messages passed to the write_message
method to see they have been received.
"""
TIMEOUT = datetime.timedelta(milliseconds=0)
def __init__(self,
io_loop,
request,
on_message_callback=None,
compression_options=None):
del request, on_message_callback, compression_options
self.connect_future = TracebackFuture()
self.written_messages = []
io_loop.add_timeout(self.TIMEOUT, self._complete)
def _complete(self):
self.connect_future.set_result(self)
def write_message(self, message):
"""Adds writen message to the written_messages buffer."""
self.written_messages.append(message)
def create(self, param, req):
tableName = param['tableName'].name
initVal = param['initVal']
if tableName in self.tables:
raise KeyError, 'Table name already existed'
#print 'MASTER CREATE:', initVal
self.tables[tableName] = {
'len': 0,
}
#Data Partition
vals = {}
for worker_id in xrange(self.num_workers):
vals[worker_id] = {}
for key in initVal:
#Bookkeeping
self.tables[tableName]['len'] += 1
self.tables[tableName][key] = type(initVal[key])
worker = hash(key) % self.num_workers
vals[worker][key] = initVal[key]
futures = []
for worker_id in xrange(self.num_workers):
args = {'tableName': tableName, 'initVal': vals[worker_id]}
futures.append(
self.client.fetch(
formatQuery(self.workers[worker_id], 'create', args)))
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
class StubWebSocketClientConnection(object):
"""A stub version of tornado.websocket.WebSocketClientConnection for use in
unit tests.
Attributes:
TIMEOUT: class constant, which should be overridden in subclasses, in
order to vary the time from when the class is instantiated until the
returned future is resolved.
connect_future: the Future resolved when the client "connection" is
made.
written_messages: a list of messages passed to the write_message
method to see they have been received.
"""
TIMEOUT = datetime.timedelta(milliseconds=0)
def __init__(self, io_loop, request, on_message_callback=None,
compression_options=None):
del request, on_message_callback, compression_options
self.connect_future = TracebackFuture()
self.written_messages = []
io_loop.add_timeout(self.TIMEOUT, self._complete)
def _complete(self):
self.connect_future.set_result(self)
def write_message(self, message):
"""Adds writen message to the written_messages buffer."""
self.written_messages.append(message)
def create(self, param, req):
tableName = param['tableName'].name
initVal = param['initVal']
if tableName in self.tables:
raise KeyError, 'Table name already existed'
#print 'MASTER CREATE:', initVal
self.tables[tableName] = {'len': 0, }
#Data Partition
vals = {}
for worker_id in xrange(self.num_workers):
vals[worker_id] = {}
for key in initVal:
#Bookkeeping
self.tables[tableName]['len'] += 1
self.tables[tableName][key] = type(initVal[key])
worker = hash(key) % self.num_workers
vals[worker][key] = initVal[key]
futures = []
for worker_id in xrange(self.num_workers):
args = {'tableName': tableName, 'initVal': vals[worker_id]}
futures.append(self.client.fetch(formatQuery(self.workers[worker_id], 'create', args)))
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def wrapper(*args, **kwargs):
runner = None
future = TracebackFuture()
if 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
def handle_exception(typ, value, tb):
try:
if runner is not None and runner.handle_exception(typ, value, tb):
return True
except Exception:
typ, value, tb = sys.exc_info()
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception):
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
def final_callback(value):
future.set_result(value)
runner = Runner(result, final_callback)
runner.run()
return future
future.set_result(result)
return future
def wrapper(*args, **kwargs):
future = TracebackFuture()
if replace_callback and 'callback' in kwargs:
# 调用的参数有callback，不实用future使用异步callback
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
try:
# 调用实际的处理函数如果返回GeneratorType，那么进入Runner类循环执行直到终止
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = _value_from_stopiteration(e)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
# 生成器
if isinstance(result, GeneratorType):
# Inline the first iteration of Runner.run. This lets us
# avoid the cost of creating a Runner when the coroutine
# never actually yields, which in turn allows us to
# use "optional" coroutines in critical path code without
# performance penalty for the synchronous case.
try:
orig_stack_contexts = stack_context._state.contexts
# 返回第一个Future，异步调用返回的Future，
# 在异步调用成功后会将这个Future给set_done
yielded = next(result)
if stack_context._state.contexts is not orig_stack_contexts:
yielded = TracebackFuture()
yielded.set_exception(
stack_context.StackContextInconsistentError(
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)')
)
except (StopIteration, Return) as e:
future.set_result(_value_from_stopiteration(e))
except Exception:
future.set_exc_info(sys.exc_info())
else:
Runner(result, future, yielded)
try:
# 进入Runner，返回
return future
finally:
# Subtle memory optimization: if next() raised an exception,
# the future's exc_info contains a traceback which
# includes this stack frame. This creates a cycle,
# which will be collected at the next full GC but has
# been shown to greatly increase memory usage of
# benchmarks (relative to the refcount-based scheme
# used in the absence of cycles). We can avoid the
# cycle by clearing the local variable after we return it.
future = None
future.set_result(result)
return future
def get(self, param, req):
tableName = param['tableName']
key = param['key']
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200,
buffer=cStringIO.StringIO(pickle.dumps(self.tables[tableName][key]))))
return fu
def len(self, param, req):
tableName = param['tableName'].name
if not tableName in self.tables:
raise KeyError, 'Table Not Found'
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO(str(self.tables[tableName]['len']))))
return fu
def fetch_all(self, param, req):
tableName = param['tableName']
if not tableName in self.tables:
raise KeyError, 'Table Not Found!'
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO(pickle.dumps(self.tables[tableName]))))
return fu
def close(self):
if self._cursor is None:
future = TracebackFuture()
future.set_result(None)
return future
future = async_call_method(self._cursor.close)
def do_close(future):
self._cursor = None
future.add_done_callback(do_close)
return future
def get(self, param, req):
tableName = param['tableName']
key = param['key']
fu = Future()
req.url = self.host
fu.set_result(
HTTPResponse(req,
200,
buffer=cStringIO.StringIO(
pickle.dumps(self.tables[tableName][key]))))
return fu
def wrapper(*args, **kwargs):
future = TracebackFuture()
if replace_callback and "callback" in kwargs:
callback = kwargs.pop("callback")
IOLoop.current().add_future(future, lambda future: callback(future.result()))
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = _value_from_stopiteration(e)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, GeneratorType):
# Inline the first iteration of Runner.run. This lets us
# avoid the cost of creating a Runner when the coroutine
# never actually yields, which in turn allows us to
# use "optional" coroutines in critical path code without
# performance penalty for the synchronous case.
try:
orig_stack_contexts = stack_context._state.contexts
yielded = next(result)
if stack_context._state.contexts is not orig_stack_contexts:
yielded = TracebackFuture()
yielded.set_exception(
stack_context.StackContextInconsistentError(
"stack_context inconsistency (probably caused "
'by yield within a "with StackContext" block)'
)
)
except (StopIteration, Return) as e:
future.set_result(_value_from_stopiteration(e))
except Exception:
future.set_exc_info(sys.exc_info())
else:
_futures_to_runners[future] = Runner(result, future, yielded)
try:
return future
finally:
# Subtle memory optimization: if next() raised an exception,
# the future's exc_info contains a traceback which
# includes this stack frame. This creates a cycle,
# which will be collected at the next full GC but has
# been shown to greatly increase memory usage of
# benchmarks (relative to the refcount-based scheme
# used in the absence of cycles). We can avoid the
# cycle by clearing the local variable after we return it.
future = None
future.set_result(result)
return future
def set(self, param, req):
tableName = param['tableName']
key = param['key']
val = param['val']
print 'Worker {0} SET: {1} with key {2} to {3}'.format(self.host, tableName, key, val)
self.tables[tableName][key] = val
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def len(self, param, req):
tableName = param['tableName'].name
if not tableName in self.tables:
raise KeyError, 'Table Not Found'
fu = Future()
req.url = self.host
fu.set_result(
HTTPResponse(req,
200,
buffer=cStringIO.StringIO(
str(self.tables[tableName]['len']))))
return fu
def create(self, param, req):
tableName = param['tableName']
initVal = param['initVal']
self.tables[tableName] = {}
for key in initVal:
self.tables[tableName][key] = initVal[key]
print 'Worker {0} CREATE:'.format(self.host), self.tables[tableName]
#Release
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def fetch_all(self, param, req):
tableName = param['tableName']
if not tableName in self.tables:
raise KeyError, 'Table Not Found!'
fu = Future()
req.url = self.host
fu.set_result(
HTTPResponse(req,
200,
buffer=cStringIO.StringIO(
pickle.dumps(self.tables[tableName]))))
return fu
def set(self, param, req):
tableName = param['tableName']
key = param['key']
val = param['val']
#print 'Worker {0} SET: {1} with key {2} to {3}'.format(self.host, tableName, key, val)
self.tables[tableName][key] = val
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def create(self, param, req):
tableName = param['tableName']
initVal = param['initVal']
self.tables[tableName] = {}
for key in initVal:
self.tables[tableName][key] = initVal[key]
#print 'Worker {0} CREATE:'.format(self.host), self.tables[tableName]
#Release
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def fetch(self, request, callback=None, **kwargs):
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
response_partial = RequestCollection.find(request)
if response_partial:
resp = response_partial(request)
else:
resp = HTTPResponse(request, 404)
if callback is not None:
callback(resp)
else:
future = TracebackFuture()
future.set_result(resp)
return future
def mock_fetch(self, request, callback=None, **kwargs):
future = TracebackFuture()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
res = MagicMock()
future.set_result(res)
return future
def remove(self, param, req):
tableName = param['tableName']
key = param['key']
if not key in self.tables[tableName]:
return Future().set_result('Key Error')
if key is None:
del self.tables[tableName]
else:
del self.tables[tableName][key]
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return Future().set_result('OK')
def remove(self, param, req):
tableName = param['tableName']
key = param['key']
if not key in self.tables[tableName]:
return Future().set_result('Key Error')
if key is None:
del self.tables[tableName]
else:
del self.tables[tableName][key]
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return Future().set_result('OK')
def start_tls(self, server_side, ssl_options, server_hostname=None):
if not isinstance(ssl_options, SSL.Context):
raise ValueError("ssl_options is not SSL.Context")
_socket = self.detach()
_socket = SSL.Connection(ssl_options, _socket)
if server_side:
_socket.set_accept_state()
else:
_socket.set_connect_state()
if server_hostname:
_socket.set_tlsext_host_name(server_hostname.encode("idna"))
orig_close_callback = self._close_callback
self._close_callback = None
future = TracebackFuture()
ssl_stream = MicroProxySSLIOStream(_socket,
server_hostname=server_hostname,
ssl_options=ssl_options,
io_loop=self.io_loop)
def close_callback():
if not future.done():
future.set_exception(ssl_stream.error or StreamClosedError())
if orig_close_callback is not None:
orig_close_callback()
ssl_stream.set_close_callback(close_callback)
ssl_stream._ssl_connect_callback = lambda: future.set_result(ssl_stream)
ssl_stream.max_buffer_size = self.max_buffer_size
ssl_stream.read_chunk_size = self.read_chunk_size
return future
def wrapper(*args, **kwargs):
future = TracebackFuture()
# ipdb.set_trace()
print args, kwargs
# callback, args, kwargs = replacer.replace(lambda value=_NO_RESULT: future.set_result(value),args, kwargs)
callback = None
kwargs['callback'] = lambda value = _NO_RESULT: future.set_result(value)
print callback, args, kwargs
def handle_error(typ, value, tb):
future.set_exc_info((typ, value, tb))
return True
exc_info = None
with ExceptionStackContext(handle_error):
try:
result = f(*args, **kwargs)
if result is not None:
raise ReturnValueIgnoredError('@return_future should not be used with function that return values')
except:
exc_info = sys.exc_info()
raise
if exc_info is not None:
future.result()
if callback is not None:
def run_callback(future):
result = future.result()
if result is _NO_RESULT:
callback('result is NULL')
else:
callback(future.result())
future.add_done_callback(wrap(run_callback))
# print future.result(), 'in the wapper'
return future
def open(self):
future = TracebackFuture()
try:
self._stream.open(lambda :future.set_result(None))
except Exception:
future.set_exc_info(sys.exc_info())
return future
def read_message(self, callback=None):
"""读取来自 WebSocket 服务器的消息.
如果在 WebSocket 初始化时指定了 on_message_callback ,那么这个方法永远不会返回消息
如果连接已经关闭,返回结果会是一个结果是 message 的 future 对象或者是 None.
如果 future 给出了回调参数, 这个参数将会在 future 完成时调用.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def read_message(self, callback=None):
"""Reads a message from the WebSocket server.
Returns a future whose result is the message, or None
if the connection is closed. If a callback argument
is given it will be called with the future when it is
ready.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def read_message(self, callback=None):
"""Reads a message from the WebSocket server.
Returns a future whose result is the message, or None
if the connection is closed. If a callback argument
is given it will be called with the future when it is
ready.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def read_message(self, callback=None):
"""读取来自 WebSocket 服务器的消息.
如果在 WebSocket 初始化时指定了 on_message_callback ,那么这个方法永远不会返回消息
如果连接已经关闭,返回结果会是一个结果是 message 的 future 对象或者是 None.
如果 future 给出了回调参数, 这个参数将会在 future 完成时调用.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def append(self, param, req):
tableName = param['tableName'].name
val = param['val']
if not tableName in self.tables:
raise KeyError, 'Table Not Found!'
args = {'tableName': tableName, 'key': self.tables[tableName]['len'], 'val': val}
worker = self.workers[hash(self.tables[tableName]['len']) % self.num_workers]
#print 'MASTER -> {0} Append: {1} to key {2}'.format(worker, val, self.tables[tableName]['len'])
self.client.fetch(formatQuery(worker, 'set', args))
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO(str(self.tables[tableName]['len']))))
self.tables[tableName]['len'] += 1
return fu
def remove(self, param, req):
tableName = param['tableName'].name
key = param['key']
if not tableName in self.tables:
#Silently fails
return
args = {'tableName': tableName, 'key': key}
futures = []
for worker in self.workers:
futures.append(self.client.fetch(formatQuery(worker, 'remove', args)))
if key is None:
del self.tables[tableName]
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def set(self, param, req):
tableName = param['tableName'].name
key = param['key']
val = param['val']
if not tableName in self.tables:
raise KeyError, 'Table not Found!'
worker = self.workers[hash(key) % self.num_workers]
args = {'tableName': tableName, 'key': key, 'val': val}
print 'MASTER -> {0} SET: {1} with key {2} to {3}'.format(worker, tableName, key, val)
futures = []
futures.append(self.client.fetch(formatQuery(worker, 'set', args)))
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def wrapper(*args, **kwargs):
runner = None
future = TracebackFuture()
# 注册进去了......
if 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
def handle_exception(typ, value, tb):
try:
if runner is not None and runner.handle_exception(typ, value, tb):
return True # 这样就吃掉exception了?
except Exception:
typ, value, tb = sys.exc_info()
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception) as deactivate:
try:
result = func(*args, **kwargs) # result是func调用返回的一个结果，有可能是一个iterator，比如我们的get、post方法里面塞几个yield的调用
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
deactivate()
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
def final_callback(value):
deactivate()
future.set_result(value)
# 这里的result很可能是一个iterator，看看头部的调用例子
runner = Runner(result, final_callback) # 关键点
runner.run()
return future
deactivate()
future.set_result(result)
return future
def wrapper(*args, **kwargs):
runner = None
future = TracebackFuture()
# 注册进去了......
if 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
def handle_exception(typ, value, tb):
try:
if runner is not None and runner.handle_exception(
typ, value, tb):
return True # 这样就吃掉exception了?
except Exception:
typ, value, tb = sys.exc_info()
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception) as deactivate:
try:
result = func(*args,
**kwargs) # result是func调用返回的一个结果，有可能是一个iterator?
except (Return, StopIteration) as e:
result = getattr(e, 'value', None) #TODO 这里也是
except Exception:
deactivate()
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
def final_callback(value):
deactivate()
future.set_result(value)
# 这里的result很可能是一个iterator，看看头部的调用例子
runner = Runner(result, final_callback) # 关键点
runner.run()
return future
deactivate()
future.set_result(result)
return future
def wrapper(*args, **kwargs):
runner = None
future = TracebackFuture()
if 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
def handle_exception(typ, value, tb):
try:
if runner is not None and runner.handle_exception(typ, value, tb):
return True
except Exception:
typ, value, tb = sys.exc_info()
if sys.getrefcount(future) == 2:
# there are no references left to this future object
# to prevent the error passing silently throw the exception right here
# so it can be catched and handled by IOLoop.handle_callback_exception
raise
else:
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception) as deactivate:
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
deactivate()
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
def final_callback(value):
deactivate()
future.set_result(value)
runner = Runner(result, final_callback)
runner.run()
return future
deactivate()
future.set_result(result)
return future
def remove(self, param, req):
tableName = param['tableName'].name
key = param['key']
if not tableName in self.tables:
#Silently fails
return
args = {'tableName': tableName, 'key': key}
futures = []
for worker in self.workers:
futures.append(
self.client.fetch(formatQuery(worker, 'remove', args)))
if key is None:
del self.tables[tableName]
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def set(self, param, req):
tableName = param['tableName'].name
key = param['key']
val = param['val']
if not tableName in self.tables:
raise KeyError, 'Table not Found!'
worker = self.workers[hash(key) % self.num_workers]
args = {'tableName': tableName, 'key': key, 'val': val}
print 'MASTER -> {0} SET: {1} with key {2} to {3}'.format(
worker, tableName, key, val)
futures = []
futures.append(self.client.fetch(formatQuery(worker, 'set', args)))
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def wrapper(*args, **kwargs):
future = TracebackFuture()
if replace_callback and 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
try:
with Measure(func):
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
try:
orig_stack_contexts = stack_context._state.contexts
with Measure(result, first=True):
yielded = next(result)
if stack_context._state.contexts is not orig_stack_contexts:
yielded = TracebackFuture()
yielded.set_exception(
stack_context.StackContextInconsistentError(
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)'))
except (StopIteration, Return) as e:
future.set_result(getattr(e, 'value', None))
except Exception:
future.set_exc_info(sys.exc_info())
else:
Runner(result, future, yielded)
try:
return future
finally:
future = None
future.set_result(result)
return future
def wrapper(*args, **kwargs):
future = TracebackFuture()
if replace_callback and 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
# Inline the first iteration of Runner.run. This lets us
# avoid the cost of creating a Runner when the coroutine
# never actually yields, which in turn allows us to
# use "optional" coroutines in critical path code without
# performance penalty for the synchronous case.
try:
orig_stack_contexts = stack_context._state.contexts
yielded = next(result)
if stack_context._state.contexts is not orig_stack_contexts:
yielded = TracebackFuture()
yielded.set_exception(
stack_context.StackContextInconsistentError(
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)')
)
except (StopIteration, Return) as e:
future.set_result(getattr(e, 'value', None))
except Exception:
future.set_exc_info(sys.exc_info())
else:
Runner(result, future, yielded)
return future
future.set_result(result)
return future
def wrapper(*args, **kwargs):
runner = None
future = TracebackFuture()
if 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
def handle_exception(typ, value, tb):
try:
if runner is not None and runner.handle_exception(
typ, value, tb):
return True
except Exception:
typ, value, tb = sys.exc_info()
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception) as deactivate:
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
deactivate()
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
def final_callback(value):
deactivate()
future.set_result(value)
runner = Runner(result, final_callback)
runner.run()
return future
deactivate()
future.set_result(result)
return future
def wrapper(*args, **kwargs):
future = TracebackFuture()
if replace_callback and "callback" in kwargs:
callback = kwargs.pop("callback")
IOLoop.current().add_future(future, lambda future: callback(future.result()))
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, "value", None)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
# Inline the first iteration of Runner.run. This lets us
# avoid the cost of creating a Runner when the coroutine
# never actually yields, which in turn allows us to
# use "optional" coroutines in critical path code without
# performance penalty for the synchronous case.
try:
orig_stack_contexts = stack_context._state.contexts
yielded = next(result)
if stack_context._state.contexts is not orig_stack_contexts:
yielded = TracebackFuture()
yielded.set_exception(
stack_context.StackContextInconsistentError(
"stack_context inconsistency (probably caused "
'by yield within a "with StackContext" block)'
)
)
except (StopIteration, Return) as e:
future.set_result(getattr(e, "value", None))
except Exception:
future.set_exc_info(sys.exc_info())
else:
Runner(result, future, yielded)
return future
future.set_result(result)
return future
def get_connection(self):
future = TracebackFuture()
if self._closed:
future.set_exception(ConnectionPoolClosedError())
return future
if not self._connections:
if self._connections_count < self._max_connections:
def _(succed, result):
if succed:
self._used_connections.append(result)
future.set_result(result)
else:
future.set_exc_info(result)
self.init_connection(_)
else:
self._wait_connections.append(future)
else:
connection = self._connections.popleft()
self._used_connections.append(connection)
future.set_result(connection)
return future
def wrapper(*args, **kwargs):
future = TracebackFuture()
if replace_callback and 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
try:
orig_stack_contexts = stack_context._state.contexts
yielded = next(result)
if stack_context._state.contexts is not orig_stack_contexts:
yielded = TracebackFuture()
yielded.set_exception(
stack_context.StackContextInconsistentError(
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)')
)
except (StopIteration, Return) as e:
future.set_result(getattr(e, 'value', None))
except Exception:
future.set_exc_info(sys.exc_info())
else:
Runner(result, future, yielded)
try:
return future
finally:
future = None
future.set_result(result)
return future
def wrapper(*args, **kwargs):
runner = None
future = TracebackFuture()
if replace_callback and 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
runner = Runner(result, future)
runner.run()
return future
future.set_result(result)
return future
def wrapper(*args, **kwargs):
runner = None
future = TracebackFuture()
if replace_callback and 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
result = getattr(e, 'value', None)
except Exception:
future.set_exc_info(sys.exc_info())
return future
else:
if isinstance(result, types.GeneratorType):
runner = Runner(result, future)
runner.run()
return future
future.set_result(result)
return future
def wrapper(*args, **kwargs):
if replacer.get_old_value(args, kwargs) is not None:
# If a callaback is present, just call in to the decorated
# function. This is a slight optimization (by not creating a
# Future that is unlikely to be used), but mainly avoids the
# complexity of running the callback in the expected way.
return f(*args, **kwargs)
future = TracebackFuture()
callback, args, kwargs = replacer.replace(
lambda value=None: future.set_result(value), args, kwargs)
f(*args, **kwargs)
stream = args[0]
stream._pending_futures.add(future)
future.add_done_callback(
lambda fut: stream._pending_futures.discard(fut))
return future
def wrapper(*args, **kwargs):
if replacer.get_old_value(args, kwargs) is not None:
# If a callaback is present, just call in to the decorated
# function. This is a slight optimization (by not creating a
# Future that is unlikely to be used), but mainly avoids the
# complexity of running the callback in the expected way.
return f(*args, **kwargs)
future = TracebackFuture()
callback, args, kwargs = replacer.replace(
lambda value=None: future.set_result(value),
args, kwargs)
f(*args, **kwargs)
stream = args[0]
stream._pending_futures.add(future)
future.add_done_callback(
lambda fut: stream._pending_futures.discard(fut))
return future
def wrapper(*args, **kwargs):
ipdb.set_trace()
future = TracebackFuture()
if replace_callback and 'callback' in kwargs:
callback = kwargs.pop('callback')
IOLoop.current().add_future(future, lambda future: callback(future.result()))
try:
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
# ipdb.set_trace()
result = gen._value_from_stopiteration(e)
except Exception:
# ipdb.set_trace()
future.set_exc_info(sys.exc_info())
return future
else:
# ipdb.set_trace()
if isinstance(result, GeneratorType):
try:
orig_stack_contexts = stack_context._state.contexts
yielded = next(result)
if stack_context._state.contexts is not orig_stack_contexts:
yielded = TracebackFuture()
yielded.set_exception(stack_context.StackContextInconsistentError('stack_context inconsistency (probably caused'))
except (StopIteration, Return) as e:
future.set_result(gen._value_from_stopiteration(e))
except Exception:
future.set_exc_info(sys.exc_info())
else:
try:
result.send(yielded.result())
except (StopIteration, Return) as e:
ipdb.set_trace()
future.set_result(gen._value_from_stopiteration(e))
# gen.Runner(result, future, yielded)
try:
return future
finally:
future = None
future.set_result(result)
return future
class WebSocketClientConnection(simple_httpclient._HTTPConnection):
"""WebSocket 客户端连接
这个类不应当直接被实例化, 请使用 `websocket_connect`
"""
def __init__(self,
io_loop,
request,
on_message_callback=None,
compression_options=None):
self.compression_options = compression_options
self.connect_future = TracebackFuture()
self.protocol = None
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
self._on_message_callback = on_message_callback
self.close_code = self.close_reason = None
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
if self.compression_options is not None:
# Always offer to let the server set our max_wbits (and even though
# we don't offer it, we will accept a client_no_context_takeover
# from the server).
# TODO: set server parameters for deflate extension
# if requested in self.compression_options.
request.headers['Sec-WebSocket-Extensions'] = (
'permessage-deflate; client_max_window_bits')
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection,
self).__init__(io_loop, None, request, lambda: None,
self._on_http_response, 104857600,
self.tcp_client, 65536, 104857600)
def close(self, code=None, reason=None):
"""关闭 websocket 连接
``code`` 和 ``reason`` 的文档在 `WebSocketHandler.close` 下已给出.
.. versionadded:: 3.2
.. versionchanged:: 4.0
添加 ``code`` 和 ``reason`` 这两个参数
"""
if self.protocol is not None:
self.protocol.close(code, reason)
self.protocol = None
def on_connection_close(self):
if not self.connect_future.done():
self.connect_future.set_exception(StreamClosedError())
self.on_message(None)
self.tcp_client.close()
super(WebSocketClientConnection, self).on_connection_close()
def _on_http_response(self, response):
if not self.connect_future.done():
if response.error:
self.connect_future.set_exception(response.error)
else:
self.connect_future.set_exception(
WebSocketError("Non-websocket response"))
def headers_received(self, start_line, headers):
if start_line.code != 101:
return super(WebSocketClientConnection,
self).headers_received(start_line, headers)
self.headers = headers
self.protocol = self.get_websocket_protocol()
self.protocol._process_server_headers(self.key, self.headers)
self.protocol._receive_frame()
if self._timeout is not None:
self.io_loop.remove_timeout(self._timeout)
self._timeout = None
self.stream = self.connection.detach()
self.stream.set_close_callback(self.on_connection_close)
# Once we've taken over the connection, clear the final callback
# we set on the http request. This deactivates the error handling
# in simple_httpclient that would otherwise interfere with our
# ability to see exceptions.
self.final_callback = None
self.connect_future.set_result(self)
def write_message(self, message, binary=False):
"""发送消息到 websocket 服务器."""
return self.protocol.write_message(message, binary)
def read_message(self, callback=None):
"""读取来自 WebSocket 服务器的消息.
如果在 WebSocket 初始化时指定了 on_message_callback ,那么这个方法永远不会返回消息
如果连接已经关闭,返回结果会是一个结果是 message 的 future 对象或者是 None.
如果 future 给出了回调参数, 这个参数将会在 future 完成时调用.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def on_message(self, message):
if self._on_message_callback:
self._on_message_callback(message)
elif self.read_future is not None:
self.read_future.set_result(message)
self.read_future = None
else:
self.read_queue.append(message)
def on_pong(self, data):
pass
def get_websocket_protocol(self):
return WebSocketProtocol13(
self,
mask_outgoing=True,
compression_options=self.compression_options)
class Runner(object):
"""Internal implementation of `tornado.gen.engine`.
Maintains information about pending callbacks and their results.
The results of the generator are stored in ``result_future`` (a
`.TracebackFuture`)
"""
def __init__(self, gen, result_future, first_yielded):
self.gen = gen
self.result_future = result_future
self.future = _null_future
self.yield_point = None
self.pending_callbacks = None
self.results = None
self.running = False
self.finished = False
self.had_exception = False
self.io_loop = IOLoop.current()
# For efficiency, we do not create a stack context until we
# reach a YieldPoint (stack contexts are required for the historical
# semantics of YieldPoints, but not for Futures). When we have
# done so, this field will be set and must be called at the end
# of the coroutine.
self.stack_context_deactivate = None
if self.handle_yield(first_yielded):
self.run()
def register_callback(self, key):
"""Adds ``key`` to the list of callbacks."""
if self.pending_callbacks is None:
# Lazily initialize the old-style YieldPoint data structures.
self.pending_callbacks = set()
self.results = {}
if key in self.pending_callbacks:
raise KeyReuseError("key %r is already pending" % (key, ))
self.pending_callbacks.add(key)
def is_ready(self, key):
"""Returns true if a result is available for ``key``."""
if self.pending_callbacks is None or key not in self.pending_callbacks:
raise UnknownKeyError("key %r is not pending" % (key, ))
return key in self.results
def set_result(self, key, result):
"""Sets the result for ``key`` and attempts to resume the generator."""
self.results[key] = result
if self.yield_point is not None and self.yield_point.is_ready():
try:
self.future.set_result(self.yield_point.get_result())
except:
self.future.set_exc_info(sys.exc_info())
self.yield_point = None
self.run()
def pop_result(self, key):
"""Returns the result for ``key`` and unregisters it."""
self.pending_callbacks.remove(key)
return self.results.pop(key)
def run(self):
"""Starts or resumes the generator, running until it reaches a
yield point that is not ready.
"""
if self.running or self.finished:
return
try:
self.running = True
while True:
future = self.future
if not future.done():
return
self.future = None
try:
orig_stack_contexts = stack_context._state.contexts
try:
value = future.result()
except Exception:
self.had_exception = True
yielded = self.gen.throw(*sys.exc_info())
else:
yielded = self.gen.send(value)
if stack_context._state.contexts is not orig_stack_contexts:
self.gen.throw(
stack_context.StackContextInconsistentError(
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)')
)
except (StopIteration, Return) as e:
self.finished = True
self.future = _null_future
if self.pending_callbacks and not self.had_exception:
# If we ran cleanly without waiting on all callbacks
# raise an error (really more of a warning). If we
# had an exception then some callbacks may have been
# orphaned, so skip the check in that case.
raise LeakedCallbackError(
"finished without waiting for callbacks %r" %
self.pending_callbacks)
self.result_future.set_result(getattr(e, 'value', None))
self.result_future = None
self._deactivate_stack_context()
return
except Exception:
self.finished = True
self.future = _null_future
self.result_future.set_exc_info(sys.exc_info())
self.result_future = None
self._deactivate_stack_context()
return
if not self.handle_yield(yielded):
return
finally:
self.running = False
def handle_yield(self, yielded):
if isinstance(yielded, list):
if all(is_future(f) for f in yielded):
yielded = multi_future(yielded)
else:
yielded = Multi(yielded)
elif isinstance(yielded, dict):
if all(is_future(f) for f in yielded.values()):
yielded = multi_future(yielded)
else:
yielded = Multi(yielded)
if isinstance(yielded, YieldPoint):
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
elif is_future(yielded):
self.future = yielded
if not self.future.done() or self.future is moment:
self.io_loop.add_future(self.future, lambda f: self.run())
return False
else:
self.future = TracebackFuture()
self.future.set_exception(
BadYieldError("yielded unknown object %r" % (yielded, )))
return True
def result_callback(self, key):
return stack_context.wrap(
_argument_adapter(functools.partial(self.set_result, key)))
def handle_exception(self, typ, value, tb):
if not self.running and not self.finished:
self.future = TracebackFuture()
self.future.set_exc_info((typ, value, tb))
self.run()
return True
else:
return False
def _deactivate_stack_context(self):
if self.stack_context_deactivate is not None:
self.stack_context_deactivate()
self.stack_context_deactivate = None
class WebSocketClientConnection(simple_httpclient._HTTPConnection):
"""WebSocket client connection.
This class should not be instantiated directly; use the
`websocket_connect` function instead.
"""
def __init__(self, io_loop, request):
self.connect_future = TracebackFuture()
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection,
self).__init__(io_loop, None, request, lambda: None,
self._on_http_response, 104857600,
self.tcp_client, 65536)
def close(self, code=None, reason=None):
"""Closes the websocket connection.
``code`` and ``reason`` are documented under
`WebSocketHandler.close`.
.. versionadded:: 3.2
.. versionchanged:: 4.0
Added the ``code`` and ``reason`` arguments.
"""
if self.protocol is not None:
self.protocol.close(code, reason)
self.protocol = None
def _on_close(self):
self.on_message(None)
self.resolver.close()
super(WebSocketClientConnection, self)._on_close()
def _on_http_response(self, response):
if not self.connect_future.done():
if response.error:
self.connect_future.set_exception(response.error)
else:
self.connect_future.set_exception(
WebSocketError("Non-websocket response"))
def headers_received(self, start_line, headers):
if start_line.code != 101:
return super(WebSocketClientConnection,
self).headers_received(start_line, headers)
self.headers = headers
assert self.headers['Upgrade'].lower() == 'websocket'
assert self.headers['Connection'].lower() == 'upgrade'
accept = WebSocketProtocol13.compute_accept_value(self.key)
assert self.headers['Sec-Websocket-Accept'] == accept
self.protocol = WebSocketProtocol13(self, mask_outgoing=True)
self.protocol._receive_frame()
if self._timeout is not None:
self.io_loop.remove_timeout(self._timeout)
self._timeout = None
self.stream = self.connection.detach()
self.stream.set_close_callback(self._on_close)
self.connect_future.set_result(self)
def write_message(self, message, binary=False):
"""Sends a message to the WebSocket server."""
self.protocol.write_message(message, binary)
def read_message(self, callback=None):
"""Reads a message from the WebSocket server.
Returns a future whose result is the message, or None
if the connection is closed. If a callback argument
is given it will be called with the future when it is
ready.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def on_message(self, message):
if self.read_future is not None:
self.read_future.set_result(message)
self.read_future = None
else:
self.read_queue.append(message)
def on_pong(self, data):
pass
class SSHClient(object):
_host_map = {}
def __init__(self, io_loop=None):
self.client = paramiko.SSHClient()
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# self.transport = None
self.channel = None
self.io_loop = io_loop or IOLoop.current()
self.future = None
self._result = ''
self.sftp_client = None
self.count = 0
def connect(self, host, uesrname, port, **kwargs):
self.client.connect(host,
username=uesrname,
port=port,
allow_agent=False,
**kwargs)
def put_file(self, filename):
try:
self.future = TracebackFuture()
transport = self.client.get_transport()
self.sftp_client = SFTPClient.from_transport(transport)
self.remote = self.sftp_client.open(filename, 'wb')
self.local = open(filename, 'rb')
# self.remote.setblocking(0)
# _buf = self.local.read(100)
# self.remote.write(_buf)
self.io_loop.add_handler(self.sftp_client.sock, self._handle_write,
self.io_loop.WRITE | self.io_loop.ERROR)
return self.future
except Exception as e:
print(e)
pass
def _handle_write(self, fd, events):
# print("EVENT=%d,fd=%s" % (events, fd))
if events & self.io_loop.WRITE:
_buf = self.local.read(BLOCK_SIZE)
if _buf:
try:
a = time.time()
self.remote.write(_buf)
b = time.time()
self.count += len(_buf)
c = b - a
except IOError as e:
print(e)
else:
self.io_loop.remove_handler(fd)
self.remote.close()
self.local.close()
self.future.set_result('FINISH')
def exec_command(self, command):
request_log.info('EXECUTE %s', command)
try:
self.future = TracebackFuture()
transport = self.client.get_transport()
self.channel = transport.open_session()
self.channel.exec_command(command)
self.io_loop.add_handler(self.channel.fileno(),
self._handle_events,
self.io_loop.READ | self.io_loop.ERROR)
return self.future
except Exception as e:
print(e)
pass
def _handle_events(self, fd, events):
# print("EVENT=" + str(events))
if events & self.io_loop.READ:
_buf = self.channel.recv(1024)
if len(_buf):
print(_buf.decode())
self._result += _buf.decode()
else:
self.io_loop.remove_handler(self.channel.fileno())
self.future.set_result(self._result)
if events & self.io_loop.ERROR:
self.io_loop.remove_handler(self.channel.fileno())
self.future.set_result('ERROR')
class WebSocketClientConnection(simple_httpclient._HTTPConnection):
"""WebSocket client connection.
This class should not be instantiated directly; use the
`websocket_connect` function instead.
"""
def __init__(self, io_loop, request, on_message_callback=None,
compression_options=None):
self.compression_options = compression_options
self.connect_future = TracebackFuture()
self.protocol = None
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
self._on_message_callback = on_message_callback
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
if self.compression_options is not None:
# Always offer to let the server set our max_wbits (and even though
# we don't offer it, we will accept a client_no_context_takeover
# from the server).
# TODO: set server parameters for deflate extension
# if requested in self.compression_options.
request.headers['Sec-WebSocket-Extensions'] = (
'permessage-deflate; client_max_window_bits')
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection, self).__init__(
io_loop, None, request, lambda: None, self._on_http_response,
104857600, self.tcp_client, 65536)
def close(self, code=None, reason=None):
"""Closes the websocket connection.
``code`` and ``reason`` are documented under
`WebSocketHandler.close`.
.. versionadded:: 3.2
.. versionchanged:: 4.0
Added the ``code`` and ``reason`` arguments.
"""
if self.protocol is not None:
self.protocol.close(code, reason)
self.protocol = None
def on_connection_close(self):
if not self.connect_future.done():
self.connect_future.set_exception(StreamClosedError())
self.on_message(None)
self.tcp_client.close()
super(WebSocketClientConnection, self).on_connection_close()
def _on_http_response(self, response):
if not self.connect_future.done():
if response.error:
self.connect_future.set_exception(response.error)
else:
self.connect_future.set_exception(WebSocketError(
"Non-websocket response"))
def headers_received(self, start_line, headers):
if start_line.code != 101:
return super(WebSocketClientConnection, self).headers_received(
start_line, headers)
self.headers = headers
self.protocol = self.get_websocket_protocol()
self.protocol._process_server_headers(self.key, self.headers)
self.protocol._receive_frame()
if self._timeout is not None:
self.io_loop.remove_timeout(self._timeout)
self._timeout = None
self.stream = self.connection.detach()
self.stream.set_close_callback(self.on_connection_close)
# Once we've taken over the connection, clear the final callback
# we set on the http request. This deactivates the error handling
# in simple_httpclient that would otherwise interfere with our
# ability to see exceptions.
self.final_callback = None
self.connect_future.set_result(self)
def write_message(self, message, binary=False):
"""Sends a message to the WebSocket server."""
self.protocol.write_message(message, binary)
def read_message(self, callback=None):
"""Reads a message from the WebSocket server.
If on_message_callback was specified at WebSocket
initialization, this function will never return messages
Returns a future whose result is the message, or None
if the connection is closed. If a callback argument
is given it will be called with the future when it is
ready.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def on_message(self, message):
if self._on_message_callback:
self._on_message_callback(message)
elif self.read_future is not None:
self.read_future.set_result(message)
self.read_future = None
else:
self.read_queue.append(message)
def on_pong(self, data):
pass
def get_websocket_protocol(self):
return WebSocketProtocol13(self, mask_outgoing=True,
compression_options=self.compression_options)
class WebSocketClientConnection(simple_httpclient._HTTPConnection):
"""WebSocket client connection."""
def __init__(self, io_loop, request):
self.connect_future = TracebackFuture()
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
self.resolver = Resolver(io_loop=io_loop)
super(WebSocketClientConnection,
self).__init__(io_loop, None, request, lambda: None,
self._on_http_response, 104857600, self.resolver)
def _on_close(self):
self.on_message(None)
self.resolver.close()
def _on_http_response(self, response):
if not self.connect_future.done():
if response.error:
self.connect_future.set_exception(response.error)
else:
self.connect_future.set_exception(
WebSocketError("Non-websocket response"))
def _handle_1xx(self, code):
assert code == 101
assert self.headers['Upgrade'].lower() == 'websocket'
assert self.headers['Connection'].lower() == 'upgrade'
accept = WebSocketProtocol13.compute_accept_value(self.key)
assert self.headers['Sec-Websocket-Accept'] == accept
self.protocol = WebSocketProtocol13(self, mask_outgoing=True)
self.protocol._receive_frame()
if self._timeout is not None:
self.io_loop.remove_timeout(self._timeout)
self._timeout = None
self.connect_future.set_result(self)
def write_message(self, message, binary=False):
"""Sends a message to the WebSocket server."""
self.protocol.write_message(message, binary)
def read_message(self, callback=None):
"""Reads a message from the WebSocket server.
Returns a future whose result is the message, or None
if the connection is closed. If a callback argument
is given it will be called with the future when it is
ready.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def on_message(self, message):
if self.read_future is not None:
self.read_future.set_result(message)
self.read_future = None
else:
self.read_queue.append(message)
def on_pong(self, data):
pass
class Runner(object):
"""Internal implementation of `tornado.gen.engine`.
Maintains information about pending callbacks and their results.
The results of the generator are stored in ``result_future`` (a
`.TracebackFuture`)
"""
def __init__(self, gen, result_future, first_yielded):
self.gen = gen
self.result_future = result_future
# 第一次在handle_yield指向first_yielded，之后指向gen()
self.future = _null_future
self.yield_point = None
self.pending_callbacks = None
self.results = None
self.running = False
self.finished = False
self.had_exception = False
self.io_loop = IOLoop.current()
# For efficiency, we do not create a stack context until we
# reach a YieldPoint (stack contexts are required for the historical
# semantics of YieldPoints, but not for Futures). When we have
# done so, this field will be set and must be called at the end
# of the coroutine.
self.stack_context_deactivate = None
if self.handle_yield(first_yielded):
self.run()
def register_callback(self, key):
"""Adds ``key`` to the list of callbacks."""
if self.pending_callbacks is None:
# Lazily initialize the old-style YieldPoint data structures.
self.pending_callbacks = set()
self.results = {}
if key in self.pending_callbacks:
raise KeyReuseError("key %r is already pending" % (key, ))
self.pending_callbacks.add(key)
def is_ready(self, key):
"""Returns true if a result is available for ``key``."""
if self.pending_callbacks is None or key not in self.pending_callbacks:
raise UnknownKeyError("key %r is not pending" % (key, ))
return key in self.results
def set_result(self, key, result):
"""Sets the result for ``key`` and attempts to resume the generator."""
self.results[key] = result
if self.yield_point is not None and self.yield_point.is_ready():
try:
self.future.set_result(self.yield_point.get_result())
except:
self.future.set_exc_info(sys.exc_info())
self.yield_point = None
self.run()
def pop_result(self, key):
"""Returns the result for ``key`` and unregisters it."""
self.pending_callbacks.remove(key)
return self.results.pop(key)
def run(self):
"""Starts or resumes the generator, running until it reaches a
yield point that is not ready.
"""
if self.running or self.finished:
return
try:
self.running = True
while True:
future = self.future
if not future.done():
return
self.future = None
try:
orig_stack_contexts = stack_context._state.contexts
exc_info = None
try:
value = future.result()
except Exception:
self.had_exception = True
exc_info = sys.exc_info()
if exc_info is not None:
yielded = self.gen.throw(*exc_info)
exc_info = None
else:
# 发送异步调用结果，通知任务完成
yielded = self.gen.send(value)
# self.gen为生成器，即被@gen.coroutine修饰的函数，
# self.gen.send发送结果到 yield语句进行赋值，即
# result = yield clien.fetch(url)，这里的result
# 即为value，生成器结束会得到StopIteration异常结束
# 调用，否则继续调用self.handle_yield
if stack_context._state.contexts is not orig_stack_contexts:
self.gen.throw(
stack_context.StackContextInconsistentError(
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)')
)
except (StopIteration, Return) as e:
# 捕捉到StopIteration 或者 Return 异常，说明 self.gen生成器
# 结束，Return 异常一般由用户调用 raise Return(True)
self.finished = True
self.future = _null_future
if self.pending_callbacks and not self.had_exception:
# If we ran cleanly without waiting on all callbacks
# raise an error (really more of a warning). If we
# had an exception then some callbacks may have been
# orphaned, so skip the check in that case.
raise LeakedCallbackError(
"finished without waiting for callbacks %r" %
self.pending_callbacks)
# 如果是 Return 异常设置返回值
self.result_future.set_result(_value_from_stopiteration(e))
self.result_future = None
self._deactivate_stack_context()
return
except Exception:
self.finished = True
self.future = _null_future
self.result_future.set_exc_info(sys.exc_info())
self.result_future = None
self._deactivate_stack_context()
return
# 返回True说明yielded已经完成done，继续运行返回结果给上层，如果yielded
# 还在等待Future完成，那么直接返回，handle_yield已经将yielded放到ioloop
# 进行处理，完成后回调run运行
if not self.handle_yield(yielded):
return
finally:
self.running = False
def handle_yield(self, yielded):
# 返回True已经完成，False表示Future未完成，添加到ioloop中
# Lists containing YieldPoints require stack contexts;
# other lists are handled in convert_yielded.
# 处理 yield list or dict 情况
if _contains_yieldpoint(yielded):
yielded = multi(yielded)
if isinstance(yielded, YieldPoint):
# YieldPoints are too closely coupled to the Runner to go
# through the generic convert_yielded mechanism.
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
else:
# Future情况，为r = yield fetch()之类返回的Future
try:
self.future = convert_yielded(yielded)
except BadYieldError:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if not self.future.done() or self.future is moment:
self.io_loop.add_future(self.future, lambda f: self.run())
return False
return True
def result_callback(self, key):
return stack_context.wrap(
_argument_adapter(functools.partial(self.set_result, key)))
def handle_exception(self, typ, value, tb):
if not self.running and not self.finished:
self.future = TracebackFuture()
self.future.set_exc_info((typ, value, tb))
self.run()
return True
else:
return False
def _deactivate_stack_context(self):
if self.stack_context_deactivate is not None:
self.stack_context_deactivate()
self.stack_context_deactivate = None
class Runner(object):
"""Internal implementation of `tornado.gen.engine`.
Maintains information about pending callbacks and their results.
The results of the generator are stored in ``result_future`` (a
`.TracebackFuture`)
"""
def __init__(self, gen, result_future, first_yielded):
self.gen = gen
self.result_future = result_future
self.future = _null_future
self.yield_point = None
self.pending_callbacks = None
self.results = None
self.running = False
self.finished = False
self.had_exception = False
self.io_loop = IOLoop.current()
# For efficiency, we do not create a stack context until we
# reach a YieldPoint (stack contexts are required for the historical
# semantics of YieldPoints, but not for Futures). When we have
# done so, this field will be set and must be called at the end
# of the coroutine.
self.stack_context_deactivate = None
if self.handle_yield(first_yielded):
# handle_yield 在 coroutine yield 的对象有了结果时返回
# True，这个时候就会马上执行 run 方法。
self.run()
def register_callback(self, key):
"""Adds ``key`` to the list of callbacks."""
if self.pending_callbacks is None:
# Lazily initialize the old-style YieldPoint data structures.
self.pending_callbacks = set()
self.results = {}
if key in self.pending_callbacks:
raise KeyReuseError("key %r is already pending" % (key, ))
self.pending_callbacks.add(key)
def is_ready(self, key):
"""Returns true if a result is available for ``key``."""
if self.pending_callbacks is None or key not in self.pending_callbacks:
raise UnknownKeyError("key %r is not pending" % (key, ))
return key in self.results
def set_result(self, key, result):
"""Sets the result for ``key`` and attempts to resume the generator."""
self.results[key] = result
if self.yield_point is not None and self.yield_point.is_ready():
try:
self.future.set_result(self.yield_point.get_result())
except:
self.future.set_exc_info(sys.exc_info())
self.yield_point = None
self.run()
def pop_result(self, key):
"""Returns the result for ``key`` and unregisters it."""
self.pending_callbacks.remove(key)
return self.results.pop(key)
def run(self):
"""Starts or resumes the generator, running until it reaches a
yield point that is not ready.
"""
if self.running or self.finished:
# 如果 run 方法已经在被执行，则返回。
return
try:
self.running = True
while True:
future = self.future
if not future.done():
# Runner 对应的 coroutine yield
# 的对象还没有准备好，则返回。
return
self.future = None
try:
orig_stack_contexts = stack_context._state.contexts
exc_info = None
try:
value = future.result()
except Exception:
# 如果执行过程中有抛出异常，则捕获之，并让 coroutine
# 也抛出该异常。Python 中的 generator 在 2.5 版本后才
# 提供了 throw 方法来提供实现 coroutine
# 的可能性，详情可以参考：
# https://www.python.org/dev/peps/pep-0342/
self.had_exception = True
exc_info = sys.exc_info()
if exc_info is not None:
# 如果有异常，让 coroutine 抛出。
yielded = self.gen.throw(*exc_info)
exc_info = None
else:
# 如果一切正常，则将结果反馈给 coroutine。
yielded = self.gen.send(value)
if stack_context._state.contexts \
is not orig_stack_contexts:
self.gen.throw(
stack_context.StackContextInconsistentError(
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)')
)
except (StopIteration, Return) as e:
# coroutine 的执行绪执行完毕，Runner 的任务也完成了。
self.finished = True
self.future = _null_future
if self.pending_callbacks and not self.had_exception:
# If we ran cleanly without waiting on all callbacks
# raise an error (really more of a warning). If we
# had an exception then some callbacks may have been
# orphaned, so skip the check in that case.
raise LeakedCallbackError(
"finished without waiting for callbacks %r" %
self.pending_callbacks)
# 从 Exception 对象中获得最终结果，并反馈给
# result_future，进而触发 result_future 的 done callbacks。
self.result_future.set_result(_value_from_stopiteration(e))
self.result_future = None
self._deactivate_stack_context()
return
except Exception:
self.finished = True
self.future = _null_future
self.result_future.set_exc_info(sys.exc_info())
self.result_future = None
self._deactivate_stack_context()
return
if not self.handle_yield(yielded):
return
finally:
self.running = False
def handle_yield(self, yielded):
# handle_yield 方法会对 coroutine yield 的结果进行转化，进而判断 yield
# 的异步执行的代码是否已经执行完毕并且已经得到了结果，如果没有，则将相
# 应得 future 对象放到 Event Loop 中，等待下一次 Event Loop
# 执行到时再判断执行结果
# Lists containing YieldPoints require stack contexts;
# other lists are handled in convert_yielded.
if _contains_yieldpoint(yielded):
# 如果 coroutine yield 的结果中包含 YieldPoint，则用 stack context
# 将其包装一次
yielded = multi(yielded)
if isinstance(yielded, YieldPoint):
# YieldPoints are too closely coupled to the Runner to go
# through the generic convert_yielded mechanism.
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
else:
# 经过前面的处理，YieldPoint 都已经被正确包装、转化，接下来就可以将
# coroutine yield 的内容转化成一个 future 对象。
try:
self.future = convert_yielded(yielded)
except BadYieldError:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if not self.future.done() or self.future is moment:
# 如果 future 还没有收到结果，不是 done 的状态，则将其添加到 IOLoop
# 实例中，在后续某一次 Event Loop 中，future 被赋予了结果后再去执行
# run 方法，从而让 coroutine 获得结果并继续执行下去。
self.io_loop.add_future(self.future, lambda f: self.run())
# 这时便不用去执行 run 方法去获得 future 中的结果并反馈给 coroutine
# 了，一切都等到之后的 Event Loop 中见分晓。
return False
# Future 对象这个时候已经有了结果，便可以去执行 run 方法，并将结果反馈给
# coroutine，让之继续后面的执行绪了。
return True
def result_callback(self, key):
return stack_context.wrap(
_argument_adapter(functools.partial(self.set_result, key)))
def handle_exception(self, typ, value, tb):
if not self.running and not self.finished:
self.future = TracebackFuture()
self.future.set_exc_info((typ, value, tb))
self.run()
return True
else:
return False
def _deactivate_stack_context(self):
if self.stack_context_deactivate is not None:
self.stack_context_deactivate()
self.stack_context_deactivate = None
def wrapper(*args, **kwargs):
future = TracebackFuture() # 一个future的跟踪对象，这个是关键
# 原来的func不支持callback的参数，修饰之后是可以使用的
# 下面的这个future好好读懂，然后好好理解传说中的future功能
if replace_callback and 'callback' in kwargs:
callback = kwargs.pop('callback')
# 就是future成功了，执行lambda future: callback(future.result())函数，参数就是这个future
# 也就是把这个可能执行的语句放到IOLoop里面去，这个就是这个add_future的功能
IOLoop.current().add_future(
future, lambda future: callback(future.result()))
# 异步的HTTP和异步的IOStream也许返回的类型不一样，
# 感觉两个异步的东西才是tornado的关键
try:
# 尝试运行函数，这里会yield出来，所以不会被阻塞的
# 1. 返回一个Return 2. 返回一个StopIteration（这个不是很清楚？？）
# 3. 返回一个Exception?就是运行错误了
result = func(*args, **kwargs)
except (Return, StopIteration) as e:
# 到这里说明结果已经出来了，future的任务结束了
# ？StopIteration无法理解
result = getattr(e, 'value', None)
except Exception: # 如果出错的话
# 运行出错了,这个时候依旧会添加到IOLoop._callbacks
# 下面这句话执行这个future全部的回调
future.set_exc_info(sys.exc_info())
return future # 一但结果确定，直接返回一个future
else:
# 如果内部有yield，直接返回一个types.GeneratorType
if isinstance(result, types.GeneratorType):
# Inline the first iteration of Runner.run. This lets us
# avoid the cost of creating a Runner when the coroutine
# never actually yields, which in turn allows us to
# use "optional" coroutines in critical path code without
# performance penalty for the synchronous case.
try:
orig_stack_contexts = stack_context._state.contexts
# 这里是获取yield的结果的地方，一般还是一个future的东西
yielded = next(result) # 获取yield的结果，这里一般是一个future类型yield出来的东西
# 先忽略下面的出错处理
if stack_context._state.contexts is not orig_stack_contexts:
yielded = TracebackFuture()
yielded.set_exception(
stack_context.StackContextInconsistentError(
'stack_context inconsistency (probably caused '
'by yield within a "with StackContext" block)'))
except (StopIteration, Return) as e: # 直接出结果
future.set_result(getattr(e, 'value', None))
except Exception: # bug了
future.set_exc_info(sys.exc_info())
else: # 返回一个future
Runner(result, future, yielded)
# 如果是一个Exception，这里记录到future，然后返回了
return future
future.set_result(result) # 这个时候添加回调到了IOLoop
return future
