def create(self, param, req):
tableName = param['tableName'].name
initVal = param['initVal']
if tableName in self.tables:
raise KeyError, 'Table name already existed'
#print 'MASTER CREATE:', initVal
self.tables[tableName] = {'len': 0, }
#Data Partition
vals = {}
for worker_id in xrange(self.num_workers):
vals[worker_id] = {}
for key in initVal:
#Bookkeeping
self.tables[tableName]['len'] += 1
self.tables[tableName][key] = type(initVal[key])
worker = hash(key) % self.num_workers
vals[worker][key] = initVal[key]
futures = []
for worker_id in xrange(self.num_workers):
args = {'tableName': tableName, 'initVal': vals[worker_id]}
futures.append(self.client.fetch(formatQuery(self.workers[worker_id], 'create', args)))
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def start_tls(self, server_side, ssl_options, server_hostname=None):
if not isinstance(ssl_options, SSL.Context):
raise ValueError("ssl_options is not SSL.Context")
_socket = self.detach()
_socket = SSL.Connection(ssl_options, _socket)
if server_side:
_socket.set_accept_state()
else:
_socket.set_connect_state()
if server_hostname:
_socket.set_tlsext_host_name(server_hostname.encode("idna"))
orig_close_callback = self._close_callback
self._close_callback = None
future = TracebackFuture()
ssl_stream = MicroProxySSLIOStream(_socket,
server_hostname=server_hostname,
ssl_options=ssl_options,
io_loop=self.io_loop)
def close_callback():
if not future.done():
future.set_exception(ssl_stream.error or StreamClosedError())
if orig_close_callback is not None:
orig_close_callback()
ssl_stream.set_close_callback(close_callback)
ssl_stream._ssl_connect_callback = lambda: future.set_result(ssl_stream)
ssl_stream.max_buffer_size = self.max_buffer_size
ssl_stream.read_chunk_size = self.read_chunk_size
return future
def _defer_to_future(defer):
future = TracebackFuture()
defer.addCallbacks(
future.set_result,
lambda failure: future.set_exc_info(
(failure.type, failure.value, failure.tb)))
return future
class StubWebSocketClientConnection(object):
"""A stub version of tornado.websocket.WebSocketClientConnection for use in
unit tests.
Attributes:
TIMEOUT: class constant, which should be overridden in subclasses, in
order to vary the time from when the class is instantiated until the
returned future is resolved.
connect_future: the Future resolved when the client "connection" is
made.
written_messages: a list of messages passed to the write_message
method to see they have been received.
"""
TIMEOUT = datetime.timedelta(milliseconds=0)
def __init__(self, io_loop, request, on_message_callback=None,
compression_options=None):
del request, on_message_callback, compression_options
self.connect_future = TracebackFuture()
self.written_messages = []
io_loop.add_timeout(self.TIMEOUT, self._complete)
def _complete(self):
self.connect_future.set_result(self)
def write_message(self, message):
"""Adds writen message to the written_messages buffer."""
self.written_messages.append(message)
def open(self):
future = TracebackFuture()
try:
self._stream.open(lambda :future.set_result(None))
except Exception:
future.set_exc_info(sys.exc_info())
return future
def fetch_all(self, param, req):
tableName = param['tableName']
if not tableName in self.tables:
raise KeyError, 'Table Not Found!'
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO(pickle.dumps(self.tables[tableName]))))
return fu
def get(self, param, req):
tableName = param['tableName']
key = param['key']
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200,
buffer=cStringIO.StringIO(pickle.dumps(self.tables[tableName][key]))))
return fu
def len(self, param, req):
tableName = param['tableName'].name
if not tableName in self.tables:
raise KeyError, 'Table Not Found'
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO(str(self.tables[tableName]['len']))))
return fu
def fetch(self, request, callback=None, raise_error=True, **kwargs):
"""Executes a request, asynchronously returning an `HTTPResponse`.
The request may be either a string URL or an `HTTPRequest` object.
If it is a string, we construct an `HTTPRequest` using any additional
kwargs: ``HTTPRequest(request, **kwargs)``
This method returns a `.Future` whose result is an
`HTTPResponse`. By default, the ``Future`` will raise an
`HTTPError` if the request returned a non-200 response code
(other errors may also be raised if the server could not be
contacted). Instead, if ``raise_error`` is set to False, the
response will always be returned regardless of the response
code.
If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
In the callback interface, `HTTPError` is not automatically raised.
Instead, you must check the response's ``error`` attribute or
call its `~HTTPResponse.rethrow` method.
"""
if self._closed:
raise RuntimeError("fetch() called on closed AsyncHTTPClient")
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
else:
if kwargs:
raise ValueError("kwargs can't be used if request is an HTTPRequest object")
# We may modify this (to add Host, Accept-Encoding, etc),
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
request.headers = httputil.HTTPHeaders(request.headers)
request = _RequestProxy(request, self.defaults)
future = TracebackFuture()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
exc = future.exception()
if isinstance(exc, HTTPError) and exc.response is not None:
response = exc.response
elif exc is not None:
response = HTTPResponse(
request, 599, error=exc,
request_time=time.time() - request.start_time)
else:
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def handle_response(response):
if raise_error and response.error:
future.set_exception(response.error)
else:
future.set_result(response)
self.fetch_impl(request, handle_response)
return future
def close(self):
if self._cursor is None:
future = TracebackFuture()
future.set_result(None)
return future
future = async_call_method(self._cursor.close)
def do_close(future):
self._cursor = None
future.add_done_callback(do_close)
return future
def register(self, name, callback=None):
msg_id = next(self._generator)
request = packer.dumps(('register', name))
buff = struct.pack('!ibi', len(request), RPC_REGISTER, msg_id) + request
future = TracebackFuture()
if callback:
future.add_done_callback(callback)
self.add_request_table(msg_id, future)
self.write(buff)
return future
def fetch(self, url, headers=None, body=None, method="GET", callback=None, raise_error=True, cache=None, **kwargs):
headers = headers or {}
body = body or "{}"
"""very simlar with AsyncHTTPClient.fetch
"""
if self._closed:
raise RuntimeError("fetch() called on closed AsyncHTTPClient")
future = TracebackFuture()
if isinstance(body, dict):
for k,v in body.items():
if v is None:
del body[k]
body = urllib.urlencode(body)
for k,v in headers.items(): #headers 只能接收str
if v:
headers[k] = str(headers[k])
else:
del headers[k]
request = HTTPRequest(url=url,method=method,headers=headers,body=body, allow_nonstandard_methods=True, request_timeout=600 ,**kwargs)
# We may modify this (to add Host, Accept-Encoding, etc),
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
request.headers = httputil.HTTPHeaders(request.headers)
request = _RequestProxy(request, self.defaults)
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
exc = future.exception()
if isinstance(exc, HTTPError) and exc.response is not None:
response = exc.response
elif exc is not None:
response = HTTPResponse(
request, 599, error=exc,
request_time=time.time() - request.start_time)
else:
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def handle_response(response):
if raise_error and response.error:
future.set_exception(response.error)
else:
try:
resp = json.loads(str(response.body))
if resp.get("statusCode") and resp.get("statusCode")==800:
future.set_result(resp)
log.info(json.dumps({"response":resp,"body":body,"headers":headers,"url":url}))
else:
future.set_result({"error_type":"statusCode is not 800", "response":resp,"body":body,"headers":headers,"url":url})
log.error(json.dumps({"error_type":"statusCode is not 800", "response":resp,"body":body,"headers":headers,"url":url}))
except Exception,e:
future.set_result({"error_type":"json.loads failed!","error":str(e),"response.body":response.body,"body":body,"headers":headers,"url":url})
log.error(json.dumps({"error_type":"json.loads failed!","error":str(e),"response.body":response.body,"body":body,"headers":headers,"url":url}))
def handle_yield(self, yielded):
# Lists containing YieldPoints require stack contexts;
# other lists are handled in convert_yielded.
if _contains_yieldpoint(yielded):
yielded = multi(yielded)
if isinstance(yielded, YieldPoint):
# YieldPoints are too closely coupled to the Runner to go
# through the generic convert_yielded mechanism.
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(
yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
else:
try:
self.future = convert_yielded(yielded)
except BadYieldError:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if not self.future.done() or self.future is moment:
def inner(f):
# Break a reference cycle to speed GC.
f = None # noqa
self.run()
self.io_loop.add_future(
self.future, inner)
return False
return True
def handle_yield(self, yielded):
# Lists containing YieldPoints require stack contexts;
# other lists are handled via multi_future in convert_yielded.
if (isinstance(yielded, list) and
any(isinstance(f, YieldPoint) for f in yielded)):
yielded = Multi(yielded)
elif (isinstance(yielded, dict) and
any(isinstance(f, YieldPoint) for f in yielded.values())):
yielded = Multi(yielded)
if isinstance(yielded, YieldPoint):
# YieldPoints are too closely coupled to the Runner to go
# through the generic convert_yielded mechanism.
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(
yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
else:
try:
self.future = convert_yielded(yielded)
except BadYieldError:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if not self.future.done() or self.future is moment:
self.io_loop.add_future(
self.future, lambda f: self.run())
return False
return True
def call(self, method_name, *arg, **kwargs):
_callback = kwargs.get('callback')
msg_id = next(self._generator)
request = packer.dumps((method_name, arg))
buff = struct.pack('!ibi', len(request), RPC_REQUEST, msg_id) + request
future = TracebackFuture()
self.add_request_table(msg_id, future)
if _callback:
future.add_done_callback(_callback)
self.write(buff)
return future
def create(self, param, req):
tableName = param['tableName']
initVal = param['initVal']
self.tables[tableName] = {}
for key in initVal:
self.tables[tableName][key] = initVal[key]
print 'Worker {0} CREATE:'.format(self.host), self.tables[tableName]
#Release
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def set(self, param, req):
tableName = param['tableName']
key = param['key']
val = param['val']
print 'Worker {0} SET: {1} with key {2} to {3}'.format(self.host, tableName, key, val)
self.tables[tableName][key] = val
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def fetch(self, request, callback=None, **kwargs):
"""Executes a request, asynchronously returning an `HTTPResponse`.
The request may be either a string URL or an `HTTPRequest` object.
If it is a string, we construct an `HTTPRequest` using any additional
kwargs: ``HTTPRequest(request, **kwargs)``
This method returns a `.Future` whose result is an
`HTTPResponse`. The ``Future`` wil raise an `HTTPError` if
the request returned a non-200 response code.
If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
In the callback interface, `HTTPError` is not automatically raised.
Instead, you must check the response's ``error`` attribute or
call its `~HTTPResponse.rethrow` method.
"""
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
# We may modify this (to add Host, Accept-Encoding, etc),
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
request.headers = httputil.HTTPHeaders(request.headers)
if options.client_trace:
request.trace.record(Annotation.client_send())
request = _RequestProxy(request, self.defaults)
future = TracebackFuture()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
exc = future.exception()
if isinstance(exc, HTTPError) and exc.response is not None:
response = exc.response
elif exc is not None:
response = HTTPResponse(
request, 599, error=exc,
request_time=time.time() - request.start_time)
else:
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def handle_response(response):
if response.error:
future.set_exception(response.error)
else:
future.set_result(response)
if options.client_trace:
request.trace.record(Annotation.client_recv())
self.fetch_impl(request, handle_response)
return future
def mock_fetch(self, request, callback=None, **kwargs):
future = TracebackFuture()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
res = MagicMock()
future.set_result(res)
return future
def fetch(self, request, callback=None, **kwargs):
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
response_partial = RequestCollection.find(request)
if response_partial:
resp = response_partial(request)
else:
resp = HTTPResponse(request, 404)
if callback is not None:
callback(resp)
else:
future = TracebackFuture()
future.set_result(resp)
return future
def handle_yield(self, yielded):
if isinstance(yielded, list):
if all(is_future(f) for f in yielded):
yielded = multi_future(yielded)
else:
yielded = Multi(yielded)
elif isinstance(yielded, dict):
if all(is_future(f) for f in yielded.values()):
yielded = multi_future(yielded)
else:
yielded = Multi(yielded)
if isinstance(yielded, YieldPoint):
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(
yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
elif is_future(yielded):
self.future = yielded
if not self.future.done() or self.future is moment:
self.io_loop.add_future(
self.future, lambda f: self.run())
return False
else:
self.future = TracebackFuture()
self.future.set_exception(BadYieldError(
"yielded unknown object %r" % (yielded,)))
return True
def wrapper(*args, **kwargs):
future = TracebackFuture()
callback, args, kwargs = replacer.replace(future, args, kwargs)
if callback is not None:
future.add_done_callback(
functools.partial(_auth_future_to_callback, callback))
def handle_exception(typ, value, tb):
if future.done():
return False
else:
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception):
f(*args, **kwargs)
return future
def remove(self, param, req):
tableName = param['tableName']
key = param['key']
if not key in self.tables[tableName]:
return Future().set_result('Key Error')
if key is None:
del self.tables[tableName]
else:
del self.tables[tableName][key]
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return Future().set_result('OK')
def handle_yield(self, yielded):
# Lists containing YieldPoints require stack contexts;
# other lists are handled in convert_yielded.
if _contains_yieldpoint(yielded):
yielded = multi(yielded)
if isinstance(yielded, YieldPoint):
# YieldPoints are too closely coupled to the Runner to go
# through the generic convert_yielded mechanism.
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
else:
try:
self.future = convert_yielded(yielded)
except BadYieldError:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if not self.future.done() or self.future is moment:
self.io_loop.add_future(self.future, lambda f: self.run())
return False
return True
def http_retry(client,
request,
raise_error=True,
attempts=5,
retry_wait=1,
**kwargs):
attempt = 1
future = TracebackFuture()
ioloop = IOLoop.current()
def _do_request(attempt):
http_future = client.fetch(request, raise_error=False, **kwargs)
http_future.add_done_callback(partial(handle_response, attempt))
def handle_response(attempt, future_response):
attempt += 1
result = future_response.result()
if result.error and\
attempt <= 5 and\
result.code >= 500 and\
result.code <= 599:
logging.error(result.error)
logging.error(result.body)
return ioloop.call_later(retry_wait, lambda: _do_request(attempt))
else:
if raise_error and result.error:
return future.set_exception(result.error)
future.set_result(result)
_do_request(attempt)
return future
def invoke(self, *args):
"""异步调用redis相关接口
:param args: 多条redis指令
"""
#默认开启multi
active_trans = True
write_buf = _chain_cmds(active_trans, *args)
future = TracebackFuture()
def handle_resp(resp):
f = resp.get(_RESP_FUTURE) or future
err = resp.get(RESP_ERR)
result = resp.get(RESP_RESULT)
if err:
f.set_exception(err)
else:
f.set_result(result)
with NullContext():
if self.__conn is None:
self.__conn = _RedisConnection(self.__io_loop, write_buf, handle_resp, self.__uri, self.__password)
self.__conn.connect(future, self.__uri, active_trans, len(args))
else:
self.__conn.write(write_buf, future, False, active_trans, len(args))
return future
def __init__(self, io_loop, request, compression_options=None):
self.compression_options = compression_options
self.connect_future = TracebackFuture()
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
scheme, sep, rest = request.url.partition(":")
scheme = {"ws": "http", "wss": "https"}[scheme]
request.url = scheme + sep + rest
request.headers.update(
{
"Upgrade": "websocket",
"Connection": "Upgrade",
"Sec-WebSocket-Key": self.key,
"Sec-WebSocket-Version": "13",
}
)
if self.compression_options is not None:
# Always offer to let the server set our max_wbits (and even though
# we don't offer it, we will accept a client_no_context_takeover
# from the server).
# TODO: set server parameters for deflate extension
# if requested in self.compression_options.
request.headers["Sec-WebSocket-Extensions"] = "permessage-deflate; client_max_window_bits"
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection, self).__init__(
io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.tcp_client, 65536
)
def write(self, data, callback=None):
"""Write the given data to this stream.
If ``callback`` is given, we call it when all of the buffered write
data has been successfully written to the stream. If there was
previously buffered write data and an old write callback, that
callback is simply overwritten with this new callback.
"""
assert isinstance(data, bytes_type)
self._check_closed()
# We use bool(_write_buffer) as a proxy for write_buffer_size>0,
# so never put empty strings in the buffer.
if data:
# Break up large contiguous strings before inserting them in the
# write buffer, so we don't have to recopy the entire thing
# as we slice off pieces to send to the socket.
WRITE_BUFFER_CHUNK_SIZE = 128 * 1024
for i in range(0, len(data), WRITE_BUFFER_CHUNK_SIZE):
self._write_buffer.append(data[i:i + WRITE_BUFFER_CHUNK_SIZE])
if callback is not None:
self._write_callback = stack_context.wrap(callback)
future = None
else:
future = self._write_future = TracebackFuture()
if not self._connecting:
self._handle_write()
if self._write_buffer:
self._add_io_state(self.io_loop.WRITE)
self._maybe_add_error_listener()
return future
def wrapper(*args, **kwargs):
future = TracebackFuture()
callback, args, kwargs = replacer.replace(future, args, kwargs)
if callback is not None:
future.add_done_callback(
functools.partial(_auth_future_to_callback, callback))
def handle_exception(typ, value, tb):
if future.done():
return False
else:
future.set_exc_info((typ, value, tb))
return True
with ExceptionStackContext(handle_exception):
f(*args, **kwargs)
return future
def run():
try:
result = func()
if result is not None:
from tornado.gen import convert_yielded
result = convert_yielded(result)
except Exception:
future_cell[0] = TracebackFuture()
future_cell[0].set_exc_info(sys.exc_info())
else:
if is_future(result):
future_cell[0] = result
else:
future_cell[0] = TracebackFuture()
future_cell[0].set_result(result)
self.add_future(future_cell[0], lambda future: self.stop())
def wrapper(*args, **kwargs):
if replacer.get_old_value(args, kwargs) is not None:
# If a callaback is present, just call in to the decorated
# function. This is a slight optimization (by not creating a
# Future that is unlikely to be used), but mainly avoids the
# complexity of running the callback in the expected way.
return f(*args, **kwargs)
future = TracebackFuture()
callback, args, kwargs = replacer.replace(
lambda value=None: future.set_result(value), args, kwargs)
f(*args, **kwargs)
stream = args[0]
stream._pending_futures.add(future)
future.add_done_callback(
lambda fut: stream._pending_futures.discard(fut))
return future
def __init__(self, future=None):
if future is not None:
self._future = future
else:
self._future = TracebackFuture()
self._future.add_done_callback(functools.partial(async_result_complete, self))
self._condition = threading.Condition()
def __init__(self, io_loop, request, on_message_callback=None,
compression_options=None):
del request, on_message_callback, compression_options
self.connect_future = TracebackFuture()
self.written_messages = []
io_loop.add_timeout(self.TIMEOUT, self._complete)
def find_one(self, **kwargs):
"""Returns future.
Executes collection's find_one method based on keyword args
maps result ( dict to instance ) and return future
Example::
manager = EntityManager(Product)
product_saved = yield manager.find_one(_id=object_id)
"""
future = TracebackFuture()
def handle_response(result, error):
if error:
future.set_exception(error)
else:
instance = self.__entity()
instance.map_dict(result)
future.set_result(instance)
self.__collection.find_one(kwargs, callback=handle_response)
return future
def method(self, *args, **kwargs):
#loop = self.get_io_loop()
loop = ioloop.IOLoop.current()
callback = kwargs.pop('callback', None)
if callback:
if not callable(callback):
raise callback_type_error
future = None
else:
future = TracebackFuture()
def call_method():
# Runs on child greenlet.
try:
result = sync_method(self, *args, **kwargs) # 用fun传对象调用方法
if callback:
# Schedule callback(result, None) on main greenlet.
loop.add_callback(functools.partial(
callback, result, None))
else:
# Schedule future to be resolved on main greenlet.
loop.add_callback(functools.partial(
future.set_result, result))
except Exception as e:
if callback:
loop.add_callback(functools.partial(
callback, None, e))
else:
loop.add_callback(functools.partial(
future.set_exc_info, sys.exc_info()))
# Start running the operation on a greenlet.
greenlet.greenlet(call_method).switch()
return future
def send_request(self, request):
method = request.method
data = request.body
headers = {'Content-Type': 'application/json'}
future = TracebackFuture()
def process_response_future(response):
if response.exc_info() is not None:
future.set_exc_info(response.exc_info())
elif response.exception() is not None:
future.set_exception(response.exception())
else:
result = response.result()
code = result.code
body = (result.body or b'').decode('utf8')
future.set_result(process_response(code, body))
request = tornado.httpclient.HTTPRequest(
request.url,
method=method,
body=data,
headers=headers,
request_timeout=self.client.timeout)
response_future = self.http.fetch(request, raise_error=False)
response_future.add_done_callback(process_response_future)
return future
def __init__(self, io_loop, request, on_message_callback=None,
compression_options=None):
self.compression_options = compression_options
self.connect_future = TracebackFuture()
self.protocol = None
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
self._on_message_callback = on_message_callback
self.close_code = self.close_reason = None
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
if self.compression_options is not None:
# Always offer to let the server set our max_wbits (and even though
# we don't offer it, we will accept a client_no_context_takeover
# from the server).
# TODO: set server parameters for deflate extension
# if requested in self.compression_options.
request.headers['Sec-WebSocket-Extensions'] = (
'permessage-deflate; client_max_window_bits')
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection, self).__init__(
io_loop, None, request, lambda: None, self._on_http_response,
104857600, self.tcp_client, 65536, 104857600)
def fetch(self, request, callback=None, **kwargs):
"""Executes a request, asynchronously returning an `HTTPResponse`.
The request may be either a string URL or an `HTTPRequest` object.
If it is a string, we construct an `HTTPRequest` using any additional
kwargs: ``HTTPRequest(request, **kwargs)``
This method returns a `.Future` whose result is an
`HTTPResponse`. The ``Future`` will raise an `HTTPError` if
the request returned a non-200 response code.
If a ``callback`` is given, it will be invoked with the `HTTPResponse`.
In the callback interface, `HTTPError` is not automatically raised.
Instead, you must check the response's ``error`` attribute or
call its `~HTTPResponse.rethrow` method.
"""
if not isinstance(request, HTTPRequest):
request = HTTPRequest(url=request, **kwargs)
# We may modify this (to add Host, Accept-Encoding, etc),
# so make sure we don't modify the caller's object. This is also
# where normal dicts get converted to HTTPHeaders objects.
request.headers = httputil.HTTPHeaders(request.headers)
request = _RequestProxy(request, self.defaults)
future = TracebackFuture()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
exc = future.exception()
if isinstance(exc, HTTPError) and exc.response is not None:
response = exc.response
elif exc is not None:
response = HTTPResponse(
request, 599, error=exc,
request_time=time.time() - request.start_time)
else:
response = future.result()
self.io_loop.add_callback(callback, response)
future.add_done_callback(handle_future)
def handle_response(response):
if response.error:
future.set_exception(response.error)
else:
future.set_result(response)
self.fetch_impl(request, handle_response)
return future
def wrapper(*args, **kwargs):
if replacer.get_old_value(args, kwargs) is not None:
# If a callaback is present, just call in to the decorated
# function. This is a slight optimization (by not creating a
# Future that is unlikely to be used), but mainly avoids the
# complexity of running the callback in the expected way.
return f(*args, **kwargs)
future = TracebackFuture()
callback, args, kwargs = replacer.replace(
lambda value=None: future.set_result(value),
args, kwargs)
f(*args, **kwargs)
stream = args[0]
stream._pending_futures.add(future)
future.add_done_callback(
lambda fut: stream._pending_futures.discard(fut))
return future
def exec_command(self, command):
request_log.info('EXECUTE %s', command)
try:
self.future = TracebackFuture()
transport = self.client.get_transport()
self.channel = transport.open_session()
self.channel.exec_command(command)
self.io_loop.add_handler(self.channel.fileno(),
self._handle_events,
self.io_loop.READ | self.io_loop.ERROR)
return self.future
except Exception as e:
print(e)
pass
def read_message(self, callback=None):
"""读取来自 WebSocket 服务器的消息.
如果在 WebSocket 初始化时指定了 on_message_callback ,那么这个方法永远不会返回消息
如果连接已经关闭,返回结果会是一个结果是 message 的 future 对象或者是 None.
如果 future 给出了回调参数, 这个参数将会在 future 完成时调用.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def submit(self, fn, *args, **kwargs):
future = TracebackFuture()
def _cb():
chain_future(self._executor.submit(fn, *args, **kwargs), future)
IOLoop.current().add_callback(_cb)
return future
def read_message(self, callback=None):
"""Reads a message from the WebSocket server.
Returns a future whose result is the message, or None
if the connection is closed. If a callback argument
is given it will be called with the future when it is
ready.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def _set_read_callback(self, callback):
assert self._read_callback is None, "Already reading"
assert self._read_future is None, "Already reading"
if callback is not None:
self._read_callback = stack_context.wrap(callback)
else:
self._read_future = TracebackFuture()
return self._read_future
def handle_exception(self, typ, value, tb):
if not self.running and not self.finished:
self.future = TracebackFuture()
self.future.set_exc_info((typ, value, tb))
self.run()
return True
else:
return False
def read_message(self, callback=None):
"""Reads a message from the WebSocket server.
Returns a future whose result is the message, or None
if the connection is closed. If a callback argument
is given it will be called with the future when it is
ready.
"""
assert self.read_future is None
future = TracebackFuture()
if self.read_queue:
future.set_result(self.read_queue.popleft())
else:
self.read_future = future
if callback is not None:
self.io_loop.add_future(future, callback)
return future
def exec_command(self, command):
try:
self.future = TracebackFuture()
self.channel = self.client.get_transport().open_session()
self.channel.exec_command(command)
# t=self.channel.recv(1024)
# print(t)
print("EXEC COMMAND[%s] STATUS: %s" %
(command, self.channel.recv_exit_status()))
self.io_loop.add_handler(self.channel.fileno(),
self._handle_events,
self.io_loop.READ | self.io_loop.ERROR)
return self.future
except Exception as e:
print(e)
pass
def fetch_msg(uid, msg_type, data, qos, timeout, callback=None):
future = TracebackFuture()
if callback is not None:
callback = stack_context.wrap(callback)
def handle_future(future):
response = future.result()
IOLoop.current().add_callback(callback, response)
future.add_done_callback(handle_future)
def web_handle_response(response=''):
future.set_result(response)
WebSocketHandler.send_message(
uid, msg_type, data,
qos, timeout, web_handle_response)
return future
def next(self):
"""Returns a `.Future` that will yield the next available result.
Note that this `.Future` will not be the same object as any of
the inputs.
"""
self._running_future = TracebackFuture()
# As long as there is an active _running_future, we must
# ensure that the WaitIterator is not GC'd (due to the
# use of weak references in __init__). Add a callback that
# references self so there is a hard reference that will be
# cleared automatically when this Future finishes.
self._running_future.add_done_callback(lambda f: self)
if self._finished:
self._return_result(self._finished.popleft())
return self._running_future
def __init__(self, io_loop, request):
self.connect_future = TracebackFuture()
self.read_future = None
self.read_queue = collections.deque()
self.events = []
if old_tornado:
self.resolver = Resolver(io_loop=io_loop)
super(EventSourceClient,
self).__init__(io_loop, None, request, lambda: None,
self._on_http_response, 104857600,
self.resolver)
else:
self.tcp_client = TCPClient(io_loop=io_loop)
super(EventSourceClient,
self).__init__(io_loop, None, request, lambda: None,
self._on_http_response, 104857600,
self.tcp_client, 65536)
def __init__(self,
io_loop,
request,
on_message_callback=None,
compression_options=None,
ping_interval=None,
ping_timeout=None,
max_message_size=None):
self.compression_options = compression_options
self.connect_future = TracebackFuture()
self.protocol = None
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
self._on_message_callback = on_message_callback
self.close_code = self.close_reason = None
self.ping_interval = ping_interval
self.ping_timeout = ping_timeout
self.max_message_size = max_message_size
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
if self.compression_options is not None:
# Always offer to let the server set our max_wbits (and even though
# we don't offer it, we will accept a client_no_context_takeover
# from the server).
# TODO: set server parameters for deflate extension
# if requested in self.compression_options.
request.headers['Sec-WebSocket-Extensions'] = (
'permessage-deflate; client_max_window_bits')
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection,
self).__init__(io_loop, None, request, lambda: None,
self._on_http_response, 104857600,
self.tcp_client, 65536, 104857600)
def remove(self, param, req):
tableName = param['tableName'].name
key = param['key']
if not tableName in self.tables:
#Silently fails
return
args = {'tableName': tableName, 'key': key}
futures = []
for worker in self.workers:
futures.append(
self.client.fetch(formatQuery(worker, 'remove', args)))
if key is None:
del self.tables[tableName]
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def put_file(self, filename):
try:
self.future = TracebackFuture()
transport = self.client.get_transport()
self.sftp_client = SFTPClient.from_transport(transport)
self.remote = self.sftp_client.open(filename, 'wb')
self.local = open(filename, 'rb')
# self.remote.setblocking(0)
# _buf = self.local.read(100)
# self.remote.write(_buf)
self.io_loop.add_handler(self.sftp_client.sock, self._handle_write,
self.io_loop.WRITE | self.io_loop.ERROR)
return self.future
except Exception as e:
print(e)
pass
def set(self, param, req):
tableName = param['tableName'].name
key = param['key']
val = param['val']
if not tableName in self.tables:
raise KeyError, 'Table not Found!'
worker = self.workers[hash(key) % self.num_workers]
args = {'tableName': tableName, 'key': key, 'val': val}
print 'MASTER -> {0} SET: {1} with key {2} to {3}'.format(
worker, tableName, key, val)
futures = []
futures.append(self.client.fetch(formatQuery(worker, 'set', args)))
fu = Future()
req.url = self.host
fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK')))
return fu
def handle_yield(self, yielded):
if isinstance(yielded, (list, dict)):
yielded = Multi(yielded)
if isinstance(yielded, YieldPoint):
self.future = TracebackFuture()
def start_yield_point():
try:
yielded.start(self)
if yielded.is_ready():
self.future.set_result(yielded.get_result())
else:
self.yield_point = yielded
except Exception:
self.future = TracebackFuture()
self.future.set_exc_info(sys.exc_info())
if self.stack_context_deactivate is None:
# Start a stack context if this is the first
# YieldPoint we've seen.
with stack_context.ExceptionStackContext(
self.handle_exception) as deactivate:
self.stack_context_deactivate = deactivate
def cb():
start_yield_point()
self.run()
self.io_loop.add_callback(cb)
return False
else:
start_yield_point()
elif is_future(yielded):
self.future = yielded
if not self.future.done() or self.future is moment:
self.io_loop.add_future(self.future, lambda f: self.run())
return False
else:
self.future = TracebackFuture()
self.future.set_exception(
BadYieldError("yielded unknown object %r" % (yielded, )))
return True
def read(self, callback=None):
assert not self.reading(), 'Already reading'
self.bytes_to_read = None
if callback is not None:
self._read_callback = stack_context.wrap(callback)
future = None
else:
future = self._read_callback = TracebackFuture()
future.add_done_callback(lambda f: f.exception())
return future
def __init__(self, io_loop, request):
self.connect_future = TracebackFuture()
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
self.resolver = Resolver(io_loop=io_loop)
super(WebSocketClientConnection,
self).__init__(io_loop, None, request, lambda: None,
self._on_http_response, 104857600, self.resolver)
def read(self, callback=None):
assert not self.reading(), 'Already reading'
if callback is not None:
self._read_callback = stack_context.wrap(callback)
future = None
else:
future = self._read_future = TracebackFuture()
future.add_done_callback(lambda f: f.exception())
self._add_io_state(self.io_loop.READ)
return future
def execute(self, request):
''' Starts the producer to send the next request to
consumer.write(Frame(request))
'''
request.transaction_id = self.transaction.getNextTID()
packet = self.framer.buildPacket(request)
if not self.transport:
raise ConnectionException("Slave not connected")
yield self.transport.write(packet)
future = TracebackFuture()
self.transaction.addTransaction(future, request.transaction_id)
res = yield future
raise gen.Return(res)
def run():
try:
result = func()
except Exception:
future_cell[0] = TracebackFuture()
future_cell[0].set_exc_info(sys.exc_info())
else:
if isinstance(result, Future):
future_cell[0] = result
else:
future_cell[0] = Future()
future_cell[0].set_result(result)
self.add_future(future_cell[0], lambda future: self.stop())