def create(self, param, req): tableName = param['tableName'].name initVal = param['initVal'] if tableName in self.tables: raise KeyError, 'Table name already existed' #print 'MASTER CREATE:', initVal self.tables[tableName] = {'len': 0, } #Data Partition vals = {} for worker_id in xrange(self.num_workers): vals[worker_id] = {} for key in initVal: #Bookkeeping self.tables[tableName]['len'] += 1 self.tables[tableName][key] = type(initVal[key]) worker = hash(key) % self.num_workers vals[worker][key] = initVal[key] futures = [] for worker_id in xrange(self.num_workers): args = {'tableName': tableName, 'initVal': vals[worker_id]} futures.append(self.client.fetch(formatQuery(self.workers[worker_id], 'create', args))) fu = Future() req.url = self.host fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK'))) return fu
def start_tls(self, server_side, ssl_options, server_hostname=None): if not isinstance(ssl_options, SSL.Context): raise ValueError("ssl_options is not SSL.Context") _socket = self.detach() _socket = SSL.Connection(ssl_options, _socket) if server_side: _socket.set_accept_state() else: _socket.set_connect_state() if server_hostname: _socket.set_tlsext_host_name(server_hostname.encode("idna")) orig_close_callback = self._close_callback self._close_callback = None future = TracebackFuture() ssl_stream = MicroProxySSLIOStream(_socket, server_hostname=server_hostname, ssl_options=ssl_options, io_loop=self.io_loop) def close_callback(): if not future.done(): future.set_exception(ssl_stream.error or StreamClosedError()) if orig_close_callback is not None: orig_close_callback() ssl_stream.set_close_callback(close_callback) ssl_stream._ssl_connect_callback = lambda: future.set_result(ssl_stream) ssl_stream.max_buffer_size = self.max_buffer_size ssl_stream.read_chunk_size = self.read_chunk_size return future
def _defer_to_future(defer): future = TracebackFuture() defer.addCallbacks( future.set_result, lambda failure: future.set_exc_info( (failure.type, failure.value, failure.tb))) return future
class StubWebSocketClientConnection(object): """A stub version of tornado.websocket.WebSocketClientConnection for use in unit tests. Attributes: TIMEOUT: class constant, which should be overridden in subclasses, in order to vary the time from when the class is instantiated until the returned future is resolved. connect_future: the Future resolved when the client "connection" is made. written_messages: a list of messages passed to the write_message method to see they have been received. """ TIMEOUT = datetime.timedelta(milliseconds=0) def __init__(self, io_loop, request, on_message_callback=None, compression_options=None): del request, on_message_callback, compression_options self.connect_future = TracebackFuture() self.written_messages = [] io_loop.add_timeout(self.TIMEOUT, self._complete) def _complete(self): self.connect_future.set_result(self) def write_message(self, message): """Adds writen message to the written_messages buffer.""" self.written_messages.append(message)
def open(self): future = TracebackFuture() try: self._stream.open(lambda :future.set_result(None)) except Exception: future.set_exc_info(sys.exc_info()) return future
def fetch_all(self, param, req): tableName = param['tableName'] if not tableName in self.tables: raise KeyError, 'Table Not Found!' fu = Future() req.url = self.host fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO(pickle.dumps(self.tables[tableName])))) return fu
def get(self, param, req): tableName = param['tableName'] key = param['key'] fu = Future() req.url = self.host fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO(pickle.dumps(self.tables[tableName][key])))) return fu
def len(self, param, req): tableName = param['tableName'].name if not tableName in self.tables: raise KeyError, 'Table Not Found' fu = Future() req.url = self.host fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO(str(self.tables[tableName]['len'])))) return fu
def fetch(self, request, callback=None, raise_error=True, **kwargs): """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` This method returns a `.Future` whose result is an `HTTPResponse`. By default, the ``Future`` will raise an `HTTPError` if the request returned a non-200 response code (other errors may also be raised if the server could not be contacted). Instead, if ``raise_error`` is set to False, the response will always be returned regardless of the response code. If a ``callback`` is given, it will be invoked with the `HTTPResponse`. In the callback interface, `HTTPError` is not automatically raised. Instead, you must check the response's ``error`` attribute or call its `~HTTPResponse.rethrow` method. """ if self._closed: raise RuntimeError("fetch() called on closed AsyncHTTPClient") if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) else: if kwargs: raise ValueError("kwargs can't be used if request is an HTTPRequest object") # We may modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) request = _RequestProxy(request, self.defaults) future = TracebackFuture() if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): exc = future.exception() if isinstance(exc, HTTPError) and exc.response is not None: response = exc.response elif exc is not None: response = HTTPResponse( request, 599, error=exc, request_time=time.time() - request.start_time) else: response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) def handle_response(response): if raise_error and response.error: future.set_exception(response.error) else: future.set_result(response) self.fetch_impl(request, handle_response) return future
def close(self): if self._cursor is None: future = TracebackFuture() future.set_result(None) return future future = async_call_method(self._cursor.close) def do_close(future): self._cursor = None future.add_done_callback(do_close) return future
def register(self, name, callback=None): msg_id = next(self._generator) request = packer.dumps(('register', name)) buff = struct.pack('!ibi', len(request), RPC_REGISTER, msg_id) + request future = TracebackFuture() if callback: future.add_done_callback(callback) self.add_request_table(msg_id, future) self.write(buff) return future
def fetch(self, url, headers=None, body=None, method="GET", callback=None, raise_error=True, cache=None, **kwargs): headers = headers or {} body = body or "{}" """very simlar with AsyncHTTPClient.fetch """ if self._closed: raise RuntimeError("fetch() called on closed AsyncHTTPClient") future = TracebackFuture() if isinstance(body, dict): for k,v in body.items(): if v is None: del body[k] body = urllib.urlencode(body) for k,v in headers.items(): #headers 只能接收str if v: headers[k] = str(headers[k]) else: del headers[k] request = HTTPRequest(url=url,method=method,headers=headers,body=body, allow_nonstandard_methods=True, request_timeout=600 ,**kwargs) # We may modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) request = _RequestProxy(request, self.defaults) if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): exc = future.exception() if isinstance(exc, HTTPError) and exc.response is not None: response = exc.response elif exc is not None: response = HTTPResponse( request, 599, error=exc, request_time=time.time() - request.start_time) else: response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) def handle_response(response): if raise_error and response.error: future.set_exception(response.error) else: try: resp = json.loads(str(response.body)) if resp.get("statusCode") and resp.get("statusCode")==800: future.set_result(resp) log.info(json.dumps({"response":resp,"body":body,"headers":headers,"url":url})) else: future.set_result({"error_type":"statusCode is not 800", "response":resp,"body":body,"headers":headers,"url":url}) log.error(json.dumps({"error_type":"statusCode is not 800", "response":resp,"body":body,"headers":headers,"url":url})) except Exception,e: future.set_result({"error_type":"json.loads failed!","error":str(e),"response.body":response.body,"body":body,"headers":headers,"url":url}) log.error(json.dumps({"error_type":"json.loads failed!","error":str(e),"response.body":response.body,"body":body,"headers":headers,"url":url}))
def handle_yield(self, yielded): # Lists containing YieldPoints require stack contexts; # other lists are handled in convert_yielded. if _contains_yieldpoint(yielded): yielded = multi(yielded) if isinstance(yielded, YieldPoint): # YieldPoints are too closely coupled to the Runner to go # through the generic convert_yielded mechanism. self.future = TracebackFuture() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): self.future.set_result( yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() else: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if not self.future.done() or self.future is moment: def inner(f): # Break a reference cycle to speed GC. f = None # noqa self.run() self.io_loop.add_future( self.future, inner) return False return True
def handle_yield(self, yielded): # Lists containing YieldPoints require stack contexts; # other lists are handled via multi_future in convert_yielded. if (isinstance(yielded, list) and any(isinstance(f, YieldPoint) for f in yielded)): yielded = Multi(yielded) elif (isinstance(yielded, dict) and any(isinstance(f, YieldPoint) for f in yielded.values())): yielded = Multi(yielded) if isinstance(yielded, YieldPoint): # YieldPoints are too closely coupled to the Runner to go # through the generic convert_yielded mechanism. self.future = TracebackFuture() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): self.future.set_result( yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() else: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if not self.future.done() or self.future is moment: self.io_loop.add_future( self.future, lambda f: self.run()) return False return True
def call(self, method_name, *arg, **kwargs): _callback = kwargs.get('callback') msg_id = next(self._generator) request = packer.dumps((method_name, arg)) buff = struct.pack('!ibi', len(request), RPC_REQUEST, msg_id) + request future = TracebackFuture() self.add_request_table(msg_id, future) if _callback: future.add_done_callback(_callback) self.write(buff) return future
def create(self, param, req): tableName = param['tableName'] initVal = param['initVal'] self.tables[tableName] = {} for key in initVal: self.tables[tableName][key] = initVal[key] print 'Worker {0} CREATE:'.format(self.host), self.tables[tableName] #Release fu = Future() req.url = self.host fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK'))) return fu
def set(self, param, req): tableName = param['tableName'] key = param['key'] val = param['val'] print 'Worker {0} SET: {1} with key {2} to {3}'.format(self.host, tableName, key, val) self.tables[tableName][key] = val fu = Future() req.url = self.host fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK'))) return fu
def fetch(self, request, callback=None, **kwargs): """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` This method returns a `.Future` whose result is an `HTTPResponse`. The ``Future`` wil raise an `HTTPError` if the request returned a non-200 response code. If a ``callback`` is given, it will be invoked with the `HTTPResponse`. In the callback interface, `HTTPError` is not automatically raised. Instead, you must check the response's ``error`` attribute or call its `~HTTPResponse.rethrow` method. """ if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) # We may modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) if options.client_trace: request.trace.record(Annotation.client_send()) request = _RequestProxy(request, self.defaults) future = TracebackFuture() if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): exc = future.exception() if isinstance(exc, HTTPError) and exc.response is not None: response = exc.response elif exc is not None: response = HTTPResponse( request, 599, error=exc, request_time=time.time() - request.start_time) else: response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) def handle_response(response): if response.error: future.set_exception(response.error) else: future.set_result(response) if options.client_trace: request.trace.record(Annotation.client_recv()) self.fetch_impl(request, handle_response) return future
def mock_fetch(self, request, callback=None, **kwargs): future = TracebackFuture() if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) res = MagicMock() future.set_result(res) return future
def fetch(self, request, callback=None, **kwargs): if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) response_partial = RequestCollection.find(request) if response_partial: resp = response_partial(request) else: resp = HTTPResponse(request, 404) if callback is not None: callback(resp) else: future = TracebackFuture() future.set_result(resp) return future
def handle_yield(self, yielded): if isinstance(yielded, list): if all(is_future(f) for f in yielded): yielded = multi_future(yielded) else: yielded = Multi(yielded) elif isinstance(yielded, dict): if all(is_future(f) for f in yielded.values()): yielded = multi_future(yielded) else: yielded = Multi(yielded) if isinstance(yielded, YieldPoint): self.future = TracebackFuture() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): self.future.set_result( yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() elif is_future(yielded): self.future = yielded if not self.future.done() or self.future is moment: self.io_loop.add_future( self.future, lambda f: self.run()) return False else: self.future = TracebackFuture() self.future.set_exception(BadYieldError( "yielded unknown object %r" % (yielded,))) return True
def wrapper(*args, **kwargs): future = TracebackFuture() callback, args, kwargs = replacer.replace(future, args, kwargs) if callback is not None: future.add_done_callback( functools.partial(_auth_future_to_callback, callback)) def handle_exception(typ, value, tb): if future.done(): return False else: future.set_exc_info((typ, value, tb)) return True with ExceptionStackContext(handle_exception): f(*args, **kwargs) return future
def remove(self, param, req): tableName = param['tableName'] key = param['key'] if not key in self.tables[tableName]: return Future().set_result('Key Error') if key is None: del self.tables[tableName] else: del self.tables[tableName][key] fu = Future() req.url = self.host fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK'))) return Future().set_result('OK')
def handle_yield(self, yielded): # Lists containing YieldPoints require stack contexts; # other lists are handled in convert_yielded. if _contains_yieldpoint(yielded): yielded = multi(yielded) if isinstance(yielded, YieldPoint): # YieldPoints are too closely coupled to the Runner to go # through the generic convert_yielded mechanism. self.future = TracebackFuture() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): self.future.set_result(yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() else: try: self.future = convert_yielded(yielded) except BadYieldError: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if not self.future.done() or self.future is moment: self.io_loop.add_future(self.future, lambda f: self.run()) return False return True
def http_retry(client, request, raise_error=True, attempts=5, retry_wait=1, **kwargs): attempt = 1 future = TracebackFuture() ioloop = IOLoop.current() def _do_request(attempt): http_future = client.fetch(request, raise_error=False, **kwargs) http_future.add_done_callback(partial(handle_response, attempt)) def handle_response(attempt, future_response): attempt += 1 result = future_response.result() if result.error and\ attempt <= 5 and\ result.code >= 500 and\ result.code <= 599: logging.error(result.error) logging.error(result.body) return ioloop.call_later(retry_wait, lambda: _do_request(attempt)) else: if raise_error and result.error: return future.set_exception(result.error) future.set_result(result) _do_request(attempt) return future
def invoke(self, *args): """异步调用redis相关接口 :param args: 多条redis指令 """ #默认开启multi active_trans = True write_buf = _chain_cmds(active_trans, *args) future = TracebackFuture() def handle_resp(resp): f = resp.get(_RESP_FUTURE) or future err = resp.get(RESP_ERR) result = resp.get(RESP_RESULT) if err: f.set_exception(err) else: f.set_result(result) with NullContext(): if self.__conn is None: self.__conn = _RedisConnection(self.__io_loop, write_buf, handle_resp, self.__uri, self.__password) self.__conn.connect(future, self.__uri, active_trans, len(args)) else: self.__conn.write(write_buf, future, False, active_trans, len(args)) return future
def __init__(self, io_loop, request, compression_options=None): self.compression_options = compression_options self.connect_future = TracebackFuture() self.read_future = None self.read_queue = collections.deque() self.key = base64.b64encode(os.urandom(16)) scheme, sep, rest = request.url.partition(":") scheme = {"ws": "http", "wss": "https"}[scheme] request.url = scheme + sep + rest request.headers.update( { "Upgrade": "websocket", "Connection": "Upgrade", "Sec-WebSocket-Key": self.key, "Sec-WebSocket-Version": "13", } ) if self.compression_options is not None: # Always offer to let the server set our max_wbits (and even though # we don't offer it, we will accept a client_no_context_takeover # from the server). # TODO: set server parameters for deflate extension # if requested in self.compression_options. request.headers["Sec-WebSocket-Extensions"] = "permessage-deflate; client_max_window_bits" self.tcp_client = TCPClient(io_loop=io_loop) super(WebSocketClientConnection, self).__init__( io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.tcp_client, 65536 )
def write(self, data, callback=None): """Write the given data to this stream. If ``callback`` is given, we call it when all of the buffered write data has been successfully written to the stream. If there was previously buffered write data and an old write callback, that callback is simply overwritten with this new callback. """ assert isinstance(data, bytes_type) self._check_closed() # We use bool(_write_buffer) as a proxy for write_buffer_size>0, # so never put empty strings in the buffer. if data: # Break up large contiguous strings before inserting them in the # write buffer, so we don't have to recopy the entire thing # as we slice off pieces to send to the socket. WRITE_BUFFER_CHUNK_SIZE = 128 * 1024 for i in range(0, len(data), WRITE_BUFFER_CHUNK_SIZE): self._write_buffer.append(data[i:i + WRITE_BUFFER_CHUNK_SIZE]) if callback is not None: self._write_callback = stack_context.wrap(callback) future = None else: future = self._write_future = TracebackFuture() if not self._connecting: self._handle_write() if self._write_buffer: self._add_io_state(self.io_loop.WRITE) self._maybe_add_error_listener() return future
def run(): try: result = func() if result is not None: from tornado.gen import convert_yielded result = convert_yielded(result) except Exception: future_cell[0] = TracebackFuture() future_cell[0].set_exc_info(sys.exc_info()) else: if is_future(result): future_cell[0] = result else: future_cell[0] = TracebackFuture() future_cell[0].set_result(result) self.add_future(future_cell[0], lambda future: self.stop())
def wrapper(*args, **kwargs): if replacer.get_old_value(args, kwargs) is not None: # If a callaback is present, just call in to the decorated # function. This is a slight optimization (by not creating a # Future that is unlikely to be used), but mainly avoids the # complexity of running the callback in the expected way. return f(*args, **kwargs) future = TracebackFuture() callback, args, kwargs = replacer.replace( lambda value=None: future.set_result(value), args, kwargs) f(*args, **kwargs) stream = args[0] stream._pending_futures.add(future) future.add_done_callback( lambda fut: stream._pending_futures.discard(fut)) return future
def __init__(self, future=None): if future is not None: self._future = future else: self._future = TracebackFuture() self._future.add_done_callback(functools.partial(async_result_complete, self)) self._condition = threading.Condition()
def __init__(self, io_loop, request, on_message_callback=None, compression_options=None): del request, on_message_callback, compression_options self.connect_future = TracebackFuture() self.written_messages = [] io_loop.add_timeout(self.TIMEOUT, self._complete)
def find_one(self, **kwargs): """Returns future. Executes collection's find_one method based on keyword args maps result ( dict to instance ) and return future Example:: manager = EntityManager(Product) product_saved = yield manager.find_one(_id=object_id) """ future = TracebackFuture() def handle_response(result, error): if error: future.set_exception(error) else: instance = self.__entity() instance.map_dict(result) future.set_result(instance) self.__collection.find_one(kwargs, callback=handle_response) return future
def method(self, *args, **kwargs): #loop = self.get_io_loop() loop = ioloop.IOLoop.current() callback = kwargs.pop('callback', None) if callback: if not callable(callback): raise callback_type_error future = None else: future = TracebackFuture() def call_method(): # Runs on child greenlet. try: result = sync_method(self, *args, **kwargs) # 用fun传对象调用方法 if callback: # Schedule callback(result, None) on main greenlet. loop.add_callback(functools.partial( callback, result, None)) else: # Schedule future to be resolved on main greenlet. loop.add_callback(functools.partial( future.set_result, result)) except Exception as e: if callback: loop.add_callback(functools.partial( callback, None, e)) else: loop.add_callback(functools.partial( future.set_exc_info, sys.exc_info())) # Start running the operation on a greenlet. greenlet.greenlet(call_method).switch() return future
def send_request(self, request): method = request.method data = request.body headers = {'Content-Type': 'application/json'} future = TracebackFuture() def process_response_future(response): if response.exc_info() is not None: future.set_exc_info(response.exc_info()) elif response.exception() is not None: future.set_exception(response.exception()) else: result = response.result() code = result.code body = (result.body or b'').decode('utf8') future.set_result(process_response(code, body)) request = tornado.httpclient.HTTPRequest( request.url, method=method, body=data, headers=headers, request_timeout=self.client.timeout) response_future = self.http.fetch(request, raise_error=False) response_future.add_done_callback(process_response_future) return future
def __init__(self, io_loop, request, on_message_callback=None, compression_options=None): self.compression_options = compression_options self.connect_future = TracebackFuture() self.protocol = None self.read_future = None self.read_queue = collections.deque() self.key = base64.b64encode(os.urandom(16)) self._on_message_callback = on_message_callback self.close_code = self.close_reason = None scheme, sep, rest = request.url.partition(':') scheme = {'ws': 'http', 'wss': 'https'}[scheme] request.url = scheme + sep + rest request.headers.update({ 'Upgrade': 'websocket', 'Connection': 'Upgrade', 'Sec-WebSocket-Key': self.key, 'Sec-WebSocket-Version': '13', }) if self.compression_options is not None: # Always offer to let the server set our max_wbits (and even though # we don't offer it, we will accept a client_no_context_takeover # from the server). # TODO: set server parameters for deflate extension # if requested in self.compression_options. request.headers['Sec-WebSocket-Extensions'] = ( 'permessage-deflate; client_max_window_bits') self.tcp_client = TCPClient(io_loop=io_loop) super(WebSocketClientConnection, self).__init__( io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.tcp_client, 65536, 104857600)
def fetch(self, request, callback=None, **kwargs): """Executes a request, asynchronously returning an `HTTPResponse`. The request may be either a string URL or an `HTTPRequest` object. If it is a string, we construct an `HTTPRequest` using any additional kwargs: ``HTTPRequest(request, **kwargs)`` This method returns a `.Future` whose result is an `HTTPResponse`. The ``Future`` will raise an `HTTPError` if the request returned a non-200 response code. If a ``callback`` is given, it will be invoked with the `HTTPResponse`. In the callback interface, `HTTPError` is not automatically raised. Instead, you must check the response's ``error`` attribute or call its `~HTTPResponse.rethrow` method. """ if not isinstance(request, HTTPRequest): request = HTTPRequest(url=request, **kwargs) # We may modify this (to add Host, Accept-Encoding, etc), # so make sure we don't modify the caller's object. This is also # where normal dicts get converted to HTTPHeaders objects. request.headers = httputil.HTTPHeaders(request.headers) request = _RequestProxy(request, self.defaults) future = TracebackFuture() if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): exc = future.exception() if isinstance(exc, HTTPError) and exc.response is not None: response = exc.response elif exc is not None: response = HTTPResponse( request, 599, error=exc, request_time=time.time() - request.start_time) else: response = future.result() self.io_loop.add_callback(callback, response) future.add_done_callback(handle_future) def handle_response(response): if response.error: future.set_exception(response.error) else: future.set_result(response) self.fetch_impl(request, handle_response) return future
def exec_command(self, command): request_log.info('EXECUTE %s', command) try: self.future = TracebackFuture() transport = self.client.get_transport() self.channel = transport.open_session() self.channel.exec_command(command) self.io_loop.add_handler(self.channel.fileno(), self._handle_events, self.io_loop.READ | self.io_loop.ERROR) return self.future except Exception as e: print(e) pass
def read_message(self, callback=None): """读取来自 WebSocket 服务器的消息. 如果在 WebSocket 初始化时指定了 on_message_callback ,那么这个方法永远不会返回消息 如果连接已经关闭,返回结果会是一个结果是 message 的 future 对象或者是 None. 如果 future 给出了回调参数, 这个参数将会在 future 完成时调用. """ assert self.read_future is None future = TracebackFuture() if self.read_queue: future.set_result(self.read_queue.popleft()) else: self.read_future = future if callback is not None: self.io_loop.add_future(future, callback) return future
def submit(self, fn, *args, **kwargs): future = TracebackFuture() def _cb(): chain_future(self._executor.submit(fn, *args, **kwargs), future) IOLoop.current().add_callback(_cb) return future
def read_message(self, callback=None): """Reads a message from the WebSocket server. Returns a future whose result is the message, or None if the connection is closed. If a callback argument is given it will be called with the future when it is ready. """ assert self.read_future is None future = TracebackFuture() if self.read_queue: future.set_result(self.read_queue.popleft()) else: self.read_future = future if callback is not None: self.io_loop.add_future(future, callback) return future
def _set_read_callback(self, callback): assert self._read_callback is None, "Already reading" assert self._read_future is None, "Already reading" if callback is not None: self._read_callback = stack_context.wrap(callback) else: self._read_future = TracebackFuture() return self._read_future
def handle_exception(self, typ, value, tb): if not self.running and not self.finished: self.future = TracebackFuture() self.future.set_exc_info((typ, value, tb)) self.run() return True else: return False
def exec_command(self, command): try: self.future = TracebackFuture() self.channel = self.client.get_transport().open_session() self.channel.exec_command(command) # t=self.channel.recv(1024) # print(t) print("EXEC COMMAND[%s] STATUS: %s" % (command, self.channel.recv_exit_status())) self.io_loop.add_handler(self.channel.fileno(), self._handle_events, self.io_loop.READ | self.io_loop.ERROR) return self.future except Exception as e: print(e) pass
def fetch_msg(uid, msg_type, data, qos, timeout, callback=None): future = TracebackFuture() if callback is not None: callback = stack_context.wrap(callback) def handle_future(future): response = future.result() IOLoop.current().add_callback(callback, response) future.add_done_callback(handle_future) def web_handle_response(response=''): future.set_result(response) WebSocketHandler.send_message( uid, msg_type, data, qos, timeout, web_handle_response) return future
def next(self): """Returns a `.Future` that will yield the next available result. Note that this `.Future` will not be the same object as any of the inputs. """ self._running_future = TracebackFuture() # As long as there is an active _running_future, we must # ensure that the WaitIterator is not GC'd (due to the # use of weak references in __init__). Add a callback that # references self so there is a hard reference that will be # cleared automatically when this Future finishes. self._running_future.add_done_callback(lambda f: self) if self._finished: self._return_result(self._finished.popleft()) return self._running_future
def __init__(self, io_loop, request): self.connect_future = TracebackFuture() self.read_future = None self.read_queue = collections.deque() self.events = [] if old_tornado: self.resolver = Resolver(io_loop=io_loop) super(EventSourceClient, self).__init__(io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.resolver) else: self.tcp_client = TCPClient(io_loop=io_loop) super(EventSourceClient, self).__init__(io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.tcp_client, 65536)
def __init__(self, io_loop, request, on_message_callback=None, compression_options=None, ping_interval=None, ping_timeout=None, max_message_size=None): self.compression_options = compression_options self.connect_future = TracebackFuture() self.protocol = None self.read_future = None self.read_queue = collections.deque() self.key = base64.b64encode(os.urandom(16)) self._on_message_callback = on_message_callback self.close_code = self.close_reason = None self.ping_interval = ping_interval self.ping_timeout = ping_timeout self.max_message_size = max_message_size scheme, sep, rest = request.url.partition(':') scheme = {'ws': 'http', 'wss': 'https'}[scheme] request.url = scheme + sep + rest request.headers.update({ 'Upgrade': 'websocket', 'Connection': 'Upgrade', 'Sec-WebSocket-Key': self.key, 'Sec-WebSocket-Version': '13', }) if self.compression_options is not None: # Always offer to let the server set our max_wbits (and even though # we don't offer it, we will accept a client_no_context_takeover # from the server). # TODO: set server parameters for deflate extension # if requested in self.compression_options. request.headers['Sec-WebSocket-Extensions'] = ( 'permessage-deflate; client_max_window_bits') self.tcp_client = TCPClient(io_loop=io_loop) super(WebSocketClientConnection, self).__init__(io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.tcp_client, 65536, 104857600)
def remove(self, param, req): tableName = param['tableName'].name key = param['key'] if not tableName in self.tables: #Silently fails return args = {'tableName': tableName, 'key': key} futures = [] for worker in self.workers: futures.append( self.client.fetch(formatQuery(worker, 'remove', args))) if key is None: del self.tables[tableName] fu = Future() req.url = self.host fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK'))) return fu
def put_file(self, filename): try: self.future = TracebackFuture() transport = self.client.get_transport() self.sftp_client = SFTPClient.from_transport(transport) self.remote = self.sftp_client.open(filename, 'wb') self.local = open(filename, 'rb') # self.remote.setblocking(0) # _buf = self.local.read(100) # self.remote.write(_buf) self.io_loop.add_handler(self.sftp_client.sock, self._handle_write, self.io_loop.WRITE | self.io_loop.ERROR) return self.future except Exception as e: print(e) pass
def set(self, param, req): tableName = param['tableName'].name key = param['key'] val = param['val'] if not tableName in self.tables: raise KeyError, 'Table not Found!' worker = self.workers[hash(key) % self.num_workers] args = {'tableName': tableName, 'key': key, 'val': val} print 'MASTER -> {0} SET: {1} with key {2} to {3}'.format( worker, tableName, key, val) futures = [] futures.append(self.client.fetch(formatQuery(worker, 'set', args))) fu = Future() req.url = self.host fu.set_result(HTTPResponse(req, 200, buffer=cStringIO.StringIO('OK'))) return fu
def handle_yield(self, yielded): if isinstance(yielded, (list, dict)): yielded = Multi(yielded) if isinstance(yielded, YieldPoint): self.future = TracebackFuture() def start_yield_point(): try: yielded.start(self) if yielded.is_ready(): self.future.set_result(yielded.get_result()) else: self.yield_point = yielded except Exception: self.future = TracebackFuture() self.future.set_exc_info(sys.exc_info()) if self.stack_context_deactivate is None: # Start a stack context if this is the first # YieldPoint we've seen. with stack_context.ExceptionStackContext( self.handle_exception) as deactivate: self.stack_context_deactivate = deactivate def cb(): start_yield_point() self.run() self.io_loop.add_callback(cb) return False else: start_yield_point() elif is_future(yielded): self.future = yielded if not self.future.done() or self.future is moment: self.io_loop.add_future(self.future, lambda f: self.run()) return False else: self.future = TracebackFuture() self.future.set_exception( BadYieldError("yielded unknown object %r" % (yielded, ))) return True
def read(self, callback=None): assert not self.reading(), 'Already reading' self.bytes_to_read = None if callback is not None: self._read_callback = stack_context.wrap(callback) future = None else: future = self._read_callback = TracebackFuture() future.add_done_callback(lambda f: f.exception()) return future
def __init__(self, io_loop, request): self.connect_future = TracebackFuture() self.read_future = None self.read_queue = collections.deque() self.key = base64.b64encode(os.urandom(16)) scheme, sep, rest = request.url.partition(':') scheme = {'ws': 'http', 'wss': 'https'}[scheme] request.url = scheme + sep + rest request.headers.update({ 'Upgrade': 'websocket', 'Connection': 'Upgrade', 'Sec-WebSocket-Key': self.key, 'Sec-WebSocket-Version': '13', }) self.resolver = Resolver(io_loop=io_loop) super(WebSocketClientConnection, self).__init__(io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.resolver)
def read(self, callback=None): assert not self.reading(), 'Already reading' if callback is not None: self._read_callback = stack_context.wrap(callback) future = None else: future = self._read_future = TracebackFuture() future.add_done_callback(lambda f: f.exception()) self._add_io_state(self.io_loop.READ) return future
def execute(self, request): ''' Starts the producer to send the next request to consumer.write(Frame(request)) ''' request.transaction_id = self.transaction.getNextTID() packet = self.framer.buildPacket(request) if not self.transport: raise ConnectionException("Slave not connected") yield self.transport.write(packet) future = TracebackFuture() self.transaction.addTransaction(future, request.transaction_id) res = yield future raise gen.Return(res)
def run(): try: result = func() except Exception: future_cell[0] = TracebackFuture() future_cell[0].set_exc_info(sys.exc_info()) else: if isinstance(result, Future): future_cell[0] = result else: future_cell[0] = Future() future_cell[0].set_result(result) self.add_future(future_cell[0], lambda future: self.stop())