def _connect(self, timeout=None): ''' Connect to a running IPCServer ''' if isinstance(self.socket_path, int): sock_type = socket.AF_INET sock_addr = ('127.0.0.1', self.socket_path) else: sock_type = socket.AF_UNIX sock_addr = self.socket_path self.stream = None if timeout is not None: timeout_at = time.time() + timeout while True: if self._closing: break if self.stream is None: with salt.utils.asynchronous.current_ioloop(self.io_loop): self.stream = IOStream( socket.socket(sock_type, socket.SOCK_STREAM)) try: log.trace('IPCClient: Connecting to socket: %s', self.socket_path) yield self.stream.connect(sock_addr) self._connecting_future.set_result(True) break except Exception as e: # pylint: disable=broad-except if self.stream.closed(): self.stream = None if timeout is None or time.time() > timeout_at: if self.stream is not None: self.stream.close() self.stream = None self._connecting_future.set_exception(e) break yield tornado.gen.sleep(1)
def _connect(self): addrinfo = yield self.resolver.resolve(self.host, self.port, 0) err = None for af, addr in addrinfo: try: s = socket.socket(af) self._stream = IOStream(s, io_loop=self.io_loop) self._timeout = self.io_loop.add_timeout( self.io_loop.time() + self.socket_connect_timeout, stack_context.wrap(self._on_timeout)) stream = yield self._stream.connect(addr) self._remove_timeout() raise gen.Return(stream) except (StreamClosedError, socket.error) as _: err = _ self.disconnect() if err is not None: raise ConnectionError(self._error_message(err)) raise socket.error("socket.getaddrinfo returned an empty list")
def test_body_size_override_reset(self): # The max_body_size override is reset between requests. stream = IOStream(socket.socket()) try: yield stream.connect(("127.0.0.1", self.get_http_port())) # Use a raw stream so we can make sure it's all on one connection. stream.write(b"PUT /streaming?expected_size=10240 HTTP/1.1\r\n" b"Content-Length: 10240\r\n\r\n") stream.write(b"a" * 10240) start_line, headers, response = yield read_stream_body(stream) self.assertEqual(response, b"10240") # Without the ?expected_size parameter, we get the old default value stream.write(b"PUT /streaming HTTP/1.1\r\n" b"Content-Length: 10240\r\n\r\n") with ExpectLog(gen_log, ".*Content-Length too long", level=logging.INFO): data = yield stream.read_until_close() self.assertEqual(data, b"HTTP/1.1 400 Bad Request\r\n\r\n") finally: stream.close()
def make_iostream_pair(self, **kwargs): port = get_unused_port() [listener] = netutil.bind_sockets(port, '127.0.0.1', family=socket.AF_INET) streams = [None, None] def accept_callback(connection, address): streams[0] = IOStream(connection, io_loop=self.io_loop, **kwargs) self.stop() def connect_callback(): streams[1] = client_stream self.stop() netutil.add_accept_handler(listener, accept_callback, io_loop=self.io_loop) client_stream = IOStream(socket.socket(), io_loop=self.io_loop, **kwargs) client_stream.connect(('127.0.0.1', port), callback=connect_callback) self.wait(condition=lambda: all(streams)) self.io_loop.remove_handler(listener.fileno()) listener.close() return streams
def test_body_size_override_reset(self): # The max_body_size override is reset between requests. stream = IOStream(socket.socket()) try: yield stream.connect(('127.0.0.1', self.get_http_port())) # Use a raw stream so we can make sure it's all on one connection. stream.write(b'PUT /streaming?expected_size=10240 HTTP/1.1\r\n' b'Content-Length: 10240\r\n\r\n') stream.write(b'a' * 10240) fut = Future() read_stream_body(stream, callback=fut.set_result) start_line, headers, response = yield fut self.assertEqual(response, b'10240') # Without the ?expected_size parameter, we get the old default value stream.write(b'PUT /streaming HTTP/1.1\r\n' b'Content-Length: 10240\r\n\r\n') with ExpectLog(gen_log, '.*Content-Length too long'): data = yield stream.read_until_close() self.assertEqual(data, b'HTTP/1.1 400 Bad Request\r\n\r\n') finally: stream.close()
def connect(self, init_future, redis_tuple, active_trans, cmd_count): """ :param init_future: 第一个future对象 :param redis_tuple: (ip, port, db) :param active_trans: 事务是否激活 :param cmd_count: 指令个数 """ if self.__stream is not None: return #future, connect_count, transaction, cmd_count self.__cmd_env.append((init_future, 1 + int(self.__haspass), False, 0)) self.__cmd_env.append((init_future, 0, active_trans, cmd_count)) with ExceptionStackContext(self.__handle_ex): self.__stream = IOStream(socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0), io_loop=self.__io_loop) self.__stream.set_close_callback(self.__on_close) self.__stream.connect(redis_tuple[:2], self.__on_connect) self.__connect_state = CONNECT_ING
def test_connection_refused(self): # When a connection is refused, the connect callback should not # be run. (The kqueue IOLoop used to behave differently from the # epoll IOLoop in this respect) cleanup_func, port = refusing_port() self.addCleanup(cleanup_func) stream = IOStream(socket.socket()) stream.set_close_callback(self.stop) # log messages vary by platform and ioloop implementation with ExpectLog(gen_log, ".*", required=False): with self.assertRaises(StreamClosedError): yield stream.connect(("127.0.0.1", port)) self.assertTrue(isinstance(stream.error, socket.error), stream.error) if sys.platform != 'cygwin': _ERRNO_CONNREFUSED = (errno.ECONNREFUSED, ) if hasattr(errno, "WSAECONNREFUSED"): _ERRNO_CONNREFUSED += (errno.WSAECONNREFUSED, ) # cygwin's errnos don't match those used on native windows python self.assertTrue(stream.error.args[0] in _ERRNO_CONNREFUSED)
async def open_dbus_connection(bus='SESSION'): bus_addr = get_bus(bus) stream = IOStream(socket.socket(family=socket.AF_UNIX)) await stream.connect(bus_addr) await stream.write(b'\0' + make_auth_external()) auth_parser = SASLParser() while not auth_parser.authenticated: auth_parser.feed(await stream.read_bytes(1024, partial=True)) if auth_parser.error: raise AuthenticationError(auth_parser.error) await stream.write(BEGIN) conn = DBusConnection(stream) with DBusRouter(conn) as router: reply_body = await wait_for(Proxy(message_bus, router).Hello(), 10) conn.unique_name = reply_body[0] return conn
def test_connection_refused(self): # When a connection is refused, the connect callback should not # be run. (The kqueue IOLoop used to behave differently from the # epoll IOLoop in this respect) server_socket, port = bind_unused_port() server_socket.close() stream = IOStream(socket.socket(), self.io_loop) self.connect_called = False def connect_callback(): self.connect_called = True stream.set_close_callback(self.stop) # log messages vary by platform and ioloop implementation with ExpectLog(gen_log, ".*", required=False): stream.connect(("localhost", port), connect_callback) self.wait() self.assertFalse(self.connect_called) self.assertTrue(isinstance(stream.error, socket.error), stream.error) if sys.platform != 'cygwin': # cygwin's errnos don't match those used on native windows python self.assertEqual(stream.error.args[0], errno.ECONNREFUSED)
def _handle_connection(self, connection, address): if self.ssl_options is not None: assert ssl, "Python 2.6+ and OpenSSL required for SSL" try: connection = ssl_wrap_socket(connection, self.ssl_options, server_side=True, do_handshake_on_connect=False) except ssl.SSLError as err: if err.args[0] == ssl.SSL_ERROR_EOF: return connection.close() else: raise except socket.error as err: # If the connection is closed immediately after it is created # (as in a port scan), we can get one of several errors. # wrap_socket makes an internal call to getpeername, # which may return either EINVAL (Mac OS X) or ENOTCONN # (Linux). If it returns ENOTCONN, this error is # silently swallowed by the ssl module, so we need to # catch another error later on (AttributeError in # SSLIOStream._do_ssl_handshake). # To test this behavior, try nmap with the -sT flag. # https://github.com/tornadoweb/tornado/pull/750 if errno_from_exception(err) in (errno.ECONNABORTED, errno.EINVAL): return connection.close() else: raise try: if self.ssl_options is not None: stream = SSLIOStream(connection, io_loop=self.io_loop, max_buffer_size=self.max_buffer_size, read_chunk_size=self.read_chunk_size) else: stream = IOStream(connection, io_loop=self.io_loop, max_buffer_size=self.max_buffer_size, read_chunk_size=self.read_chunk_size) self.handle_stream(stream, address) except Exception: app_log.error("Error in connection callback", exc_info=True)
async def _send_tplink_command(self, command): out_cmd = {} if command in ["on", "off"]: out_cmd = { 'system': {'set_relay_state': {'state': int(command == "on")}} } # TPLink device controls multiple devices if len(self.addr) == 2: sysinfo = await self._send_tplink_command("info") dev_id = sysinfo["system"]["get_sysinfo"]["deviceId"] out_cmd["context"] = { 'child_ids': [f"{dev_id}{int(self.addr[1]):02}"] } elif command == "info": out_cmd = {'system': {'get_sysinfo': {}}} else: raise self.server.error(f"Invalid tplink command: {command}") s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) stream = IOStream(s) try: await stream.connect((self.addr[0], self.port)) await stream.write(self._encrypt(out_cmd)) data = await stream.read_bytes(2048, partial=True) length = struct.unpack(">I", data[:4])[0] data = data[4:] retries = 5 remaining = length - len(data) while remaining and retries: data += await stream.read_bytes(remaining) remaining = length - len(data) retries -= 1 if not retries: raise self.server.error("Unable to read tplink packet") except Exception: msg = f"Error sending tplink command: {command}" logging.exception(msg) raise self.server.error(msg) finally: stream.close() return json.loads(self._decrypt(data))
def _create_stream(self, max_buffer_size, af, addr, source_ip=None, source_port=None): # Always connect in plaintext; we'll convert to ssl if necessary # after one connection has completed. source_port_bind = source_port if isinstance(source_port, int) else 0 source_ip_bind = source_ip if source_port_bind and not source_ip: # User required a specific port, but did not specify # a certain source IP, will bind to the default loopback. source_ip_bind = '::1' if af == socket.AF_INET6 else '127.0.0.1' # Trying to use the same address family as the requested af socket: # - 127.0.0.1 for IPv4 # - ::1 for IPv6 socket_obj = socket.socket(af) set_close_exec(socket_obj.fileno()) try: stream = IOStream(socket_obj, io_loop=self.io_loop, max_buffer_size=max_buffer_size) except socket.error as e: fu = Future() fu.set_exception(e) return fu else: if source_port_bind or source_ip_bind: @gen.coroutine def _(addr): r = yield stream.connect( (source_ip_bind, source_port_bind)) yield self._connect_tunnel(stream, addr, {}) raise gen.Return(r) return _(addr) else: return stream.connect(addr)
def async_connect(self): sock = socket.socket(socket.AF_UNIX) workerlog.debug("connecting to %s", self.endpoint) try: io_stream = IOStream(sock, io_loop=self.io_loop) self.pipe = yield io_stream.connect(self.endpoint, callback=None) workerlog.debug("connected to %s %s", self.endpoint, self.pipe) self.pipe.read_until_close(callback=self.on_failure, streaming_callback=self.on_message) except Exception as err: workerlog.error("unable to connect to '%s' %s", self.endpoint, err) self.on_failure() return workerlog.debug("sending handshake") self.send_handshake() workerlog.debug("sending heartbeat") self.do_heartbeat() # start heartbeat timer self.heartbeat_timer.start() workerlog.debug("start threaded_disown_timer") self.threaded_disown_timer.start()
def test_read_zero_bytes(self): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) s.connect(("localhost", self.get_http_port())) self.stream = IOStream(s, io_loop=self.io_loop) self.stream.write(b("GET / HTTP/1.0\r\n\r\n")) # normal read self.stream.read_bytes(9, self.stop) data = self.wait() self.assertEqual(data, b("HTTP/1.0 ")) # zero bytes self.stream.read_bytes(0, self.stop) data = self.wait() self.assertEqual(data, b("")) # another normal read self.stream.read_bytes(3, self.stop) data = self.wait() self.assertEqual(data, b("200")) s.close()
def test_100_continue(self): # Run through a 100-continue interaction by hand: # When given Expect: 100-continue, we get a 100 response after the # headers, and then the real response after the body. stream = IOStream(socket.socket()) yield stream.connect(("127.0.0.1", self.get_http_port())) yield stream.write(b"\r\n".join([ b"POST /hello HTTP/1.1", b"Content-Length: 1024", b"Expect: 100-continue", b"Connection: close", b"\r\n"])) data = yield stream.read_until(b"\r\n\r\n") self.assertTrue(data.startswith(b"HTTP/1.1 100 "), data) stream.write(b"a" * 1024) first_line = yield stream.read_until(b"\r\n") self.assertTrue(first_line.startswith(b"HTTP/1.1 200"), first_line) header_data = yield stream.read_until(b"\r\n\r\n") headers = HTTPHeaders.parse(native_str(header_data.decode('latin1'))) body = yield stream.read_bytes(int(headers["Content-Length"])) self.assertEqual(body, b"Got 1024 bytes in POST") stream.close()
def start(self, socket_or_fd, read_mtu): """Starts the decoder. If already started, this does nothing. """ if self._started: return # process self._process = Subprocess([ "sox", "-t", self._codec, "-", "--bits", str(self._sample_size), "--channels", str(self._channels), "--rate", str(self._sample_rate), "-t", "wav", "-" ], stdin=Subprocess.STREAM, stdout=Subprocess.STREAM, stderr=Subprocess.STREAM) self._process.stdout.set_close_callback(self._on_close) self._process.stdout.read_until_close( streaming_callback=self._out_data_ready) self._process.stderr.read_until_close( streaming_callback=self._sox_error) # did we get socket or fd? sock = socket_or_fd if isinstance(socket_or_fd, int): logger.debug("SoxDecoder received fd, building socket...") sock = socket.socket(fileno=socket_or_fd) sock.setblocking(True) # input pump self._input = IOStream(socket=sock) self._input.read_until_close(self._in_data_ready) # start self._started = True # we know WAV params already if self.on_wav_params_ready: self.on_wav_params_ready()
def echo(socket, address, io_loop): """A server connection handler that repeats back lines received from the client. The server stops echoing if the client says 'goodbye'.""" stream = IOStream(socket, io_loop=io_loop) def write(data, *args): print 'S: %r' % data stream.write(data, *args) def handle(data): if data == 'goodbye\n': write('See you later.\n', stream.close) else: write('You said: "%s".\n' % data.strip()) loop() def loop(): stream.read_until("\n", handle) loop()
def _create_stream( self, max_buffer_size: int, af: socket.AddressFamily, addr: Tuple, source_ip: str = None, source_port: int = None, ) -> Tuple[IOStream, "Future[IOStream]"]: # Always connect in plaintext; we'll convert to ssl if necessary # after one connection has completed. source_port_bind = source_port if isinstance(source_port, int) else 0 source_ip_bind = source_ip if source_port_bind and not source_ip: # User required a specific port, but did not specify # a certain source IP, will bind to the default loopback. source_ip_bind = "::1" if af == socket.AF_INET6 else "127.0.0.1" # Trying to use the same address family as the requested af socket: # - 127.0.0.1 for IPv4 # - ::1 for IPv6 socket_obj = socket.socket(af) set_close_exec(socket_obj.fileno()) if source_port_bind or source_ip_bind: # If the user requires binding also to a specific IP/port. try: socket_obj.bind((source_ip_bind, source_port_bind)) except socket.error: socket_obj.close() # Fail loudly if unable to use the IP/port. raise try: stream = IOStream(socket_obj, max_buffer_size=max_buffer_size) except socket.error as e: fu = Future() # type: Future[IOStream] fu.set_exception(e) return stream, fu else: return stream, stream.connect(addr)
def accept_callback(conn, address): # fake an HTTP server using chunked encoding where the final chunks # and connection close all happen at once stream = IOStream(conn) request_data = yield stream.read_until(b"\r\n\r\n") if b"HTTP/1." not in request_data: self.skipTest("requires HTTP/1.x") yield stream.write( b"""\ HTTP/1.1 200 OK Transfer-Encoding: chunked 1 1 1 2 0 """.replace( b"\n", b"\r\n" ) ) stream.close()
def setUp(self): super(TestWebSocketBase, self).setUp() self.application = Application() self.server = HTTPServer(self.application) self.socket, self.port = testing.bind_unused_port() self.server.add_socket(self.socket) self.instance = WebSocketBase( self.application, HTTPServerRequest( method="GET", uri='/', version="HTTP/1.0", headers=HTTPHeaders(), body=BytesIO(), host=None, files=None, connection=HTTP1Connection(stream=IOStream(socket.socket()), is_client=False), start_line=RequestStartLine(method='GET', path='/', version='HTTP/1.1'), )) self.instance.open()
def connect(self): if not self.closed(): return # Assume host is an ipv6 address if it has a colon. if ':' in self.host: self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) else: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.settimeout(self.timeout) self.socket.setblocking(0) self.stream = IOStream(self.socket) self.stream.set_close_callback(self._socket_close) self.stream.set_nodelay(True) self.state = CONNECTING self.on(event.CONNECT, self._on_connect) self.on(event.DATA, self._on_data) fut = self.stream.connect((self.host, self.port)) IOLoop.current().add_future(fut, self._connect_callback)
def _handle_connection(self, connection, address): if self.ssl_options is not None: assert ssl, "Python 2.6+ and OpenSSL required for SSL" try: connection = ssl_wrap_socket(connection, self.ssl_options, server_side=True, do_handshake_on_connect=False) except ssl.SSLError as err: if err.args[0] == ssl.SSL_ERROR_EOF: return connection.close() else: raise except socket.error as err: if errno_from_exception(err) in (errno.ECONNABORTED, errno.EINVAL): return connection.close() else: raise try: if self.ssl_options is not None: stream = SSLIOStream(connection, io_loop=self.io_loop, max_buffer_size=self.max_buffer_size, read_chunk_size=self.read_chunk_size) else: stream = IOStream(connection, io_loop=self.io_loop, max_buffer_size=self.max_buffer_size, read_chunk_size=self.read_chunk_size) future = self.handle_stream(stream, address) if future is not None: self.io_loop.add_future(future, lambda f: f.result()) except Exception: app_log.error("Error in connection callback", exc_info=True)
def _create_stream(self, max_buffer_size, af, addr, source_ip=None, source_port=None): # Always connect in plaintext; we'll convert to ssl if necessary # after one connection has completed. source_port_bind = source_port if isinstance(source_port, int) else 0 source_ip_bind = source_ip if source_port_bind and not source_ip: # User required a specific port, but did not specify # a certain source IP, will bind to the default loopback. source_ip_bind = '::1' if af == socket.AF_INET6 else '127.0.0.1' # Trying to use the same address family as the requested af socket: # - 127.0.0.1 for IPv4 # - ::1 for IPv6 socket_obj = socket.socket(af) if source_port_bind or source_ip_bind: # If the user requires binding also to a specific IP/port. try: socket_obj.bind((source_ip_bind, source_port_bind)) except socket.error: socket_obj.close() # Fail loudly if unable to use the IP/port. raise try: stream = IOStream(socket_obj, io_loop=self.io_loop, max_buffer_size=max_buffer_size) except socket.error as e: fu = Future() fu.set_exception(e) return fu else: return stream.connect(addr)
async def handle_connection(connection, address): stream = IOStream(connection) message = '1' while message: try: if await is_frame_header(stream): version = await get_device_version(stream) device_id = await get_device_id(stream) session = await get_session(stream) ctrl_sign = await get_ctrl_sign(stream) content = await get_content(stream) await parse_content(content) checksum = await get_checksum(stream) except Exception as e: print("exception: " + str(e)) message = ''
def test_indexing_line(self): client = AsyncHTTPClient(io_loop=self.io_loop) ping = yield client.fetch("http://*****:*****@version'], 1) self.assertEqual(doc['message'], "My name is Yuri and I'm 6 years old.")
def accept_callback(conn, address): # fake an HTTP server using chunked encoding where the final chunks # and connection close all happen at once stream = IOStream(conn, io_loop=self.io_loop) stream.read_until(b("\r\n\r\n"), functools.partial(write_response, stream))
def _make_client_iostream(self, connection, **kwargs): return IOStream(connection, **kwargs)
def _make_client_iostream(self): return IOStream(socket.socket(), io_loop=self.io_loop)
def _make_server_iostream(self, connection, **kwargs): return IOStream(connection, **kwargs)
def __init__(self, io_loop, client, request, release_callback, final_callback, max_buffer_size): self.start_time = time.time() self.io_loop = io_loop self.client = client self.request = request self.release_callback = release_callback self.final_callback = final_callback self.code = None self.headers = None self.chunks = None self._decompressor = None # Timeout handle returned by IOLoop.add_timeout self._timeout = None with stack_context.StackContext(self.cleanup): parsed = urlparse.urlsplit(_unicode(self.request.url)) if ssl is None and parsed.scheme == "https": raise ValueError("HTTPS requires either python2.6+ or " "curl_httpclient") if parsed.scheme not in ("http", "https"): raise ValueError("Unsupported url scheme: %s" % self.request.url) # urlsplit results have hostname and port results, but they # didn't support ipv6 literals until python 2.7. netloc = parsed.netloc if "@" in netloc: userpass, _, netloc = netloc.rpartition("@") match = re.match(r'^(.+):(\d+)$', netloc) if match: host = match.group(1) port = int(match.group(2)) else: host = netloc port = 443 if parsed.scheme == "https" else 80 if re.match(r'^\[.*\]$', host): # raw ipv6 addresses in urls are enclosed in brackets host = host[1:-1] if self.client.hostname_mapping is not None: host = self.client.hostname_mapping.get(host, host) if request.allow_ipv6: af = socket.AF_UNSPEC else: # We only try the first IP we get from getaddrinfo, # so restrict to ipv4 by default. af = socket.AF_INET addrinfo = socket.getaddrinfo(host, port, af, socket.SOCK_STREAM, 0, 0) af, socktype, proto, canonname, sockaddr = addrinfo[0] if parsed.scheme == "https": ssl_options = {} if request.validate_cert: ssl_options["cert_reqs"] = ssl.CERT_REQUIRED if request.ca_certs is not None: ssl_options["ca_certs"] = request.ca_certs else: ssl_options["ca_certs"] = _DEFAULT_CA_CERTS if request.client_key is not None: ssl_options["keyfile"] = request.client_key if request.client_cert is not None: ssl_options["certfile"] = request.client_cert self.stream = SSLIOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop, ssl_options=ssl_options, max_buffer_size=max_buffer_size) else: self.stream = IOStream(socket.socket(af, socktype, proto), io_loop=self.io_loop, max_buffer_size=max_buffer_size) timeout = min(request.connect_timeout, request.request_timeout) if timeout: self._timeout = self.io_loop.add_timeout( self.start_time + timeout, self._on_timeout) self.stream.set_close_callback(self._on_close) self.stream.connect(sockaddr, functools.partial(self._on_connect, parsed))