def __init__(self,
host,
port,
io_loop,
ssl_options,
max_connect_backoff=MAX_CONNECT_BACKOFF,
initial_window_size=DEFAULT_WINDOW_SIZE,
connect_timeout=5,
connect_callback=None):
self.host = host
self.port = port
self.io_loop = io_loop
self.tcp_client = TCPClient()
self.h2conn = None
self.io_stream = None
self.window_manager = None
self.connect_timeout = connect_timeout
self._connect_timeout_handle = None
self._connect_future = None
self._streams = {}
self.ssl_context = None
self.ssl_options = ssl_options
self.parse_ssl_opts()
self.initial_window_size = initial_window_size
self.max_connect_backoff = max_connect_backoff
self.consecutive_connect_fails = 0
self.connect_callback = connect_callback
self._closed = False
def _run_traffic_jam(nsends, nbytes):
# This test eats `nsends * nbytes` bytes in RAM
np = pytest.importorskip('numpy')
from distributed.protocol import to_serialize
data = bytes(np.random.randint(0, 255, size=(nbytes,)).astype('u1').data)
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=0.01)
b.start(stream)
msg = {'x': to_serialize(data)}
for i in range(nsends):
b.send(assoc(msg, 'i', i))
if np.random.random() > 0.5:
yield gen.sleep(0.001)
results = []
count = 0
while len(results) < nsends:
# If this times out then I think it's a backpressure issue
# Somehow we're able to flood the socket so that the receiving end
# loses some of our messages
L = yield gen.with_timeout(timedelta(seconds=5), read(stream))
count += 1
results.extend(r['i'] for r in L)
assert count == b.batch_count == e.count
assert b.message_count == nsends
assert results == list(range(nsends))
stream.close() # external closing
yield b.close(ignore_closed=True)
class TCPClientTest(AsyncTestCase):
def setUp(self):
super(TCPClientTest, self).setUp()
self.server = None
self.client = TCPClient()
def start_server(self, family):
if family == socket.AF_UNSPEC and "TRAVIS" in os.environ:
self.skipTest("dual-stack servers often have port conflicts on travis")
self.server = TestTCPServer(family)
return self.server.port
def stop_server(self):
if self.server is not None:
self.server.stop()
self.server = None
def tearDown(self):
self.client.close()
self.stop_server()
super(TCPClientTest, self).tearDown()
def skipIf10.0.0.7V4(self):
# The port used here doesn't matter, but some systems require it
# to be non-zero if we do not also pass AI_PASSIVE.
addrinfo = self.io_loop.run_sync(lambda: Resolver().resolve("10.0.0.7", 80))
families = set(addr[0] for addr in addrinfo)
if socket.AF_INET6 not in families:
self.skipTest("10.0.0.7 does not resolve to ipv6")
def __init__(self, routes, node, pipe):
"""
Application instantiates and registers handlers for each message type,
and routes messages to the pre-instantiated instances of each message handler
:param routes: list of tuples in the form of (<message type str>, <MessageHandler class>)
:param node: Node instance of the local node
:param pipe: Instance of multiprocessing.Pipe for communicating with the parent process
"""
# We don't really have to worry about synchronization
# so long as we're careful about explicit context switching
self.nodes = {node.node_id: node}
self.local_node = node
self.handlers = {}
self.tcpclient = TCPClient()
self.gossip_inbox = Queue()
self.gossip_outbox = Queue()
self.sequence_number = 0
if routes:
self.add_handlers(routes)
self.pipe = pipe
self.ioloop = IOLoop.current()
self.add_node_event = Event()
def __init__(self, io_loop, request, on_message_callback=None,
compression_options=None):
self.compression_options = compression_options
self.connect_future = TracebackFuture()
self.protocol = None
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
self._on_message_callback = on_message_callback
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
if self.compression_options is not None:
# Always offer to let the server set our max_wbits (and even though
# we don't offer it, we will accept a client_no_context_takeover
# from the server).
# TODO: set server parameters for deflate extension
# if requested in self.compression_options.
request.headers['Sec-WebSocket-Extensions'] = (
'permessage-deflate; client_max_window_bits')
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection, self).__init__(
io_loop, None, request, lambda: None, self._on_http_response,
104857600, self.tcp_client, 65536)
def connect(self):
client = TCPClient()
self.stream = yield client.connect(self.host, self.port)
# sock = None
# try:
# if self.unix_socket and self.host in ('localhost', '127.0.0.1'):
# sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# t = sock.gettimeout()
# sock.settimeout(self.connect_timeout)
# sock.connect(self.unix_socket)
# sock.settimeout(t)
# self.host_info = "Localhost via UNIX socket"
# if DEBUG: print('connected using unix_socket')
# else:
# while True:
# try:
# sock = socket.create_connection(
# (self.host, self.port), self.connect_timeout)
# break
# except (OSError, IOError) as e:
# if e.errno == errno.EINTR:
# continue
# raise
# self.host_info = "socket %s:%d" % (self.host, self.port)
# if DEBUG: print('connected using socket')
# sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
# if self.no_delay:
# sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# self.socket = sock
# self._rfile = _makefile(sock, 'rb')
yield self._get_server_information()
yield self._request_authentication()
def __init__(self):
debug('Starting Envisalink Client')
# Register events for alarmserver requests -> envisalink
events.register('alarm_update', self.request_action)
# Register events for envisalink proxy
events.register('envisalink', self.envisalink_proxy)
# Create TCP Client
self.tcpclient = TCPClient()
# Connection
self._connection = None
# Set our terminator to \r\n
self._terminator = b"\r\n"
# Reconnect delay
self._retrydelay = 10
# Connect to Envisalink
self.do_connect()
# Setup timer to refresh envisalink
tornado.ioloop.PeriodicCallback(self.check_connection, 1000).start()
# Last activity
self._last_activity = time.time()
def test_BatchedSend():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=10)
assert str(len(b.buffer)) in str(b)
assert str(len(b.buffer)) in repr(b)
b.start(stream)
yield gen.sleep(0.020)
b.send('hello')
b.send('hello')
b.send('world')
yield gen.sleep(0.020)
b.send('HELLO')
b.send('HELLO')
result = yield read(stream)
assert result == ['hello', 'hello', 'world']
result = yield read(stream)
assert result == ['HELLO', 'HELLO']
assert b.byte_count > 1
def whois_async(query, fields=None):
"""
Perform whois request
:param query:
:param fields:
:return:
"""
logger.debug("whois %s", query)
# Get appropriate whois server
if is_fqdn(query):
# Use TLD.whois-servers.net for domain lookup
tld = query.split(".")[-1]
server = "%s.whois-servers.net" % tld
else:
server = DEFAULT_WHOIS_SERVER
# Perform query
try:
client = TCPClient()
stream = yield client.connect(server, DEFAULT_WHOIS_PORT)
except IOError as e:
logger.error("Cannot resolve host '%s': %s", server, e)
raise tornado.gen.Return()
try:
yield stream.write(str(query) + "\r\n")
data = yield stream.read_until_close()
finally:
yield stream.close()
data = parse_response(data)
if fields:
data = [(k, v) for k, v in data if k in fields]
raise tornado.gen.Return(data)
def _run_traffic_jam(nsends, nbytes):
# This test eats `nsends * nbytes` bytes in RAM
np = pytest.importorskip('numpy')
from distributed.protocol import to_serialize
data = bytes(np.random.randint(0, 255, size=(nbytes, )).astype('u1').data)
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=0.01)
b.start(stream)
msg = {'x': to_serialize(data)}
for i in range(nsends):
b.send(assoc(msg, 'i', i))
if np.random.random() > 0.5:
yield gen.sleep(0.001)
results = []
count = 0
while len(results) < nsends:
# If this times out then I think it's a backpressure issue
# Somehow we're able to flood the socket so that the receiving end
# loses some of our messages
L = yield gen.with_timeout(timedelta(seconds=5), read(stream))
count += 1
results.extend(r['i'] for r in L)
assert count == b.batch_count == e.count
assert b.message_count == nsends
assert results == list(range(nsends))
stream.close() # external closing
yield b.close(ignore_closed=True)
def __init__(self, *args, **kwargs):
TCPClient.__init__(self, kwargs.pop("resolver", None))
Connection.__init__(self, parser_class=AsyncParser, *args, **kwargs)
self._stream = None
def connect(self, address, deserialize=True, **connection_args):
self._check_encryption(address, connection_args)
ip, port = parse_host_port(address)
kwargs = self._get_connect_args(**connection_args)
client = TCPClient()
try:
stream = yield client.connect(ip, port,
max_buffer_size=MAX_BUFFER_SIZE,
**kwargs)
# Under certain circumstances tornado will have a closed connnection with an error and not raise
# a StreamClosedError.
#
# This occurs with tornado 5.x and openssl 1.1+
if stream.closed() and stream.error:
raise StreamClosedError(stream.error)
except StreamClosedError as e:
# The socket connect() call failed
convert_stream_closed_error(self, e)
local_address = self.prefix + get_stream_address(stream)
raise gen.Return(self.comm_class(stream,
local_address,
self.prefix + address,
deserialize))
def test_stress():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
L = []
@gen.coroutine
def send():
b = BatchedSend(interval=3)
b.start(stream)
for i in range(0, 10000, 2):
b.send(i)
b.send(i + 1)
yield gen.sleep(0.00001 * random.randint(1, 10))
@gen.coroutine
def recv():
while True:
result = yield gen.with_timeout(timedelta(seconds=1),
read(stream))
print(result)
L.extend(result)
if result[-1] == 9999:
break
yield All([send(), recv()])
assert L == list(range(0, 10000, 1))
stream.close()
def __init__(self, endpoint=DEFAULT_ENDPOINT_DEALER, timeout=5.0):
self.endpoint = endpoint
self._id = cast_bytes(uuid.uuid4().hex)
self.timeout = timeout
self.stream = None
self.endpoint = endpoint
self.client = TCPClient()
def connect(self):
client = TCPClient(io_loop=self.io_loop)
self.stream = yield client.connect(self.host, self.port)
# sock = None
# try:
# if self.unix_socket and self.host in ('localhost', '127.0.0.1'):
# sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# t = sock.gettimeout()
# sock.settimeout(self.connect_timeout)
# sock.connect(self.unix_socket)
# sock.settimeout(t)
# self.host_info = "Localhost via UNIX socket"
# if DEBUG: print('connected using unix_socket')
# else:
# while True:
# try:
# sock = socket.create_connection(
# (self.host, self.port), self.connect_timeout)
# break
# except (OSError, IOError) as e:
# if e.errno == errno.EINTR:
# continue
# raise
# self.host_info = "socket %s:%d" % (self.host, self.port)
# if DEBUG: print('connected using socket')
# sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
# if self.no_delay:
# sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# self.socket = sock
# self._rfile = _makefile(sock, 'rb')
yield self._get_server_information()
yield self._request_authentication()
def initialize(self,
max_clients=10,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
max_header_size=None,
max_body_size=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
``force_instance=True`` may be used to suppress this behavior.
Note that because of this implicit reuse, unless ``force_instance``
is used, only the first call to the constructor actually uses
its arguments. It is recommended to use the ``configure`` method
instead of the constructor to ensure that arguments take effect.
``max_clients`` is the number of concurrent requests that can be
in progress; when this limit is reached additional requests will be
queued. Note that time spent waiting in this queue still counts
against the ``request_timeout``.
``hostname_mapping`` is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like ``/etc/hosts`` is not possible or desirable (e.g. in
unittests).
``max_buffer_size`` (default 100MB) is the number of bytes
that can be read into memory at once. ``max_body_size``
(defaults to ``max_buffer_size``) is the largest response body
that the client will accept. Without a
``streaming_callback``, the smaller of these two limits
applies; with a ``streaming_callback`` only ``max_body_size``
does.
.. versionchanged:: 4.2
Added the ``max_body_size`` argument.
"""
super(SimpleAsyncHTTPClient, self).initialize(defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver()
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver)
def test_stress():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
L = []
@gen.coroutine
def send():
b = BatchedSend(interval=3)
b.start(stream)
for i in range(0, 10000, 2):
b.send(i)
b.send(i + 1)
yield gen.sleep(0.00001 * random.randint(1, 10))
@gen.coroutine
def recv():
while True:
result = yield gen.with_timeout(timedelta(seconds=1), read(stream))
print(result)
L.extend(result)
if result[-1] == 9999:
break
yield All([send(), recv()])
assert L == list(range(0, 10000, 1))
stream.close()
def __init__(self, *args, **kwargs):
TCPClient.__init__(self, kwargs.pop("resolver", None),
kwargs.pop("io_loop", None))
Connection.__init__(self, parser_class=AsyncParser, *args, **kwargs)
self._stream = None
def test_close_twice():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=10)
b.start(stream)
yield b.close()
yield b.close()
def connect(self, timeout):
deadline = None if timeout is None else self._io_loop.time() + timeout
try:
if len(self._parent.ssl) > 0:
ssl_options = {}
if self._parent.ssl["ca_certs"]:
ssl_options['ca_certs'] = self._parent.ssl["ca_certs"]
ssl_options['cert_reqs'] = 2 # ssl.CERT_REQUIRED
stream_future = TCPClient().connect(self._parent.host,
self._parent.port,
ssl_options=ssl_options)
else:
stream_future = TCPClient().connect(self._parent.host,
self._parent.port)
self._stream = yield with_absolute_timeout(
deadline,
stream_future,
io_loop=self._io_loop,
quiet_exceptions=(iostream.StreamClosedError))
except Exception as err:
raise ReqlDriverError(
'Could not connect to %s:%s. Error: %s' %
(self._parent.host, self._parent.port, str(err)))
self._stream.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY,
1)
self._stream.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE,
1)
try:
self._stream.write(self._parent.handshake)
response = yield with_absolute_timeout(
deadline,
self._stream.read_until(b'\0'),
io_loop=self._io_loop,
quiet_exceptions=(iostream.StreamClosedError))
except Exception as err:
raise ReqlDriverError(
'Connection interrupted during handshake with %s:%s. Error: %s'
% (self._parent.host, self._parent.port, str(err)))
message = decodeUTF(response[:-1]).split('\n')[0]
if message != 'SUCCESS':
yield self.close(False, None)
if message == "ERROR: Incorrect authorization key":
raise ReqlAuthError(self._parent.host, self._parent.port)
else:
raise ReqlDriverError(
'Server dropped connection with message: "%s"' %
(message, ))
# Start a parallel function to perform reads
self._io_loop.add_callback(self._reader)
raise gen.Return(self._parent)
def test_close_twice():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=10)
b.start(stream)
yield b.close()
yield b.close()
def start(self):
self.client = TCPClient()
self.pcb = PeriodicCallback(self.send, 1000.0 / self.n)
self.pcb.start()
IOLoop.current().call_later(self.duration + 0.5, self.stop)
IOLoop.current().start()
IOLoop.clear_current()
class CircleClient(object):
def __init__(self, endpoint=DEFAULT_ENDPOINT_DEALER, timeout=5.0):
self.endpoint = endpoint
self._id = cast_bytes(uuid.uuid4().hex)
self.timeout = timeout
self.stream = None
self.endpoint = endpoint
self.client = TCPClient()
def stop(self):
self.client.close()
def send_message(self, command, **props):
return self.call(make_message(command, **props))
def call(self, cmd):
result = IOLoop.instance().run_sync(lambda: self._call(cmd))
return result
@gen.coroutine
def _call(self, cmd):
if isinstance(cmd, basestring):
raise DeprecationWarning('call() takes a mapping')
call_id = uuid.uuid4().hex
cmd['id'] = call_id
host, port = self.endpoint.split(':')
try:
cmd = json.dumps(cmd)
self.stream = yield gen_timeout(self.timeout,
self.client.connect(host, port))
yield gen_timeout(self.timeout, self.stream.write(cmd + MSG_END))
except StreamClosedError:
raise CallError("Can't connect circled. Maybe it is closed.")
except gen.TimeoutError:
raise CallError('Connect timed out ({} seconds).'.format(
self.timeout))
except ValueError as e:
raise CallError(str(e))
while True:
try:
msg = yield gen_timeout(self.timeout,
self.stream.read_until(MSG_END))
msg = rstrip(msg, MSG_END)
res = json.loads(msg)
if 'id' in res and res['id'] not in (call_id, None):
# we got the wrong message
continue
raise gen.Return(res)
except gen.TimeoutError:
raise CallError('Run timed out ({} seconds).'.format(
self.timeout))
except ValueError as e:
raise CallError(str(e))
def __init__(self, ssl_options=None, gp_module=False, **kwargs):
self.logger = logging.getLogger(self.__class__.__name__)
self.gp_module = gp_module
try:
TCPClient.__init__(self, ssl_options=ssl_options, **kwargs)
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not create tcp client. Exception: %s, Error: %s." % (etype, evalue))
self.gp_module.shutDown()
def test_send_after_stream_finish():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=10)
b.start(stream)
yield b.last_send
b.send('hello')
result = yield read(stream); assert result == ['hello']
def connect(self, host, port):
self.host = host
self.port = port
client = TCPClient()
try:
self.stream = yield client.connect(self.host, self.port)
except IOError as e:
log.error("%s", repr(e))
raise gen.Return((False, 'Failed to connect'))
self.trigger(Event.CONNECT, self)
raise gen.Return((True, "OK"))
def test_close_closed():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=10)
b.start(stream)
b.send(123)
stream.close() # external closing
yield b.close(ignore_closed=True)
def handle_stream():
while True:
ip = '192.168.199.126'
ip = '45.77.214.165'
yield gen.sleep(3)
streamCli = yield TCPClient().connect(ip, 8889)
stream = yield TCPClient().connect('192.168.199.126', 5900)
print 'client ok'
IOLoop.instance().add_callback(
functools.partial(run_client, stream, streamCli))
IOLoop.instance().add_callback(
functools.partial(run_client, streamCli, stream))
def test_send_after_stream_finish():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=10)
b.start(stream)
yield b.last_send
b.send('hello')
result = yield read(stream)
assert result == ['hello']
def test_close_closed():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=10)
b.start(stream)
b.send(123)
stream.close() # external closing
yield b.close(ignore_closed=True)
def test_send_before_start():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=10)
b.send('hello')
b.send('world')
b.start(stream)
result = yield read(stream); assert result == ['hello', 'world']
def connect(ip, port, timeout=1):
client = TCPClient()
start = time()
while True:
try:
stream = yield client.connect(ip, port)
raise Return(stream)
except StreamClosedError:
if time() - start < timeout:
yield gen.sleep(0.01)
logger.debug("sleeping on connect")
else:
raise
def __init__(self,
p_mq,
p_server_map,
p_callback=None,
max_buffer_size=None,
timeout=None):
self._host = p_server_map["host"]
self._port = p_server_map["port"]
self._max_buf_size = max_buffer_size
self._timeout = timeout
self._mq = p_mq
self._handler = p_callback
self._tcpclient = TCPClient()
def connect(self, address, deserialize=True):
ip, port = parse_host_port(address)
client = TCPClient()
try:
stream = yield client.connect(ip,
port,
max_buffer_size=MAX_BUFFER_SIZE)
except StreamClosedError as e:
# The socket connect() call failed
convert_stream_closed_error(e)
raise gen.Return(TCP(stream, 'tcp://' + address, deserialize))
def connect(ip, port, timeout=1):
client = TCPClient()
start = time()
while True:
try:
stream = yield client.connect(ip, port)
raise Return(stream)
except StreamClosedError:
if time() - start < timeout:
yield gen.sleep(0.01)
logger.debug("sleeping on connect")
else:
raise
def initialize(self,
io_loop,
max_clients=10,
hostname_mapping=None,
max_buffer_size=104857600,
resolver=None,
defaults=None,
max_header_size=None):
"""Creates a AsyncHTTPClient.
# 一个IOLoop实例只有一个AsyncHTTPClient实例
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients # 最大连接数
self.queue = collections.deque() # 来一个小小的队列
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
# 这里估计是一个解析DNS的配置，先不理会
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
# 这里来了一个tcp的连接
self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop)
def start_app():
tcpClient = TCPClient()
try:
stream = yield tcpClient.connect('127.0.0.1', 9999)
print 'Connection started'
app = LaternController(LanternDriver())
client = TLVClient(stream)
executer = ThreadPoolExecutor(max_workers=5)
while True:
command = yield client.getCommand()
executer.submit(app.handle, command)
except Exception as e:
print 'Caught Error: %s' % e
IOLoop.instance().add_callback(IOLoop.instance().stop)
def __init__(self, host, port, user_connection_id, docker_id, io_loop):
self.__host = host
self.__port = port
self.__user_connection_id = user_connection_id
self.__docker_id = docker_id
self._io_loop = io_loop
self.client = TCPClient(io_loop=self._io_loop)
self._stream = None
self._requests = dict()
self._requests_signals = {
packet.InPacket.METHOD_SELECT_RESULT: Signal('data'),
packet.InPacket.METHOD_GET_STATUS: Signal('data'),
packet.InPacket.METHOD_CANCEL: Signal('data'),
}
def __init__(self,
io_loop=None,
max_clients=10,
max_buffer_size=None,
max_response_size=None):
self._io_loop = io_loop or IOLoop.instance()
self.max_clients = max_clients
self.max_buffer_size = max_buffer_size or 104857600 # 100M
self.max_response_size = max_response_size or 10 * 1024 * 1024 # 10M
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self._client_closed = False
self.tcp_client = TCPClient(io_loop=self._io_loop)
def test_BatchedStream_raises():
port = 3435
server = MyServer()
server.listen(port)
client = TCPClient()
stream = yield client.connect('127.0.0.1', port)
b = BatchedStream(stream, interval=20)
stream.close()
with pytest.raises(StreamClosedError):
yield b.recv()
with pytest.raises(StreamClosedError):
yield b.send('123')
def connect(ip, port, timeout=3):
client = TCPClient()
start = time()
while True:
try:
future = client.connect(ip, port, max_buffer_size=MAX_BUFFER_SIZE)
stream = yield gen.with_timeout(timedelta(seconds=timeout), future)
raise Return(stream)
except StreamClosedError:
if time() - start < timeout:
yield gen.sleep(0.01)
logger.debug("sleeping on connect")
else:
raise
except gen.TimeoutError:
raise IOError("Timed out while connecting to %s:%d" % (ip, port))
class GatewayConnector(Connector):
_executor = ThreadPoolExecutor(2)
_resolver = netutil.ExecutorResolver(close_executor=False,
executor=_executor)
client = TCPClient(resolver=_resolver)
async def connect(self, address, deserialize=True, **connection_args):
ip, port, path = parse_gateway_address(address)
sni = "daskgateway-" + path
ctx = connection_args.get("ssl_context")
if not isinstance(ctx, ssl.SSLContext):
raise TypeError("Gateway expects a `ssl_context` argument of type "
"ssl.SSLContext, instead got %s" % ctx)
try:
plain_stream = await self.client.connect(
ip, port, max_buffer_size=MAX_BUFFER_SIZE)
stream = await plain_stream.start_tls(False,
ssl_options=ctx,
server_hostname=sni)
if stream.closed() and stream.error:
raise StreamClosedError(stream.error)
except StreamClosedError as e:
# The socket connect() call failed
convert_stream_closed_error(self, e)
local_address = "tls://" + get_stream_address(stream)
peer_address = "gateway://" + address
return TLS(stream, local_address, peer_address, deserialize)
def __init__(self):
logger.debug('Starting Envisalink Client')
# Register events for alarmserver requests -> envisalink
events.register('alarm_update', self.request_action)
# Register events for envisalink proxy
events.register('envisalink', self.envisalink_proxy)
# Create TCP Client
self.tcpclient = TCPClient()
# Connection
self._connection = None
# Set our terminator to \r\n
self._terminator = b"\r\n"
# Reconnect delay
self._retrydelay = 10
# Connect to Envisalink
self.do_connect()
# Setup timer to refresh envisalink
tornado.ioloop.PeriodicCallback(self.check_connection, 1000).start()
# Last activity
self._last_activity = time.time()
def __init__(self, io_loop, request, on_message_callback=None,
compression_options=None):
self.compression_options = compression_options
self.connect_future = TracebackFuture()
self.protocol = None
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
self._on_message_callback = on_message_callback
self.close_code = self.close_reason = None
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
if self.compression_options is not None:
# Always offer to let the server set our max_wbits (and even though
# we don't offer it, we will accept a client_no_context_takeover
# from the server).
# TODO: set server parameters for deflate extension
# if requested in self.compression_options.
request.headers['Sec-WebSocket-Extensions'] = (
'permessage-deflate; client_max_window_bits')
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection, self).__init__(
io_loop, None, request, lambda: None, self._on_http_response,
104857600, self.tcp_client, 65536, 104857600)
class BaseTCPConnector(Connector, RequireEncryptionMixin):
_executor = ThreadPoolExecutor(2, thread_name_prefix="TCP-Executor")
_resolver = netutil.ExecutorResolver(close_executor=False, executor=_executor)
client = TCPClient(resolver=_resolver)
async def connect(self, address, deserialize=True, **connection_args):
self._check_encryption(address, connection_args)
ip, port = parse_host_port(address)
kwargs = self._get_connect_args(**connection_args)
try:
stream = await self.client.connect(
ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs
)
# Under certain circumstances tornado will have a closed connnection with an error and not raise
# a StreamClosedError.
#
# This occurs with tornado 5.x and openssl 1.1+
if stream.closed() and stream.error:
raise StreamClosedError(stream.error)
except StreamClosedError as e:
# The socket connect() call failed
convert_stream_closed_error(self, e)
local_address = self.prefix + get_stream_address(stream)
return self.comm_class(
stream, local_address, self.prefix + address, deserialize
)
def __init__(self, io_loop, request, compression_options=None):
self.compression_options = compression_options
self.connect_future = TracebackFuture()
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
scheme, sep, rest = request.url.partition(":")
scheme = {"ws": "http", "wss": "https"}[scheme]
request.url = scheme + sep + rest
request.headers.update(
{
"Upgrade": "websocket",
"Connection": "Upgrade",
"Sec-WebSocket-Key": self.key,
"Sec-WebSocket-Version": "13",
}
)
if self.compression_options is not None:
# Always offer to let the server set our max_wbits (and even though
# we don't offer it, we will accept a client_no_context_takeover
# from the server).
# TODO: set server parameters for deflate extension
# if requested in self.compression_options.
request.headers["Sec-WebSocket-Extensions"] = "permessage-deflate; client_max_window_bits"
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection, self).__init__(
io_loop, None, request, lambda: None, self._on_http_response, 104857600, self.tcp_client, 65536
)
def __init__(self, routes, node, pipe):
"""
Application instantiates and registers handlers for each message type,
and routes messages to the pre-instantiated instances of each message handler
:param routes: list of tuples in the form of (<message type str>, <MessageHandler class>)
:param node: Node instance of the local node
:param pipe: Instance of multiprocessing.Pipe for communicating with the parent process
"""
# We don't really have to worry about synchronization
# so long as we're careful about explicit context switching
self.nodes = {node.node_id: node}
self.local_node = node
self.handlers = {}
self.tcpclient = TCPClient()
self.gossip_inbox = Queue()
self.gossip_outbox = Queue()
self.sequence_number = 0
if routes:
self.add_handlers(routes)
self.pipe = pipe
self.ioloop = IOLoop.current()
self.add_node_event = Event()
def initialize(self, io_loop=None, max_clients=10,
max_simultaneous_connections=None,
hostname_mapping=None, max_buffer_size=104857600,
resolver=None, defaults=None, max_header_size=None,
max_body_size=None):
"""Creates a AsyncHTTPClient.
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
``force_instance=True`` may be used to suppress this behavior.
Note that because of this implicit reuse, unless ``force_instance``
is used, only the first call to the constructor actually uses
its arguments. It is recommended to use the ``configure`` method
instead of the constructor to ensure that arguments take effect.
``max_clients`` is the number of concurrent requests that can be
in progress; when this limit is reached additional requests will be
queued. Note that time spent waiting in this queue still counts
against the ``request_timeout``.
``hostname_mapping`` is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like ``/etc/hosts`` is not possible or desirable (e.g. in
unittests).
``max_buffer_size`` (default 100MB) is the number of bytes
that can be read into memory at once. ``max_body_size``
(defaults to ``max_buffer_size``) is the largest response body
that the client will accept. Without a
``streaming_callback``, the smaller of these two limits
applies; with a ``streaming_callback`` only ``max_body_size``
does.
.. versionchanged:: 4.2
Added the ``max_body_size`` argument.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
self.max_body_size = max_body_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop)
def test_BatchedStream():
port = 3434
server = MyServer()
server.listen(port)
client = TCPClient()
stream = yield client.connect('127.0.0.1', port)
b = BatchedStream(stream, interval=20)
b.send('hello')
b.send('world')
result = yield b.recv(); assert result == 'hello'
result = yield b.recv(); assert result == 'hello'
result = yield b.recv(); assert result == 'world'
result = yield b.recv(); assert result == 'world'
b.close()
def start(self):
self.client = TCPClient()
self.pcb = PeriodicCallback(self.send, 1000.0 / self.n)
self.pcb.start()
IOLoop.current().call_later(self.duration + 0.5, self.stop)
IOLoop.current().start()
IOLoop.clear_current()
def run_task(self, host):
client = TCPClient()
LOG.debug("connecting to `%s:%s'" % (host, self.port))
try:
stream = yield client.connect(host, port=self.port)
LOG.debug("sending query `%s' to `%s:%s'" % (
self.content.encode('string-escape'), host, self.port))
yield stream.write(self.content)
ret = yield stream.read_until_close()
resp = str(ret).encode('string-escape')[:DEBUG_CONTENT_LENGTH]
LOG.debug("`%s:%s' returns `%s'" % (host, self.port, resp))
stream.close()
del stream
self.returns[host] = ret
except:
LOG.warn("`%s:%s' return status unknown" % (host, self.port))
self.returns[host] = None
finally:
client = None
def test_BatchedSend():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=10)
b.start(stream)
yield b.last_send
yield gen.sleep(0.020)
b.send('hello')
b.send('hello')
b.send('world')
yield gen.sleep(0.020)
b.send('HELLO')
b.send('HELLO')
result = yield read(stream); assert result == ['hello', 'hello', 'world']
result = yield read(stream); assert result == ['HELLO', 'HELLO']
def __init__(self, io_loop=None, max_clients=10, max_buffer_size=None, max_response_size=None):
self._io_loop = io_loop or IOLoop.instance()
self.max_clients = max_clients
self.max_buffer_size = max_buffer_size or 104857600 # 100M
self.max_response_size = max_response_size or 10 * 1024 * 1024 # 10M
self.queue = collections.deque()
self.active = {}
self.waiting = {}
self._client_closed = False
self.tcp_client = TCPClient(io_loop=self._io_loop)
def test_send_before_close():
with echo_server() as e:
client = TCPClient()
stream = yield client.connect('127.0.0.1', e.port)
b = BatchedSend(interval=10)
b.start(stream)
cnt = int(e.count)
b.send('hello')
yield b.close() # close immediately after sending
assert not b.buffer
start = time()
while e.count != cnt + 1:
yield gen.sleep(0.01)
assert time() < start + 5
with pytest.raises(StreamClosedError):
b.send('123')
class SafeTcpStream:
def __init__(self, host, port):
self.host = host
self.port = port
self.stream = None
self.tcp_client = TCPClient()
self.make_tcp_connection_loop()
@gen.coroutine
def make_tcp_connection_loop(self, once=False):
while True:
if self.stream is None:
try:
if once:
self.stream = yield gen.with_timeout(
ioloop.IOLoop.instance().time() + ONCE_CONNECT_TIMEOUT,
self.tcp_client.connect(self.host, self.port))
else:
self.stream = yield self.tcp_client.connect(self.host, self.port)
if self.stream is not None:
self.stream.set_close_callback(self.disconnected)
except Exception:
pass
if once:
break
yield gen.Task(ioloop.IOLoop.instance().add_timeout, ioloop.IOLoop.instance().time() + RECONNECT_INTERVAL)
def disconnected(self):
try:
self.stream.close()
except Exception:
pass
self.stream = None
def write(self, data):
if self.stream is not None:
return self.stream.write(data)
def __init__(self, host, port, user_connection_id, docker_id, io_loop):
self.__host = host
self.__port = port
self.__user_connection_id = user_connection_id
self.__docker_id = docker_id
self._io_loop = io_loop
self.client = TCPClient(io_loop=self._io_loop)
self._stream = None
self._requests = dict()
self._requests_signals = {
packet.InPacket.METHOD_SELECT_RESULT: Signal("data"),
packet.InPacket.METHOD_GET_STATUS: Signal("data"),
packet.InPacket.METHOD_CANCEL: Signal("data"),
}
def initialize(self, io_loop, max_clients=10,
hostname_mapping=None, max_buffer_size=104857600,
resolver=None, defaults=None, max_header_size=None):
"""Creates a AsyncHTTPClient.
# 一个IOLoop实例只有一个AsyncHTTPClient实例
Only a single AsyncHTTPClient instance exists per IOLoop
in order to provide limitations on the number of pending connections.
force_instance=True may be used to suppress this behavior.
max_clients is the number of concurrent requests that can be
in progress. Note that this arguments are only used when the
client is first created, and will be ignored when an existing
client is reused.
hostname_mapping is a dictionary mapping hostnames to IP addresses.
It can be used to make local DNS changes when modifying system-wide
settings like /etc/hosts is not possible or desirable (e.g. in
unittests).
max_buffer_size is the number of bytes that can be read by IOStream. It
defaults to 100mb.
"""
super(SimpleAsyncHTTPClient, self).initialize(io_loop,
defaults=defaults)
self.max_clients = max_clients # 最大连接数
self.queue = collections.deque() # 来一个小小的队列
self.active = {}
self.waiting = {}
self.max_buffer_size = max_buffer_size
self.max_header_size = max_header_size
# TCPClient could create a Resolver for us, but we have to do it
# ourselves to support hostname_mapping.
# 这里估计是一个解析DNS的配置，先不理会
if resolver:
self.resolver = resolver
self.own_resolver = False
else:
self.resolver = Resolver(io_loop=io_loop)
self.own_resolver = True
if hostname_mapping is not None:
self.resolver = OverrideResolver(resolver=self.resolver,
mapping=hostname_mapping)
# 这里来了一个tcp的连接
self.tcp_client = TCPClient(resolver=self.resolver, io_loop=io_loop)
def __init__(self, io_loop, request):
self.connect_future = TracebackFuture()
self.read_future = None
self.read_queue = collections.deque()
self.key = base64.b64encode(os.urandom(16))
scheme, sep, rest = request.url.partition(':')
scheme = {'ws': 'http', 'wss': 'https'}[scheme]
request.url = scheme + sep + rest
request.headers.update({
'Upgrade': 'websocket',
'Connection': 'Upgrade',
'Sec-WebSocket-Key': self.key,
'Sec-WebSocket-Version': '13',
})
self.tcp_client = TCPClient(io_loop=io_loop)
super(WebSocketClientConnection, self).__init__(
io_loop, None, request, lambda: None, self._on_http_response,
104857600, self.tcp_client, 65536)
