def _maybe_connect(self, to_pid, callback=None):
"""Asynchronously establish a connection to the remote pid."""
callback = stack_context.wrap(callback or (lambda stream: None))
def streaming_callback(data):
# we are not guaranteed to get an acknowledgment, but log and discard bytes if we do.
log.info('Received %d bytes from %s, discarding.' %
(len(data), to_pid))
log.debug(' data: %r' % (data, ))
def on_connect(exit_cb, stream):
log.info('Connection to %s established' % to_pid)
with self._connection_callbacks_lock:
self._connections[to_pid] = stream
self.__dispatch_on_connect_callbacks(to_pid, stream)
self.__loop.add_callback(stream.read_until_close,
exit_cb,
streaming_callback=streaming_callback)
create = False
with self._connection_callbacks_lock:
stream = self._connections.get(to_pid)
callbacks = self._connection_callbacks.get(to_pid)
if not stream:
self._connection_callbacks[to_pid].append(callback)
if not callbacks:
create = True
if stream:
self.__loop.add_callback(callback, stream)
return
if not create:
return
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
if not sock:
raise self.SocketError('Failed opening socket')
stream = IOStream(sock, io_loop=self.__loop)
stream.set_nodelay(True)
stream.set_close_callback(
partial(self.__on_exit, to_pid, b'reached end of stream'))
connect_callback = partial(on_connect, partial(self.__on_exit, to_pid),
stream)
log.info('Establishing connection to %s' % to_pid)
stream.connect((to_pid.ip, to_pid.port), callback=connect_callback)
if stream.closed():
raise self.SocketError('Failed to initiate stream connection')
log.info('Maybe connected to %s' % to_pid)
def _maybe_connect(self, to_pid, callback=None):
"""Asynchronously establish a connection to the remote pid."""
callback = stack_context.wrap(callback or (lambda stream: None))
def streaming_callback(data):
# we are not guaranteed to get an acknowledgment, but log and discard bytes if we do.
log.info('Received %d bytes from %s, discarding.' % (len(data), to_pid))
log.debug(' data: %r' % (data,))
def on_connect(exit_cb, stream):
log.info('Connection to %s established' % to_pid)
with self._connection_callbacks_lock:
self._connections[to_pid] = stream
self.__dispatch_on_connect_callbacks(to_pid, stream)
self.__loop.add_callback(
stream.read_until_close,
exit_cb,
streaming_callback=streaming_callback)
create = False
with self._connection_callbacks_lock:
stream = self._connections.get(to_pid)
callbacks = self._connection_callbacks.get(to_pid)
if not stream:
self._connection_callbacks[to_pid].append(callback)
if not callbacks:
create = True
if stream:
self.__loop.add_callback(callback, stream)
return
if not create:
return
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
if not sock:
raise self.SocketError('Failed opening socket')
stream = IOStream(sock, io_loop=self.__loop)
stream.set_nodelay(True)
stream.set_close_callback(partial(self.__on_exit, to_pid, b'reached end of stream'))
connect_callback = partial(on_connect, partial(self.__on_exit, to_pid), stream)
log.info('Establishing connection to %s' % to_pid)
stream.connect((to_pid.ip, to_pid.port), callback=connect_callback)
if stream.closed():
raise self.SocketError('Failed to initiate stream connection')
log.info('Maybe connected to %s' % to_pid)
def _create_connection(self, stream: IOStream) -> HTTP1Connection:
stream.set_nodelay(True)
connection = HTTP1Connection(
stream,
True,
HTTP1ConnectionParameters(
no_keep_alive=True,
max_header_size=self.max_header_size,
max_body_size=self.max_body_size,
decompress=bool(self.request.decompress_response),
),
self._sockaddr,
)
return connection
def _create_connection(self, stream: IOStream) -> HTTP1Connection:
stream.set_nodelay(True)
connection = HTTP1Connection(
stream,
True,
HTTP1ConnectionParameters(
no_keep_alive=True,
max_header_size=self.max_header_size,
max_body_size=self.max_body_size,
decompress=bool(self.request.decompress_response),
),
self._sockaddr,
)
return connection
def io_stream(request, io_loop):
'''Create an instance of the `tornado.iostream.IOStream`.
Current `tornado.ioloop.IOLoop` is used for the stream, that is
provided by `io_loop` fixture.
No-delay flag is set for this stream. The no-delay flag requests that data
should be written as soon as possible, even if doing so would consume
additional bandwidth.
'''
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
stream = IOStream(s)
stream.set_nodelay(True)
yield stream
stream.close()
def io_stream(request, io_loop):
"""Create an instance of the `tornado.iostream.IOStream`.
Current `tornado.ioloop.IOLoop` is used for the stream, that is
provided by `io_loop` fixture.
No-delay flag is set for this stream. The no-delay flag requests that data
should be written as soon as possible, even if doing so would consume
additional bandwidth.
"""
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
stream = IOStream(s)
stream.set_nodelay(True)
yield stream
stream.close()
def maybe_connect(self, to_pid, callback=None):
"""Synchronously open a connection to to_pid or return a connection if it exists."""
callback = stack_context.wrap(callback or (lambda stream: None))
def streaming_callback(data):
# we are not guaranteed to get an acknowledgment, but log and discard bytes if we do.
log.info('Received %d bytes from %s, discarding.' % (len(data), to_pid))
def on_connect(exit_cb, stream):
log.info('Connection to %s established' % to_pid)
self._connections[to_pid] = stream
callback(stream)
self.loop.add_callback(stream.read_until_close, exit_cb,
streaming_callback=streaming_callback)
stream = self._connections.get(to_pid)
if stream is not None:
callback(stream)
return
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
if not sock:
raise self.SocketError("Failed opening socket")
# Set the socket non-blocking
sock.setblocking(0)
stream = IOStream(sock, io_loop=self.loop)
stream.set_nodelay(True)
stream.set_close_callback(partial(self.__on_exit, to_pid, b'closed from maybe_connect'))
connect_callback = partial(on_connect, partial(self.__on_exit, to_pid), stream)
log.info('Establishing connection to %s' % to_pid)
stream.connect((to_pid.ip, to_pid.port), callback=connect_callback)
if stream.closed():
raise self.SocketError("Failed to initiate stream connection")
log.info('Maybe connected to %s' % to_pid)
class _RedisConnection(object):
def __init__(self, io_loop, write_buf, final_callback, redis_tuple,
redis_pass):
"""
:param io_loop: 你懂的
:param write_buf: 第一次写入
:param final_callback: resp赋值时调用
:param redis_tuple: (ip, port, db)
:param redis_pass: redis密码
"""
self.__io_loop = io_loop
self.__final_cb = final_callback
self.__stream = None
#redis应答解析remain
self.__recv_buf = ''
self.__write_buf = write_buf
init_buf = ''
init_buf = chain_select_cmd(redis_tuple[2], init_buf)
if redis_pass is None:
self.__init_buf = (init_buf, )
else:
assert redis_pass and isinstance(redis_pass, str)
self.__init_buf = (redis_auth(redis_pass), init_buf)
self.__haspass = redis_pass is not None
self.__init_buf = ''.join(self.__init_buf)
self.__connect_state = CONNECT_INIT
#redis指令上下文, connect指令个数(AUTH, SELECT .etc)，trans，cmd_count
self.__cmd_env = deque()
self.__written = False
def connect(self, init_future, redis_tuple, active_trans, cmd_count):
"""
:param init_future: 第一个future对象
:param redis_tuple: (ip, port, db)
:param active_trans: 事务是否激活
:param cmd_count: 指令个数
"""
if self.__stream is not None:
return
#future, connect_count, transaction, cmd_count
self.__cmd_env.append((init_future, 1 + int(self.__haspass), False, 0))
self.__cmd_env.append((init_future, 0, active_trans, cmd_count))
with ExceptionStackContext(self.__handle_ex):
self.__stream = IOStream(socket.socket(socket.AF_INET,
socket.SOCK_STREAM, 0),
io_loop=self.__io_loop)
self.__stream.set_close_callback(self.__on_close)
self.__stream.connect(redis_tuple[:2], self.__on_connect)
self.__connect_state = CONNECT_ING
def connect_state(self):
return self.__connect_state
def write(self,
write_buf,
new_future,
include_select,
active_trans,
cmd_count,
by_connect=False):
"""
:param new_future: 由于闭包的影响，在resp回调函数中会保存上一次的future对象，该对象必须得到更新
:param include_select: 是否包含SELECT指令
:param active_trans: 事务是否激活
:param cmd_count: 指令个数
"""
if by_connect:
self.__stream.write(self.__init_buf)
self.__init_buf = None
if self.__write_buf:
self.__stream.write(self.__write_buf)
self.__write_buf = None
return
self.__cmd_env.append(
(new_future, int(include_select), active_trans, cmd_count))
if self.__connect_state == CONNECT_ING:
self.__write_buf = ''.join((self.__write_buf, write_buf))
return
if self.__write_buf:
write_buf = ''.join((self.__write_buf, write_buf))
self.__stream.write(write_buf)
self.__write_buf = None
def __on_connect(self):
"""连接，只需要发送初始cmd即可
"""
self.__connect_state = CONNECT_SUCC
self.__stream.set_nodelay(True)
self.write(None, None, None, None, None, True)
self.__stream.read_until_close(None, self.__on_resp)
def __on_resp(self, recv):
"""
:param recv: 收到的buf
"""
recv = ''.join((self.__recv_buf, recv))
idx = 0
for future, connect_count, trans, count in self.__cmd_env:
ok, payload, recv = decode_resp_ondemand(recv, connect_count,
trans, count)
if not ok:
break
idx += 1
if count > 0:
self.__run_callback({
_RESP_FUTURE: future,
RESP_RESULT: payload
})
self.__recv_buf = recv
for _ in xrange(idx):
self.__cmd_env.popleft()
def __on_close(self):
self.__connect_state = CONNECT_INIT
if self.__final_cb:
if self.__stream.error:
self.__run_callback({RESP_ERR: self.__stream.error})
def __run_callback(self, resp):
if self.__final_cb is None:
return
self.__io_loop.add_callback(self.__final_cb, resp)
def __handle_ex(self, typ, value, tb):
"""
:param typ: 异常类型
"""
if self.__final_cb:
self.__run_callback({RESP_ERR: value})
return True
return False
class _RedisConnection(object):
def __init__(self, final_callback, redis_tuple, redis_pwd):
"""
:param final_callback: resp赋值时调用
:param redis_tuple: (ip, port, db)
:param redis_pwd: redis密码
"""
self.__io_loop = IOLoop.instance()
self.__resp_cb = final_callback
self.__stream = None
#redis应答解析remain
self.__recv_buf = ''
self.__redis_tuple = redis_tuple
self.__redis_pwd = redis_pwd
#redis指令上下文, connect指令个数(AUTH, SELECT .etc)，trans，cmd_count
self.__cmd_env = deque()
self.__cache_before_connect = []
self.__connected = False
def con_ok(self):
"""
连接对象是否ok
:return:
"""
return self.__connected
def connect(self, init_future):
"""
connect指令包括：AUTH, SELECT
:param init_future: 第一个future对象
"""
#future, connect_count, transaction, cmd_count
self.__cmd_env.append((init_future, 1 + int(bool(self.__redis_pwd)), False, 0))
self.__stream = IOStream(socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0),
io_loop=self.__io_loop)
self.__stream.set_close_callback(self.__on_close)
self.__stream.connect(self.__redis_tuple[:2], self.__on_connect)
def __on_connect(self):
"""连接，只需要发送初始cmd即可
"""
self.__connected = True
self.__stream.set_nodelay(True)
self.__stream.read_until_close(self.__last_closd_recv, self.__on_resp)
self.__stream.write(chain_select_cmd(self.__redis_pwd, self.__redis_tuple[-1]))
for x in self.__cache_before_connect:
self.__stream.write(x)
self.__cache_before_connect = []
def write(self, buf, new_future, active_trans, cmd_count):
"""
:param new_future: 由于闭包的影响，在resp回调函数中会保存上一次的future对象，该对象必须得到更新
:param active_trans: 事务是否激活
:param cmd_count: 指令个数
"""
self.__cmd_env.append((new_future, 0, active_trans, cmd_count))
if not self.__connected:
self.__cache_before_connect.append(buf)
return
self.__stream.write(buf)
def __last_closd_recv(self, data):
"""
socket关闭时最后几个字节
"""
if not data:
return
self.__on_resp(data)
def __on_resp(self, recv):
"""
:param recv: 收到的buf
"""
recv = ''.join((self.__recv_buf, recv))
idx = 0
for future, connect, trans, cmd in self.__cmd_env:
ok, payload, recv = decode_resp_ondemand(recv, connect, trans, cmd)
if not ok:
break
idx += 1
if not connect:
self.__run_callback({_RESP_FUTURE: future, RESP_RESULT: payload})
self.__recv_buf = recv
for _ in xrange(idx):
self.__cmd_env.popleft()
def __run_callback(self, resp):
if self.__resp_cb is None:
return
self.__io_loop.add_callback(self.__resp_cb, resp)
def __on_close(self):
self.__connected = False
while len(self.__cmd_env) > 0:
self.__run_callback({_RESP_FUTURE: self.__cmd_env.popleft(), RESP_RESULT: 0})
self.__cmd_env.clear()
class redisSub:
def __init__(self, host="localhost", port=6379, db=0, pw=None):
self.socket = socket.socket()
self.socket.connect((host, port))
self.fp = self.socket.makefile("rb")
self.stream = None
if pw:
self._command("AUTH", pw)
self._get_reply()
if db:
self._command("SELECT", db)
self._get_reply()
def _command(self, *args):
cmd = [b"*", b(len(args)), b"\r\n"]
for val in args:
val = b(val)
cmd += [b"$", b(len(val)), b"\r\n", val, b"\r\n"]
cmd = b"".join(cmd)
self.socket.sendall(cmd)
def _get_reply(self):
data = self.fp.readline()
flag, data = data[0], data[1:-2]
if flag == 36: # b"$" bulk
length = int(data)
if length == -1:
data = None
else:
data = self.fp.readline()
data = data[:-2].decode()
elif flag == 42: # b"*" multi bulk
length = int(data)
if length == -1:
data = None
else:
data = tuple(self._get_reply() for i in range(length))
elif flag == 58: # b":" integer
data = int(data)
elif flag == 43: # b"+" status
data = data.decode()
elif flag == 45: # b"-" error
raise Exception(data)
return data
def subscribe(self, *channels):
self._command("SUBSCRIBE", *channels)
return tuple(self._get_reply() for i in channels)
def unsubscribe(self, *channels):
self._command("UNSUBSCRIBE", *channels)
return tuple(self._get_reply() for i in channels)
def psubscribe(self, *pattenns):
self._command("PSUBSCRIBE", *pattenns)
return tuple(self._get_reply() for i in pattenns)
def punsubscribe(self, *pattenns):
self._command("PUNSUBSCRIBE", *pattenns)
return tuple(self._get_reply() for i in pattenns)
def close(self):
if self.stream is not None:
self.stream.close()
else:
self.fp.close()
self.socket.close()
@gen.coroutine
def listen(self, callback):
if self.stream is not None:
return
self.fp.close()
self.stream = IOStream(self.socket)
self.stream.set_nodelay(True)
while not self.stream.closed():
data = yield gen.Task(self.stream.read_until, b"\n")
flag, data = data[0], data[1:-2]
if flag != 42:
raise Exception("unexpected reply flag: " + chr(flag))
length = int(data)
if length == -1:
reply = None
else:
reply = []
for i in range(length):
data = yield gen.Task(self.stream.read_until, b"\n")
flag, data = data[0], data[1:-2]
if flag != 36:
raise Exception("unexpected reply flag: " + chr(flag))
length = int(data)
if length == -1:
reply.append(None)
else:
data = yield gen.Task(self.stream.read_until, b"\n")
data = data[:-2].decode()
reply.append(data)
reply = tuple(reply)
callback(reply)
class AsyncConn(event.EventedMixin):
"""
Low level object representing a TCP connection to nsqd.
When a message on this connection is requeued and the requeue delay
has not been specified, it calculates the delay automatically by an
increasing multiple of ``requeue_delay``.
Generates the following events that can be listened to with
:meth:`nsq.AsyncConn.on`:
* ``connect``
* ``close``
* ``error``
* ``identify``
* ``identify_response``
* ``auth``
* ``auth_response``
* ``heartbeat``
* ``ready``
* ``message``
* ``response``
* ``backoff``
* ``resume``
:param host: the host to connect to
:param port: the post to connect to
:param timeout: the timeout for read/write operations (in seconds)
:param heartbeat_interval: the amount of time (in seconds) to negotiate
with the connected producers to send heartbeats (requires nsqd 0.2.19+)
:param requeue_delay: the base multiple used when calculating requeue delay
(multiplied by # of attempts)
:param tls_v1: enable TLS v1 encryption (requires nsqd 0.2.22+)
:param tls_options: dictionary of options to pass to `ssl.wrap_socket()
<http://docs.python.org/2/library/ssl.html#ssl.wrap_socket>`_ as
``**kwargs``
:param snappy: enable Snappy stream compression (requires nsqd 0.2.23+)
:param deflate: enable deflate stream compression (requires nsqd 0.2.23+)
:param deflate_level: configure the deflate compression level for this
connection (requires nsqd 0.2.23+)
:param output_buffer_size: size of the buffer (in bytes) used by nsqd
for buffering writes to this connection
:param output_buffer_timeout: timeout (in ms) used by nsqd before
flushing buffered writes (set to 0 to disable). **Warning**:
configuring clients with an extremely low (``< 25ms``)
``output_buffer_timeout`` has a significant effect on ``nsqd``
CPU usage (particularly with ``> 50`` clients connected).
:param sample_rate: take only a sample of the messages being sent
to the client. Not setting this or setting it to 0 will ensure
you get all the messages destined for the client.
Sample rate can be greater than 0 or less than 100 and the client
will receive that percentage of the message traffic.
(requires nsqd 0.2.25+)
:param user_agent: a string identifying the agent for this client
in the spirit of HTTP (default: ``<client_library_name>/<version>``)
(requires nsqd 0.2.25+)
:param auth_secret: a string passed when using nsq auth
(requires nsqd 1.0+)
:param msg_timeout: the amount of time (in seconds) that nsqd will wait
before considering messages that have been delivered to this
consumer timed out (requires nsqd 0.2.28+)
:param hostname: a string identifying the host where this client runs
(default: ``<hostname>``)
"""
def __init__(self,
host,
port,
timeout=1.0,
heartbeat_interval=30,
requeue_delay=90,
tls_v1=False,
tls_options=None,
snappy=False,
deflate=False,
deflate_level=6,
user_agent=DEFAULT_USER_AGENT,
output_buffer_size=16 * 1024,
output_buffer_timeout=250,
sample_rate=0,
auth_secret=None,
msg_timeout=None,
hostname=None):
assert isinstance(host, string_types)
assert isinstance(port, int)
assert isinstance(timeout, float)
assert isinstance(tls_options, (dict, None.__class__))
assert isinstance(deflate_level, int)
assert isinstance(heartbeat_interval, int) and heartbeat_interval >= 1
assert isinstance(requeue_delay, int) and requeue_delay >= 0
assert isinstance(output_buffer_size, int) and output_buffer_size >= 0
assert isinstance(output_buffer_timeout,
int) and output_buffer_timeout >= 0
assert isinstance(sample_rate,
int) and sample_rate >= 0 and sample_rate < 100
assert msg_timeout is None or (isinstance(msg_timeout, (float, int))
and msg_timeout > 0)
# auth_secret validated by to_bytes() below
self.state = INIT
self.host = host
self.port = port
self.timeout = timeout
self.last_recv_timestamp = time.time()
self.last_msg_timestamp = time.time()
self.in_flight = 0
self.rdy = 0
self.rdy_timeout = None
# for backwards compatibility when interacting with older nsqd
# (pre 0.2.20), default this to their hard-coded max
self.max_rdy_count = 2500
self.tls_v1 = tls_v1
self.tls_options = tls_options
self.snappy = snappy
self.deflate = deflate
self.deflate_level = deflate_level
self.hostname = hostname
if self.hostname is None:
self.hostname = socket.gethostname()
self.short_hostname = self.hostname.split('.')[0]
self.heartbeat_interval = heartbeat_interval * 1000
self.msg_timeout = int(msg_timeout * 1000) if msg_timeout else None
self.requeue_delay = requeue_delay
self.output_buffer_size = output_buffer_size
self.output_buffer_timeout = output_buffer_timeout
self.sample_rate = sample_rate
self.user_agent = user_agent
self._authentication_required = False # tracking server auth state
self.auth_secret = to_bytes(auth_secret) if auth_secret else None
self.socket = None
self.stream = None
self._features_to_enable = []
self.last_rdy = 0
self.rdy = 0
self.callback_queue = []
self.encoder = DefaultEncoder()
super(AsyncConn, self).__init__()
@property
def id(self):
return str(self)
def __str__(self):
return self.host + ':' + str(self.port)
def connected(self):
return self.state == CONNECTED
def connecting(self):
return self.state == CONNECTING
def closed(self):
return self.state in (INIT, DISCONNECTED)
def connect(self):
if not self.closed():
return
# Assume host is an ipv6 address if it has a colon.
if ':' in self.host:
self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
else:
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.settimeout(self.timeout)
self.socket.setblocking(0)
self.stream = IOStream(self.socket)
self.stream.set_close_callback(self._socket_close)
self.stream.set_nodelay(True)
self.state = CONNECTING
self.on(event.CONNECT, self._on_connect)
self.on(event.DATA, self._on_data)
fut = self.stream.connect((self.host, self.port))
IOLoop.current().add_future(fut, self._connect_callback)
def _connect_callback(self, fut):
fut.result()
self.state = CONNECTED
self.stream.write(protocol.MAGIC_V2)
self._start_read()
self.trigger(event.CONNECT, conn=self)
def _read_bytes(self, size, callback):
try:
fut = self.stream.read_bytes(size)
IOLoop.current().add_future(fut, callback)
except IOError:
self.close()
self.trigger(
event.ERROR,
conn=self,
error=protocol.ConnectionClosedError('Stream is closed'),
)
def _start_read(self):
if self.stream is None:
return # IOStream.start_tls() invalidates stream, will call again when ready
self._read_bytes(4, self._read_size)
def _socket_close(self):
self.state = DISCONNECTED
self.trigger(event.CLOSE, conn=self)
def close(self):
self.stream.close()
def _read_size(self, fut):
try:
data = fut.result()
size = struct_l.unpack(data)[0]
except Exception:
self.close()
self.trigger(
event.ERROR,
conn=self,
error=protocol.IntegrityError('failed to unpack size'),
)
return
self._read_bytes(size, self._read_body)
def _read_body(self, fut):
try:
data = fut.result()
self.trigger(event.DATA, conn=self, data=data)
except Exception:
logger.exception('uncaught exception in data event')
self._start_read()
def send(self, data):
return self.stream.write(self.encoder.encode(data))
def upgrade_to_tls(self, options=None):
# in order to upgrade to TLS we need to *replace* the IOStream...
opts = {
'cert_reqs': ssl.CERT_REQUIRED,
'ssl_version': ssl.PROTOCOL_TLSv1_2
}
opts.update(options or {})
fut = self.stream.start_tls(False,
ssl_options=opts,
server_hostname=self.host)
self.stream = None
def finish_upgrade_tls(fut):
try:
self.stream = fut.result()
self.socket = self.stream.socket
self._start_read()
except Exception as e:
# skip self.close() because no stream
self.trigger(
event.ERROR,
conn=self,
error=protocol.SendError('failed to upgrade to TLS', e),
)
IOLoop.current().add_future(fut, finish_upgrade_tls)
def upgrade_to_snappy(self):
assert SnappySocket, 'snappy requires the python-snappy package'
# in order to upgrade to Snappy we need to use whatever IOStream
# is currently in place (normal or SSL)...
#
# first read any compressed bytes the existing IOStream might have
# already buffered and use that to bootstrap the SnappySocket, then
# monkey patch the existing IOStream by replacing its socket
# with a wrapper that will automagically handle compression.
existing_data = self.stream._consume(self.stream._read_buffer_size)
self.socket = SnappySocket(self.socket)
self.socket.bootstrap(existing_data)
self.stream.socket = self.socket
self.encoder = SnappyEncoder()
def upgrade_to_deflate(self):
# in order to upgrade to DEFLATE we need to use whatever IOStream
# is currently in place (normal or SSL)...
#
# first read any compressed bytes the existing IOStream might have
# already buffered and use that to bootstrap the DeflateSocket, then
# monkey patch the existing IOStream by replacing its socket
# with a wrapper that will automagically handle compression.
existing_data = self.stream._consume(self.stream._read_buffer_size)
self.socket = DeflateSocket(self.socket, self.deflate_level)
self.socket.bootstrap(existing_data)
self.stream.socket = self.socket
self.encoder = DeflateEncoder(level=self.deflate_level)
def send_rdy(self, value):
if self.last_rdy != value:
try:
self.send(protocol.ready(value))
except Exception as e:
self.close()
self.trigger(
event.ERROR,
conn=self,
error=protocol.SendError('failed to send RDY %d' % value,
e),
)
return False
self.last_rdy = value
self.rdy = value
return True
def _on_connect(self, **kwargs):
identify_data = {
'short_id': self.
short_hostname, # TODO remove when deprecating pre 1.0 support
'long_id':
self.hostname, # TODO remove when deprecating pre 1.0 support
'client_id': self.short_hostname,
'hostname': self.hostname,
'heartbeat_interval': self.heartbeat_interval,
'feature_negotiation': True,
'tls_v1': self.tls_v1,
'snappy': self.snappy,
'deflate': self.deflate,
'deflate_level': self.deflate_level,
'output_buffer_timeout': self.output_buffer_timeout,
'output_buffer_size': self.output_buffer_size,
'sample_rate': self.sample_rate,
'user_agent': self.user_agent
}
if self.msg_timeout:
identify_data['msg_timeout'] = self.msg_timeout
self.trigger(event.IDENTIFY, conn=self, data=identify_data)
self.on(event.RESPONSE, self._on_identify_response)
try:
self.send(protocol.identify(identify_data))
except Exception as e:
self.close()
self.trigger(
event.ERROR,
conn=self,
error=protocol.SendError('failed to bootstrap connection', e),
)
def _on_identify_response(self, data, **kwargs):
self.off(event.RESPONSE, self._on_identify_response)
if data == b'OK':
logger.warning(
'nsqd version does not support feature netgotiation')
return self.trigger(event.READY, conn=self)
try:
data = json.loads(data.decode('utf-8'))
except ValueError:
self.close()
self.trigger(
event.ERROR,
conn=self,
error=protocol.IntegrityError(
'failed to parse IDENTIFY response JSON from nsqd - %r' %
data),
)
return
self.trigger(event.IDENTIFY_RESPONSE, conn=self, data=data)
if self.tls_v1 and data.get('tls_v1'):
self._features_to_enable.append('tls_v1')
if self.snappy and data.get('snappy'):
self._features_to_enable.append('snappy')
if self.deflate and data.get('deflate'):
self._features_to_enable.append('deflate')
if data.get('auth_required'):
self._authentication_required = True
if data.get('max_rdy_count'):
self.max_rdy_count = data.get('max_rdy_count')
else:
# for backwards compatibility when interacting with older nsqd
# (pre 0.2.20), default this to their hard-coded max
logger.warn('setting max_rdy_count to default value of 2500')
self.max_rdy_count = 2500
self.on(event.RESPONSE, self._on_response_continue)
self._on_response_continue(conn=self, data=None)
def _on_response_continue(self, data, **kwargs):
if self._features_to_enable:
feature = self._features_to_enable.pop(0)
if feature == 'tls_v1':
self.upgrade_to_tls(self.tls_options)
elif feature == 'snappy':
self.upgrade_to_snappy()
elif feature == 'deflate':
self.upgrade_to_deflate()
# the server will 'OK' after these connection upgrades triggering another response
return
self.off(event.RESPONSE, self._on_response_continue)
if self.auth_secret and self._authentication_required:
self.on(event.RESPONSE, self._on_auth_response)
self.trigger(event.AUTH, conn=self, data=self.auth_secret)
try:
self.send(protocol.auth(self.auth_secret))
except Exception as e:
self.close()
self.trigger(
event.ERROR,
conn=self,
error=protocol.SendError('Error sending AUTH', e),
)
return
self.trigger(event.READY, conn=self)
def _on_auth_response(self, data, **kwargs):
try:
data = json.loads(data.decode('utf-8'))
except ValueError:
self.close()
self.trigger(
event.ERROR,
conn=self,
error=protocol.IntegrityError(
'failed to parse AUTH response JSON from nsqd - %r' %
data),
)
return
self.off(event.RESPONSE, self._on_auth_response)
self.trigger(event.AUTH_RESPONSE, conn=self, data=data)
return self.trigger(event.READY, conn=self)
def _on_data(self, data, **kwargs):
self.last_recv_timestamp = time.time()
frame, data = protocol.unpack_response(data)
if frame == protocol.FRAME_TYPE_MESSAGE:
self.last_msg_timestamp = time.time()
self.in_flight += 1
message = protocol.decode_message(data)
message.on(event.FINISH, self._on_message_finish)
message.on(event.REQUEUE, self._on_message_requeue)
message.on(event.TOUCH, self._on_message_touch)
self.trigger(event.MESSAGE, conn=self, message=message)
elif frame == protocol.FRAME_TYPE_RESPONSE and data == b'_heartbeat_':
self.send(protocol.nop())
self.trigger(event.HEARTBEAT, conn=self)
elif frame == protocol.FRAME_TYPE_RESPONSE:
self.trigger(event.RESPONSE, conn=self, data=data)
elif frame == protocol.FRAME_TYPE_ERROR:
self.trigger(event.ERROR, conn=self, error=protocol.Error(data))
def _on_message_requeue(self, message, backoff=True, time_ms=-1, **kwargs):
if backoff:
self.trigger(event.BACKOFF, conn=self)
else:
self.trigger(event.CONTINUE, conn=self)
self.in_flight -= 1
try:
time_ms = self.requeue_delay * message.attempts * 1000 if time_ms < 0 else time_ms
self.send(protocol.requeue(message.id, time_ms))
except Exception as e:
self.close()
self.trigger(event.ERROR,
conn=self,
error=protocol.SendError(
'failed to send REQ %s @ %d' %
(message.id, time_ms), e))
def _on_message_finish(self, message, **kwargs):
self.trigger(event.RESUME, conn=self)
self.in_flight -= 1
try:
self.send(protocol.finish(message.id))
except Exception as e:
self.close()
self.trigger(
event.ERROR,
conn=self,
error=protocol.SendError('failed to send FIN %s' % message.id,
e),
)
def _on_message_touch(self, message, **kwargs):
try:
self.send(protocol.touch(message.id))
except Exception as e:
self.close()
self.trigger(
event.ERROR,
conn=self,
error=protocol.SendError(
'failed to send TOUCH %s' % message.id, e),
)
class _RedisConnection(object):
def __init__(self, io_loop, init_buf, final_callback, redis_tuple, redis_pass):
"""
:param io_loop: 你懂的
:param init_buf: 第一次写入
:param final_callback: resp赋值时调用
:param redis_tuple: (ip, port, db)
:param redis_pass: redis密码
"""
self.__io_loop = io_loop
self.__final_cb = final_callback
self.__stream = None
#redis应答解析remain
self.__recv_buf = ''
init_buf = init_buf or ''
init_buf = chain_select_cmd(redis_tuple[2], init_buf)
if redis_pass is None:
self.__init_buf = (init_buf,)
else:
assert redis_pass and isinstance(redis_pass, str)
self.__init_buf = (redis_auth(redis_pass), init_buf)
self.__haspass = redis_pass is not None
self.__init_buf = ''.join(self.__init_buf)
self.__connected = False
#redis指令上下文, connect指令个数(AUTH, SELECT .etc)，trans，cmd_count
self.__cmd_env = deque()
self.__written = False
def connect(self, init_future, redis_tuple, active_trans, cmd_count):
"""
:param init_future: 第一个future对象
:param redis_tuple: (ip, port, db)
:param active_trans: 事务是否激活
:param cmd_count: 指令个数
"""
if self.__stream is not None:
return
#future, connect_count, transaction, cmd_count
self.__cmd_env.append((init_future, 1 + int(self.__haspass), active_trans, cmd_count))
with ExceptionStackContext(self.__handle_ex):
self.__stream = IOStream(socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0),
io_loop=self.__io_loop)
self.__stream.connect(redis_tuple[:2], self.__on_connect)
def write(self, write_buf, new_future, include_select, active_trans, cmd_count, by_connect=False):
"""
:param new_future: 由于闭包的影响，在resp回调函数中会保存上一次的future对象，该对象必须得到更新
:param include_select: 是否包含SELECT指令
:param active_trans: 事务是否激活
:param cmd_count: 指令个数
"""
if by_connect:
self.__stream.write(self.__init_buf)
self.__init_buf = None
return
self.__cmd_env.append((new_future, int(include_select), active_trans, cmd_count))
if not self.__connected:
self.__init_buf = ''.join((self.__init_buf, write_buf))
return
if self.__init_buf:
write_buf = ''.join((self.__init_buf, write_buf))
self.__stream.write(write_buf)
self.__init_buf = None
def __on_connect(self):
"""连接，只需要发送初始cmd即可
"""
self.__connected = True
self.__stream.set_nodelay(True)
self.write(None, None, None, None, None, True)
self.__stream.read_until_close(None, self.__on_resp)
def __on_resp(self, recv):
"""
:param recv: 收到的buf
"""
recv = ''.join((self.__recv_buf, recv))
idx = 0
for future, connect_count, trans, count in self.__cmd_env:
ok, payload, recv = decode_resp_ondemand(recv, connect_count, trans, count)
if not ok:
break
idx += 1
self.__run_callback({_RESP_FUTURE: future, RESP_RESULT: payload})
self.__recv_buf = recv
for _ in xrange(idx):
self.__cmd_env.popleft()
def __run_callback(self, resp):
if self.__final_cb is None:
return
self.__io_loop.add_callback(self.__final_cb, resp)
def __handle_ex(self, typ, value, tb):
"""
:param typ: 异常类型
"""
if self.__final_cb:
self.__run_callback({RESP_ERR: value})
return True
return False
class AsyncSocket(object):
def __init__(self, sock):
self._iostream = IOStream(sock)
self._resolver = Resolver()
self._readtimeout = 0
self._connecttimeout = 0
def set_readtimeout(self, timeout):
self._readtimeout = timeout
def set_connecttimeout(self, timeout):
self._connecttimeout = timeout
@synclize
def connect(self, address):
host, port = address
timer = None
try:
if self._connecttimeout:
timer = Timeout(self._connecttimeout)
timer.start()
resolved_addrs = yield self._resolver.resolve(host, port, family=socket.AF_INET)
for addr in resolved_addrs:
family, host_port = addr
yield self._iostream.connect(host_port)
break
except TimeoutException:
self.close()
raise
finally:
if timer:
timer.cancel()
#@synclize
def sendall(self, buff):
self._iostream.write(buff)
@synclize
def read(self, nbytes, partial=False):
timer = None
try:
if self._readtimeout:
timer = Timeout(self._readtimeout)
timer.start()
buff = yield self._iostream.read_bytes(nbytes, partial=partial)
raise Return(buff)
except TimeoutException:
self.close()
raise
finally:
if timer:
timer.cancel()
def recv(self, nbytes):
return self.read(nbytes, partial=True)
@synclize
def readline(self, max_bytes=-1):
timer = None
if self._readtimeout:
timer = Timeout(self._readtimeout)
timer.start()
try:
if max_bytes > 0:
buff = yield self._iostream.read_until('\n', max_bytes=max_bytes)
else:
buff = yield self._iostream.read_until('\n')
raise Return(buff)
except TimeoutException:
self.close()
raise
finally:
if timer:
timer.cancel()
def close(self):
self._iostream.close()
def set_nodelay(self, flag):
self._iostream.set_nodelay(flag)
def settimeout(self, timeout):
pass
def shutdown(self, direction):
if self._iostream.fileno():
self._iostream.fileno().shutdown(direction)
def recv_into(self, buff):
expected_rbytes = len(buff)
data = self.read(expected_rbytes, True)
srcarray = bytearray(data)
nbytes = len(srcarray)
buff[0:nbytes] = srcarray
return nbytes
def makefile(self, mode, other):
return self
class _PrxConn(object):
def __init__(self, handle_resp, svr_addr):
assert callable(handle_resp)
self._io_loop = IOLoop.instance()
self.__resp_cb = handle_resp
self.__svr_addr = svr_addr
self._stream = None
self._send_buf = deque()
self._recv_buf = ''
self.__cmd_env = deque()
self.__con_ok = False
def con_ok(self):
"""
连接不可用
"""
return self.__con_ok
def connect(self):
self._stream = IOStream(socket.socket(socket.AF_INET,
socket.SOCK_STREAM, 0),
io_loop=self._io_loop)
self._stream.set_close_callback(self._on_close)
self._stream.connect(self.__svr_addr, self._on_connect)
def _on_connect(self):
self._stream.set_nodelay(True)
while len(self._send_buf) > 0:
self._stream.write(self._send_buf.popleft())
self._stream.read_until_close(self._last_closd_recv, self._on_recv)
self.__con_ok = True
def write(self, future, encode_result):
self.__cmd_env.append(future)
if not self.__con_ok:
self._send_buf.append(encode_result)
else:
self._stream.write(encode_result)
def _last_closd_recv(self, data):
"""
socket关闭时最后几个字节
"""
self._on_recv(data)
def _on_recv(self, buf):
self._recv_buf += buf
while 1:
if not self._recv_buf:
break
ok, payload, self._recv_buf = decode_resp_ondemand(
self._recv_buf, 0, False, 1)
if not ok:
break
if payload and isinstance(payload,
(list, tuple)) and 1 == len(payload):
payload = payload[0]
self.__run_callback({
_RESP_FUTURE: self.__cmd_env.popleft(),
RESP_RESULT: payload
})
def __run_callback(self, resp):
if self.__resp_cb is None:
return
self._io_loop.add_callback(self.__resp_cb, resp)
def _on_close(self):
self.__con_ok = False
while len(self.__cmd_env) > 0:
self.__run_callback({
_RESP_FUTURE: self.__cmd_env.popleft(),
RESP_RESULT: 0
})
self.__cmd_env.clear()
class PAConnection(object):
def __init__(self, host, port, io_loop, key):
self.io_loop = io_loop
self.resolver = Resolver()
self._callbacks = {}
self._connected = False
self.queue = deque()
self.key = key
self.stream = None
self.pepv_act_resp = None
self.prof = {}
with stack_context.ExceptionStackContext(self._handle_exception):
self.resolver.resolve(host,
port,
socket.AF_INET,
callback=self._on_resolve)
def _handle_exception(self, typ, value, tb):
gen_log.exception("pa connection error [%s] [%s] %s", typ, value, tb)
def _on_resolve(self, addrinfo):
af = addrinfo[0][0]
self.stream = IOStream(socket.socket(af))
self.stream.set_nodelay(True)
self.stream.set_close_callback(self._on_close)
sockaddr = addrinfo[0][1]
# gen_log.info("sock addr {0}".format(sockaddr))
self.stream.connect(sockaddr, self._on_connect)
def _on_close(self):
gen_log.info("pa conn close")
def _on_connect(self):
# gen_log.info("start conn to pa")
self._connected = True
self.stream.write('\xab\xcd') # magic number of act protocol
# gen_log.info('write data {0}'.format(repr(encode_act_key(self.key))))
self.stream.write(encode_act_key(self.key))
self._process_queue()
self.stream.read_bytes(4, self._on_id)
def _on_id(self, data):
resp = ActResponse()
resp.Id = bytes2int(data)
self.prof[resp.Id].append(time.time())
self.pepv_act_resp = resp
self.stream.read_bytes(4, self._on_rlen)
def _on_rlen(self, data):
self.stream.read_bytes(bytes2int(data), self._on_res_body)
def _on_res_body(self, data):
resp = self.pepv_act_resp
resp.result = data
cb = self._callbacks[resp.Id]
t = time.time()
# gen_log.info(
# "ActID[{0}]: {1}, {2}, {3}, {4}, {5}, {6}, {7}".format(resp.Id, self.prof[resp.Id][0], self.prof[resp.Id][1], self.prof[resp.Id][2], t, self.prof[resp.Id][1]-self.prof[resp.Id][0], self.prof[resp.Id][2]-self.prof[resp.Id][0], t-self.prof[resp.Id][0]))
del self.prof[resp.Id]
del self._callbacks[resp.Id]
# self.io_loop.add_callback(cb, resp)
cb(resp)
self.stream.read_bytes(4, self._on_id)
def fetch(self, act_request, callback):
if act_request.Id in self._callbacks:
gen_log.error("act Id {0} already in cbs !!".format(
act_request.Id))
self._callbacks[act_request.Id] = callback
self.prof[act_request.Id] = [
time.time(),
]
self.queue.append(act_request)
self._process_queue()
def _process_queue(self):
if not self._connected:
# gen_log.info("act connection not ready, wait an other turn")
return
with stack_context.NullContext():
while self.queue:
act_request = self.queue.popleft()
self.prof[act_request.Id].append(time.time())
self.stream.write(act_request.encode_body())
class DubboConnection(object):
READ_HEAD = 0x01
READ_BODY = 0x02
def __init__(self, host, port, io_loop):
self.io_loop = io_loop
self.resolver = Resolver()
self.stream = None
self.queue = deque()
self._callbacks = {}
self._connected = False
self.read_state = self.READ_HEAD
self.prev_response = None
self.prof = {}
with stack_context.ExceptionStackContext(self._handle_exception):
self.resolver.resolve(host,
port,
socket.AF_INET,
callback=self._on_resolve)
def _on_resolve(self, addrinfo):
af = addrinfo[0][0]
self.stream = IOStream(socket.socket(af))
self.stream.set_nodelay(True)
self.stream.set_close_callback(self._on_close)
sockaddr = addrinfo[0][1]
gen_log.info("sock addr {0}".format(sockaddr))
self.stream.connect(sockaddr, self._on_connect)
def _on_close(self):
gen_log.info("close dubbo connect")
def _on_connect(self):
gen_log.info("dubbo connect ready")
self._connected = True
self._process_queue()
self.stream.read_bytes(16, self._on_header)
def _on_header(self, data):
# print 'read header', data
resp = Response()
resp.decode_header(data)
self.prof[resp.Id].append(time.time())
self.prev_response = resp
self.stream.read_bytes(resp.data_len, self._on_body)
def _on_body(self, data):
resp = self.prev_response
resp.decode_body(data)
cb = self._callbacks[resp.Id]
t = time.time()
# gen_log.info(
# "DubboID[{0}]: {1}, {2}, {3}, {4}, {5}, {6}, {7}".format(resp.Id, self.prof[resp.Id][0], self.prof[resp.Id][1], self.prof[resp.Id][2], t,
# self.prof[resp.Id][1] - self.prof[resp.Id][0], self.prof[resp.Id][2] - self.prof[resp.Id][0],
# t - self.prof[resp.Id][0]))
del self._callbacks[resp.Id]
del self.prof[resp.Id]
# self.io_loop.add_callback(cb, resp) # 调用 callback
cb(resp)
self.stream.read_bytes(16, self._on_header)
def fetch(self, dubbo_request, callback):
if dubbo_request.Id in self._callbacks:
gen_log.error("dubbo Id {0} already in cbs !!".format(
dubbo_request.Id))
self._callbacks[dubbo_request.Id] = callback
self.prof[dubbo_request.Id] = [
time.time(),
]
self.queue.append(dubbo_request)
self._process_queue()
def _process_queue(self):
if not self._connected:
gen_log.info("dubbo connection not ready")
return
with stack_context.NullContext():
while self.queue:
dubbo_request = self.queue.popleft()
self.prof[dubbo_request.Id].append(time.time())
# print 'write data', dubbo_request.encode()
self.stream.write(dubbo_request.encode())
def _handle_exception(self, typ, value, tb):
gen_log.exception("dubbo connection error [%s] [%s] %s", typ, value,
tb)
class AsyncSocket(object):
def __init__(self, sock):
self._iostream = IOStream(sock)
self._resolver = Resolver()
self._readtimeout = 0
self._connecttimeout = 0
def set_readtimeout(self, timeout):
self._readtimeout = timeout
def set_connecttimeout(self, timeout):
self._connecttimeout = timeout
@synclize
def connect(self, address):
host, port = address
timer = None
try:
if self._connecttimeout:
timer = Timeout(self._connecttimeout)
timer.start()
resolved_addrs = yield self._resolver.resolve(
host, port, family=socket.AF_INET)
for addr in resolved_addrs:
family, host_port = addr
yield self._iostream.connect(host_port)
break
except TimeoutException:
self.close()
raise
finally:
if timer:
timer.cancel()
#@synclize
def sendall(self, buff):
self._iostream.write(buff)
@synclize
def read(self, nbytes, partial=False):
timer = None
try:
if self._readtimeout:
timer = Timeout(self._readtimeout)
timer.start()
buff = yield self._iostream.read_bytes(nbytes, partial=partial)
raise Return(buff)
except TimeoutException:
self.close()
raise
finally:
if timer:
timer.cancel()
def recv(self, nbytes):
return self.read(nbytes, partial=True)
@synclize
def readline(self, max_bytes=-1):
timer = None
if self._readtimeout:
timer = Timeout(self._readtimeout)
timer.start()
try:
if max_bytes > 0:
buff = yield self._iostream.read_until('\n',
max_bytes=max_bytes)
else:
buff = yield self._iostream.read_until('\n')
raise Return(buff)
except TimeoutException:
self.close()
raise
finally:
if timer:
timer.cancel()
def close(self):
self._iostream.close()
def set_nodelay(self, flag):
self._iostream.set_nodelay(flag)
def settimeout(self, timeout):
pass
def shutdown(self, direction):
if self._iostream.fileno():
self._iostream.fileno().shutdown(direction)
def recv_into(self, buff):
expected_rbytes = len(buff)
data = self.read(expected_rbytes, True)
srcarray = bytearray(data)
nbytes = len(srcarray)
buff[0:nbytes] = srcarray
return nbytes
def makefile(self, mode, other):
return self