def test_protocol6(self):
wb = WriteBuffer()
wb.append(b"foobar")
res = bytes(format_args_in_redis_protocol("SET", "key", wb))
self.assertEqual(
res, b"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n"
b"$6\r\nfoobar\r\n")
def _pipelined_call(self, pipeline, callback):
buf = WriteBuffer()
replies = len(pipeline.pipelined_args)
cb = functools.partial(self._reply_aggregator, callback, replies)
for args in pipeline.pipelined_args:
self.__callback_queue.append(cb)
tmp_buf = format_args_in_redis_protocol(*args)
buf.append(tmp_buf)
self.__connection.write(buf)
def _pipelined_call(self, pipeline, callback):
buf = WriteBuffer()
replies = len(pipeline.pipelined_args)
cb = functools.partial(self._reply_aggregator, callback, replies)
for args in pipeline.pipelined_args:
self.__callback_queue.append(cb)
tmp_buf = format_args_in_redis_protocol(*args)
buf.append(tmp_buf)
self.__connection.write(buf)
def test_write_buffer3(self):
b = WriteBuffer()
b.append(b"x" * 10000)
chunk = self._pop_chunk_as_str(b, 4000)
self.assertEquals(len(chunk), 4000)
chunk = self._pop_chunk_as_str(b, 4000)
self.assertEquals(len(chunk), 4000)
chunk = self._pop_chunk_as_str(b, 4000)
self.assertEquals(len(chunk), 2000)
def test_write_buffer3(self):
b = WriteBuffer()
b.append(b"x" * 10000)
chunk = self._pop_chunk_as_str(b, 4000)
self.assertEquals(len(chunk), 4000)
chunk = self._pop_chunk_as_str(b, 4000)
self.assertEquals(len(chunk), 4000)
chunk = self._pop_chunk_as_str(b, 4000)
self.assertEquals(len(chunk), 2000)
def _make_test_buffer(self):
x = WriteBuffer()
x.append(b"23")
x.append(b"4")
x.append(b"")
x.append(b"56789")
x.appendleft(b"1")
return x
def _make_test_buffer(self):
x = WriteBuffer()
x.append(b"23")
x.append(b"4")
x.append(b"")
x.append(b"56789")
x.appendleft(b"1")
return x
class Connection(object):
"""Low level connection object."""
def __init__(self, read_callback, close_callback,
host=tornadis.DEFAULT_HOST,
port=tornadis.DEFAULT_PORT, unix_domain_socket=None,
read_page_size=tornadis.DEFAULT_READ_PAGE_SIZE,
write_page_size=tornadis.DEFAULT_WRITE_PAGE_SIZE,
connect_timeout=tornadis.DEFAULT_CONNECT_TIMEOUT,
tcp_nodelay=False, aggressive_write=False, ioloop=None):
"""Constructor.
Args:
read_callback: callback called when there is something to read.
close_callback: callback called when the connection is closed.
host (string): the host name to connect to.
port (int): the port to connect to.
unix_domain_socket (string): path to a unix socket to connect to
(if set, overrides host/port parameters).
read_page_size (int): page size for reading.
write_page_size (int): page size for writing.
connect_timeout (int): timeout (in seconds) for connecting.
tcp_nodelay (boolean): set TCP_NODELAY on socket.
aggressive_write (boolean): try to minimize write latency over
global throughput (default False).
ioloop (IOLoop): the tornado ioloop to use.
"""
self.host = host
self.port = port
self.unix_domain_socket = unix_domain_socket
self._state = ConnectionState()
self._ioloop = ioloop or tornado.ioloop.IOLoop.instance()
cb = tornado.ioloop.PeriodicCallback(self._on_every_second, 1000,
self._ioloop)
self.__periodic_callback = cb
self._read_callback = read_callback
self._close_callback = close_callback
self.read_page_size = read_page_size
self.write_page_size = write_page_size
self.connect_timeout = connect_timeout
self.tcp_nodelay = tcp_nodelay
self.aggressive_write = aggressive_write
self._write_buffer = WriteBuffer()
self._listened_events = 0
def _redis_server(self):
if self.unix_domain_socket:
return self.unix_domain_socket
return "%s:%i" % (self.host, self.port)
def is_connecting(self):
"""Returns True if the object is connecting."""
return self._state.is_connecting()
def is_connected(self):
"""Returns True if the object is connected."""
return self._state.is_connected()
@tornado.gen.coroutine
def connect(self):
"""Connects the object to the host:port.
Returns:
Future: a Future object with True as result if the connection
process was ok.
"""
if self.is_connected() or self.is_connecting():
raise tornado.gen.Return(True)
if self.unix_domain_socket is None:
self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if self.tcp_nodelay:
self.__socket.setsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY, 1)
else:
if not os.path.exists(self.unix_domain_socket):
LOG.warning("can't connect to %s, file does not exist",
self.unix_domain_socket)
raise tornado.gen.Return(False)
self.__socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.__socket.setblocking(0)
self.__periodic_callback.start()
try:
LOG.debug("connecting to %s...", self._redis_server())
self._state.set_connecting()
if self.unix_domain_socket is None:
self.__socket.connect((self.host, self.port))
else:
self.__socket.connect(self.unix_domain_socket)
except socket.error as e:
if (errno_from_exception(e) not in _ERRNO_INPROGRESS and
errno_from_exception(e) not in _ERRNO_WOULDBLOCK):
self.disconnect()
LOG.warning("can't connect to %s", self._redis_server())
raise tornado.gen.Return(False)
self.__socket_fileno = self.__socket.fileno()
self._register_or_update_event_handler()
yield self._state.get_changed_state_future()
if not self.is_connected():
LOG.warning("can't connect to %s", self._redis_server())
raise tornado.gen.Return(False)
else:
LOG.debug("connected to %s", self._redis_server())
self.__socket_fileno = self.__socket.fileno()
self._state.set_connected()
self._register_or_update_event_handler()
raise tornado.gen.Return(True)
def _on_every_second(self):
if self.is_connecting():
dt = self._state.get_last_state_change_timedelta()
if dt.total_seconds() > self.connect_timeout:
self.disconnect()
def _register_or_update_event_handler(self, write=True):
if write:
listened_events = READ_EVENT | WRITE_EVENT | ERROR_EVENT
else:
listened_events = READ_EVENT | ERROR_EVENT
if self._listened_events == 0:
try:
self._ioloop.add_handler(self.__socket_fileno,
self._handle_events, listened_events)
except (OSError, IOError, ValueError):
self.disconnect()
return
else:
if self._listened_events != listened_events:
try:
self._ioloop.update_handler(self.__socket_fileno,
listened_events)
except (OSError, IOError, ValueError):
self.disconnect()
return
self._listened_events = listened_events
def disconnect(self):
"""Disconnects the object.
Safe method (no exception, even if it's already disconnected or if
there are some connection errors).
"""
if not self.is_connected() and not self.is_connecting():
return
LOG.debug("disconnecting from %s...", self._redis_server())
self.__periodic_callback.stop()
try:
self._ioloop.remove_handler(self.__socket_fileno)
self._listened_events = 0
except:
pass
self.__socket_fileno = -1
try:
self.__socket.close()
except:
pass
self._state.set_disconnected()
self._close_callback()
LOG.debug("disconnected from %s", self._redis_server())
def _handle_events(self, fd, event):
if self.is_connecting():
err = self.__socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0:
LOG.debug("connecting error in _handle_events")
self.disconnect()
return
self._state.set_connected()
LOG.debug("connected to %s", self._redis_server())
if not self.is_connected():
return
if event & self._ioloop.READ:
self._handle_read()
if not self.is_connected():
return
if event & self._ioloop.WRITE:
self._handle_write()
if not self.is_connected():
return
if event & self._ioloop.ERROR:
LOG.debug("unknown socket error")
self.disconnect()
def _handle_read(self):
chunk = self._read(self.read_page_size)
if chunk is not None:
self._read_callback(chunk)
def _handle_write(self):
while not self._write_buffer.is_empty():
ps = self.write_page_size
data = self._write_buffer.pop_chunk(ps)
if len(data) > 0:
try:
size = self.__socket.send(data)
except (socket.error, IOError, OSError) as e:
if e.args[0] in _ERRNO_WOULDBLOCK:
LOG.debug("write would block")
self._write_buffer.appendleft(data)
break
else:
self.disconnect()
return
else:
LOG.debug("%i bytes written to the socket", size)
if size < len(data):
self._write_buffer.appendleft(data[size:])
break
if self._write_buffer.is_empty():
self._register_or_update_event_handler(write=False)
def _read(self, size):
try:
chunk = self.__socket.recv(size)
chunk_length = len(chunk)
if chunk_length > 0:
LOG.debug("%i bytes read from socket", chunk_length)
return chunk
else:
LOG.debug("closed socket => disconnecting")
self.disconnect()
except socket.error as e:
if e.args[0] in _ERRNO_WOULDBLOCK:
LOG.debug("read would block")
return None
else:
self.disconnect()
def write(self, data):
"""Buffers some data to be sent to the host:port in a non blocking way.
So the data is always buffered and not sent on the socket in a
synchronous way.
You can give a WriteBuffer as parameter. The internal Connection
WriteBuffer will be extended with this one (without copying).
Args:
data (str or WriteBuffer): string (or WriteBuffer) to write to
the host:port.
"""
if isinstance(data, WriteBuffer):
self._write_buffer.append(data)
else:
if len(data) > 0:
self._write_buffer.append(data)
if self.aggressive_write:
self._handle_write()
if self._write_buffer._total_length > 0:
self._register_or_update_event_handler(write=True)
def format_args_in_redis_protocol(*args):
"""Formats arguments into redis protocol...
This function makes and returns a string/buffer corresponding to
given arguments formated with the redis protocol.
integer, text, string or binary types are automatically converted
(using utf8 if necessary).
More informations about the protocol: http://redis.io/topics/protocol
Args:
*args: full redis command as variable length argument list
Returns:
binary string (arguments in redis protocol)
Examples:
>>> format_args_in_redis_protocol("HSET", "key", "field", "value")
'*4\r\n$4\r\nHSET\r\n$3\r\nkey\r\n$5\r\nfield\r\n$5\r\nvalue\r\n'
"""
buf = WriteBuffer()
l = "*%d\r\n" % len(args) # noqa: E741
if six.PY2:
buf.append(l)
else: # pragma: no cover
buf.append(l.encode('utf-8'))
for arg in args:
if isinstance(arg, six.text_type):
# it's a unicode string in Python2 or a standard (unicode)
# string in Python3, let's encode it in utf-8 to get raw bytes
arg = arg.encode('utf-8')
elif isinstance(arg, six.string_types):
# it's a basestring in Python2 => nothing to do
pass
elif isinstance(arg, six.binary_type): # pragma: no cover
# it's a raw bytes string in Python3 => nothing to do
pass
elif isinstance(arg, six.integer_types):
tmp = "%d" % arg
if six.PY2:
arg = tmp
else: # pragma: no cover
arg = tmp.encode('utf-8')
elif isinstance(arg, WriteBuffer):
# it's a WriteBuffer object => nothing to do
pass
else:
raise Exception("don't know what to do with %s" % type(arg))
l = "$%d\r\n" % len(arg) # noqa: E741
if six.PY2:
buf.append(l)
else: # pragma: no cover
buf.append(l.encode('utf-8'))
buf.append(arg)
buf.append(b"\r\n")
return buf
def test_protocol6(self):
wb = WriteBuffer()
wb.append(b"foobar")
res = bytes(format_args_in_redis_protocol("SET", "key", wb))
self.assertEqual(res, b"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n"
b"$6\r\nfoobar\r\n")
class Connection(object):
"""Low level connection object.
Attributes:
host (string): the host name to connect to.
port (int): the port to connect to.
unix_domain_socket (string): path to a unix socket to connect to
(if set, overrides host/port parameters).
read_page_size (int): page size for reading.
write_page_size (int): page size for writing.
connect_timeout (int): timeout (in seconds) for connecting.
tcp_nodelay (boolean): set TCP_NODELAY on socket.
aggressive_write (boolean): try to minimize write latency over
global throughput (default False).
read_timeout (int): timeout (in seconds) to read something on
the socket (if nothing is read during this time, the
connection is closed) (default: 0 means no timeout)
"""
def __init__(self,
read_callback,
close_callback,
host=tornadis.DEFAULT_HOST,
port=tornadis.DEFAULT_PORT,
unix_domain_socket=None,
read_page_size=tornadis.DEFAULT_READ_PAGE_SIZE,
write_page_size=tornadis.DEFAULT_WRITE_PAGE_SIZE,
connect_timeout=tornadis.DEFAULT_CONNECT_TIMEOUT,
tcp_nodelay=False,
aggressive_write=False,
read_timeout=tornadis.DEFAULT_READ_TIMEOUT,
ioloop=None):
"""Constructor.
Args:
read_callback: callback called when there is something to read
(private, do not use from Client constructor).
close_callback: callback called when the connection is closed
(private, do not use from Client constructor).
host (string): the host name to connect to.
port (int): the port to connect to.
unix_domain_socket (string): path to a unix socket to connect to
(if set, overrides host/port parameters).
read_page_size (int): page size for reading.
write_page_size (int): page size for writing.
connect_timeout (int): timeout (in seconds) for connecting.
tcp_nodelay (boolean): set TCP_NODELAY on socket.
aggressive_write (boolean): try to minimize write latency over
global throughput (default False).
read_timeout (int): timeout (in seconds) to read something on
the socket (if nothing is read during this time, the
connection is closed) (default: 0 means no timeout)
ioloop (IOLoop): the tornado ioloop to use.
"""
self.host = host
self.port = port
self.unix_domain_socket = unix_domain_socket
self._state = ConnectionState()
self._ioloop = ioloop or tornado.ioloop.IOLoop.instance()
cb = tornado.ioloop.PeriodicCallback(self._on_every_second, 1000,
self._ioloop)
self.__periodic_callback = cb
self._read_callback = read_callback
self._close_callback = close_callback
self.read_page_size = read_page_size
self.write_page_size = write_page_size
self.connect_timeout = connect_timeout
self.read_timeout = read_timeout
self.tcp_nodelay = tcp_nodelay
self.aggressive_write = aggressive_write
self._write_buffer = WriteBuffer()
self._listened_events = 0
self._last_read = datetime.now()
def _redis_server(self):
if self.unix_domain_socket:
return self.unix_domain_socket
return "%s:%i" % (self.host, self.port)
def is_connecting(self):
"""Returns True if the object is connecting."""
return self._state.is_connecting()
def is_connected(self):
"""Returns True if the object is connected."""
return self._state.is_connected()
@tornado.gen.coroutine
def connect(self):
"""Connects the object to the host:port.
Returns:
Future: a Future object with True as result if the connection
process was ok.
"""
if self.is_connected() or self.is_connecting():
raise tornado.gen.Return(True)
if self.unix_domain_socket is None:
self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if self.tcp_nodelay:
self.__socket.setsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY, 1)
else:
if not os.path.exists(self.unix_domain_socket):
LOG.warning("can't connect to %s, file does not exist",
self.unix_domain_socket)
raise tornado.gen.Return(False)
self.__socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.__socket.setblocking(0)
self.__periodic_callback.start()
try:
LOG.debug("connecting to %s...", self._redis_server())
self._state.set_connecting()
if self.unix_domain_socket is None:
self.__socket.connect((self.host, self.port))
else:
self.__socket.connect(self.unix_domain_socket)
except socket.error as e:
if (errno_from_exception(e) not in _ERRNO_INPROGRESS
and errno_from_exception(e) not in _ERRNO_WOULDBLOCK):
self.disconnect()
LOG.warning("can't connect to %s", self._redis_server())
raise tornado.gen.Return(False)
self.__socket_fileno = self.__socket.fileno()
self._register_or_update_event_handler()
yield self._state.get_changed_state_future()
if not self.is_connected():
LOG.warning("can't connect to %s", self._redis_server())
raise tornado.gen.Return(False)
else:
LOG.debug("connected to %s", self._redis_server())
self.__socket_fileno = self.__socket.fileno()
self._state.set_connected()
self._register_or_update_event_handler()
raise tornado.gen.Return(True)
def _on_every_second(self):
if self.is_connecting():
dt = self._state.get_last_state_change_timedelta()
if dt.total_seconds() > self.connect_timeout:
self.disconnect()
if self.read_timeout > 0:
dt = datetime.now() - self._last_read
if dt.total_seconds() > self.read_timeout:
LOG.warning("read timeout => disconnecting")
self.disconnect()
def _register_or_update_event_handler(self, write=True):
if write:
listened_events = READ_EVENT | WRITE_EVENT | ERROR_EVENT
else:
listened_events = READ_EVENT | ERROR_EVENT
if self._listened_events == 0:
try:
self._ioloop.add_handler(self.__socket_fileno,
self._handle_events, listened_events)
except (OSError, IOError, ValueError):
self.disconnect()
return
else:
if self._listened_events != listened_events:
try:
self._ioloop.update_handler(self.__socket_fileno,
listened_events)
except (OSError, IOError, ValueError):
self.disconnect()
return
self._listened_events = listened_events
def disconnect(self):
"""Disconnects the object.
Safe method (no exception, even if it's already disconnected or if
there are some connection errors).
"""
if not self.is_connected() and not self.is_connecting():
return
LOG.debug("disconnecting from %s...", self._redis_server())
self.__periodic_callback.stop()
try:
self._ioloop.remove_handler(self.__socket_fileno)
self._listened_events = 0
except:
pass
self.__socket_fileno = -1
try:
self.__socket.close()
except:
pass
self._state.set_disconnected()
self._close_callback()
LOG.debug("disconnected from %s", self._redis_server())
def _handle_events(self, fd, event):
if self.is_connecting():
err = self.__socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0:
LOG.debug("connecting error in _handle_events")
self.disconnect()
return
self._state.set_connected()
LOG.debug("connected to %s", self._redis_server())
if not self.is_connected():
return
if event & self._ioloop.READ:
self._handle_read()
if not self.is_connected():
return
if event & self._ioloop.WRITE:
self._handle_write()
if not self.is_connected():
return
if event & self._ioloop.ERROR:
LOG.debug("unknown socket error")
self.disconnect()
def _handle_read(self):
chunk = self._read(self.read_page_size)
if chunk is not None:
if self.read_timeout > 0:
self._last_read = datetime.now()
self._read_callback(chunk)
def _handle_write(self):
while not self._write_buffer.is_empty():
ps = self.write_page_size
data = self._write_buffer.pop_chunk(ps)
if len(data) > 0:
try:
size = self.__socket.send(data)
except (socket.error, IOError, OSError) as e:
if e.args[0] in _ERRNO_WOULDBLOCK:
LOG.debug("write would block")
self._write_buffer.appendleft(data)
break
else:
self.disconnect()
return
else:
LOG.debug("%i bytes written to the socket", size)
if size < len(data):
self._write_buffer.appendleft(data[size:])
break
if self._write_buffer.is_empty():
self._register_or_update_event_handler(write=False)
def _read(self, size):
try:
chunk = self.__socket.recv(size)
chunk_length = len(chunk)
if chunk_length > 0:
LOG.debug("%i bytes read from socket", chunk_length)
return chunk
else:
LOG.debug("closed socket => disconnecting")
self.disconnect()
except socket.error as e:
if e.args[0] in _ERRNO_WOULDBLOCK:
LOG.debug("read would block")
return None
else:
self.disconnect()
def write(self, data):
"""Buffers some data to be sent to the host:port in a non blocking way.
So the data is always buffered and not sent on the socket in a
synchronous way.
You can give a WriteBuffer as parameter. The internal Connection
WriteBuffer will be extended with this one (without copying).
Args:
data (str or WriteBuffer): string (or WriteBuffer) to write to
the host:port.
"""
if isinstance(data, WriteBuffer):
self._write_buffer.append(data)
else:
if len(data) > 0:
self._write_buffer.append(data)
if self.aggressive_write:
self._handle_write()
if self._write_buffer._total_length > 0:
self._register_or_update_event_handler(write=True)
def format_args_in_redis_protocol(*args):
"""Formats arguments into redis protocol...
This function makes and returns a string/buffer corresponding to
given arguments formated with the redis protocol.
integer, text, string or binary types are automatically converted
(using utf8 if necessary).
More informations about the protocol: http://redis.io/topics/protocol
Args:
*args: full redis command as variable length argument list
Returns:
binary string (arguments in redis protocol)
Examples:
>>> format_args_in_redis_protocol("HSET", "key", "field", "value")
'*4\r\n$4\r\nHSET\r\n$3\r\nkey\r\n$5\r\nfield\r\n$5\r\nvalue\r\n'
"""
buf = WriteBuffer()
l = "*%d\r\n" % len(args) # noqa: E741
if six.PY2:
buf.append(l)
else: # pragma: no cover
buf.append(l.encode('utf-8'))
for arg in args:
if isinstance(arg, six.text_type):
# it's a unicode string in Python2 or a standard (unicode)
# string in Python3, let's encode it in utf-8 to get raw bytes
arg = arg.encode('utf-8')
elif isinstance(arg, six.string_types):
# it's a basestring in Python2 => nothing to do
pass
elif isinstance(arg, six.binary_type): # pragma: no cover
# it's a raw bytes string in Python3 => nothing to do
pass
elif isinstance(arg, six.integer_types):
tmp = "%d" % arg
if six.PY2:
arg = tmp
else: # pragma: no cover
arg = tmp.encode('utf-8')
elif isinstance(arg, WriteBuffer):
# it's a WriteBuffer object => nothing to do
pass
else:
raise Exception("don't know what to do with %s" % type(arg))
l = "$%d\r\n" % len(arg) # noqa: E741
if six.PY2:
buf.append(l)
else: # pragma: no cover
buf.append(l.encode('utf-8'))
buf.append(arg)
buf.append(b"\r\n")
return buf