def test_empty_write_buffer(self): x = WriteBuffer() s = bytes(x) self.assertEquals(s, b"") c = x.pop_chunk(4096) self.assertEquals(c, b"") self.assertEquals(len(x), 0)
class Connection(object): """Low level connection object.""" def __init__(self, read_callback, close_callback, host=tornadis.DEFAULT_HOST, port=tornadis.DEFAULT_PORT, unix_domain_socket=None, read_page_size=tornadis.DEFAULT_READ_PAGE_SIZE, write_page_size=tornadis.DEFAULT_WRITE_PAGE_SIZE, connect_timeout=tornadis.DEFAULT_CONNECT_TIMEOUT, tcp_nodelay=False, aggressive_write=False, ioloop=None): """Constructor. Args: read_callback: callback called when there is something to read. close_callback: callback called when the connection is closed. host (string): the host name to connect to. port (int): the port to connect to. unix_domain_socket (string): path to a unix socket to connect to (if set, overrides host/port parameters). read_page_size (int): page size for reading. write_page_size (int): page size for writing. connect_timeout (int): timeout (in seconds) for connecting. tcp_nodelay (boolean): set TCP_NODELAY on socket. aggressive_write (boolean): try to minimize write latency over global throughput (default False). ioloop (IOLoop): the tornado ioloop to use. """ self.host = host self.port = port self.unix_domain_socket = unix_domain_socket self._state = ConnectionState() self._ioloop = ioloop or tornado.ioloop.IOLoop.instance() cb = tornado.ioloop.PeriodicCallback(self._on_every_second, 1000, self._ioloop) self.__periodic_callback = cb self._read_callback = read_callback self._close_callback = close_callback self.read_page_size = read_page_size self.write_page_size = write_page_size self.connect_timeout = connect_timeout self.tcp_nodelay = tcp_nodelay self.aggressive_write = aggressive_write self._write_buffer = WriteBuffer() self._listened_events = 0 def _redis_server(self): if self.unix_domain_socket: return self.unix_domain_socket return "%s:%i" % (self.host, self.port) def is_connecting(self): """Returns True if the object is connecting.""" return self._state.is_connecting() def is_connected(self): """Returns True if the object is connected.""" return self._state.is_connected() @tornado.gen.coroutine def connect(self): """Connects the object to the host:port. Returns: Future: a Future object with True as result if the connection process was ok. """ if self.is_connected() or self.is_connecting(): raise tornado.gen.Return(True) if self.unix_domain_socket is None: self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self.tcp_nodelay: self.__socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) else: if not os.path.exists(self.unix_domain_socket): LOG.warning("can't connect to %s, file does not exist", self.unix_domain_socket) raise tornado.gen.Return(False) self.__socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.__socket.setblocking(0) self.__periodic_callback.start() try: LOG.debug("connecting to %s...", self._redis_server()) self._state.set_connecting() if self.unix_domain_socket is None: self.__socket.connect((self.host, self.port)) else: self.__socket.connect(self.unix_domain_socket) except socket.error as e: if (errno_from_exception(e) not in _ERRNO_INPROGRESS and errno_from_exception(e) not in _ERRNO_WOULDBLOCK): self.disconnect() LOG.warning("can't connect to %s", self._redis_server()) raise tornado.gen.Return(False) self.__socket_fileno = self.__socket.fileno() self._register_or_update_event_handler() yield self._state.get_changed_state_future() if not self.is_connected(): LOG.warning("can't connect to %s", self._redis_server()) raise tornado.gen.Return(False) else: LOG.debug("connected to %s", self._redis_server()) self.__socket_fileno = self.__socket.fileno() self._state.set_connected() self._register_or_update_event_handler() raise tornado.gen.Return(True) def _on_every_second(self): if self.is_connecting(): dt = self._state.get_last_state_change_timedelta() if dt.total_seconds() > self.connect_timeout: self.disconnect() def _register_or_update_event_handler(self, write=True): if write: listened_events = READ_EVENT | WRITE_EVENT | ERROR_EVENT else: listened_events = READ_EVENT | ERROR_EVENT if self._listened_events == 0: try: self._ioloop.add_handler(self.__socket_fileno, self._handle_events, listened_events) except (OSError, IOError, ValueError): self.disconnect() return else: if self._listened_events != listened_events: try: self._ioloop.update_handler(self.__socket_fileno, listened_events) except (OSError, IOError, ValueError): self.disconnect() return self._listened_events = listened_events def disconnect(self): """Disconnects the object. Safe method (no exception, even if it's already disconnected or if there are some connection errors). """ if not self.is_connected() and not self.is_connecting(): return LOG.debug("disconnecting from %s...", self._redis_server()) self.__periodic_callback.stop() try: self._ioloop.remove_handler(self.__socket_fileno) self._listened_events = 0 except: pass self.__socket_fileno = -1 try: self.__socket.close() except: pass self._state.set_disconnected() self._close_callback() LOG.debug("disconnected from %s", self._redis_server()) def _handle_events(self, fd, event): if self.is_connecting(): err = self.__socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if err != 0: LOG.debug("connecting error in _handle_events") self.disconnect() return self._state.set_connected() LOG.debug("connected to %s", self._redis_server()) if not self.is_connected(): return if event & self._ioloop.READ: self._handle_read() if not self.is_connected(): return if event & self._ioloop.WRITE: self._handle_write() if not self.is_connected(): return if event & self._ioloop.ERROR: LOG.debug("unknown socket error") self.disconnect() def _handle_read(self): chunk = self._read(self.read_page_size) if chunk is not None: self._read_callback(chunk) def _handle_write(self): while not self._write_buffer.is_empty(): ps = self.write_page_size data = self._write_buffer.pop_chunk(ps) if len(data) > 0: try: size = self.__socket.send(data) except (socket.error, IOError, OSError) as e: if e.args[0] in _ERRNO_WOULDBLOCK: LOG.debug("write would block") self._write_buffer.appendleft(data) break else: self.disconnect() return else: LOG.debug("%i bytes written to the socket", size) if size < len(data): self._write_buffer.appendleft(data[size:]) break if self._write_buffer.is_empty(): self._register_or_update_event_handler(write=False) def _read(self, size): try: chunk = self.__socket.recv(size) chunk_length = len(chunk) if chunk_length > 0: LOG.debug("%i bytes read from socket", chunk_length) return chunk else: LOG.debug("closed socket => disconnecting") self.disconnect() except socket.error as e: if e.args[0] in _ERRNO_WOULDBLOCK: LOG.debug("read would block") return None else: self.disconnect() def write(self, data): """Buffers some data to be sent to the host:port in a non blocking way. So the data is always buffered and not sent on the socket in a synchronous way. You can give a WriteBuffer as parameter. The internal Connection WriteBuffer will be extended with this one (without copying). Args: data (str or WriteBuffer): string (or WriteBuffer) to write to the host:port. """ if isinstance(data, WriteBuffer): self._write_buffer.append(data) else: if len(data) > 0: self._write_buffer.append(data) if self.aggressive_write: self._handle_write() if self._write_buffer._total_length > 0: self._register_or_update_event_handler(write=True)
class Connection(object): """Low level connection object. Attributes: host (string): the host name to connect to. port (int): the port to connect to. unix_domain_socket (string): path to a unix socket to connect to (if set, overrides host/port parameters). read_page_size (int): page size for reading. write_page_size (int): page size for writing. connect_timeout (int): timeout (in seconds) for connecting. tcp_nodelay (boolean): set TCP_NODELAY on socket. aggressive_write (boolean): try to minimize write latency over global throughput (default False). read_timeout (int): timeout (in seconds) to read something on the socket (if nothing is read during this time, the connection is closed) (default: 0 means no timeout) """ def __init__(self, read_callback, close_callback, host=tornadis.DEFAULT_HOST, port=tornadis.DEFAULT_PORT, unix_domain_socket=None, read_page_size=tornadis.DEFAULT_READ_PAGE_SIZE, write_page_size=tornadis.DEFAULT_WRITE_PAGE_SIZE, connect_timeout=tornadis.DEFAULT_CONNECT_TIMEOUT, tcp_nodelay=False, aggressive_write=False, read_timeout=tornadis.DEFAULT_READ_TIMEOUT, ioloop=None): """Constructor. Args: read_callback: callback called when there is something to read (private, do not use from Client constructor). close_callback: callback called when the connection is closed (private, do not use from Client constructor). host (string): the host name to connect to. port (int): the port to connect to. unix_domain_socket (string): path to a unix socket to connect to (if set, overrides host/port parameters). read_page_size (int): page size for reading. write_page_size (int): page size for writing. connect_timeout (int): timeout (in seconds) for connecting. tcp_nodelay (boolean): set TCP_NODELAY on socket. aggressive_write (boolean): try to minimize write latency over global throughput (default False). read_timeout (int): timeout (in seconds) to read something on the socket (if nothing is read during this time, the connection is closed) (default: 0 means no timeout) ioloop (IOLoop): the tornado ioloop to use. """ self.host = host self.port = port self.unix_domain_socket = unix_domain_socket self._state = ConnectionState() self._ioloop = ioloop or tornado.ioloop.IOLoop.instance() cb = tornado.ioloop.PeriodicCallback(self._on_every_second, 1000, self._ioloop) self.__periodic_callback = cb self._read_callback = read_callback self._close_callback = close_callback self.read_page_size = read_page_size self.write_page_size = write_page_size self.connect_timeout = connect_timeout self.read_timeout = read_timeout self.tcp_nodelay = tcp_nodelay self.aggressive_write = aggressive_write self._write_buffer = WriteBuffer() self._listened_events = 0 self._last_read = datetime.now() def _redis_server(self): if self.unix_domain_socket: return self.unix_domain_socket return "%s:%i" % (self.host, self.port) def is_connecting(self): """Returns True if the object is connecting.""" return self._state.is_connecting() def is_connected(self): """Returns True if the object is connected.""" return self._state.is_connected() @tornado.gen.coroutine def connect(self): """Connects the object to the host:port. Returns: Future: a Future object with True as result if the connection process was ok. """ if self.is_connected() or self.is_connecting(): raise tornado.gen.Return(True) if self.unix_domain_socket is None: self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self.tcp_nodelay: self.__socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) else: if not os.path.exists(self.unix_domain_socket): LOG.warning("can't connect to %s, file does not exist", self.unix_domain_socket) raise tornado.gen.Return(False) self.__socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.__socket.setblocking(0) self.__periodic_callback.start() try: LOG.debug("connecting to %s...", self._redis_server()) self._state.set_connecting() if self.unix_domain_socket is None: self.__socket.connect((self.host, self.port)) else: self.__socket.connect(self.unix_domain_socket) except socket.error as e: if (errno_from_exception(e) not in _ERRNO_INPROGRESS and errno_from_exception(e) not in _ERRNO_WOULDBLOCK): self.disconnect() LOG.warning("can't connect to %s", self._redis_server()) raise tornado.gen.Return(False) self.__socket_fileno = self.__socket.fileno() self._register_or_update_event_handler() yield self._state.get_changed_state_future() if not self.is_connected(): LOG.warning("can't connect to %s", self._redis_server()) raise tornado.gen.Return(False) else: LOG.debug("connected to %s", self._redis_server()) self.__socket_fileno = self.__socket.fileno() self._state.set_connected() self._register_or_update_event_handler() raise tornado.gen.Return(True) def _on_every_second(self): if self.is_connecting(): dt = self._state.get_last_state_change_timedelta() if dt.total_seconds() > self.connect_timeout: self.disconnect() if self.read_timeout > 0: dt = datetime.now() - self._last_read if dt.total_seconds() > self.read_timeout: LOG.warning("read timeout => disconnecting") self.disconnect() def _register_or_update_event_handler(self, write=True): if write: listened_events = READ_EVENT | WRITE_EVENT | ERROR_EVENT else: listened_events = READ_EVENT | ERROR_EVENT if self._listened_events == 0: try: self._ioloop.add_handler(self.__socket_fileno, self._handle_events, listened_events) except (OSError, IOError, ValueError): self.disconnect() return else: if self._listened_events != listened_events: try: self._ioloop.update_handler(self.__socket_fileno, listened_events) except (OSError, IOError, ValueError): self.disconnect() return self._listened_events = listened_events def disconnect(self): """Disconnects the object. Safe method (no exception, even if it's already disconnected or if there are some connection errors). """ if not self.is_connected() and not self.is_connecting(): return LOG.debug("disconnecting from %s...", self._redis_server()) self.__periodic_callback.stop() try: self._ioloop.remove_handler(self.__socket_fileno) self._listened_events = 0 except: pass self.__socket_fileno = -1 try: self.__socket.close() except: pass self._state.set_disconnected() self._close_callback() LOG.debug("disconnected from %s", self._redis_server()) def _handle_events(self, fd, event): if self.is_connecting(): err = self.__socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if err != 0: LOG.debug("connecting error in _handle_events") self.disconnect() return self._state.set_connected() LOG.debug("connected to %s", self._redis_server()) if not self.is_connected(): return if event & self._ioloop.READ: self._handle_read() if not self.is_connected(): return if event & self._ioloop.WRITE: self._handle_write() if not self.is_connected(): return if event & self._ioloop.ERROR: LOG.debug("unknown socket error") self.disconnect() def _handle_read(self): chunk = self._read(self.read_page_size) if chunk is not None: if self.read_timeout > 0: self._last_read = datetime.now() self._read_callback(chunk) def _handle_write(self): while not self._write_buffer.is_empty(): ps = self.write_page_size data = self._write_buffer.pop_chunk(ps) if len(data) > 0: try: size = self.__socket.send(data) except (socket.error, IOError, OSError) as e: if e.args[0] in _ERRNO_WOULDBLOCK: LOG.debug("write would block") self._write_buffer.appendleft(data) break else: self.disconnect() return else: LOG.debug("%i bytes written to the socket", size) if size < len(data): self._write_buffer.appendleft(data[size:]) break if self._write_buffer.is_empty(): self._register_or_update_event_handler(write=False) def _read(self, size): try: chunk = self.__socket.recv(size) chunk_length = len(chunk) if chunk_length > 0: LOG.debug("%i bytes read from socket", chunk_length) return chunk else: LOG.debug("closed socket => disconnecting") self.disconnect() except socket.error as e: if e.args[0] in _ERRNO_WOULDBLOCK: LOG.debug("read would block") return None else: self.disconnect() def write(self, data): """Buffers some data to be sent to the host:port in a non blocking way. So the data is always buffered and not sent on the socket in a synchronous way. You can give a WriteBuffer as parameter. The internal Connection WriteBuffer will be extended with this one (without copying). Args: data (str or WriteBuffer): string (or WriteBuffer) to write to the host:port. """ if isinstance(data, WriteBuffer): self._write_buffer.append(data) else: if len(data) > 0: self._write_buffer.append(data) if self.aggressive_write: self._handle_write() if self._write_buffer._total_length > 0: self._register_or_update_event_handler(write=True)