def test_already_connected(self): c = Connection(self._read_cb, self._close_cb, connect_timeout=2) res = yield c.connect() self.assertTrue(res) res = yield c.connect() self.assertTrue(res) c.disconnect()
def connect(self): """Connects the client object to redis. It's safe to use this method even if you are already connected. Note: this method is useless with autoconnect mode (default). Returns: a Future object with True as result if the connection was ok. """ if self.is_connected(): raise tornado.gen.Return(True) cb1 = self._read_callback cb2 = self._close_callback self.__callback_queue = collections.deque() self._reply_list = [] self.__reader = hiredis.Reader(replyError=ClientError) kwargs = self.connection_kwargs self.__connection = Connection(cb1, cb2, **kwargs) connection_status = yield self.__connection.connect() if connection_status is not True or not self.password: # nothing left to do here, return raise tornado.gen.Return(connection_status) authentication_status = yield self._call('AUTH', self.password) if authentication_status == b'OK': # correct password, it worked raise tornado.gen.Return(True) else: # incorrect password, return back the result LOG.warning("impossible to connect: bad password") self.__connection.disconnect() raise tornado.gen.Return(False)
def test_timeout_connect(self): orig_constructor = socket.socket socket.socket = functools.partial(fake_socket_constructor, FakeSocketObject1) c = Connection(self._read_cb, self._close_cb, connect_timeout=2) res = yield c.connect() self.assertFalse(res) socket.socket = orig_constructor
def test_bigwrite(self): c = Connection(self._read_cb, self._close_cb) yield c.connect() data1 = format_args_in_redis_protocol("SET", "___foobar", BIG_VALUE) c.write(data1) data2 = format_args_in_redis_protocol("GET", "___foobar") c.write(data2) reply1 = yield self.reply_queue.get() reply2 = yield self.reply_queue.get() self.assertEquals(reply1, b"OK") self.assertEquals(reply2, BIG_VALUE) c.disconnect()
def connect(self): """Connects the client object to redis. It's safe to use this method even if you are already connected. Returns: a Future object with True as result if the connection was ok. """ if self.is_connected(): raise tornado.gen.Return(True) cb1 = self._read_callback cb2 = self._close_callback self.__callback_queue = collections.deque() self._reply_list = [] self.__reader = hiredis.Reader(replyError=ClientError) kwargs = self.connection_kwargs self.__connection = Connection(cb1, cb2, **kwargs) return self.__connection.connect()
def test_bigwrite(self): c = Connection(self._read_cb, self._close_cb) yield c.connect() data1 = format_args_in_redis_protocol("SET", "___foobar", BIG_VALUE) c.write(data1) data2 = format_args_in_redis_protocol("GET", "___foobar") c.write(data2) reply1 = yield self.reply_queue.get() reply2 = yield self.reply_queue.get() self.assertEqual(reply1, b"OK") self.assertEqual(reply2, BIG_VALUE) c.disconnect()
def _test_basic_write(self): c = Connection(self._read_cb, self._close_cb, connect_timeout=2) yield c.connect() data1 = format_args_in_redis_protocol("PING") data2 = b"*1\r\b$4\r\nQUIT\r\n" c.write(data1) c.write(data2) reply1 = yield self.reply_queue.get() self.assertEquals(reply1, b"PONG") reply2 = yield self.reply_queue.get() self.assertEquals(reply2, b"OK") c.disconnect()
def _test_basic_write(self): c = Connection(self._read_cb, self._close_cb, connect_timeout=2) yield c.connect() data1 = format_args_in_redis_protocol("PING") data2 = b"*1\r\b$4\r\nQUIT\r\n" c.write(data1) c.write(data2) reply1 = yield self.reply_queue.get() self.assertEqual(reply1, b"PONG") reply2 = yield self.reply_queue.get() self.assertEqual(reply2, b"OK") c.disconnect()
def test_bad_connect(self): c = Connection(self._read_cb, self._close_cb, host="bad_host__") res = yield c.connect() self.assertFalse(res) c.disconnect()
def test_init_with_tcp_nodelay(self): c = Connection(self._read_cb, self._close_cb, tcp_nodelay=True) yield c.connect() c.disconnect()
def test_init(self): c = Connection(self._read_cb, self._close_cb) yield c.connect() c.disconnect()
class Client(object): """High level object to interact with redis. Attributes: autoconnect (boolean): True if the client is in autoconnect mode (and in autoreconnection mode) (default True). password (string): the password to authenticate with. connection_kwargs (dict): :class:`Connection` object kwargs (note that read_callback and close_callback args are set automatically). """ def __init__(self, autoconnect=True, password=None, **connection_kwargs): """Constructor. Args: autoconnect (boolean): True if the client is in autoconnect mode (and in autoreconnection mode) (default True). password (string): the password to authenticate with. **connection_kwargs: :class:`Connection` object kwargs. """ if 'read_callback' in connection_kwargs or \ 'close_callback' in connection_kwargs: raise Exception("read_callback and close_callback are not allowed " "to be used here.") self.connection_kwargs = connection_kwargs self.autoconnect = autoconnect self.password = password self.__connection = None self.subscribed = False self.__connection = None self.__reader = None # Used for normal clients self.__callback_queue = None # Used for subscribed clients self._condition = tornado.locks.Condition() self._reply_list = None @property def title(self): return self.__connection._redis_server() def is_connected(self): """Returns True is the client is connected to redis. Returns: True if the client if connected to redis. """ return (self.__connection is not None) and \ (self.__connection.is_connected()) @tornado.gen.coroutine def connect(self): """Connects the client object to redis. It's safe to use this method even if you are already connected. Note: this method is useless with autoconnect mode (default). Returns: a Future object with True as result if the connection was ok. """ if self.is_connected(): raise tornado.gen.Return(True) cb1 = self._read_callback cb2 = self._close_callback self.__callback_queue = collections.deque() self._reply_list = [] self.__reader = hiredis.Reader(replyError=ClientError) kwargs = self.connection_kwargs self.__connection = Connection(cb1, cb2, **kwargs) connection_status = yield self.__connection.connect() if connection_status is not True or not self.password: # nothing left to do here, return raise tornado.gen.Return(connection_status) authentication_status = yield self._call('AUTH', self.password) if authentication_status == b'OK': # correct password, it worked raise tornado.gen.Return(True) else: # incorrect password, return back the result LOG.warning("impossible to connect: bad password") self.__connection.disconnect() raise tornado.gen.Return(False) def disconnect(self): """Disconnects the client object from redis. It's safe to use this method even if you are already disconnected. """ if not self.is_connected(): return if self.__connection is not None: self.__connection.disconnect() def _close_callback(self): """Callback called when redis closed the connection. The callback queue is emptied and we call each callback found with None or with an exception object to wake up blocked client. """ while True: try: callback = self.__callback_queue.popleft() callback(ConnectionError("closed connection")) except IndexError: break if self.subscribed: # pubsub clients self._reply_list.append(ConnectionError("closed connection")) self._condition.notify_all() def _read_callback(self, data=None): """Callback called when some data are read on the socket. The buffer is given to the hiredis parser. If a reply is complete, we put the decoded reply to on the reply queue. Args: data (str): string (buffer) read on the socket. """ try: if data is not None: self.__reader.feed(data) while True: reply = self.__reader.gets() if reply is not False: try: callback = self.__callback_queue.popleft() # normal client (1 reply = 1 callback) callback(reply) except IndexError: # pubsub clients self._reply_list.append(reply) self._condition.notify_all() else: break except hiredis.ProtocolError: # something nasty occured (corrupt stream => no way to recover) LOG.warning("corrupted stream => disconnect") self.disconnect() def call(self, *args, **kwargs): """Calls a redis command and returns a Future of the reply. Args: *args: full redis command as variable length argument list or a Pipeline object (as a single argument). **kwargs: internal private options (do not use). Returns: a Future with the decoded redis reply as result (when available) or a ConnectionError object in case of connection error. Raises: ClientError: your Pipeline object is empty. Examples: >>> @tornado.gen.coroutine def foobar(): client = Client() result = yield client.call("HSET", "key", "field", "val") """ if not self.is_connected(): if self.autoconnect: # We use this method only when we are not contected # to void performance penaly due to gen.coroutine decorator return self._call_with_autoconnect(*args, **kwargs) else: error = ConnectionError("you are not connected and " "autoconnect=False") return tornado.gen.maybe_future(error) return self._call(*args, **kwargs) @tornado.gen.coroutine def _call_with_autoconnect(self, *args, **kwargs): yield self.connect() if not self.is_connected(): raise tornado.gen.Return(ConnectionError("impossible to connect")) res = yield self._call(*args, **kwargs) raise tornado.gen.Return(res) def async_call(self, *args, **kwargs): """Calls a redis command, waits for the reply and call a callback. Following options are available (not part of the redis command itself): - callback Function called (with the result as argument) when the result is available. If not set, the reply is silently discarded. In case of errors, the callback is called with a TornadisException object as argument. Args: *args: full redis command as variable length argument list or a Pipeline object (as a single argument). **kwargs: options as keyword parameters. Examples: >>> def cb(result): pass >>> client.async_call("HSET", "key", "field", "val", callback=cb) """ def after_autoconnect_callback(future): if self.is_connected(): self._call(*args, **kwargs) else: # FIXME pass if 'callback' not in kwargs: kwargs['callback'] = discard_reply_cb if not self.is_connected(): if self.autoconnect: connect_future = self.connect() cb = after_autoconnect_callback self.__connection._ioloop.add_future(connect_future, cb) else: error = ConnectionError("you are not connected and " "autoconnect=False") kwargs['callback'](error) else: self._call(*args, **kwargs) def _call(self, *args, **kwargs): callback = False if 'callback' in kwargs: callback = True if len(args) == 1 and isinstance(args[0], Pipeline): fn = self._pipelined_call pipeline = args[0] if pipeline.number_of_stacked_calls == 0: excep = ClientError("empty pipeline") if callback: kwargs['callback'](excep) else: return tornado.gen.maybe_future(excep) arguments = (pipeline,) else: if "__multiple_replies" in kwargs: fn = self._simple_call_with_multiple_replies arguments = tuple([kwargs["__multiple_replies"]] + list(args)) else: fn = self._simple_call arguments = args if callback: fn(*arguments, **kwargs) else: return tornado.gen.Task(fn, *arguments, **kwargs) def _reply_aggregator(self, callback, replies, reply): self._reply_list.append(reply) if len(self._reply_list) == replies: callback(self._reply_list) self._reply_list = [] def _simple_call(self, *args, **kwargs): callback = kwargs['callback'] msg = format_args_in_redis_protocol(*args) self.__callback_queue.append(callback) self.__connection.write(msg) def _simple_call_with_multiple_replies(self, replies, *args, **kwargs): original_callback = kwargs['callback'] msg = format_args_in_redis_protocol(*args) callback = functools.partial(self._reply_aggregator, original_callback, replies) for _ in range(0, replies): self.__callback_queue.append(callback) self.__connection.write(msg) def _pipelined_call(self, pipeline, callback): buf = WriteBuffer() replies = len(pipeline.pipelined_args) cb = functools.partial(self._reply_aggregator, callback, replies) for args in pipeline.pipelined_args: self.__callback_queue.append(cb) tmp_buf = format_args_in_redis_protocol(*args) buf.append(tmp_buf) self.__connection.write(buf) def get_last_state_change_timedelta(self): return self.__connection._state.get_last_state_change_timedelta()
def test_write_on_closed_socket(self): c = Connection(self._read_cb, self._close_cb, connect_timeout=2) yield c.connect() data1 = format_args_in_redis_protocol("PING") data2 = b"*1\r\b$4\r\nQUIT\r\n" c.write(data1) c.write(data2) reply1 = yield self.reply_queue.get() self.assertEquals(reply1, b"PONG") reply2 = yield self.reply_queue.get() self.assertEquals(reply2, b"OK") c.write(data1) # Wait a short moment while the server closes the socket yield tornado.gen.sleep(.0001) c._handle_write() c._handle_read() self.assertFalse(c.is_connected()) c.disconnect()
def test_write_on_closed_socket(self): c = Connection(self._read_cb, self._close_cb, connect_timeout=2) yield c.connect() data1 = format_args_in_redis_protocol("PING") data2 = b"*1\r\b$4\r\nQUIT\r\n" c.write(data1) c.write(data2) reply1 = yield self.reply_queue.get() self.assertEqual(reply1, b"PONG") reply2 = yield self.reply_queue.get() self.assertEqual(reply2, b"OK") c.write(data1) # Wait a short moment while the server closes the socket yield tornado.gen.sleep(.0001) c._handle_write() c._handle_read() self.assertFalse(c.is_connected()) c.disconnect()
class Client(object): """High level object to interact with redis. Attributes: autoconnect (boolean): True if the client is in autoconnect mode (and in autoreconnection mode) (default True). password (string): the password to authenticate with. connection_kwargs (dict): :class:`Connection` object kwargs (note that read_callback and close_callback args are set automatically). """ def __init__(self, autoconnect=True, password=None, **connection_kwargs): """Constructor. Args: autoconnect (boolean): True if the client is in autoconnect mode (and in autoreconnection mode) (default True). password (string): the password to authenticate with. **connection_kwargs: :class:`Connection` object kwargs. """ if 'read_callback' in connection_kwargs or \ 'close_callback' in connection_kwargs: raise Exception("read_callback and close_callback are not allowed " "to be used here.") self.connection_kwargs = connection_kwargs self.autoconnect = autoconnect self.password = password self.__connection = None self.subscribed = False self.__connection = None self.__reader = None # Used for normal clients self.__callback_queue = None # Used for subscribed clients self._condition = tornado.locks.Condition() self._reply_list = None @property def title(self): return self.__connection._redis_server() def is_connected(self): """Returns True is the client is connected to redis. Returns: True if the client if connected to redis. """ return (self.__connection is not None) and \ (self.__connection.is_connected()) @tornado.gen.coroutine def connect(self): """Connects the client object to redis. It's safe to use this method even if you are already connected. Note: this method is useless with autoconnect mode (default). Returns: a Future object with True as result if the connection was ok. """ if self.is_connected(): raise tornado.gen.Return(True) cb1 = self._read_callback cb2 = self._close_callback self.__callback_queue = collections.deque() self._reply_list = [] self.__reader = hiredis.Reader(replyError=ClientError) kwargs = self.connection_kwargs self.__connection = Connection(cb1, cb2, **kwargs) connection_status = yield self.__connection.connect() if connection_status is not True or not self.password: # nothing left to do here, return raise tornado.gen.Return(connection_status) authentication_status = yield self._call('AUTH', self.password) if authentication_status == b'OK': # correct password, it worked raise tornado.gen.Return(True) else: # incorrect password, return back the result LOG.warning("impossible to connect: bad password") self.__connection.disconnect() raise tornado.gen.Return(False) def disconnect(self): """Disconnects the client object from redis. It's safe to use this method even if you are already disconnected. """ if not self.is_connected(): return if self.__connection is not None: self.__connection.disconnect() def _close_callback(self): """Callback called when redis closed the connection. The callback queue is emptied and we call each callback found with None or with an exception object to wake up blocked client. """ while True: try: callback = self.__callback_queue.popleft() callback(ConnectionError("closed connection")) except IndexError: break if self.subscribed: # pubsub clients self._reply_list.append(ConnectionError("closed connection")) self._condition.notify_all() def _read_callback(self, data=None): """Callback called when some data are read on the socket. The buffer is given to the hiredis parser. If a reply is complete, we put the decoded reply to on the reply queue. Args: data (str): string (buffer) read on the socket. """ try: if data is not None: self.__reader.feed(data) while True: reply = self.__reader.gets() if reply is not False: try: callback = self.__callback_queue.popleft() # normal client (1 reply = 1 callback) callback(reply) except IndexError: # pubsub clients self._reply_list.append(reply) self._condition.notify_all() else: break except hiredis.ProtocolError: # something nasty occured (corrupt stream => no way to recover) LOG.warning("corrupted stream => disconnect") self.disconnect() def call(self, *args, **kwargs): """Calls a redis command and returns a Future of the reply. Args: *args: full redis command as variable length argument list or a Pipeline object (as a single argument). **kwargs: internal private options (do not use). Returns: a Future with the decoded redis reply as result (when available) or a ConnectionError object in case of connection error. Raises: ClientError: your Pipeline object is empty. Examples: >>> @tornado.gen.coroutine def foobar(): client = Client() result = yield client.call("HSET", "key", "field", "val") """ if not self.is_connected(): if self.autoconnect: # We use this method only when we are not contected # to void performance penaly due to gen.coroutine decorator return self._call_with_autoconnect(*args, **kwargs) else: error = ConnectionError("you are not connected and " "autoconnect=False") return tornado.gen.maybe_future(error) return self._call(*args, **kwargs) @tornado.gen.coroutine def _call_with_autoconnect(self, *args, **kwargs): yield self.connect() if not self.is_connected(): raise tornado.gen.Return(ConnectionError("impossible to connect")) res = yield self._call(*args, **kwargs) raise tornado.gen.Return(res) def async_call(self, *args, **kwargs): """Calls a redis command, waits for the reply and call a callback. Following options are available (not part of the redis command itself): - callback Function called (with the result as argument) when the result is available. If not set, the reply is silently discarded. In case of errors, the callback is called with a TornadisException object as argument. Args: *args: full redis command as variable length argument list or a Pipeline object (as a single argument). **kwargs: options as keyword parameters. Examples: >>> def cb(result): pass >>> client.async_call("HSET", "key", "field", "val", callback=cb) """ def after_autoconnect_callback(future): if self.is_connected(): self._call(*args, **kwargs) else: # FIXME pass if 'callback' not in kwargs: kwargs['callback'] = discard_reply_cb if not self.is_connected(): if self.autoconnect: connect_future = self.connect() cb = after_autoconnect_callback self.__connection._ioloop.add_future(connect_future, cb) else: error = ConnectionError("you are not connected and " "autoconnect=False") kwargs['callback'](error) else: self._call(*args, **kwargs) def _call(self, *args, **kwargs): callback = False if 'callback' in kwargs: callback = True if len(args) == 1 and isinstance(args[0], Pipeline): fn = self._pipelined_call pipeline = args[0] if pipeline.number_of_stacked_calls == 0: excep = ClientError("empty pipeline") if callback: kwargs['callback'](excep) else: return tornado.gen.maybe_future(excep) arguments = (pipeline, ) else: if "__multiple_replies" in kwargs: fn = self._simple_call_with_multiple_replies arguments = tuple([kwargs["__multiple_replies"]] + list(args)) else: fn = self._simple_call arguments = args if callback: fn(*arguments, **kwargs) else: return tornado.gen.Task(fn, *arguments, **kwargs) def _reply_aggregator(self, callback, replies, reply): self._reply_list.append(reply) if len(self._reply_list) == replies: callback(self._reply_list) self._reply_list = [] def _simple_call(self, *args, **kwargs): callback = kwargs['callback'] msg = format_args_in_redis_protocol(*args) self.__callback_queue.append(callback) self.__connection.write(msg) def _simple_call_with_multiple_replies(self, replies, *args, **kwargs): original_callback = kwargs['callback'] msg = format_args_in_redis_protocol(*args) callback = functools.partial(self._reply_aggregator, original_callback, replies) for _ in range(0, replies): self.__callback_queue.append(callback) self.__connection.write(msg) def _pipelined_call(self, pipeline, callback): buf = WriteBuffer() replies = len(pipeline.pipelined_args) cb = functools.partial(self._reply_aggregator, callback, replies) for args in pipeline.pipelined_args: self.__callback_queue.append(cb) tmp_buf = format_args_in_redis_protocol(*args) buf.append(tmp_buf) self.__connection.write(buf) def get_last_state_change_timedelta(self): return self.__connection._state.get_last_state_change_timedelta()
def test_init(self): c = Connection(self._read_cb, self._close_cb, unix_domain_socket="/tmp/redis.sock") yield c.connect() c.disconnect()