def _reset_zk(self): tmp_list = [] log_warning("reset zk proxy list...") for child in self.zk_client.get_children(self.zk_proxy_dir): try: child_path = '/'.join((self.zk_proxy_dir, child)) data, stat = self.zk_client.get(child_path) proxy_info = json.loads(data) state, addr = proxy_info["state"], proxy_info["addr"] # if state != _CODIS_PROXY_STATE_ONLINE: # continue # a smart way here we should listen the new proxy state # util be changed to online # pass addr = addr.split(':') tmp_list.append((addr[0], int(addr[1]))) except Exception, e: raise ConnectionError("Error while parse zk proxy(%s): %s" % (child, e.args))
def _read_from_socket(self, length=None, timeout=SENTINEL, raise_on_timeout=True): sock = self._sock socket_read_size = self.socket_read_size buf = self._buffer buf.seek(self.bytes_written) marker = 0 custom_timeout = timeout is not SENTINEL try: if custom_timeout: sock.settimeout(timeout) while True: data = recv(self._sock, socket_read_size) # an empty string indicates the server shutdown the socket if isinstance(data, bytes) and len(data) == 0: raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) buf.write(data) data_length = len(data) self.bytes_written += data_length marker += data_length if length is not None and length > marker: continue return True except NONBLOCKING_EXCEPTIONS as ex: # if we're in nonblocking mode and the recv raises a # blocking error, simply return False indicating that # there's no data to be read. otherwise raise the # original exception. allowed_errno = NONBLOCKING_EXCEPTION_ERROR_NUMBERS[ex.__class__] if raise_on_timeout or ex.errno != allowed_errno: raise return False except socket.timeout: if raise_on_timeout: raise return False finally: if custom_timeout: sock.settimeout(self.socket_timeout)
def ws_listening(self): if self.call < 2: res = r.ping() # 检测是否能够连接上 redis server print u'开始监听消息' # 一直连接直到连接成功 try: # simulate(r) ws_app = websocket.WebSocketApp(url=url, on_message=on_message, on_open=on_open) ws_app.run_forever() except ConnectionError, e: print u'连接redis失败' raise ConnectionError('redis-server未启动') except Exception, e: # 连接不上是继续连接,递归调用 self.ws_listening()
def _read_from_socket(self, length=None): socket_read_size = self.socket_read_size if length is None: length = socket_read_size buf = self._buffer buf.seek(self.bytes_written) marker = 0 try: while length > marker: data = self._sock.recv(socket_read_size) buf.write(data) self.bytes_written += len(data) marker += socket_read_size except (socket.error, socket.timeout): e = sys.exc_info()[1] raise ConnectionError("Error while reading from socket: %s" % (e.args, ))
def test_ratelimit_exceptions(self, mock_get, mock_pipeline, mock_error): with self.app.test_request_context(): g.db = database.get_session() self.assertFalse(RateLimit().call(uid=None), msg='Anonymous requests should not be limited') mock_get.side_effect = ConnectionError('testlimit') self.assertFalse(RateLimit().call(uid=self.test_uid)) mock_error.assert_called_with( dict(action='ratelimit.call', message='testlimit')) mock_get.side_effect = None mock_pipeline.side_effect = DataError('key not set') self.assertFalse(RateLimit().call(uid=self.test_uid)) mock_error.assert_called_with( dict(action='ratelimit.call', message='key not set')) g.db.remove()
def read_response(self): "Read the response from a previously sent command" try: response = self._parser.read_response() except socket.timeout: self.disconnect() raise TimeoutError("Timeout reading from %s:%s" % (self.host, self.port)) except socket.error: self.disconnect() e = sys.exc_info()[1] raise ConnectionError("Error while reading from %s:%s : %s" % (self.host, self.port, e.args)) except: # noqa: E722 self.disconnect() raise if isinstance(response, ResponseError): raise response return response
def send_packed_command(self, command): "Send an already packed command to the Redis server" if not self._sock: self.connect() try: # 发送命令 self._sock.sendall(command) except socket.error: e = sys.exc_info()[1] self.disconnect() if len(e.args) == 1: _errno, errmsg = 'UNKNOWN', e.args[0] else: _errno, errmsg = e.args raise ConnectionError("Error %s while writing to socket. %s." % (_errno, errmsg)) except: self.disconnect() raise
def get_connection(self, command_name, shard_hint=None): host_id = shard_hint if host_id is None: raise RuntimeError('The routing pool requires the host id ' 'as shard hint') real_pool = self.cluster.get_pool_for_host(host_id) # When we check something out from the real underlying pool it's # very much possible that the connection is stale. This is why we # check out up to 10 connections which are either not connected # yet or verified alive. for _ in xrange(10): con = real_pool.get_connection(command_name) if con._sock is None or not is_closed(con._sock): con.__creating_pool = weakref(real_pool) return con raise ConnectionError('Failed to check out a valid connection ' '(host %s)' % host_id)
def after_hook(self, command, command_name, args, completer): # === After hook === # SELECT db on AUTH if command_name.upper() == "AUTH": if self.db: select_result = self.execute_command_and_read_response( completer, "SELECT", self.db ) if nativestr(select_result) != "OK": raise ConnectionError("Invalid Database") # When the connection is TimeoutError or ConnectionError, reconnect the connection will use it self.connection.password = args[0] elif command_name.upper() == "SELECT": logger.debug("[After hook] Command is SELECT, change self.db.") self.db = int(args[0]) # When the connection is TimeoutError or ConnectionError, reconnect the connection will use it self.connection.db = self.db if command_name.upper() == "MULTI": logger.debug("[After hook] Command is MULTI, start transaction.") config.transaction = True
def read(self, length=None): """ Read a line from the socket is length is None, otherwise read ``length`` bytes """ try: if length is not None: result = self._fp.read(length) else: result = self._fp.readline() if sys.version_info.major >= 3: result = result.decode() return result except socket.error: e = sys.exc_info()[1] self.disconnect() if e.args and e.args[0] == errno.EAGAIN: raise ConnectionError("Error while reading from socket: %s" % \ e.args[1]) return ''
def read_response(self): try: response = yield self._stream.read_until(SYM_CRLF) except StreamClosedError: raise socket.error(SERVER_CLOSED_CONNECTION_ERROR) response = response[:-2] if not response: raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) byte, response = byte_to_chr(response[0]), response[1:] if byte not in ('-', '+', ':', '$', '*'): raise InvalidResponse("Protocol Error: %s, %s" % (str(byte), str(response))) if byte == '-': response = nativestr(response) error = self.parse_error(response) if isinstance(error, ConnectionError): raise error raise gen.Return(error) elif byte == '+': pass elif byte == ':': response = long(response) elif byte == '$': length = int(response) if length == -1: raise gen.Return(None) response = yield self._stream.read_bytes( length + 2) # make sure to read the '\r\n' response = response[:-2] elif byte == '*': length = int(response) if length == -1: raise gen.Return(None) response = [] for i in xrange(length): part = yield self.read_response() response.append(part) if isinstance(response, bytes): response = self.encoder.decode(response) raise gen.Return(response)
def _read_from_socket(self, length=None): socket_read_size = self.socket_read_size buf = self._buffer buf.seek(self.bytes_written) marker = 0 try: while True: data = self._sock.recv(socket_read_size) buf.write(data) data_length = len(data) self.bytes_written += data_length marker += data_length if length is not None and length > marker: continue break except (socket.error, socket.timeout): e = sys.exc_info()[1] raise ConnectionError("Error while reading from socket: %s" % (e.args, ))
def _connect(self): addrinfo = yield self.resolver.resolve(self.host, self.port, 0) err = None for af, addr in addrinfo: try: s = socket.socket(af) self._stream = IOStream(s, io_loop=self.io_loop) self._timeout = self.io_loop.add_timeout( self.io_loop.time() + self.socket_connect_timeout, stack_context.wrap(self._on_timeout)) stream = yield self._stream.connect(addr) self._remove_timeout() raise gen.Return(stream) except (StreamClosedError, socket.error) as _: err = _ self.disconnect() if err is not None: raise ConnectionError(self._error_message(err)) raise socket.error("socket.getaddrinfo returned an empty list")
def send_packed_command(self, command): "Send an already packed command to the Redis server" current = self._get_current_greenlet() parent = current.parent if not self._iostream: self.connect() try: handle = generate_handle() self._events[handle] = True self._iostream.set_close_callback( functools.partial(self._handle_write_error, current, handle) ) cb = functools.partial(self._handle_write_complete, current, handle) self._timeout_handle = self._ioloop.add_timeout( datetime.timedelta(seconds=self.socket_timeout), functools.partial(self._handle_write_timeout, current) ) # command is always a list cmd_str = ''.join(command) try: self._iostream.write(cmd_str, callback=cb) parent.switch() except IOError as e: # When connection closed tornado will throw an IOError # with the text 'Stream is closed' raise ConnectionError(str(e)) finally: if self._timeout_handle: self._ioloop.remove_timeout(self._timeout_handle) self._timeout_handle = None self._iostream.set_close_callback(None) del self._events[handle] except: self.disconnect() raise
def _parse_response(self, command_name): conn = self.connection response = conn.read().strip() if not response: self.connection.disconnect() raise ConnectionError("Socket closed on remote end") # server returned a null value if response in ('$-1', '*-1'): return None byte, response = response[0], response[1:] # server returned an error if byte == '-': if response.startswith('ERR '): response = response[4:] raise ResponseError(response) # single value elif byte == '+': return response # int value elif byte == ':': return int(response) # bulk response elif byte == '$': length = int(response) if length == -1: return None response = length and conn.read(length) or '' conn.read(2) # read the \r\n delimiter return response # multi-bulk response elif byte == '*': length = int(response) if length == -1: return None return [self._parse_response(command_name) for i in range(length)] raise InvalidResponse("Unknown response type for: %s" % command_name)
def connect(self): """通过这个方法来建立与redis的连接""" if self._sock: return try: sock = self._connect() # 创建socket except socket.error: e = sys.exc_info()[1] raise ConnectionError(self._error_message(e)) self._sock = sock # 保存tcp连接 try: self.on_connect() # 如果需要验证的话,那么搞一下 except RedisError: # clean up after any error in on_connect self.disconnect() raise # run any user callbacks. right now the only internal callback # is for pubsub channel/pattern resubscription for callback in self._connect_callbacks: # 调用一些连接回调 callback(self)
def connect(self): "Connects to the Redis server if not already connected" if self._sock: return try: sock = self._connect() except socket.error: e = sys.exc_info()[1] raise ConnectionError(self._error_message(e)) self._sock = sock try: self.on_connect() except RedisError: # clean up after any error in on_connect self.disconnect() raise # run any user callbacks. right now the only internal callback # is for pubsub channel/pattern resubscription for callback in self._connect_callbacks: callback(self)
def start(self, timeout=5.0): """ 循环启动 """ timeout = float(timeout) self.running = True # 孵化心跳循环线程 eventlet.spawn_n(self.heart_beat_loop) eventlet.sleep(0.1) overtime = monotonic() + timeout while True: if self.connection_pool._created_connections > 0: break if monotonic() > overtime: LOG.error('Redis connection pool init fail') self.running = False # 等待前面的绿色线程结束 # eventlet.sleep(1.0) raise ConnectionError('redis connection pool empty over %1.2f seconds' % timeout) eventlet.sleep(timeout/5.0) # 孵化垃圾key删除循环 eventlet.spawn_n(self.garbage_collector_loop)
def send_packed_command(self, command): "Send an already packed command to the Redis server" if not self._sock: self.connect() try: python_version = sys.version_info.major # Python 2.7 and above except AttributeError: # Below 2.7 python_version = sys.version_info[0] try: if isinstance(command, str): # Works in Python 2 only, must be <class 'bytes'> in 3 if python_version == 2: command = [command] elif python_version == 3: raise TypeError("Expected <class 'bytes'> argument," + " got string instead." + " Use string.encode(encoding) method" + " to convert the" + " argument before passing.") elif isinstance(command, bytes): # Works both in Python 2 and 3 command = [command] for item in command: self._sock.sendall(item) except socket.timeout: self.disconnect() raise TimeoutError("Timeout writing to socket") except socket.error: e = sys.exc_info()[1] self.disconnect() if len(e.args) == 1: _errno, errmsg = 'UNKNOWN', e.args[0] else: _errno, errmsg = e.args raise ConnectionError("Error %s while writing to socket. %s." % (_errno, errmsg)) except: self.disconnect() raise
def send_buffer(self): """Utility function that sends the buffer into the provided socket. The buffer itself will slowly clear out and is modified in place. """ buf = self._send_buf sock = self.connection._sock try: timeout = sock.gettimeout() sock.setblocking(False) try: for idx, item in enumerate(buf): sent = 0 while 1: try: sent = sock.send(item) except IOError as e: if e.errno == errno.EAGAIN: continue elif e.errno == errno.EWOULDBLOCK: break raise self.sent_something = True break if sent < len(item): buf[:idx + 1] = [item[sent:]] break else: del buf[:] finally: sock.settimeout(timeout) except IOError as e: self.connection.disconnect() if isinstance(e, socket.timeout): raise TimeoutError('Timeout writing to socket (host %s)' % self.host_id) raise ConnectionError( 'Error while writing to socket (host %s): %s' % (self.host_id, e))
def get_connection(self, command_name, *keys, **options): """ Get a connection, blocking for ``self.timeout`` until a connection is available from the pool. If the connection returned is ``None`` then creates a new connection. Because we use a last-in first-out queue, the existing connections (having been returned to the pool after the initial ``None`` values were added) will be returned before ``None`` values. This means we only create new connections when we need to, i.e.: the actual number of connections will only increase in response to demand. """ # Make sure we haven't changed process. self._checkpid() # Try and get a connection from the pool. If one isn't available within # self.timeout then raise a ``ConnectionError``. connection = None try: connection = self.pool.get(block=True, timeout=self.timeout) except Empty: # Note that this is not caught by the redis client and will be # raised unless handled by application code. If you want never to raise ConnectionError("No connection available.") # If the pool generation differs, close the connection and open a new one. if connection is not None and connection.pool_generation != self.generation: self._remove_connection(connection) connection.disconnect() connection = None # If the ``connection`` is actually ``None`` then that's a cue to make # a new connection to add to the pool. if connection is None: connection = self.make_connection() return connection
def read_response(self): response = self.read() if not response: raise ConnectionError("Socket closed on remote end") byte, response = byte_to_chr(response[0]), response[1:] if byte not in ('-', '+', ':', '$', '*'): raise InvalidResponse("Protocol Error") # server returned an error if byte == '-': response = nativestr(response) # *return*, not raise the exception class. if it is meant to be # raised, it will be at a higher level. return self.parse_error(response) # single value elif byte == '+': pass # int value elif byte == ':': response = long(response) # bulk response elif byte == '$': length = int(response) if length == -1: return None response = self.read(length) # multi-bulk response elif byte == '*': length = int(response) if length == -1: return None response = [self.read_response() for i in xrange(length)] if isinstance(response, bytes) and self.encoding: response = response.decode(self.encoding) return response
def connect(self): """ Connect to one of the Disque nodes. You can get current connection with connected_node property :returns: nothing """ self.connected_node = None for i, node in self.nodes.items(): host, port = i.split(':') port = int(port) redis_client = redis.Redis(host, port) try: ret = redis_client.execute_command('HELLO') format_version, node_id = ret[0], ret[1] others = ret[2:] self.nodes[i] = Node(node_id, host, port, redis_client) self.connected_node = self.nodes[i] except redis.exceptions.ConnectionError: pass if not self.connected_node: raise ConnectionError('couldnt connect to any nodes') logger.info("connected to node %s" % self.connected_node)
def send_packed_command(self, command): "Send an already packed command to the Redis server" if not self._sock: self.connect() try: if isinstance(command, str): command = [command] for item in command: self._sock.sendall(item) except socket.timeout: self.disconnect() raise TimeoutError("Timeout writing to socket") except socket.error: e = sys.exc_info()[1] self.disconnect() if len(e.args) == 1: _errno, errmsg = 'UNKNOWN', e.args[0] else: _errno, errmsg = e.args raise ConnectionError("Error %s while writing to socket. %s." % (_errno, errmsg)) except: self.disconnect() raise
def _read_from_socket(self, length=None): socket_read_size = self.socket_read_size buf = self._buffer buf.seek(self.bytes_written) marker = 0 try: while True: data = self._sock.recv(socket_read_size) # an empty string indicates the server shutdown the socket if isinstance(data, str) and len(data) == 0: raise socket.error("Connection closed by remote server.") buf.write(data) data_length = len(data) self.bytes_written += data_length marker += data_length if length is not None and length > marker: continue break except (socket.error, socket.timeout): e = sys.exc_info()[1] raise ConnectionError("Error while reading from socket: %s" % (e.args, ))
def get_connection(self, command_name, *keys, **options): """ Get a connection, blocking for ``self.timeout`` until a connection is available from the pool. If the connection returned is ``None`` then creates a new connection. Because we use a last-in first-out queue, the existing connections (having been returned to the pool after the initial ``None`` values were added) will be returned before ``None`` values. This means we only create new connections when we need to, i.e.: the actual number of connections will only increase in response to demand. """ self._checkpid() connection = None try: # 从连接池队列中获取一个连接 connection = self.pool.get(block=True, timeout=self.timeout) except Empty: raise ConnectionError("No connection available.") # 若获取的connection对象为None,则创建一个新的连接 if connection is None: connection = self.make_connection() return connection
def _try_send_buffer(self): sock = self.connection._sock try: timeout = sock.gettimeout() sock.setblocking(False) try: for i, item in enumerate(self._send_buf): sent = 0 while 1: try: sent = sock.send(item) except socket.error as e: if e.errno == errno.EAGAIN: continue elif e.errno == errno.EWOULDBLOCK: break raise break if sent < len(item): self._send_buf[:i + 1] = [item[sent:]] break else: del self._send_buf[:] finally: sock.settimeout(timeout) except socket.timeout: self.connection.disconnect() raise TimeoutError('Timeout writing to socket (%s)' % self.host_name) except socket.error: self.connection.disconnect() raise ConnectionError('Error while writing to socket (%s)' % self.host_name) except: self.connection.disconnect() raise
def execute_command_via_connection(r, *argv, **kwargs): # the first time this is called, simulate an ASK exception. # after that behave normally. # capture all the requests and responses. if not test.execute_command_calls: e = ConnectionError('FAKE: could not connect') test.execute_command_calls.append({'exception': e}) raise e try: result = execute_command_via_connection_original( r, *argv, **kwargs) test.execute_command_calls.append({ 'argv': argv, 'kwargs': kwargs, 'result': result }) return result except Exception as e: test.execute_command_calls.append({ 'argv': argv, 'kwargs': kwargs, 'exception': e }) raise e
def make_connection(self): "Create a new connection" if self._created_connections >= self.max_connections: raise ConnectionError("Too many connections") self._created_connections += 1 return self.connection_class(**self.connection_kwargs)
def get_build_by_attr_value(self, attr): try: neoDriver.connect() return neoDriver.generate_build_one(attr) except: raise (ConnectionError('Service Unavailable'))