def _reset_zk(self):
tmp_list = []
log_warning("reset zk proxy list...")
for child in self.zk_client.get_children(self.zk_proxy_dir):
try:
child_path = '/'.join((self.zk_proxy_dir, child))
data, stat = self.zk_client.get(child_path)
proxy_info = json.loads(data)
state, addr = proxy_info["state"], proxy_info["addr"]
# if state != _CODIS_PROXY_STATE_ONLINE:
# continue
# a smart way here we should listen the new proxy state
# util be changed to online
# pass
addr = addr.split(':')
tmp_list.append((addr[0], int(addr[1])))
except Exception, e:
raise ConnectionError("Error while parse zk proxy(%s): %s" %
(child, e.args))
def _read_from_socket(self,
length=None,
timeout=SENTINEL,
raise_on_timeout=True):
sock = self._sock
socket_read_size = self.socket_read_size
buf = self._buffer
buf.seek(self.bytes_written)
marker = 0
custom_timeout = timeout is not SENTINEL
try:
if custom_timeout:
sock.settimeout(timeout)
while True:
data = recv(self._sock, socket_read_size)
# an empty string indicates the server shutdown the socket
if isinstance(data, bytes) and len(data) == 0:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
buf.write(data)
data_length = len(data)
self.bytes_written += data_length
marker += data_length
if length is not None and length > marker:
continue
return True
except NONBLOCKING_EXCEPTIONS as ex:
# if we're in nonblocking mode and the recv raises a
# blocking error, simply return False indicating that
# there's no data to be read. otherwise raise the
# original exception.
allowed_errno = NONBLOCKING_EXCEPTION_ERROR_NUMBERS[ex.__class__]
if raise_on_timeout or ex.errno != allowed_errno:
raise
return False
except socket.timeout:
if raise_on_timeout:
raise
return False
finally:
if custom_timeout:
sock.settimeout(self.socket_timeout)
def ws_listening(self):
if self.call < 2:
res = r.ping() # 检测是否能够连接上 redis server
print u'开始监听消息'
# 一直连接直到连接成功
try:
# simulate(r)
ws_app = websocket.WebSocketApp(url=url,
on_message=on_message,
on_open=on_open)
ws_app.run_forever()
except ConnectionError, e:
print u'连接redis失败'
raise ConnectionError('redis-server未启动')
except Exception, e:
# 连接不上是继续连接，递归调用
self.ws_listening()
def _read_from_socket(self, length=None):
socket_read_size = self.socket_read_size
if length is None:
length = socket_read_size
buf = self._buffer
buf.seek(self.bytes_written)
marker = 0
try:
while length > marker:
data = self._sock.recv(socket_read_size)
buf.write(data)
self.bytes_written += len(data)
marker += socket_read_size
except (socket.error, socket.timeout):
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from socket: %s" %
(e.args, ))
def test_ratelimit_exceptions(self, mock_get, mock_pipeline, mock_error):
with self.app.test_request_context():
g.db = database.get_session()
self.assertFalse(RateLimit().call(uid=None),
msg='Anonymous requests should not be limited')
mock_get.side_effect = ConnectionError('testlimit')
self.assertFalse(RateLimit().call(uid=self.test_uid))
mock_error.assert_called_with(
dict(action='ratelimit.call', message='testlimit'))
mock_get.side_effect = None
mock_pipeline.side_effect = DataError('key not set')
self.assertFalse(RateLimit().call(uid=self.test_uid))
mock_error.assert_called_with(
dict(action='ratelimit.call', message='key not set'))
g.db.remove()
def read_response(self):
"Read the response from a previously sent command"
try:
response = self._parser.read_response()
except socket.timeout:
self.disconnect()
raise TimeoutError("Timeout reading from %s:%s" %
(self.host, self.port))
except socket.error:
self.disconnect()
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from %s:%s : %s" %
(self.host, self.port, e.args))
except: # noqa: E722
self.disconnect()
raise
if isinstance(response, ResponseError):
raise response
return response
def send_packed_command(self, command):
"Send an already packed command to the Redis server"
if not self._sock:
self.connect()
try:
# 发送命令
self._sock.sendall(command)
except socket.error:
e = sys.exc_info()[1]
self.disconnect()
if len(e.args) == 1:
_errno, errmsg = 'UNKNOWN', e.args[0]
else:
_errno, errmsg = e.args
raise ConnectionError("Error %s while writing to socket. %s." %
(_errno, errmsg))
except:
self.disconnect()
raise
def get_connection(self, command_name, shard_hint=None):
host_id = shard_hint
if host_id is None:
raise RuntimeError('The routing pool requires the host id '
'as shard hint')
real_pool = self.cluster.get_pool_for_host(host_id)
# When we check something out from the real underlying pool it's
# very much possible that the connection is stale. This is why we
# check out up to 10 connections which are either not connected
# yet or verified alive.
for _ in xrange(10):
con = real_pool.get_connection(command_name)
if con._sock is None or not is_closed(con._sock):
con.__creating_pool = weakref(real_pool)
return con
raise ConnectionError('Failed to check out a valid connection '
'(host %s)' % host_id)
def after_hook(self, command, command_name, args, completer):
# === After hook ===
# SELECT db on AUTH
if command_name.upper() == "AUTH":
if self.db:
select_result = self.execute_command_and_read_response(
completer, "SELECT", self.db
)
if nativestr(select_result) != "OK":
raise ConnectionError("Invalid Database")
# When the connection is TimeoutError or ConnectionError, reconnect the connection will use it
self.connection.password = args[0]
elif command_name.upper() == "SELECT":
logger.debug("[After hook] Command is SELECT, change self.db.")
self.db = int(args[0])
# When the connection is TimeoutError or ConnectionError, reconnect the connection will use it
self.connection.db = self.db
if command_name.upper() == "MULTI":
logger.debug("[After hook] Command is MULTI, start transaction.")
config.transaction = True
def read(self, length=None):
"""
Read a line from the socket is length is None,
otherwise read ``length`` bytes
"""
try:
if length is not None:
result = self._fp.read(length)
else:
result = self._fp.readline()
if sys.version_info.major >= 3:
result = result.decode()
return result
except socket.error:
e = sys.exc_info()[1]
self.disconnect()
if e.args and e.args[0] == errno.EAGAIN:
raise ConnectionError("Error while reading from socket: %s" % \
e.args[1])
return ''
def read_response(self):
try:
response = yield self._stream.read_until(SYM_CRLF)
except StreamClosedError:
raise socket.error(SERVER_CLOSED_CONNECTION_ERROR)
response = response[:-2]
if not response:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error: %s, %s" %
(str(byte), str(response)))
if byte == '-':
response = nativestr(response)
error = self.parse_error(response)
if isinstance(error, ConnectionError):
raise error
raise gen.Return(error)
elif byte == '+':
pass
elif byte == ':':
response = long(response)
elif byte == '$':
length = int(response)
if length == -1:
raise gen.Return(None)
response = yield self._stream.read_bytes(
length + 2) # make sure to read the '\r\n'
response = response[:-2]
elif byte == '*':
length = int(response)
if length == -1:
raise gen.Return(None)
response = []
for i in xrange(length):
part = yield self.read_response()
response.append(part)
if isinstance(response, bytes):
response = self.encoder.decode(response)
raise gen.Return(response)
def _read_from_socket(self, length=None):
socket_read_size = self.socket_read_size
buf = self._buffer
buf.seek(self.bytes_written)
marker = 0
try:
while True:
data = self._sock.recv(socket_read_size)
buf.write(data)
data_length = len(data)
self.bytes_written += data_length
marker += data_length
if length is not None and length > marker:
continue
break
except (socket.error, socket.timeout):
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from socket: %s" %
(e.args, ))
def _connect(self):
addrinfo = yield self.resolver.resolve(self.host, self.port, 0)
err = None
for af, addr in addrinfo:
try:
s = socket.socket(af)
self._stream = IOStream(s, io_loop=self.io_loop)
self._timeout = self.io_loop.add_timeout(
self.io_loop.time() + self.socket_connect_timeout,
stack_context.wrap(self._on_timeout))
stream = yield self._stream.connect(addr)
self._remove_timeout()
raise gen.Return(stream)
except (StreamClosedError, socket.error) as _:
err = _
self.disconnect()
if err is not None:
raise ConnectionError(self._error_message(err))
raise socket.error("socket.getaddrinfo returned an empty list")
def send_packed_command(self, command):
"Send an already packed command to the Redis server"
current = self._get_current_greenlet()
parent = current.parent
if not self._iostream:
self.connect()
try:
handle = generate_handle()
self._events[handle] = True
self._iostream.set_close_callback(
functools.partial(self._handle_write_error, current, handle)
)
cb = functools.partial(self._handle_write_complete, current, handle)
self._timeout_handle = self._ioloop.add_timeout(
datetime.timedelta(seconds=self.socket_timeout),
functools.partial(self._handle_write_timeout, current)
)
# command is always a list
cmd_str = ''.join(command)
try:
self._iostream.write(cmd_str, callback=cb)
parent.switch()
except IOError as e:
# When connection closed tornado will throw an IOError
# with the text 'Stream is closed'
raise ConnectionError(str(e))
finally:
if self._timeout_handle:
self._ioloop.remove_timeout(self._timeout_handle)
self._timeout_handle = None
self._iostream.set_close_callback(None)
del self._events[handle]
except:
self.disconnect()
raise
def _parse_response(self, command_name):
conn = self.connection
response = conn.read().strip()
if not response:
self.connection.disconnect()
raise ConnectionError("Socket closed on remote end")
# server returned a null value
if response in ('$-1', '*-1'):
return None
byte, response = response[0], response[1:]
# server returned an error
if byte == '-':
if response.startswith('ERR '):
response = response[4:]
raise ResponseError(response)
# single value
elif byte == '+':
return response
# int value
elif byte == ':':
return int(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = length and conn.read(length) or ''
conn.read(2) # read the \r\n delimiter
return response
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
return [self._parse_response(command_name) for i in range(length)]
raise InvalidResponse("Unknown response type for: %s" % command_name)
def connect(self):
"""通过这个方法来建立与redis的连接"""
if self._sock:
return
try:
sock = self._connect() # 创建socket
except socket.error:
e = sys.exc_info()[1]
raise ConnectionError(self._error_message(e))
self._sock = sock # 保存tcp连接
try:
self.on_connect() # 如果需要验证的话，那么搞一下
except RedisError:
# clean up after any error in on_connect
self.disconnect()
raise
# run any user callbacks. right now the only internal callback
# is for pubsub channel/pattern resubscription
for callback in self._connect_callbacks: # 调用一些连接回调
callback(self)
def connect(self):
"Connects to the Redis server if not already connected"
if self._sock:
return
try:
sock = self._connect()
except socket.error:
e = sys.exc_info()[1]
raise ConnectionError(self._error_message(e))
self._sock = sock
try:
self.on_connect()
except RedisError:
# clean up after any error in on_connect
self.disconnect()
raise
# run any user callbacks. right now the only internal callback
# is for pubsub channel/pattern resubscription
for callback in self._connect_callbacks:
callback(self)
def start(self, timeout=5.0):
"""
循环启动
"""
timeout = float(timeout)
self.running = True
# 孵化心跳循环线程
eventlet.spawn_n(self.heart_beat_loop)
eventlet.sleep(0.1)
overtime = monotonic() + timeout
while True:
if self.connection_pool._created_connections > 0:
break
if monotonic() > overtime:
LOG.error('Redis connection pool init fail')
self.running = False
# 等待前面的绿色线程结束
# eventlet.sleep(1.0)
raise ConnectionError('redis connection pool empty over %1.2f seconds' % timeout)
eventlet.sleep(timeout/5.0)
# 孵化垃圾key删除循环
eventlet.spawn_n(self.garbage_collector_loop)
def send_packed_command(self, command):
"Send an already packed command to the Redis server"
if not self._sock:
self.connect()
try:
python_version = sys.version_info.major # Python 2.7 and above
except AttributeError: # Below 2.7
python_version = sys.version_info[0]
try:
if isinstance(command, str):
# Works in Python 2 only, must be <class 'bytes'> in 3
if python_version == 2:
command = [command]
elif python_version == 3:
raise TypeError("Expected <class 'bytes'> argument," +
" got string instead." +
" Use string.encode(encoding) method" +
" to convert the" +
" argument before passing.")
elif isinstance(command, bytes): # Works both in Python 2 and 3
command = [command]
for item in command:
self._sock.sendall(item)
except socket.timeout:
self.disconnect()
raise TimeoutError("Timeout writing to socket")
except socket.error:
e = sys.exc_info()[1]
self.disconnect()
if len(e.args) == 1:
_errno, errmsg = 'UNKNOWN', e.args[0]
else:
_errno, errmsg = e.args
raise ConnectionError("Error %s while writing to socket. %s." %
(_errno, errmsg))
except:
self.disconnect()
raise
def send_buffer(self):
"""Utility function that sends the buffer into the provided socket.
The buffer itself will slowly clear out and is modified in place.
"""
buf = self._send_buf
sock = self.connection._sock
try:
timeout = sock.gettimeout()
sock.setblocking(False)
try:
for idx, item in enumerate(buf):
sent = 0
while 1:
try:
sent = sock.send(item)
except IOError as e:
if e.errno == errno.EAGAIN:
continue
elif e.errno == errno.EWOULDBLOCK:
break
raise
self.sent_something = True
break
if sent < len(item):
buf[:idx + 1] = [item[sent:]]
break
else:
del buf[:]
finally:
sock.settimeout(timeout)
except IOError as e:
self.connection.disconnect()
if isinstance(e, socket.timeout):
raise TimeoutError('Timeout writing to socket (host %s)' %
self.host_id)
raise ConnectionError(
'Error while writing to socket (host %s): %s' %
(self.host_id, e))
def get_connection(self, command_name, *keys, **options):
"""
Get a connection, blocking for ``self.timeout`` until a connection
is available from the pool.
If the connection returned is ``None`` then creates a new connection.
Because we use a last-in first-out queue, the existing connections
(having been returned to the pool after the initial ``None`` values
were added) will be returned before ``None`` values. This means we only
create new connections when we need to, i.e.: the actual number of
connections will only increase in response to demand.
"""
# Make sure we haven't changed process.
self._checkpid()
# Try and get a connection from the pool. If one isn't available within
# self.timeout then raise a ``ConnectionError``.
connection = None
try:
connection = self.pool.get(block=True, timeout=self.timeout)
except Empty:
# Note that this is not caught by the redis client and will be
# raised unless handled by application code. If you want never to
raise ConnectionError("No connection available.")
# If the pool generation differs, close the connection and open a new one.
if connection is not None and connection.pool_generation != self.generation:
self._remove_connection(connection)
connection.disconnect()
connection = None
# If the ``connection`` is actually ``None`` then that's a cue to make
# a new connection to add to the pool.
if connection is None:
connection = self.make_connection()
return connection
def read_response(self):
response = self.read()
if not response:
raise ConnectionError("Socket closed on remote end")
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error")
# server returned an error
if byte == '-':
response = nativestr(response)
# *return*, not raise the exception class. if it is meant to be
# raised, it will be at a higher level.
return self.parse_error(response)
# single value
elif byte == '+':
pass
# int value
elif byte == ':':
response = long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = self.read(length)
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
response = [self.read_response() for i in xrange(length)]
if isinstance(response, bytes) and self.encoding:
response = response.decode(self.encoding)
return response
def connect(self):
"""
Connect to one of the Disque nodes.
You can get current connection with connected_node property
:returns: nothing
"""
self.connected_node = None
for i, node in self.nodes.items():
host, port = i.split(':')
port = int(port)
redis_client = redis.Redis(host, port)
try:
ret = redis_client.execute_command('HELLO')
format_version, node_id = ret[0], ret[1]
others = ret[2:]
self.nodes[i] = Node(node_id, host, port, redis_client)
self.connected_node = self.nodes[i]
except redis.exceptions.ConnectionError:
pass
if not self.connected_node:
raise ConnectionError('couldnt connect to any nodes')
logger.info("connected to node %s" % self.connected_node)
def send_packed_command(self, command):
"Send an already packed command to the Redis server"
if not self._sock:
self.connect()
try:
if isinstance(command, str):
command = [command]
for item in command:
self._sock.sendall(item)
except socket.timeout:
self.disconnect()
raise TimeoutError("Timeout writing to socket")
except socket.error:
e = sys.exc_info()[1]
self.disconnect()
if len(e.args) == 1:
_errno, errmsg = 'UNKNOWN', e.args[0]
else:
_errno, errmsg = e.args
raise ConnectionError("Error %s while writing to socket. %s." %
(_errno, errmsg))
except:
self.disconnect()
raise
def _read_from_socket(self, length=None):
socket_read_size = self.socket_read_size
buf = self._buffer
buf.seek(self.bytes_written)
marker = 0
try:
while True:
data = self._sock.recv(socket_read_size)
# an empty string indicates the server shutdown the socket
if isinstance(data, str) and len(data) == 0:
raise socket.error("Connection closed by remote server.")
buf.write(data)
data_length = len(data)
self.bytes_written += data_length
marker += data_length
if length is not None and length > marker:
continue
break
except (socket.error, socket.timeout):
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from socket: %s" %
(e.args, ))
def get_connection(self, command_name, *keys, **options):
"""
Get a connection, blocking for ``self.timeout`` until a connection
is available from the pool.
If the connection returned is ``None`` then creates a new connection.
Because we use a last-in first-out queue, the existing connections
(having been returned to the pool after the initial ``None`` values
were added) will be returned before ``None`` values. This means we only
create new connections when we need to, i.e.: the actual number of
connections will only increase in response to demand.
"""
self._checkpid()
connection = None
try:
# 从连接池队列中获取一个连接
connection = self.pool.get(block=True, timeout=self.timeout)
except Empty:
raise ConnectionError("No connection available.")
# 若获取的connection对象为None，则创建一个新的连接
if connection is None:
connection = self.make_connection()
return connection
def _try_send_buffer(self):
sock = self.connection._sock
try:
timeout = sock.gettimeout()
sock.setblocking(False)
try:
for i, item in enumerate(self._send_buf):
sent = 0
while 1:
try:
sent = sock.send(item)
except socket.error as e:
if e.errno == errno.EAGAIN:
continue
elif e.errno == errno.EWOULDBLOCK:
break
raise
break
if sent < len(item):
self._send_buf[:i + 1] = [item[sent:]]
break
else:
del self._send_buf[:]
finally:
sock.settimeout(timeout)
except socket.timeout:
self.connection.disconnect()
raise TimeoutError('Timeout writing to socket (%s)' %
self.host_name)
except socket.error:
self.connection.disconnect()
raise ConnectionError('Error while writing to socket (%s)' %
self.host_name)
except:
self.connection.disconnect()
raise
def execute_command_via_connection(r, *argv, **kwargs):
# the first time this is called, simulate an ASK exception.
# after that behave normally.
# capture all the requests and responses.
if not test.execute_command_calls:
e = ConnectionError('FAKE: could not connect')
test.execute_command_calls.append({'exception': e})
raise e
try:
result = execute_command_via_connection_original(
r, *argv, **kwargs)
test.execute_command_calls.append({
'argv': argv,
'kwargs': kwargs,
'result': result
})
return result
except Exception as e:
test.execute_command_calls.append({
'argv': argv,
'kwargs': kwargs,
'exception': e
})
raise e
def make_connection(self):
"Create a new connection"
if self._created_connections >= self.max_connections:
raise ConnectionError("Too many connections")
self._created_connections += 1
return self.connection_class(**self.connection_kwargs)
def get_build_by_attr_value(self, attr):
try:
neoDriver.connect()
return neoDriver.generate_build_one(attr)
except:
raise (ConnectionError('Service Unavailable'))
