def read_response(self):
if not self._reader:
raise ConnectionError("Socket closed on remote end")
response = self._reader.gets()
while response is False:
try:
buffer = self._sock.recv(4096)
except (socket.error, socket.timeout):
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from socket: %s" %
(e.args, ))
if not buffer:
raise ConnectionError("Socket closed on remote end")
self._reader.feed(buffer)
# proactively, but not conclusively, check if more data is in the
# buffer. if the data received doesn't end with \n, there's more.
if not buffer.endswith(SYM_LF):
continue
response = self._reader.gets()
if isinstance(response, ResponseError):
response = self.parse_error(response.args[0])
return response
def __init__(self, test=False):
if not test:
if not self.__read_config_file('./feature-flag.ini'):
raise IOError('Please check your FF configuration')
else:
self._redis_client = FeatureFlag._init_redis_client(
self.config)
try:
self._redis_client.ping()
except ConnectionError:
raise ConnectionError('Invalid Redis server')
else:
self._redis_client = None
def on_connect(self):
self._parser.on_connect(self)
# 若redis服务器设置了密码，那么在使用之前需要先经过AUTH认证
if self.password:
self.send_command('AUTH', self.password)
if nativestr(self.read_response()) != 'OK':
raise AuthenticationError('Invalid Password')
# 选择使用编号为db的数据库
if self.db:
self.send_command('SELECT', self.db)
if nativestr(self.read_response()) != 'OK':
raise ConnectionError('Invalid Database')
def read_response(self):
try:
return super(SentinelManagedConnection, self).read_response()
except ReadOnlyError:
if self.connection_pool.is_master:
# When talking to a master, a ReadOnlyError when likely
# indicates that the previous master that we're still connected
# to has been demoted to a slave and there's a new master.
# calling disconnect will force the connection to re-query
# sentinel during the next connect() attempt.
self.disconnect()
raise ConnectionError('The previous master is now a slave')
raise
def read_response(self, command_name, catch_errors):
response = self.read()[:-2] # strip last two characters (\r\n)
if not response:
self.disconnect()
raise ConnectionError("Socket closed on remote end")
# server returned a null value
if response in ('$-1', '*-1'):
return None
byte, response = response[0], response[1:]
# server returned an error
if byte == '-':
if response.startswith('ERR '):
response = response[4:]
raise ResponseError(response)
# single value
elif byte == '+':
return response
# int value
elif byte == ':':
return long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = length and self.read(length) or ''
self.read(2) # read the \r\n delimiter
return response
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
if not catch_errors:
return [self.read_response(command_name, catch_errors)
for i in range(length)]
else:
# for pipelines, we need to read everything,
# including response errors. otherwise we'd
# completely mess up the receive buffer
data = []
for i in range(length):
try:
data.append(
self.read_response(command_name, catch_errors)
)
except Exception, e:
data.append(e)
return data
def read_response(self, command_name, catch_errors):
response = self._reader.gets()
while response is False:
buffer = self._sock.recv(4096)
if not buffer:
self.disconnect()
raise ConnectionError("Socket closed on remote end")
self._reader.feed(buffer)
response = self._reader.gets()
if isinstance(response, ResponseError):
raise response
return response
def send_packed_command(self, command):
"Send an already packed command to the Redis server"
if not self._sock:
self.connect()
try:
self._sock.sendall(command)
except socket.error, e:
self.disconnect()
if len(e.args) == 1:
_errno, errmsg = 'UNKNOWN', e.args[0]
else:
_errno, errmsg = e.args
raise ConnectionError("Error %s while writing to socket. %s." % \
(_errno, errmsg))
def wrapper(*args, **kwargs):
self = args[0]
for i in range(self.RETRY_N):
logging.debug('try to %s', f.__name__)
try:
return f(*args, **kwargs)
except Exception as e:
logging.warning('catched error "%s", retry "%s"', e,
f.__name__)
time.sleep(self.RETRY_INTERVAL_SEC)
if raise_:
raise ConnectionError('gave up after {} retries'.format(
self.RETRY_N))
def send(self, command, redis_instance):
"Send ``command`` to the Redis server. Return the result."
self.connect(redis_instance)
try:
self._sock.sendall(command)
except socket.error, e:
if e.args[0] == errno.EPIPE:
self.disconnect()
if isinstance(e.args, basestring):
_errno, errmsg = 'UNKNOWN', e.args
else:
_errno, errmsg = e.args
raise ConnectionError("Error %s while writing to socket. %s." % \
(_errno, errmsg))
def read(self, length=None):
"""
Read a line from the socket is length is None,
otherwise read ``length`` bytes
"""
try:
if length is not None:
return self._fp.read(length)
return self._fp.readline()
except socket.error, e:
self.disconnect()
if e.args and e.args[0] == errno.EAGAIN:
raise ConnectionError("Error while reading from socket: %s" % \
e.args[1])
def after_hook(self, command, command_name, args, completer):
# === After hook ===
# SELECT db on AUTH
if command_name.upper() == "AUTH" and self.db:
select_result = self.execute_command_and_read_response(
completer, "SELECT", self.db)
if nativestr(select_result) != "OK":
raise ConnectionError("Invalid Database")
elif command_name.upper() == "SELECT":
logger.debug("[After hook] Command is SELECT, change self.db.")
self.db = int(args[0])
if command_name.upper() == "MULTI":
logger.debug("[After hook] Command is MULTI, start transaction.")
config.transaction = True
def on_connect(self):
'''
Initialize the connection, authenticate and select a database and send READONLY if it is
set during object initialization.
'''
super(SSLClusterConnection, self).on_connect()
if self.readonly:
log.debug("Sending READONLY command to server to configure connection as readonly")
self.send_command('READONLY')
if nativestr(self.read_response()) != 'OK':
raise ConnectionError('READONLY command failed')
def on_connect(self):
self._parser.on_connect(self)
if self.password:
yield self.send_command('AUTH', self.password)
response = yield self.read_response()
if nativestr(response) != 'OK':
raise AuthenticationError('Invalid Password')
if self.db:
yield self.send_command('SELECT', self.db)
response = yield self.read_response()
if nativestr(response) != 'OK':
raise ConnectionError('Invalid Database')
async def read_response(self, disable_decoding: bool = False):
try:
return await super().read_response(
disable_decoding=disable_decoding)
except ReadOnlyError:
if self.connection_pool.is_master:
# When talking to a master, a ReadOnlyError when likely
# indicates that the previous master that we're still connected
# to has been demoted to a slave and there's a new master.
# calling disconnect will force the connection to re-query
# sentinel during the next connect() attempt.
await self.disconnect()
raise ConnectionError("The previous master is now a slave")
raise
def _certificate_components(self, cert):
"""Given an SSL certificate, retract the useful components for
validating the certificate status with an OCSP server.
Args:
cert ([bytes]): A PEM encoded ssl certificate
"""
try:
aia = cert.extensions.get_extension_for_oid(
x509.oid.ExtensionOID.AUTHORITY_INFORMATION_ACCESS).value
except cryptography.x509.extensions.ExtensionNotFound:
raise ConnectionError(
"No AIA information present in ssl certificate")
# fetch certificate issuers
issuers = [
i for i in aia if i.access_method ==
x509.oid.AuthorityInformationAccessOID.CA_ISSUERS
]
try:
issuer = issuers[0].access_location.value
except IndexError:
issuer = None
# now, the series of ocsp server entries
ocsps = [
i for i in aia
if i.access_method == x509.oid.AuthorityInformationAccessOID.OCSP
]
try:
ocsp = ocsps[0].access_location.value
except IndexError:
raise ConnectionError("no ocsp servers in certificate")
return cert, issuer, ocsp
def read_from_socket(self, timeout=SENTINEL, raise_on_timeout=True):
sock = self._sock
custom_timeout = timeout is not SENTINEL
try:
if custom_timeout:
sock.settimeout(timeout)
if HIREDIS_USE_BYTE_BUFFER:
bufflen = recv_into(self._sock, self._buffer)
if bufflen == 0:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
self._reader.feed(self._buffer, 0, bufflen)
else:
buffer = recv(self._sock, self.socket_read_size)
# an empty string indicates the server shutdown the socket
if not isinstance(buffer, bytes) or len(buffer) == 0:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
self._reader.feed(buffer)
# data was read from the socket and added to the buffer.
# return True to indicate that data was read.
return True
except socket.timeout:
if raise_on_timeout:
raise TimeoutError("Timeout reading from socket")
return False
except NONBLOCKING_EXCEPTIONS as ex:
# if we're in nonblocking mode and the recv raises a
# blocking error, simply return False indicating that
# there's no data to be read. otherwise raise the
# original exception.
allowed = NONBLOCKING_EXCEPTION_ERROR_NUMBERS.get(ex.__class__, -1)
if not raise_on_timeout and ex.errno == allowed:
return False
raise ConnectionError("Error while reading from socket: %s" %
(ex.args, ))
finally:
if custom_timeout:
sock.settimeout(self._socket_timeout)
def read_response(self):
try:
response = self._buffer.readline()
except:
print("SC Debug: race condition that the _buffer is already released")
return None
if not response:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error: %s, %s" %
(str(byte), str(response)))
# server returned an error
if byte == '-':
response = nativestr(response)
error = self.parse_error(response)
# if the error is a ConnectionError, raise immediately so the user
# is notified
if isinstance(error, ConnectionError):
raise error
# otherwise, we're dealing with a ResponseError that might belong
# inside a pipeline response. the connection's read_response()
# and/or the pipeline's execute() will raise this error if
# necessary, so just return the exception instance here.
return error
# single value
elif byte == '+':
pass
# int value
elif byte == ':':
response = long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = self._buffer.read(length)
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
response = [self.read_response() for i in xrange(length)]
if isinstance(response, bytes):
response = self.encoder.decode(response)
return response
def ws_listening():
res = r.ping() # 检测是否能够连接上 redis server
print u'开始监听消息'
# 一直连接直到连接成功
try:
# simulate(r)
ws_app = websocket.WebSocketApp(url=url,
on_message=on_message,
on_open=on_open)
ws_app.run_forever()
except ConnectionError, e:
print u'连接redis失败'
raise ConnectionError('redis-server未启动')
def init_db(redis=None):
if redis:
Customer.redis = redis
try:
Customer.redis.ping()
except ConnectionError:
Customer.redis = None
raise ConnectionError('Could not connect to the Redis Service')
return
# Get the credentials from the Bluemix environment
if 'VCAP_SERVICES' in os.environ:
vcap_services = os.environ['VCAP_SERVICES']
services = json.loads(vcap_services)
creds = services['rediscloud'][0]['credentials']
Customer.connect_to_redis(creds['hostname'], creds['port'],
creds['password'])
else:
Customer.connect_to_redis('127.0.0.1', 6379, None)
if not Customer.redis:
Customer.connect_to_redis('redis', 6379, None)
if not Customer.redis:
# if you end up here, redis instance is down.
raise ConnectionError('Could not connect to the Redis Service')
def on_connect(self):
"Initialize the connection, authenticate and select a database"
self._parser.on_connect(self)
# if a password is specified, authenticate
if self.password:
self.send_command('AUTH', self.password)
if nativestr(self.read_response()) != 'OK':
raise AuthenticationError('Invalid Password')
# if a database is specified, switch to it
if self.db:
self.send_command('SELECT', self.db)
if nativestr(self.read_response()) != 'OK':
raise ConnectionError('Invalid Database')
def read_response(self):
response = self.read()
if not response:
raise ConnectionError("Socket closed on remote end")
byte, response = response[0], response[1:]
# server returned an error
if byte == '-':
if response.startswith('ERR '):
response = response[4:]
return ResponseError(response)
if response.startswith('LOADING '):
# If we're loading the dataset into memory, kill the socket
# so we re-initialize (and re-SELECT) next time.
raise ConnectionError("Redis is loading data into memory")
# single value
elif byte == '+':
return response
# int value
elif byte == ':':
return long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = self.read(length)
return response
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
return [self.read_response() for i in xrange(length)]
raise InvalidResponse("Protocol Error")
def _send(self, command):
"Send the command to the socket"
if not self._sock:
self.connect()
try:
self._sock.sendall(command)
except socket.error, e:
if e.args[0] == errno.EPIPE:
self.disconnect()
if len(e.args) == 1:
_errno, errmsg = 'UNKNOWN', e.args[0]
else:
_errno, errmsg = e.args
raise ConnectionError("Error %s while writing to socket. %s." % \
(_errno, errmsg))
def init_db(redis=None):
"""
Initialized Redis database connection
This method will work in the following conditions:
1) In Bluemix with Redis bound through VCAP_SERVICES
2) With Redis running on the local server as with Travis CI
3) With Redis --link in a Docker container called 'redis'
4) Passing in your own Redis connection object
Exception:
----------
redis.ConnectionError - if ping() test fails
"""
# if redis:
# Promotion.logger.info("Using client connection...")
# Promotion.redis = redis
# try:
# Promotion.redis.ping()
# Promotion.logger.info("Connection established")
# except ConnectionError:
# Promotion.logger.error("Client Connection Error!")
# Promotion.redis = None
# raise ConnectionError('Could not connect to the Redis Service')
# return
# Get the credentials from the Bluemix environment
if 'VCAP_SERVICES' in os.environ:
Promotion.logger.info("Using VCAP_SERVICES...")
vcap_services = os.environ['VCAP_SERVICES']
services = json.loads(vcap_services)
creds = services['rediscloud'][0]['credentials']
Promotion.logger.info("Conecting to Redis on host %s port %s",
creds['hostname'], creds['port'])
Promotion.connect_to_redis(creds['hostname'], creds['port'],
creds['password'])
else:
Promotion.logger.info(
"VCAP_SERVICES not found, checking localhost for Redis")
Promotion.connect_to_redis('127.0.0.1', 6379, None)
if not Promotion.redis:
Promotion.logger.info(
"No Redis on localhost, looking for redis host")
Promotion.connect_to_redis('redis', 6379, None)
if not Promotion.redis:
# if you end up here, redis instance is down.
Promotion.logger.fatal(
'*** FATAL ERROR: Could not connect to the Redis Service')
raise ConnectionError('Could not connect to the Redis Service')
def test_redis_conn_fail(self, mock_logger, mocker):
"""Test the redis constructor fails."""
mock_get_redis_url = mocker.patch(
"pgsync.redisqueue.get_redis_url",
return_value="redis://*****:*****@some-host:6379/0",
)
mock_ping = mocker.patch(
"redis.Redis.ping", side_effect=ConnectionError("pong")
)
with pytest.raises(ConnectionError) as excinfo:
queue = RedisQueue("something", namespace="foo")
mock_get_redis_url.assert_called_once()
mock_ping.assert_called_once()
mock_logger.exception.assert_called_once_with(
"Redis server is not running: pong"
)
def test_storage_connection_error(self, mock_redis):
mock_redis.side_effect = ConnectionError()
original_count = Document.objects.count()
connection_error = False
try:
file = SimpleUploadedFile('file.txt', b'this is some content')
test = Document()
test.file = file
test.save()
except ConnectionError:
connection_error = True
self.assertTrue(connection_error)
self.assertEqual(Document.objects.count(), original_count)
def connect(self):
"Connects to the Redis server if not already connected"
if self._sock:
return
try:
sock = self._connect()
except socket.error, e:
# args for socket.error can either be (errno, "message")
# or just "message"
if len(e.args) == 1:
error_message = "Error connecting to %s:%s. %s." % \
(self.host, self.port, e.args[0])
else:
error_message = "Error %s connecting %s:%s. %s." % \
(e.args[0], self.host, self.port, e.args[1])
raise ConnectionError(error_message)
def connect(self):
"Connects to the Redis server if not already connected"
if self._sock:
return
try:
sock = self._connect()
except socket.error:
e = sys.exc_info()[1]
raise ConnectionError(self._error_message(e))
self._sock = sock
try:
self.on_connect()
except RedisError:
# clean up after any error in on_connect
self.disconnect()
raise
def execute_command_and_read_response(
self, completer, command_name, *args, **options
):
"Execute a command and return a parsed response"
# === pre hook ===
# TRANSATION state chage
if command_name.upper() in ["EXEC", "DISCARD"]:
logger.debug(f"[After hook] Command is {command_name}, unset transaction.")
config.transaction = False
if command_name.upper() in ["ZSCAN", "ZPOPMAX", "ZPOPMIN"]:
config.withscores = True
try:
self.connection.send_command(command_name, *args)
resp = self.parse_response(
self.connection, completer, command_name, **options
)
# retry on timeout
except (ConnectionError, TimeoutError) as e:
self.connection.disconnect()
if not (self.connection.retry_on_timeout and isinstance(e, TimeoutError)):
raise
self.connection.send_command(command_name, *args)
resp = self.parse_response(
self.connection, completer, command_name, **options
)
except redis.exceptions.ExecAbortError:
config.transaction = False
raise
# === After hook ===
# SELECT db on AUTH
if command_name.upper() == "AUTH" and self.db:
select_result = self.execute_command_and_read_response(
completer, "SELECT", self.db
)
if nativestr(select_result) != "OK":
raise ConnectionError("Invalid Database")
elif command_name.upper() == "SELECT":
logger.debug("[After hook] Command is SELECT, change self.db.")
self.db = int(args[0])
if command_name.upper() == "MULTI":
logger.debug("[After hook] Command is MULTI, start transaction.")
config.transaction = True
return resp
def on_connect(self):
"Initialize the connection, authenticate and select a database"
self._parser.on_connect(self)
# if a password is specified, authenticate
if self.password:
# avoid checking health here -- PING will fail if we try
# to check the health prior to the AUTH
self.send_command('AUTH', self.password, check_health=False)
if nativestr(self.read_response()) != 'OK':
raise AuthenticationError('Invalid Password')
# if a database is specified, switch to it
if self.db:
self.send_command('SELECT', self.db)
if nativestr(self.read_response()) != 'OK':
raise ConnectionError('Invalid Database')
def send(self, command, redis_instance):
"Send ``command`` to the Redis server. Return the result."
self.connect(redis_instance)
try:
if MAJOR_VERSION >= 3:
command = command.encode()
self._sock.sendall(command)
except socket.error:
e = sys.exc_info()[1]
if e.args[0] == errno.EPIPE:
self.disconnect()
if len(e.args) == 1:
_errno, errmsg = 'UNKNOWN', e.args[0]
else:
_errno, errmsg = e.args
raise ConnectionError("Error %s while writing to socket. %s." % \
(_errno, errmsg))
