class PythonConnection(BaseConnection):
def read_response(self, command_name, catch_errors):
response = self.read()[:-2] # strip last two characters (\r\n)
if not response:
self.disconnect()
raise ConnectionError("Socket closed on remote end")
# server returned a null value
if response in ('$-1', '*-1'):
return None
byte, response = response[0], response[1:]
# server returned an error
if byte == '-':
if response.startswith('ERR '):
response = response[4:]
if response.startswith('LOADING '):
# If we're loading the dataset into memory, kill the socket
# so we re-initialize (and re-SELECT) next time.
self.disconnect()
response = response[8:]
raise ResponseError(response)
# single value
elif byte == '+':
return response
# int value
elif byte == ':':
return long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = length and self.read(length) or ''
self.read(2) # read the \r\n delimiter
return response
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
if not catch_errors:
return [
self.read_response(command_name, catch_errors)
for i in range(length)
]
else:
# for pipelines, we need to read everything,
# including response errors. otherwise we'd
# completely mess up the receive buffer
data = []
for i in range(length):
try:
data.append(
self.read_response(command_name, catch_errors))
except Exception, e:
data.append(e)
return data
raise InvalidResponse("Unknown response type for: %s" % command_name)
def read_response(self, command=None, index=0):
response = self._buffer.readline()
if not response:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error: %s, %s" %
(str(byte), str(response)))
# server returned an error
if byte == '-':
response = nativestr(response)
error = self.parse_error(response)
# if the error is a ConnectionError, raise immediately so the user
# is notified
if isinstance(error, ConnectionError):
raise error
# otherwise, we're dealing with a ResponseError that might belong
# inside a pipeline response. the connection's read_response()
# and/or the pipeline's execute() will raise this error if
# necessary, so just return the exception instance here.
return error
# single value
elif byte == '+':
pass
# int value
elif byte == ':':
response = long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
# TODO: IF command=HGETALL and it is value (odd indexed), then decode it
if command == 'HGETALL' and index % 2 == 1:
# print('Unpacking through here...')
# start_time = time.time()
# unpacker = msgpack.Unpacker(self._buffer, length)
response = self._buffer.read_and_decode(length)
# print('Time taken: {}'.format(time.time() - start_time))
else:
response = self._buffer.read(length)
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
response = [self.read_response(command, i) for i in xrange(length)]
if isinstance(response, bytes):
response = self.encoder.decode(response)
return response
def read_response(self):
response = yield self._buffer.readline()
if not response:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error: %s, %s" %
(str(byte), str(response)))
# server returned an error
if byte == '-':
response = nativestr(response)
error = self.parse_error(response)
# if the error is a ConnectionError, raise immediately so the user
# is notified
if isinstance(error, ConnectionError):
raise error
# otherwise, we're dealing with a ResponseError that might belong
# inside a pipeline response. the connection's read_response()
# and/or the pipeline's execute() will raise this error if
# necessary, so just return the exception instance here.
raise gen.Return(error)
# single value
elif byte == '+':
pass
# int value
elif byte == ':':
response = int(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
raise gen.Return(None)
response = yield self._buffer.read(length)
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
raise gen.Return(None)
response = []
for i in range(length):
res = yield self.read_response()
response.append(res)
if isinstance(response, bytes):
response = self.encoder.decode(response)
raise gen.Return(response)
def execute_command(self, *args, **kwargs):
pool = kwargs.pop('pool', None)
ask = kwargs.pop('ask', False)
if not pool:
if len(args) < 2:
pool = self.get_pool()
else:
key = str(args[1])
pool = self.get_pool(key=key)
try:
if ask:
pool.execute_command('ASKING')
ask = False
response = pool.execute_command(*args, **kwargs)
return response
except (ConnectionError, ResponseError) as e:
msg = str(e)
ask = msg.startswith('ASK ')
moved = msg.startswith('MOVED ')
num_resets = kwargs.pop('num_resets', 0)
if num_resets > self.max_resets:
raise ClusterError('Too many resets happened.')
# Disconnect current pool and remove pool from cluster
current_addr = pool.addr
pool.disconnect()
self.cluster_pools.pop(current_addr, None)
self.reset_slots()
num_resets += 1
kwargs.update({'num_resets': num_resets})
if ask or moved:
_, addr = self.redirect_info(msg)
if not addr or addr not in self.cluster_pools:
raise InvalidResponse('Invalid redirect node %s.' % addr)
pool = self.cluster_pools[addr]
kwargs.update({'ask': ask, 'pool': pool})
elif isinstance(e, ResponseError):
raise
return self.execute_command(*args, **kwargs)
def read_response(self):
response = self._buffer.readline()
if not response:
raise ConnectionError("Socket closed on remote end")
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error: %s, %s" %
(str(byte), str(response)))
# server returned an error
if byte == '-':
response = nativestr(response)
error = self.parse_error(response)
# if the error is a ConnectionError, raise immediately so the user
# is notified
if isinstance(error, ConnectionError):
raise error
# otherwise, we're dealing with a ResponseError that might belong
# inside a pipeline response. the connection's read_response()
# and/or the pipeline's execute() will raise this error if
# necessary, so just return the exception instance here.
return error
# single value
elif byte == '+':
pass
# int value
elif byte == ':':
response = long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = self._buffer.read(length)
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
response = [self.read_response() for i in xrange(length)]
if isinstance(response, bytes) and self.encoding:
response = response.decode(self.encoding)
return response
def read_response(self):
response = self.read()
if not response:
raise ConnectionError("Socket closed on remote end")
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error")
# server returned an error
if byte == '-':
if nativestr(response).startswith('LOADING '):
# if we're loading the dataset into memory, kill the socket
# so we re-initialize (and re-SELECT) next time.
raise ConnectionError("Redis is loading data into memory")
# if the error starts with ERR, trim that off
if nativestr(response).startswith('ERR '):
response = response[4:]
# *return*, not raise the exception class. if it is meant to be
# raised, it will be at a higher level.
return ResponseError(response)
# single value
elif byte == '+':
pass
# int value
elif byte == ':':
response = long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = self.read(length)
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
response = [self.read_response() for i in xrange(length)]
if isinstance(response, bytes) and self.encoding:
response = response.decode(self.encoding)
return response
def read_response(self):
try:
response = yield self._stream.read_until(SYM_CRLF)
except StreamClosedError:
raise socket.error(SERVER_CLOSED_CONNECTION_ERROR)
response = response[:-2]
if not response:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error: %s, %s" %
(str(byte), str(response)))
if byte == '-':
response = nativestr(response)
error = self.parse_error(response)
if isinstance(error, ConnectionError):
raise error
raise gen.Return(error)
elif byte == '+':
pass
elif byte == ':':
response = long(response)
elif byte == '$':
length = int(response)
if length == -1:
raise gen.Return(None)
response = yield self._stream.read_bytes(
length + 2) # make sure to read the '\r\n'
response = response[:-2]
elif byte == '*':
length = int(response)
if length == -1:
raise gen.Return(None)
response = []
for i in xrange(length):
part = yield self.read_response()
response.append(part)
if isinstance(response, bytes):
response = self.encoder.decode(response)
raise gen.Return(response)
def _parse_response(self, command_name):
conn = self.connection
response = conn.read().strip()
if not response:
self.connection.disconnect()
raise ConnectionError("Socket closed on remote end")
# server returned a null value
if response in ('$-1', '*-1'):
return None
byte, response = response[0], response[1:]
# server returned an error
if byte == '-':
if response.startswith('ERR '):
response = response[4:]
raise ResponseError(response)
# single value
elif byte == '+':
return response
# int value
elif byte == ':':
return int(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = length and conn.read(length) or ''
conn.read(2) # read the \r\n delimiter
return response
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
return [self._parse_response(command_name) for i in range(length)]
raise InvalidResponse("Unknown response type for: %s" % command_name)
def read_response(self):
response = self.read()
if not response:
raise ConnectionError("Socket closed on remote end")
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error")
# server returned an error
if byte == '-':
response = nativestr(response)
# *return*, not raise the exception class. if it is meant to be
# raised, it will be at a higher level.
return self.parse_error(response)
# single value
elif byte == '+':
pass
# int value
elif byte == ':':
response = long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = self.read(length)
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
response = [self.read_response() for i in xrange(length)]
if isinstance(response, bytes) and self.encoding:
response = response.decode(self.encoding)
return response
def read_response(self):
response = self.read()
if not response:
raise ConnectionError("Socket closed on remote end")
byte, response = response[0], response[1:]
# server returned an error
if byte == '-':
if response.startswith('ERR '):
response = response[4:]
return ResponseError(response)
if response.startswith('LOADING '):
# If we're loading the dataset into memory, kill the socket
# so we re-initialize (and re-SELECT) next time.
raise ConnectionError("Redis is loading data into memory")
# single value
elif byte == '+':
return response
# int value
elif byte == ':':
return long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = self.read(length)
return response
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
return [self.read_response() for i in xrange(length)]
raise InvalidResponse("Protocol Error")
class Redis(threading.local):
"""
Implementation of the Redis protocol.
This abstract class provides a Python interface to all Redis commands
and an implementation of the Redis protocol.
Connection and Pipeline derive from this, implementing how
the commands are sent and received to the Redis server
"""
RESPONSE_CALLBACKS = dict_merge(
string_keys_to_dict(
'AUTH DEL EXISTS EXPIRE MOVE MSETNX RENAMENX SADD SISMEMBER SMOVE '
'SETNX SREM ZADD ZREM', bool),
string_keys_to_dict(
'DECRBY INCRBY LLEN SCARD SDIFFSTORE SINTERSTORE SUNIONSTORE '
'ZCARD ZREMRANGEBYSCORE', int),
string_keys_to_dict(
# these return OK, or int if redis-server is >=1.3.4
'LPUSH RPUSH',
lambda r: isinstance(r, int) and r or r == 'OK'),
string_keys_to_dict('ZSCORE ZINCRBY',
lambda r: r is not None and float(r) or r),
string_keys_to_dict(
'FLUSHALL FLUSHDB LSET LTRIM MSET RENAME '
'SAVE SELECT SET SHUTDOWN', lambda r: r == 'OK'),
string_keys_to_dict('SDIFF SINTER SMEMBERS SUNION',
lambda r: r and set(r) or r),
string_keys_to_dict('ZRANGE ZRANGEBYSCORE ZREVRANGE',
zset_score_pairs),
{
'BGSAVE': lambda r: r == 'Background saving started',
'INFO': parse_info,
'LASTSAVE': timestamp_to_datetime,
'PING': lambda r: r == 'PONG',
'RANDOMKEY': lambda r: r and r or None,
'TTL': lambda r: r != -1 and r or None,
})
def __init__(self,
host='localhost',
port=6379,
db=0,
password=None,
charset='utf-8',
errors='strict'):
self.encoding = charset
self.errors = errors
self.select(host, port, db, password)
#### Legacty accessors of connection information ####
def _get_host(self):
return self.connection.host
host = property(_get_host)
def _get_port(self):
return self.connection.port
port = property(_get_port)
def _get_db(self):
return self.connection.db
db = property(_get_db)
def pipeline(self):
return Pipeline(self.connection, self.encoding, self.errors)
#### COMMAND EXECUTION AND PROTOCOL PARSING ####
def _execute_command(self, command_name, command, **options):
self.connection.send(command, self)
return self.parse_response(command_name, **options)
def execute_command(self, command_name, command, **options):
"Sends the command to the Redis server and returns it's response"
try:
return self._execute_command(command_name, command, **options)
except ConnectionError:
self.connection.disconnect()
return self._execute_command(command_name, command, **options)
def _parse_response(self, command_name, catch_errors):
conn = self.connection
response = conn.read().strip()
if not response:
self.connection.disconnect()
raise ConnectionError("Socket closed on remote end")
# server returned a null value
if response in ('$-1', '*-1'):
return None
byte, response = response[0], response[1:]
# server returned an error
if byte == '-':
if response.startswith('ERR '):
response = response[4:]
raise ResponseError(response)
# single value
elif byte == '+':
return response
# int value
elif byte == ':':
return int(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = length and conn.read(length) or ''
conn.read(2) # read the \r\n delimiter
return response
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
if not catch_errors:
return [
self._parse_response(command_name, catch_errors)
for i in range(length)
]
else:
# for pipelines, we need to read everything, including response errors.
# otherwise we'd completely mess up the receive buffer
data = []
for i in range(length):
try:
data.append(
self._parse_response(command_name, catch_errors))
except Exception, e:
data.append(e)
return data
raise InvalidResponse("Unknown response type for: %s" % command_name)
def read_response(self):
'''
Reads one line from the wire, and interprets it.
Example: the acknowledgment to an unsubscribe
from topic myTopic on the wire looks like this:
*3\r\n$11\r\nUNSUBSCRIBE\r\n$7\r\nmyTopic\r\n:1\r\n'
*3 # three items to follow
$11 # string of 11 chars
UNSUBSCRIBE
$7 # string of 7 chars
myTopic
:1 # one topic subscribed to now
Each line will cause a recursive call to this method
(see elif byte == '*' below).
Simpler calls will be individual elements, such
as ':12', which returns the integer 12.
These are the possible prefixes; each item
is followed by a \r\n, which is stripped
by SocketLineReader:
+<str> simple string
:<int> integer
$<n> string of length <n>
*<num> start of array with <num> elements
When the message to parse is the acknowledgment of
a SUBSCRIBE or UNSUBSCRIBE command, this method
will set() event self.unsubscribeAckEvent/self.unsubscribeAckEvent.
:return: response string
:rtype: string
'''
response = self._buffer.readline()
if not response:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error: %s, %s" %
(str(byte), str(response)))
# server returned an error
if byte == '-':
response = nativestr(response)
error = self.parse_error(response)
# if the error is a ConnectionError, raise immediately so the user
# is notified
if isinstance(error, ConnectionError):
raise error
# otherwise, we're dealing with a ResponseError that might belong
# inside a pipeline response. the connection's read_response()
# and/or the pipeline's execute() will raise this error if
# necessary, so just return the exception instance here.
return error
# simple-string: response holds result:
elif byte == '+':
pass
# int value
elif byte == ':':
response = long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
# Null string:
return None
response = self._buffer.read(length)
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
response = [self.read_response() for _ in xrange(length)]
if isinstance(response, bytes) and self.encoding:
response = response.decode(self.encoding)
#***********
#print('Response: %s' % byte + '|' + str(response))
#***********
return response