def test_update_slot(self):
clustermap = ClusterMap(self.seeds)
clustermap._update_slot(0, ['127.0.0.1', 7000], [['127.0.0.1', 7003]])
self.assertEqual(clustermap._map[0], {
'master': '127.0.0.1_7000',
'slave': '127.0.0.1_7003'
})
def __init__(self,
seeds=None,
database=0,
password=None,
encoding=None,
slave_ok=False,
conn_timeout=2,
read_timeout=2,
cluster_map=None):
super(ClusterClient, self).__init__()
if not bool(seeds) != bool(cluster_map):
raise PyRedisError(
u'Ether seeds or cluster_map has to be provided')
self._cluster = True
self._conns = dict()
self._conn_timeout = conn_timeout
self._read_timeout = read_timeout
self._encoding = encoding
self._password = password
self._database = database
self._slave_ok = slave_ok
if cluster_map:
self._map = cluster_map
else:
self._map = ClusterMap(seeds=seeds)
self._map_id = self._map.id
def test_update_slot(self):
clustermap = ClusterMap(self.seeds)
clustermap._update_slot(0, ['127.0.0.1', 7000], [['127.0.0.1', 7003]])
self.assertEqual(
clustermap._map[0],
{'master': '127.0.0.1_7000', 'slave': '127.0.0.1_7003'}
)
def test_get_slot_slaves(self):
clustermap = ClusterMap(self.seeds)
clustermap._fetch_map = Mock()
clustermap._fetch_map.return_value = self.map
clustermap.update(clustermap.id)
result = clustermap.get_slot('blarg')
self.assertEqual(result, '127.0.0.1_7001')
def test_hosts_master(self):
clustermap = ClusterMap(self.seeds)
clustermap._fetch_map = Mock()
clustermap._fetch_map.return_value = self.minimap
clustermap.update(clustermap.id)
self.assertEqual(
({'127.0.0.1_7000', '127.0.0.1_7001', '127.0.0.1_7002'}),
clustermap.hosts())
def test_hosts_slaves(self):
clustermap = ClusterMap(self.seeds)
clustermap._fetch_map = Mock()
clustermap._fetch_map.return_value = self.minimap
clustermap.update(clustermap.id)
self.assertEqual(
({'127.0.0.1_7003', '127.0.0.1_7004', '127.0.0.1_7005'}),
clustermap.hosts(slave=True))
def test__fetch_map_first_try_ok(self):
conn1 = Mock()
conn1.read.return_value = self.map
self.connection_mock.return_value = conn1
clustermap = ClusterMap(self.seeds)
result = clustermap._fetch_map()
self.assertEqual(result, self.map)
conn1.write.assert_called_with('CLUSTER', 'SLOTS')
self.assertTrue(conn1.close.called)
def test__fetch_map_first_try_ok(self):
conn1 = Mock()
conn1.read.return_value = self.map
self.connection_mock.return_value = conn1
clustermap = ClusterMap(self.seeds)
result = clustermap._fetch_map()
self.assertEqual(result, self.map)
conn1.write.assert_called_with(u'CLUSTER', u'SLOTS')
self.assertTrue(conn1.close.called)
def test__fetch_map_exception(self):
conn1 = Mock()
conn1.read.side_effect = PyRedisError
conn2 = Mock()
conn2.read.side_effect = PyRedisError
conn3 = Mock()
conn3.read.side_effect = PyRedisError
self.connection_mock.side_effect = [conn1, conn2, conn3]
clustermap = ClusterMap(self.seeds)
self.assertRaises(PyRedisError, clustermap._fetch_map)
self.connection_mock.assert_has_calls([
call(host=u'host1', port=12345, encoding=u'utf-8'),
call(host=u'host2', port=12345, encoding=u'utf-8'),
call(host=u'host3', port=12345, encoding=u'utf-8')
])
conn1.write.assert_called_with(u'CLUSTER', u'SLOTS')
self.assertTrue(conn1.close.called)
conn2.write.assert_called_with(u'CLUSTER', u'SLOTS')
self.assertTrue(conn2.close.called)
conn3.write.assert_called_with(u'CLUSTER', u'SLOTS')
self.assertTrue(conn3.close.called)
def __init__(
self,
seeds=None,
database=0,
password=None,
encoding=None,
slave_ok=False,
conn_timeout=2,
read_timeout=2,
cluster_map=None):
super().__init__()
if not bool(seeds) != bool(cluster_map):
raise PyRedisError('Ether seeds or cluster_map has to be provided')
self._cluster = True
self._conns = {}
self._conn_timeout = conn_timeout
self._read_timeout = read_timeout
self._encoding = encoding
self._password = password
self._database = database
self._slave_ok = slave_ok
if cluster_map:
self._map = cluster_map
else:
self._map = ClusterMap(seeds=seeds)
self._map_id = self._map.id
def test__fetch_map_second_try_ok(self):
conn1 = Mock()
conn1.read.side_effect = PyRedisError
conn2 = Mock()
conn2.read.return_value = self.map
self.connection_mock.side_effect = [conn1, conn2]
clustermap = ClusterMap(self.seeds)
result = clustermap._fetch_map()
self.assertEqual(result, self.map)
conn1.write.assert_called_with('CLUSTER', 'SLOTS')
self.assertTrue(conn1.close.called)
conn2.write.assert_called_with('CLUSTER', 'SLOTS')
self.assertTrue(conn2.close.called)
def test__fetch_map_second_try_ok(self):
conn1 = Mock()
conn1.read.side_effect = PyRedisError
conn2 = Mock()
conn2.read.return_value = self.map
self.connection_mock.side_effect = [conn1, conn2]
clustermap = ClusterMap(self.seeds)
result = clustermap._fetch_map()
self.assertEqual(result, self.map)
conn1.write.assert_called_with(u'CLUSTER', u'SLOTS')
self.assertTrue(conn1.close.called)
conn2.write.assert_called_with(u'CLUSTER', u'SLOTS')
self.assertTrue(conn2.close.called)
def test__fetch_map_first_try_ok(self):
conn1 = Mock()
conn1.read.return_value = self.map
self.connection_mock.return_value = conn1
clustermap = ClusterMap(self.seeds, password="******")
result = clustermap._fetch_map()
self.assertEqual(result, self.map)
conn1.write.assert_called_with(b'CLUSTER', b'SLOTS')
self.assertTrue(conn1.close.called)
self.connection_mock.assert_called_with(host='host1',
port=12345,
encoding='utf-8',
password='******')
def test_update(self):
clustermap = ClusterMap(self.seeds)
update = Mock()
id = clustermap.id
clustermap._update_slot = update
clustermap._fetch_map = Mock()
clustermap._fetch_map.return_value = self.minimap
id_new = clustermap.update(clustermap.id)
clustermap._update_slot.assert_has_calls([
call(0, [u'127.0.0.1', 7000], [[u'127.0.0.1', 7003]]),
call(1, [u'127.0.0.1', 7000], [[u'127.0.0.1', 7003]]),
call(2, [u'127.0.0.1', 7000], [[u'127.0.0.1', 7003]]),
call(3, [u'127.0.0.1', 7001], [[u'127.0.0.1', 7004]]),
call(4, [u'127.0.0.1', 7002], [[u'127.0.0.1', 7005]]),
call(5, [u'127.0.0.1', 7002], [[u'127.0.0.1', 7005]])
])
self.assertNotEqual(clustermap.id, id)
self.assertEqual(clustermap.id, id_new)
def test_update(self):
clustermap = ClusterMap(self.seeds)
update = Mock()
id = clustermap.id
clustermap._update_slot = update
clustermap._fetch_map = Mock()
clustermap._fetch_map.return_value = self.minimap
id_new = clustermap.update(clustermap.id)
clustermap._update_slot.assert_has_calls(
[
call(0, ['127.0.0.1', 7000], [['127.0.0.1', 7003]]),
call(1, ['127.0.0.1', 7000], [['127.0.0.1', 7003]]),
call(2, ['127.0.0.1', 7000], [['127.0.0.1', 7003]]),
call(3, ['127.0.0.1', 7001], [['127.0.0.1', 7004]]),
call(4, ['127.0.0.1', 7002], [['127.0.0.1', 7005]]),
call(5, ['127.0.0.1', 7002], [['127.0.0.1', 7005]])
]
)
self.assertNotEqual(clustermap.id, id)
self.assertEqual(clustermap.id, id_new)
def test_get_slot_slaves(self):
clustermap = ClusterMap(self.seeds)
clustermap._fetch_map = Mock()
clustermap._fetch_map.return_value = self.map
clustermap.update(clustermap.id)
result = clustermap.get_slot(u'blarg')
self.assertEqual(result, u'127.0.0.1_7001')
def test_hosts_slaves(self):
clustermap = ClusterMap(self.seeds)
clustermap._fetch_map = Mock()
clustermap._fetch_map.return_value = self.minimap
clustermap.update(clustermap.id)
self.assertEqual(
(set([u'127.0.0.1_7003', u'127.0.0.1_7004', u'127.0.0.1_7005'])),
clustermap.hosts(slave=True))
def test_hosts_master(self):
clustermap = ClusterMap(self.seeds)
clustermap._fetch_map = Mock()
clustermap._fetch_map.return_value = self.minimap
clustermap.update(clustermap.id)
self.assertEqual(
(set([u'127.0.0.1_7000', u'127.0.0.1_7001', u'127.0.0.1_7002'])),
clustermap.hosts())
class ClusterClient(commands.Connection, commands.Hash, commands.HyperLogLog,
commands.Key, commands.List, commands.Scripting,
commands.Set, commands.SSet, commands.String,
commands.Transaction):
u""" Base Client for Talking to Redis Cluster.
Inherits the following Commmand classes:
- commands.Connection,
- commands.Hash,
- commands.HyperLogLog,
- commands.Key,
- commands.List,
- commands.Scripting,
- commands.Set,
- commands.SSet,
- commands.String,
- commands.Transaction
:param seeds:
Accepts a list of seed nodes in this form: [('seed1', 6379), ('seed2', 6379), ('seed3', 6379)]
:type seeds: list
:param slave_ok:
Set to True if this Client should use slave nodes.
:type slave_ok: bool
:param database:
Select which db should be used for this connection
:type database: int
:param password:
Password used for authentication. If None, no authentication is done
:type password: str
:param encoding:
Convert result strings with this encoding. If None, no encoding is done.
:type encoding: str
:param conn_timeout:
Connect Timeout.
:type conn_timeout: float
:param read_timeout:
Read Timeout.
:type read_timeout: float
"""
def __init__(self,
seeds=None,
database=0,
password=None,
encoding=None,
slave_ok=False,
conn_timeout=2,
read_timeout=2,
cluster_map=None):
super(ClusterClient, self).__init__()
if not bool(seeds) != bool(cluster_map):
raise PyRedisError(
u'Ether seeds or cluster_map has to be provided')
self._cluster = True
self._conns = dict()
self._conn_timeout = conn_timeout
self._read_timeout = read_timeout
self._encoding = encoding
self._password = password
self._database = database
self._slave_ok = slave_ok
if cluster_map:
self._map = cluster_map
else:
self._map = ClusterMap(seeds=seeds)
self._map_id = self._map.id
def _cleanup_conns(self):
hosts = self._map.hosts(slave=self._slave_ok)
wipe = set()
for conn in self._conns.keys():
if conn not in hosts:
wipe.add(conn)
for conn in wipe:
self._conns[conn].close()
del self._conns[conn]
def _connect(self, sock):
host, port = sock.split(u'_')
client = Connection(host=host,
port=int(port),
conn_timeout=self._conn_timeout,
read_timeout=self._read_timeout,
encoding=self._encoding,
password=self._password,
database=self._database)
self._conns[sock] = client
def _get_slot_info(self, shard_key):
if self._map_id != self._map.id:
self._map_id = self._map.id
self._cleanup_conns()
try:
return self._map.get_slot(shard_key, self._slave_ok)
except KeyError:
self._map_id = self._map.update(self._map_id)
self._cleanup_conns()
return self._map.get_slot(shard_key, self._slave_ok)
@property
def closed(self):
return False
def execute(self, *args, **_3to2kwargs):
if 'retries' in _3to2kwargs:
retries = _3to2kwargs['retries']
del _3to2kwargs['retries']
else:
retries = 3
if 'asking' in _3to2kwargs:
asking = _3to2kwargs['asking']
del _3to2kwargs['asking']
else:
asking = False
if 'sock' in _3to2kwargs:
sock = _3to2kwargs['sock']
del _3to2kwargs['sock']
else:
sock = None
if 'shard_key' in _3to2kwargs:
shard_key = _3to2kwargs['shard_key']
del _3to2kwargs['shard_key']
else:
shard_key = None
u""" Execute arbitrary redis command.
:param args:
:type args: list, int, float
:param shard_key: (optional)
Should be set to the key name you try to work with.
Can not be used if sock is set.
:type shard_key: string
:param sock: (optional)
The string representation of a socket, the command should be executed against.
For example: "testhost_6379"
Can not be used if shard_key is set.
:type sock: string
:return: result, exception
"""
if not bool(shard_key) != bool(sock):
raise PyRedisError(u'Ether shard_key or sock has to be provided')
if not sock:
sock = self._get_slot_info(shard_key)
if sock not in self._conns.keys():
self._connect(sock)
try:
if asking:
self._conns[sock].write(u'ASKING', *args)
else:
self._conns[sock].write(*args)
return self._conns[sock].read()
except ReplyError, err:
errstr = unicode(err)
if retries <= 1 and (errstr.startswith(u'MOVED')
or errstr.startswith(u'ASK')):
raise PyRedisError(
u'Slot moved to often or wrong shard_key, giving up,')
if errstr.startswith(u'MOVED'):
if not shard_key:
raise ReplyError(
u'Explicitly set socket, but key does not belong to this redis: {0}'
.format(sock))
self._map_id = self._map.update(self._map_id)
self._cleanup_conns()
return self.execute(*args,
shard_key=shard_key,
retries=retries - 1)
elif errstr.startswith(u'ASK'):
sock = errstr.split()[2].replace(u':', u'_')
return self.execute(*args,
sock=sock,
retries=retries - 1,
asking=True)
else:
raise err
except (PyRedisConnError, PyRedisConnReadTimeout), err:
self._conns[sock].close()
del self._conns[sock]
self._map.update(self._map_id)
raise err
def test__make_str(self):
result = ClusterMap._make_str(['127.0.0.1', 7002])
self.assertEqual(result, '127.0.0.1_7002')
def test__make_str(self):
result = ClusterMap._make_str([u'127.0.0.1', 7002])
self.assertEqual(result, u'127.0.0.1_7002')
def test___init__(self):
clustermap = ClusterMap(self.seeds)
self.assertEqual(clustermap._map, {})
self.assertEqual(clustermap._seeds, deque(self.seeds))
def test___init__(self):
clustermap = ClusterMap(self.seeds, password='******')
self.assertEqual(clustermap._map, {})
self.assertEqual(clustermap._seeds, deque(self.seeds))
self.assertEqual(clustermap._password, 'blubber')
def __init__(self, seeds, slave_ok=False, password=None, **kwargs):
super().__init__(password=password, **kwargs)
self._map = ClusterMap(seeds=seeds, password=password)
self._slave_ok = slave_ok
self._cluster = True
def __init__(self, seeds, slave_ok=False, **kwargs):
super().__init__(**kwargs)
self._map = ClusterMap(seeds=seeds)
self._slave_ok = slave_ok
self._cluster = True
class ClusterClient(
commands.Connection,
commands.Hash,
commands.HyperLogLog,
commands.Key,
commands.List,
commands.Scripting,
commands.Set,
commands.SSet,
commands.String,
commands.Transaction
):
""" Base Client for Talking to Redis Cluster.
Inherits the following Commmand classes:
- commands.Connection,
- commands.Hash,
- commands.HyperLogLog,
- commands.Key,
- commands.List,
- commands.Scripting,
- commands.Set,
- commands.SSet,
- commands.String,
- commands.Transaction
:param seeds:
Accepts a list of seed nodes in this form: [('seed1', 6379), ('seed2', 6379), ('seed3', 6379)]
:type seeds: list
:param slave_ok:
Set to True if this Client should use slave nodes.
:type slave_ok: bool
:param database:
Select which db should be used for this connection
:type database: int
:param password:
Password used for authentication. If None, no authentication is done
:type password: str
:param encoding:
Convert result strings with this encoding. If None, no encoding is done.
:type encoding: str
:param conn_timeout:
Connect Timeout.
:type conn_timeout: float
:param read_timeout:
Read Timeout.
:type read_timeout: float
"""
def __init__(
self,
seeds=None,
database=0,
password=None,
encoding=None,
slave_ok=False,
conn_timeout=2,
read_timeout=2,
cluster_map=None):
super().__init__()
if not bool(seeds) != bool(cluster_map):
raise PyRedisError('Ether seeds or cluster_map has to be provided')
self._cluster = True
self._conns = {}
self._conn_timeout = conn_timeout
self._read_timeout = read_timeout
self._encoding = encoding
self._password = password
self._database = database
self._slave_ok = slave_ok
if cluster_map:
self._map = cluster_map
else:
self._map = ClusterMap(seeds=seeds)
self._map_id = self._map.id
def _cleanup_conns(self):
hosts = self._map.hosts(slave=self._slave_ok)
wipe = set()
for conn in self._conns.keys():
if conn not in hosts:
wipe.add(conn)
for conn in wipe:
self._conns[conn].close()
del self._conns[conn]
def _connect(self, sock):
host, port = sock.split('_')
client = Connection(
host=host, port=int(port),
conn_timeout=self._conn_timeout,
read_timeout=self._read_timeout,
encoding=self._encoding,
password=self._password,
database=self._database
)
self._conns[sock] = client
def _get_slot_info(self, shard_key):
if self._map_id != self._map.id:
self._map_id = self._map.id
self._cleanup_conns()
try:
return self._map.get_slot(shard_key, self._slave_ok)
except KeyError:
self._map_id = self._map.update(self._map_id)
self._cleanup_conns()
return self._map.get_slot(shard_key, self._slave_ok)
@property
def closed(self):
return False
def execute(self, *args, shard_key=None, sock=None, asking=False, retries=3):
""" Execute arbitrary redis command.
:param args:
:type args: list, int, float
:param shard_key: (optional)
Should be set to the key name you try to work with.
Can not be used if sock is set.
:type shard_key: string
:param sock: (optional)
The string representation of a socket, the command should be executed against.
For example: "testhost_6379"
Can not be used if shard_key is set.
:type sock: string
:return: result, exception
"""
if not bool(shard_key) != bool(sock):
raise PyRedisError('Ether shard_key or sock has to be provided')
if not sock:
sock = self._get_slot_info(shard_key)
if sock not in self._conns.keys():
self._connect(sock)
try:
if asking:
self._conns[sock].write('ASKING', *args)
else:
self._conns[sock].write(*args)
return self._conns[sock].read()
except ReplyError as err:
errstr = str(err)
if retries <= 1 and (errstr.startswith('MOVED') or errstr.startswith('ASK')):
raise PyRedisError('Slot moved to often or wrong shard_key, giving up,')
if errstr.startswith('MOVED'):
if not shard_key:
raise ReplyError('Explicitly set socket, but key does not belong to this redis: {0}'.format(sock))
self._map_id = self._map.update(self._map_id)
self._cleanup_conns()
return self.execute(*args, shard_key=shard_key, retries=retries-1)
elif errstr.startswith('ASK'):
sock = errstr.split()[2].replace(':', '_')
return self.execute(*args, sock=sock, retries=retries-1, asking=True)
else:
raise err
except (PyRedisConnError, PyRedisConnReadTimeout) as err:
self._conns[sock].close()
del self._conns[sock]
self._map.update(self._map_id)
raise err
