def redis_cluster():
redis_nodes = [{
"host": "192.168.0.6",
"port": "6066"
}, {
"host": "192.168.0.4",
"port": "6064"
}, {
"host": "192.168.0.7",
"port": "6061"
}, {
"host": "192.168.0.5",
"port": "6062"
}, {
"host": "192.168.0.3",
"port": "6065"
}, {
"host": "192.168.0.2",
"port": "6063"
}]
try:
redisconn = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print(e)
sys.exit(1)
redisconn.set('name', 'admin')
redisconn.set('age', 18)
print("name is: ", redisconn.get('name'))
print("age is: ", redisconn.get('age'))
def test_access_correct_slave_with_readonly_mode_client(sr):
"""
Test that the client can get value normally with readonly mode
when we connect to correct slave.
"""
# we assume this key is set on 127.0.0.1:7000(7003)
sr.set('foo16706', 'foo')
import time
time.sleep(1)
with patch.object(ClusterReadOnlyConnectionPool, 'get_node_by_slot') as return_slave_mock:
return_slave_mock.return_value = {
'name': '127.0.0.1:7003',
'host': '127.0.0.1',
'port': 7003,
'server_type': 'slave',
}
master_value = {'host': '127.0.0.1', 'name': '127.0.0.1:7000', 'port': 7000, 'server_type': 'master'}
with patch.object(
ClusterConnectionPool,
'get_master_node_by_slot',
return_value=master_value) as return_master_mock:
readonly_client = StrictRedisCluster(host="127.0.0.1", port=7000, readonly_mode=True)
assert b('foo') == readonly_client.get('foo16706')
assert return_master_mock.call_count == 0
readonly_client = StrictRedisCluster.from_url(url="redis://127.0.0.1:7000/0", readonly_mode=True)
assert b('foo') == readonly_client.get('foo16706')
assert return_master_mock.call_count == 0
class SmsRedis(object):
def __init__(self):
# self.host = conf['server']['redis_sms']['host']
self.host = conf['dev_server']['redis_sms']['host']
# self.port = conf['server']['redis_sms']['port']
self.port = conf['dev_server']['redis_sms']['port']
nodes = [{"host": self.host, "port": self.port}]
self.r = StrictRedisCluster(startup_nodes=nodes, decode_responses=True)
# x = self.r.get("user:sms:13116285391_5")
# print(x)
def get_value(self, phone, types=None, only_value=True):
kv = {}
if types is not None:
key = 'user:sms:' + phone + '_' + str(types)
# print(key)
try:
kv[str(phone) + '_' + str(types)] = self.r.get(key)
# print(kv)
except ResponseError:
pass
else:
for i in [1, 3, 4, 5, 6, 7, 8]:
key = 'user:sms:' + phone + '_' + str(i)
try:
if self.r.get(key) is not None:
kv[key] = self.r.get(key)
except ResponseError:
pass
if only_value:
return kv[str(phone) + '_' + str(types)]
else:
return kv
def redis_cluster():
redis_nodes = [{
'host': '192.168.118.15',
'port': 6379
}, {
'host': '192.168.118.15',
'port': 6380
}, {
'host': '192.168.118.15',
'port': 6381
}, {
'host': '192.168.118.15',
'port': 6382
}, {
'host': '192.168.118.15',
'port': 6383
}, {
'host': '192.168.118.15',
'port': 6384
}]
try:
redisconn = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print("Connect Error!")
sys.exit(1)
# redisconn.set('name', 'hkey')
# redisconn.set('age',18)
print("name is: ", redisconn.get('name'))
print("age is: ", redisconn.get('age'))
def redis_cluster():
redis_nodes = [{
'host': '10.101.104.132',
'port': 1321
}, {
'host': '10.101.104.132',
'port': 1322
}, {
'host': '10.101.104.132',
'port': 1323
}]
try:
redisconn = StrictRedisCluster(startup_nodes=redis_nodes,
decode_responses=True)
except Exception as e:
print("Connect Error!")
sys.exit(1)
#redisconn.set('name','admin')
#redisconn.set('age',18)
#print("name is: ", redisconn.get('name'))
#print("age is: ", redisconn.get('age'))
print(redisconn.get("creative:app:com.DreamonStudios.BouncyBasketball"))
print(redisconn.get('billing:ad:1'))
print(redisconn.hgetall('billing:creative:spent:2019040417:5'))
def test_access_correct_slave_with_readonly_mode_client(sr):
"""
Test that the client can get value normally with readonly mode
when we connect to correct slave.
"""
# we assume this key is set on 127.0.0.1:7000(7003)
sr.set('foo16706', 'foo')
import time
time.sleep(1)
with patch.object(ClusterReadOnlyConnectionPool, 'get_node_by_slot') as return_slave_mock:
return_slave_mock.return_value = {
'name': '127.0.0.1:7003',
'host': '127.0.0.1',
'port': 7003,
'server_type': 'slave',
}
master_value = {'host': '127.0.0.1', 'name': '127.0.0.1:7000', 'port': 7000, 'server_type': 'master'}
with patch.object(
ClusterConnectionPool,
'get_master_node_by_slot',
return_value=master_value) as return_master_mock:
readonly_client = StrictRedisCluster(host="127.0.0.1", port=7000, readonly_mode=True)
assert b('foo') == readonly_client.get('foo16706')
readonly_client = StrictRedisCluster.from_url(url="redis://127.0.0.1:7000/0", readonly_mode=True)
assert b('foo') == readonly_client.get('foo16706')
def redisCluster(classid, recordid):
startup_nodes = [{
'host': '192.168.40.112',
'port': '7000'
}, {
'host': '192.168.40.133',
'port': '7000'
}, {
'host': '192.168.40.141',
'port': '7000'
}, {
'host': '192.168.40.112',
'port': '7001'
}, {
'host': '192.168.40.133',
'port': '7001'
}, {
'host': '192.168.40.141',
'port': '7001'
}]
r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
data1 = 'music-class_room-' + str(classid)
data2 = 'music-class_record-' + str(recordid)
rs = r.get(data1)
rs1 = r.get(data2)
print rs
print rs1
# redisCluster(1814805, 12347)
class myRedisCluster(object):
conn = ''
redis_nodes = [
{
'host': '192.168.128.128',
'port': 6379
},
]
def __init__(self):
try:
self.conn = StrictRedisCluster(startup_nodes=self.redis_nodes)
except Exception as e:
print e
sys.exit(1)
def add(self, key, value):
self.conn.set(key, value)
def get(self, key):
self.conn.get(key)
def rem(self, key):
result = self.conn.delete(key)
if result != 0:
return 1
else:
return 0
def test_fix(self):
def migrate_one_slot(nodes, _):
if nodes[0].port == 7100:
source, target = nodes
else:
target, source = nodes
return [(source, target, 1)]
comm.create([('127.0.0.1', 7100)])
rc = StrictRedisCluster(startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
comm.join_cluster('127.0.0.1',
7100,
'127.0.0.1',
7101,
balance_plan=migrate_one_slot)
rc.set('h-893', 'I am in slot 0')
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
t7100 = Connection('127.0.0.1', 7100)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
t7100.execute('cluster', 'setslot', 0, 'importing', n7101.node_id)
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
self.assertEqual(16384, len(n7100.assigned_slots))
self.assertEqual(0, len(n7101.assigned_slots))
t7101 = Connection('127.0.0.1', 7101)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
self.assertEqual(16384, len(n7100.assigned_slots))
self.assertEqual(0, len(n7101.assigned_slots))
t7100.execute('cluster', 'setslot', 0, 'migrating', n7101.node_id)
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
comm.quit_cluster('127.0.0.1', 7101)
rc.delete('h-893')
comm.shutdown_cluster('127.0.0.1', 7100)
t7100.close()
t7101.close()
def test_start_with_max_slots_set(self):
comm.create([('127.0.0.1', 7100)], max_slots=7000)
rc = StrictRedisCluster(
startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
rc.set('key', 'value')
self.assertEqual('value', rc.get('key'))
rc.delete('key')
comm.shutdown_cluster('127.0.0.1', 7100)
comm.start_cluster_on_multi(
[('127.0.0.1', 7100), ('127.0.0.1', 7101)], max_slots=7000)
rc = StrictRedisCluster(
startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
rc.set('key', 'value')
self.assertEqual('value', rc.get('key'))
rc.delete('key')
comm.quit_cluster('127.0.0.1', 7101)
comm.shutdown_cluster('127.0.0.1', 7100)
def redis_cluster():
redis_nodes = [{
"host": "192.168.1.161",
"port": 7000
}, {
"host": "192.168.1.161",
"port": 7001
}, {
"host": "192.168.1.161",
"port": 7002
}, {
"host": "192.168.1.147",
"port": 7000
}, {
"host": "192.168.1.147",
"port": 7001
}, {
"host": "192.168.1.147",
"port": 7002
}]
try:
redisconn = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print("connect error!!")
sys.exit(1)
redisconn.set("ni", "hao")
print(redisconn.get("ni"))
def test_quit_problems(self):
comm.start_cluster('127.0.0.1', 7100)
comm.join_cluster('127.0.0.1', 7100, '127.0.0.1', 7101)
comm.replicate('127.0.0.1', 7100, '127.0.0.1', 7102)
time.sleep(1)
rc = StrictRedisCluster(startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
for i in xrange(20):
rc.set('key_%s' % i, 'value_%s' % i)
for i in xrange(20):
self.assertEqual('value_%s' % i, rc.get('key_%s' % i))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(3, len(nodes))
self.assertEqual(range(8192),
nodes[('127.0.0.1', 7101)].assigned_slots)
self.assertEqual(range(8192, 16384),
nodes[('127.0.0.1', 7100)].assigned_slots)
for i in xrange(20):
rc.delete('key_%s' % i)
self.assertRaisesRegexp(ValueError, '^The master still has slaves$',
comm.quit_cluster, '127.0.0.1', 7100)
comm.quit_cluster('127.0.0.1', 7102)
comm.quit_cluster('127.0.0.1', 7101)
self.assertRaisesRegexp(ValueError, '^This is the last node',
comm.quit_cluster, '127.0.0.1', 7100)
comm.shutdown_cluster('127.0.0.1', 7100)
def get_doc_data():
timestamp = time.time()
ltime = time.localtime(timestamp)
date = time.strftime("%Y%m%d", ltime)
doc_list = get_doc_list(date)
redis_nodes = [{
'host': '10.200.131.32',
'port': 6101
}, {
'host': '10.200.131.31',
'port': 6102
}, {
'host': '10.200.131.27',
'port': 6101
}, {
'host': '10.200.131.28',
'port': 6102
}]
r = StrictRedisCluster(startup_nodes=redis_nodes)
key_prefix = 'headline_'
filepath = 'data/doc/doc_data_' + date
fw = open(filepath, 'w')
doc_data = []
for d_key in doc_list:
key = key_prefix + d_key
doc = r.get(key)
if isinstance(doc, basestring):
fw.write(doc + '\n')
doc_data.append(doc)
fw.close()
return doc_list, doc_data
def redis_cluster_get_verifyCode():
send_SMS()
redis_nodes = [{
'host': '192.168.1.162',
'port': 7001
}, {
'host': '192.168.1.162',
'port': 7002
}, {
'host': '192.168.1.162',
'port': 7003
}, {
'host': '192.168.1.162',
'port': 7004
}, {
'host': '192.168.1.162',
'port': 7005
}]
try:
rediscoon = StrictRedisCluster(startup_nodes=redis_nodes,
decode_responses=True)
except Exception as e:
print(e)
verifyCode = rediscoon.get('SMS_VERIFY_' + mobile)
if verifyCode:
return verifyCode
def redis_time():
redis_nodes = [{
'host': '10.12.28.222',
'port': 6380
}, {
'host': '10.12.28.222',
'port': 6381
}, {
'host': '10.12.28.224',
'port': 6380
}, {
'host': '10.12.28.224',
'port': 6381
}, {
'host': '10.12.28.227',
'port': 6380
}, {
'host': '10.12.28.227',
'port': 6381
}]
try:
r = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print("connect error %s" % e)
# print(r.set('thoth-ai','过期测试'))
#
# print(r.expire('thoth-ai',10)) # 设定key过期时间 10s
print(r.get('thoth-ai:robotId:'))
def get_video_data(date):
video_keys = get_list(date)
redis_nodes = [{
'host': '10.200.131.32',
'port': 6101
}, {
'host': '10.200.131.31',
'port': 6102
}, {
'host': '10.200.131.27',
'port': 6101
}, {
'host': '10.200.131.28',
'port': 6102
}]
r = StrictRedisCluster(startup_nodes=redis_nodes)
filepath = '../normal_knn/jobs/data/doc/doc_data_' + date
fw = open(filepath, 'w')
key_prefix = 'headline_'
for v_key in video_keys:
key = key_prefix + v_key
video = r.get(key)
if isinstance(video, basestring):
fw.write(video + '\n')
fw.close()
def get_peer_info_from_redis(peer_ids):
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
if type(peer_ids) is not list:
peer_ids = [peer_ids]
peer_info_list = list()
peer_info_keys = [
"peer_id", "version", "natType", "publicIP", "publicPort", "privateIP",
"privatePort", "stunIP"
]
for peer_id in peer_ids:
try:
# 从redis-cluster中获取节点PNIC信息
pnic = rc.get("PNIC_{0}".format(peer_id))
pnic = json.loads(pnic)
peer_info = dict()
for k in peer_info_keys:
peer_info[k] = pnic[k]
# 当播放节点与供源节点使用线上stun探测natType相加≥7无法穿透而播放节点与供源节点又处于同一局域网时，
# 强行将seedslist中供源节点的natType该为0或1即可使其穿透成功
# peer_info.update({"natType": 0})
peer_info_list.append(peer_info)
except:
pass
return peer_info_list
def test_access_correct_slave_with_readonly_mode_client(sr):
"""
Test that the client can get value normally with readonly mode
when we connect to correct slave.
"""
# we assume this key is set on 127.0.0.1:7000(7003)
sr.set("foo16706", "foo")
import time
time.sleep(1)
with patch.object(ClusterReadOnlyConnectionPool, "get_node_by_slot") as return_slave_mock:
return_slave_mock.return_value = {
"name": "127.0.0.1:7003",
"host": "127.0.0.1",
"port": 7003,
"server_type": "slave",
}
master_value = {"host": "127.0.0.1", "name": "127.0.0.1:7000", "port": 7000, "server_type": "master"}
with patch.object(
ClusterConnectionPool, "get_master_node_by_slot", return_value=master_value
) as return_master_mock:
readonly_client = StrictRedisCluster(host="127.0.0.1", port=7000, readonly_mode=True)
assert b("foo") == readonly_client.get("foo16706")
assert return_master_mock.call_count == 0
def use_cluster():
# 设置uabntud的桥接ip地址即可
startup_nodes = [
{
"host": "192.168.75.132",
"post": 7001
},
{
"host": "192.168.75.132",
"post": 7002
},
{
"host": "192.168.75.132",
"post": 7003
},
]
# 1.创建集群对象
cluster = StrictRedisCluster(startup_nodes=startup_nodes,
decode_responses=True)
# 2.设置值
cluster.set("name", "laowang")
# 3.获取值
print(cluster.get("name"))
class selectclass:
def __init__(self):
self.rc = StrictRedisCluster(startup_nodes=config.startup_nodes)
def _select(self, key_type, kwargs):
partition = str(random.randint(1, kwargs['partition']))
_key_name = key_type + '_r_test_' + partition + ':00' + str(
random.randint(1, kwargs['key_num']))
if key_type == 'string':
self.rc.get(_key_name)
elif key_type == 'hash':
_filed = str(random.randint(1, int(
kwargs['filed_num']))) + ':value_test'
self.rc.hget(_key_name, _filed)
elif key_type == 'list':
_num_1 = random.randint(1, int(kwargs['filed_num']))
_num_2 = random.randint(1, int(kwargs['filed_num']))
if _num_1 < _num_2:
self.rc.lrange(_key_name, _num_1, _num_2)
else:
self.rc.lrange(_key_name, _num_2, _num_1)
elif key_type == 'set':
self.rc.srandmember(_key_name)
else:
exit()
def _update(self, key_type, kwargs):
_value = ''.join(
map(lambda xx: (hex(ord(xx))[2:]),
os.urandom(random.randint(1, 200) / 2)))
partition = str(random.randint(1, kwargs['partition']))
_key_name = key_type + '_r_test_' + partition + ':00' + str(
random.randint(1, kwargs['key_num']))
if key_type == 'string':
self.rc.set(_key_name, _value)
elif key_type == 'hash':
_filed_name = str(random.randint(
1, kwargs['filed_num'])) + ':_value_test'
self.rc.hset(_key_name, _filed_name, _value)
elif key_type == 'list':
_filed_int = random.randint(0, kwargs['filed_num'] - 1)
self.rc.lset(_key_name, _filed_int, _value)
elif key_type == 'set':
self.rc.spop(_key_name)
self.rc.sadd(_key_name, _value)
else:
exit()
class RedisEngine:
@property
def engine(self):
return self.redis_eng
def __init__(self, redis_host, redis_port, redis_password, db=0):
if isinstance(redis_host, list) and len(redis_host) == 1:
redis_host = redis_host[0]
if isinstance(redis_host, str):
conn_pool = ConnectionPool(host=redis_host,
port=redis_port,
db=db,
password=redis_password)
self.redis_eng = StrictRedis(
connection_pool=conn_pool,
max_connections=10,
)
elif isinstance(redis_host, list):
if isinstance(redis_port, int):
startup_nodes = [{
"host": host,
"port": redis_port
} for host in redis_host]
elif isinstance(redis_port, list):
startup_nodes = [{
"host": host,
"port": port
} for host, port in zip(redis_host, redis_port)]
self.redis_eng = StrictRedisCluster(startup_nodes=startup_nodes,
password=redis_password)
def lpush(self, key, val):
self.redis_eng.lpush(key, val)
def lpop(self, key):
return self.redis_eng.lpop(key)
def rpush(self, key, val):
self.redis_eng.rpush(key, val)
def rpop(self, key):
return self.redis_eng.rpop(key)
def add_task(self, key, val, **kwargs):
self.redis_eng.lpush(key, val)
def get_task(self, key):
return self.rpop(key)
def save_result(self, key, val, expire_time):
res = self.redis_eng.set(key, val, nx=False, px=expire_time)
if not res:
print("id : %s has exists!" % key)
def get_result(self, key):
res = self.redis_eng.get(key)
if res is not None:
self.redis_eng.delete(key)
return res
class PullProxy(object):
startup_nodes = [
{
"host": "192.168.50.144",
"port": 6003
},
{
"host": "192.168.50.144",
"port": 6004
},
{
"host": "192.168.50.181",
"port": 6003
},
{
"host": "192.168.50.181",
"port": 6004
},
{
"host": "192.168.50.200",
"port": 6003
},
{
"host": "192.168.50.200",
"port": 6004
},
]
def __init__(self):
self.key = 'sdg_spider_proxy'
self.rc = StrictRedisCluster(startup_nodes=self.startup_nodes,
decode_responses=True)
@property
def proxy(self):
return self.rc.get(self.key)
@proxy.setter
def proxy(self, value):
self.rc.set(self.key, value)
print('end pull!')
def pull(self):
while True:
try:
print('start pull!')
sf = paramiko.Transport("%s:%s" % (SFTP_HOST, SFTP_PORT))
sf.connect(username=SFTP_USERNAME, password=SFTP_PASSWORD)
sftp = paramiko.SFTPClient.from_transport(sf)
sftp.get(REMOTE, LOCAL)
sf.close()
with open(LOCAL) as f:
value = f.readline()
self.proxy = value
print('proxy server: ', value)
time.sleep(1)
except Exception as e:
print(e)
def fetch_test_redis():
""" Connect to a Redis cluster """
startup_nodes = [{'host': 'hmscloud-dyn-cache.5evooa.clustercfg.euw1.cache.amazonaws.com', 'port': '6379'}]
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True, skip_full_coverage_check=True)
keys = {}
key = '1020_1540442262000_918359a632f442e18242b0db9b0586b1_DEV-336-62964'
print('Key: ' + key)
print("Value: " + str(rc.get(key)))
def test_redis_cluster():
cluster_id = 0
ip_list = config.get_node_ip_list(cluster_id)
port_list = config.get_master_port_list(cluster_id)
startup_nodes = utils.get_ip_port_dict_list(ip_list, port_list)
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
d = {'k': {'k2': [1, 2, 3]}}
# s = json.dumps(d)
rc.set('cols', d)
print(rc.get('cols'))
def create_client(self):
if config.get("IS_REDIS_CLUSTER"):
redis_client = StrictRedisCluster(startup_nodes=self.session_node, encoding="utf-8", decode_responses=True,\
skip_full_coverage_check=True)
else:
redis_client = StrictRedis(host=self.redis_host, port=self.redis_port, encoding="utf-8", decode_responses=True)
try:
redis_client.get("None")
logger.info("Connected to redis")
except RedisClusterExceptions.RedisClusterException as err:
logger.error(f"Error connecting to redis cluster\n {err}")
raise
except RedisExceptions.ConnectionError as err:
logger.error(f"Error connecting to redis\n {err}")
raise
return redis_client
def test_api(self):
comm.start_cluster('127.0.0.1', 7100)
comm.join_cluster('127.0.0.1', 7100, '127.0.0.1', 7101)
comm.replicate('127.0.0.1', 7100, '127.0.0.1', 7102)
time.sleep(1)
rc = StrictRedisCluster(
startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
for i in range(20):
rc.set('key_%s' % i, 'value_%s' % i)
for i in range(20):
self.assertEqual('value_%s' % i, rc.get('key_%s' % i))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(3, len(nodes))
self.assertEqual(
list(range(8192)), nodes[('127.0.0.1', 7101)].assigned_slots)
self.assertEqual(
list(range(8192, 16384)), nodes[('127.0.0.1',
7100)].assigned_slots)
comm.quit_cluster('127.0.0.1', 7101)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(
list(range(16384)), nodes[('127.0.0.1', 7100)].assigned_slots)
for i in range(20):
self.assertEqual('value_%s' % i, rc.get('key_%s' % i))
for i in range(20):
rc.delete('key_%s' % i)
comm.quit_cluster('127.0.0.1', 7102)
comm.shutdown_cluster('127.0.0.1', 7100)
class RedisManager:
def __init__(self, redis_server, pw=None):
#self.redis_client_ = redis.Redis(host='127.0.0.1', port=9966)
if pw:
pool = ClusterConnectionPool(startup_nodes=redis_server, password=pw, skip_full_coverage_check=True, decode_responses=True)
self.redis_client_ = StrictRedisCluster(connection_pool=pool)
else:
pool = ClusterConnectionPool(startup_nodes=redis_server, skip_full_coverage_check=True, decode_responses=True)
self.redis_client_ = StrictRedisCluster(connection_pool=pool)
def Set(self, k, v, is_str, expire=None):
if not is_str:
v = json.dumps(v)
return self.redis_client_.set(k, v, ex=expire)
def Get(self, k):
return self.redis_client_.get(k)
def Delete(self, k):
return self.redis_client_.delete(k)
def HashMultiSet(self, k, d, expire=None):
self.redis_client_.hmset(k, d)
if expire:
self.redis_client_.expire(k, expire)
def HashGetAll(self, k):
return self.redis_client_.hgetall(k)
def Pushback(self, k, v, expire=None):
self.redis_client_.rpush(k, v)
if expire:
self.redis_client_.expire(k, expire)
def SetList(self, k, l, expire=None):
self.redis_client_.rpush(k, *l)
if expire:
self.redis_client_.expire(k, expire)
def SetSet(self, k, v, expire=None):
self.redis_client_.sadd(k, v)
if expire:
self.redis_client_.expire(k, expire)
def SortSetSet(self, k, v, expire=None):
self.redis_client_.zadd(k, v[0], v[1])
if expire:
self.redis_client_.expire(k, expire)
def Handle(self):
return self.redis_client_
def redis_cluster():
'''集群操作'''
redis_nodes = [{
'host': '10.12.28.222',
'port': 6380
}, {
'host': '10.12.28.222',
'port': 6381
}, {
'host': '10.12.28.224',
'port': 6380
}, {
'host': '10.12.28.224',
'port': 6381
}, {
'host': '10.12.28.227',
'port': 6380
}, {
'host': '10.12.28.227',
'port': 6381
}]
try:
r = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print("connect error %s" % e)
# string 操作
r.set('thoth:thoth-ai:robot:1', 'kk')
# r.delete('thoth:thoth-ai:robot:1')
print("name is", r.get('thoth:thoth-ai:robot:1'))
# list 操作
r.lpush('thoth:thoth-ai:robot:2', [[1, 2, 3], [2, 3, 4]])
print('list len:', r.llen("thoth:thoth-ai:robot:2")) # list size
print("list ", r.lindex('thoth:thoth-ai:robot:2', 0))
# hash 操作
r.hset('thoth:thoth-ai:robot:3', 'private_vector', [[1, 2, 3], [2, 3, 4]])
r.hset('thoth:thoth-ai:robot:3', 'public_vector', [['4', 3, 2], [0, 1, 1]])
pv = r.hget(
'thoth:thoth-ai:robot:3',
'public_vector',
)
print('hash.robot3.public_vector:', pv)
aaa = pv.decode('utf-8')
print(type(aaa), aaa)
b = eval(aaa) # eval 函数妙用：将string‘[1,2,3]’--->list [1,2,3]
print(type(b), b)
def threadedClientWithCluster(cid, clients):
hclients = [{'host': c[0], 'port': c[1]} for c in clients]
credis = StrictRedisCluster(startup_nodes=hclients)
logging.debug('client %d connected to redis servers %s' % (cid, hclients))
count = 0
while True:
time.sleep(1)
count += 1
print 'client: %d, servers value are ' % (cid, ),
credis.set(cid, count)
v = credis.get(cid)
print v,
print "end"
def test_start_with_max_slots_set(self):
comm.create([('127.0.0.1', 7100)], max_slots=7000)
rc = StrictRedisCluster(startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
rc.set('key', 'value')
self.assertEqual('value', rc.get('key'))
rc.delete('key')
comm.shutdown_cluster('127.0.0.1', 7100)
comm.start_cluster_on_multi([('127.0.0.1', 7100), ('127.0.0.1', 7101)],
max_slots=7000)
rc = StrictRedisCluster(startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
rc.set('key', 'value')
self.assertEqual('value', rc.get('key'))
rc.delete('key')
comm.quit_cluster('127.0.0.1', 7101)
comm.shutdown_cluster('127.0.0.1', 7100)
def demo():
startup_nodes = [
{"host": "172.16.28.143", "port": 7000},
{"host": "172.16.28.143", "port": 7001},
{"host": "172.16.28.143", "port": 7002}
]
src = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
try:
result = src.set('name', 'laowang')
print(result)
value = src.get('name')
print(value)
except Exception as e:
print(e)
class RedisClusterTest(unittest.TestCase):
"""
集群测试
"""
def setUp(self):
self.redis_nodes = [
{
'host': '127.0.0.1',
'port': '6381'
},
{
'host': '127.0.0.1',
'port': '6382'
},
{
'host': '127.0.0.1',
'port': '6383'
},
{
'host': '127.0.0.1',
'port': '6384'
},
{
'host': '127.0.0.1',
'port': '6385'
},
{
'host': '127.0.0.1',
'port': '6386'
},
]
self.rc = StrictRedisCluster(startup_nodes=self.redis_nodes,
decode_responses=True)
def test_cluster(self):
"""
测试集群操作
:return:
"""
value = 'bar'
self.rc.set('foo', value)
result = self.rc.get('foo')
self.assertEqual(value, result)
def tearDown(self):
pass
def conn_redis_cluster():
redis_nodes = [{
'host': '192.168.100.8',
'port': 6379
}, {
'host': '192.168.100.9',
'port': 6380
}]
try:
redisconn = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print('error', e)
sys.exit()
print(redisconn.get('name1'))
print(redisconn.get('name2'))
def test_quit_problems(self):
comm.start_cluster('127.0.0.1', 7100)
comm.join_cluster('127.0.0.1', 7100, '127.0.0.1', 7101)
comm.replicate('127.0.0.1', 7100, '127.0.0.1', 7102)
time.sleep(1)
rc = StrictRedisCluster(
startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
for i in range(20):
rc.set('key_%s' % i, 'value_%s' % i)
for i in range(20):
self.assertEqual('value_%s' % i, rc.get('key_%s' % i))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(3, len(nodes))
self.assertEqual(
list(range(8192)), nodes[('127.0.0.1', 7101)].assigned_slots)
self.assertEqual(
list(range(8192, 16384)), nodes[('127.0.0.1',
7100)].assigned_slots)
for i in range(20):
rc.delete('key_%s' % i)
six.assertRaisesRegex(self, ValueError,
'^The master still has slaves$',
comm.quit_cluster, '127.0.0.1', 7100)
comm.quit_cluster('127.0.0.1', 7102)
comm.quit_cluster('127.0.0.1', 7101)
six.assertRaisesRegex(self, ValueError, '^This is the last node',
comm.quit_cluster, '127.0.0.1', 7100)
comm.shutdown_cluster('127.0.0.1', 7100)
class RedisCluster(object):
def __init__(self, redis_nodes):
self.cluster = StrictRedisCluster(startup_nodes=redis_nodes)
# 无差别的方法
def set(self, name, value, ex=None, px=None, nx=False, xx=False):
return self.cluster.set(name, value, ex, px, nx, xx)
# 无差别的方法
def get(self, name):
return self.cluster.get(name)
# 扇形发送的命令
def cluster_info(self):
return self.cluster.cluster_info()
# 重写StrictRedis的方法
def mset(self, *args, **kwargs):
return self.cluster.mset(args, kwargs)
# 重写StrictRedis的方法
def mget(self, keys, *args):
return self.cluster.mget(keys, args)
lines = f.readlines()
startup_nodes = []
for ip in lines:
startup_nodes.append({"host": ip[:-1], "port": "6379"})
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
print "Starting timer..."
startTotal = start = time.time()
key_pair_values= {}
for i in range(operations):
key_pair_values[string_generator(10)] = string_generator(90)
for key in key_pair_values:
rc.set(key, key_pair_values[key])
print "Insert Time:",time.time() - start,"seconds"
start = time.time()
for key in key_pair_values:
rc.get(key)
print "Lookup Time:",time.time() - start,"seconds"
start = time.time()
for key in key_pair_values:
rc.delete(key)
print "Delete Time:",time.time() - start,"seconds"
print "Overall Time:",time.time() - startTotal,"seconds"
from rediscluster.cluster_mgt import RedisClusterMgt
from rediscluster import StrictRedisCluster
startup_nodes = [{"host":"127.0.0.1","port":"7000"}, {"host":"127.0.0.1","port":"7001"}]
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
rc.set("foo", "bar")
print(rc.get("foo"))
def test_api(self):
comm.create([('127.0.0.1', 7100)])
rc = StrictRedisCluster(
startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
rc.set('key', 'value')
self.assertEqual('value', rc.get('key'))
comm.join_cluster('127.0.0.1', 7100, '127.0.0.1', 7101)
for i in range(20):
rc.set('key_%s' % i, 'value_%s' % i)
for i in range(20):
self.assertEqual('value_%s' % i, rc.get('key_%s' % i))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
self.assertEqual(
list(range(8192)), nodes[('127.0.0.1', 7101)].assigned_slots)
self.assertEqual(
list(range(8192, 16384)), nodes[('127.0.0.1',
7100)].assigned_slots)
comm.migrate_slots('127.0.0.1', 7100, '127.0.0.1', 7101, [8192])
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
self.assertEqual(
list(range(8193)), nodes[('127.0.0.1', 7101)].assigned_slots)
self.assertEqual(
list(range(8193, 16384)), nodes[('127.0.0.1',
7100)].assigned_slots)
comm.migrate_slots('127.0.0.1', 7100, '127.0.0.1', 7101,
[8193, 8194, 8195])
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
self.assertEqual(
list(range(8196)), nodes[('127.0.0.1', 7101)].assigned_slots)
self.assertEqual(
list(range(8196, 16384)), nodes[('127.0.0.1',
7100)].assigned_slots)
six.assertRaisesRegex(self, ValueError, 'Not all slot held by',
comm.migrate_slots, '127.0.0.1', 7100,
'127.0.0.1', 7101, [8192])
six.assertRaisesRegex(self, ValueError, 'Not all slot held by',
comm.migrate_slots, '127.0.0.1', 7100,
'127.0.0.1', 7101, [8195, 8196])
six.assertRaisesRegex(
self, ValueError, 'Two nodes are not in the same cluster',
comm.migrate_slots, '127.0.0.1', 7100, '127.0.0.1', 7102, [8196])
comm.quit_cluster('127.0.0.1', 7100)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(1, len(nodes))
self.assertEqual(0, len(nodes[('127.0.0.1', 7100)].assigned_slots))
nodes = base.list_nodes('127.0.0.1', 7101)
self.assertEqual(1, len(nodes))
self.assertEqual(
list(range(16384)), nodes[('127.0.0.1', 7101)].assigned_slots)
rc = StrictRedisCluster(
startup_nodes=[{
'host': '127.0.0.1',
'port': 7101
}],
decode_responses=True)
for i in range(20):
self.assertEqual('value_%s' % i, rc.get('key_%s' % i))
self.assertEqual('value', rc.get('key'))
six.assertRaisesRegex(self, RedisStatusError, 'still contains keys',
comm.shutdown_cluster, '127.0.0.1', 7101)
rc.delete('key', *['key_%s' % i for i in range(20)])
comm.shutdown_cluster('127.0.0.1', 7101)
six.assertRaisesRegex(
self, RedisClusterException,
'All slots are not covered after query all startup_nodes. .*',
rc.get, 'key')
def test_fix(self):
def migrate_one_slot(nodes, _):
if nodes[0].port == 7100:
source, target = nodes
else:
target, source = nodes
return [(source, target, 1)]
comm.create([('127.0.0.1', 7100)])
rc = StrictRedisCluster(
startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
comm.join_cluster(
'127.0.0.1',
7100,
'127.0.0.1',
7101,
balance_plan=migrate_one_slot)
rc.set('h-893', 'I am in slot 0')
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
t7100 = Connection('127.0.0.1', 7100)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
t7100.execute('cluster', 'setslot', 0, 'importing', n7101.node_id)
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
self.assertEqual(16384, len(n7100.assigned_slots))
self.assertEqual(0, len(n7101.assigned_slots))
t7101 = Connection('127.0.0.1', 7101)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(2, len(nodes))
n7100 = nodes[('127.0.0.1', 7100)]
n7101 = nodes[('127.0.0.1', 7101)]
self.assertEqual(16384, len(n7100.assigned_slots))
self.assertEqual(0, len(n7101.assigned_slots))
t7100.execute('cluster', 'setslot', 0, 'migrating', n7101.node_id)
comm.fix_migrating('127.0.0.1', 7100)
self.assertEqual('I am in slot 0', rc.get('h-893'))
comm.quit_cluster('127.0.0.1', 7101)
rc.delete('h-893')
comm.shutdown_cluster('127.0.0.1', 7100)
t7100.close()
t7101.close()
pip install redis-py-cluster
redis-py-cluster : https://github.com/Grokzen/redis-py-cluster/tree/1.0.0
"""
startup_nodes = [{"host": "127.0.0.1", "port": "6379"}]
# This line initialize a redis cluster client obj.
# Will raise RedisClusterException on connection failed.
# The 'decode_responses=True' is needed for Python3
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
# For Redis command per call, please refer:
# https://redis-py.readthedocs.org/en/latest/
maxacc = rc.get('UniqueID:Account')
print('Maximum Account ID is ' + maxacc)
# Demo of pipeline (batch) usage
rcp = rc.pipeline(transaction=False)
rcp.command_stack = []
curacci = 133589
maxacci = int(maxacc)
for acc in range(curacci, maxacci+1):
rcp.command_stack.append((['HGETALL', 'Account:-' + str(acc)], {}))
print('Pipelining ' + str(len(rcp.command_stack)) + ' Redis queries.')
# A list of dict. One dict is a HGETALL result of an account.
result = rcp.execute()
u = randint(0,len(k)-1)
#Insert into redis
start = current_milli_time()
r.delete(name + str(k[u]))
end = current_milli_time()
print(str(end-start)+'ms del ' + str(k[u]))
deletes.append(k[u])
del(sessions[k[u]])
except (ClusterError, ConnectionError):
print('Cluster Timeout')
#Compare redis sessions to
for key in sessions.keys():
a = sessions[key]
result = r.get(name + str(key))
if not result:
print('error missing ' + str(key))
else:
b = int(result)
if a != b:
print('error unequal '+str(a)+','+str(b))
for item in deletes:
result = r.get(name + str(item))
if result:
print('error not deleted ' + str(item))
import sys
from datetime import datetime
from time import sleep
from rediscluster import StrictRedisCluster
import logging
startup_nodes = [
{"host": "192.168.56.21", "port": "6379"},
{"host": "192.168.56.22", "port": "6379"},
{"host": "192.168.56.23", "port": "6379"}]
r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
print(r.set('foo', 'bar'))
print(r.get('foo'))
p = r.pubsub()
def my_handler(message):
global LastReceivedMessage
#print "Received (%s,%s)" % (message['channel'], message['data'])
if 'data' in message.keys():
shard = int(message['channel'])
actualMessage = int(message['data'])
print "%s%sReceived %s" % (' '*26, '\t'*shard, actualMessage)
if LastReceivedMessage+1 != actualMessage:
print "Missed %s messages" %(actualMessage-LastReceivedMessage-1)
LastReceivedMessage = actualMessage
#p.psubscribe(**{'my-*': my_handler}) # throws an error
p.subscribe(**{'0': my_handler})
p.subscribe(**{'1': my_handler})
p.subscribe(**{'2': my_handler})
p.subscribe(**{'3': my_handler})