def getRedisKeyValues(host, port, keyFilter):
try:
startup_nodes = [{"host": host, "port": port}]
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
info = rc.info()
for key in info:
print "%s: %s" % (key, info[key])
result = []
# keysCount = len(rc.keys())
count = 0
for key in rc.keys():
if count >= 1024:
break
else:
if key.find(keyFilter) != -1:
count += 1
# 查询类型
# (key, value)
result.append([key, getValue(rc, key)])
# pprint.pprint("type:" + rc.type(key) + "$key:" + key + "$value:" + getValue(rc, key))
# print rc.get(key)
return result
except Exception, e:
return [["<font color='red'>Error Info</font>", "<font color='red'>%s</font>" % (str(e) + "<br/>请检查ip和端口是否正确")]]
def __init__(self, key, startup_nodes):
"""
init cluster
"""
self.key = key
self.conn = StrictRedisCluster(startup_nodes=startup_nodes,
decode_responses=True)
def __init__(self,
name,
host='localhost',
port=6379,
db=0,
maxsize=0,
lazy_limit=True,
password=None,
cluster_nodes=None,
redis_lock_config={},
use_lock=False):
"""
Constructor for RedisQueue
maxsize: an integer that sets the upperbound limit on the number of
items that can be placed in the queue.
lazy_limit: redis queue is shared via instance, a lazy size limit is used
for better performance.
"""
self.name = name
if (cluster_nodes is not None):
from rediscluster import StrictRedisCluster
self.redis = StrictRedisCluster(startup_nodes=cluster_nodes)
else:
self.redis = redis.StrictRedis(host=host,
port=port,
db=db,
password=password)
self.maxsize = maxsize
self.lazy_limit = lazy_limit
self.last_qsize = 0
self.redis_lock_config = redis_lock_config
self.lock = None
self.use_lock = use_lock
def test_moved_redirection_pipeline():
"""
Test that the server handles MOVED response when used in pipeline.
At first call it should return a MOVED ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
with patch.object(StrictRedisCluster, 'parse_response') as parse_response:
def moved_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7002
return "MOCK_OK"
parse_response.side_effect = ok_response
raise MovedError("12182 127.0.0.1:7002")
parse_response.side_effect = moved_redirect_effect
r = StrictRedisCluster(host="127.0.0.1", port=7000)
p = r.pipeline()
p.set("foo", "bar")
assert p.execute() == ["MOCK_OK"]
def test_moved_redirection():
"""
Test that the client handles MOVED response.
At first call it should return a MOVED ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
m = Mock(autospec=True)
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7002
return "MOCK_OK"
m.side_effect = ok_response
raise MovedError("12182 127.0.0.1:7002")
m.side_effect = ask_redirect_effect
r.parse_response = m
assert r.set("foo", "bar") == "MOCK_OK"
def test_moved_redirection_pipeline():
"""
Test that the server handles MOVED response when used in pipeline.
At first call it should return a MOVED ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
p = r.pipeline()
m = Mock(autospec=True)
def moved_redirect_effect(connection, command_name, **options):
def ok_response(connection, command_name, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7002
return "MOCK_OK"
m.side_effect = ok_response
raise MovedError("12182 127.0.0.1:7002")
m.side_effect = moved_redirect_effect
p.parse_response = m
p.set("foo", "bar")
assert p.execute() == ["MOCK_OK"]
def test_moved_redirection_pipeline():
"""
Test that the server handles MOVED response when used in pipeline.
At first call it should return a MOVED ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
with patch.object(StrictRedisCluster, 'parse_response') as parse_response:
def moved_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7002
return "MOCK_OK"
parse_response.side_effect = ok_response
raise MovedError("12182 127.0.0.1:7002")
parse_response.side_effect = moved_redirect_effect
r = StrictRedisCluster(host="127.0.0.1", port=7000)
p = r.pipeline()
p.set("foo", "bar")
assert p.execute() == ["MOCK_OK"]
class RedisPubSub():
def __init__(self):
redis_nodes = [{'host': '10.12.28.222', 'port': 6380},
{'host': '10.12.28.222', 'port': 6381},
{'host': '10.12.28.224', 'port': 6380},
{'host': '10.12.28.224', 'port': 6381},
{'host': '10.12.28.227', 'port': 6380},
{'host': '10.12.28.227', 'port': 6381}
]
try:
self.__r = StrictRedisCluster(startup_nodes=redis_nodes)
self.chan_sub = 'fm99'
self.chan_pub = 'fm99'
except Exception as e:
print("connect error %s" % e)
def pub_fun(self, msg):
self.__r.publish(self.chan_pub, msg)
return True
def sub_fun(self):
pub = self.__r.pubsub()
pub.subscribe(self.chan_sub)
# pub.psubscribe('*') # 订阅所有频道
pub.parse_response()
return pub
class myRedisCluster(object):
conn = ''
redis_nodes = [
{
'host': '192.168.128.128',
'port': 6379
},
]
def __init__(self):
try:
self.conn = StrictRedisCluster(startup_nodes=self.redis_nodes)
except Exception as e:
print e
sys.exit(1)
def add(self, key, value):
self.conn.set(key, value)
def get(self, key):
self.conn.get(key)
def rem(self, key):
result = self.conn.delete(key)
if result != 0:
return 1
else:
return 0
def __init__(self, name='common', db=0, host=None, port=None):
if host is None and port is None:
self.config = DBConfigParser().get_config(
server='redis_common_colony', key='colony')
# self.config = DBConfigParser().get_config(server='redis_common_colony', key='105-62-93colony')
self.host = self.config.get('host')
self.port = self.config.get('port')
self.db = self.config.get('db')
if '|' in self.host:
host_list = self.host.split('|')
redis_nodes = []
for ho in host_list:
redis_nodes.append({
'host': str(ho),
'port': self.port,
'db': self.db
})
self.conn = StrictRedisCluster(startup_nodes=redis_nodes)
else:
self.conn = redis.Redis(host=self.host,
port=self.port,
db=self.db)
else:
self.host = host
self.port = port
self.db = db
self.conn = redis.Redis(host=self.host, port=self.port, db=self.db)
self.name = name
def __init__(self,
name,
host='localhost',
port=6379,
db=0,
maxsize=0,
lazy_limit=True,
password=None,
cluster_nodes=None):
"""
Constructor for RedisQueue
maxsize: an integer that sets the upperbound limit on the number of
items that can be placed in the queue.
lazy_limit: redis queue is shared via instance, a lazy size limit is used
for better performance.
"""
self.name = name
if (cluster_nodes is not None):
from rediscluster import StrictRedisCluster
# cluster_nodes = [
# {'host': '172.16.179.131', 'port': '7000'},
# {'host': '172.16.179.142', 'port': '7003'},
# {'host': '172.16.179.131', 'port': '7001'},
# ]
self.redis = StrictRedisCluster(
startup_nodes=cluster_nodes) # redis集群交互 startup_nodes为集群节点的配置
else:
self.redis = redis.StrictRedis(host=host,
port=port,
db=db,
password=password)
self.maxsize = maxsize
self.lazy_limit = lazy_limit
self.last_qsize = 0
def __init__(self, redis_host, redis_port, redis_password, db=0):
if isinstance(redis_host, list) and len(redis_host) == 1:
redis_host = redis_host[0]
if isinstance(redis_host, str):
conn_pool = ConnectionPool(host=redis_host,
port=redis_port,
db=db,
password=redis_password)
self.redis_eng = StrictRedis(
connection_pool=conn_pool,
max_connections=10,
)
elif isinstance(redis_host, list):
if isinstance(redis_port, int):
startup_nodes = [{
"host": host,
"port": redis_port
} for host in redis_host]
elif isinstance(redis_port, list):
startup_nodes = [{
"host": host,
"port": port
} for host, port in zip(redis_host, redis_port)]
self.redis_eng = StrictRedisCluster(startup_nodes=startup_nodes,
password=redis_password)
def redis_cluster():
redis_nodes = [{
'host': '192.168.118.15',
'port': 6379
}, {
'host': '192.168.118.15',
'port': 6380
}, {
'host': '192.168.118.15',
'port': 6381
}, {
'host': '192.168.118.15',
'port': 6382
}, {
'host': '192.168.118.15',
'port': 6383
}, {
'host': '192.168.118.15',
'port': 6384
}]
try:
redisconn = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print("Connect Error!")
sys.exit(1)
# redisconn.set('name', 'hkey')
# redisconn.set('age',18)
print("name is: ", redisconn.get('name'))
print("age is: ", redisconn.get('age'))
def test_access_correct_slave_with_readonly_mode_client(sr):
"""
Test that the client can get value normally with readonly mode
when we connect to correct slave.
"""
# we assume this key is set on 127.0.0.1:7000(7003)
sr.set("foo16706", "foo")
import time
time.sleep(1)
with patch.object(ClusterReadOnlyConnectionPool, "get_node_by_slot") as return_slave_mock:
return_slave_mock.return_value = {
"name": "127.0.0.1:7003",
"host": "127.0.0.1",
"port": 7003,
"server_type": "slave",
}
master_value = {"host": "127.0.0.1", "name": "127.0.0.1:7000", "port": 7000, "server_type": "master"}
with patch.object(
ClusterConnectionPool, "get_master_node_by_slot", return_value=master_value
) as return_master_mock:
readonly_client = StrictRedisCluster(host="127.0.0.1", port=7000, readonly_mode=True)
assert b("foo") == readonly_client.get("foo16706")
assert return_master_mock.call_count == 0
def test_ask_redirection():
"""
Test that the server handles ASK response.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.nodes['127.0.0.1:7001'] = {
'host': u'127.0.0.1',
'server_type': 'master',
'port': 7001,
'name': '127.0.0.1:7001'
}
m = Mock(autospec=True)
host_ip = find_node_ip_based_on_port(r, '7001')
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == host_ip
assert connection.port == 7001
return "MOCK_OK"
m.side_effect = ok_response
raise AskError("1337 {0}:7001".format(host_ip))
m.side_effect = ask_redirect_effect
r.parse_response = m
assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
class SmsRedis(object):
def __init__(self):
# self.host = conf['server']['redis_sms']['host']
self.host = conf['dev_server']['redis_sms']['host']
# self.port = conf['server']['redis_sms']['port']
self.port = conf['dev_server']['redis_sms']['port']
nodes = [{"host": self.host, "port": self.port}]
self.r = StrictRedisCluster(startup_nodes=nodes, decode_responses=True)
# x = self.r.get("user:sms:13116285391_5")
# print(x)
def get_value(self, phone, types=None, only_value=True):
kv = {}
if types is not None:
key = 'user:sms:' + phone + '_' + str(types)
# print(key)
try:
kv[str(phone) + '_' + str(types)] = self.r.get(key)
# print(kv)
except ResponseError:
pass
else:
for i in [1, 3, 4, 5, 6, 7, 8]:
key = 'user:sms:' + phone + '_' + str(i)
try:
if self.r.get(key) is not None:
kv[key] = self.r.get(key)
except ResponseError:
pass
if only_value:
return kv[str(phone) + '_' + str(types)]
else:
return kv
def use_cluster():
# 设置uabntud的桥接ip地址即可
startup_nodes = [
{
"host": "192.168.75.132",
"post": 7001
},
{
"host": "192.168.75.132",
"post": 7002
},
{
"host": "192.168.75.132",
"post": 7003
},
]
# 1.创建集群对象
cluster = StrictRedisCluster(startup_nodes=startup_nodes,
decode_responses=True)
# 2.设置值
cluster.set("name", "laowang")
# 3.获取值
print(cluster.get("name"))
def test_moved_redirection():
"""
Test that the client handles MOVED response.
At first call it should return a MOVED ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
m = Mock(autospec=True)
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7002
return "MOCK_OK"
m.side_effect = ok_response
raise MovedError("12182 127.0.0.1:7002")
m.side_effect = ask_redirect_effect
r.parse_response = m
assert r.set("foo", "bar") == "MOCK_OK"
def test_ask_redirection():
"""
Test that the server handles ASK response.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
m = Mock(autospec=True)
def ask_redirect_effect(connection, command_name, **options):
def ok_response(connection, command_name, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7001
return "MOCK_OK"
m.side_effect = ok_response
raise AskError("1337 127.0.0.1:7001")
m.side_effect = ask_redirect_effect
r.parse_response = m
assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def test_pipeline_ask_redirection():
"""
Test that the server handles ASK response when used in pipeline.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
with patch.object(StrictRedisCluster, "parse_response") as parse_response:
def response(connection, *args, **options):
def response(connection, *args, **options):
def response(connection, *args, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7001
return "MOCK_OK"
parse_response.side_effect = response
raise AskError("12182 127.0.0.1:7001")
parse_response.side_effect = response
raise AskError("12182 127.0.0.1:7001")
parse_response.side_effect = response
p = r.pipeline()
p.set("foo", "bar")
assert p.execute() == ["MOCK_OK"]
def test_ask_redirection():
"""
Test that the server handles ASK response.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.nodes["127.0.0.1:7001"] = {
"host": u"127.0.0.1",
"server_type": "master",
"port": 7001,
"name": "127.0.0.1:7001",
}
with patch.object(StrictRedisCluster, "parse_response") as parse_response:
host_ip = find_node_ip_based_on_port(r, "7001")
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == host_ip
assert connection.port == 7001
return "MOCK_OK"
parse_response.side_effect = ok_response
raise AskError("1337 {0}:7001".format(host_ip))
parse_response.side_effect = ask_redirect_effect
assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def test_access_correct_slave_with_readonly_mode_client(sr):
"""
Test that the client can get value normally with readonly mode
when we connect to correct slave.
"""
# we assume this key is set on 127.0.0.1:7000(7003)
sr.set('foo16706', 'foo')
import time
time.sleep(1)
with patch.object(ClusterReadOnlyConnectionPool, 'get_node_by_slot') as return_slave_mock:
return_slave_mock.return_value = {
'name': '127.0.0.1:7003',
'host': '127.0.0.1',
'port': 7003,
'server_type': 'slave',
}
master_value = {'host': '127.0.0.1', 'name': '127.0.0.1:7000', 'port': 7000, 'server_type': 'master'}
with patch.object(
ClusterConnectionPool,
'get_master_node_by_slot',
return_value=master_value) as return_master_mock:
readonly_client = StrictRedisCluster(host="127.0.0.1", port=7000, readonly_mode=True)
assert b('foo') == readonly_client.get('foo16706')
assert return_master_mock.call_count == 0
def main():
startup_nodes = [{'host': 'redis3', 'port': '6379'}]
r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
pubg_friends_length = r.llen(
'spider:python:pubg_friends:keyword:dest') # redis的数据量
print time.strftime(
'[%Y-%m-%d %H:%M:%S]'), 'redis中pubg_friends的数据量：', pubg_friends_length
lock = threading.Lock()
first_hour = time.strftime('%H') # 小时
date = time.strftime('%Y%m%d') # 数据文件日期
dest_path = '/ftp_samba/112/spider/python/pubg' # linux上的文件目录
if not os.path.exists(dest_path):
os.makedirs(dest_path)
dest_file_name = os.path.join(dest_path, 'pubg_friends_' + date)
fileout = open(dest_file_name, 'a')
threads = []
for i in xrange(1):
t = threading.Thread(target=pubg_friends,
args=(lock, r, first_hour, fileout))
t.start()
threads.append(t)
for t in threads:
t.join()
try:
fileout.flush()
fileout.close()
except IOError as e:
time.sleep(2)
fileout.close()
print time.strftime('[%Y-%m-%d %H:%M:%S]：'), 'over'
def __init__(self, sc, dataset_path):
logger.info("Starting Spark Query Engine..")
self.sc = sc
self.sqlContext = SQLContext(sc)
startup_nodes = [{
"host": cfg.redis['host'],
"port": cfg.redis['port']
}]
self.rc = StrictRedisCluster(startup_nodes=startup_nodes,
decode_responses=True)
# Loading data from data.CSV file
logger.info("Loading the data ....")
data = (
self.sqlContext.read.format('com.databricks.spark.csv').options(
header='true',
inferschema='true').load(os.path.join(dataset_path,
'data.csv')))
logger.info("Cleaning the dataset ....")
self.df = data.select(
'id', 'brand', 'colors',
'dateAdded') # Choosing only the required columns
self.df = self.df.dropDuplicates(['id']) # Dropping duplicate rows
self.df = self.df.dropna(
how='any') # Dropping rows with atleast one null value
logger.info("Processing and storing necessary data in Redis ... ")
self.__get_recent_items()
def get_peer_info_from_redis(peer_ids):
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
if type(peer_ids) is not list:
peer_ids = [peer_ids]
peer_info_list = list()
peer_info_keys = [
"peer_id", "version", "natType", "publicIP", "publicPort", "privateIP",
"privatePort", "stunIP"
]
for peer_id in peer_ids:
try:
# 从redis-cluster中获取节点PNIC信息
pnic = rc.get("PNIC_{0}".format(peer_id))
pnic = json.loads(pnic)
peer_info = dict()
for k in peer_info_keys:
peer_info[k] = pnic[k]
# 当播放节点与供源节点使用线上stun探测natType相加≥7无法穿透而播放节点与供源节点又处于同一局域网时，
# 强行将seedslist中供源节点的natType该为0或1即可使其穿透成功
# peer_info.update({"natType": 0})
peer_info_list.append(peer_info)
except:
pass
return peer_info_list
def redis_cluster():
redis_nodes = [{
'host': '10.101.104.132',
'port': 1321
}, {
'host': '10.101.104.132',
'port': 1322
}, {
'host': '10.101.104.132',
'port': 1323
}]
try:
redisconn = StrictRedisCluster(startup_nodes=redis_nodes,
decode_responses=True)
except Exception as e:
print("Connect Error!")
sys.exit(1)
#redisconn.set('name','admin')
#redisconn.set('age',18)
#print("name is: ", redisconn.get('name'))
#print("age is: ", redisconn.get('age'))
print(redisconn.get("creative:app:com.DreamonStudios.BouncyBasketball"))
print(redisconn.get('billing:ad:1'))
print(redisconn.hgetall('billing:creative:spent:2019040417:5'))
def test_refresh_table_asap():
"""
If this variable is set externally, initialize() should be called.
"""
with patch.object(NodeManager, "initialize") as mock_initialize:
mock_initialize.return_value = None
# Patch parse_response to avoid issues when the cluster sometimes return MOVED
with patch.object(StrictRedisCluster, "parse_response") as mock_parse_response:
def side_effect(self, *args, **kwargs):
return None
mock_parse_response.side_effect = side_effect
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.slots[12182] = [
{"host": "127.0.0.1", "port": 7002, "name": "127.0.0.1:7002", "server_type": "master"}
]
r.refresh_table_asap = True
i = len(mock_initialize.mock_calls)
r.execute_command("SET", "foo", "bar")
assert len(mock_initialize.mock_calls) - i == 1
assert r.refresh_table_asap is False
def test_pipeline_ask_redirection():
"""
Test that the server handles ASK response when used in pipeline.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
p = r.pipeline()
m = Mock(autospec=True)
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7001
return "MOCK_OK"
m.side_effect = ok_response
raise AskError("12182 127.0.0.1:7001")
m.side_effect = ask_redirect_effect
p.parse_response = m
p.set("foo", "bar")
assert p.execute() == ["MOCK_OK"]
def main():
startup_nodes = [{'host': 'redis3', 'port': '6379'}]
r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
gaode_length = r.llen('spider:python:gaode:keyword:dest') # redis的数据量
print time.strftime(
'[%Y-%m-%d %H:%M:%S]'), 'redis中gaode的数据量：', gaode_length
lock = threading.Lock()
first_hour = time.strftime('%H')
date = time.strftime('%Y%m%d')
dest_path = '/ftp_samba/112/spider/python/gd_location/' # 线上数据存储文件
if not os.path.exists(dest_path):
os.makedirs(dest_path)
dest_file_name = os.path.join(dest_path, 'gd_location_' + date)
fileout = open(dest_file_name, 'a')
threads = []
for i in xrange(1):
t = threading.Thread(target=gaode, args=(lock, r, first_hour, fileout))
t.start()
threads.append(t)
for t in threads:
t.join()
try:
fileout.flush()
fileout.close()
except IOError as e:
time.sleep(2)
fileout.close()
def __init__(self, ip_ports = IP_PORTS, db = DB, user_pass = USER_PASS):
# super(RedisDB, self).__init__()
if not hasattr(self,'_redis'):
self._is_redis_cluster = False
try:
if len(ip_ports) > 1:
startup_nodes = []
for ip_port in ip_ports:
ip, port = ip_port.split(':')
startup_nodes.append({"host":ip, "port":port})
self._redis = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
self._pipe = self._redis.pipeline(transaction=False)
self._is_redis_cluster = True
else:
ip, port = ip_ports[0].split(':')
self._redis = redis.Redis(host = ip, port = port, db = db, password = user_pass, decode_responses=True) # redis默认端口是6379
self._pipe = self._redis.pipeline(transaction=True) # redis-py默认在执行每次请求都会创建（连接池申请连接）和断开（归还连接池）一次连接操作，如果想要在一次请求中指定多个命令，则可以使用pipline实现一次请求指定多个命令，并且默认情况下一次pipline 是原子性操作。
except Exception as e:
raise
else:
log.info('连接到redis数据库 %s'%(tools.dumps_json(ip_ports)))
def monkey_link(host=None, port=None, decode_responses=False):
"""
Helper function to return custom slots cache data from different redis nodes
"""
if port == 7000:
result = [[0, 5460, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]],
[5461, 10922, [b'127.0.0.1', 7001], [b'127.0.0.1', 7004]]]
elif port == 7001:
result = [[0, 5460, [b'127.0.0.1', 7001], [b'127.0.0.1', 7003]],
[5461, 10922, [b'127.0.0.1', 7000], [b'127.0.0.1', 7004]]]
else:
result = []
r = StrictRedisCluster(host=host, port=port, decode_responses=True)
orig_execute_command = r.execute_command
def execute_command(*args, **kwargs):
if args == ("cluster", "slots"):
return result
return orig_execute_command(*args, **kwargs)
r.execute_command = execute_command
return r
def connect(self):
try:
self.redisconn = StrictRedisCluster(
startup_nodes=self.cluster_nodes)
except Exception as e:
print("Connect Error!")
sys.exit(1)
def get_video_data(date):
video_keys = get_list(date)
redis_nodes = [{
'host': '10.200.131.32',
'port': 6101
}, {
'host': '10.200.131.31',
'port': 6102
}, {
'host': '10.200.131.27',
'port': 6101
}, {
'host': '10.200.131.28',
'port': 6102
}]
r = StrictRedisCluster(startup_nodes=redis_nodes)
filepath = '../normal_knn/jobs/data/doc/doc_data_' + date
fw = open(filepath, 'w')
key_prefix = 'headline_'
for v_key in video_keys:
key = key_prefix + v_key
video = r.get(key)
if isinstance(video, basestring):
fw.write(video + '\n')
fw.close()
def redis_time():
redis_nodes = [{
'host': '10.12.28.222',
'port': 6380
}, {
'host': '10.12.28.222',
'port': 6381
}, {
'host': '10.12.28.224',
'port': 6380
}, {
'host': '10.12.28.224',
'port': 6381
}, {
'host': '10.12.28.227',
'port': 6380
}, {
'host': '10.12.28.227',
'port': 6381
}]
try:
r = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print("connect error %s" % e)
# print(r.set('thoth-ai','过期测试'))
#
# print(r.expire('thoth-ai',10)) # 设定key过期时间 10s
print(r.get('thoth-ai:robotId:'))
def redisCluster(classid, recordid):
startup_nodes = [{
'host': '192.168.40.112',
'port': '7000'
}, {
'host': '192.168.40.133',
'port': '7000'
}, {
'host': '192.168.40.141',
'port': '7000'
}, {
'host': '192.168.40.112',
'port': '7001'
}, {
'host': '192.168.40.133',
'port': '7001'
}, {
'host': '192.168.40.141',
'port': '7001'
}]
r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
data1 = 'music-class_room-' + str(classid)
data2 = 'music-class_record-' + str(recordid)
rs = r.get(data1)
rs1 = r.get(data2)
print rs
print rs1
# redisCluster(1814805, 12347)
def __init__(self, ci):
log.debug('create connection = %s', ci)
t = ci.type
self.t = t
if t == 1:
log.debug('create redis connection.')
self.conn = StrictRedis(host=ci.host, port=ci.port, db=ci.db)
elif t == 2:
log.debug('create redis cluster connection.')
nodes = json.loads(ci.host)
pool = ClusterConnectionPool(startup_nodes=nodes)
self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True)
elif t == 3:
log.debug('create redis connection from zookeeper.')
client = zk.Client(hosts=ci.host, read_only=True)
node = client.get(ci.path)
arr = str(node[0], encoding='utf-8').split('\n')
address = []
for h in arr:
if h is '':
continue
a = h.split(':')
address.append({'host': a[0], 'port': int(a[1])})
pool = ClusterConnectionPool(startup_nodes=address)
self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True)
else:
raise AttributeError('illegal ConnInfo type.')
if self.test():
self.ci = ci
log.info('connect redis(%s) success', ci.host)
def __connect(self):
try:
self.connect = StrictRedisCluster(startup_nodes=self.node_list)
except Exception as e:
self.connect = None
print("Connect redisCluster node error!", e)
return
def test_refresh_table_asap():
"""
If this variable is set externally, initialize() should be called.
"""
with patch.object(NodeManager, 'initialize') as mock_initialize:
mock_initialize.return_value = None
# Patch parse_response to avoid issues when the cluster sometimes return MOVED
with patch.object(StrictRedisCluster, 'parse_response') as mock_parse_response:
def side_effect(self, *args, **kwargs):
return None
mock_parse_response.side_effect = side_effect
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.slots[12182] = [{
"host": "127.0.0.1",
"port": 7002,
"name": "127.0.0.1:7002",
"server_type": "master",
}]
r.refresh_table_asap = True
i = len(mock_initialize.mock_calls)
r.execute_command("SET", "foo", "bar")
assert len(mock_initialize.mock_calls) - i == 1
assert r.refresh_table_asap is False
def __init__(self, host='127.0.0.1', port=6379):
self.r = StrictRedisCluster(startup_nodes=[{
"host": host,
"port": port
}],
decode_responses=True,
skip_full_coverage_check=True)
def test_ask_redirection():
"""
Test that the server handles ASK response.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.nodes['127.0.0.1:7001'] = {
'host': u'127.0.0.1',
'server_type': 'master',
'port': 7001,
'name': '127.0.0.1:7001'
}
with patch.object(StrictRedisCluster,
'parse_response') as parse_response:
host_ip = find_node_ip_based_on_port(r, '7001')
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == host_ip
assert connection.port == 7001
return "MOCK_OK"
parse_response.side_effect = ok_response
raise AskError("1337 {0}:7001".format(host_ip))
parse_response.side_effect = ask_redirect_effect
assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def monkey_link(host=None, port=None, decode_responses=False):
"""
Helper function to return custom slots cache data from different redis nodes
"""
if port == 7000:
result = [[0, 5460, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]],
[
5461, 10922, [b'127.0.0.1', 7001],
[b'127.0.0.1', 7004]
]]
elif port == 7001:
result = [[0, 5460, [b'127.0.0.1', 7001], [b'127.0.0.1', 7003]],
[
5461, 10922, [b'127.0.0.1', 7000],
[b'127.0.0.1', 7004]
]]
else:
result = []
r = StrictRedisCluster(host=host, port=port, decode_responses=True)
orig_execute_command = r.execute_command
def execute_command(*args, **kwargs):
if args == ("cluster", "slots"):
return result
return orig_execute_command(*args, **kwargs)
r.execute_command = execute_command
return r
def test_access_correct_slave_with_readonly_mode_client(sr):
"""
Test that the client can get value normally with readonly mode
when we connect to correct slave.
"""
# we assume this key is set on 127.0.0.1:7000(7003)
sr.set('foo16706', 'foo')
import time
time.sleep(1)
with patch.object(ClusterReadOnlyConnectionPool, 'get_node_by_slot') as return_slave_mock:
return_slave_mock.return_value = {
'name': '127.0.0.1:7003',
'host': '127.0.0.1',
'port': 7003,
'server_type': 'slave',
}
master_value = {'host': '127.0.0.1', 'name': '127.0.0.1:7000', 'port': 7000, 'server_type': 'master'}
with patch.object(
ClusterConnectionPool,
'get_master_node_by_slot',
return_value=master_value) as return_master_mock:
readonly_client = StrictRedisCluster(host="127.0.0.1", port=7000, readonly_mode=True)
assert b('foo') == readonly_client.get('foo16706')
assert return_master_mock.call_count == 0
def test_refresh_using_specific_nodes(r):
"""
Test making calls on specific nodes when the cluster has failed over to
another node
"""
with patch.object(StrictRedisCluster, 'parse_response') as parse_response_mock:
with patch.object(NodeManager, 'initialize', autospec=True) as init_mock:
# simulate 7006 as a failed node
def side_effect(self, *args, **kwargs):
if self.port == 7006:
parse_response_mock.failed_calls += 1
raise ClusterDownError('CLUSTERDOWN The cluster is down. Use CLUSTER INFO for more information')
elif self.port == 7007:
parse_response_mock.successful_calls += 1
def side_effect_rebuild_slots_cache(self):
# start with all slots mapped to 7006
self.nodes = {'127.0.0.1:7006': {'host': '127.0.0.1', 'server_type': 'master', 'port': 7006, 'name': '127.0.0.1:7006'}}
self.slots = {}
for i in range(0, 16383):
self.slots[i] = [{
'host': '127.0.0.1',
'server_type': 'master',
'port': 7006,
'name': '127.0.0.1:7006',
}]
# After the first connection fails, a reinitialize should follow the cluster to 7007
def map_7007(self):
self.nodes = {'127.0.0.1:7007': {'host': '127.0.0.1', 'server_type': 'master', 'port': 7007, 'name': '127.0.0.1:7007'}}
self.slots = {}
for i in range(0, 16383):
self.slots[i] = [{
'host': '127.0.0.1',
'server_type': 'master',
'port': 7007,
'name': '127.0.0.1:7007',
}]
init_mock.side_effect = map_7007
parse_response_mock.side_effect = side_effect
parse_response_mock.successful_calls = 0
parse_response_mock.failed_calls = 0
init_mock.side_effect = side_effect_rebuild_slots_cache
rc = StrictRedisCluster(host='127.0.0.1', port=7006)
assert len(rc.connection_pool.nodes.nodes) == 1
assert '127.0.0.1:7006' in rc.connection_pool.nodes.nodes
rc.ping()
# Cluster should now point to 7006, and there should be one failed and one succesful call
assert len(rc.connection_pool.nodes.nodes) == 1
assert '127.0.0.1:7007' in rc.connection_pool.nodes.nodes
assert parse_response_mock.failed_calls == 1
assert parse_response_mock.successful_calls == 1
def test_cluster_of_one_instance():
"""
Test a cluster that starts with only one redis server and ends up with
one server.
There is another redis server joining the cluster, hold slot 0, and
eventually quit the cluster. The StrictRedisCluster instance may get confused
when slots mapping and nodes change during the test.
"""
with patch.object(StrictRedisCluster, 'parse_response') as parse_response_mock:
with patch.object(NodeManager, 'initialize', autospec=True) as init_mock:
def side_effect(self, *args, **kwargs):
def ok_call(self, *args, **kwargs):
assert self.port == 7007
return "OK"
parse_response_mock.side_effect = ok_call
raise ClusterDownError('CLUSTERDOWN The cluster is down. Use CLUSTER INFO for more information')
def side_effect_rebuild_slots_cache(self):
# make new node cache that points to 7007 instead of 7006
self.nodes = [{'host': '127.0.0.1', 'server_type': 'master', 'port': 7006, 'name': '127.0.0.1:7006'}]
self.slots = {}
for i in range(0, 16383):
self.slots[i] = [{
'host': '127.0.0.1',
'server_type': 'master',
'port': 7006,
'name': '127.0.0.1:7006',
}]
# Second call should map all to 7007
def map_7007(self):
self.nodes = [{'host': '127.0.0.1', 'server_type': 'master', 'port': 7007, 'name': '127.0.0.1:7007'}]
self.slots = {}
for i in range(0, 16383):
self.slots[i] = [{
'host': '127.0.0.1',
'server_type': 'master',
'port': 7007,
'name': '127.0.0.1:7007',
}]
# First call should map all to 7006
init_mock.side_effect = map_7007
parse_response_mock.side_effect = side_effect
init_mock.side_effect = side_effect_rebuild_slots_cache
rc = StrictRedisCluster(host='127.0.0.1', port=7006)
rc.set("foo", "bar")
def redis_write(nodes, dict_data):
"""
将字典写入redis集群，并计算时间
"""
time1 = time.time()
conn_rc = StrictRedisCluster(startup_nodes=nodes, decode_responses=True)
for k in dict_data:
# print k, func_dict[k]
key_name = 'access_log' + '-' + str(k)
conn_rc.set(key_name, dict_data[k])
# print conn_rc.get(key_name)
time2 = time.time()
time3 = time2 - time1
print time3
def redis_cluster():
redis_nodes = [
{'host':HOSTNAME,'port':7001},
{'host':HOSTNAME,'port':7002},
{'host':HOSTNAME,'port':7003},
{'host':HOSTNAME,'port':7004},
{'host':HOSTNAME,'port':7005},
{'host':HOSTNAME,'port':7006},
]
try:
r = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print(e)
else:
r.set('test','test')
def test_clusterdown_exception_handling():
"""
Test that if exception message starts with CLUSTERDOWN it should
disconnect the connection pool and set refresh_table_asap to True.
"""
with patch.object(ClusterConnectionPool, 'disconnect') as mock_disconnect:
with patch.object(ClusterConnectionPool, 'reset') as mock_reset:
r = StrictRedisCluster(host="127.0.0.1", port=7000)
i = len(mock_reset.mock_calls)
assert r.handle_cluster_command_exception(Exception("CLUSTERDOWN")) == {"method": "clusterdown"}
assert r.refresh_table_asap is True
mock_disconnect.assert_called_once_with()
# reset() should only be called once inside `handle_cluster_command_exception`
assert len(mock_reset.mock_calls) - i == 1
class RedisPubSubHelper():
def __init__(self):
# redis连接对象
self.__conn = StrictRedisCluster(startup_nodes=redis_nodes, decode_responses=True)
def publish(self, channel, message):
# redis对象的publish方法(发布)
# 往指定的频道中发布信息
self.__conn.publish(channel, message)
return True
def subscribe(self, channel):
# 返回了一个发布订阅的对象
pub = self.__conn.pubsub()
# 订阅指定频道
pub.subscribe(channel)
pub.parse_response()
return pub
def test_refresh_table_asap():
"""
If this variable is set externally, initialize() should be called.
"""
with patch.object(NodeManager, 'initialize') as mock_initialize:
mock_initialize.return_value = None
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.slots[12182] = {
"host": "127.0.0.1",
"port": 7002,
"name": "127.0.0.1:7002",
"server_type": "master",
}
r.refresh_table_asap = True
i = len(mock_initialize.mock_calls)
r.execute_command("SET", "foo", "bar")
assert len(mock_initialize.mock_calls) - i == 1
assert r.refresh_table_asap is False
class RedisCluster(object):
def __init__(self, redis_nodes):
self.cluster = StrictRedisCluster(startup_nodes=redis_nodes)
# 无差别的方法
def set(self, name, value, ex=None, px=None, nx=False, xx=False):
return self.cluster.set(name, value, ex, px, nx, xx)
# 无差别的方法
def get(self, name):
return self.cluster.get(name)
# 扇形发送的命令
def cluster_info(self):
return self.cluster.cluster_info()
# 重写StrictRedis的方法
def mset(self, *args, **kwargs):
return self.cluster.mset(args, kwargs)
# 重写StrictRedis的方法
def mget(self, keys, *args):
return self.cluster.mget(keys, args)
def __init__(self, name, host='localhost', port=6379, db=0,
maxsize=0, lazy_limit=True, password=None, cluster_nodes=None):
"""
Constructor for RedisQueue
maxsize: an integer that sets the upperbound limit on the number of
items that can be placed in the queue.
lazy_limit: redis queue is shared via instance, a lazy size limit is used
for better performance.
"""
self.name = name
if(cluster_nodes is not None):
from rediscluster import StrictRedisCluster
self.redis = StrictRedisCluster(startup_nodes=cluster_nodes)
else:
self.redis = redis.StrictRedis(host=host, port=port, db=db, password=password)
self.maxsize = maxsize
self.lazy_limit = lazy_limit
self.last_qsize = 0
def test_api(self):
comm.start_cluster('127.0.0.1', 7100)
comm.join_cluster('127.0.0.1', 7100, '127.0.0.1', 7101)
comm.replicate('127.0.0.1', 7100, '127.0.0.1', 7102)
time.sleep(1)
rc = StrictRedisCluster(
startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
for i in range(20):
rc.set('key_%s' % i, 'value_%s' % i)
for i in range(20):
self.assertEqual('value_%s' % i, rc.get('key_%s' % i))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(3, len(nodes))
self.assertEqual(
list(range(8192)), nodes[('127.0.0.1', 7101)].assigned_slots)
self.assertEqual(
list(range(8192, 16384)), nodes[('127.0.0.1',
7100)].assigned_slots)
comm.quit_cluster('127.0.0.1', 7101)
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(
list(range(16384)), nodes[('127.0.0.1', 7100)].assigned_slots)
for i in range(20):
self.assertEqual('value_%s' % i, rc.get('key_%s' % i))
for i in range(20):
rc.delete('key_%s' % i)
comm.quit_cluster('127.0.0.1', 7102)
comm.shutdown_cluster('127.0.0.1', 7100)
def test_quit_problems(self):
comm.start_cluster('127.0.0.1', 7100)
comm.join_cluster('127.0.0.1', 7100, '127.0.0.1', 7101)
comm.replicate('127.0.0.1', 7100, '127.0.0.1', 7102)
time.sleep(1)
rc = StrictRedisCluster(
startup_nodes=[{
'host': '127.0.0.1',
'port': 7100
}],
decode_responses=True)
for i in range(20):
rc.set('key_%s' % i, 'value_%s' % i)
for i in range(20):
self.assertEqual('value_%s' % i, rc.get('key_%s' % i))
nodes = base.list_nodes('127.0.0.1', 7100)
self.assertEqual(3, len(nodes))
self.assertEqual(
list(range(8192)), nodes[('127.0.0.1', 7101)].assigned_slots)
self.assertEqual(
list(range(8192, 16384)), nodes[('127.0.0.1',
7100)].assigned_slots)
for i in range(20):
rc.delete('key_%s' % i)
six.assertRaisesRegex(self, ValueError,
'^The master still has slaves$',
comm.quit_cluster, '127.0.0.1', 7100)
comm.quit_cluster('127.0.0.1', 7102)
comm.quit_cluster('127.0.0.1', 7101)
six.assertRaisesRegex(self, ValueError, '^This is the last node',
comm.quit_cluster, '127.0.0.1', 7100)
comm.shutdown_cluster('127.0.0.1', 7100)
from rediscluster.cluster_mgt import RedisClusterMgt
from rediscluster import StrictRedisCluster
startup_nodes = [{"host":"127.0.0.1","port":"7000"}, {"host":"127.0.0.1","port":"7001"}]
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
rc.set("foo", "bar")
print(rc.get("foo"))
def __init__(self):
# redis连接对象
self.__conn = StrictRedisCluster(startup_nodes=redis_nodes, decode_responses=True)
from rediscluster import StrictRedisCluster
startup_nodes = [{"host": "192.168.1.168", "port": "7000"}]
rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
ps = rc.pubsub()
ps.subscribe(['foo', 'bar'])
rc.publish('foo', 'hello foo from python')
rc.publish('bar', 'hello bar from python')
# chapter 2 : redis cluster
from rediscluster import StrictRedisCluster
import csv
import time
def multiLoader(user_list, r):
for email, user in user_list:
r.set(email, user)
startup_nodes = [
{"host":"127.0.0.1", "port":"6379"},
{"host":"127.0.0.1", "port":"6380"},
{"host":"127.0.0.1", "port":"6381"}
]
rc = StrictRedisCluster(startup_nodes=startup_nodes)
set_count = 0
start_time = int(time.time())
rc.flushall()
with open('users.csv', 'rU') as fd:
csvreader = csv.reader(fd)
user_list = []
for line in csvreader:
if csvreader.line_num == 1:
continue
email = line[0].strip()
first_name = line[1].strip()
import time
from rediscluster import StrictRedisCluster
from rediscluster.exceptions import ClusterError
from redis.exceptions import ConnectionError
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-n", "--name", dest = "name", default = "a", help="Name that will be added to key")
args = parser.parse_args()
name = args.name
current_milli_time = lambda: int(round(time.time() * 1000))
startup_nodes = [{"host":"10.240.0.8","port":"6379"},{"host":"10.240.0.10","port":"6379"},{"host":"10.240.0.12","port":"6379"}]
r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
id = 0
sessions = {}
deletes = []
#Create 100 sessions
while id < 200:
#Insert into redis
sessions[id] = 0
start = current_milli_time()
r.set(name + str(id),0)
end = current_milli_time()
print(str(end-start)+'ms new '+str(id))
id = id + 1
#!/usr/bin/env python
# _*_coding:utf-8_*_
# Author: create by yang.hong
# Time: 2018-08-17 11:06
"""
集群模式下，匹配方式删除多个key
"""
from rediscluster import StrictRedisCluster
redis_cluster = [{'host': '172.28.246.12', 'port': '28000'},
{'host': '172.28.246.12', 'port': '28001'},
{'host': '172.28.246.12', 'port': '28002'},
{'host': '172.28.246.12', 'port': '28003'},
{'host': '172.28.246.12', 'port': '28004'},
{'host': '172.28.246.12', 'port': '28005'},
]
r = StrictRedisCluster(startup_nodes=redis_cluster, decode_responses=True)
print r.keys(pattern="*mc_payChannel_router*")
r.delete(*r.keys(pattern="*mc_payChannel_router*"))
def test_cluster_of_one_instance():
"""
Test a cluster that starts with only one redis server and ends up with
one server.
There is another redis server joining the cluster, hold slot 0, and
eventually quit the cluster. The StrictRedisCluster instance may get confused
when slots mapping and nodes change during the test.
"""
with patch.object(StrictRedisCluster, "parse_response") as parse_response_mock:
with patch.object(NodeManager, "initialize", autospec=True) as init_mock:
def side_effect(self, *args, **kwargs):
def ok_call(self, *args, **kwargs):
assert self.port == 7007
return "OK"
parse_response_mock.side_effect = ok_call
raise ClusterDownError("CLUSTERDOWN The cluster is down. Use CLUSTER INFO for more information")
def side_effect_rebuild_slots_cache(self):
# make new node cache that points to 7007 instead of 7006
self.nodes = [{"host": "127.0.0.1", "server_type": "master", "port": 7006, "name": "127.0.0.1:7006"}]
self.slots = {}
for i in range(0, 16383):
self.slots[i] = [
{"host": "127.0.0.1", "server_type": "master", "port": 7006, "name": "127.0.0.1:7006"}
]
# Second call should map all to 7007
def map_7007(self):
self.nodes = [
{"host": "127.0.0.1", "server_type": "master", "port": 7007, "name": "127.0.0.1:7007"}
]
self.slots = {}
for i in range(0, 16383):
self.slots[i] = [
{"host": "127.0.0.1", "server_type": "master", "port": 7007, "name": "127.0.0.1:7007"}
]
# First call should map all to 7006
init_mock.side_effect = map_7007
parse_response_mock.side_effect = side_effect
init_mock.side_effect = side_effect_rebuild_slots_cache
rc = StrictRedisCluster(host="127.0.0.1", port=7006)
rc.set("foo", "bar")
#####
# Test that CLUSTERDOWN is handled the same way when used via pipeline
parse_response_mock.side_effect = side_effect
init_mock.side_effect = side_effect_rebuild_slots_cache
rc = StrictRedisCluster(host="127.0.0.1", port=7006)
p = rc.pipeline()
p.set("bar", "foo")
p.execute()
