def getRedisKeyValues(host, port, keyFilter): try: startup_nodes = [{"host": host, "port": port}] rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) info = rc.info() for key in info: print "%s: %s" % (key, info[key]) result = [] # keysCount = len(rc.keys()) count = 0 for key in rc.keys(): if count >= 1024: break else: if key.find(keyFilter) != -1: count += 1 # 查询类型 # (key, value) result.append([key, getValue(rc, key)]) # pprint.pprint("type:" + rc.type(key) + "$key:" + key + "$value:" + getValue(rc, key)) # print rc.get(key) return result except Exception, e: return [["<font color='red'>Error Info</font>", "<font color='red'>%s</font>" % (str(e) + "<br/>请检查ip和端口是否正确")]]
def __init__(self, key, startup_nodes): """ init cluster """ self.key = key self.conn = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
def __init__(self, name, host='localhost', port=6379, db=0, maxsize=0, lazy_limit=True, password=None, cluster_nodes=None, redis_lock_config={}, use_lock=False): """ Constructor for RedisQueue maxsize: an integer that sets the upperbound limit on the number of items that can be placed in the queue. lazy_limit: redis queue is shared via instance, a lazy size limit is used for better performance. """ self.name = name if (cluster_nodes is not None): from rediscluster import StrictRedisCluster self.redis = StrictRedisCluster(startup_nodes=cluster_nodes) else: self.redis = redis.StrictRedis(host=host, port=port, db=db, password=password) self.maxsize = maxsize self.lazy_limit = lazy_limit self.last_qsize = 0 self.redis_lock_config = redis_lock_config self.lock = None self.use_lock = use_lock
def test_moved_redirection_pipeline(): """ Test that the server handles MOVED response when used in pipeline. At first call it should return a MOVED ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ with patch.object(StrictRedisCluster, 'parse_response') as parse_response: def moved_redirect_effect(connection, *args, **options): def ok_response(connection, *args, **options): assert connection.host == "127.0.0.1" assert connection.port == 7002 return "MOCK_OK" parse_response.side_effect = ok_response raise MovedError("12182 127.0.0.1:7002") parse_response.side_effect = moved_redirect_effect r = StrictRedisCluster(host="127.0.0.1", port=7000) p = r.pipeline() p.set("foo", "bar") assert p.execute() == ["MOCK_OK"]
def test_moved_redirection(): """ Test that the client handles MOVED response. At first call it should return a MOVED ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ r = StrictRedisCluster(host="127.0.0.1", port=7000) m = Mock(autospec=True) def ask_redirect_effect(connection, *args, **options): def ok_response(connection, *args, **options): assert connection.host == "127.0.0.1" assert connection.port == 7002 return "MOCK_OK" m.side_effect = ok_response raise MovedError("12182 127.0.0.1:7002") m.side_effect = ask_redirect_effect r.parse_response = m assert r.set("foo", "bar") == "MOCK_OK"
def test_moved_redirection_pipeline(): """ Test that the server handles MOVED response when used in pipeline. At first call it should return a MOVED ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ r = StrictRedisCluster(host="127.0.0.1", port=7000) p = r.pipeline() m = Mock(autospec=True) def moved_redirect_effect(connection, command_name, **options): def ok_response(connection, command_name, **options): assert connection.host == "127.0.0.1" assert connection.port == 7002 return "MOCK_OK" m.side_effect = ok_response raise MovedError("12182 127.0.0.1:7002") m.side_effect = moved_redirect_effect p.parse_response = m p.set("foo", "bar") assert p.execute() == ["MOCK_OK"]
class RedisPubSub(): def __init__(self): redis_nodes = [{'host': '10.12.28.222', 'port': 6380}, {'host': '10.12.28.222', 'port': 6381}, {'host': '10.12.28.224', 'port': 6380}, {'host': '10.12.28.224', 'port': 6381}, {'host': '10.12.28.227', 'port': 6380}, {'host': '10.12.28.227', 'port': 6381} ] try: self.__r = StrictRedisCluster(startup_nodes=redis_nodes) self.chan_sub = 'fm99' self.chan_pub = 'fm99' except Exception as e: print("connect error %s" % e) def pub_fun(self, msg): self.__r.publish(self.chan_pub, msg) return True def sub_fun(self): pub = self.__r.pubsub() pub.subscribe(self.chan_sub) # pub.psubscribe('*') # 订阅所有频道 pub.parse_response() return pub
class myRedisCluster(object): conn = '' redis_nodes = [ { 'host': '192.168.128.128', 'port': 6379 }, ] def __init__(self): try: self.conn = StrictRedisCluster(startup_nodes=self.redis_nodes) except Exception as e: print e sys.exit(1) def add(self, key, value): self.conn.set(key, value) def get(self, key): self.conn.get(key) def rem(self, key): result = self.conn.delete(key) if result != 0: return 1 else: return 0
def __init__(self, name='common', db=0, host=None, port=None): if host is None and port is None: self.config = DBConfigParser().get_config( server='redis_common_colony', key='colony') # self.config = DBConfigParser().get_config(server='redis_common_colony', key='105-62-93colony') self.host = self.config.get('host') self.port = self.config.get('port') self.db = self.config.get('db') if '|' in self.host: host_list = self.host.split('|') redis_nodes = [] for ho in host_list: redis_nodes.append({ 'host': str(ho), 'port': self.port, 'db': self.db }) self.conn = StrictRedisCluster(startup_nodes=redis_nodes) else: self.conn = redis.Redis(host=self.host, port=self.port, db=self.db) else: self.host = host self.port = port self.db = db self.conn = redis.Redis(host=self.host, port=self.port, db=self.db) self.name = name
def __init__(self, name, host='localhost', port=6379, db=0, maxsize=0, lazy_limit=True, password=None, cluster_nodes=None): """ Constructor for RedisQueue maxsize: an integer that sets the upperbound limit on the number of items that can be placed in the queue. lazy_limit: redis queue is shared via instance, a lazy size limit is used for better performance. """ self.name = name if (cluster_nodes is not None): from rediscluster import StrictRedisCluster # cluster_nodes = [ # {'host': '172.16.179.131', 'port': '7000'}, # {'host': '172.16.179.142', 'port': '7003'}, # {'host': '172.16.179.131', 'port': '7001'}, # ] self.redis = StrictRedisCluster( startup_nodes=cluster_nodes) # redis集群交互 startup_nodes为集群节点的配置 else: self.redis = redis.StrictRedis(host=host, port=port, db=db, password=password) self.maxsize = maxsize self.lazy_limit = lazy_limit self.last_qsize = 0
def __init__(self, redis_host, redis_port, redis_password, db=0): if isinstance(redis_host, list) and len(redis_host) == 1: redis_host = redis_host[0] if isinstance(redis_host, str): conn_pool = ConnectionPool(host=redis_host, port=redis_port, db=db, password=redis_password) self.redis_eng = StrictRedis( connection_pool=conn_pool, max_connections=10, ) elif isinstance(redis_host, list): if isinstance(redis_port, int): startup_nodes = [{ "host": host, "port": redis_port } for host in redis_host] elif isinstance(redis_port, list): startup_nodes = [{ "host": host, "port": port } for host, port in zip(redis_host, redis_port)] self.redis_eng = StrictRedisCluster(startup_nodes=startup_nodes, password=redis_password)
def redis_cluster(): redis_nodes = [{ 'host': '192.168.118.15', 'port': 6379 }, { 'host': '192.168.118.15', 'port': 6380 }, { 'host': '192.168.118.15', 'port': 6381 }, { 'host': '192.168.118.15', 'port': 6382 }, { 'host': '192.168.118.15', 'port': 6383 }, { 'host': '192.168.118.15', 'port': 6384 }] try: redisconn = StrictRedisCluster(startup_nodes=redis_nodes) except Exception as e: print("Connect Error!") sys.exit(1) # redisconn.set('name', 'hkey') # redisconn.set('age',18) print("name is: ", redisconn.get('name')) print("age is: ", redisconn.get('age'))
def test_access_correct_slave_with_readonly_mode_client(sr): """ Test that the client can get value normally with readonly mode when we connect to correct slave. """ # we assume this key is set on 127.0.0.1:7000(7003) sr.set("foo16706", "foo") import time time.sleep(1) with patch.object(ClusterReadOnlyConnectionPool, "get_node_by_slot") as return_slave_mock: return_slave_mock.return_value = { "name": "127.0.0.1:7003", "host": "127.0.0.1", "port": 7003, "server_type": "slave", } master_value = {"host": "127.0.0.1", "name": "127.0.0.1:7000", "port": 7000, "server_type": "master"} with patch.object( ClusterConnectionPool, "get_master_node_by_slot", return_value=master_value ) as return_master_mock: readonly_client = StrictRedisCluster(host="127.0.0.1", port=7000, readonly_mode=True) assert b("foo") == readonly_client.get("foo16706") assert return_master_mock.call_count == 0
def test_ask_redirection(): """ Test that the server handles ASK response. At first call it should return a ASK ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ r = StrictRedisCluster(host="127.0.0.1", port=7000) r.connection_pool.nodes.nodes['127.0.0.1:7001'] = { 'host': u'127.0.0.1', 'server_type': 'master', 'port': 7001, 'name': '127.0.0.1:7001' } m = Mock(autospec=True) host_ip = find_node_ip_based_on_port(r, '7001') def ask_redirect_effect(connection, *args, **options): def ok_response(connection, *args, **options): assert connection.host == host_ip assert connection.port == 7001 return "MOCK_OK" m.side_effect = ok_response raise AskError("1337 {0}:7001".format(host_ip)) m.side_effect = ask_redirect_effect r.parse_response = m assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
class SmsRedis(object): def __init__(self): # self.host = conf['server']['redis_sms']['host'] self.host = conf['dev_server']['redis_sms']['host'] # self.port = conf['server']['redis_sms']['port'] self.port = conf['dev_server']['redis_sms']['port'] nodes = [{"host": self.host, "port": self.port}] self.r = StrictRedisCluster(startup_nodes=nodes, decode_responses=True) # x = self.r.get("user:sms:13116285391_5") # print(x) def get_value(self, phone, types=None, only_value=True): kv = {} if types is not None: key = 'user:sms:' + phone + '_' + str(types) # print(key) try: kv[str(phone) + '_' + str(types)] = self.r.get(key) # print(kv) except ResponseError: pass else: for i in [1, 3, 4, 5, 6, 7, 8]: key = 'user:sms:' + phone + '_' + str(i) try: if self.r.get(key) is not None: kv[key] = self.r.get(key) except ResponseError: pass if only_value: return kv[str(phone) + '_' + str(types)] else: return kv
def use_cluster(): # 设置uabntud的桥接ip地址即可 startup_nodes = [ { "host": "192.168.75.132", "post": 7001 }, { "host": "192.168.75.132", "post": 7002 }, { "host": "192.168.75.132", "post": 7003 }, ] # 1.创建集群对象 cluster = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) # 2.设置值 cluster.set("name", "laowang") # 3.获取值 print(cluster.get("name"))
def test_ask_redirection(): """ Test that the server handles ASK response. At first call it should return a ASK ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ r = StrictRedisCluster(host="127.0.0.1", port=7000) m = Mock(autospec=True) def ask_redirect_effect(connection, command_name, **options): def ok_response(connection, command_name, **options): assert connection.host == "127.0.0.1" assert connection.port == 7001 return "MOCK_OK" m.side_effect = ok_response raise AskError("1337 127.0.0.1:7001") m.side_effect = ask_redirect_effect r.parse_response = m assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def test_pipeline_ask_redirection(): """ Test that the server handles ASK response when used in pipeline. At first call it should return a ASK ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ r = StrictRedisCluster(host="127.0.0.1", port=7000) with patch.object(StrictRedisCluster, "parse_response") as parse_response: def response(connection, *args, **options): def response(connection, *args, **options): def response(connection, *args, **options): assert connection.host == "127.0.0.1" assert connection.port == 7001 return "MOCK_OK" parse_response.side_effect = response raise AskError("12182 127.0.0.1:7001") parse_response.side_effect = response raise AskError("12182 127.0.0.1:7001") parse_response.side_effect = response p = r.pipeline() p.set("foo", "bar") assert p.execute() == ["MOCK_OK"]
def test_ask_redirection(): """ Test that the server handles ASK response. At first call it should return a ASK ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ r = StrictRedisCluster(host="127.0.0.1", port=7000) r.connection_pool.nodes.nodes["127.0.0.1:7001"] = { "host": u"127.0.0.1", "server_type": "master", "port": 7001, "name": "127.0.0.1:7001", } with patch.object(StrictRedisCluster, "parse_response") as parse_response: host_ip = find_node_ip_based_on_port(r, "7001") def ask_redirect_effect(connection, *args, **options): def ok_response(connection, *args, **options): assert connection.host == host_ip assert connection.port == 7001 return "MOCK_OK" parse_response.side_effect = ok_response raise AskError("1337 {0}:7001".format(host_ip)) parse_response.side_effect = ask_redirect_effect assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def test_access_correct_slave_with_readonly_mode_client(sr): """ Test that the client can get value normally with readonly mode when we connect to correct slave. """ # we assume this key is set on 127.0.0.1:7000(7003) sr.set('foo16706', 'foo') import time time.sleep(1) with patch.object(ClusterReadOnlyConnectionPool, 'get_node_by_slot') as return_slave_mock: return_slave_mock.return_value = { 'name': '127.0.0.1:7003', 'host': '127.0.0.1', 'port': 7003, 'server_type': 'slave', } master_value = {'host': '127.0.0.1', 'name': '127.0.0.1:7000', 'port': 7000, 'server_type': 'master'} with patch.object( ClusterConnectionPool, 'get_master_node_by_slot', return_value=master_value) as return_master_mock: readonly_client = StrictRedisCluster(host="127.0.0.1", port=7000, readonly_mode=True) assert b('foo') == readonly_client.get('foo16706') assert return_master_mock.call_count == 0
def main(): startup_nodes = [{'host': 'redis3', 'port': '6379'}] r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) pubg_friends_length = r.llen( 'spider:python:pubg_friends:keyword:dest') # redis的数据量 print time.strftime( '[%Y-%m-%d %H:%M:%S]'), 'redis中pubg_friends的数据量:', pubg_friends_length lock = threading.Lock() first_hour = time.strftime('%H') # 小时 date = time.strftime('%Y%m%d') # 数据文件日期 dest_path = '/ftp_samba/112/spider/python/pubg' # linux上的文件目录 if not os.path.exists(dest_path): os.makedirs(dest_path) dest_file_name = os.path.join(dest_path, 'pubg_friends_' + date) fileout = open(dest_file_name, 'a') threads = [] for i in xrange(1): t = threading.Thread(target=pubg_friends, args=(lock, r, first_hour, fileout)) t.start() threads.append(t) for t in threads: t.join() try: fileout.flush() fileout.close() except IOError as e: time.sleep(2) fileout.close() print time.strftime('[%Y-%m-%d %H:%M:%S]:'), 'over'
def __init__(self, sc, dataset_path): logger.info("Starting Spark Query Engine..") self.sc = sc self.sqlContext = SQLContext(sc) startup_nodes = [{ "host": cfg.redis['host'], "port": cfg.redis['port'] }] self.rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) # Loading data from data.CSV file logger.info("Loading the data ....") data = ( self.sqlContext.read.format('com.databricks.spark.csv').options( header='true', inferschema='true').load(os.path.join(dataset_path, 'data.csv'))) logger.info("Cleaning the dataset ....") self.df = data.select( 'id', 'brand', 'colors', 'dateAdded') # Choosing only the required columns self.df = self.df.dropDuplicates(['id']) # Dropping duplicate rows self.df = self.df.dropna( how='any') # Dropping rows with atleast one null value logger.info("Processing and storing necessary data in Redis ... ") self.__get_recent_items()
def get_peer_info_from_redis(peer_ids): rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) if type(peer_ids) is not list: peer_ids = [peer_ids] peer_info_list = list() peer_info_keys = [ "peer_id", "version", "natType", "publicIP", "publicPort", "privateIP", "privatePort", "stunIP" ] for peer_id in peer_ids: try: # 从redis-cluster中获取节点PNIC信息 pnic = rc.get("PNIC_{0}".format(peer_id)) pnic = json.loads(pnic) peer_info = dict() for k in peer_info_keys: peer_info[k] = pnic[k] # 当播放节点与供源节点使用线上stun探测natType相加≥7无法穿透而播放节点与供源节点又处于同一局域网时, # 强行将seedslist中供源节点的natType该为0或1即可使其穿透成功 # peer_info.update({"natType": 0}) peer_info_list.append(peer_info) except: pass return peer_info_list
def redis_cluster(): redis_nodes = [{ 'host': '10.101.104.132', 'port': 1321 }, { 'host': '10.101.104.132', 'port': 1322 }, { 'host': '10.101.104.132', 'port': 1323 }] try: redisconn = StrictRedisCluster(startup_nodes=redis_nodes, decode_responses=True) except Exception as e: print("Connect Error!") sys.exit(1) #redisconn.set('name','admin') #redisconn.set('age',18) #print("name is: ", redisconn.get('name')) #print("age is: ", redisconn.get('age')) print(redisconn.get("creative:app:com.DreamonStudios.BouncyBasketball")) print(redisconn.get('billing:ad:1')) print(redisconn.hgetall('billing:creative:spent:2019040417:5'))
def test_refresh_table_asap(): """ If this variable is set externally, initialize() should be called. """ with patch.object(NodeManager, "initialize") as mock_initialize: mock_initialize.return_value = None # Patch parse_response to avoid issues when the cluster sometimes return MOVED with patch.object(StrictRedisCluster, "parse_response") as mock_parse_response: def side_effect(self, *args, **kwargs): return None mock_parse_response.side_effect = side_effect r = StrictRedisCluster(host="127.0.0.1", port=7000) r.connection_pool.nodes.slots[12182] = [ {"host": "127.0.0.1", "port": 7002, "name": "127.0.0.1:7002", "server_type": "master"} ] r.refresh_table_asap = True i = len(mock_initialize.mock_calls) r.execute_command("SET", "foo", "bar") assert len(mock_initialize.mock_calls) - i == 1 assert r.refresh_table_asap is False
def test_pipeline_ask_redirection(): """ Test that the server handles ASK response when used in pipeline. At first call it should return a ASK ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ r = StrictRedisCluster(host="127.0.0.1", port=7000) p = r.pipeline() m = Mock(autospec=True) def ask_redirect_effect(connection, *args, **options): def ok_response(connection, *args, **options): assert connection.host == "127.0.0.1" assert connection.port == 7001 return "MOCK_OK" m.side_effect = ok_response raise AskError("12182 127.0.0.1:7001") m.side_effect = ask_redirect_effect p.parse_response = m p.set("foo", "bar") assert p.execute() == ["MOCK_OK"]
def main(): startup_nodes = [{'host': 'redis3', 'port': '6379'}] r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) gaode_length = r.llen('spider:python:gaode:keyword:dest') # redis的数据量 print time.strftime( '[%Y-%m-%d %H:%M:%S]'), 'redis中gaode的数据量:', gaode_length lock = threading.Lock() first_hour = time.strftime('%H') date = time.strftime('%Y%m%d') dest_path = '/ftp_samba/112/spider/python/gd_location/' # 线上数据存储文件 if not os.path.exists(dest_path): os.makedirs(dest_path) dest_file_name = os.path.join(dest_path, 'gd_location_' + date) fileout = open(dest_file_name, 'a') threads = [] for i in xrange(1): t = threading.Thread(target=gaode, args=(lock, r, first_hour, fileout)) t.start() threads.append(t) for t in threads: t.join() try: fileout.flush() fileout.close() except IOError as e: time.sleep(2) fileout.close()
def __init__(self, ip_ports = IP_PORTS, db = DB, user_pass = USER_PASS): # super(RedisDB, self).__init__() if not hasattr(self,'_redis'): self._is_redis_cluster = False try: if len(ip_ports) > 1: startup_nodes = [] for ip_port in ip_ports: ip, port = ip_port.split(':') startup_nodes.append({"host":ip, "port":port}) self._redis = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) self._pipe = self._redis.pipeline(transaction=False) self._is_redis_cluster = True else: ip, port = ip_ports[0].split(':') self._redis = redis.Redis(host = ip, port = port, db = db, password = user_pass, decode_responses=True) # redis默认端口是6379 self._pipe = self._redis.pipeline(transaction=True) # redis-py默认在执行每次请求都会创建(连接池申请连接)和断开(归还连接池)一次连接操作,如果想要在一次请求中指定多个命令,则可以使用pipline实现一次请求指定多个命令,并且默认情况下一次pipline 是原子性操作。 except Exception as e: raise else: log.info('连接到redis数据库 %s'%(tools.dumps_json(ip_ports)))
def monkey_link(host=None, port=None, decode_responses=False): """ Helper function to return custom slots cache data from different redis nodes """ if port == 7000: result = [[0, 5460, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]], [5461, 10922, [b'127.0.0.1', 7001], [b'127.0.0.1', 7004]]] elif port == 7001: result = [[0, 5460, [b'127.0.0.1', 7001], [b'127.0.0.1', 7003]], [5461, 10922, [b'127.0.0.1', 7000], [b'127.0.0.1', 7004]]] else: result = [] r = StrictRedisCluster(host=host, port=port, decode_responses=True) orig_execute_command = r.execute_command def execute_command(*args, **kwargs): if args == ("cluster", "slots"): return result return orig_execute_command(*args, **kwargs) r.execute_command = execute_command return r
def connect(self): try: self.redisconn = StrictRedisCluster( startup_nodes=self.cluster_nodes) except Exception as e: print("Connect Error!") sys.exit(1)
def get_video_data(date): video_keys = get_list(date) redis_nodes = [{ 'host': '10.200.131.32', 'port': 6101 }, { 'host': '10.200.131.31', 'port': 6102 }, { 'host': '10.200.131.27', 'port': 6101 }, { 'host': '10.200.131.28', 'port': 6102 }] r = StrictRedisCluster(startup_nodes=redis_nodes) filepath = '../normal_knn/jobs/data/doc/doc_data_' + date fw = open(filepath, 'w') key_prefix = 'headline_' for v_key in video_keys: key = key_prefix + v_key video = r.get(key) if isinstance(video, basestring): fw.write(video + '\n') fw.close()
def redis_time(): redis_nodes = [{ 'host': '10.12.28.222', 'port': 6380 }, { 'host': '10.12.28.222', 'port': 6381 }, { 'host': '10.12.28.224', 'port': 6380 }, { 'host': '10.12.28.224', 'port': 6381 }, { 'host': '10.12.28.227', 'port': 6380 }, { 'host': '10.12.28.227', 'port': 6381 }] try: r = StrictRedisCluster(startup_nodes=redis_nodes) except Exception as e: print("connect error %s" % e) # print(r.set('thoth-ai','过期测试')) # # print(r.expire('thoth-ai',10)) # 设定key过期时间 10s print(r.get('thoth-ai:robotId:'))
def redisCluster(classid, recordid): startup_nodes = [{ 'host': '192.168.40.112', 'port': '7000' }, { 'host': '192.168.40.133', 'port': '7000' }, { 'host': '192.168.40.141', 'port': '7000' }, { 'host': '192.168.40.112', 'port': '7001' }, { 'host': '192.168.40.133', 'port': '7001' }, { 'host': '192.168.40.141', 'port': '7001' }] r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) data1 = 'music-class_room-' + str(classid) data2 = 'music-class_record-' + str(recordid) rs = r.get(data1) rs1 = r.get(data2) print rs print rs1 # redisCluster(1814805, 12347)
def __init__(self, ci): log.debug('create connection = %s', ci) t = ci.type self.t = t if t == 1: log.debug('create redis connection.') self.conn = StrictRedis(host=ci.host, port=ci.port, db=ci.db) elif t == 2: log.debug('create redis cluster connection.') nodes = json.loads(ci.host) pool = ClusterConnectionPool(startup_nodes=nodes) self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True) elif t == 3: log.debug('create redis connection from zookeeper.') client = zk.Client(hosts=ci.host, read_only=True) node = client.get(ci.path) arr = str(node[0], encoding='utf-8').split('\n') address = [] for h in arr: if h is '': continue a = h.split(':') address.append({'host': a[0], 'port': int(a[1])}) pool = ClusterConnectionPool(startup_nodes=address) self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True) else: raise AttributeError('illegal ConnInfo type.') if self.test(): self.ci = ci log.info('connect redis(%s) success', ci.host)
def __connect(self): try: self.connect = StrictRedisCluster(startup_nodes=self.node_list) except Exception as e: self.connect = None print("Connect redisCluster node error!", e) return
def test_refresh_table_asap(): """ If this variable is set externally, initialize() should be called. """ with patch.object(NodeManager, 'initialize') as mock_initialize: mock_initialize.return_value = None # Patch parse_response to avoid issues when the cluster sometimes return MOVED with patch.object(StrictRedisCluster, 'parse_response') as mock_parse_response: def side_effect(self, *args, **kwargs): return None mock_parse_response.side_effect = side_effect r = StrictRedisCluster(host="127.0.0.1", port=7000) r.connection_pool.nodes.slots[12182] = [{ "host": "127.0.0.1", "port": 7002, "name": "127.0.0.1:7002", "server_type": "master", }] r.refresh_table_asap = True i = len(mock_initialize.mock_calls) r.execute_command("SET", "foo", "bar") assert len(mock_initialize.mock_calls) - i == 1 assert r.refresh_table_asap is False
def __init__(self, host='127.0.0.1', port=6379): self.r = StrictRedisCluster(startup_nodes=[{ "host": host, "port": port }], decode_responses=True, skip_full_coverage_check=True)
def test_ask_redirection(): """ Test that the server handles ASK response. At first call it should return a ASK ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ r = StrictRedisCluster(host="127.0.0.1", port=7000) r.connection_pool.nodes.nodes['127.0.0.1:7001'] = { 'host': u'127.0.0.1', 'server_type': 'master', 'port': 7001, 'name': '127.0.0.1:7001' } with patch.object(StrictRedisCluster, 'parse_response') as parse_response: host_ip = find_node_ip_based_on_port(r, '7001') def ask_redirect_effect(connection, *args, **options): def ok_response(connection, *args, **options): assert connection.host == host_ip assert connection.port == 7001 return "MOCK_OK" parse_response.side_effect = ok_response raise AskError("1337 {0}:7001".format(host_ip)) parse_response.side_effect = ask_redirect_effect assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def monkey_link(host=None, port=None, decode_responses=False): """ Helper function to return custom slots cache data from different redis nodes """ if port == 7000: result = [[0, 5460, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]], [ 5461, 10922, [b'127.0.0.1', 7001], [b'127.0.0.1', 7004] ]] elif port == 7001: result = [[0, 5460, [b'127.0.0.1', 7001], [b'127.0.0.1', 7003]], [ 5461, 10922, [b'127.0.0.1', 7000], [b'127.0.0.1', 7004] ]] else: result = [] r = StrictRedisCluster(host=host, port=port, decode_responses=True) orig_execute_command = r.execute_command def execute_command(*args, **kwargs): if args == ("cluster", "slots"): return result return orig_execute_command(*args, **kwargs) r.execute_command = execute_command return r
def test_refresh_using_specific_nodes(r): """ Test making calls on specific nodes when the cluster has failed over to another node """ with patch.object(StrictRedisCluster, 'parse_response') as parse_response_mock: with patch.object(NodeManager, 'initialize', autospec=True) as init_mock: # simulate 7006 as a failed node def side_effect(self, *args, **kwargs): if self.port == 7006: parse_response_mock.failed_calls += 1 raise ClusterDownError('CLUSTERDOWN The cluster is down. Use CLUSTER INFO for more information') elif self.port == 7007: parse_response_mock.successful_calls += 1 def side_effect_rebuild_slots_cache(self): # start with all slots mapped to 7006 self.nodes = {'127.0.0.1:7006': {'host': '127.0.0.1', 'server_type': 'master', 'port': 7006, 'name': '127.0.0.1:7006'}} self.slots = {} for i in range(0, 16383): self.slots[i] = [{ 'host': '127.0.0.1', 'server_type': 'master', 'port': 7006, 'name': '127.0.0.1:7006', }] # After the first connection fails, a reinitialize should follow the cluster to 7007 def map_7007(self): self.nodes = {'127.0.0.1:7007': {'host': '127.0.0.1', 'server_type': 'master', 'port': 7007, 'name': '127.0.0.1:7007'}} self.slots = {} for i in range(0, 16383): self.slots[i] = [{ 'host': '127.0.0.1', 'server_type': 'master', 'port': 7007, 'name': '127.0.0.1:7007', }] init_mock.side_effect = map_7007 parse_response_mock.side_effect = side_effect parse_response_mock.successful_calls = 0 parse_response_mock.failed_calls = 0 init_mock.side_effect = side_effect_rebuild_slots_cache rc = StrictRedisCluster(host='127.0.0.1', port=7006) assert len(rc.connection_pool.nodes.nodes) == 1 assert '127.0.0.1:7006' in rc.connection_pool.nodes.nodes rc.ping() # Cluster should now point to 7006, and there should be one failed and one succesful call assert len(rc.connection_pool.nodes.nodes) == 1 assert '127.0.0.1:7007' in rc.connection_pool.nodes.nodes assert parse_response_mock.failed_calls == 1 assert parse_response_mock.successful_calls == 1
def test_cluster_of_one_instance(): """ Test a cluster that starts with only one redis server and ends up with one server. There is another redis server joining the cluster, hold slot 0, and eventually quit the cluster. The StrictRedisCluster instance may get confused when slots mapping and nodes change during the test. """ with patch.object(StrictRedisCluster, 'parse_response') as parse_response_mock: with patch.object(NodeManager, 'initialize', autospec=True) as init_mock: def side_effect(self, *args, **kwargs): def ok_call(self, *args, **kwargs): assert self.port == 7007 return "OK" parse_response_mock.side_effect = ok_call raise ClusterDownError('CLUSTERDOWN The cluster is down. Use CLUSTER INFO for more information') def side_effect_rebuild_slots_cache(self): # make new node cache that points to 7007 instead of 7006 self.nodes = [{'host': '127.0.0.1', 'server_type': 'master', 'port': 7006, 'name': '127.0.0.1:7006'}] self.slots = {} for i in range(0, 16383): self.slots[i] = [{ 'host': '127.0.0.1', 'server_type': 'master', 'port': 7006, 'name': '127.0.0.1:7006', }] # Second call should map all to 7007 def map_7007(self): self.nodes = [{'host': '127.0.0.1', 'server_type': 'master', 'port': 7007, 'name': '127.0.0.1:7007'}] self.slots = {} for i in range(0, 16383): self.slots[i] = [{ 'host': '127.0.0.1', 'server_type': 'master', 'port': 7007, 'name': '127.0.0.1:7007', }] # First call should map all to 7006 init_mock.side_effect = map_7007 parse_response_mock.side_effect = side_effect init_mock.side_effect = side_effect_rebuild_slots_cache rc = StrictRedisCluster(host='127.0.0.1', port=7006) rc.set("foo", "bar")
def redis_write(nodes, dict_data): """ 将字典写入redis集群,并计算时间 """ time1 = time.time() conn_rc = StrictRedisCluster(startup_nodes=nodes, decode_responses=True) for k in dict_data: # print k, func_dict[k] key_name = 'access_log' + '-' + str(k) conn_rc.set(key_name, dict_data[k]) # print conn_rc.get(key_name) time2 = time.time() time3 = time2 - time1 print time3
def redis_cluster(): redis_nodes = [ {'host':HOSTNAME,'port':7001}, {'host':HOSTNAME,'port':7002}, {'host':HOSTNAME,'port':7003}, {'host':HOSTNAME,'port':7004}, {'host':HOSTNAME,'port':7005}, {'host':HOSTNAME,'port':7006}, ] try: r = StrictRedisCluster(startup_nodes=redis_nodes) except Exception as e: print(e) else: r.set('test','test')
def test_clusterdown_exception_handling(): """ Test that if exception message starts with CLUSTERDOWN it should disconnect the connection pool and set refresh_table_asap to True. """ with patch.object(ClusterConnectionPool, 'disconnect') as mock_disconnect: with patch.object(ClusterConnectionPool, 'reset') as mock_reset: r = StrictRedisCluster(host="127.0.0.1", port=7000) i = len(mock_reset.mock_calls) assert r.handle_cluster_command_exception(Exception("CLUSTERDOWN")) == {"method": "clusterdown"} assert r.refresh_table_asap is True mock_disconnect.assert_called_once_with() # reset() should only be called once inside `handle_cluster_command_exception` assert len(mock_reset.mock_calls) - i == 1
class RedisPubSubHelper(): def __init__(self): # redis连接对象 self.__conn = StrictRedisCluster(startup_nodes=redis_nodes, decode_responses=True) def publish(self, channel, message): # redis对象的publish方法(发布) # 往指定的频道中发布信息 self.__conn.publish(channel, message) return True def subscribe(self, channel): # 返回了一个发布订阅的对象 pub = self.__conn.pubsub() # 订阅指定频道 pub.subscribe(channel) pub.parse_response() return pub
def test_refresh_table_asap(): """ If this variable is set externally, initialize() should be called. """ with patch.object(NodeManager, 'initialize') as mock_initialize: mock_initialize.return_value = None r = StrictRedisCluster(host="127.0.0.1", port=7000) r.connection_pool.nodes.slots[12182] = { "host": "127.0.0.1", "port": 7002, "name": "127.0.0.1:7002", "server_type": "master", } r.refresh_table_asap = True i = len(mock_initialize.mock_calls) r.execute_command("SET", "foo", "bar") assert len(mock_initialize.mock_calls) - i == 1 assert r.refresh_table_asap is False
class RedisCluster(object): def __init__(self, redis_nodes): self.cluster = StrictRedisCluster(startup_nodes=redis_nodes) # 无差别的方法 def set(self, name, value, ex=None, px=None, nx=False, xx=False): return self.cluster.set(name, value, ex, px, nx, xx) # 无差别的方法 def get(self, name): return self.cluster.get(name) # 扇形发送的命令 def cluster_info(self): return self.cluster.cluster_info() # 重写StrictRedis的方法 def mset(self, *args, **kwargs): return self.cluster.mset(args, kwargs) # 重写StrictRedis的方法 def mget(self, keys, *args): return self.cluster.mget(keys, args)
def __init__(self, name, host='localhost', port=6379, db=0, maxsize=0, lazy_limit=True, password=None, cluster_nodes=None): """ Constructor for RedisQueue maxsize: an integer that sets the upperbound limit on the number of items that can be placed in the queue. lazy_limit: redis queue is shared via instance, a lazy size limit is used for better performance. """ self.name = name if(cluster_nodes is not None): from rediscluster import StrictRedisCluster self.redis = StrictRedisCluster(startup_nodes=cluster_nodes) else: self.redis = redis.StrictRedis(host=host, port=port, db=db, password=password) self.maxsize = maxsize self.lazy_limit = lazy_limit self.last_qsize = 0
def test_api(self): comm.start_cluster('127.0.0.1', 7100) comm.join_cluster('127.0.0.1', 7100, '127.0.0.1', 7101) comm.replicate('127.0.0.1', 7100, '127.0.0.1', 7102) time.sleep(1) rc = StrictRedisCluster( startup_nodes=[{ 'host': '127.0.0.1', 'port': 7100 }], decode_responses=True) for i in range(20): rc.set('key_%s' % i, 'value_%s' % i) for i in range(20): self.assertEqual('value_%s' % i, rc.get('key_%s' % i)) nodes = base.list_nodes('127.0.0.1', 7100) self.assertEqual(3, len(nodes)) self.assertEqual( list(range(8192)), nodes[('127.0.0.1', 7101)].assigned_slots) self.assertEqual( list(range(8192, 16384)), nodes[('127.0.0.1', 7100)].assigned_slots) comm.quit_cluster('127.0.0.1', 7101) nodes = base.list_nodes('127.0.0.1', 7100) self.assertEqual( list(range(16384)), nodes[('127.0.0.1', 7100)].assigned_slots) for i in range(20): self.assertEqual('value_%s' % i, rc.get('key_%s' % i)) for i in range(20): rc.delete('key_%s' % i) comm.quit_cluster('127.0.0.1', 7102) comm.shutdown_cluster('127.0.0.1', 7100)
def test_quit_problems(self): comm.start_cluster('127.0.0.1', 7100) comm.join_cluster('127.0.0.1', 7100, '127.0.0.1', 7101) comm.replicate('127.0.0.1', 7100, '127.0.0.1', 7102) time.sleep(1) rc = StrictRedisCluster( startup_nodes=[{ 'host': '127.0.0.1', 'port': 7100 }], decode_responses=True) for i in range(20): rc.set('key_%s' % i, 'value_%s' % i) for i in range(20): self.assertEqual('value_%s' % i, rc.get('key_%s' % i)) nodes = base.list_nodes('127.0.0.1', 7100) self.assertEqual(3, len(nodes)) self.assertEqual( list(range(8192)), nodes[('127.0.0.1', 7101)].assigned_slots) self.assertEqual( list(range(8192, 16384)), nodes[('127.0.0.1', 7100)].assigned_slots) for i in range(20): rc.delete('key_%s' % i) six.assertRaisesRegex(self, ValueError, '^The master still has slaves$', comm.quit_cluster, '127.0.0.1', 7100) comm.quit_cluster('127.0.0.1', 7102) comm.quit_cluster('127.0.0.1', 7101) six.assertRaisesRegex(self, ValueError, '^This is the last node', comm.quit_cluster, '127.0.0.1', 7100) comm.shutdown_cluster('127.0.0.1', 7100)
from rediscluster.cluster_mgt import RedisClusterMgt from rediscluster import StrictRedisCluster startup_nodes = [{"host":"127.0.0.1","port":"7000"}, {"host":"127.0.0.1","port":"7001"}] rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) rc.set("foo", "bar") print(rc.get("foo"))
def __init__(self): # redis连接对象 self.__conn = StrictRedisCluster(startup_nodes=redis_nodes, decode_responses=True)
from rediscluster import StrictRedisCluster startup_nodes = [{"host": "192.168.1.168", "port": "7000"}] rc = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) ps = rc.pubsub() ps.subscribe(['foo', 'bar']) rc.publish('foo', 'hello foo from python') rc.publish('bar', 'hello bar from python')
# chapter 2 : redis cluster from rediscluster import StrictRedisCluster import csv import time def multiLoader(user_list, r): for email, user in user_list: r.set(email, user) startup_nodes = [ {"host":"127.0.0.1", "port":"6379"}, {"host":"127.0.0.1", "port":"6380"}, {"host":"127.0.0.1", "port":"6381"} ] rc = StrictRedisCluster(startup_nodes=startup_nodes) set_count = 0 start_time = int(time.time()) rc.flushall() with open('users.csv', 'rU') as fd: csvreader = csv.reader(fd) user_list = [] for line in csvreader: if csvreader.line_num == 1: continue email = line[0].strip() first_name = line[1].strip()
import time from rediscluster import StrictRedisCluster from rediscluster.exceptions import ClusterError from redis.exceptions import ConnectionError import argparse parser = argparse.ArgumentParser() parser.add_argument("-n", "--name", dest = "name", default = "a", help="Name that will be added to key") args = parser.parse_args() name = args.name current_milli_time = lambda: int(round(time.time() * 1000)) startup_nodes = [{"host":"10.240.0.8","port":"6379"},{"host":"10.240.0.10","port":"6379"},{"host":"10.240.0.12","port":"6379"}] r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) id = 0 sessions = {} deletes = [] #Create 100 sessions while id < 200: #Insert into redis sessions[id] = 0 start = current_milli_time() r.set(name + str(id),0) end = current_milli_time() print(str(end-start)+'ms new '+str(id)) id = id + 1
#!/usr/bin/env python # _*_coding:utf-8_*_ # Author: create by yang.hong # Time: 2018-08-17 11:06 """ 集群模式下,匹配方式删除多个key """ from rediscluster import StrictRedisCluster redis_cluster = [{'host': '172.28.246.12', 'port': '28000'}, {'host': '172.28.246.12', 'port': '28001'}, {'host': '172.28.246.12', 'port': '28002'}, {'host': '172.28.246.12', 'port': '28003'}, {'host': '172.28.246.12', 'port': '28004'}, {'host': '172.28.246.12', 'port': '28005'}, ] r = StrictRedisCluster(startup_nodes=redis_cluster, decode_responses=True) print r.keys(pattern="*mc_payChannel_router*") r.delete(*r.keys(pattern="*mc_payChannel_router*"))
def test_cluster_of_one_instance(): """ Test a cluster that starts with only one redis server and ends up with one server. There is another redis server joining the cluster, hold slot 0, and eventually quit the cluster. The StrictRedisCluster instance may get confused when slots mapping and nodes change during the test. """ with patch.object(StrictRedisCluster, "parse_response") as parse_response_mock: with patch.object(NodeManager, "initialize", autospec=True) as init_mock: def side_effect(self, *args, **kwargs): def ok_call(self, *args, **kwargs): assert self.port == 7007 return "OK" parse_response_mock.side_effect = ok_call raise ClusterDownError("CLUSTERDOWN The cluster is down. Use CLUSTER INFO for more information") def side_effect_rebuild_slots_cache(self): # make new node cache that points to 7007 instead of 7006 self.nodes = [{"host": "127.0.0.1", "server_type": "master", "port": 7006, "name": "127.0.0.1:7006"}] self.slots = {} for i in range(0, 16383): self.slots[i] = [ {"host": "127.0.0.1", "server_type": "master", "port": 7006, "name": "127.0.0.1:7006"} ] # Second call should map all to 7007 def map_7007(self): self.nodes = [ {"host": "127.0.0.1", "server_type": "master", "port": 7007, "name": "127.0.0.1:7007"} ] self.slots = {} for i in range(0, 16383): self.slots[i] = [ {"host": "127.0.0.1", "server_type": "master", "port": 7007, "name": "127.0.0.1:7007"} ] # First call should map all to 7006 init_mock.side_effect = map_7007 parse_response_mock.side_effect = side_effect init_mock.side_effect = side_effect_rebuild_slots_cache rc = StrictRedisCluster(host="127.0.0.1", port=7006) rc.set("foo", "bar") ##### # Test that CLUSTERDOWN is handled the same way when used via pipeline parse_response_mock.side_effect = side_effect init_mock.side_effect = side_effect_rebuild_slots_cache rc = StrictRedisCluster(host="127.0.0.1", port=7006) p = rc.pipeline() p.set("bar", "foo") p.execute()