def __init__(self, ci): log.debug('create connection = %s', ci) t = ci.type self.t = t if t == 1: log.debug('create redis connection.') self.conn = StrictRedis(host=ci.host, port=ci.port, db=ci.db) elif t == 2: log.debug('create redis cluster connection.') nodes = json.loads(ci.host) pool = ClusterConnectionPool(startup_nodes=nodes) self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True) elif t == 3: log.debug('create redis connection from zookeeper.') client = zk.Client(hosts=ci.host, read_only=True) node = client.get(ci.path) arr = str(node[0], encoding='utf-8').split('\n') address = [] for h in arr: if h is '': continue a = h.split(':') address.append({'host': a[0], 'port': int(a[1])}) pool = ClusterConnectionPool(startup_nodes=address) self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True) else: raise AttributeError('illegal ConnInfo type.') if self.test(): self.ci = ci log.info('connect redis(%s) success', ci.host)
def connect(host, port, passwd): startup_nodes = [{"host": host, "port": str(port)}] if passwd: return StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True, password=passwd) else: return StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
def __init__(self, redis_server, pw=None): #self.redis_client_ = redis.Redis(host='127.0.0.1', port=9966) if pw: pool = ClusterConnectionPool(startup_nodes=redis_server, password=pw, skip_full_coverage_check=True, decode_responses=True) self.redis_client_ = StrictRedisCluster(connection_pool=pool) else: pool = ClusterConnectionPool(startup_nodes=redis_server, skip_full_coverage_check=True, decode_responses=True) self.redis_client_ = StrictRedisCluster(connection_pool=pool)
def config_redis(self): revisions = None redis_nodes = self.get_nodes() try: if self.password == "": revisions = StrictRedisCluster(startup_nodes=redis_nodes) else: revisions = StrictRedisCluster(startup_nodes=redis_nodes, password=self.password) except Exception as e: logging.error(str(e)) return revisions
def test_redis_cluster_connect(): """ 测试redis集群链接。 :return: """ try: rds_cluster = StrictRedisCluster(startup_nodes=redis_nodes, password=redis_pass, decode_responses=True) except RedisClusterException: rds_cluster = StrictRedisCluster(startup_nodes=redis_nodes, decode_responses=True) value = rds_cluster.hget('my_hash', "my_key") common_logger.info(value)
def __init__(self, name, host='localhost', port=6379, db=0, maxsize=0, lazy_limit=True, password=None, cluster_nodes=None): """ Constructor for RedisQueue maxsize: an integer that sets the upperbound limit on the number of items that can be placed in the queue. lazy_limit: redis queue is shared via instance, a lazy size limit is used for better performance. """ self.name = name if (cluster_nodes is not None): from rediscluster import StrictRedisCluster # cluster_nodes = [ # {'host': '172.16.179.131', 'port': '7000'}, # {'host': '172.16.179.142', 'port': '7003'}, # {'host': '172.16.179.131', 'port': '7001'}, # ] self.redis = StrictRedisCluster( startup_nodes=cluster_nodes) # redis集群交互 startup_nodes为集群节点的配置 else: self.redis = redis.StrictRedis(host=host, port=port, db=db, password=password) self.maxsize = maxsize self.lazy_limit = lazy_limit self.last_qsize = 0
def redisCluster(classid, recordid): startup_nodes = [{ 'host': '192.168.40.112', 'port': '7000' }, { 'host': '192.168.40.133', 'port': '7000' }, { 'host': '192.168.40.141', 'port': '7000' }, { 'host': '192.168.40.112', 'port': '7001' }, { 'host': '192.168.40.133', 'port': '7001' }, { 'host': '192.168.40.141', 'port': '7001' }] r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) data1 = 'music-class_room-' + str(classid) data2 = 'music-class_record-' + str(recordid) rs = r.get(data1) rs1 = r.get(data2) print rs print rs1 # redisCluster(1814805, 12347)
def __init__(self, name='common', db=0, host=None, port=None): if host is None and port is None: self.config = DBConfigParser().get_config( server='redis_common_colony', key='colony') # self.config = DBConfigParser().get_config(server='redis_common_colony', key='105-62-93colony') self.host = self.config.get('host') self.port = self.config.get('port') self.db = self.config.get('db') if '|' in self.host: host_list = self.host.split('|') redis_nodes = [] for ho in host_list: redis_nodes.append({ 'host': str(ho), 'port': self.port, 'db': self.db }) self.conn = StrictRedisCluster(startup_nodes=redis_nodes) else: self.conn = redis.Redis(host=self.host, port=self.port, db=self.db) else: self.host = host self.port = port self.db = db self.conn = redis.Redis(host=self.host, port=self.port, db=self.db) self.name = name
def _get_redis_server(opts=None): """ Return the Redis server instance. Caching the object instance. """ global REDIS_SERVER if REDIS_SERVER: return REDIS_SERVER if not opts: opts = _get_redis_cache_opts() if opts["cluster_mode"]: REDIS_SERVER = StrictRedisCluster( startup_nodes=opts["startup_nodes"], skip_full_coverage_check=opts["skip_full_coverage_check"], ) else: REDIS_SERVER = redis.StrictRedis( opts["host"], opts["port"], unix_socket_path=opts["unix_socket_path"], db=opts["db"], password=opts["password"], ) return REDIS_SERVER
def redis_cluster(): redis_nodes = [{ 'host': '192.168.30.132', 'port': 7009 }, { 'host': '192.168.30.132', 'port': 7010 }, { 'host': '192.168.30.132', 'port': 7011 }, { 'host': '192.168.30.120', 'port': 7000 }, { 'host': '192.168.30.120', 'port': 7001 }, { 'host': '192.168.30.120', 'port': 7002 }] try: redisconn = StrictRedisCluster(startup_nodes=redis_nodes) except Exception, e: print "connect error" sys.exit(1)
def main(): startup_nodes = [{'host': 'redis3', 'port': '6379'}] r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) pubg_friends_length = r.llen( 'spider:python:pubg_friends:keyword:dest') # redis的数据量 print time.strftime( '[%Y-%m-%d %H:%M:%S]'), 'redis中pubg_friends的数据量:', pubg_friends_length lock = threading.Lock() first_hour = time.strftime('%H') # 小时 date = time.strftime('%Y%m%d') # 数据文件日期 dest_path = '/ftp_samba/112/spider/python/pubg' # linux上的文件目录 if not os.path.exists(dest_path): os.makedirs(dest_path) dest_file_name = os.path.join(dest_path, 'pubg_friends_' + date) fileout = open(dest_file_name, 'a') threads = [] for i in xrange(1): t = threading.Thread(target=pubg_friends, args=(lock, r, first_hour, fileout)) t.start() threads.append(t) for t in threads: t.join() try: fileout.flush() fileout.close() except IOError as e: time.sleep(2) fileout.close() print time.strftime('[%Y-%m-%d %H:%M:%S]:'), 'over'
def test_access_correct_slave_with_readonly_mode_client(sr): """ Test that the client can get value normally with readonly mode when we connect to correct slave. """ # we assume this key is set on 127.0.0.1:7000(7003) sr.set('foo16706', 'foo') import time time.sleep(1) with patch.object(ClusterReadOnlyConnectionPool, 'get_node_by_slot') as return_slave_mock: return_slave_mock.return_value = { 'name': '127.0.0.1:7003', 'host': '127.0.0.1', 'port': 7003, 'server_type': 'slave', } master_value = {'host': '127.0.0.1', 'name': '127.0.0.1:7000', 'port': 7000, 'server_type': 'master'} with patch.object( ClusterConnectionPool, 'get_master_node_by_slot', return_value=master_value) as return_master_mock: readonly_client = StrictRedisCluster(host="127.0.0.1", port=7000, readonly_mode=True) assert b('foo') == readonly_client.get('foo16706') assert return_master_mock.call_count == 0
def test_moved_redirection_pipeline(): """ Test that the server handles MOVED response when used in pipeline. At first call it should return a MOVED ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ with patch.object(StrictRedisCluster, 'parse_response') as parse_response: def moved_redirect_effect(connection, *args, **options): def ok_response(connection, *args, **options): assert connection.host == "127.0.0.1" assert connection.port == 7002 return "MOCK_OK" parse_response.side_effect = ok_response raise MovedError("12182 127.0.0.1:7002") parse_response.side_effect = moved_redirect_effect r = StrictRedisCluster(host="127.0.0.1", port=7000) p = r.pipeline() p.set("foo", "bar") assert p.execute() == ["MOCK_OK"]
def test_moved_redirection(): """ Test that the client handles MOVED response. At first call it should return a MOVED ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ r = StrictRedisCluster(host="127.0.0.1", port=7000) m = Mock(autospec=True) def ask_redirect_effect(connection, *args, **options): def ok_response(connection, *args, **options): assert connection.host == "127.0.0.1" assert connection.port == 7002 return "MOCK_OK" m.side_effect = ok_response raise MovedError("12182 127.0.0.1:7002") m.side_effect = ask_redirect_effect r.parse_response = m assert r.set("foo", "bar") == "MOCK_OK"
def test_ask_redirection(): """ Test that the server handles ASK response. At first call it should return a ASK ResponseError that will point the client to the next server it should talk to. Important thing to verify is that it tries to talk to the second node. """ r = StrictRedisCluster(host="127.0.0.1", port=7000) r.connection_pool.nodes.nodes['127.0.0.1:7001'] = { 'host': u'127.0.0.1', 'server_type': 'master', 'port': 7001, 'name': '127.0.0.1:7001' } with patch.object(StrictRedisCluster, 'parse_response') as parse_response: host_ip = find_node_ip_based_on_port(r, '7001') def ask_redirect_effect(connection, *args, **options): def ok_response(connection, *args, **options): assert connection.host == host_ip assert connection.port == 7001 return "MOCK_OK" parse_response.side_effect = ok_response raise AskError("1337 {0}:7001".format(host_ip)) parse_response.side_effect = ask_redirect_effect assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def conn_redis(cls): redis_node_list = [{ 'host': '10.162.129.38', 'port': '7001' }, { 'host': '10.162.129.38', 'port': '7002' }, { 'host': '10.162.133.41', 'port': '7001' }, { 'host': '10.162.133.41', 'port': '7002' }, { 'host': '10.162.133.42', 'port': '7001' }, { 'host': '10.162.133.42', 'port': '7002' }] nodes_index = randint(0, len(redis_node_list) - 1) #logging.info("********* connecting redis_node_list %d ************"%(nodes_index,)) try: return StrictRedisCluster( startup_nodes=[redis_node_list[nodes_index]], max_connections=500) except Exception, e: logging.error( "Filter Error connecting to database of redis cluster.") sys.exit(1)
def __init__(self, host='127.0.0.1', port=6379): self.r = StrictRedisCluster(startup_nodes=[{ "host": host, "port": port }], decode_responses=True, skip_full_coverage_check=True)
def main(): startup_nodes = [{'host': 'redis3', 'port': '6379'}] r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) gaode_length = r.llen('spider:python:gaode:keyword:dest') # redis的数据量 print time.strftime( '[%Y-%m-%d %H:%M:%S]'), 'redis中gaode的数据量:', gaode_length lock = threading.Lock() first_hour = time.strftime('%H') date = time.strftime('%Y%m%d') dest_path = '/ftp_samba/112/spider/python/gd_location/' # 线上数据存储文件 if not os.path.exists(dest_path): os.makedirs(dest_path) dest_file_name = os.path.join(dest_path, 'gd_location_' + date) fileout = open(dest_file_name, 'a') threads = [] for i in xrange(1): t = threading.Thread(target=gaode, args=(lock, r, first_hour, fileout)) t.start() threads.append(t) for t in threads: t.join() try: fileout.flush() fileout.close() except IOError as e: time.sleep(2) fileout.close()
def monkey_link(host=None, port=None, decode_responses=False): """ Helper function to return custom slots cache data from different redis nodes """ if port == 7000: result = [[0, 5460, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]], [ 5461, 10922, [b'127.0.0.1', 7001], [b'127.0.0.1', 7004] ]] elif port == 7001: result = [[0, 5460, [b'127.0.0.1', 7001], [b'127.0.0.1', 7003]], [ 5461, 10922, [b'127.0.0.1', 7000], [b'127.0.0.1', 7004] ]] else: result = [] r = StrictRedisCluster(host=host, port=port, decode_responses=True) orig_execute_command = r.execute_command def execute_command(*args, **kwargs): if args == ("cluster", "slots"): return result return orig_execute_command(*args, **kwargs) r.execute_command = execute_command return r
def __init__(self, key, startup_nodes): """ init cluster """ self.key = key self.conn = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
def __init__(self, redis_host, redis_port, redis_password, db=0): if isinstance(redis_host, list) and len(redis_host) == 1: redis_host = redis_host[0] if isinstance(redis_host, str): conn_pool = ConnectionPool(host=redis_host, port=redis_port, db=db, password=redis_password) self.redis_eng = StrictRedis( connection_pool=conn_pool, max_connections=10, ) elif isinstance(redis_host, list): if isinstance(redis_port, int): startup_nodes = [{ "host": host, "port": redis_port } for host in redis_host] elif isinstance(redis_port, list): startup_nodes = [{ "host": host, "port": port } for host, port in zip(redis_host, redis_port)] self.redis_eng = StrictRedisCluster(startup_nodes=startup_nodes, password=redis_password)
def redis_cluster(): redis_nodes = [{ 'host': '192.168.118.15', 'port': 6379 }, { 'host': '192.168.118.15', 'port': 6380 }, { 'host': '192.168.118.15', 'port': 6381 }, { 'host': '192.168.118.15', 'port': 6382 }, { 'host': '192.168.118.15', 'port': 6383 }, { 'host': '192.168.118.15', 'port': 6384 }] try: redisconn = StrictRedisCluster(startup_nodes=redis_nodes) except Exception as e: print("Connect Error!") sys.exit(1) # redisconn.set('name', 'hkey') # redisconn.set('age',18) print("name is: ", redisconn.get('name')) print("age is: ", redisconn.get('age'))
def config(self, cfg=None): """ The callback function for setting up the reader. New readers should implement this method. """ if cfg is None: cfg = default_conf print 'debug mode!!' print cfg self.fail_time = 0 self.th = float(cfg.get('th', 50)) self.addr = cfg['addr'].split(":") self.listfile = cfg['listfile'] self.prefix = cfg.get('prefix', "") self.delim = cfg.get('delim', " ,\t") self.shuffle_epoch_num = int(cfg.get('shuffle_epoch_num', 1)) self.shuffle = cfg.get('shuffle', True) print 'init iter next' self.iter = self._read_iter() self.startup_nodes = [{"host": self.addr[0], "port": self.addr[1]}] self.rc = StrictRedisCluster(startup_nodes=self.startup_nodes, decode_responses=False) if use_pool: try: self.pool = mp.Pool(processes=128) except KeyboardInterrupt as inst: print inst self.pool.close() self.pool.join() self.queue = collections.deque(maxlen=128)
def get_video_data(date): video_keys = get_list(date) redis_nodes = [{ 'host': '10.200.131.32', 'port': 6101 }, { 'host': '10.200.131.31', 'port': 6102 }, { 'host': '10.200.131.27', 'port': 6101 }, { 'host': '10.200.131.28', 'port': 6102 }] r = StrictRedisCluster(startup_nodes=redis_nodes) filepath = '../normal_knn/jobs/data/doc/doc_data_' + date fw = open(filepath, 'w') key_prefix = 'headline_' for v_key in video_keys: key = key_prefix + v_key video = r.get(key) if isinstance(video, basestring): fw.write(video + '\n') fw.close()