def __init__(self, ci):
log.debug('create connection = %s', ci)
t = ci.type
self.t = t
if t == 1:
log.debug('create redis connection.')
self.conn = StrictRedis(host=ci.host, port=ci.port, db=ci.db)
elif t == 2:
log.debug('create redis cluster connection.')
nodes = json.loads(ci.host)
pool = ClusterConnectionPool(startup_nodes=nodes)
self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True)
elif t == 3:
log.debug('create redis connection from zookeeper.')
client = zk.Client(hosts=ci.host, read_only=True)
node = client.get(ci.path)
arr = str(node[0], encoding='utf-8').split('\n')
address = []
for h in arr:
if h is '':
continue
a = h.split(':')
address.append({'host': a[0], 'port': int(a[1])})
pool = ClusterConnectionPool(startup_nodes=address)
self.conn = StrictRedisCluster(connection_pool=pool, decode_responses=True)
else:
raise AttributeError('illegal ConnInfo type.')
if self.test():
self.ci = ci
log.info('connect redis(%s) success', ci.host)
def connect(host, port, passwd):
startup_nodes = [{"host": host, "port": str(port)}]
if passwd:
return StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True, password=passwd)
else:
return StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
def __init__(self, redis_server, pw=None):
#self.redis_client_ = redis.Redis(host='127.0.0.1', port=9966)
if pw:
pool = ClusterConnectionPool(startup_nodes=redis_server, password=pw, skip_full_coverage_check=True, decode_responses=True)
self.redis_client_ = StrictRedisCluster(connection_pool=pool)
else:
pool = ClusterConnectionPool(startup_nodes=redis_server, skip_full_coverage_check=True, decode_responses=True)
self.redis_client_ = StrictRedisCluster(connection_pool=pool)
def config_redis(self):
revisions = None
redis_nodes = self.get_nodes()
try:
if self.password == "":
revisions = StrictRedisCluster(startup_nodes=redis_nodes)
else:
revisions = StrictRedisCluster(startup_nodes=redis_nodes, password=self.password)
except Exception as e:
logging.error(str(e))
return revisions
def test_redis_cluster_connect():
"""
测试redis集群链接。
:return:
"""
try:
rds_cluster = StrictRedisCluster(startup_nodes=redis_nodes,
password=redis_pass,
decode_responses=True)
except RedisClusterException:
rds_cluster = StrictRedisCluster(startup_nodes=redis_nodes,
decode_responses=True)
value = rds_cluster.hget('my_hash', "my_key")
common_logger.info(value)
def __init__(self,
name,
host='localhost',
port=6379,
db=0,
maxsize=0,
lazy_limit=True,
password=None,
cluster_nodes=None):
"""
Constructor for RedisQueue
maxsize: an integer that sets the upperbound limit on the number of
items that can be placed in the queue.
lazy_limit: redis queue is shared via instance, a lazy size limit is used
for better performance.
"""
self.name = name
if (cluster_nodes is not None):
from rediscluster import StrictRedisCluster
# cluster_nodes = [
# {'host': '172.16.179.131', 'port': '7000'},
# {'host': '172.16.179.142', 'port': '7003'},
# {'host': '172.16.179.131', 'port': '7001'},
# ]
self.redis = StrictRedisCluster(
startup_nodes=cluster_nodes) # redis集群交互 startup_nodes为集群节点的配置
else:
self.redis = redis.StrictRedis(host=host,
port=port,
db=db,
password=password)
self.maxsize = maxsize
self.lazy_limit = lazy_limit
self.last_qsize = 0
def redisCluster(classid, recordid):
startup_nodes = [{
'host': '192.168.40.112',
'port': '7000'
}, {
'host': '192.168.40.133',
'port': '7000'
}, {
'host': '192.168.40.141',
'port': '7000'
}, {
'host': '192.168.40.112',
'port': '7001'
}, {
'host': '192.168.40.133',
'port': '7001'
}, {
'host': '192.168.40.141',
'port': '7001'
}]
r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
data1 = 'music-class_room-' + str(classid)
data2 = 'music-class_record-' + str(recordid)
rs = r.get(data1)
rs1 = r.get(data2)
print rs
print rs1
# redisCluster(1814805, 12347)
def __init__(self, name='common', db=0, host=None, port=None):
if host is None and port is None:
self.config = DBConfigParser().get_config(
server='redis_common_colony', key='colony')
# self.config = DBConfigParser().get_config(server='redis_common_colony', key='105-62-93colony')
self.host = self.config.get('host')
self.port = self.config.get('port')
self.db = self.config.get('db')
if '|' in self.host:
host_list = self.host.split('|')
redis_nodes = []
for ho in host_list:
redis_nodes.append({
'host': str(ho),
'port': self.port,
'db': self.db
})
self.conn = StrictRedisCluster(startup_nodes=redis_nodes)
else:
self.conn = redis.Redis(host=self.host,
port=self.port,
db=self.db)
else:
self.host = host
self.port = port
self.db = db
self.conn = redis.Redis(host=self.host, port=self.port, db=self.db)
self.name = name
def _get_redis_server(opts=None):
"""
Return the Redis server instance.
Caching the object instance.
"""
global REDIS_SERVER
if REDIS_SERVER:
return REDIS_SERVER
if not opts:
opts = _get_redis_cache_opts()
if opts["cluster_mode"]:
REDIS_SERVER = StrictRedisCluster(
startup_nodes=opts["startup_nodes"],
skip_full_coverage_check=opts["skip_full_coverage_check"],
)
else:
REDIS_SERVER = redis.StrictRedis(
opts["host"],
opts["port"],
unix_socket_path=opts["unix_socket_path"],
db=opts["db"],
password=opts["password"],
)
return REDIS_SERVER
def redis_cluster():
redis_nodes = [{
'host': '192.168.30.132',
'port': 7009
}, {
'host': '192.168.30.132',
'port': 7010
}, {
'host': '192.168.30.132',
'port': 7011
}, {
'host': '192.168.30.120',
'port': 7000
}, {
'host': '192.168.30.120',
'port': 7001
}, {
'host': '192.168.30.120',
'port': 7002
}]
try:
redisconn = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception, e:
print "connect error"
sys.exit(1)
def main():
startup_nodes = [{'host': 'redis3', 'port': '6379'}]
r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
pubg_friends_length = r.llen(
'spider:python:pubg_friends:keyword:dest') # redis的数据量
print time.strftime(
'[%Y-%m-%d %H:%M:%S]'), 'redis中pubg_friends的数据量：', pubg_friends_length
lock = threading.Lock()
first_hour = time.strftime('%H') # 小时
date = time.strftime('%Y%m%d') # 数据文件日期
dest_path = '/ftp_samba/112/spider/python/pubg' # linux上的文件目录
if not os.path.exists(dest_path):
os.makedirs(dest_path)
dest_file_name = os.path.join(dest_path, 'pubg_friends_' + date)
fileout = open(dest_file_name, 'a')
threads = []
for i in xrange(1):
t = threading.Thread(target=pubg_friends,
args=(lock, r, first_hour, fileout))
t.start()
threads.append(t)
for t in threads:
t.join()
try:
fileout.flush()
fileout.close()
except IOError as e:
time.sleep(2)
fileout.close()
print time.strftime('[%Y-%m-%d %H:%M:%S]：'), 'over'
def test_access_correct_slave_with_readonly_mode_client(sr):
"""
Test that the client can get value normally with readonly mode
when we connect to correct slave.
"""
# we assume this key is set on 127.0.0.1:7000(7003)
sr.set('foo16706', 'foo')
import time
time.sleep(1)
with patch.object(ClusterReadOnlyConnectionPool, 'get_node_by_slot') as return_slave_mock:
return_slave_mock.return_value = {
'name': '127.0.0.1:7003',
'host': '127.0.0.1',
'port': 7003,
'server_type': 'slave',
}
master_value = {'host': '127.0.0.1', 'name': '127.0.0.1:7000', 'port': 7000, 'server_type': 'master'}
with patch.object(
ClusterConnectionPool,
'get_master_node_by_slot',
return_value=master_value) as return_master_mock:
readonly_client = StrictRedisCluster(host="127.0.0.1", port=7000, readonly_mode=True)
assert b('foo') == readonly_client.get('foo16706')
assert return_master_mock.call_count == 0
def test_moved_redirection_pipeline():
"""
Test that the server handles MOVED response when used in pipeline.
At first call it should return a MOVED ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
with patch.object(StrictRedisCluster, 'parse_response') as parse_response:
def moved_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7002
return "MOCK_OK"
parse_response.side_effect = ok_response
raise MovedError("12182 127.0.0.1:7002")
parse_response.side_effect = moved_redirect_effect
r = StrictRedisCluster(host="127.0.0.1", port=7000)
p = r.pipeline()
p.set("foo", "bar")
assert p.execute() == ["MOCK_OK"]
def test_moved_redirection():
"""
Test that the client handles MOVED response.
At first call it should return a MOVED ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
m = Mock(autospec=True)
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == "127.0.0.1"
assert connection.port == 7002
return "MOCK_OK"
m.side_effect = ok_response
raise MovedError("12182 127.0.0.1:7002")
m.side_effect = ask_redirect_effect
r.parse_response = m
assert r.set("foo", "bar") == "MOCK_OK"
def test_ask_redirection():
"""
Test that the server handles ASK response.
At first call it should return a ASK ResponseError that will point
the client to the next server it should talk to.
Important thing to verify is that it tries to talk to the second node.
"""
r = StrictRedisCluster(host="127.0.0.1", port=7000)
r.connection_pool.nodes.nodes['127.0.0.1:7001'] = {
'host': u'127.0.0.1',
'server_type': 'master',
'port': 7001,
'name': '127.0.0.1:7001'
}
with patch.object(StrictRedisCluster,
'parse_response') as parse_response:
host_ip = find_node_ip_based_on_port(r, '7001')
def ask_redirect_effect(connection, *args, **options):
def ok_response(connection, *args, **options):
assert connection.host == host_ip
assert connection.port == 7001
return "MOCK_OK"
parse_response.side_effect = ok_response
raise AskError("1337 {0}:7001".format(host_ip))
parse_response.side_effect = ask_redirect_effect
assert r.execute_command("SET", "foo", "bar") == "MOCK_OK"
def conn_redis(cls):
redis_node_list = [{
'host': '10.162.129.38',
'port': '7001'
}, {
'host': '10.162.129.38',
'port': '7002'
}, {
'host': '10.162.133.41',
'port': '7001'
}, {
'host': '10.162.133.41',
'port': '7002'
}, {
'host': '10.162.133.42',
'port': '7001'
}, {
'host': '10.162.133.42',
'port': '7002'
}]
nodes_index = randint(0, len(redis_node_list) - 1)
#logging.info("********* connecting redis_node_list %d ************"%(nodes_index,))
try:
return StrictRedisCluster(
startup_nodes=[redis_node_list[nodes_index]],
max_connections=500)
except Exception, e:
logging.error(
"Filter Error connecting to database of redis cluster.")
sys.exit(1)
def __init__(self, host='127.0.0.1', port=6379):
self.r = StrictRedisCluster(startup_nodes=[{
"host": host,
"port": port
}],
decode_responses=True,
skip_full_coverage_check=True)
def main():
startup_nodes = [{'host': 'redis3', 'port': '6379'}]
r = StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
gaode_length = r.llen('spider:python:gaode:keyword:dest') # redis的数据量
print time.strftime(
'[%Y-%m-%d %H:%M:%S]'), 'redis中gaode的数据量：', gaode_length
lock = threading.Lock()
first_hour = time.strftime('%H')
date = time.strftime('%Y%m%d')
dest_path = '/ftp_samba/112/spider/python/gd_location/' # 线上数据存储文件
if not os.path.exists(dest_path):
os.makedirs(dest_path)
dest_file_name = os.path.join(dest_path, 'gd_location_' + date)
fileout = open(dest_file_name, 'a')
threads = []
for i in xrange(1):
t = threading.Thread(target=gaode, args=(lock, r, first_hour, fileout))
t.start()
threads.append(t)
for t in threads:
t.join()
try:
fileout.flush()
fileout.close()
except IOError as e:
time.sleep(2)
fileout.close()
def monkey_link(host=None, port=None, decode_responses=False):
"""
Helper function to return custom slots cache data from different redis nodes
"""
if port == 7000:
result = [[0, 5460, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]],
[
5461, 10922, [b'127.0.0.1', 7001],
[b'127.0.0.1', 7004]
]]
elif port == 7001:
result = [[0, 5460, [b'127.0.0.1', 7001], [b'127.0.0.1', 7003]],
[
5461, 10922, [b'127.0.0.1', 7000],
[b'127.0.0.1', 7004]
]]
else:
result = []
r = StrictRedisCluster(host=host, port=port, decode_responses=True)
orig_execute_command = r.execute_command
def execute_command(*args, **kwargs):
if args == ("cluster", "slots"):
return result
return orig_execute_command(*args, **kwargs)
r.execute_command = execute_command
return r
def __init__(self, key, startup_nodes):
"""
init cluster
"""
self.key = key
self.conn = StrictRedisCluster(startup_nodes=startup_nodes,
decode_responses=True)
def __init__(self, redis_host, redis_port, redis_password, db=0):
if isinstance(redis_host, list) and len(redis_host) == 1:
redis_host = redis_host[0]
if isinstance(redis_host, str):
conn_pool = ConnectionPool(host=redis_host,
port=redis_port,
db=db,
password=redis_password)
self.redis_eng = StrictRedis(
connection_pool=conn_pool,
max_connections=10,
)
elif isinstance(redis_host, list):
if isinstance(redis_port, int):
startup_nodes = [{
"host": host,
"port": redis_port
} for host in redis_host]
elif isinstance(redis_port, list):
startup_nodes = [{
"host": host,
"port": port
} for host, port in zip(redis_host, redis_port)]
self.redis_eng = StrictRedisCluster(startup_nodes=startup_nodes,
password=redis_password)
def redis_cluster():
redis_nodes = [{
'host': '192.168.118.15',
'port': 6379
}, {
'host': '192.168.118.15',
'port': 6380
}, {
'host': '192.168.118.15',
'port': 6381
}, {
'host': '192.168.118.15',
'port': 6382
}, {
'host': '192.168.118.15',
'port': 6383
}, {
'host': '192.168.118.15',
'port': 6384
}]
try:
redisconn = StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print("Connect Error!")
sys.exit(1)
# redisconn.set('name', 'hkey')
# redisconn.set('age',18)
print("name is: ", redisconn.get('name'))
print("age is: ", redisconn.get('age'))
def config(self, cfg=None):
"""
The callback function for setting up the reader.
New readers should implement this method.
"""
if cfg is None:
cfg = default_conf
print 'debug mode!!'
print cfg
self.fail_time = 0
self.th = float(cfg.get('th', 50))
self.addr = cfg['addr'].split(":")
self.listfile = cfg['listfile']
self.prefix = cfg.get('prefix', "")
self.delim = cfg.get('delim', " ,\t")
self.shuffle_epoch_num = int(cfg.get('shuffle_epoch_num', 1))
self.shuffle = cfg.get('shuffle', True)
print 'init iter next'
self.iter = self._read_iter()
self.startup_nodes = [{"host": self.addr[0], "port": self.addr[1]}]
self.rc = StrictRedisCluster(startup_nodes=self.startup_nodes,
decode_responses=False)
if use_pool:
try:
self.pool = mp.Pool(processes=128)
except KeyboardInterrupt as inst:
print inst
self.pool.close()
self.pool.join()
self.queue = collections.deque(maxlen=128)
def get_video_data(date):
video_keys = get_list(date)
redis_nodes = [{
'host': '10.200.131.32',
'port': 6101
}, {
'host': '10.200.131.31',
'port': 6102
}, {
'host': '10.200.131.27',
'port': 6101
}, {
'host': '10.200.131.28',
'port': 6102
}]
r = StrictRedisCluster(startup_nodes=redis_nodes)
filepath = '../normal_knn/jobs/data/doc/doc_data_' + date
fw = open(filepath, 'w')
key_prefix = 'headline_'
for v_key in video_keys:
key = key_prefix + v_key
video = r.get(key)
if isinstance(video, basestring):
fw.write(video + '\n')
fw.close()
