def __init__(self, key, value=None, host='localhost', port=6379, db=0, password=None, ttl=3600, redis_cluster=False):
self.key = key
self.ttl = ttl
self.value = value if value is not None else str(uuid.uuid4())
if redis_cluster:
self.redis = rediscluster.StrictRedisCluster(host=host, port=port, password=password)
else:
self.redis = redis.StrictRedis(host=host, port=port, db=db, password=password)
self._acquire_lock = self.redis.register_script("""
local key = KEYS[1]
local value = ARGV[1]
local ttl = ARGV[2]
local current_value = redis.call('GET', key)
if current_value == value or current_value == false then
redis.call('PSETEX', key, ttl, value)
return 1
end
return 0
""")
self._release = self.redis.register_script("""
local key = KEYS[1]
local value = ARGV[1]
if redis.call("GET", key) == value then
return redis.call("DEL", key)
else
return 0
end
""")
self.locked = False
self.expiration = None
def getClusterRedisClient(self):
try:
import rediscluster
except ImportError:
self.logger.error("Could not import rediscluster module. To install follow instructions @https://github.com/salimane/rediscluster-py")
self.lumbermill.shutDown()
# TODO: Implement a locking mechanism for the cluster client.
# Some modules like Facet depend on this.
cluster = {'nodes': {}, 'master_of': {}}
counter = 1
for master_node, slave_nodes in self.getConfigurationValue('cluster').items():
master_node_key = "node_%d" % counter
node_name_or_ip, node_port = self._parseRedisServerAddress(master_node)
cluster['nodes'].update({master_node_key: {'host': node_name_or_ip, 'port': node_port}})
if 'default_node' not in cluster:
cluster['default_node'] = master_node
if type(slave_nodes) is str:
slave_nodes = [slave_nodes]
for slave_node in slave_nodes:
counter += 1
slave_node_key = "node_%d" % counter
node_name_or_ip, node_port = self._parseRedisServerAddress(slave_node)
cluster['nodes'].update({slave_node_key: {'host':node_name_or_ip, 'port': node_port}})
cluster['master_of'].update({master_node_key: slave_node_key})
try:
client = rediscluster.StrictRedisCluster(cluster=cluster, db=self.getConfigurationValue('db'))
except:
etype, evalue, etb = sys.exc_info()
self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (self.getConfigurationValue['cluster'], etype, evalue))
self.lumbermill.shutDown()
return client
def __init__(self, redis_type=None, **args):
if redis_type == "cluster":
import rediscluster
self.r_conn = rediscluster.StrictRedisCluster(**args)
else:
import redis
self.r_conn = redis.StrictRedis(**args)
def redisCluster():
redis_nodes = [{
'host': '192.168.9.71',
'port': 20006
}, {
'host': '192.168.9.71',
'port': 20007
}, {
'host': '192.168.9.71',
'port': 20008
}, {
'host': '192.168.9.71',
'port': 20009
}, {
'host': '192.168.9.71',
'port': 20010
}, {
'host': '192.168.9.71',
'port': 20011
}]
try:
conn = rediscluster.StrictRedisCluster(startup_nodes=redis_nodes)
except Exception as e:
print e.message
print conn.info()
conn.set("fk:fk", 123)
print conn.get("fk:fk")
def redis_init(self, params):
global Redis_Cluster_instances, Core_param_list
try:
import rediscluster
except:
print("RUN $HOME/anaconda2/bin/pip install redis")
raise Exception("RUN $HOME/anaconda2/bin/pip install rediscluster")
m_db = params["db"] if "db" in params.keys() else 0
''' module level same instance: '''
core_params = {"nodes": self.cluster, "db": m_db}
ind = 0
while ind < len(Core_param_list):
if core_params == Core_param_list[ind]:
self.r = Redis_Cluster_instances[ind]
if iprint >= 2:
print("\n\nRe-using %d'th redis instance.\n\n" % ind)
break
ind += 1
if ind == len(Core_param_list):
if iprint >= 2: print("\n\nCreating new redis instance\n\n")
Core_param_list.append(
copy.deepcopy(core_params)) # must deep copy!!!!
self.r = rediscluster.StrictRedisCluster(cluster=self.cluster,
db=m_db)
Redis_Cluster_instances.append(self.r)
def test_store_create(self, mock_get_config):
"""An entry should be created in Redis."""
mock_get_config.return_value = {
'JWT_SECRET': self.secret,
'REDIS_HOST': os.environ.get('REDIS_HOST', 'localhost'),
'REDIS_PORT': os.environ.get('REDIS_PORT', '7000'),
'REDIS_CLUSTER': os.environ.get('REDIS_CLUSTER', '1')
}
ip = '127.0.0.1'
remote_host = 'foo-host.foo.com'
user = domain.User(
user_id='1',
username='******',
email='*****@*****.**',
)
authorizations = domain.Authorizations(
classic=2,
scopes=['foo:write'],
endorsements=[]
)
session = store.create(authorizations, ip, remote_host, user=user)
cookie = store.generate_cookie(session)
# API still works as expected.
self.assertIsInstance(session, domain.Session)
self.assertTrue(bool(session.session_id))
self.assertIsNotNone(cookie)
r = rediscluster.StrictRedisCluster(
startup_nodes=[dict(host='localhost', port='7000')]
)
raw = r.get(session.session_id)
stored_data = jwt.decode(raw, self.secret, algorithms=['HS256'])
cookie_data = jwt.decode(cookie, self.secret, algorithms=['HS256'])
self.assertEqual(stored_data['nonce'], cookie_data['nonce'])
def __init__(self, redis_key='ip_table', redis_cluster=False, **kwargs):
self.redis_key = redis_key
if redis_cluster:
self.redis = rediscluster.StrictRedisCluster(**kwargs)
else:
self.redis = redis.StrictRedis(**kwargs)
def getClusterConnection(self):
return rediscluster.StrictRedisCluster(startup_nodes=[{
'host':
'localhost',
'port':
self.shards[0].getMasterPort()
}],
decode_responses=True)
def __init__(self, redisIP="127.0.0.1", redisPort="7000"):
startupN = [{"host": redisIP, "port": redisPort}]
try:
self.r = rediscluster.StrictRedisCluster(startup_nodes=startupN)
except Exception as e:
self.rErr = e
else:
self.rErr = False
def conn_redis_cluster(self):
redis_nodes = [{"host": "127.0.0.1", "port": "6379"},
{"host": "127.0.0.1", "port": "6479"},
{"host": "127.0.0.1", "port": "7379"},
{"host": "127.0.0.1", "port": "7479"},
{"host": "127.0.0.1", "port": "8379"},
{"host": "127.0.0.1", "port": "8479"}
]
self.r = rediscluster.StrictRedisCluster(startup_nodes=redis_nodes)
def _get_test_client(self):
startup_nodes = [{
'host': self.TEST_HOST,
'port': int(port)
} for port in self.TEST_PORTS.split(',')]
if REDISCLUSTER_VERSION >= (2, 0, 0):
return rediscluster.RedisCluster(startup_nodes=startup_nodes)
else:
return rediscluster.StrictRedisCluster(startup_nodes=startup_nodes)
def check_rediscluster():
test_host = REDISCLUSTER_CONFIG['host']
test_ports = REDISCLUSTER_CONFIG['ports']
startup_nodes = [
{'host': test_host, 'port': int(port)}
for port in test_ports.split(',')
]
r = rediscluster.StrictRedisCluster(startup_nodes=startup_nodes)
r.flushall()
def __init__(self):
self.retry_count = 1
self.lock_labels = map(lambda x: "%i" % (x), range(0, 1))
self.uuid = str(uuid.uuid1())
config = get_config()
if config.redis_cluster_enabled():
startup_nodes = map(lambda x: {"host": x, "port": "%s" % (config.port())}, config.cluster_nodes())
self.redis_client = rediscluster.StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True)
else:
self.redis_client = redis.StrictRedis(host=config.host(), port=config.port(), db=0)
def redis_cluster(self):
redis_nodes = self.conf.getRedisNodes()
password = self.conf.getConfig("redisCluster", "password")
try:
redisconn = rediscluster.StrictRedisCluster(
startup_nodes=redis_nodes, password=password)
except Exception as e:
print 'redis连接失败，原因', format(e)
sys.exit(1)
return redisconn
def _connection(self):
if self.is_cluster:
self._connect = rediscluster.StrictRedisCluster(
startup_nodes=self.startup_nodes,
decode_responses=self.decode_responses)
else:
self._connect = redis.StrictRedis(host=self.host,
port=self.port,
db=self.db,
password=self.password,
**self.kwargs)
return self._connect
def _redis_client(opts):
'''
Connect to the redis host and return a StrictRedisCluster client object.
If connection fails then return None.
'''
redis_host = opts.get("eauth_redis_host", "localhost")
redis_port = opts.get("eauth_redis_port", 6379)
try:
return rediscluster.StrictRedisCluster(host=redis_host,
port=redis_port)
except rediscluster.exceptions.RedisClusterException as err:
log.warning('Failed to connect to redis at %s:%s - %s', redis_host,
redis_port, err)
return None
def redis_type(request):
if request.param == REDIS_REGULAR:
os.environ["REDIS_HOST"] = 'redis'
os.environ["REDIS_PORT"] = '6379'
r = redis.StrictRedis(host=os.getenv('REDIS_HOST'),
port=os.getenv('REDIS_PORT'),
db=TEST_DB)
else:
os.environ["REDIS_HOST"] = 'redis-cluster'
os.environ["REDIS_PORT"] = '7000'
r = rediscluster.StrictRedisCluster(host=os.getenv('REDIS_HOST'),
port=os.getenv('REDIS_PORT'))
r.flushdb()
return request.param
def initialize(self, nodes):
"""
初始化connection
:param nodes: [{"host": "172.0.0.1", "port": 6379}...]
:return:
"""
try:
self._connection = rediscluster.StrictRedisCluster(
startup_nodes=nodes, skip_full_coverage_check=True)
except:
logging.error("Failed to connect to redis cluster")
sys.exit(0)
return -1
return 0
def main():
if os.path.exists('/tmp/stopredis'):
os.unlink('/tmp/stopredis')
manager = multiprocessing.Manager()
result = manager.dict()
result['generator_enabled'] = True
result['total'] = 0
config = manager.dict()
config['host'] = os.environ.get('REDIS_HOST', None) or DEFAULT_REDIS_HOST
config['port'] = os.environ.get('REDIS_PORT', None) or DEFAULT_REDIS_PORT
queue_data = multiprocessing.JoinableQueue()
print('Configuration: {}'.format(config))
print('Allocating feeders')
feeders = [Feeder(queue_data, config, result) for _ in xrange(WORKERS)]
for feeder in feeders:
feeder.start()
print('Allocating generator')
generator = IBANGenerator(queue_data, result)
generator.start()
startup_nodes = [{'host': config['host'], 'port': config['port']}]
client = rediscluster.StrictRedisCluster(startup_nodes=startup_nodes,
decode_responses=True,
readonly_mode=False,
skip_full_coverage_check=True)
while True:
sleep(1)
if os.path.exists('/tmp/stopredis'):
break
if result['total'] == 0:
continue
if result['total'] % 100000 == 0 and client.get(
IBANGenerator.CANARY[0]) is None:
break
if result['total'] % 10000 == 0:
print('Generation in progress [{}]'.format(result['total']))
# canary lost poisoning pill
result['generator_enabled'] = False
queue_data.join()
for _ in feeders:
queue_data.put(None)
print('WAITING FOR FEEDER PROCESSES')
queue_data.join()
for p in feeders:
p.join()
print('JOB DONE!')
print('RESULTS\n{}'.format(result))
def test_delete_session(self, mock_get_config):
"""Delete a session from the datastore."""
mock_get_config.return_value = {
'JWT_SECRET': self.secret,
'REDIS_HOST': os.environ.get('REDIS_HOST', 'localhost'),
'REDIS_PORT': os.environ.get('REDIS_PORT', '7000'),
'REDIS_CLUSTER': os.environ.get('REDIS_CLUSTER', '1')
}
r = rediscluster.StrictRedisCluster(
startup_nodes=[dict(host='localhost', port='7000')]
)
r.set('fookey', b'foovalue')
store.delete_by_id('fookey')
self.assertIsNone(r.get('fookey'))
def __init__(self):
config = get_config()
startup_nodes = map(
lambda x: {
"host": x,
"port": "%s" % (config.port())
}, config.cluster_nodes())
self.redis_client = rediscluster.StrictRedisCluster(
startup_nodes=startup_nodes, decode_responses=True)
self.dlm = Redlock([
{
"host": "localhost",
"port": 6379,
"db": 0
},
],
retry_count=10)
def connection(self):
startup_nodes = list()
for item in self.redis_hosts:
pair = dict()
pair['host'], pair['port'] = item.split(':')
startup_nodes.append(pair)
if len(startup_nodes) > 1:
import rediscluster
self._conn = rediscluster.StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True,
retry_on_timeout=True)
elif len(startup_nodes) == 1:
import redis
self._conn = redis.StrictRedis(host=startup_nodes[0]['host'], port=startup_nodes[0]['port'],
retry_on_timeout=True)
else:
self._conn = None
return self._conn
def __init__(self,
host: str,
port: int,
db: int,
secret: str,
duration: int = 7200,
token: str = None) -> None:
"""Open the connection to Redis."""
# params = #, db=db)
logger.debug('New Redis connection at %s, port %s', host, port)
self.r = rediscluster.StrictRedisCluster(startup_nodes=[{
'host':
host,
'port':
str(port)
}],
skip_full_coverage_check=True)
self._secret = secret
self._duration = duration
def __init__(self, host: str, port: int, db: int, secret: str,
duration: int = 7200, token: Optional[str] = None,
cluster: bool = True, fake: bool = False) -> None:
"""Open the connection to Redis."""
self._secret = secret
self._duration = duration
if fake:
logger.warn('Using FakeRedis')
import fakeredis # this is a dev dependency needed during testing
self.r = fakeredis.FakeStrictRedis()
else:
logger.debug('New Redis connection at %s, port %s', host, port)
if cluster:
self.r = rediscluster.StrictRedisCluster(
startup_nodes=[{'host': host, 'port': str(port)}],
skip_full_coverage_check=True
)
else:
self.r = redis.StrictRedis(host=host, port=port)
def __init__(self, task_queue, config, result):
"""
:type task_queue: multiprocessing.JoinableQueue
"""
super(Feeder, self).__init__()
self.task_queue = task_queue
self.config = config
self.result = result
self.result[self.name] = 0
# self.client = redis.StrictRedis(host=config['host'], port=config['port'])
startup_nodes = [{
'host': self.config['host'],
'port': self.config['port']
}]
self.client = rediscluster.StrictRedisCluster(
startup_nodes=startup_nodes,
decode_responses=True,
readonly_mode=False,
skip_full_coverage_check=True)
def __init__(self):
config = get_config()
if config.redis_cluster_enabled():
startup_nodes = map(
lambda x: {
"host": x,
"port": "%s" % (config.port())
}, config.cluster_nodes())
self.redis_client = rediscluster.StrictRedisCluster(
startup_nodes=startup_nodes, decode_responses=True)
else:
self.redis_client = redis.StrictRedis(host=config.host(),
port=config.port(),
db=0)
self.cluster = Cluster(contact_points=config.cluster_nodes(),
protocol_version=3)
self.session = self.cluster.connect('mykeyspace')
self.session.row_factory = decoded_dict_factory
self._tables = {}
self.table_columns_metadata = {}
def init(config_file_path=None, **kwargs):
'''Init the storage client. Basically, we set the Redis client and connects it
to the instance/cluster
'''
global redis_connection
global hosts
# If config_file_path is None we will assume that we only have localhost
# as storage node
if config_file_path is None:
try:
import StringIO as sio
except ImportError:
from io import StringIO as sio
config_file_handler = sio.StringIO('localhost\n')
else:
config_file_handler = open(config_file_path)
# As accorded in the API standar, this file must contain all the hosts names
# with no port, one per line
hosts = [x.strip() for x in config_file_handler.readlines()]
config_file_handler.close()
# If we have more than one host then we will assume that our backend is a Redis
# cluster. If not, we will assume that we are dealing with a Redis standalone
# instance
if len(hosts) > 1:
# Given that cluster clients are capable to perform master
# slave hierarchy discovery, we will simply connect to the first
# node we got
redis_connection = \
rediscluster.StrictRedisCluster(host=hosts[0], port=REDIS_PORT)
else:
# We are in standalone mode
redis_connection = \
redis.StrictRedis(host=hosts[0], port=REDIS_PORT)
# StrictRedis is not capable to know if we had success when connecting by
# simply calling the constructor. We need to perform an actual query to
# the backend
# If we had no success this first line should crash
redis_connection.set('PYCOMPSS_TEST', 'OK')
# Beware py2 vs py3 - b'string' works for both.
assert redis_connection.get('PYCOMPSS_TEST') == b'OK'
redis_connection.delete('PYCOMPSS_TEST')
def __init__(self):
list_host = REDIS.split(",")
redis_nodes = []
for i in list_host:
i = i.split(":")
redis_nodes.append({'host': i[0], 'port': i[1]})
# list_host = HOST.split(",")
# list_port = PORT.split(",")
# redis_nodes = [
# {'host': list_host[0], 'port': list_port[0]},
# {'host': list_host[0], 'port': list_port[1]},
# {'host': list_host[0], 'port': list_port[2]},
# {'host': list_host[0], 'port': list_port[3]},
# {'host': list_host[0], 'port': list_port[4]},
# {'host': list_host[0], 'port': list_port[5]},
# ]
self.cluster = rediscluster.StrictRedisCluster(
startup_nodes=redis_nodes)
def __init__(self,
domain,
hosts_and_ports='localhost:6379',
password=None,
event_broker=None,
record_exp=None):
logger.debug('Using redis cluster {}'.format(hosts_and_ports))
host_and_port = hosts_and_ports.split(',')
startup_nodes = []
for hap in host_and_port:
part = hap.split(':')
tmp = dict()
if (len(part) == 2):
tmp['host'] = part[0]
tmp['port'] = part[1]
startup_nodes.append(tmp)
else:
logger.warning('Parsing error in host and port {}'.format(hap))
import rediscluster
self.red = rediscluster.StrictRedisCluster(startup_nodes=startup_nodes,
password=password)
self.record_exp = record_exp
super(RedisClusterTrackerStore, self).__init__(domain, event_broker)
def get_redis_client(redis_type='single',
host='127.0.0.1',
port=6379,
db=0,
pwd=None,
nodes=None,
timeout=3):
if redis_type == 'single':
pool = redis.ConnectionPool(host=host,
port=port,
db=db,
password=pwd,
socket_timeout=timeout,
socket_connect_timeout=timeout,
encoding='utf-8',
decode_responses=True)
client = redis.StrictRedis(connection_pool=pool)
else:
client = rediscluster.StrictRedisCluster(
startup_nodes=nodes,
decode_responses=True,
socket_timeout=timeout,
socket_connect_timeout=timeout)
return client
