def __init__(self, key, value=None, host='localhost', port=6379, db=0, password=None, ttl=3600, redis_cluster=False): self.key = key self.ttl = ttl self.value = value if value is not None else str(uuid.uuid4()) if redis_cluster: self.redis = rediscluster.StrictRedisCluster(host=host, port=port, password=password) else: self.redis = redis.StrictRedis(host=host, port=port, db=db, password=password) self._acquire_lock = self.redis.register_script(""" local key = KEYS[1] local value = ARGV[1] local ttl = ARGV[2] local current_value = redis.call('GET', key) if current_value == value or current_value == false then redis.call('PSETEX', key, ttl, value) return 1 end return 0 """) self._release = self.redis.register_script(""" local key = KEYS[1] local value = ARGV[1] if redis.call("GET", key) == value then return redis.call("DEL", key) else return 0 end """) self.locked = False self.expiration = None
def getClusterRedisClient(self): try: import rediscluster except ImportError: self.logger.error("Could not import rediscluster module. To install follow instructions @https://github.com/salimane/rediscluster-py") self.lumbermill.shutDown() # TODO: Implement a locking mechanism for the cluster client. # Some modules like Facet depend on this. cluster = {'nodes': {}, 'master_of': {}} counter = 1 for master_node, slave_nodes in self.getConfigurationValue('cluster').items(): master_node_key = "node_%d" % counter node_name_or_ip, node_port = self._parseRedisServerAddress(master_node) cluster['nodes'].update({master_node_key: {'host': node_name_or_ip, 'port': node_port}}) if 'default_node' not in cluster: cluster['default_node'] = master_node if type(slave_nodes) is str: slave_nodes = [slave_nodes] for slave_node in slave_nodes: counter += 1 slave_node_key = "node_%d" % counter node_name_or_ip, node_port = self._parseRedisServerAddress(slave_node) cluster['nodes'].update({slave_node_key: {'host':node_name_or_ip, 'port': node_port}}) cluster['master_of'].update({master_node_key: slave_node_key}) try: client = rediscluster.StrictRedisCluster(cluster=cluster, db=self.getConfigurationValue('db')) except: etype, evalue, etb = sys.exc_info() self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (self.getConfigurationValue['cluster'], etype, evalue)) self.lumbermill.shutDown() return client
def __init__(self, redis_type=None, **args): if redis_type == "cluster": import rediscluster self.r_conn = rediscluster.StrictRedisCluster(**args) else: import redis self.r_conn = redis.StrictRedis(**args)
def redisCluster(): redis_nodes = [{ 'host': '192.168.9.71', 'port': 20006 }, { 'host': '192.168.9.71', 'port': 20007 }, { 'host': '192.168.9.71', 'port': 20008 }, { 'host': '192.168.9.71', 'port': 20009 }, { 'host': '192.168.9.71', 'port': 20010 }, { 'host': '192.168.9.71', 'port': 20011 }] try: conn = rediscluster.StrictRedisCluster(startup_nodes=redis_nodes) except Exception as e: print e.message print conn.info() conn.set("fk:fk", 123) print conn.get("fk:fk")
def redis_init(self, params): global Redis_Cluster_instances, Core_param_list try: import rediscluster except: print("RUN $HOME/anaconda2/bin/pip install redis") raise Exception("RUN $HOME/anaconda2/bin/pip install rediscluster") m_db = params["db"] if "db" in params.keys() else 0 ''' module level same instance: ''' core_params = {"nodes": self.cluster, "db": m_db} ind = 0 while ind < len(Core_param_list): if core_params == Core_param_list[ind]: self.r = Redis_Cluster_instances[ind] if iprint >= 2: print("\n\nRe-using %d'th redis instance.\n\n" % ind) break ind += 1 if ind == len(Core_param_list): if iprint >= 2: print("\n\nCreating new redis instance\n\n") Core_param_list.append( copy.deepcopy(core_params)) # must deep copy!!!! self.r = rediscluster.StrictRedisCluster(cluster=self.cluster, db=m_db) Redis_Cluster_instances.append(self.r)
def test_store_create(self, mock_get_config): """An entry should be created in Redis.""" mock_get_config.return_value = { 'JWT_SECRET': self.secret, 'REDIS_HOST': os.environ.get('REDIS_HOST', 'localhost'), 'REDIS_PORT': os.environ.get('REDIS_PORT', '7000'), 'REDIS_CLUSTER': os.environ.get('REDIS_CLUSTER', '1') } ip = '127.0.0.1' remote_host = 'foo-host.foo.com' user = domain.User( user_id='1', username='******', email='*****@*****.**', ) authorizations = domain.Authorizations( classic=2, scopes=['foo:write'], endorsements=[] ) session = store.create(authorizations, ip, remote_host, user=user) cookie = store.generate_cookie(session) # API still works as expected. self.assertIsInstance(session, domain.Session) self.assertTrue(bool(session.session_id)) self.assertIsNotNone(cookie) r = rediscluster.StrictRedisCluster( startup_nodes=[dict(host='localhost', port='7000')] ) raw = r.get(session.session_id) stored_data = jwt.decode(raw, self.secret, algorithms=['HS256']) cookie_data = jwt.decode(cookie, self.secret, algorithms=['HS256']) self.assertEqual(stored_data['nonce'], cookie_data['nonce'])
def __init__(self, redis_key='ip_table', redis_cluster=False, **kwargs): self.redis_key = redis_key if redis_cluster: self.redis = rediscluster.StrictRedisCluster(**kwargs) else: self.redis = redis.StrictRedis(**kwargs)
def getClusterConnection(self): return rediscluster.StrictRedisCluster(startup_nodes=[{ 'host': 'localhost', 'port': self.shards[0].getMasterPort() }], decode_responses=True)
def __init__(self, redisIP="127.0.0.1", redisPort="7000"): startupN = [{"host": redisIP, "port": redisPort}] try: self.r = rediscluster.StrictRedisCluster(startup_nodes=startupN) except Exception as e: self.rErr = e else: self.rErr = False
def conn_redis_cluster(self): redis_nodes = [{"host": "127.0.0.1", "port": "6379"}, {"host": "127.0.0.1", "port": "6479"}, {"host": "127.0.0.1", "port": "7379"}, {"host": "127.0.0.1", "port": "7479"}, {"host": "127.0.0.1", "port": "8379"}, {"host": "127.0.0.1", "port": "8479"} ] self.r = rediscluster.StrictRedisCluster(startup_nodes=redis_nodes)
def _get_test_client(self): startup_nodes = [{ 'host': self.TEST_HOST, 'port': int(port) } for port in self.TEST_PORTS.split(',')] if REDISCLUSTER_VERSION >= (2, 0, 0): return rediscluster.RedisCluster(startup_nodes=startup_nodes) else: return rediscluster.StrictRedisCluster(startup_nodes=startup_nodes)
def check_rediscluster(): test_host = REDISCLUSTER_CONFIG['host'] test_ports = REDISCLUSTER_CONFIG['ports'] startup_nodes = [ {'host': test_host, 'port': int(port)} for port in test_ports.split(',') ] r = rediscluster.StrictRedisCluster(startup_nodes=startup_nodes) r.flushall()
def __init__(self): self.retry_count = 1 self.lock_labels = map(lambda x: "%i" % (x), range(0, 1)) self.uuid = str(uuid.uuid1()) config = get_config() if config.redis_cluster_enabled(): startup_nodes = map(lambda x: {"host": x, "port": "%s" % (config.port())}, config.cluster_nodes()) self.redis_client = rediscluster.StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True) else: self.redis_client = redis.StrictRedis(host=config.host(), port=config.port(), db=0)
def redis_cluster(self): redis_nodes = self.conf.getRedisNodes() password = self.conf.getConfig("redisCluster", "password") try: redisconn = rediscluster.StrictRedisCluster( startup_nodes=redis_nodes, password=password) except Exception as e: print 'redis连接失败,原因', format(e) sys.exit(1) return redisconn
def _connection(self): if self.is_cluster: self._connect = rediscluster.StrictRedisCluster( startup_nodes=self.startup_nodes, decode_responses=self.decode_responses) else: self._connect = redis.StrictRedis(host=self.host, port=self.port, db=self.db, password=self.password, **self.kwargs) return self._connect
def _redis_client(opts): ''' Connect to the redis host and return a StrictRedisCluster client object. If connection fails then return None. ''' redis_host = opts.get("eauth_redis_host", "localhost") redis_port = opts.get("eauth_redis_port", 6379) try: return rediscluster.StrictRedisCluster(host=redis_host, port=redis_port) except rediscluster.exceptions.RedisClusterException as err: log.warning('Failed to connect to redis at %s:%s - %s', redis_host, redis_port, err) return None
def redis_type(request): if request.param == REDIS_REGULAR: os.environ["REDIS_HOST"] = 'redis' os.environ["REDIS_PORT"] = '6379' r = redis.StrictRedis(host=os.getenv('REDIS_HOST'), port=os.getenv('REDIS_PORT'), db=TEST_DB) else: os.environ["REDIS_HOST"] = 'redis-cluster' os.environ["REDIS_PORT"] = '7000' r = rediscluster.StrictRedisCluster(host=os.getenv('REDIS_HOST'), port=os.getenv('REDIS_PORT')) r.flushdb() return request.param
def initialize(self, nodes): """ 初始化connection :param nodes: [{"host": "172.0.0.1", "port": 6379}...] :return: """ try: self._connection = rediscluster.StrictRedisCluster( startup_nodes=nodes, skip_full_coverage_check=True) except: logging.error("Failed to connect to redis cluster") sys.exit(0) return -1 return 0
def main(): if os.path.exists('/tmp/stopredis'): os.unlink('/tmp/stopredis') manager = multiprocessing.Manager() result = manager.dict() result['generator_enabled'] = True result['total'] = 0 config = manager.dict() config['host'] = os.environ.get('REDIS_HOST', None) or DEFAULT_REDIS_HOST config['port'] = os.environ.get('REDIS_PORT', None) or DEFAULT_REDIS_PORT queue_data = multiprocessing.JoinableQueue() print('Configuration: {}'.format(config)) print('Allocating feeders') feeders = [Feeder(queue_data, config, result) for _ in xrange(WORKERS)] for feeder in feeders: feeder.start() print('Allocating generator') generator = IBANGenerator(queue_data, result) generator.start() startup_nodes = [{'host': config['host'], 'port': config['port']}] client = rediscluster.StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True, readonly_mode=False, skip_full_coverage_check=True) while True: sleep(1) if os.path.exists('/tmp/stopredis'): break if result['total'] == 0: continue if result['total'] % 100000 == 0 and client.get( IBANGenerator.CANARY[0]) is None: break if result['total'] % 10000 == 0: print('Generation in progress [{}]'.format(result['total'])) # canary lost poisoning pill result['generator_enabled'] = False queue_data.join() for _ in feeders: queue_data.put(None) print('WAITING FOR FEEDER PROCESSES') queue_data.join() for p in feeders: p.join() print('JOB DONE!') print('RESULTS\n{}'.format(result))
def test_delete_session(self, mock_get_config): """Delete a session from the datastore.""" mock_get_config.return_value = { 'JWT_SECRET': self.secret, 'REDIS_HOST': os.environ.get('REDIS_HOST', 'localhost'), 'REDIS_PORT': os.environ.get('REDIS_PORT', '7000'), 'REDIS_CLUSTER': os.environ.get('REDIS_CLUSTER', '1') } r = rediscluster.StrictRedisCluster( startup_nodes=[dict(host='localhost', port='7000')] ) r.set('fookey', b'foovalue') store.delete_by_id('fookey') self.assertIsNone(r.get('fookey'))
def __init__(self): config = get_config() startup_nodes = map( lambda x: { "host": x, "port": "%s" % (config.port()) }, config.cluster_nodes()) self.redis_client = rediscluster.StrictRedisCluster( startup_nodes=startup_nodes, decode_responses=True) self.dlm = Redlock([ { "host": "localhost", "port": 6379, "db": 0 }, ], retry_count=10)
def connection(self): startup_nodes = list() for item in self.redis_hosts: pair = dict() pair['host'], pair['port'] = item.split(':') startup_nodes.append(pair) if len(startup_nodes) > 1: import rediscluster self._conn = rediscluster.StrictRedisCluster(startup_nodes=startup_nodes, decode_responses=True, retry_on_timeout=True) elif len(startup_nodes) == 1: import redis self._conn = redis.StrictRedis(host=startup_nodes[0]['host'], port=startup_nodes[0]['port'], retry_on_timeout=True) else: self._conn = None return self._conn
def __init__(self, host: str, port: int, db: int, secret: str, duration: int = 7200, token: str = None) -> None: """Open the connection to Redis.""" # params = #, db=db) logger.debug('New Redis connection at %s, port %s', host, port) self.r = rediscluster.StrictRedisCluster(startup_nodes=[{ 'host': host, 'port': str(port) }], skip_full_coverage_check=True) self._secret = secret self._duration = duration
def __init__(self, host: str, port: int, db: int, secret: str, duration: int = 7200, token: Optional[str] = None, cluster: bool = True, fake: bool = False) -> None: """Open the connection to Redis.""" self._secret = secret self._duration = duration if fake: logger.warn('Using FakeRedis') import fakeredis # this is a dev dependency needed during testing self.r = fakeredis.FakeStrictRedis() else: logger.debug('New Redis connection at %s, port %s', host, port) if cluster: self.r = rediscluster.StrictRedisCluster( startup_nodes=[{'host': host, 'port': str(port)}], skip_full_coverage_check=True ) else: self.r = redis.StrictRedis(host=host, port=port)
def __init__(self, task_queue, config, result): """ :type task_queue: multiprocessing.JoinableQueue """ super(Feeder, self).__init__() self.task_queue = task_queue self.config = config self.result = result self.result[self.name] = 0 # self.client = redis.StrictRedis(host=config['host'], port=config['port']) startup_nodes = [{ 'host': self.config['host'], 'port': self.config['port'] }] self.client = rediscluster.StrictRedisCluster( startup_nodes=startup_nodes, decode_responses=True, readonly_mode=False, skip_full_coverage_check=True)
def __init__(self): config = get_config() if config.redis_cluster_enabled(): startup_nodes = map( lambda x: { "host": x, "port": "%s" % (config.port()) }, config.cluster_nodes()) self.redis_client = rediscluster.StrictRedisCluster( startup_nodes=startup_nodes, decode_responses=True) else: self.redis_client = redis.StrictRedis(host=config.host(), port=config.port(), db=0) self.cluster = Cluster(contact_points=config.cluster_nodes(), protocol_version=3) self.session = self.cluster.connect('mykeyspace') self.session.row_factory = decoded_dict_factory self._tables = {} self.table_columns_metadata = {}
def init(config_file_path=None, **kwargs): '''Init the storage client. Basically, we set the Redis client and connects it to the instance/cluster ''' global redis_connection global hosts # If config_file_path is None we will assume that we only have localhost # as storage node if config_file_path is None: try: import StringIO as sio except ImportError: from io import StringIO as sio config_file_handler = sio.StringIO('localhost\n') else: config_file_handler = open(config_file_path) # As accorded in the API standar, this file must contain all the hosts names # with no port, one per line hosts = [x.strip() for x in config_file_handler.readlines()] config_file_handler.close() # If we have more than one host then we will assume that our backend is a Redis # cluster. If not, we will assume that we are dealing with a Redis standalone # instance if len(hosts) > 1: # Given that cluster clients are capable to perform master # slave hierarchy discovery, we will simply connect to the first # node we got redis_connection = \ rediscluster.StrictRedisCluster(host=hosts[0], port=REDIS_PORT) else: # We are in standalone mode redis_connection = \ redis.StrictRedis(host=hosts[0], port=REDIS_PORT) # StrictRedis is not capable to know if we had success when connecting by # simply calling the constructor. We need to perform an actual query to # the backend # If we had no success this first line should crash redis_connection.set('PYCOMPSS_TEST', 'OK') # Beware py2 vs py3 - b'string' works for both. assert redis_connection.get('PYCOMPSS_TEST') == b'OK' redis_connection.delete('PYCOMPSS_TEST')
def __init__(self): list_host = REDIS.split(",") redis_nodes = [] for i in list_host: i = i.split(":") redis_nodes.append({'host': i[0], 'port': i[1]}) # list_host = HOST.split(",") # list_port = PORT.split(",") # redis_nodes = [ # {'host': list_host[0], 'port': list_port[0]}, # {'host': list_host[0], 'port': list_port[1]}, # {'host': list_host[0], 'port': list_port[2]}, # {'host': list_host[0], 'port': list_port[3]}, # {'host': list_host[0], 'port': list_port[4]}, # {'host': list_host[0], 'port': list_port[5]}, # ] self.cluster = rediscluster.StrictRedisCluster( startup_nodes=redis_nodes)
def __init__(self, domain, hosts_and_ports='localhost:6379', password=None, event_broker=None, record_exp=None): logger.debug('Using redis cluster {}'.format(hosts_and_ports)) host_and_port = hosts_and_ports.split(',') startup_nodes = [] for hap in host_and_port: part = hap.split(':') tmp = dict() if (len(part) == 2): tmp['host'] = part[0] tmp['port'] = part[1] startup_nodes.append(tmp) else: logger.warning('Parsing error in host and port {}'.format(hap)) import rediscluster self.red = rediscluster.StrictRedisCluster(startup_nodes=startup_nodes, password=password) self.record_exp = record_exp super(RedisClusterTrackerStore, self).__init__(domain, event_broker)
def get_redis_client(redis_type='single', host='127.0.0.1', port=6379, db=0, pwd=None, nodes=None, timeout=3): if redis_type == 'single': pool = redis.ConnectionPool(host=host, port=port, db=db, password=pwd, socket_timeout=timeout, socket_connect_timeout=timeout, encoding='utf-8', decode_responses=True) client = redis.StrictRedis(connection_pool=pool) else: client = rediscluster.StrictRedisCluster( startup_nodes=nodes, decode_responses=True, socket_timeout=timeout, socket_connect_timeout=timeout) return client