def _add_new_nodes(self, cluster_size): old_nodes = self.nodes.copy() nodes_before = self._get_nodes_primitive() self._docker_scale(cluster_size) nodes_after = self._get_nodes_primitive() new_ips = [ ':'.join(map(str, x)) for x in set(nodes_after) - set(nodes_before) ] print(new_ips) master_ip_port = old_nodes[0]['ip_port'] master_ip, master_port = master_ip_port.split(':') master_conn = Redis(master_ip, master_port) print("Adding nodes to the cluster") for ip in new_ips: new_ip, new_port = ip.split(':') master_conn.execute_command('CLUSTER MEET', new_ip, new_port) print("Preventive fix") sleep(3) fix = subprocess.Popen( ['ruby', 'redis-trib.rb', 'fix', master_ip_port], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL) fix.communicate(b'yes\n') fix.wait() sleep(3) new_nodes = [ x for x in self._get_running_nodes() if x['ip_port'] in new_ips ] slots_per_node = round(16384 / cluster_size) old_redises = { x[0]: Redis(x[0], x[1]) for x in (y['ip_port'].split(':') for y in old_nodes) } new_redises = [ Redis(x[0], x[1]) for x in (y['ip_port'].split(':') for y in new_nodes) ] slots_repartition = self._get_slots_repartition( list(old_redises.values())[0]) for dest_node, dest_redis, i in zip(new_nodes, new_redises, range(len(new_nodes))): slots = slots_repartition[i * slots_per_node:(i + 1) * slots_per_node] sources_ip = {x[1] for x in slots} for source_ip in sources_ip: slots_for_source = [x for x in slots if x[1] == source_ip] source_redis = old_redises[source_ip] self._transfer_slots(source_redis, slots_for_source[0][3], dest_redis, dest_node['id'], [x[0] for x in slots_for_source]) subprocess.check_call( ['ruby', 'redis-trib.rb', 'info', master_ip_port])
def new_watcher(addr: str, option: WatcherOptions) -> Casbin_Watcher: option.addr = addr option.init_config() w = Watcher() w.sub_client = Redis().client() w.pub_client = Redis().client() w.ctx = None w.close = None
def init_redis_client(cls): # 防止多线程竞争,获取到锁后依然可能已经初始化了 if cls.redis_objs: return cls.instance_name = conf.get('sentinel', "") cls.socket_timeout = conf.get("socket_timeout", 5) cls.connect_timeout = conf.get("connect_timeout", 0.1) if not cls.instance_name: # 单例模式 host = conf["host"] cls.redis_objs[cls.MODE_READ] = Redis( host=host.split(':')[0], port=host.split(':')[1], socket_timeout=cls.socket_timeout, socket_connect_timeout=cls.connect_timeout, retry_on_timeout=1 ) cls.redis_objs[cls.MODE_WRITE] = cls.redis_objs[cls.MODE_READ] else: # 哨兵模式 sentinel = Sentinel( cls.parse_config(), socket_timeout=cls.socket_timeout, socket_connect_timeout=cls.connect_timeout, retry_on_timeout=1 ) cls.redis_objs[cls.MODE_READ] = sentinel.slave_for(cls.instance_name) cls.redis_objs[cls.MODE_WRITE] = sentinel.master_for(cls.instance_name)
def post(self, request): form = AccessTokenForm.create_from_request(request) if not form.is_valid(): raise ValidationException(request, form) backend = ModelBackend() user = backend.authenticate(request, username=form.cleaned_data['username'], password=form.cleaned_data['password']) if not user: raise UnauthorizedException(request) access_token = JWTFactory(user.pk).access() jti, refresh_token = JWTFactory(user.pk).refresh() redis = Redis(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.REDIS_DATABASE) redis.set(f"refresh_token:{jti}", jti) redis.expire(f"refresh_token:{jti}", settings.SECURED_VIEW_JWT_REFRESH_TOKEN_EXPIRATION) return SingleResponse(request, { 'access_token': access_token, 'refresh_token': refresh_token }, status=HTTPStatus.OK)
def post(self, request): form = RefreshTokenForm.create_from_request(request) if not form.is_valid(): raise ValidationException(request, form) try: claims = JWTFactory.decode(form.cleaned_data['refresh']) except JoseError as e: raise ProblemDetailException(request, _('Invalid token.'), status=HTTPStatus.UNAUTHORIZED, previous=e) redis = Redis(host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.REDIS_DATABASE) if not redis.exists(f"refresh_token:{claims['jti']}"): raise UnauthorizedException(request) access_token = JWTFactory(claims['sub']).access() return SingleResponse(request, {'access_token': access_token}, status=HTTPStatus.OK)
def setup_rq_connection(): redis_conn = get_current_connection() if redis_conn == None: opts = OPTIONS.get('connection') logger.debug('Establishing Redis connection to DB %(db)s at %(host)s:%(port)s' % opts) redis_conn = Redis(**opts) push_connection(redis_conn)
def __init__(self, client=None): if not client: self.client = Redis(host='localhost', port=6379, db=13, password=None) else: self.client = client
def _build_redis(host: str, port: int=6379, db: int=0, password: str=None) -> Redis: return Redis(host=host, port=port, db=db, password=password)
def init_config(self, option: WatcherOptions): if option.optional_update_callback: self.set_update_callback(option.optional_update_callback) else: raise WatcherError("Casbin Redis Watcher callback not " "set when an update was received") if option.sub_client: self.sub_client = option.sub_client else: # TODO self.sub_client = Redis().client() if option.pub_client: self.pub_client = option.pub_client else: # TODO self.pub_client = Redis().client()
def connect(self): conf = parse_uri(self._uri) self._channel = conf.pop('channel', self._name) self._method = conf.pop('method', 'pubsub') self._client = Redis(**conf) if self._method == 'pubsub': self._pubsub = self._client.pubsub() self._pubsub.subscribe(self._channel)
def connection(self): """Get Redis connection from the pool.""" if not self._connection: self._connection = Redis(connection_pool=_connection_pool) try: self._connection.ping() except exceptions.RedisError: self._connection = None raise return self._connection
def __init__(self, app, config): if 'host' not in config: raise ConfigurationError("Redis extension needs 'host' configured") if 'password' not in config: raise ConfigurationError( "Postmark extension needs 'password' configured") host = config['host'] password = config['password'] redis = Redis(host=host, password=password) register_singleton(redis, 'redis')
def testRegisterSensorDataDbmsListener(self): #check for true when the connection variables are ok self.assertEqual(True, self.persistenceUtil.registerSensorDataDbmsListener()) #add an invalid port to the jedisSensor self.persistenceUtil.r_sensor = Redis(host = "localhost", port = 6890) #check for false when connection variables are invalid self.assertEqual(False, self.persistenceUtil.registerSensorDataDbmsListener())
def flushall(self): if self.cluster_size == 1: subprocess.check_call([ 'redis-cli', '-h', 'erasuretester_redis-standalone_1', 'FLUSHALL' ]) elif self.cluster_size >= 2: nodes = self._get_running_nodes() for redis in (Redis(x[0], x[1]) for x in (y['ip_port'].split(':') for y in nodes)): redis.flushall()
def redis_client(self) -> Redis: """Provide an instance of Redis client.""" if self._redis_client is None: redis_client = Redis(connection_pool=self.redis_conn_pool) self._redis_client = redis_client self._logger.debug("[%s]: Initialized Redis client: %s", self.__name__, self._redis_client) return self._redis_client
def redis(self): """Returns the redis.client object for the given host/port Performing lazy initialization if it hasn't already been created """ if not self.__redis: redis = Redis( host=self.__host, port=self.__port, password=self.__password, socket_timeout=self.__connection_timeout ) self.__update_latest( redis ) #__update_latest can raise an exception, so don't assign to self until it #completes successfully self.__redis = redis return self.__redis
def testWriteSensorDataToDbms(self): #Create ActuatorData instance sensorData = SensorData(); #write to redis and check if it returns true self.assertEqual(True, self.persistenceUtil.writeSensorDataToDbms(sensorData)); #add an invalid port to the jedisSensor self.persistenceUtil.r_sensor = Redis(host = "localhost", port = 6890) #write to redis and check if it returns false self.assertEqual(False, self.persistenceUtil.writeSensorDataToDbms(sensorData));
def Connect(self): from redis.client import Redis try: WriteBehindLog("Connect: connecting to {}:{} password: {}".format( self.host, self.port, self.password)) r = Redis(host=self.host, port=self.port, password=self.password, decode_responses=True) except Exception as e: msg = "Cannot connect to Redis. Exception: {}".format(e) WriteBehindLog(msg) raise Exception(msg) from None return r
def testWriteActuatorDataToDbms(self): #Create ActuatorData instance actuatorData = ActuatorData() #write to redis and check if it returns true self.assertEqual( True, self.persistenceUtil.writeActuatorDataToDbms(actuatorData)) #add an invalid port to the jedisActuator self.persistenceUtil.r_actuator = Redis(host="localhost", port=6379) #write to redis and check if it returns false self.assertEqual( True, self.persistenceUtil.writeActuatorDataToDbms(actuatorData))
class Statistics: CON = Redis( host=settings.REDIS_STATISTICS_HOST, port=settings.REDIS_STATISTICS_PORT, db=settings.REDIS_STATISTICS_DB, decode_responses=True, ) def __init__(self, product_id): self.con = self.CON self.product_id = product_id @property def key(self): return f'flow_{self.product_id}' @property def key_uv_add(self): return f'flow_uv_{self.product_id}' def add_uv(self, info): identify = info['identify'] meta = info['meta'] key = self.key_uv_add root_key = self.key def func(pipe): if identify and pipe.sismember(key, identify): return ip = meta.get('HTTP_X_FORWARDED_FOR') or meta.get('REMOTE_ADDR') if not ip: return if pipe.sismember(key, ip): return pipe.multi() if identify: pipe.sadd(key, identify) pipe.sadd(key, ip) pipe.incr(root_key, 1) return 'success' return self.con.transaction(func, key, value_from_callable=True) def get_uv_num(self): n = self.con.get(self.key) n = n and int(n) or 0 return n
def __get_redis_client(self) -> RedisCluster: if not self.__redis_client: if len(startup_nodes) > 1: self.__redis_client = RedisCluster(startup_nodes=startup_nodes, decode_responses=False, password=server_config.redis_server_password) else: node = startup_nodes[0] host = node["host"] port = node["port"] self.__redis_client = Redis(connection_pool=BlockingConnectionPool( host=host, port=port, password=server_config.redis_server_password, db=release_cache_db_index) ) return self.__redis_client
def __init__(self, sentinels, min_other_sentinels=0, sentinel_kwargs=None, **connection_kwargs): # if sentinel_kwargs isn't defined, use the socket_* options from # connection_kwargs if sentinel_kwargs is None: sentinel_kwargs = { k: v for k, v in connection_kwargs.items() if k.startswith('socket_') } self.sentinel_kwargs = sentinel_kwargs self.sentinels = [Redis(hostname, port, **self.sentinel_kwargs) for hostname, port in sentinels] self.min_other_sentinels = min_other_sentinels self.connection_kwargs = connection_kwargs
def delete(): r = Redis() # # iterate a list in batches of size n # def batcher(iterable, n): # args = [iter(iterable)] * n # return zip_longest(*args) # # in batches of 500 delete keys matching user:* # for keybatch in batcher(r.scan_iter('user:*'),500): # print("keybatch", keybatch) # r.delete(*keybatch) for key in r.scan_iter("*"): # delete the key r.delete(key)
def __init__(self, file, static_folder='static', static_url='/static', template_path='templates'): self.redis = Redis(host='localhost') self.static_folder = os.path.join(os.path.abspath(file), static_folder) self.static_url = static_url self.jinja_env = Environment(loader=FileSystemLoader(template_path), autoescape=True) self.url_map = Map([ Rule('/', endpoint='welcome'), Rule('/a', endpoint='a'), Rule('/b', endpoint='b'), Rule('/c', endpoint='c'), Rule(static_url, endpoint='file_system') ])
def get_client(self, mode, timeout=None): # def _curr_session_time_out(): # return timeout if timeout is not None else self.sock_timeout if timeout is not None: # 如果指定了超时,就临时生成一个符合这个超时的连接,使用短连接,这个场景主要应对DI轮训服务,不能频繁超时 if not self.__class__.instance_name: host = conf["host"] return Redis( host=host.split(':')[0], port=host.split(':')[1], socket_timeout=self.socket_timeout+2, socket_connect_timeout=self.connect_timeout, retry_on_timeout=1, socket_keepalive=False ) curr_sentinel = Sentinel( self.parse_config(), socket_timeout=self.socket_timeout, socket_connect_timeout=self.connect_timeout, ) if mode == self.MODE_WRITE: return curr_sentinel.master_for( self.instance_name, socket_timeout=timeout+2, socket_connect_timeout=self.connect_timeout, socket_keepalive=False, # 短连接 retry_on_timeout=True ) else: return curr_sentinel.slave_for( self.instance_name, socket_timeout=timeout+2, socket_connect_timeout=self.connect_timeout, socket_keepalive=False, retry_on_timeout=True ) else: return self.__class__.redis_objs[mode]
def count_calls(method: Callable) -> Callable: """ Takes a single method Callable argument and returns a Callable """ redis = Redis() @wraps(method) def wrapper(url): """ Function wrapper count calls """ redis.incr(f'count:{url}') res = redis.get(url) if res: return res.decode("utf-8") result = method(url) redis.setex(url, 10, result) return result return wrapper
def __new__(cls, *args, **kwargs): pid = os.getpid() if not hasattr(cls, '_instance') or pid != cls._pid: print("PID is {} and father PID is {}".format( os.getpid(), os.getppid())) if hasattr(cls, "_pid"): print("Instance PID is {} and PID is {}".format(cls._pid, pid)) if REDIS_CLUSTER: pool = ClusterConnectionPool(*args, **kwargs) cls._instance = RedisCluster(connection_pool=pool, decode_responses=True, password=REDIS_PASSWORD) cls._pid = pid else: pool = ConnectionPool(*args, **kwargs) cls._instance = Redis(connection_pool=pool, password=REDIS_PASSWORD) cls._pid = pid return cls._instance
def main(argv=sys.argv): if len(argv) < 2: usage(argv) config_uri = argv[1] options = parse_vars(argv[2:]) setup_logging(config_uri) settings = get_appsettings(config_uri, options=options) logging.getLogger('c2corg_ui').setLevel(logging.INFO) log.info('Cache Redis: {0}'.format(settings['redis.url'])) # we don't really need a connection pool here, but the `from_url` # function is convenient redis_pool = ConnectionPool.from_url( settings['redis.url'], max_connections=1) # remove all keys from the database r = Redis(connection_pool=redis_pool) r.flushdb() log.info('Flushed cache')
def connect(self): if (len(self.startup_nodes) < 2): host = self.startup_nodes[0].get('host') port = self.startup_nodes[0].get('port') if self.password: self.redis_pool = redis.ConnectionPool(host=host, port=port, db=0) else: self.redis_pool = redis.ConnectionPool(host=host, port=port, password=self.password, db=0) self.redis_instance = Redis(connection_pool=self.redis_pool, decode_responses=True) return self.redis_instance #, skip_full_coverage_check=True self.redis_cluster = RedisCluster(startup_nodes=self.startup_nodes, password=self.password) return self.redis_cluster
def _connect(self) -> NoReturn: """ Connection to redis server with retries :param retry_counter: counter of times of retrying :return: Redis connection if it was successful and None otherwise """ try: self._kv_storage = Redis( host=self.host, port=self.port, socket_timeout=self.timeout ) self._kv_storage.ping() except Exception: logging.error( "Got error while connecting to redis with host %s and port %d.", self.host, self.port )