def __init__(self, config): BaseCache.__init__(self, config) self.key_prefix = config.get('key_prefix', '') try: import redis except ImportError: raise RuntimeError('no redis module found') kwargs = dict((k, v) for k, v in config.items() if k not in _redis_kwargs_exclusions) if 'socket_timeout' not in kwargs: kwargs['socket_timeout'] = _DEFAULT_SOCKET_TIMEOUT if 'socket_connect_timeout' not in kwargs: kwargs['socket_connect_timeout'] = _DEFAULT_SOCKET_TIMEOUT if 'socket_keepalive' not in kwargs: kwargs['socket_keepalive'] = 1 if 'socket_keepalive_options' not in kwargs: kwargs['socket_keepalive_options'] = _TCP_KEEP_ALIVE_OPTIONS if kwargs.pop('blocking_pool', False): if 'blocking_pool_timeout' in kwargs: kwargs['timeout'] = kwargs.pop('blocking_pool_timeout') else: kwargs['timeout'] = _DEFAULT_REDIS_BLOCKING_POOL_TIMEOUT connection_pool = redis.BlockingConnectionPool(**kwargs) else: connection_pool = redis.ConnectionPool(**kwargs) self._client = redis.Redis(connection_pool=connection_pool)
def get_pool(self, connection_info=None, max_connections=10, timeout=20): connection_info = connection_info or {'a': 1, 'b': 2, 'c': 3} pool = redis.BlockingConnectionPool(connection_class=DummyConnection, max_connections=max_connections, timeout=timeout, **connection_info) return pool
def __init__(self, host, port, db, max_connections=100): super(RedisManager, self).__init__() pool = redis.BlockingConnectionPool(host=host, port=port, db=db, max_connections=max_connections) self._redis = redis.StrictRedis(connection_pool=pool)
def get_pool(self, connection_kwargs=None, max_connections=10, timeout=20): connection_kwargs = connection_kwargs or {} pool = redis.BlockingConnectionPool(connection_class=DummyConnection, max_connections=max_connections, timeout=timeout, **connection_kwargs) return pool
def make_redis_client(redis_config): """ :param redis_config: redis 配置 :return: """ try: if cmp(redis.VERSION, (2, 10, 1)) >= 0: pool = redis.BlockingConnectionPool(retry_on_timeout=True, **redis_config) else: pool = redis.BlockingConnectionPool(**redis_config) except: pool = redis.BlockingConnectionPool(**redis_config) redis_client = redis.Redis(connection_pool=pool) return redis_client
class RedisCache(BaseCache): """Redis caching implementation.""" _host = getattr(settings, 'redis_host') if hasattr(settings, 'redis_host') else 'localhost' _port = getattr(settings, 'redis_port') if hasattr(settings, 'redis_port') else 6379 _db = getattr(settings, 'redis_db') if hasattr(settings, 'redis_db') else 0 _pool = redis.BlockingConnectionPool(host=_host, port=_port, db=_db) logger.info('Create Redis connection pool %s', _pool) def __init__(self): self._connection = redis.Redis(connection_pool=RedisCache._pool) def get(self, key: str, field: str=None) -> Union[bytes, dict]: if field is not None: return self._connection.hget(key, field) else: return self._connection.hgetall(key) def put(self, key: str, field: str, value: str) -> int: return self._connection.hset(key, field, value) def get_ttl(self, key: str) -> int: return self._connection.ttl(key) def set_ttl(self, key: str, ttl: int) -> None: if ttl == -1: self._connection.persist(key) else: self._connection.expire(key, ttl) def del_record(self, key: str, field: str) -> None: return self._connection.hdel(key, field) def del_key(self, key): return self._connection.delete(key)
def __init__(self, file_size_db_path: str = None, redis_url: str = None, bar_position: int = 0): """Prepare the environment for statistic extraction. Args: file_size_db_path (str): the folder path to file size database in sqlite redis_url (str): the redis cache url (ex: 'localhost') bar_position (int): position for progress bar """ self.__columns = [ 'day', 'filename', 'protocol', 'task_monitor_id', 'task_id', 'job_id', 'site_name', 'job_success', 'job_length_h', 'job_length_m', 'user', 'cpu_time', 'io_time', 'size' ] self._data = pd.DataFrame( columns=self.__columns ) self.__buffer = [] self.__last_date = None self.__cur_date = None self.__cursors = None self.__conn_mc = None self.__conn_data = None self.__redis = None self.__tmp_cache = {} self.__tmp_cache_sets = set() self.__tmp_cache_set_added = set() self.__bar_position = bar_position # SQLite databases are used as source for file sizes if file_size_db_path: self.__conn_mc = sqlite3.connect(os.path.join( file_size_db_path, "mc_file_sizes.db")) self.__conn_data = sqlite3.connect(os.path.join( file_size_db_path, "data_file_sizes.db")) self.__cursors = { 'mc': self.__conn_mc.cursor(), 'data': self.__conn_data.cursor() } # Redis cache is used to speedup the extraction if redis_url: self.__redis = redis.Redis( connection_pool=redis.BlockingConnectionPool( host=redis_url, port=6379, db=0 ) )
def make_redis_client(redis_config): """make_redis_client: docstring args: redis_config: --- arg returns: 0 --- """ try: if cmp(redis.VERSION, (2, 10, 1)) >= 0: pool = redis.BlockingConnectionPool(retry_on_timeout=True, **redis_config) else: pool = redis.BlockingConnectionPool(**redis_config) except: pool = redis.BlockingConnectionPool(**redis_config) redis_client = redis.Redis(connection_pool=pool) return redis_client
def _get_redis_connection(self, group, shard): """ Create and return a Redis Connection for the given group Returns: redis.StrictRedis: The Redis Connection Raises: Exception: Passes through any exceptions that happen in trying to get the connection pool """ redis_group = self.__config.redis_urls_by_group[group][shard] self.__logger.info( 'Attempting to connect to Redis for group "{}", shard "{}", url "{}"' .format(group, shard, redis_group)) redis_pool = redis.BlockingConnectionPool( host=redis_group.host, port=redis_group.port, db=redis_group.db, password=redis_group.password) redis_connection = redis.StrictRedis(connection_pool=redis_pool) self.__logger.info( 'Successfully connected to Redis for group "{}", shard "{}", url "{}"' .format(group, shard, redis_group)) return redis_connection
def connectRedis(): global redisClient redisClient = redis.StrictRedis(host=argServer, port=argServerPort, db=0, ssl=argServerSSL) redis.BlockingConnectionPool(timeout=5)
def redis_server(host=local_redis_host, port=local_redis_port, password=local_redis_password): rdp = redis.BlockingConnectionPool(host=host, port=port, password=password) server = redis.StrictRedis(connection_pool=rdp) return server
def get_redis_db(): pool = redis.BlockingConnectionPool(host=REDIS_DATABASE['host'], port=REDIS_DATABASE['port'], db=REDIS_DATABASE['db'], password=REDIS_DATABASE['password'], decode_responses='utf-8', max_connections=3) return redis.StrictRedis(connection_pool=pool)
class RedisPool(object): __metaclass__ = SingletonType blocking_pool = redis.BlockingConnectionPool( host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.REDIS_DB, max_connections=settings.REDIS_MAX_CONNECTIONS)
def get_connection(self): pool = redis.BlockingConnectionPool( host=self.host, port=self.port, db=self.index_db, password=self.password, max_connections=self.max_connections, timeout=self.timeout) return redis.Redis(connection_pool=pool, charset='utf-8')
def __init__(self): if not type(self).Init: try: type(self).Pool = redis.BlockingConnectionPool(host="127.0.0.1", port=6379) type(self).Init = True except redis.exceptions.ConnectionError as _: self.conn = None print("Cannot create ConnectionPool") self.conn = redis.StrictRedis(connection_pool=type(self).Pool)
def _connect(self): self.conn = redis.Redis( host=self.redis_host, port=self.redis_port, socket_keepalive=True, socket_timeout=300, connection_pool=redis.BlockingConnectionPool( max_connections=self.max_connections ), )
def create_redis_cli(server, db): host, port = server args = { "host": host, "port": int(port), "socket_timeout": 2, "db": db } pool = redis.BlockingConnectionPool(**args) return redis.StrictRedis(connection_pool=pool)
def getConnection(self, **kwargs): """ Get a connection """ db = int(kwargs.get('db', 0)) if db not in self.pool: self.pool[db] = redis.BlockingConnectionPool(db=db) client = redis.StrictRedis(connection_pool=self.pool[db]) yield client client.close()
def test_repr_contains_db_info_unix(self): pool = redis.BlockingConnectionPool( connection_class=redis.UnixDomainSocketConnection, path='abc', db=0) assert re.match('(.*)<(.*)<(.*)>>', repr(pool)).groups() == ( 'BlockingConnectionPool', 'UnixDomainSocketConnection', 'path=abc,db=0', )
def _get_redis_connection(self, group, shard): """ Create and return a Redis Connection for the given group Returns: redis.StrictRedis: The Redis Connection Raises: Exception: Passes through any exceptions that happen in trying to get the connection pool """ redis_group = self.__config.redis_urls_by_group[group][shard] self.__logger.info( u'Attempting to connect to Redis for group "{}", shard "{}", url "{}"' .format(group, shard, redis_group)) if isinstance(redis_group, PanoptesRedisConnectionConfiguration): redis_pool = redis.BlockingConnectionPool( host=redis_group.host, port=redis_group.port, db=redis_group.db, password=redis_group.password) redis_connection = redis.StrictRedis(connection_pool=redis_pool) elif isinstance(redis_group, PanoptesRedisSentinelConnectionConfiguration): sentinels = [(sentinel.host, sentinel.port) for sentinel in redis_group.sentinels] self.__logger.info( u'Querying Redis Sentinels "{}" for group "{}", shard "{}"'. format(repr(redis_group), group, shard)) sentinel = redis.sentinel.Sentinel(sentinels) master = sentinel.discover_master(redis_group.master_name) password_present = u'yes' if redis_group.master_password else u'no' self.__logger.info( u'Going to connect to master "{}" ({}:{}, password: {}) for group "{}", shard "{}""' .format(redis_group.master_name, master[0], master[1], password_present, group, shard)) redis_connection = sentinel.master_for( redis_group.master_name, password=redis_group.master_password) else: self.__logger.info( u'Unknown Redis configuration object type: {}'.format( type(redis_group))) return self.__logger.info( u'Successfully connected to Redis for group "{}", shard "{}", url "{}"' .format(group, shard, redis_group)) return redis_connection
def test_repr_contains_db_info_tcp(self): pool = redis.BlockingConnectionPool( host='localhost', port=6379, db=0, ) assert re.match('(.*)<(.*)<(.*)>>', repr(pool)).groups() == ( 'BlockingConnectionPool', 'Connection', 'host=localhost,port=6379,db=0', )
def get_redis_client(cls, redis_config): """# make_redis_client: docstring args: redis_config: --- arg returns: 0 --- """ client_key = '_'.join([ redis_config['host'], str(redis_config['port']), str(redis_config['db']) ]) if client_key not in cls.REDIS_CLIENT_DICT: if cmp(redis.VERSION, (2, 10, 1)) >= 0: pool = redis.BlockingConnectionPool(retry_on_timeout=True, **redis_config) else: pool = redis.BlockingConnectionPool(**redis_config) client = redis.Redis(connection_pool=pool) cls.REDIS_CLIENT_DICT[client_key] = client return cls.REDIS_CLIENT_DICT[client_key]
def main(argv=None): if not argv: argv = sys.argv global pool pool = redis.BlockingConnectionPool(max_connections=10, host=REDIS_HOST, port=REDIS_PORT) # 使用此方法连接redis时 所有连接参数的配置在BlockingConnectionPool中配置,例如db=0,1,2,3... r = redis.Redis(connection_pool=pool) r1 = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=1) String_examples(r, r1)
def __init__(self, config={}): # ./redis-cli.exe -h 127.0.0.1 -p 6379 -a '' -n 1 db = int(config['db']) if not self.__config.has_key(db): self.__config[db] = { #'path': '/tmp/redis.sock', # 若选用unix domain socket, 请关闭注释 'host': '127.0.0.1', 'port': 6666, 'password': '******', 'db': 1, # 使用第几个库 'socket_timeout': None, # specific timeout(seconds) for reading; None for blocking. 'socket_connect_timeout': 2, # specific timeout(seconds) for connect; None for blocking. } self.__config[db].update(config) if not self.__pool.has_key(db): if self.__config[db].has_key('path') and self.__config[db]['path']: # print 'new redis Connection[{}], unix socket:{}'.format(db, self.__config[db]['path']) self.__pool = redis.BlockingConnectionPool( connection_class=redis.UnixDomainSocketConnection, path=self.__config[db]['path'], password=self.__config[db]['password'], db=self.__config[db]['db'], socket_timeout=self.__config[db]['socket_timeout'], ) else: # print 'new redis Connection[{}], socket. host:{}, port:{}'.format(db, self.__config[db]['host'], self.__config[db]['port']) self.__pool = redis.BlockingConnectionPool( connection_class=redis.Connection, host=self.__config[db]['host'], port=self.__config[db]['port'], password=self.__config[db]['password'], db=self.__config[db]['db'], socket_timeout=self.__config[db]['socket_timeout'], socket_connect_timeout=self.__config[db] ['socket_connect_timeout'], ) super(self.__class__, self).__init__(connection_pool=self.__pool)
def wrapper(*args, **kwargs): arg_conn = 'redis_conn' func_params = fn.__code__.co_varnames conn_in_args = arg_conn in func_params and func_params.index( arg_conn) < len(args) conn_in_kwargs = arg_conn in kwargs if conn_in_args or conn_in_kwargs: return fn(*args, **kwargs) else: connection_pool = redis.BlockingConnectionPool(**REDIS_CONN_CONF) with create_redis_conn(connection_pool) as redis_obj: kwargs[arg_conn] = redis_obj return fn(*args, **kwargs)
def __init__(self, host='127.0.0.1', port=6379, db=0, password=None): base_config = current_app.config.copy() config = { 'host': host or base_config['REDIS_HOST'], 'password': password or base_config['REDIS_PASSWORD'], 'port': port or base_config['REDIS_PORT'], 'db': db or base_config['REDIS_DB'], 'max_connections': 15, } pool = redis.BlockingConnectionPool(**config) self.__conn = redis.StrictRedis(connection_pool=pool)
def create_client(**pool_args): if 'url' in pool_args: return redis.Redis(connection_pool=redis.BlockingConnectionPool. from_url(pool_args['url'])) if 'port' not in pool_args: pool_args['port'] = 6379 if 'host' not in pool_args: raise ValueError('"host" is not defined.') pool = redis.BlockingConnectionPool(**pool_args) return redis.Redis(connection_pool=pool)
def setup_redis_broker() -> None: _connection_pool: redis.BlockingConnectionPool = redis.BlockingConnectionPool( host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, password=REDIS_PASSWORD ) _redis_db: redis.StrictRedis = redis.StrictRedis(connection_pool=_connection_pool) _result_backend = RedisBackend(encoder=PickleEncoder(), client=_redis_db) _result_middleware = Results(backend=_result_backend) broker: Broker = RedisBroker( connection_pool=_connection_pool, middleware=[_result_middleware], namespace="lrw", ) dramatiq.set_broker(broker) dramatiq.set_encoder(dramatiq.PickleEncoder())
def get_pool(host, port): key = (host, port) connection_pool = pool.get(key, None) if not connection_pool: connection_pool = \ redis.BlockingConnectionPool( host=host, port=port, max_connections=200 ) pool[key] = connection_pool return connection_pool
def create_from_config(config, prefix=''): """Redis client instantiation from settings. """ settings = config.get_settings() uri = settings[prefix + 'url'] uri = urlparse.urlparse(uri) pool_size = int(settings[prefix + 'pool_size']) kwargs = { "max_connections": pool_size, "host": uri.hostname or 'localhost', "port": uri.port or 6379, "password": uri.password or None, "db": int(uri.path[1:]) if uri.path else 0 } connection_pool = redis.BlockingConnectionPool(**kwargs) return redis.StrictRedis(connection_pool=connection_pool)