def __init__(self, config):
BaseCache.__init__(self, config)
self.key_prefix = config.get('key_prefix', '')
try:
import redis
except ImportError:
raise RuntimeError('no redis module found')
kwargs = dict((k, v) for k, v in config.items() if k not in _redis_kwargs_exclusions)
if 'socket_timeout' not in kwargs:
kwargs['socket_timeout'] = _DEFAULT_SOCKET_TIMEOUT
if 'socket_connect_timeout' not in kwargs:
kwargs['socket_connect_timeout'] = _DEFAULT_SOCKET_TIMEOUT
if 'socket_keepalive' not in kwargs:
kwargs['socket_keepalive'] = 1
if 'socket_keepalive_options' not in kwargs:
kwargs['socket_keepalive_options'] = _TCP_KEEP_ALIVE_OPTIONS
if kwargs.pop('blocking_pool', False):
if 'blocking_pool_timeout' in kwargs:
kwargs['timeout'] = kwargs.pop('blocking_pool_timeout')
else:
kwargs['timeout'] = _DEFAULT_REDIS_BLOCKING_POOL_TIMEOUT
connection_pool = redis.BlockingConnectionPool(**kwargs)
else:
connection_pool = redis.ConnectionPool(**kwargs)
self._client = redis.Redis(connection_pool=connection_pool)
def get_pool(self, connection_info=None, max_connections=10, timeout=20):
connection_info = connection_info or {'a': 1, 'b': 2, 'c': 3}
pool = redis.BlockingConnectionPool(connection_class=DummyConnection,
max_connections=max_connections,
timeout=timeout,
**connection_info)
return pool
def __init__(self, host, port, db, max_connections=100):
super(RedisManager, self).__init__()
pool = redis.BlockingConnectionPool(host=host,
port=port,
db=db,
max_connections=max_connections)
self._redis = redis.StrictRedis(connection_pool=pool)
def get_pool(self, connection_kwargs=None, max_connections=10, timeout=20):
connection_kwargs = connection_kwargs or {}
pool = redis.BlockingConnectionPool(connection_class=DummyConnection,
max_connections=max_connections,
timeout=timeout,
**connection_kwargs)
return pool
def make_redis_client(redis_config):
"""
:param redis_config: redis 配置
:return:
"""
try:
if cmp(redis.VERSION, (2, 10, 1)) >= 0:
pool = redis.BlockingConnectionPool(retry_on_timeout=True, **redis_config)
else:
pool = redis.BlockingConnectionPool(**redis_config)
except:
pool = redis.BlockingConnectionPool(**redis_config)
redis_client = redis.Redis(connection_pool=pool)
return redis_client
class RedisCache(BaseCache):
"""Redis caching implementation."""
_host = getattr(settings, 'redis_host') if hasattr(settings, 'redis_host') else 'localhost'
_port = getattr(settings, 'redis_port') if hasattr(settings, 'redis_port') else 6379
_db = getattr(settings, 'redis_db') if hasattr(settings, 'redis_db') else 0
_pool = redis.BlockingConnectionPool(host=_host, port=_port, db=_db)
logger.info('Create Redis connection pool %s', _pool)
def __init__(self):
self._connection = redis.Redis(connection_pool=RedisCache._pool)
def get(self, key: str, field: str=None) -> Union[bytes, dict]:
if field is not None:
return self._connection.hget(key, field)
else:
return self._connection.hgetall(key)
def put(self, key: str, field: str, value: str) -> int:
return self._connection.hset(key, field, value)
def get_ttl(self, key: str) -> int:
return self._connection.ttl(key)
def set_ttl(self, key: str, ttl: int) -> None:
if ttl == -1:
self._connection.persist(key)
else:
self._connection.expire(key, ttl)
def del_record(self, key: str, field: str) -> None:
return self._connection.hdel(key, field)
def del_key(self, key):
return self._connection.delete(key)
def __init__(self, file_size_db_path: str = None, redis_url: str = None,
bar_position: int = 0):
"""Prepare the environment for statistic extraction.
Args:
file_size_db_path (str): the folder path to file size database in
sqlite
redis_url (str): the redis cache url (ex: 'localhost')
bar_position (int): position for progress bar
"""
self.__columns = [
'day',
'filename',
'protocol',
'task_monitor_id',
'task_id',
'job_id',
'site_name',
'job_success',
'job_length_h',
'job_length_m',
'user',
'cpu_time',
'io_time',
'size'
]
self._data = pd.DataFrame(
columns=self.__columns
)
self.__buffer = []
self.__last_date = None
self.__cur_date = None
self.__cursors = None
self.__conn_mc = None
self.__conn_data = None
self.__redis = None
self.__tmp_cache = {}
self.__tmp_cache_sets = set()
self.__tmp_cache_set_added = set()
self.__bar_position = bar_position
# SQLite databases are used as source for file sizes
if file_size_db_path:
self.__conn_mc = sqlite3.connect(os.path.join(
file_size_db_path, "mc_file_sizes.db"))
self.__conn_data = sqlite3.connect(os.path.join(
file_size_db_path, "data_file_sizes.db"))
self.__cursors = {
'mc': self.__conn_mc.cursor(),
'data': self.__conn_data.cursor()
}
# Redis cache is used to speedup the extraction
if redis_url:
self.__redis = redis.Redis(
connection_pool=redis.BlockingConnectionPool(
host=redis_url,
port=6379, db=0
)
)
def make_redis_client(redis_config):
"""make_redis_client: docstring
args:
redis_config: --- arg
returns:
0 ---
"""
try:
if cmp(redis.VERSION, (2, 10, 1)) >= 0:
pool = redis.BlockingConnectionPool(retry_on_timeout=True, **redis_config)
else:
pool = redis.BlockingConnectionPool(**redis_config)
except:
pool = redis.BlockingConnectionPool(**redis_config)
redis_client = redis.Redis(connection_pool=pool)
return redis_client
def _get_redis_connection(self, group, shard):
"""
Create and return a Redis Connection for the given group
Returns:
redis.StrictRedis: The Redis Connection
Raises:
Exception: Passes through any exceptions that happen in trying to get the connection pool
"""
redis_group = self.__config.redis_urls_by_group[group][shard]
self.__logger.info(
'Attempting to connect to Redis for group "{}", shard "{}", url "{}"'
.format(group, shard, redis_group))
redis_pool = redis.BlockingConnectionPool(
host=redis_group.host,
port=redis_group.port,
db=redis_group.db,
password=redis_group.password)
redis_connection = redis.StrictRedis(connection_pool=redis_pool)
self.__logger.info(
'Successfully connected to Redis for group "{}", shard "{}", url "{}"'
.format(group, shard, redis_group))
return redis_connection
def connectRedis():
global redisClient
redisClient = redis.StrictRedis(host=argServer,
port=argServerPort,
db=0,
ssl=argServerSSL)
redis.BlockingConnectionPool(timeout=5)
def redis_server(host=local_redis_host,
port=local_redis_port,
password=local_redis_password):
rdp = redis.BlockingConnectionPool(host=host, port=port, password=password)
server = redis.StrictRedis(connection_pool=rdp)
return server
def get_redis_db():
pool = redis.BlockingConnectionPool(host=REDIS_DATABASE['host'],
port=REDIS_DATABASE['port'],
db=REDIS_DATABASE['db'],
password=REDIS_DATABASE['password'],
decode_responses='utf-8',
max_connections=3)
return redis.StrictRedis(connection_pool=pool)
class RedisPool(object):
__metaclass__ = SingletonType
blocking_pool = redis.BlockingConnectionPool(
host=settings.REDIS_HOST,
port=settings.REDIS_PORT,
db=settings.REDIS_DB,
max_connections=settings.REDIS_MAX_CONNECTIONS)
def get_connection(self):
pool = redis.BlockingConnectionPool(
host=self.host,
port=self.port,
db=self.index_db,
password=self.password,
max_connections=self.max_connections,
timeout=self.timeout)
return redis.Redis(connection_pool=pool, charset='utf-8')
def __init__(self):
if not type(self).Init:
try:
type(self).Pool = redis.BlockingConnectionPool(host="127.0.0.1", port=6379)
type(self).Init = True
except redis.exceptions.ConnectionError as _:
self.conn = None
print("Cannot create ConnectionPool")
self.conn = redis.StrictRedis(connection_pool=type(self).Pool)
def _connect(self):
self.conn = redis.Redis(
host=self.redis_host,
port=self.redis_port,
socket_keepalive=True,
socket_timeout=300,
connection_pool=redis.BlockingConnectionPool(
max_connections=self.max_connections
),
)
def create_redis_cli(server, db):
host, port = server
args = {
"host": host,
"port": int(port),
"socket_timeout": 2,
"db": db
}
pool = redis.BlockingConnectionPool(**args)
return redis.StrictRedis(connection_pool=pool)
def getConnection(self, **kwargs):
""" Get a connection """
db = int(kwargs.get('db', 0))
if db not in self.pool:
self.pool[db] = redis.BlockingConnectionPool(db=db)
client = redis.StrictRedis(connection_pool=self.pool[db])
yield client
client.close()
def test_repr_contains_db_info_unix(self):
pool = redis.BlockingConnectionPool(
connection_class=redis.UnixDomainSocketConnection,
path='abc',
db=0)
assert re.match('(.*)<(.*)<(.*)>>', repr(pool)).groups() == (
'BlockingConnectionPool',
'UnixDomainSocketConnection',
'path=abc,db=0',
)
def _get_redis_connection(self, group, shard):
"""
Create and return a Redis Connection for the given group
Returns:
redis.StrictRedis: The Redis Connection
Raises:
Exception: Passes through any exceptions that happen in trying to get the connection pool
"""
redis_group = self.__config.redis_urls_by_group[group][shard]
self.__logger.info(
u'Attempting to connect to Redis for group "{}", shard "{}", url "{}"'
.format(group, shard, redis_group))
if isinstance(redis_group, PanoptesRedisConnectionConfiguration):
redis_pool = redis.BlockingConnectionPool(
host=redis_group.host,
port=redis_group.port,
db=redis_group.db,
password=redis_group.password)
redis_connection = redis.StrictRedis(connection_pool=redis_pool)
elif isinstance(redis_group,
PanoptesRedisSentinelConnectionConfiguration):
sentinels = [(sentinel.host, sentinel.port)
for sentinel in redis_group.sentinels]
self.__logger.info(
u'Querying Redis Sentinels "{}" for group "{}", shard "{}"'.
format(repr(redis_group), group, shard))
sentinel = redis.sentinel.Sentinel(sentinels)
master = sentinel.discover_master(redis_group.master_name)
password_present = u'yes' if redis_group.master_password else u'no'
self.__logger.info(
u'Going to connect to master "{}" ({}:{}, password: {}) for group "{}", shard "{}""'
.format(redis_group.master_name, master[0], master[1],
password_present, group, shard))
redis_connection = sentinel.master_for(
redis_group.master_name, password=redis_group.master_password)
else:
self.__logger.info(
u'Unknown Redis configuration object type: {}'.format(
type(redis_group)))
return
self.__logger.info(
u'Successfully connected to Redis for group "{}", shard "{}", url "{}"'
.format(group, shard, redis_group))
return redis_connection
def test_repr_contains_db_info_tcp(self):
pool = redis.BlockingConnectionPool(
host='localhost',
port=6379,
db=0,
)
assert re.match('(.*)<(.*)<(.*)>>', repr(pool)).groups() == (
'BlockingConnectionPool',
'Connection',
'host=localhost,port=6379,db=0',
)
def get_redis_client(cls, redis_config):
"""# make_redis_client: docstring
args:
redis_config: --- arg
returns:
0 ---
"""
client_key = '_'.join([
redis_config['host'],
str(redis_config['port']),
str(redis_config['db'])
])
if client_key not in cls.REDIS_CLIENT_DICT:
if cmp(redis.VERSION, (2, 10, 1)) >= 0:
pool = redis.BlockingConnectionPool(retry_on_timeout=True,
**redis_config)
else:
pool = redis.BlockingConnectionPool(**redis_config)
client = redis.Redis(connection_pool=pool)
cls.REDIS_CLIENT_DICT[client_key] = client
return cls.REDIS_CLIENT_DICT[client_key]
def main(argv=None):
if not argv:
argv = sys.argv
global pool
pool = redis.BlockingConnectionPool(max_connections=10, host=REDIS_HOST, port=REDIS_PORT)
# 使用此方法连接redis时 所有连接参数的配置在BlockingConnectionPool中配置，例如db=0，1，2，3...
r = redis.Redis(connection_pool=pool)
r1 = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, db=1)
String_examples(r, r1)
def __init__(self, config={}):
# ./redis-cli.exe -h 127.0.0.1 -p 6379 -a '' -n 1
db = int(config['db'])
if not self.__config.has_key(db):
self.__config[db] = {
#'path': '/tmp/redis.sock', # 若选用unix domain socket, 请关闭注释
'host': '127.0.0.1',
'port': 6666,
'password': '******',
'db': 1, # 使用第几个库
'socket_timeout':
None, # specific timeout(seconds) for reading; None for blocking.
'socket_connect_timeout':
2, # specific timeout(seconds) for connect; None for blocking.
}
self.__config[db].update(config)
if not self.__pool.has_key(db):
if self.__config[db].has_key('path') and self.__config[db]['path']:
# print 'new redis Connection[{}], unix socket:{}'.format(db, self.__config[db]['path'])
self.__pool = redis.BlockingConnectionPool(
connection_class=redis.UnixDomainSocketConnection,
path=self.__config[db]['path'],
password=self.__config[db]['password'],
db=self.__config[db]['db'],
socket_timeout=self.__config[db]['socket_timeout'],
)
else:
# print 'new redis Connection[{}], socket. host:{}, port:{}'.format(db, self.__config[db]['host'], self.__config[db]['port'])
self.__pool = redis.BlockingConnectionPool(
connection_class=redis.Connection,
host=self.__config[db]['host'],
port=self.__config[db]['port'],
password=self.__config[db]['password'],
db=self.__config[db]['db'],
socket_timeout=self.__config[db]['socket_timeout'],
socket_connect_timeout=self.__config[db]
['socket_connect_timeout'],
)
super(self.__class__, self).__init__(connection_pool=self.__pool)
def wrapper(*args, **kwargs):
arg_conn = 'redis_conn'
func_params = fn.__code__.co_varnames
conn_in_args = arg_conn in func_params and func_params.index(
arg_conn) < len(args)
conn_in_kwargs = arg_conn in kwargs
if conn_in_args or conn_in_kwargs:
return fn(*args, **kwargs)
else:
connection_pool = redis.BlockingConnectionPool(**REDIS_CONN_CONF)
with create_redis_conn(connection_pool) as redis_obj:
kwargs[arg_conn] = redis_obj
return fn(*args, **kwargs)
def __init__(self, host='127.0.0.1', port=6379, db=0, password=None):
base_config = current_app.config.copy()
config = {
'host': host or base_config['REDIS_HOST'],
'password': password or base_config['REDIS_PASSWORD'],
'port': port or base_config['REDIS_PORT'],
'db': db or base_config['REDIS_DB'],
'max_connections': 15,
}
pool = redis.BlockingConnectionPool(**config)
self.__conn = redis.StrictRedis(connection_pool=pool)
def create_client(**pool_args):
if 'url' in pool_args:
return redis.Redis(connection_pool=redis.BlockingConnectionPool.
from_url(pool_args['url']))
if 'port' not in pool_args:
pool_args['port'] = 6379
if 'host' not in pool_args:
raise ValueError('"host" is not defined.')
pool = redis.BlockingConnectionPool(**pool_args)
return redis.Redis(connection_pool=pool)
def setup_redis_broker() -> None:
_connection_pool: redis.BlockingConnectionPool = redis.BlockingConnectionPool(
host=REDIS_HOST, port=REDIS_PORT, db=REDIS_DB, password=REDIS_PASSWORD
)
_redis_db: redis.StrictRedis = redis.StrictRedis(connection_pool=_connection_pool)
_result_backend = RedisBackend(encoder=PickleEncoder(), client=_redis_db)
_result_middleware = Results(backend=_result_backend)
broker: Broker = RedisBroker(
connection_pool=_connection_pool,
middleware=[_result_middleware],
namespace="lrw",
)
dramatiq.set_broker(broker)
dramatiq.set_encoder(dramatiq.PickleEncoder())
def get_pool(host, port):
key = (host, port)
connection_pool = pool.get(key, None)
if not connection_pool:
connection_pool = \
redis.BlockingConnectionPool(
host=host,
port=port,
max_connections=200
)
pool[key] = connection_pool
return connection_pool
def create_from_config(config, prefix=''):
"""Redis client instantiation from settings.
"""
settings = config.get_settings()
uri = settings[prefix + 'url']
uri = urlparse.urlparse(uri)
pool_size = int(settings[prefix + 'pool_size'])
kwargs = {
"max_connections": pool_size,
"host": uri.hostname or 'localhost',
"port": uri.port or 6379,
"password": uri.password or None,
"db": int(uri.path[1:]) if uri.path else 0
}
connection_pool = redis.BlockingConnectionPool(**kwargs)
return redis.StrictRedis(connection_pool=connection_pool)
