def _add_new_nodes(self, cluster_size):
old_nodes = self.nodes.copy()
nodes_before = self._get_nodes_primitive()
self._docker_scale(cluster_size)
nodes_after = self._get_nodes_primitive()
new_ips = [
':'.join(map(str, x)) for x in set(nodes_after) - set(nodes_before)
]
print(new_ips)
master_ip_port = old_nodes[0]['ip_port']
master_ip, master_port = master_ip_port.split(':')
master_conn = Redis(master_ip, master_port)
print("Adding nodes to the cluster")
for ip in new_ips:
new_ip, new_port = ip.split(':')
master_conn.execute_command('CLUSTER MEET', new_ip, new_port)
print("Preventive fix")
sleep(3)
fix = subprocess.Popen(
['ruby', 'redis-trib.rb', 'fix', master_ip_port],
stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL)
fix.communicate(b'yes\n')
fix.wait()
sleep(3)
new_nodes = [
x for x in self._get_running_nodes() if x['ip_port'] in new_ips
]
slots_per_node = round(16384 / cluster_size)
old_redises = {
x[0]: Redis(x[0], x[1])
for x in (y['ip_port'].split(':') for y in old_nodes)
}
new_redises = [
Redis(x[0], x[1])
for x in (y['ip_port'].split(':') for y in new_nodes)
]
slots_repartition = self._get_slots_repartition(
list(old_redises.values())[0])
for dest_node, dest_redis, i in zip(new_nodes, new_redises,
range(len(new_nodes))):
slots = slots_repartition[i * slots_per_node:(i + 1) *
slots_per_node]
sources_ip = {x[1] for x in slots}
for source_ip in sources_ip:
slots_for_source = [x for x in slots if x[1] == source_ip]
source_redis = old_redises[source_ip]
self._transfer_slots(source_redis, slots_for_source[0][3],
dest_redis, dest_node['id'],
[x[0] for x in slots_for_source])
subprocess.check_call(
['ruby', 'redis-trib.rb', 'info', master_ip_port])
def new_watcher(addr: str, option: WatcherOptions) -> Casbin_Watcher:
option.addr = addr
option.init_config()
w = Watcher()
w.sub_client = Redis().client()
w.pub_client = Redis().client()
w.ctx = None
w.close = None
def init_redis_client(cls):
# 防止多线程竞争，获取到锁后依然可能已经初始化了
if cls.redis_objs:
return
cls.instance_name = conf.get('sentinel', "")
cls.socket_timeout = conf.get("socket_timeout", 5)
cls.connect_timeout = conf.get("connect_timeout", 0.1)
if not cls.instance_name:
# 单例模式
host = conf["host"]
cls.redis_objs[cls.MODE_READ] = Redis(
host=host.split(':')[0],
port=host.split(':')[1],
socket_timeout=cls.socket_timeout,
socket_connect_timeout=cls.connect_timeout,
retry_on_timeout=1
)
cls.redis_objs[cls.MODE_WRITE] = cls.redis_objs[cls.MODE_READ]
else:
# 哨兵模式
sentinel = Sentinel(
cls.parse_config(),
socket_timeout=cls.socket_timeout,
socket_connect_timeout=cls.connect_timeout,
retry_on_timeout=1
)
cls.redis_objs[cls.MODE_READ] = sentinel.slave_for(cls.instance_name)
cls.redis_objs[cls.MODE_WRITE] = sentinel.master_for(cls.instance_name)
def post(self, request):
form = AccessTokenForm.create_from_request(request)
if not form.is_valid():
raise ValidationException(request, form)
backend = ModelBackend()
user = backend.authenticate(request,
username=form.cleaned_data['username'],
password=form.cleaned_data['password'])
if not user:
raise UnauthorizedException(request)
access_token = JWTFactory(user.pk).access()
jti, refresh_token = JWTFactory(user.pk).refresh()
redis = Redis(host=settings.REDIS_HOST,
port=settings.REDIS_PORT,
db=settings.REDIS_DATABASE)
redis.set(f"refresh_token:{jti}", jti)
redis.expire(f"refresh_token:{jti}",
settings.SECURED_VIEW_JWT_REFRESH_TOKEN_EXPIRATION)
return SingleResponse(request, {
'access_token': access_token,
'refresh_token': refresh_token
},
status=HTTPStatus.OK)
def post(self, request):
form = RefreshTokenForm.create_from_request(request)
if not form.is_valid():
raise ValidationException(request, form)
try:
claims = JWTFactory.decode(form.cleaned_data['refresh'])
except JoseError as e:
raise ProblemDetailException(request,
_('Invalid token.'),
status=HTTPStatus.UNAUTHORIZED,
previous=e)
redis = Redis(host=settings.REDIS_HOST,
port=settings.REDIS_PORT,
db=settings.REDIS_DATABASE)
if not redis.exists(f"refresh_token:{claims['jti']}"):
raise UnauthorizedException(request)
access_token = JWTFactory(claims['sub']).access()
return SingleResponse(request, {'access_token': access_token},
status=HTTPStatus.OK)
def setup_rq_connection():
redis_conn = get_current_connection()
if redis_conn == None:
opts = OPTIONS.get('connection')
logger.debug('Establishing Redis connection to DB %(db)s at %(host)s:%(port)s' % opts)
redis_conn = Redis(**opts)
push_connection(redis_conn)
def __init__(self, client=None):
if not client:
self.client = Redis(host='localhost',
port=6379,
db=13,
password=None)
else:
self.client = client
def _build_redis(host: str,
port: int=6379,
db: int=0,
password: str=None) -> Redis:
return Redis(host=host,
port=port,
db=db,
password=password)
def init_config(self, option: WatcherOptions):
if option.optional_update_callback:
self.set_update_callback(option.optional_update_callback)
else:
raise WatcherError("Casbin Redis Watcher callback not "
"set when an update was received")
if option.sub_client:
self.sub_client = option.sub_client
else:
# TODO
self.sub_client = Redis().client()
if option.pub_client:
self.pub_client = option.pub_client
else:
# TODO
self.pub_client = Redis().client()
def connect(self):
conf = parse_uri(self._uri)
self._channel = conf.pop('channel', self._name)
self._method = conf.pop('method', 'pubsub')
self._client = Redis(**conf)
if self._method == 'pubsub':
self._pubsub = self._client.pubsub()
self._pubsub.subscribe(self._channel)
def connection(self):
"""Get Redis connection from the pool."""
if not self._connection:
self._connection = Redis(connection_pool=_connection_pool)
try:
self._connection.ping()
except exceptions.RedisError:
self._connection = None
raise
return self._connection
def __init__(self, app, config):
if 'host' not in config:
raise ConfigurationError("Redis extension needs 'host' configured")
if 'password' not in config:
raise ConfigurationError(
"Postmark extension needs 'password' configured")
host = config['host']
password = config['password']
redis = Redis(host=host, password=password)
register_singleton(redis, 'redis')
def testRegisterSensorDataDbmsListener(self): #check for true when the connection variables are ok self.assertEqual(True, self.persistenceUtil.registerSensorDataDbmsListener()) #add an invalid port to the jedisSensor self.persistenceUtil.r_sensor = Redis(host = "localhost", port = 6890) #check for false when connection variables are invalid self.assertEqual(False, self.persistenceUtil.registerSensorDataDbmsListener())
def flushall(self):
if self.cluster_size == 1:
subprocess.check_call([
'redis-cli', '-h', 'erasuretester_redis-standalone_1',
'FLUSHALL'
])
elif self.cluster_size >= 2:
nodes = self._get_running_nodes()
for redis in (Redis(x[0], x[1])
for x in (y['ip_port'].split(':') for y in nodes)):
redis.flushall()
def redis_client(self) -> Redis:
"""Provide an instance of Redis client."""
if self._redis_client is None:
redis_client = Redis(connection_pool=self.redis_conn_pool)
self._redis_client = redis_client
self._logger.debug("[%s]: Initialized Redis client: %s",
self.__name__, self._redis_client)
return self._redis_client
def redis(self):
"""Returns the redis.client object for the given host/port
Performing lazy initialization if it hasn't already been created
"""
if not self.__redis:
redis = Redis( host=self.__host, port=self.__port, password=self.__password, socket_timeout=self.__connection_timeout )
self.__update_latest( redis )
#__update_latest can raise an exception, so don't assign to self until it
#completes successfully
self.__redis = redis
return self.__redis
def testWriteSensorDataToDbms(self): #Create ActuatorData instance sensorData = SensorData(); #write to redis and check if it returns true self.assertEqual(True, self.persistenceUtil.writeSensorDataToDbms(sensorData)); #add an invalid port to the jedisSensor self.persistenceUtil.r_sensor = Redis(host = "localhost", port = 6890) #write to redis and check if it returns false self.assertEqual(False, self.persistenceUtil.writeSensorDataToDbms(sensorData));
def Connect(self):
from redis.client import Redis
try:
WriteBehindLog("Connect: connecting to {}:{} password: {}".format(
self.host, self.port, self.password))
r = Redis(host=self.host,
port=self.port,
password=self.password,
decode_responses=True)
except Exception as e:
msg = "Cannot connect to Redis. Exception: {}".format(e)
WriteBehindLog(msg)
raise Exception(msg) from None
return r
def testWriteActuatorDataToDbms(self):
#Create ActuatorData instance
actuatorData = ActuatorData()
#write to redis and check if it returns true
self.assertEqual(
True, self.persistenceUtil.writeActuatorDataToDbms(actuatorData))
#add an invalid port to the jedisActuator
self.persistenceUtil.r_actuator = Redis(host="localhost", port=6379)
#write to redis and check if it returns false
self.assertEqual(
True, self.persistenceUtil.writeActuatorDataToDbms(actuatorData))
class Statistics:
CON = Redis(
host=settings.REDIS_STATISTICS_HOST, port=settings.REDIS_STATISTICS_PORT,
db=settings.REDIS_STATISTICS_DB, decode_responses=True,
)
def __init__(self, product_id):
self.con = self.CON
self.product_id = product_id
@property
def key(self):
return f'flow_{self.product_id}'
@property
def key_uv_add(self):
return f'flow_uv_{self.product_id}'
def add_uv(self, info):
identify = info['identify']
meta = info['meta']
key = self.key_uv_add
root_key = self.key
def func(pipe):
if identify and pipe.sismember(key, identify):
return
ip = meta.get('HTTP_X_FORWARDED_FOR') or meta.get('REMOTE_ADDR')
if not ip:
return
if pipe.sismember(key, ip):
return
pipe.multi()
if identify:
pipe.sadd(key, identify)
pipe.sadd(key, ip)
pipe.incr(root_key, 1)
return 'success'
return self.con.transaction(func, key, value_from_callable=True)
def get_uv_num(self):
n = self.con.get(self.key)
n = n and int(n) or 0
return n
def __get_redis_client(self) -> RedisCluster:
if not self.__redis_client:
if len(startup_nodes) > 1:
self.__redis_client = RedisCluster(startup_nodes=startup_nodes,
decode_responses=False,
password=server_config.redis_server_password)
else:
node = startup_nodes[0]
host = node["host"]
port = node["port"]
self.__redis_client = Redis(connection_pool=BlockingConnectionPool(
host=host, port=port,
password=server_config.redis_server_password,
db=release_cache_db_index)
)
return self.__redis_client
def __init__(self, sentinels, min_other_sentinels=0, sentinel_kwargs=None,
**connection_kwargs):
# if sentinel_kwargs isn't defined, use the socket_* options from
# connection_kwargs
if sentinel_kwargs is None:
sentinel_kwargs = {
k: v
for k, v in connection_kwargs.items()
if k.startswith('socket_')
}
self.sentinel_kwargs = sentinel_kwargs
self.sentinels = [Redis(hostname, port, **self.sentinel_kwargs)
for hostname, port in sentinels]
self.min_other_sentinels = min_other_sentinels
self.connection_kwargs = connection_kwargs
def delete():
r = Redis()
# # iterate a list in batches of size n
# def batcher(iterable, n):
# args = [iter(iterable)] * n
# return zip_longest(*args)
# # in batches of 500 delete keys matching user:*
# for keybatch in batcher(r.scan_iter('user:*'),500):
# print("keybatch", keybatch)
# r.delete(*keybatch)
for key in r.scan_iter("*"):
# delete the key
r.delete(key)
def __init__(self,
file,
static_folder='static',
static_url='/static',
template_path='templates'):
self.redis = Redis(host='localhost')
self.static_folder = os.path.join(os.path.abspath(file), static_folder)
self.static_url = static_url
self.jinja_env = Environment(loader=FileSystemLoader(template_path),
autoescape=True)
self.url_map = Map([
Rule('/', endpoint='welcome'),
Rule('/a', endpoint='a'),
Rule('/b', endpoint='b'),
Rule('/c', endpoint='c'),
Rule(static_url, endpoint='file_system')
])
def get_client(self, mode, timeout=None):
# def _curr_session_time_out():
# return timeout if timeout is not None else self.sock_timeout
if timeout is not None:
# 如果指定了超时，就临时生成一个符合这个超时的连接，使用短连接，这个场景主要应对DI轮训服务，不能频繁超时
if not self.__class__.instance_name:
host = conf["host"]
return Redis(
host=host.split(':')[0],
port=host.split(':')[1],
socket_timeout=self.socket_timeout+2,
socket_connect_timeout=self.connect_timeout,
retry_on_timeout=1,
socket_keepalive=False
)
curr_sentinel = Sentinel(
self.parse_config(),
socket_timeout=self.socket_timeout,
socket_connect_timeout=self.connect_timeout,
)
if mode == self.MODE_WRITE:
return curr_sentinel.master_for(
self.instance_name,
socket_timeout=timeout+2,
socket_connect_timeout=self.connect_timeout,
socket_keepalive=False, # 短连接
retry_on_timeout=True
)
else:
return curr_sentinel.slave_for(
self.instance_name,
socket_timeout=timeout+2,
socket_connect_timeout=self.connect_timeout,
socket_keepalive=False,
retry_on_timeout=True
)
else:
return self.__class__.redis_objs[mode]
def count_calls(method: Callable) -> Callable:
""" Takes a single method Callable argument and returns a Callable
"""
redis = Redis()
@wraps(method)
def wrapper(url):
""" Function wrapper count calls
"""
redis.incr(f'count:{url}')
res = redis.get(url)
if res:
return res.decode("utf-8")
result = method(url)
redis.setex(url, 10, result)
return result
return wrapper
def __new__(cls, *args, **kwargs):
pid = os.getpid()
if not hasattr(cls, '_instance') or pid != cls._pid:
print("PID is {} and father PID is {}".format(
os.getpid(), os.getppid()))
if hasattr(cls, "_pid"):
print("Instance PID is {} and PID is {}".format(cls._pid, pid))
if REDIS_CLUSTER:
pool = ClusterConnectionPool(*args, **kwargs)
cls._instance = RedisCluster(connection_pool=pool,
decode_responses=True,
password=REDIS_PASSWORD)
cls._pid = pid
else:
pool = ConnectionPool(*args, **kwargs)
cls._instance = Redis(connection_pool=pool,
password=REDIS_PASSWORD)
cls._pid = pid
return cls._instance
def main(argv=sys.argv):
if len(argv) < 2:
usage(argv)
config_uri = argv[1]
options = parse_vars(argv[2:])
setup_logging(config_uri)
settings = get_appsettings(config_uri, options=options)
logging.getLogger('c2corg_ui').setLevel(logging.INFO)
log.info('Cache Redis: {0}'.format(settings['redis.url']))
# we don't really need a connection pool here, but the `from_url`
# function is convenient
redis_pool = ConnectionPool.from_url(
settings['redis.url'], max_connections=1)
# remove all keys from the database
r = Redis(connection_pool=redis_pool)
r.flushdb()
log.info('Flushed cache')
def connect(self):
if (len(self.startup_nodes) < 2):
host = self.startup_nodes[0].get('host')
port = self.startup_nodes[0].get('port')
if self.password:
self.redis_pool = redis.ConnectionPool(host=host,
port=port,
db=0)
else:
self.redis_pool = redis.ConnectionPool(host=host,
port=port,
password=self.password,
db=0)
self.redis_instance = Redis(connection_pool=self.redis_pool,
decode_responses=True)
return self.redis_instance
#, skip_full_coverage_check=True
self.redis_cluster = RedisCluster(startup_nodes=self.startup_nodes,
password=self.password)
return self.redis_cluster
def _connect(self) -> NoReturn:
"""
Connection to redis server with retries
:param retry_counter: counter of times of retrying
:return: Redis connection if it was successful and None otherwise
"""
try:
self._kv_storage = Redis(
host=self.host,
port=self.port,
socket_timeout=self.timeout
)
self._kv_storage.ping()
except Exception:
logging.error(
"Got error while connecting to redis with host %s and port %d.",
self.host,
self.port
)
