def get_redis_link_wrapper(*args, **kwargs):
link = StrictRedis(host="127.0.0.1", port=7000, decode_responses=True)
orig_exec_method = link.execute_command
async def patch_execute_command(*args, **kwargs):
if args == ('CLUSTER SLOTS',):
# Missing slot 5460
return {
(0, 5459): [{'host': '127.0.0.1', 'port': 7000, 'node_id': str(uuid.uuid4()), 'server_type': 'master'},
{'host': '127.0.0.1', 'port': 7003, 'node_id': str(uuid.uuid4()), 'server_type': 'slave'}],
(5461, 10922): [{'host': '127.0.0.1', 'port': 7001, 'node_id': str(uuid.uuid4()), 'server_type': 'master'},
{'host': '127.0.0.1', 'port': 7004, 'node_id': str(uuid.uuid4()), 'server_type': 'slave'}],
(10923, 16383): [{'host': '127.0.0.1', 'port': 7002, 'node_id': str(uuid.uuid4()), 'server_type': 'master'},
{'host': '127.0.0.1', 'port': 7005, 'node_id': str(uuid.uuid4()), 'server_type': 'slave'}],
}
elif args == ('CONFIG GET', 'cluster-require-full-coverage'):
return {'cluster-require-full-coverage': 'no'}
else:
return await orig_exec_method(*args, **kwargs)
# Missing slot 5460
link.execute_command = patch_execute_command
return link
def add_redis_client(self):
kvstore_host = None
kvstore_port = None
kvstore_db = None
try:
kvstore_host = os.getenv("KVSTORE_HOST",
"redis.dev.muchneededllc.com")
kvstore_port = int(os.getenv("KVSTORE_PORT", 6379))
kvstore_db = int(os.getenv("KVSTORE_DB", 0))
except OSError as e:
logger.error("Couldn't get environmental variables for kvstore. " +
str(e))
exit(1)
try:
if self._loop is not None:
self._redis_client = StrictRedis(host=kvstore_host,
port=kvstore_port,
db=kvstore_db,
loop=self._loop)
self.redis_conn = True
logger.debug("Created Redis Client.")
else:
logger.error(
"Couldn't create redis client because loop hasn't been set."
)
except Exception as e:
logger.error("couldn't open redis.")
raise Exception(e)
async def example_bitfield():
redis = StrictRedis(host='127.0.0.1')
await redis.flushdb()
bitfield = redis.bitfield('example')
res = await (bitfield.set('i8', '#1', 100).get('i8', '#1')).exc()
assert res == [0, 100]
print(res)
def setup_caches(app: WebApp, loop) -> Any:
logger.info('cache.setup_caches', when='before_server_start')
args = app.config.args
caches = []
if args.redis_url:
try:
redis_client = StrictRedis().from_url(args.redis_url)
redis_cache = Cache(redis_client)
if redis_cache:
caches.append(CacheGroupItem(cache=redis_cache,
read=False,
write=True,
speed_tier=SpeedTier.SLOW))
except Exception as e:
logger.error('failed to add redis cache to caches', exception=e)
if args.redis_read_replica_urls:
for url_string in args.redis_read_replica_urls:
url = urlparse(url_string)
logger.info('Adding redis read replicas',
read_replica=url,
host=url.hostname,
port=url.port)
redis_client = StrictRedis().from_url(url_string)
redis_cache = Cache(redis_client)
if redis_cache:
caches.append(
CacheGroupItem(cache=redis_cache,
read=True,
write=False,
speed_tier=SpeedTier.SLOW))
configured_cache_group = CacheGroup(caches=caches)
return configured_cache_group
def __init__(
self,
host: str = "localhost",
port: int = 6379,
db: int = 0,
password: str = None,
) -> None:
self._redis = StrictRedis(host=host, port=port, db=db, password=password)
self.decrease_function = self._redis.register_script(DECREASE_SCRIPT)
def start_redis_client() -> Union[StrictRedis, Exception]:
loop = asyncio.get_event_loop()
try:
pool = StrictRedis(**CONFIG['tokens']['redis'])
# Only way to ensure the pool is connected to redis
loop.run_until_complete(pool.ping())
except Exception as e:
pool = e
return pool
def __init__(
self,
host: str = "localhost",
port: int = 6379,
db: int = 0,
password: str = None,
ssl: bool = False,
) -> None:
self._redis = StrictRedis(
host=host, port=port, db=db, password=password, ssl=ssl
)
self.sliding_function = self._redis.register_script(SLIDING_WINDOW_SCRIPT)
def create(cls,
publish_stream: str,
redis_host: str,
redis_port: int,
period: float = 3600):
redis = StrictRedis(host=redis_host, port=redis_port)
return cls(publish_stream, redis, period=period)
class Producer(ABC):
publish_stream: str
redis: StrictRedis = StrictRedis(host=REDIS_HOST, port=REDIS_PORT)
@abstractmethod
async def initialize(self):
pass
@abstractmethod
async def finalize(self):
pass
@abstractmethod
async def create_event(self) -> Dict[str, Any]:
pass
async def publish(self, event: Dict[str, Any]):
await self.redis.xadd(self.publish_stream, event)
async def run(self):
await self.initialize()
try:
while True:
event = await self.create_event()
await self.publish(event)
finally:
await self.finalize()
async def lock():
from aredis import StrictRedis
redis_conn = StrictRedis(host="127.0.0.1", port=7777)
lock = DistributedLock(redis_conn, "test_key", 10)
async with lock:
time.sleep(int(sys.argv[1]))
print(11111111111, sys.argv[1])
def __init__(self,
host="localhost",
port=6379,
db=None,
password=None,
encoding='utf-8',
socket_keepalive=False,
connection_pool=None,
startup_nodes=None,
create_connection_timeout=None,
project='',
**kwargs):
if project:
project = f'{project}:'
self.cluster_flag = False
self.project = project
if startup_nodes:
from aredis import StrictRedisCluster
if isinstance(startup_nodes, (str, bytes)):
startup_nodes = _normalize_startup_nodes(startup_nodes)
self._redis = StrictRedisCluster(startup_nodes=startup_nodes,
decode_responses=True,
encoding=encoding,
skip_full_coverage_check=True,
**kwargs)
self.cluster_flag = True
else:
self._redis = StrictRedis(host=host,
port=port,
db=db,
password=password,
encoding=encoding,
socket_keepalive=socket_keepalive,
connection_pool=connection_pool,
**kwargs)
def create_app(config):
app = web.Application()
app.config = config
app.model = BlacklistModel(StrictRedis.from_url(app.config.REDIS_URL), )
app.router.add_post('/add', add)
app.router.add_post('/delete', delete)
app.router.add_post('/is_blacklisted', is_blacklisted)
return app
def __init__(self,
redis: RedisOrURL,
key_prefix: str = "JB:WHITE:") -> None:
self.key_prefix = key_prefix
if isinstance(redis, str):
self.redis = StrictRedis.from_url(redis)
else:
self.redis = redis
def redis(request):
try:
client = StrictRedis(config.get("REDIS_CONFIG"))
except Exception as e:
logging.error(
f'"MSG_EXCEPTION", {uuid.uuid1().hex}, "Redis connection fail", None, None, None, {traceback.format_exc()}'
)
return fail(traceback.format_exc())
return ok(client)
def _aredis(self):
# Lazy initialization of the async redis client
if not self._async_redis_client:
self._async_redis_client = StrictRedis(
host=self._options.get('host', 'localhost'),
port=self._options.get('port', 6379),
db=self._options.get('db', 0),
password=self._options.get('password'))
return self._async_redis_client
def test_client_creates_connection_pool(self):
r = StrictRedis.from_url('redis://myhost')
assert r.connection_pool.connection_class == Connection
assert r.connection_pool.connection_kwargs == {
'host': 'myhost',
'port': 6379,
'db': 0,
'password': None,
}
async def doRedisStuff():
print("Connecting to /tmp/redis.sock")
redisConn = StrictRedis(unix_socket_path="/tmp/redis.sock")
print("Connected, flushing db")
await redisConn.flushdb()
print("Setting values")
await redisConn.set("subscribedChannels", pickle.dumps(localData["subscribedChannels"]))
await redisConn.set("launchNotifSent", localData["launchNotifSent"])
await redisConn.set("latestLaunchInfoEmbedDict", pickle.dumps(localData["latestLaunchInfoEmbedDict"]))
print("Done")
def __init__(self, mongo_uri: str, mongo_db_name: str, redis_url: str):
"""
Constructor
:param mongo_uri: Mongo DB URI
:param mongo_db_name: Database Name
:param redis_url: Redis database URL
"""
self.__client = motor.motor_asyncio.AsyncIOMotorClient(mongo_uri)
self.__db = self.__client[mongo_db_name]
self.__redis_client = StrictRedis.from_url(redis_url)
async def example():
client = StrictRedis(host='127.0.0.1', port=6379, db=0)
await client.flushdb()
await client.set('foo', 1)
assert await client.exists('foo') is True
await client.incr('foo', 100)
assert int(await client.get('foo')) == 101
await client.expire('foo', 10)
await client.incr('foo', 100)
def get_redis_link_wrapper(*args, **kwargs):
link = StrictRedis(host="127.0.0.1", port=7000)
orig_exec_method = link.execute_command
async def patch_execute_command(*args, **kwargs):
if args == ('CLUSTER SLOTS', ):
# Missing slot 5460
return [
[0, 5459, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]],
[5461, 10922, [b'127.0.0.1', 7001], [b'127.0.0.1', 7004]],
[10923, 16383, [b'127.0.0.1', 7002], [b'127.0.0.1', 7005]],
]
return await orig_exec_method(*args, **kwargs)
# Missing slot 5460
link.execute_command = patch_execute_command
return link
async def updateTransaction():
client = StrictRedis(host=configs['redis_ip'],
port=6379,
db=0)
# print(await client.zrange("test", 0, -1))
tx_ids = []
async with await client.pipeline() as pipe:
for i in range(0, 10000):
await pipe.incr('test')
await pipe.delete('test')
tx_ids = await pipe.execute()
print("length of tx_ids:", len(tx_ids))
def create(
cls,
stream: str,
group: str,
redis_host: str,
redis_port: int,
items_per_read: int = 10,
):
consumer = f"{str(cls.__name__)}-{str(uuid4())}"
redis = StrictRedis(host=redis_host, port=redis_port)
return cls(RedisChannel(stream, group, consumer), redis,
items_per_read)
def create(
cls,
subscribe_stream: str,
group: str,
publish_stream: str,
redis_host: str,
redis_port: int,
):
consumer = f"{str(cls.__name__)}-{str(uuid4())}"
redis = StrictRedis(host=redis_host, port=redis_port)
subscribe_channel = RedisChannel(subscribe_stream, group, consumer)
return cls(subscribe_channel, publish_stream, redis)
def __init__(self, name: str, **kwargs):
self.name: str = name
self.settings: Settings = Settings(**kwargs)
self.redis = StrictRedis.from_url(self.settings.redis_url)
self.workers: Set[Worker] = set()
self.tasks: Set[Coroutine] = set()
self.processors: Dict[str, Coroutine] = {}
self.scripts: Dict[str, Callable] = {}
init_logging(
level=self.settings.log_level,
format=self.settings.log_format,
)
def get_redis_link_wrapper(*args, **kwargs):
link = StrictRedis(host="127.0.0.1", port=7000, decode_responses=True)
orig_exec_method = link.execute_command
def patch_execute_command(*args, **kwargs):
if args == ('cluster', 'slots'):
# Missing slot 5460
return [
[0, 5459, [b'127.0.0.1', 7000], [b'127.0.0.1', 7003]],
[5461, 10922, [b'127.0.0.1', 7001], [b'127.0.0.1', 7004]],
[10923, 16383, [b'127.0.0.1', 7002], [b'127.0.0.1', 7005]],
]
elif args == ('CONFIG GET', 'cluster-require-full-coverage'):
return {'cluster-require-full-coverage': 'no'}
else:
return orig_exec_method(*args, **kwargs)
# Missing slot 5460
link.execute_command = patch_execute_command
return link
class SlidingRedisBackend(BaseBackend):
def __init__(
self,
host: str = "localhost",
port: int = 6379,
db: int = 0,
password: str = None,
) -> None:
self._redis = StrictRedis(host=host,
port=port,
db=db,
password=password)
self.sliding_function = self._redis.register_script(
SLIDING_WINDOW_SCRIPT)
async def get_limits(self, path: str, user: str, rule: Rule) -> bool:
epoch = time.time()
ruleset = rule.ruleset(path, user)
keys = list(ruleset.keys())
args = [epoch, json.dumps(ruleset)]
# quoted_args = [f"'{a}'" for a in args]
# cli = f"redis-cli --ldb --eval /tmp/script.lua {' '.join(keys)} , {' '.join(quoted_args)}"
# logger.debug(cli)
r = await self.sliding_function.execute(keys=keys, args=args)
# from tests.backends.test_redis import logger
# logger.debug(f"{epoch} {r} : {all(r)}")
return all(r)
async def decrease_limit(self, path: str, user: str, rule: Rule) -> bool:
raise NotImplementedError()
async def increase_limit(self, path: str, user: str, rule: Rule) -> bool:
raise NotImplementedError()
async def set_block_time(self, user: str, block_time: int) -> None:
await self._redis.set(f"blocking:{user}", True, block_time)
async def is_blocking(self, user: str) -> bool:
return bool(await self._redis.get(f"blocking:{user}"))
async def allow_request(self, path: str, user: str, rule: Rule) -> bool:
if await self.is_blocking(user):
return False
allow = await self.get_limits(path, user, rule)
if not allow and rule.block_time:
await self.set_block_time(user, rule.block_time)
return allow
async def redis_connect() -> StrictRedis:
""" Connecting with redis """
global client
try:
client = StrictRedis.from_url(URL)
ping = await client.ping()
if ping is True:
logger.info("Connection with redis successful")
return client
except Exception:
logger.error("Connection with redis failed")
client = None
def __init__(
self, db, host="localhost", port=6379, decode_responses=True, **kwargs
):
"""Initialize Redis connection."""
self.db: int = db
self.host: str = host
self.port: int = port
self.instance: StrictRedis = StrictRedis(
db=self.db,
host=self.host,
port=self.port,
decode_responses=decode_responses,
**kwargs,
)
def config_app():
#
# REDIS (refresh_token cache)
#
config.redis_client = StrictRedis(host="127.0.0.1", port=6379, db=0)
#
# sanic_jwt configuration & setup
#
custom_claims = [NameClaim, EmailClaim]
Initialize(
config.app,
authenticate=authenticate,
custom_claims=custom_claims,
store_refresh_token=store_refresh_token,
retrieve_refresh_token=retrieve_refresh_token,
retrieve_user=retrieve_user,
add_scopes_to_payload=scope_extender,
debug=True,
claim_iat=True,
refresh_token_enabled=True,
)
#
# user routes
#
config.app.add_route(register, "/users", methods=["POST"])
config.app.add_route(get_users, "/users", methods=["GET"])
config.app.add_route(
update_user_scope, "/users/<userId>/scopes", methods=["PATCH"]
)
config.app.add_route(update_user, "/users/<userId>", methods=["PATCH"])
config.app.add_route(delete_user, "/users/<userId>", methods=["DELETE"])
#
# jogging routes
#
config.app.add_route(add_jogging_result, "/results", methods=["POST"])
config.app.add_route(
update_jogging_result, "/results/<resultId>", methods=["PATCH"]
)
config.app.add_route(
get_jogging_result, "/results/<resultId>", methods=["GET"]
)
config.app.add_route(
delete_jogging_result, "/results/<resultId>", methods=["DELETE"]
)
config.app.add_route(get_jogging_results, "/results", methods=["GET"])
config.app.add_route(
get_jogging_weekly_report, "/results/reports/weekly", methods=["GET"]
)
async def aredis_bench():
client = StrictRedis(host=HOST, port=PORT, db=0)
await client.flushdb()
start = time.time()
for x in range(NUM_INSERTS):
await client.hset("words", "word|{}".format(x), "1")
print('Aredis HSET Done. Duration=', time.time() - start)
hset_dict['aredis'] = time.time() - start
start = time.time()
for x in range(NUM_INSERTS):
await client.hget("words", "word|{}".format(x))
print('Aredis HGET Done. Duration=', time.time() - start)
hget_dict['aredis'] = time.time() - start
