async def main():
redis = aioredis.from_url("redis://localhost")
await redis.set("key", "string-value")
bin_value = await redis.get("key")
assert bin_value == b"string-value"
redis = aioredis.from_url("redis://localhost", decode_responses=True)
str_value = await redis.get("key")
assert str_value == "string-value"
await redis.close()
async def connect(self):
"""
Connects to Redis using two connection pools, one to handle
writing to stream and one for reading.
"""
logger.info(__name__, f"Connecting address={self.address}...")
try:
self._write_pool = aioredis.from_url(self.address)
self._read_pool = aioredis.from_url(self.address)
return self
except (OSError, RedisError) as e: # pragma: no cover
logger.error(__name__, e)
raise StreamOSError(e) from e
def __init__(self):
super().__init__()
self.io_loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.io_loop)
self.redis_client = aioredis.from_url(
f"redis://{config.get('REDIS', 'host', fallback='localhost')}:"
f"{config.getint('REDIS', 'port', fallback=6379)}/{config.getint('REDIS', 'db', fallback=0)}",
decode_responses=True)
self.raw_redis = redis.StrictRedis(host=config.get(
'REDIS', 'host', fallback='localhost'),
port=config.getint('REDIS',
'port',
fallback=6379),
db=config.getint('REDIS',
'db',
fallback=0),
decode_responses=True)
self.sub_client = self.redis_client.pubsub()
self.initialized = False
self.sub_tasks = list()
self.sub_channels = list()
self.channel_router = dict()
self.crontab_router = defaultdict(dict)
self.datetime = None
self.time = None
self.loop_time = None
async def run_redisclient(taskrq, **kwargs):
redis_username = os.environ['REDIS_USER'] # Fatal if missing!
redis_password = os.environ['REDIS_PASSWORD'] # Fatal if missing!
async with aioredis.from_url(
kwargs['endpoint'],
username=redis_username,
password=redis_password,
decode_responses=True,
) as rdb:
if kwargs['check_idle_time'] > 0:
await _task_submission_check_client_idle(rdb, taskrq,
kwargs['check_idle_time'])
task_id = str(uuid.uuid4())
taskctx = {
'id': task_id,
'status_path': taskrq['agent_id'] + '/task/' + task_id,
'rq': taskrq,
'lock': asyncio.Lock(),
'pushed': False,
}
def generate_cancel_handler():
return _cancel_task(
kwargs['endpoint'],
redis_username,
redis_password,
taskrq['agent_id'],
task_id,
taskrq['parent'],
kwargs.get('cancel_task_timeout', 0),
)
# Add a callback to post a cancel-task action:
kwargs['cancel_handler'].add_callback(generate_cancel_handler)
# Context is initially locked. Unlock occurs when task has been
# submitted by _acontrol_task:
await taskctx['lock'].acquire()
tcontrol = asyncio.create_task(_retry_request(_acontrol_task, rdb,
taskctx, **kwargs),
name='tcontrol')
tpoll = asyncio.create_task(_retry_request(_apoll_status, rdb,
taskctx),
name='tpoll')
done, pending = await asyncio.wait({tcontrol, tpoll},
return_when=asyncio.FIRST_COMPLETED)
# Remove the cancel callback
kwargs['cancel_handler'].remove_callback(generate_cancel_handler)
if tpoll in done:
tcontrol.cancel()
return tpoll.result()
if tcontrol in done:
tpoll.cancel()
return tcontrol.result()
async def _cancel_task(
endpoint,
username,
password,
agent_id,
task_id,
parent_task,
timeout,
):
"""Push a cancel-task action
"""
async with aioredis.from_url(
endpoint,
username=username,
password=password,
decode_responses=True,
) as rdb:
print(f'Sending cancel-task request to {agent_id}/task/{task_id}',
file=sys.stderr)
return await rdb.lpush(
f"{agent_id}/tasks",
json.dumps({
'id': str(uuid.uuid4()),
'action': 'cancel-task',
'data': {
'task': task_id,
'timeout': timeout
},
'parent': parent_task,
'extra': {
'title': f"{agent_id}/cancel-task",
'description': f"Terminate task {task_id}",
'isNotificationHidden': True,
},
}))
async def apply_policies(attributes):
"""Apply defined policies to received request."""
sasl_username = attributes.get("sasl_username")
if not sasl_username:
return SUCCESS_ACTION
rclient = aioredis.from_url(settings.REDIS_URL, encoding="utf-8", decode_responses=True)
decr_domain = False
decr_user = False
localpart, domain = split_mailbox(sasl_username)
if await rclient.hexists(constants.REDIS_HASHNAME, domain):
counter = await rclient.hget(constants.REDIS_HASHNAME, domain)
logger.info("Domain {} current counter: {}".format(domain, counter))
if int(counter) <= 0:
return FAILURE_ACTION
decr_domain = True
if await rclient.hexists(constants.REDIS_HASHNAME, sasl_username):
counter = await rclient.hget(constants.REDIS_HASHNAME, sasl_username)
logger.info("Account {} current counter: {}".format(
sasl_username, counter))
if int(counter) <= 0:
return FAILURE_ACTION
decr_user = True
if decr_domain:
await decrement_limit(rclient, "domain", domain)
if decr_user:
await decrement_limit(rclient, "account", sasl_username)
await rclient.close()
logger.debug("Let it pass")
return SUCCESS_ACTION
def __init__(self, station_address=None, **kwargs):
self._name = kwargs.get("name", str(uuid.uuid1()))
self._ready = False
self._station_address = station_address
self._address_book = {}
self._address_actors = defaultdict(list)
self._pool = aioredis.from_url(self._station_address)
async def start(self, token: str, **kwargs: Any):
"Get everything ready in async env"
cache_info = self.config["Redis Info"]
db_info = self.config["PostgreSQL Info"]
self.cache_db = aioredis.from_url(**cache_info)
self.pool, self.gtts = await asyncio.gather(
cast(Awaitable[Pool], asyncpg.create_pool(**db_info)),
asyncgTTS.setup(premium=False, session=self.session),
)
# Fill up bot.channels, as a load of webhooks
for channel_name, webhook_url in self.config["Webhook URLs"].items():
self.channels[channel_name] = discord.Webhook.from_url(
webhook_url, session=self.session, bot_token=self.http.token
)
# Load all of /cogs and /extensions
self.load_extensions("cogs")
self.load_extensions("extensions")
# Send starting message and actually start the bot
if self.shard_ids is not None:
prefix = f"`[Cluster] [ID {self.cluster_id}] [Shards {len(self.shard_ids)}]`: "
self.websocket = await self.create_websocket()
kwargs["reconnect"] = True
else:
prefix = ""
self.websocket = None
self.logger = utils.setup_logging(config["Main"]["log_level"], prefix, self.session)
self.logger.info("Starting TTS Bot!")
await automatic_update.do_normal_updates(self)
await super().start(token, **kwargs)
def __init__(self,
host='127.0.0.1',
port=6379,
socket=None,
key=None,
none_to='None',
numeric_type=float,
writer_interval=0.01,
**kwargs):
"""
setting key lets you override the prefix on the
key used in redis. The defaults are related to the data
being stored, i.e. trade, funding, etc
writer_interval: float
the frequency writer sleeps when there is nothing in the queue.
0 consumes a lot of CPU, while large interval puts pressure on queue.
"""
prefix = 'redis://'
if socket:
prefix = 'unix://'
self.redis = aioredis.from_url(f"{prefix}{host}:{port}")
self.key = key if key else self.default_key
self.numeric_type = numeric_type
self.none_to = none_to
self.running = True
self.exited = False
self.writer_interval = writer_interval
async def setup():
app.redis_client = aioredis.from_url(redis_uri,
encoding='utf-8',
decode_responses=True)
app.http_connector = aiohttp.TCPConnector(limit_per_host=10)
app.http_session_obj = aiohttp.ClientSession(connector=app.http_connector)
app.http_session = await app.http_session_obj.__aenter__()
async def on_startup(self):
await Tortoise.init(db_url=os.environ.get("DB_URL"),
modules={"models": ["common.models"]})
self.redis = aioredis.from_url(os.environ.get("REDIS_URL"),
decode_responses=True)
self.session = aiohttp.ClientSession()
auth_mgr = AuthenticationManager(
self.session,
os.environ["XBOX_CLIENT_ID"],
os.environ["XBOX_CLIENT_SECRET"],
"",
)
auth_mgr.oauth = OAuth2TokenResponse.parse_file(
os.environ["XAPI_TOKENS_LOCATION"])
await auth_mgr.refresh_tokens()
xbl_client = XboxLiveClient(auth_mgr)
self.profile = ProfileProvider(xbl_client)
self.club = ClubProvider(xbl_client)
self.realms = RealmsAPI(aiohttp.ClientSession())
headers = {
"X-Authorization": os.environ["OPENXBL_KEY"],
"Accept": "application/json",
"Accept-Language": "en-US",
}
self.openxbl_session = aiohttp.ClientSession(headers=headers)
def open_redis_client(cls):
"""Create Redis client session object instance.
Based on configuration create either Redis client or Redis Sentinel.
Returns:
aioredis.Redis: Redis object instance.
"""
if cls.redis_client is None:
cls.log.debug("Initialize Redis client.")
if config.REDIS_USER and config.REDIS_PWD:
cls.connection_kwargs = {
"username": config.REDIS_USER,
"password": config.REDIS_PWD,
}
if config.REDIS_USE_SENTINEL:
sentinel = aioredis.sentinel.Sentinel(
[(config.REDIS_HOST, config.REDIS_PORT)],
sentinel_kwargs=cls.connection_kwargs,
)
cls.redis_client = sentinel.master_for("mymaster")
else:
cls.base_redis_init_kwargs.update(cls.connection_kwargs)
cls.redis_client = aioredis.from_url(
"redis://{0:s}".format(config.REDIS_HOST),
**cls.base_redis_init_kwargs,
)
return cls.redis_client
def __init__(self, url, broadcast):
redis = aioredis.from_url(get_redis_url(), decode_responses=True)
self.pubsub = redis.pubsub(ignore_subscribe_messages=True)
self.broadcast = broadcast
self.channel_groups = defaultdict(set)
# Tracks when the bus starts running i.e on the first connection.
self._running = None
async def redis_pool():
# Redis client bound to pool of connections (auto-reconnecting).
redis = aioredis.from_url("redis://localhost",
encoding="utf-8",
decode_responses=True)
await redis.set("my-key", "value")
val = await redis.get("my-key")
print(val)
def __init__(self, prefix: str) -> None:
self._redis: Redis = aioredis.from_url(
f"redis://:{config.REDIS_PASSWORD}@{config.REDIS_HOST}:" +
f"{config.REDIS_PORT}",
encoding="utf-8",
decode_responses=True)
self._prefix = prefix
def connect(self, *, address: str) -> Any:
"""
Creates a Redis connection pool
:param address: str, address = "redis://hostname:6379/0?encoding=utf-8"
"""
self._conn = aioredis.from_url(address)
return self
async def startup():
"""Start up FastAPI server"""
settings = get_settings()
app.state.graph = TccmGraph(settings)
app.state.graph.connect()
redis = aioredis.from_url(url=settings.redis_url,
encoding="utf8",
decode_responses=True)
FastAPICache.init(backend=RedisBackend(redis), prefix="fastapi-cache")
async def startup():
update_loggers()
redis = aioredis.from_url(config.REDIS_HOST,
encoding="utf8",
decode_responses=True)
FastAPICache.init(RedisBackend(redis), prefix="fastapi-cache")
logger.info("initialization successful")
async def connect(self):
print("Here is the content of REDIS_CONFIG:", self.config)
print("Connecting to a single redis server")
config = copy.deepcopy(self.config)
redis_url = config.pop("redis", None)
if not isinstance(redis_url, str):
raise ValueError(
"redis should be specified like redis://[[username]:[password]]@localhost:6379/0")
self._redis = aioredis.from_url(redis_url, **config)
def _connect_redis(self) -> None:
self.log.debug("Connecting Redis")
self.redis = aioredis.from_url(
self._config['redis']['host'],
password=self._config['redis']['password'],
db=self._config['redis']['db-index']
)
self.redis_pubsub = self.redis.pubsub()
async def on_startup():
redis_client = aioredis.from_url(config.PM_REDIS_HOST, decode_responses=True)
## Auth
await db.init(redis_client)
## Traefik
await init_traefik(redis_client)
## PowerDNS
await init_powerdns()
async def main():
# Redis client bound to single connection (no auto reconnection).
redis = aioredis.from_url("redis://localhost",
encoding="utf-8",
decode_responses=True)
async with redis.client() as conn:
await conn.set("my-key", "value")
val = await conn.get("my-key")
print(val)
async def make_redis_pool(host, port):
redis_host = "redis://" + host + ":" + port
try:
redis = from_url(
redis_host
)
return redis
except (OSError, RedisError):
logger.error('failed to create redis connection')
async def main() -> None:
# Instantiate Redis `broker` and `result_backend` connection pools.
broker = aioredis.from_url("redis://localhost:6379/0")
result_backend = aioredis.from_url("redis://localhost:6379/1")
async with broker.client() as b:
async with result_backend.client() as rb:
queue = Queue(b, rb, "default")
funcs = [foo for _ in range(1000)]
args = [(i, j) for (i, j) in zip(range(1000), range(1000, 2000))]
# Enqueue tasks.
for func, arg in zip(funcs, args):
print("enqueing task")
await queue.enqueue(func, *arg)
# Dequeue and execute tasks.
await worker(queue)
def get_async_redis(self):
"""Gets async redis instance using provided redis host and port for this server
Returns
-------
Redis instance
"""
url = f"redis://{self.redis_host}:{self.redis_port}"
redis = aioredis.from_url(url, encoding="utf-8", decode_responses=True)
return redis
async def test_script_register(monkeypatch):
monkeypatch.setattr(redis, "read_text", lambda *args: 'return { "SUCCESS" }')
redis_url = os.environ.get("REDIS_URL", "redis://localhost")
redis_pool = aioredis.from_url(redis_url)
lua_script = redis._LuaScript("dummy")
result = await lua_script.execute(redis_pool, [], [])
assert result == [b"SUCCESS"]
async def main():
redis = aioredis.from_url("redis://localhost")
pubsub = redis.pubsub()
await pubsub.subscribe("channel:1", "channel:2")
asyncio.create_task(reader(pubsub))
await redis.publish("channel:1", "Hello")
await redis.publish("channel:2", "World")
await redis.publish("channel:1", STOPWORD)
async def main():
"""Scan command example."""
redis = aioredis.from_url("redis://localhost")
await redis.mset({"key:1": "value1", "key:2": "value2"})
async with redis.client() as conn:
cur = b"0" # set initial cursor to 0
while cur:
cur, keys = await conn.scan(cur, match="key:*")
print("Iteration results:", keys)
async def main():
redis = aioredis.from_url("redis://localhost")
# No pipelining;
async def wait_each_command():
val = await redis.get("foo") # wait until `val` is available
cnt = await redis.incr("bar") # wait until `cnt` is available
return val, cnt
# Sending multiple commands and then gathering results
async def concurrent():
fut1 = redis.get("foo") # issue command and return future
fut2 = redis.incr("bar") # issue command and return future
# block until results are available
val, cnt = await asyncio.gather(fut1, fut2)
return val, cnt
# Explicit pipeline
async def explicit_pipeline():
pipe = redis.pipeline()
pipe.get("foo").incr("bar")
result = await pipe.execute()
return result
async def context_pipeline():
async with redis.pipeline() as pipe:
pipe.get("foo").incr("bar")
result = await pipe.execute()
return result
async def pipeline_transaction():
async with redis.pipeline(transaction=True) as pipe:
pipe.get("foo").incr("bar")
result = await pipe.execute()
return result
def callback(pipe: aioredis.client.Pipeline):
pipe.get("foo").incr("bar")
async def transaction():
return await redis.transaction(callback)
res = await wait_each_command()
print(res)
res = await concurrent()
print(res)
res = await explicit_pipeline()
print(res)
res = await context_pipeline()
print(res)
res = await pipeline_transaction()
print(res)
res = await transaction()
print(res)
def main():
basicConfig(level=INFO)
redis = aioredis.from_url(getenv("REDIS_DSN", "redis:///"),
encoding="utf-8",
decode_responses=True)
server = WebsocketServer(
redis=redis,
handler_class=PublishEverythingHandler,
)
host = getenv("HOST", "localhost")
port = int(getenv("PORT", 8765))
server.listen(host, port, channel_patterns=["[a-z]*"])
