async def _startup(self) -> None:
"""Creates needed sessions.
"""
await Sessions.smtp.connect()
await Sessions.database.connect()
Sessions.aiohttp = ClientSession()
try:
Sessions.cache = Cache(Cache.REDIS)
await Sessions.cache.exists("connection")
except ConnectionRefusedError:
Sessions.cache = Cache(Cache.MEMORY)
logger.warning(
"Memory cache being used, use redis for production."
)
if self.clear_cache:
await Sessions.cache.clear()
if Config.upload_type == B2UploadSettings:
await self.b2.authorize()
self.background_tasks = await create_scheduler()
for to_spawn in TASKS_TO_SPAWN:
await self.background_tasks.spawn(to_spawn())
await cache_community_types(self.community_types)
await create_product_and_set(self.product_name)
def test_calls_parse_uri_path_from_cache(self):
with patch("aiocache.factory.Cache") as mock:
mock.get_scheme_class.return_value.parse_uri_path = Mock(return_value={"arg1": "arg1"})
Cache.from_url("redis:///")
mock.get_scheme_class.return_value.parse_uri_path.assert_called_once_with("/")
mock.assert_called_once_with(mock.get_scheme_class.return_value, arg1="arg1")
def test_new(self, cache_type):
kwargs = {"a": 1, "b": 2}
cache_class = Cache.get_scheme_class(cache_type)
with patch("aiocache.{}.__init__".format(cache_class.__name__)) as init:
cache = Cache(cache_class, **kwargs)
assert isinstance(cache, cache_class)
init.assert_called_once_with(**kwargs)
def test_cached():
cache = Cache(Cache.REDIS,
endpoint="127.0.0.1",
port=6379,
namespace="main")
loop = asyncio.get_event_loop()
loop.run_until_complete(cached_call())
assert loop.run_until_complete(cache.exists("key")) is True
loop.run_until_complete(cache.delete("key"))
loop.run_until_complete(cache.close())
def test_alias():
loop = asyncio.get_event_loop()
loop.run_until_complete(default_cache())
loop.run_until_complete(alt_cache())
cache = Cache(Cache.REDIS)
loop.run_until_complete(cache.delete("key"))
loop.run_until_complete(cache.close())
loop.run_until_complete(caches.get('default').close())
def redis_cache(event_loop):
cache = Cache(Cache.REDIS, namespace="test", pool_max_size=1)
yield cache
for _, pool in RedisBackend.pools.items():
pool.close()
event_loop.run_until_complete(pool.wait_closed())
async def test_exception_in_cache_backend(
aiohttp_client: TestClientFixture) -> None:
"""check if application still works if cache backend is misconfigured
or cache backend doesn't work"""
handler_hits = 0
@cached
async def handler(_: web.Request) -> web.Response:
nonlocal handler_hits
handler_hits += 1
return web.Response(body=b"Hello world")
app = web.Application()
app.router.add_route('GET', '/', handler)
cache = Cache(
Cache.REDIS,
endpoint="such.hostname.must.not.exist",
serializer=PickleSerializer(),
namespace="main",
ttl=60,
)
register_cache(app, cache)
client = await aiohttp_client(app)
hits = 10
for i in range(hits):
resp = await client.get("/")
assert await resp.read() == b"Hello world"
assert resp.status == 200
assert handler_hits == hits
async def clear_cache(name: str, background_tasks: BackgroundTasks) -> None:
cache = Cache(Cache.REDIS,
endpoint=app_settings.CACHE_HOST,
port=app_settings.CACHE_PORT)
if name == "all":
background_tasks.add_task(cache.delete, "/*")
else:
background_tasks.add_task(cache.delete, f"/{name}*")
def initAppObjects(app):
app["cache"] = Cache(Cache.MEMORY)
app["resolver"] = aiodns.DNSResolver(app.loop)
app["websockets"] = set()
app["subscriptions"] = WEBSOCKET_SUBSCRIPTIONS
app["publishTypes"] = WEBSOCKET_PUBLISH_TYPES
app["threads"] = {}
app["clientThreads"] = {}
app["threadStops"] = {}
app["clientThreadStops"] = {}
async def redis_prepare():
if Config.IN_MEMORY_CACHE:
cache = Cache()
else:
# noinspection PyBroadException
try:
sentinel = await aioredis.create_sentinel(
sentinels=[f'redis://{Config.REDIS_HOST}:{Config.REDIS_PORT}'],
password=Config.REDIS_PASSWORD)
cache = RedisCache(sentinel=sentinel, master=Config.REDIS_USER)
except:
cache = Cache()
caches._caches['redis'] = cache
# noinspection PyGlobalUndefined
global _get_user
_get_user = cached(alias='redis',
ttl=60,
key_builder=lambda f, request, email: f'user_{email}',
timeout=0.5)(_get_user)
async def init_cache(app: web.Application) -> None:
if app['config'].redis_host:
app['cache'] = Cache(
Cache.REDIS,
endpoint=app['config'].redis_host,
port=app['config'].redis_port,
namespace='jaundice',
serializer=JsonSerializer(),
)
else:
app['cache'] = None
async def test_cache_middleware(aiohttp_client: TestClientFixture) -> None:
"""
test if cache middleware works
and doesn't prevent other middlewares from execution
"""
handler_hits = 0
before_cache_middleware_hits = 0
after_cache_middleware_hits = 0
@web.middleware
async def before_cache_middleware(request: web.Request,
handler: Any) -> web.Response:
nonlocal before_cache_middleware_hits
before_cache_middleware_hits += 1
return await handler(request)
@web.middleware
async def after_cache_middleware(request: web.Request,
handler: Any) -> web.Response:
nonlocal after_cache_middleware_hits
after_cache_middleware_hits += 1
return await handler(request)
@cached
async def handler(_: web.Request) -> web.Response:
nonlocal handler_hits
handler_hits += 1
return web.Response(body=b"Hello world")
app = web.Application(middlewares=[before_cache_middleware])
app.router.add_route('GET', '/', handler)
cache = Cache(
Cache.MEMORY,
serializer=PickleSerializer(),
namespace="0",
ttl=60,
)
register_cache(app, cache)
# it's artificial case
app.middlewares.append(after_cache_middleware)
client = await aiohttp_client(app)
hits = 10
for i in range(hits):
resp: aiohttp.ClientResponse = await client.get("/")
assert await resp.read() == b"Hello world"
assert resp.status == 200
assert handler_hits == 1
assert after_cache_middleware_hits == 1
assert before_cache_middleware_hits == hits
async def start(cls):
cls.fernet = Fernet(cls.salt.encode())
await db_models.db.set_bind(bind=cls.db_url, min_size=1)
cls.redis = Cache.from_url(cls.redis_url)
cls.application = await Application.get_main() if not cls.test else await Application.get_test()
cls.intra = IntraAPI(config=cls)
await cls.intra.load()
courses = await Courses.get_courses()
cls.courses = {cursus.id: cursus.name for cursus in courses}
cls.cursus_id = [cursus.id for cursus in courses if cursus.is_primary][0]
cls.local = Localization()
cls.local.load(data=read_json(cls.localization))
cls.sub_apps = SubApps(intra=cls.intra, local=cls.local)
async def init(loop):
conf = load_config(PROJ_ROOT / 'config' / 'config.yml')
app = web.Application(loop=loop)
app.router.add_route('GET', "/api/v2/{item}", method)
app.router.add_route('GET', "/api/{item}", method)
cache = Cache(plugins=[HitMissRatioPlugin(), TimingPlugin()])
print(conf['allowed_items'])
if 'host' in conf:
host = conf['host']
else:
host = '127.0.0.1'
if 'port' in conf:
port = conf['port']
else:
port = '443'
if 'access_log_format' in conf:
access_log_format = conf['access_log_format']
else:
access_log_format = '%a %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"'
if 'scheme' in conf:
if conf['scheme'] == 'https':
if 'sslcertchain' and 'sslkey' in conf:
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
ssl_context.load_verify_locations(conf['sslcertchain'],
conf['sslkey'])
ssl_context.load_cert_chain(conf['sslcertchain'],
conf['sslkey'])
else:
raise NameError(
'sslcertchain / sslkey missing in the configuration')
else:
ssl_context = None
else:
ssl_context = None
app['config'] = conf
user, password, realm = conf['authentication']['user'], conf[
'authentication']['password'], conf['authentication']['realm']
await setup(app, AllowedHosts(conf['allowed_hosts']),
BasicAuth(user, password, realm))
app['cache'] = cache
return app, cache, host, port, access_log_format, ssl_context
async def clear_cache(name: str, background_tasks: BackgroundTasks) -> None:
cache = Cache(Cache.REDIS,
endpoint=app_settings.CACHE_HOST,
port=app_settings.CACHE_PORT)
# TODO: this try/except wrap temporarily fixes an issue in the aiocache library, resulting in an TypeError if
# there are no keys in Redis with the given namespace. This is solved on the main branch in aoicache but is not
# yet released (23 June 2022).
try:
if name == "all":
await cache.clear(namespace="orchestrator")
else:
await cache.clear(namespace=f"orchestrator:{name}")
except TypeError:
pass
def endpoint_cache(function: _AsyncCallable) -> _AsyncCallable:
from .routing import request_headers, response_headers # noqa:F401
vf = CachedValidatedFunction(function, config={})
cache: BaseCache = AioCache.from_url(CACHE_URI) # type:ignore
config: CacheConfig = getattr(function, "cache_config",
CacheConfig.new(function))
cache.namespace, cache.ttl = config.namespace, config.ttl.total_seconds()
if not CACHE_ENABLED:
config.enabled = False
@wraps(function)
async def wrapper(*args, **kwargs):
cache_policy: str = request_headers.get().get("cache-control",
"public")
if (not config.enabled) or (cache_policy.casefold() == "no-store"):
return await vf.call(*args, **kwargs)
key = hashlib.md5((model :=
vf.serialize(args=args, kwargs=kwargs)).json(
exclude={"self"},
sort_keys=True,
ensure_ascii=False).encode()).hexdigest()
if cache_policy.casefold() == "no-cache":
await cache.delete(key)
if await cache.exists(key):
logger.debug(f"Request to endpoint <g>{function.__qualname__}</g> "
f"restoring from <e>{key=}</e> in cache data.")
response_headers.get().setdefault("X-Cache-Hit", key)
result, cache_date = await cache.get(key)
else:
result, cache_date = await vf.execute(model), datetime.now()
await cache.set(key, (result, cache_date))
response_headers.get().update({
"Cache-Control":
"public",
"Expires":
format_date_time(cache_date.timestamp() + cache.ttl),
})
return result
return wrapper # type:ignore
async def get_with_client(url, access_token=None, cache=None, client=None):
"""Fetches the prismic api JSON.
Returns :class:`~Api` object.
:param url: URL to the api of the repository (mandatory).
:param access_token: The access token (optional).
:param cache: The cache object. Optional, will default to a in-memory cache if None is passed.
:param client: The httpx client. If not passed, one client will be created for each subsequent http request.
"""
if cache is None:
cache = Cache(Cache.MEMORY)
return Api(
await get_json(url, access_token=access_token, cache=cache, ttl=5, client=client),
access_token,
cache,
client
)
def __init__(
self,
app: "Application",
twitch_bearer_token: str,
logger: "Logger" = None,
*args,
**kwargs,
):
self.app = app
self.config = config = app["config"]
intents = discord.Intents.default()
intents.members = True
super().__init__(
owner_id=self.config["bot"].owner_id,
command_prefix=determine_prefix,
status=discord.Status.idle,
intents=intents,
*args,
**kwargs,
)
self.db = app["db"]
self.logger = logger
self.description = config["bot"].description
self.default_prefix = config["bot"].default_prefix
self.session = aiohttp.ClientSession()
self.pool = concurrent.futures.ThreadPoolExecutor()
self.launch_time = None
self.twitch_client = twitch.Client(config["twitch"].client_id,
twitch_bearer_token, self.session)
self.dbl_client = (dbl.DBLClient(
self, config["bot"].dbl_token, session=self.session, autopost=True)
if config["bot"].dbl_token else None)
self.service_guild: Optional[discord.Guild] = None
self.errors_channel: Optional[discord.TextChannel] = None
self.reports_channel: Optional[discord.TextChannel] = None
self.log_channel: Optional[discord.TextChannel] = None
cors = self.app["cors"]
resource = cors.add(
self.app.router.add_resource(r"/wh/twitch/{topic}/{id}"))
cors.add(resource.add_route("GET", self.handler_get))
cors.add(resource.add_route("POST", self.handler_post))
self.cache = Cache()
async def main():
PORT = config["server"]["port"]
DEBUG_MODE = config["debug_mode"]
# Create the global connection pool.
async with aiomysql.create_pool(
host=config["db"]["host"],
port=config["db"]["port"],
user=config["db"]["user"],
password=str(config["db"]["pwd"]),
db=config["db"]["database"],
) as db:
cache = Cache(Cache.MEMCACHED,
endpoint="127.0.0.1",
port=11211,
namespace="main")
# await maybe_create_tables(db)
app = Application(db, cache)
if DEBUG_MODE:
# 单进程启动
app.listen(PORT)
else:
# 多进程启动
server = HTTPServer(app)
server.ssl_options = {
"certfile": config["server"]["ssl_key"],
"keyfile": config["server"]["ssl_cert"]
}
# 在Linux系统bind方法不起作用，需要使用listen；在macOS系统listen方法不起作用，需要使用bind
if sys.platform == 'linux':
server.listen(PORT)
else:
server.bind(PORT)
server.start(num_processes=config["server"]["num_processes"])
# the server will simply run until interrupted
# with Ctrl-C, but if you want to shut down more gracefully,
# call shutdown_event.set().
print(f"tornado running on port {PORT}")
shutdown_event = tornado.locks.Event()
await shutdown_event.wait()
async def cache():
conf = load_config(PROJ_ROOT / 'config' / 'config-gunicorn.yml')
logging.basicConfig(level=logging.DEBUG)
app = web.Application()
app.router.add_route('GET', "/api/v2/{item}", method)
app.router.add_route('GET', "/api/v2/{item}/{domain}", method)
app.router.add_route('GET', "/api/{item}", method)
app.router.add_route('GET', "/api/{item}/{domain}", method)
memcached_host = conf['cache']['memcached_host']
memcached_port = conf['cache']['memcached_port']
#cache = Cache(plugins=[HitMissRatioPlugin(), TimingPlugin()])
lookup_type = {}
cache = Cache(Cache.MEMCACHED,
endpoint=memcached_host,
port=memcached_port,
serializer=JsonSerializer(),
plugins=[HitMissRatioPlugin(),
TimingPlugin()])
if 'statsd' in conf:
if conf['statsd']['enable']:
hostname = socket.gethostname().split('.', 1)[0]
c = statsd.StatsClient(conf['statsd']['host'],
conf['statsd']['port'],
prefix=conf['statsd']['prefix'])
t = MetricsTimer(conf['statsd']['interval'], cache_metrics, cache,
lookup_type, c, hostname)
app['config'] = conf
user, password, realm = conf['authentication']['user'], conf[
'authentication']['password'], conf['authentication']['realm']
await setup(app, AllowedHosts(conf['allowed_hosts']),
BasicAuth(user, password, realm))
app['cache'] = cache
app['lookup_type'] = lookup_type
return app
async def test_cache(self, mocked_fetch, morpher):
mocked_fetch.side_effect = fetch_return_inosmi_html
results = []
cache = Cache(Cache.MEMORY)
await process_article(
session=None,
morph=morpher,
charged_words=['бодрость'],
url='http://localhost',
results=results,
cache=cache,
)
assert mocked_fetch.call_count == 1
await process_article(
session=None,
morph=morpher,
charged_words=['бодрость'],
url='http://localhost',
results=results,
cache=cache,
)
assert mocked_fetch.call_count == 1 # second call returned from cache
async def get(url, access_token=None, cache=None, **client_kwargs):
"""Fetches the prismic api JSON. Generates only one httpx client for the async context.
Yields :class:`~Api` object.
Usage:
>>> import prismic
>>> async with prismic.get("http://your-repo.prismic.io/api", "access_token") as api:
... doc = await api.get_by_uid("speculoos-macaron")
:param url: URL to the api of the repository (mandatory).
:param access_token: The access token (optional).
:param cache: The cache object. Optional, will default to a in-memory cache if None is passed.
"""
if cache is None:
cache = Cache(Cache.MEMORY)
async with httpx.AsyncClient(**client_kwargs) as client:
yield Api(
await get_json(url, access_token=access_token, cache=cache, ttl=5, client=client),
access_token,
cache,
client
)
import asyncio
from aiocache import caches, Cache
from aiocache.serializers import PickleSerializer
cache = Cache(Cache.REDIS,
endpoint="127.0.0.1",
port=6379,
namespace="cache_hh",
serializer=PickleSerializer())
cache_time = 86400
logger = logging.getLogger(__name__)
class MyCustomPlugin(BasePlugin):
async def pre_set(self, *args, **kwargs):
logger.info("I'm the pre_set hook being called with %s %s" %
(args, kwargs))
async def post_set(self, *args, **kwargs):
logger.info("I'm the post_set hook being called with %s %s" %
(args, kwargs))
cache = Cache(plugins=[HitMissRatioPlugin(),
TimingPlugin(),
MyCustomPlugin()],
namespace="main")
async def run():
await cache.set("a", "1")
await cache.set("b", "2")
await cache.set("c", "3")
await cache.set("d", "4")
possible_keys = ["a", "b", "c", "d", "e", "f"]
for t in range(1000):
await cache.get(random.choice(possible_keys))
assert cache.hit_miss_ratio["hit_ratio"] > 0.5
import asyncio
import logging
from aiocache import Cache
from aiocache.lock import RedLock
logger = logging.getLogger(__name__)
cache = Cache(Cache.REDIS, endpoint='127.0.0.1', port=6379, namespace='main')
async def expensive_function():
logger.warning('Expensive is being executed...')
await asyncio.sleep(1)
return 'result'
async def my_view():
async with RedLock(cache, 'key', lease=2): # Wait at most 2 seconds
result = await cache.get('key')
if result is not None:
logger.info('Found the value in the cache hurray!')
return result
result = await expensive_function()
await cache.set('key', result)
return result
async def concurrent():
await asyncio.gather(my_view(), my_view(), my_view())
from aiocache import Cache from backend import config from backend.db_clients import get_db_client # Initialize cache cache = Cache() # Initialize db client db_client = get_db_client((config.DATABASE_TYPE))
async def get_cached_tags():
cache = Cache()
result = await cache.get("tags")
logger.info("from cache, tags in size = {}".format(
0 if result is None else len(result)))
return result
async def get_cached_categories():
cache = Cache()
result = await cache.get("categories")
logger.info("from cached, categories in size = {}".format(
0 if result is None else len(result)))
return result
async def get_cached_products(page):
cache = Cache()
result = await cache.get("products{}".format(page))
logger.info("from cache, products in page = {0} in size = {1}".format(
page, 0 if result is None else len(result)))
return result
"project-jupyter/jupyter-meta-documentation/translate/#{language}/{resource}/1"
"?q={query_string}"),
}
FILTER_RESOURCES_TO_BE_TRANSLATED = {
"python": lambda r: r.split("--")[0] in ["bugs", "howto", "library"],
"jupyter": None,
}
WEEK_IN_SECONDS = 604_800
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
logger.info(config.CACHE_URL)
cache = Cache.from_url(config.CACHE_URL)
STRINGS_CACHE = defaultdict(dict)
async def transifex_api(url, project, data=None, retrying=False, ttl=3600):
url = urljoin(TRANSIFEX_API[project], url)
if not data and (in_cache := await cache.get(url)):
return in_cache
if retrying:
logger.debug("retrying url=%s", url)
auth = aiohttp.BasicAuth(login="******", password=config.TRANSIFEX_TOKEN)
async with aiohttp.ClientSession(auth=auth) as session:
http_method = session.put if data else session.get
