async def _startup(self) -> None: """Creates needed sessions. """ await Sessions.smtp.connect() await Sessions.database.connect() Sessions.aiohttp = ClientSession() try: Sessions.cache = Cache(Cache.REDIS) await Sessions.cache.exists("connection") except ConnectionRefusedError: Sessions.cache = Cache(Cache.MEMORY) logger.warning( "Memory cache being used, use redis for production." ) if self.clear_cache: await Sessions.cache.clear() if Config.upload_type == B2UploadSettings: await self.b2.authorize() self.background_tasks = await create_scheduler() for to_spawn in TASKS_TO_SPAWN: await self.background_tasks.spawn(to_spawn()) await cache_community_types(self.community_types) await create_product_and_set(self.product_name)
def redis_cache(event_loop): cache = Cache(Cache.REDIS, namespace="test", pool_max_size=1) yield cache for _, pool in RedisBackend.pools.items(): pool.close() event_loop.run_until_complete(pool.wait_closed())
async def test_exception_in_cache_backend( aiohttp_client: TestClientFixture) -> None: """check if application still works if cache backend is misconfigured or cache backend doesn't work""" handler_hits = 0 @cached async def handler(_: web.Request) -> web.Response: nonlocal handler_hits handler_hits += 1 return web.Response(body=b"Hello world") app = web.Application() app.router.add_route('GET', '/', handler) cache = Cache( Cache.REDIS, endpoint="such.hostname.must.not.exist", serializer=PickleSerializer(), namespace="main", ttl=60, ) register_cache(app, cache) client = await aiohttp_client(app) hits = 10 for i in range(hits): resp = await client.get("/") assert await resp.read() == b"Hello world" assert resp.status == 200 assert handler_hits == hits
async def clear_cache(name: str, background_tasks: BackgroundTasks) -> None: cache = Cache(Cache.REDIS, endpoint=app_settings.CACHE_HOST, port=app_settings.CACHE_PORT) if name == "all": background_tasks.add_task(cache.delete, "/*") else: background_tasks.add_task(cache.delete, f"/{name}*")
def test_new(self, cache_type): kwargs = {"a": 1, "b": 2} cache_class = Cache.get_scheme_class(cache_type) with patch("aiocache.{}.__init__".format(cache_class.__name__)) as init: cache = Cache(cache_class, **kwargs) assert isinstance(cache, cache_class) init.assert_called_once_with(**kwargs)
def initAppObjects(app): app["cache"] = Cache(Cache.MEMORY) app["resolver"] = aiodns.DNSResolver(app.loop) app["websockets"] = set() app["subscriptions"] = WEBSOCKET_SUBSCRIPTIONS app["publishTypes"] = WEBSOCKET_PUBLISH_TYPES app["threads"] = {} app["clientThreads"] = {} app["threadStops"] = {} app["clientThreadStops"] = {}
def test_alias(): loop = asyncio.get_event_loop() loop.run_until_complete(default_cache()) loop.run_until_complete(alt_cache()) cache = Cache(Cache.REDIS) loop.run_until_complete(cache.delete("key")) loop.run_until_complete(cache.close()) loop.run_until_complete(caches.get('default').close())
def test_cached(): cache = Cache(Cache.REDIS, endpoint="127.0.0.1", port=6379, namespace="main") loop = asyncio.get_event_loop() loop.run_until_complete(cached_call()) assert loop.run_until_complete(cache.exists("key")) is True loop.run_until_complete(cache.delete("key")) loop.run_until_complete(cache.close())
async def redis_prepare(): if Config.IN_MEMORY_CACHE: cache = Cache() else: # noinspection PyBroadException try: sentinel = await aioredis.create_sentinel( sentinels=[f'redis://{Config.REDIS_HOST}:{Config.REDIS_PORT}'], password=Config.REDIS_PASSWORD) cache = RedisCache(sentinel=sentinel, master=Config.REDIS_USER) except: cache = Cache() caches._caches['redis'] = cache # noinspection PyGlobalUndefined global _get_user _get_user = cached(alias='redis', ttl=60, key_builder=lambda f, request, email: f'user_{email}', timeout=0.5)(_get_user)
async def init_cache(app: web.Application) -> None: if app['config'].redis_host: app['cache'] = Cache( Cache.REDIS, endpoint=app['config'].redis_host, port=app['config'].redis_port, namespace='jaundice', serializer=JsonSerializer(), ) else: app['cache'] = None
async def test_cache_middleware(aiohttp_client: TestClientFixture) -> None: """ test if cache middleware works and doesn't prevent other middlewares from execution """ handler_hits = 0 before_cache_middleware_hits = 0 after_cache_middleware_hits = 0 @web.middleware async def before_cache_middleware(request: web.Request, handler: Any) -> web.Response: nonlocal before_cache_middleware_hits before_cache_middleware_hits += 1 return await handler(request) @web.middleware async def after_cache_middleware(request: web.Request, handler: Any) -> web.Response: nonlocal after_cache_middleware_hits after_cache_middleware_hits += 1 return await handler(request) @cached async def handler(_: web.Request) -> web.Response: nonlocal handler_hits handler_hits += 1 return web.Response(body=b"Hello world") app = web.Application(middlewares=[before_cache_middleware]) app.router.add_route('GET', '/', handler) cache = Cache( Cache.MEMORY, serializer=PickleSerializer(), namespace="0", ttl=60, ) register_cache(app, cache) # it's artificial case app.middlewares.append(after_cache_middleware) client = await aiohttp_client(app) hits = 10 for i in range(hits): resp: aiohttp.ClientResponse = await client.get("/") assert await resp.read() == b"Hello world" assert resp.status == 200 assert handler_hits == 1 assert after_cache_middleware_hits == 1 assert before_cache_middleware_hits == hits
async def init(loop): conf = load_config(PROJ_ROOT / 'config' / 'config.yml') app = web.Application(loop=loop) app.router.add_route('GET', "/api/v2/{item}", method) app.router.add_route('GET', "/api/{item}", method) cache = Cache(plugins=[HitMissRatioPlugin(), TimingPlugin()]) print(conf['allowed_items']) if 'host' in conf: host = conf['host'] else: host = '127.0.0.1' if 'port' in conf: port = conf['port'] else: port = '443' if 'access_log_format' in conf: access_log_format = conf['access_log_format'] else: access_log_format = '%a %t "%r" %s %b "%{Referer}i" "%{User-Agent}i"' if 'scheme' in conf: if conf['scheme'] == 'https': if 'sslcertchain' and 'sslkey' in conf: ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ssl_context.load_verify_locations(conf['sslcertchain'], conf['sslkey']) ssl_context.load_cert_chain(conf['sslcertchain'], conf['sslkey']) else: raise NameError( 'sslcertchain / sslkey missing in the configuration') else: ssl_context = None else: ssl_context = None app['config'] = conf user, password, realm = conf['authentication']['user'], conf[ 'authentication']['password'], conf['authentication']['realm'] await setup(app, AllowedHosts(conf['allowed_hosts']), BasicAuth(user, password, realm)) app['cache'] = cache return app, cache, host, port, access_log_format, ssl_context
async def clear_cache(name: str, background_tasks: BackgroundTasks) -> None: cache = Cache(Cache.REDIS, endpoint=app_settings.CACHE_HOST, port=app_settings.CACHE_PORT) # TODO: this try/except wrap temporarily fixes an issue in the aiocache library, resulting in an TypeError if # there are no keys in Redis with the given namespace. This is solved on the main branch in aoicache but is not # yet released (23 June 2022). try: if name == "all": await cache.clear(namespace="orchestrator") else: await cache.clear(namespace=f"orchestrator:{name}") except TypeError: pass
async def get_with_client(url, access_token=None, cache=None, client=None): """Fetches the prismic api JSON. Returns :class:`~Api` object. :param url: URL to the api of the repository (mandatory). :param access_token: The access token (optional). :param cache: The cache object. Optional, will default to a in-memory cache if None is passed. :param client: The httpx client. If not passed, one client will be created for each subsequent http request. """ if cache is None: cache = Cache(Cache.MEMORY) return Api( await get_json(url, access_token=access_token, cache=cache, ttl=5, client=client), access_token, cache, client )
def __init__( self, app: "Application", twitch_bearer_token: str, logger: "Logger" = None, *args, **kwargs, ): self.app = app self.config = config = app["config"] intents = discord.Intents.default() intents.members = True super().__init__( owner_id=self.config["bot"].owner_id, command_prefix=determine_prefix, status=discord.Status.idle, intents=intents, *args, **kwargs, ) self.db = app["db"] self.logger = logger self.description = config["bot"].description self.default_prefix = config["bot"].default_prefix self.session = aiohttp.ClientSession() self.pool = concurrent.futures.ThreadPoolExecutor() self.launch_time = None self.twitch_client = twitch.Client(config["twitch"].client_id, twitch_bearer_token, self.session) self.dbl_client = (dbl.DBLClient( self, config["bot"].dbl_token, session=self.session, autopost=True) if config["bot"].dbl_token else None) self.service_guild: Optional[discord.Guild] = None self.errors_channel: Optional[discord.TextChannel] = None self.reports_channel: Optional[discord.TextChannel] = None self.log_channel: Optional[discord.TextChannel] = None cors = self.app["cors"] resource = cors.add( self.app.router.add_resource(r"/wh/twitch/{topic}/{id}")) cors.add(resource.add_route("GET", self.handler_get)) cors.add(resource.add_route("POST", self.handler_post)) self.cache = Cache()
async def main(): PORT = config["server"]["port"] DEBUG_MODE = config["debug_mode"] # Create the global connection pool. async with aiomysql.create_pool( host=config["db"]["host"], port=config["db"]["port"], user=config["db"]["user"], password=str(config["db"]["pwd"]), db=config["db"]["database"], ) as db: cache = Cache(Cache.MEMCACHED, endpoint="127.0.0.1", port=11211, namespace="main") # await maybe_create_tables(db) app = Application(db, cache) if DEBUG_MODE: # 单进程启动 app.listen(PORT) else: # 多进程启动 server = HTTPServer(app) server.ssl_options = { "certfile": config["server"]["ssl_key"], "keyfile": config["server"]["ssl_cert"] } # 在Linux系统bind方法不起作用,需要使用listen;在macOS系统listen方法不起作用,需要使用bind if sys.platform == 'linux': server.listen(PORT) else: server.bind(PORT) server.start(num_processes=config["server"]["num_processes"]) # the server will simply run until interrupted # with Ctrl-C, but if you want to shut down more gracefully, # call shutdown_event.set(). print(f"tornado running on port {PORT}") shutdown_event = tornado.locks.Event() await shutdown_event.wait()
async def test_cache(self, mocked_fetch, morpher): mocked_fetch.side_effect = fetch_return_inosmi_html results = [] cache = Cache(Cache.MEMORY) await process_article( session=None, morph=morpher, charged_words=['бодрость'], url='http://localhost', results=results, cache=cache, ) assert mocked_fetch.call_count == 1 await process_article( session=None, morph=morpher, charged_words=['бодрость'], url='http://localhost', results=results, cache=cache, ) assert mocked_fetch.call_count == 1 # second call returned from cache
async def cache(): conf = load_config(PROJ_ROOT / 'config' / 'config-gunicorn.yml') logging.basicConfig(level=logging.DEBUG) app = web.Application() app.router.add_route('GET', "/api/v2/{item}", method) app.router.add_route('GET', "/api/v2/{item}/{domain}", method) app.router.add_route('GET', "/api/{item}", method) app.router.add_route('GET', "/api/{item}/{domain}", method) memcached_host = conf['cache']['memcached_host'] memcached_port = conf['cache']['memcached_port'] #cache = Cache(plugins=[HitMissRatioPlugin(), TimingPlugin()]) lookup_type = {} cache = Cache(Cache.MEMCACHED, endpoint=memcached_host, port=memcached_port, serializer=JsonSerializer(), plugins=[HitMissRatioPlugin(), TimingPlugin()]) if 'statsd' in conf: if conf['statsd']['enable']: hostname = socket.gethostname().split('.', 1)[0] c = statsd.StatsClient(conf['statsd']['host'], conf['statsd']['port'], prefix=conf['statsd']['prefix']) t = MetricsTimer(conf['statsd']['interval'], cache_metrics, cache, lookup_type, c, hostname) app['config'] = conf user, password, realm = conf['authentication']['user'], conf[ 'authentication']['password'], conf['authentication']['realm'] await setup(app, AllowedHosts(conf['allowed_hosts']), BasicAuth(user, password, realm)) app['cache'] = cache app['lookup_type'] = lookup_type return app
async def get(url, access_token=None, cache=None, **client_kwargs): """Fetches the prismic api JSON. Generates only one httpx client for the async context. Yields :class:`~Api` object. Usage: >>> import prismic >>> async with prismic.get("http://your-repo.prismic.io/api", "access_token") as api: ... doc = await api.get_by_uid("speculoos-macaron") :param url: URL to the api of the repository (mandatory). :param access_token: The access token (optional). :param cache: The cache object. Optional, will default to a in-memory cache if None is passed. """ if cache is None: cache = Cache(Cache.MEMORY) async with httpx.AsyncClient(**client_kwargs) as client: yield Api( await get_json(url, access_token=access_token, cache=cache, ttl=5, client=client), access_token, cache, client )
DEFAULT_ENCODING = None def dumps(self, value): print("I've received:\n{}".format(value)) compressed = zlib.compress(value.encode()) print("But I'm storing:\n{}".format(compressed)) return compressed def loads(self, value): print("I've retrieved:\n{}".format(value)) decompressed = zlib.decompress(value).decode() print("But I'm returning:\n{}".format(decompressed)) return decompressed cache = Cache(Cache.REDIS, serializer=CompressionSerializer(), namespace="main") async def serializer(): text = ( "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt" "ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation" "ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in" "reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur" "sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit" "anim id est laborum.") await cache.set("key", text) print("-----------------------------------") real_value = await cache.get("key") compressed_value = await cache.raw("get", "main:key") assert len(compressed_value) < len(real_value.encode())
logger = logging.getLogger(__name__) class MyCustomPlugin(BasePlugin): async def pre_set(self, *args, **kwargs): logger.info("I'm the pre_set hook being called with %s %s" % (args, kwargs)) async def post_set(self, *args, **kwargs): logger.info("I'm the post_set hook being called with %s %s" % (args, kwargs)) cache = Cache(plugins=[HitMissRatioPlugin(), TimingPlugin(), MyCustomPlugin()], namespace="main") async def run(): await cache.set("a", "1") await cache.set("b", "2") await cache.set("c", "3") await cache.set("d", "4") possible_keys = ["a", "b", "c", "d", "e", "f"] for t in range(1000): await cache.get(random.choice(possible_keys)) assert cache.hit_miss_ratio["hit_ratio"] > 0.5
from aiocache import Cache from backend import config from backend.db_clients import get_db_client # Initialize cache cache = Cache() # Initialize db client db_client = get_db_client((config.DATABASE_TYPE))
import asyncio import logging from aiocache import Cache from aiocache.lock import RedLock logger = logging.getLogger(__name__) cache = Cache(Cache.REDIS, endpoint='127.0.0.1', port=6379, namespace='main') async def expensive_function(): logger.warning('Expensive is being executed...') await asyncio.sleep(1) return 'result' async def my_view(): async with RedLock(cache, 'key', lease=2): # Wait at most 2 seconds result = await cache.get('key') if result is not None: logger.info('Found the value in the cache hurray!') return result result = await expensive_function() await cache.set('key', result) return result async def concurrent(): await asyncio.gather(my_view(), my_view(), my_view())
async def get_cached_categories(): cache = Cache() result = await cache.get("categories") logger.info("from cached, categories in size = {}".format( 0 if result is None else len(result))) return result
async def get_cached_tags(): cache = Cache() result = await cache.get("tags") logger.info("from cache, tags in size = {}".format( 0 if result is None else len(result))) return result
async def get_cached_products(page): cache = Cache() result = await cache.get("products{}".format(page)) logger.info("from cache, products in page = {0} in size = {1}".format( page, 0 if result is None else len(result))) return result
import asyncio from aiocache import caches, Cache from aiocache.serializers import PickleSerializer cache = Cache(Cache.REDIS, endpoint="127.0.0.1", port=6379, namespace="cache_hh", serializer=PickleSerializer()) cache_time = 86400
from ast import literal_eval from concurrent.futures import ThreadPoolExecutor, as_completed from dotenv import load_dotenv from pypi_tools.logic import parse_xml_file_with_version from concurrent.futures import ThreadPoolExecutor import asyncio from random import choice import asyncio from aiocache import cached, Cache from aiocache.serializers import PickleSerializer load_dotenv() client = bigquery.Client() cache = Cache(Cache.REDIS, endpoint=os.environ.get('REDIS_HOST', '127.0.0.1'), port=6379, namespace="main") async def get_download_stats_for_period_from_today(package_name, days, current_date): """ method returns a sorted by date form yesterday to past dict with downloads numbers """ threads = [] results = [] dates = [int((current_date - timedelta(days=i+1)).isoformat().replace("-", "")) for i in range(days)] cached_data = await asyncio.gather(*[cache.get(f'{package_name}:{date}') for date in dates]) cached_result = {} dates_with_no_cache = [] for num, result in enumerate(cached_data): date_ = dates[num] if result: cached_result[date_] = result else: dates_with_no_cache.append(date_)
from bs4 import BeautifulSoup, Tag from fastapi import FastAPI, WebSocket from fastapi.encoders import jsonable_encoder from fastapi.responses import Response app = FastAPI() queue: Queue = None caches.set_config({ 'default': { 'cache': "aiocache.SimpleMemoryCache", 'serializer': { 'class': "aiocache.serializers.PickleSerializer" } } }) cache = Cache(Cache.MEMORY) Mutations = Union[Literal["PREVIOUS_NODE"], Literal["NEXT_NODE"], Literal["CLICK_NODE"], Literal["PUSH_ROUTE"]] from pydantic import BaseModel class Navigation(BaseModel): mutation: Mutations data: dict @app.post("/navigate") async def post(navigation: Navigation): resp = {
async def reuse_data(): cache = Cache(serializer=JsonSerializer()) # Not ideal to define here data = await cache.get("my_custom_key" ) # Note the key is defined in `cached` decorator return data