class BaseManager: cache = SimpleMemoryCache(serializer=JsonSerializer()) @classmethod async def get_by_uri(cls, object_id): activity = await cls.find_one({ "activity.object.id": object_id, "deleted": False }) return activity @classmethod async def delete(cls, obj_id): result = await cls.update_one( { "$or": [{ "activity.object.id": obj_id }, { "activity.id": obj_id }], "deleted": False }, {'$set': { "deleted": True }}) await cls.cache.clear() return result.modified_count @staticmethod def activity_clean(data): return [item["activity"] for item in data] @staticmethod async def get_ordered(request, model, filters, cleaner, coll_id): page = request.args.get("page") if page: total = None page = int(page) else: total = await model.count(filter=filters) page = 1 limit = request.app.config.PAGINATION_LIMIT if total != 0: data = await model.find(filter=filters, sort="activity.published desc", skip=limit * (page - 1), limit=limit) data = data.objects else: data = [] return ordered_collection(coll_id, total, page, cleaner(data)) @classmethod async def timeline_paged(cls, request, uri): filters = { "deleted": False, "activity.type": { '$in': ["Create", "Announce", "Like"] } } if cls.__coll__ == 'inbox': filters.update({ "users.0": { "$ne": "cached" }, "users": { "$size": 1 } }) return await cls.get_ordered(request, cls, filters, cls.activity_clean, uri)
def test_dumps(self): assert (JsonSerializer().dumps({"hi": 1}) == '{"hi": 1}' or # json JsonSerializer().dumps({"hi": 1}) == '{"hi":1}') # ujson
async def reuse_data(): cache = Cache(serializer=JsonSerializer()) # Not ideal to define here data = await cache.get("my_custom_key" ) # Note the key is defined in `cached` decorator return data
async def test_add_get_types(self, cache, obj): cache.serializer = JsonSerializer() assert await cache.add(pytest.KEY, obj) is True assert await cache.get(pytest.KEY) == json.loads(json.dumps(obj))
async with aiohttp.ClientSession(headers=headers) as session: async with session.get('%s' % uri, ssl=sslcontext, allow_redirects=False) as response: json = await response.json() if response.status in RESPONSE_OK: await cache.set(uri, json, ttl=ttl) except aiohttp.ClientConnectorError as e: print("Cannot connect to %s %s" % (uri, e)) pass return json @cached(ttl=60, serializer=JsonSerializer()) async def fgetp1(request): json = {} json['subtotal'] = -1 params = {} for qstring in request.rel_url.query: params[qstring] = request.rel_url.query[qstring] params['per_page'] = 1 path = request.rel_url.path url = '%s://%s:%s' % (request.app['config']['foreman']['scheme'], request.app['config']['foreman']['host'], request.app['config']['foreman']['port'])
def __init__(self, serializer=None, **kwargs): super().__init__(**kwargs) self.serializer = serializer or JsonSerializer()
and their associated metadata. .. note:: See ``beacon_api`` root folder ``__init__.py`` for changing values used here. """ from .. import __apiVersion__, __title__, __version__, __description__, __url__, __alturl__, __handover_beacon__ from .. import __createtime__, __updatetime__, __org_id__, __org_name__, __org_description__ from .. import __org_address__, __org_logoUrl__, __org_welcomeUrl__, __org_info__, __org_contactUrl__ from .. import __sample_queries__, __handover_drs__, __docs_url__, __service_type__, __service_env__ from ..utils.data_query import fetch_dataset_metadata from ..extensions.handover import make_handover from aiocache import cached from aiocache.serializers import JsonSerializer @cached(ttl=60, key="ga4gh_info", serializer=JsonSerializer()) async def ga4gh_info(host): """Construct the `Beacon` app information dict in GA4GH Discovery format. :return beacon_info: A dict that contain information about the ``Beacon`` endpoint. """ beacon_info = { # TO DO implement some fallback mechanism for ID 'id': '.'.join(reversed(host.split('.'))), 'name': __title__, "type": __service_type__, 'description': __description__, "organization": { "name": __org_name__, "url": __org_welcomeUrl__, },
def test_loads_with_none(self): assert JsonSerializer().loads(None) is None
def test_init_fails_if_msgpack_not_installed(self): with mock.patch("aiocache.serializers.serializers.msgpack", None): with pytest.raises(RuntimeError): MsgPackSerializer() assert JsonSerializer( ), "Other serializers should still initialize"
def test_set_types(self, obj): assert JsonSerializer().dumps(obj) == json.dumps(obj)
from aiocache import SimpleMemoryCache from aiocache.serializers import JsonSerializer from aiohttp import web from aiohttp_apispec import docs, response_schema, use_kwargs from marshmallow import ValidationError from {{cookiecutter.project_slug}}.entities.requests import RequestPostDummy from {{cookiecutter.project_slug}}.entities.responses import ResponseGetDummyData, ResponseGetDummy, \ ResponseInternalError from {{cookiecutter.project_slug}}.services.dummy import get_dummy, post_dummy from {{cookiecutter.project_slug}}.log_manager import log_manager LOGGER = log_manager.getLogger(module_name=__name__) cache = SimpleMemoryCache(serializer=JsonSerializer()) class DummyHandler(web.View): @docs(tags=['dummy'], summary='Get dummy', description='''Dummy resource''') @response_schema(ResponseGetDummyData.Schema(), 200, description="Single dummy", required=True) @response_schema(ResponseInternalError.Schema(), 400, description="Error description", required=True) async def get(self): dummy = await get_dummy() try: payload = dummy.make_dump() except ValidationError as err: return web.json_response({"error": err.messages}, status=400) return web.json_response({"data": payload})
from urllib.parse import unquote_plus from aiocache import Cache, cached from aiocache.serializers import JsonSerializer from sanic.log import logger from .models import (HoloBiliDB, NijiBiliDB, NijiTubeChannels, NijiTubeLive, OtherBiliDB, OtherYTChannelsDB, OtherYTDB, TwitcastingChannelsDB, TwitcastingDB, TwitchChannelsDB, TwitchDB) @cached( key="holobili", ttl=60, serializer=JsonSerializer(), ) async def fetch_holobili() -> dict: try: logger.debug("Fetching (HoloLive) database...") data = await HoloBiliDB.find_one() except Exception as e: logger.debug(e) logger.debug("Failed to fetch database, returning...") return {"upcoming": [], "live": []} logger.info("Returning...") return {"live": data["live"], "upcoming": data["upcoming"]} @cached( key="nijibili", ttl=60, serializer=JsonSerializer(), ) async def fetch_nijibili() -> dict:
import asyncio import logging from datetime import datetime from aiohttp import web from aiocache import cached from aiocache.serializers import JsonSerializer @cached(key="function_key", serializer=JsonSerializer()) async def time(): return {"time": datetime.now().isoformat()} async def handle(request): return web.json_response(await time()) # It is also possible to cache the whole route, but for this you will need to # override `cached.get_from_cache` and regenerate the response since aiohttp # forbids reusing responses class CachedOverride(cached): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) async def get_from_cache(self, key): try: value = await self.cache.get(key) if type(value) == web.Response: return web.Response( body=value.body, status=value.status,
def test_dumps_with_none(self): assert JsonSerializer().dumps(None) == 'null'
"""Cache Beacon URLs that were received from Registry's update message.""" LOG.debug('Caching Beacons from Registry\'s update message.') try: cache = SimpleMemoryCache() await cache.set('beacon_urls', beacons) LOG.debug('Cache was set.') except Exception as e: response = 500 LOG.error(f'Couldn\'t set cache: {e}.') return response # Cache Beacon URLs if they're not already cached @cached(ttl=86400, key="beacon_urls", serializer=JsonSerializer()) async def get_services(db_pool): """Return service urls.""" LOG.debug('Fetch service urls.') # Take connection from the database pool async with db_pool.acquire() as connection: services = await db_get_service_urls( connection, service_type='GA4GHRegistry' ) # service urls (in this case registries) to be queried # Query Registries for their known Beacon services, fetch only URLs service_urls = await http_get_service_urls(services, service_type='GA4GHBeacon') # Remove duplicate Beacons
def test_loads_with_null(self): assert JsonSerializer().loads('null') is None
app.config.update(SANIC_CONFIG) app.config["JWT_SECRET_KEY"] = JWT_SECRET_KEY app.config["JWT_ACCESS_TOKEN_EXPIRES"] = timedelta(minutes=30) app.config["JWT_TOKEN_LOCATION"] = "cookies" app.config["JWT_ERROR_MESSAGE_KEY"] = "error" app.config["CORS_AUTOMATIC_OPTIONS"] = True app.config["CORS_SUPPORTS_CREDENTIALS"] = True # Construct an in-memory storage redis_port = 6379 if MODE == "testing": redis_port = 63791 elif MODE == "development": redis_port = 63790 cache = RedisCache( serializer=JsonSerializer(), password=CELERY_BROKER_PASSWORD, pool_min_size=3, port=redis_port, ) # Initialize the DB before doing anything else # to avoid circular importing db.init_app(app) JWTManager(app) CORS(app, origins=CORS_ORIGINS, supports_credentials=True) # logging.getLogger("sanic_cors").level = logging.DEBUG # Register the limiter # from ora_backend.utils.limiter import get_user_id_or_ip_addr
def test_dumps_and_loads(self): obj = {"hi": 1} serializer = JsonSerializer() assert serializer.loads(serializer.dumps(obj)) == obj
async def query_params(request: web.Request) -> dict: """Parse query string params from path.""" LOG.debug("Parse query params from AAI response.") # Response from AAI must have the query params `state` and `code` if "state" in request.query and "code" in request.query: LOG.debug("AAI response contained the correct params.") return {"state": request.query["state"], "code": request.query["code"]} else: LOG.error(f"AAI response is missing mandatory params, received: {request.query}") raise web.HTTPBadRequest(text="AAI response is missing mandatory parameters.") @cached(ttl=3600, key="jwk", serializer=JsonSerializer()) async def get_jwk() -> dict: """Get a key to decode access token with.""" LOG.debug("Retrieving JWK.") try: async with aiohttp.ClientSession() as session: async with session.get(CONFIG.aai["jwk_server"]) as r: # This can be a single key or a list of JWK return await r.json() except Exception as e: # pragma: no cover # cache is preventing the testing of this block. this block has been tested manually LOG.error(f"Could not retrieve JWK: {e}") raise web.HTTPInternalServerError(text="Could not retrieve public key.")
html = get_html_by_requests(url=url, headers=headers) if html: soup = BeautifulSoup(html, 'html5lib') selector = RULES[netloc].chapter_selector if selector.get('id', None): content = soup.find_all(id=selector['id']) elif selector.get('class', None): content = soup.find_all(class_=selector['class']) else: content = soup.find_all(selector.get('tag')) # 防止章节被display:none return str(content).replace('style', '') if content else None return None @cached(ttl=10800, key_from_attr='search_ranking', serializer=JsonSerializer(), namespace="ranking") async def cache_novel_search_ranking(): motor_db = MotorBase().get_db() keyword_cursor = motor_db.search_records.find( {'count': {'$gte': 50}}, {'keyword': 1, 'count': 1, '_id': 0} ).sort('count', -1).limit(35) result = [] index = 1 async for document in keyword_cursor: result.append({'keyword': document['keyword'], 'count': document['count'], 'index': index}) index += 1 return result @cached(ttl=3600, key_from_attr='search_ranking', serializer=JsonSerializer(), namespace="ranking")
def test_set_types(self, obj): serializer = JsonSerializer() assert serializer.loads(serializer.dumps(obj)) == obj
/: Does a 3 seconds sleep. Only the first time because its using the `cached` decorator /reuse: Returns the data stored in "main" endpoint """ import asyncio from sanic import Sanic from sanic.response import json from sanic.log import log from aiocache import cached, Cache from aiocache.serializers import JsonSerializer app = Sanic(__name__) @cached(key="my_custom_key", serializer=JsonSerializer()) async def expensive_call(): log.info("Expensive has been called") await asyncio.sleep(3) return {"test": True} async def reuse_data(): cache = Cache(serializer=JsonSerializer()) # Not ideal to define here data = await cache.get("my_custom_key" ) # Note the key is defined in `cached` decorator return data @app.route("/") async def main(request):
def test_init(self): serializer = JsonSerializer() assert isinstance(serializer, BaseSerializer) assert serializer.DEFAULT_ENCODING == 'utf-8' assert serializer.encoding == 'utf-8'
def setup_cache(app: web.Application) -> None: cache = Cache.from_url(app['config']['cache_url']) cache.serializer = JsonSerializer() app['cache'] = cache
async def view_search_result(request): cache = SimpleMemoryCache(serializer=JsonSerializer()) response_dict = await cache.get("response_dict") if not response_dict: response_dict = {} return response_dict
async def test_multi_set_multi_get_types(self, cache, obj): cache.serializer = JsonSerializer() assert await cache.multi_set([(pytest.KEY, obj)]) is True assert await cache.multi_get([pytest.KEY]) == [json.loads(json.dumps(obj))]
from .models import ( HoloBiliDB, NijiBiliDB, OtherBiliDB, OtherYTChannelsDB, OtherYTDB, TwitcastingDB, TwitchDB, ) @cached( key="holobili", ttl=60, serializer=JsonSerializer(), ) async def fetch_holobili() -> dict: try: logger.debug("Fetching (HoloLive) database...") data = await HoloBiliDB.find_one() except Exception as e: logger.debug(e) logger.debug("Failed to fetch database, returning...") return {"upcoming": [], "live": []} logger.info("Returning...") return {"live": data["live"], "upcoming": data["upcoming"]} @cached( key="nijibili",