class BaseManager:
cache = SimpleMemoryCache(serializer=JsonSerializer())
@classmethod
async def get_by_uri(cls, object_id):
activity = await cls.find_one({
"activity.object.id": object_id,
"deleted": False
})
return activity
@classmethod
async def delete(cls, obj_id):
result = await cls.update_one(
{
"$or": [{
"activity.object.id": obj_id
}, {
"activity.id": obj_id
}],
"deleted": False
}, {'$set': {
"deleted": True
}})
await cls.cache.clear()
return result.modified_count
@staticmethod
def activity_clean(data):
return [item["activity"] for item in data]
@staticmethod
async def get_ordered(request, model, filters, cleaner, coll_id):
page = request.args.get("page")
if page:
total = None
page = int(page)
else:
total = await model.count(filter=filters)
page = 1
limit = request.app.config.PAGINATION_LIMIT
if total != 0:
data = await model.find(filter=filters,
sort="activity.published desc",
skip=limit * (page - 1),
limit=limit)
data = data.objects
else:
data = []
return ordered_collection(coll_id, total, page, cleaner(data))
@classmethod
async def timeline_paged(cls, request, uri):
filters = {
"deleted": False,
"activity.type": {
'$in': ["Create", "Announce", "Like"]
}
}
if cls.__coll__ == 'inbox':
filters.update({
"users.0": {
"$ne": "cached"
},
"users": {
"$size": 1
}
})
return await cls.get_ordered(request, cls, filters, cls.activity_clean,
uri)
def test_dumps(self):
assert (JsonSerializer().dumps({"hi": 1}) == '{"hi": 1}' or # json
JsonSerializer().dumps({"hi": 1}) == '{"hi":1}') # ujson
async def reuse_data():
cache = Cache(serializer=JsonSerializer()) # Not ideal to define here
data = await cache.get("my_custom_key"
) # Note the key is defined in `cached` decorator
return data
async def test_add_get_types(self, cache, obj):
cache.serializer = JsonSerializer()
assert await cache.add(pytest.KEY, obj) is True
assert await cache.get(pytest.KEY) == json.loads(json.dumps(obj))
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get('%s' % uri,
ssl=sslcontext,
allow_redirects=False) as response:
json = await response.json()
if response.status in RESPONSE_OK:
await cache.set(uri, json, ttl=ttl)
except aiohttp.ClientConnectorError as e:
print("Cannot connect to %s %s" % (uri, e))
pass
return json
@cached(ttl=60, serializer=JsonSerializer())
async def fgetp1(request):
json = {}
json['subtotal'] = -1
params = {}
for qstring in request.rel_url.query:
params[qstring] = request.rel_url.query[qstring]
params['per_page'] = 1
path = request.rel_url.path
url = '%s://%s:%s' % (request.app['config']['foreman']['scheme'],
request.app['config']['foreman']['host'],
request.app['config']['foreman']['port'])
def __init__(self, serializer=None, **kwargs):
super().__init__(**kwargs)
self.serializer = serializer or JsonSerializer()
and their associated metadata.
.. note:: See ``beacon_api`` root folder ``__init__.py`` for changing values used here.
"""
from .. import __apiVersion__, __title__, __version__, __description__, __url__, __alturl__, __handover_beacon__
from .. import __createtime__, __updatetime__, __org_id__, __org_name__, __org_description__
from .. import __org_address__, __org_logoUrl__, __org_welcomeUrl__, __org_info__, __org_contactUrl__
from .. import __sample_queries__, __handover_drs__, __docs_url__, __service_type__, __service_env__
from ..utils.data_query import fetch_dataset_metadata
from ..extensions.handover import make_handover
from aiocache import cached
from aiocache.serializers import JsonSerializer
@cached(ttl=60, key="ga4gh_info", serializer=JsonSerializer())
async def ga4gh_info(host):
"""Construct the `Beacon` app information dict in GA4GH Discovery format.
:return beacon_info: A dict that contain information about the ``Beacon`` endpoint.
"""
beacon_info = {
# TO DO implement some fallback mechanism for ID
'id': '.'.join(reversed(host.split('.'))),
'name': __title__,
"type": __service_type__,
'description': __description__,
"organization": {
"name": __org_name__,
"url": __org_welcomeUrl__,
},
def test_loads_with_none(self):
assert JsonSerializer().loads(None) is None
def test_init_fails_if_msgpack_not_installed(self):
with mock.patch("aiocache.serializers.serializers.msgpack", None):
with pytest.raises(RuntimeError):
MsgPackSerializer()
assert JsonSerializer(
), "Other serializers should still initialize"
def test_set_types(self, obj):
assert JsonSerializer().dumps(obj) == json.dumps(obj)
from aiocache import SimpleMemoryCache
from aiocache.serializers import JsonSerializer
from aiohttp import web
from aiohttp_apispec import docs, response_schema, use_kwargs
from marshmallow import ValidationError
from {{cookiecutter.project_slug}}.entities.requests import RequestPostDummy
from {{cookiecutter.project_slug}}.entities.responses import ResponseGetDummyData, ResponseGetDummy, \
ResponseInternalError
from {{cookiecutter.project_slug}}.services.dummy import get_dummy, post_dummy
from {{cookiecutter.project_slug}}.log_manager import log_manager
LOGGER = log_manager.getLogger(module_name=__name__)
cache = SimpleMemoryCache(serializer=JsonSerializer())
class DummyHandler(web.View):
@docs(tags=['dummy'],
summary='Get dummy',
description='''Dummy resource''')
@response_schema(ResponseGetDummyData.Schema(), 200, description="Single dummy", required=True)
@response_schema(ResponseInternalError.Schema(), 400, description="Error description", required=True)
async def get(self):
dummy = await get_dummy()
try:
payload = dummy.make_dump()
except ValidationError as err:
return web.json_response({"error": err.messages}, status=400)
return web.json_response({"data": payload})
from urllib.parse import unquote_plus
from aiocache import Cache, cached
from aiocache.serializers import JsonSerializer
from sanic.log import logger
from .models import (HoloBiliDB, NijiBiliDB, NijiTubeChannels,
NijiTubeLive, OtherBiliDB, OtherYTChannelsDB, OtherYTDB,
TwitcastingChannelsDB, TwitcastingDB, TwitchChannelsDB,
TwitchDB)
@cached(
key="holobili", ttl=60, serializer=JsonSerializer(),
)
async def fetch_holobili() -> dict:
try:
logger.debug("Fetching (HoloLive) database...")
data = await HoloBiliDB.find_one()
except Exception as e:
logger.debug(e)
logger.debug("Failed to fetch database, returning...")
return {"upcoming": [], "live": []}
logger.info("Returning...")
return {"live": data["live"], "upcoming": data["upcoming"]}
@cached(
key="nijibili", ttl=60, serializer=JsonSerializer(),
)
async def fetch_nijibili() -> dict:
import asyncio
import logging
from datetime import datetime
from aiohttp import web
from aiocache import cached
from aiocache.serializers import JsonSerializer
@cached(key="function_key", serializer=JsonSerializer())
async def time():
return {"time": datetime.now().isoformat()}
async def handle(request):
return web.json_response(await time())
# It is also possible to cache the whole route, but for this you will need to
# override `cached.get_from_cache` and regenerate the response since aiohttp
# forbids reusing responses
class CachedOverride(cached):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
async def get_from_cache(self, key):
try:
value = await self.cache.get(key)
if type(value) == web.Response:
return web.Response(
body=value.body,
status=value.status,
def test_dumps_with_none(self):
assert JsonSerializer().dumps(None) == 'null'
"""Cache Beacon URLs that were received from Registry's update message."""
LOG.debug('Caching Beacons from Registry\'s update message.')
try:
cache = SimpleMemoryCache()
await cache.set('beacon_urls', beacons)
LOG.debug('Cache was set.')
except Exception as e:
response = 500
LOG.error(f'Couldn\'t set cache: {e}.')
return response
# Cache Beacon URLs if they're not already cached
@cached(ttl=86400, key="beacon_urls", serializer=JsonSerializer())
async def get_services(db_pool):
"""Return service urls."""
LOG.debug('Fetch service urls.')
# Take connection from the database pool
async with db_pool.acquire() as connection:
services = await db_get_service_urls(
connection, service_type='GA4GHRegistry'
) # service urls (in this case registries) to be queried
# Query Registries for their known Beacon services, fetch only URLs
service_urls = await http_get_service_urls(services,
service_type='GA4GHBeacon')
# Remove duplicate Beacons
def test_loads_with_null(self):
assert JsonSerializer().loads('null') is None
app.config.update(SANIC_CONFIG)
app.config["JWT_SECRET_KEY"] = JWT_SECRET_KEY
app.config["JWT_ACCESS_TOKEN_EXPIRES"] = timedelta(minutes=30)
app.config["JWT_TOKEN_LOCATION"] = "cookies"
app.config["JWT_ERROR_MESSAGE_KEY"] = "error"
app.config["CORS_AUTOMATIC_OPTIONS"] = True
app.config["CORS_SUPPORTS_CREDENTIALS"] = True
# Construct an in-memory storage
redis_port = 6379
if MODE == "testing":
redis_port = 63791
elif MODE == "development":
redis_port = 63790
cache = RedisCache(
serializer=JsonSerializer(),
password=CELERY_BROKER_PASSWORD,
pool_min_size=3,
port=redis_port,
)
# Initialize the DB before doing anything else
# to avoid circular importing
db.init_app(app)
JWTManager(app)
CORS(app, origins=CORS_ORIGINS, supports_credentials=True)
# logging.getLogger("sanic_cors").level = logging.DEBUG
# Register the limiter
# from ora_backend.utils.limiter import get_user_id_or_ip_addr
def test_dumps_and_loads(self):
obj = {"hi": 1}
serializer = JsonSerializer()
assert serializer.loads(serializer.dumps(obj)) == obj
async def query_params(request: web.Request) -> dict:
"""Parse query string params from path."""
LOG.debug("Parse query params from AAI response.")
# Response from AAI must have the query params `state` and `code`
if "state" in request.query and "code" in request.query:
LOG.debug("AAI response contained the correct params.")
return {"state": request.query["state"], "code": request.query["code"]}
else:
LOG.error(f"AAI response is missing mandatory params, received: {request.query}")
raise web.HTTPBadRequest(text="AAI response is missing mandatory parameters.")
@cached(ttl=3600, key="jwk", serializer=JsonSerializer())
async def get_jwk() -> dict:
"""Get a key to decode access token with."""
LOG.debug("Retrieving JWK.")
try:
async with aiohttp.ClientSession() as session:
async with session.get(CONFIG.aai["jwk_server"]) as r:
# This can be a single key or a list of JWK
return await r.json()
except Exception as e: # pragma: no cover
# cache is preventing the testing of this block. this block has been tested manually
LOG.error(f"Could not retrieve JWK: {e}")
raise web.HTTPInternalServerError(text="Could not retrieve public key.")
html = get_html_by_requests(url=url, headers=headers)
if html:
soup = BeautifulSoup(html, 'html5lib')
selector = RULES[netloc].chapter_selector
if selector.get('id', None):
content = soup.find_all(id=selector['id'])
elif selector.get('class', None):
content = soup.find_all(class_=selector['class'])
else:
content = soup.find_all(selector.get('tag'))
# 防止章节被display:none
return str(content).replace('style', '') if content else None
return None
@cached(ttl=10800, key_from_attr='search_ranking', serializer=JsonSerializer(), namespace="ranking")
async def cache_novel_search_ranking():
motor_db = MotorBase().get_db()
keyword_cursor = motor_db.search_records.find(
{'count': {'$gte': 50}},
{'keyword': 1, 'count': 1, '_id': 0}
).sort('count', -1).limit(35)
result = []
index = 1
async for document in keyword_cursor:
result.append({'keyword': document['keyword'], 'count': document['count'], 'index': index})
index += 1
return result
@cached(ttl=3600, key_from_attr='search_ranking', serializer=JsonSerializer(), namespace="ranking")
def test_set_types(self, obj):
serializer = JsonSerializer()
assert serializer.loads(serializer.dumps(obj)) == obj
/: Does a 3 seconds sleep. Only the first time because its using the `cached` decorator
/reuse: Returns the data stored in "main" endpoint
"""
import asyncio
from sanic import Sanic
from sanic.response import json
from sanic.log import log
from aiocache import cached, Cache
from aiocache.serializers import JsonSerializer
app = Sanic(__name__)
@cached(key="my_custom_key", serializer=JsonSerializer())
async def expensive_call():
log.info("Expensive has been called")
await asyncio.sleep(3)
return {"test": True}
async def reuse_data():
cache = Cache(serializer=JsonSerializer()) # Not ideal to define here
data = await cache.get("my_custom_key"
) # Note the key is defined in `cached` decorator
return data
@app.route("/")
async def main(request):
def test_init(self):
serializer = JsonSerializer()
assert isinstance(serializer, BaseSerializer)
assert serializer.DEFAULT_ENCODING == 'utf-8'
assert serializer.encoding == 'utf-8'
def setup_cache(app: web.Application) -> None:
cache = Cache.from_url(app['config']['cache_url'])
cache.serializer = JsonSerializer()
app['cache'] = cache
async def view_search_result(request):
cache = SimpleMemoryCache(serializer=JsonSerializer())
response_dict = await cache.get("response_dict")
if not response_dict:
response_dict = {}
return response_dict
async def test_multi_set_multi_get_types(self, cache, obj):
cache.serializer = JsonSerializer()
assert await cache.multi_set([(pytest.KEY, obj)]) is True
assert await cache.multi_get([pytest.KEY]) == [json.loads(json.dumps(obj))]
from .models import (
HoloBiliDB,
NijiBiliDB,
OtherBiliDB,
OtherYTChannelsDB,
OtherYTDB,
TwitcastingDB,
TwitchDB,
)
@cached(
key="holobili",
ttl=60,
serializer=JsonSerializer(),
)
async def fetch_holobili() -> dict:
try:
logger.debug("Fetching (HoloLive) database...")
data = await HoloBiliDB.find_one()
except Exception as e:
logger.debug(e)
logger.debug("Failed to fetch database, returning...")
return {"upcoming": [], "live": []}
logger.info("Returning...")
return {"live": data["live"], "upcoming": data["upcoming"]}
@cached(
key="nijibili",