def get_connection():
try:
if db is not None:
return db
raise Exception('DB connection not found.')
except Exception as e:
logger.info("Trying reconnect to DB: %s:%s ", context.get('db_host'),
context.get('db_port'))
return connect(context.get('db_host'), context.get('db_port'))
def set(key: Hashable, value: Any) -> None:
"""
Puts the given key value pair into the session storage
:param key: The key for the value to be associated with
:param value: The value to be inserted
"""
# Check for a conflict
if aiotask_context.get(key) is not None:
raise KeyError("This key is occupied by the framework")
else:
aiotask_context.get('_<[storage]>_')[key] = value
def test_gather_context_propagation():
context.set("key", "value")
@asyncio.coroutine
def change_context():
assert context.get("key") == "value"
context.set("key", "what")
context.set("other", "data")
yield from asyncio.gather(change_context())
assert context.get("key") == "what"
assert context.get("other") == "data"
def connect(db_host, db_port):
logger.info("[DB] Establishing connection DB connection in: %s:%s ",
db_host, db_port)
if db_host is None:
db_host = context.get('db_host')
if db_port is None:
db_port = context.get('db_port')
db = (motor.motor_asyncio.AsyncIOMotorClient(db_host,
db_port)).test_database
return db
async def _get_page(cls, options={}):
"""Get current page."""
request = context.get('request')
page = options.get('page') if 'page' in options else int(
request.args.get('page', 1))
return page if page > 0 else 1
def get_log_context() -> Dict:
log_context = context.get(LOG_CONTEXT)
if log_context is None:
log_context = {}
context.set(LOG_CONTEXT, log_context)
return log_context
def get(key: Hashable, default=None) -> Any:
"""
Retrieves an value of the async context or the session storage in this order
:param key: The key to get
:param default: The value to return, if nothing is found
:return:
"""
# First try to find the value in the context
value = aiotask_context.get(key)
# If not found, try to find it in the session storage
if value is None:
value = aiotask_context.get('_<[storage]>_').get(key, default)
return value
async def load_metadata(self, version=None):
if not self.needs_update:
metadata = await super().load_metadata(version)
return metadata
# update and cache
url = self.url(version)
client_session = aiotask_context.get('client_session')
self.log.info('Loading upstream metadata', url=url)
async with client_session.get(url) as r:
if r.status == 200:
data = await r.json()
metadata = Metadata.from_dict(data)
await self.store_metadata(metadata)
self.log.info('Loading metadata from upstream', response=r)
metadata = await super().load_metadata(version)
return metadata
if r.status == 404:
log.error('Metadata not found', project=self, response=r)
raise MetadataNotFound(self, r)
self.log.error('Error while retrieving metadata',
project=self,
response=r)
raise MetadataRetrievingError(self, r)
def dummy2(a, b):
yield from asyncio.sleep(random.uniform(0, 2))
res = context.get("key")
yield from asyncio.sleep(random.uniform(0, 2))
res1 = yield from dummy3()
assert res == res1
return a, b, res
def user():
request = context.get('request')
schema = {
'name': {
'type': 'string',
'required': True
},
'email': {
'type': 'string',
'required': True
},
'password': {
'type': 'string',
'required': True
},
}
if request.method == 'PUT':
schema.update({
'password': {
'type': 'string',
'required': False
},
})
return schema
def __init__(self, row):
super().__init__(name=utils.Sem.names[3])
self.url = row.url
self.site = row.site
self.browser = row.browser
self.click = row.click
self.times_max = row.times_max
self.times_real = row.times_real
self.loop = aiotask_context.get('loop_{}'.format(utils.Sem.names[3]))
asyncio.set_event_loop(self.loop)
super().__init__(name=utils.Sem.names[3])
self.sem = asyncio.Semaphore(self.max, loop=self.loop)
self.todo = set()
self.busy = set()
self.done = {}
self.extracted = {}
self.tasks = set()
self.megagroups = None
self.error = 0
self.patterns = [
'http://t.me/', 'https://t.me/', 'http://www.t.me/',
'https://www.t.me/', 'http://telegram.me/', 'https://telegram.me/',
'http://www.telegram.me/', 'https://www.telegram.me/'
]
async def get_basic_info(cls):
""" 接口的通用参数可以统一放在 Basic-Info 这个 Header 中
本地缓存效果：第一次会请求耗时 5 秒左右，3 秒内再次请求会立即返回结果，3 秒后再请求又会耗时 5 秒
超时日志效果：函数执行超过 4 秒会打印一条超时的错误日志，但是并不会抛异常。
"""
await asyncio.sleep(5)
basic_info = aiotask_context.get("handler").basic_info
return basic_info
async def _get_limit(cls, options={}):
"""Get retrieve limit of registers."""
request = context.get('request')
limit = options.get('limit') if 'limit' in options else int(
request.args.get('limit',
cls.get_model().get_default_limit()))
return limit if limit <= cls.get_model().get_max_limit(
) else cls.get_model().get_max_limit()
async def _get_url(cls, page):
""""Get current URL with page query string."""
request = context.get('request')
url_parts = list(urllib.parse.urlparse(request.url))
query = dict(urllib.parse.parse_qsl(url_parts[4]))
query.update({'page': page})
url_parts[4] = urllib.parse.urlencode(query)
return urllib.parse.urlunparse(url_parts)
async def test_task_context_child():
aiotask_context.set("a", "b")
async def child_task(a):
aiotask_context.set("a", a)
await asyncio.gather(*[child_task(i) for i in range(5)])
a = aiotask_context.get("a")
assert a == "b"
async def get(self, request, *args, **kwargs):
user = aiotask_context.get(settings.TASK_CONTEXT_REQUEST_USER)
assert user is not None
assert user == dict(test_user)
request_user = request.user
assert user == dict(request_user)
service = request.service
assert service.request_service == "test"
return json({"hi": "hello"})
async def get(self, request, *args, **kwargs):
user = aiotask_context.get(settings.TASK_CONTEXT_REQUEST_USER)
assert user is not None
assert user == dict(test_user)
request_user = request.user
assert user == dict(request_user)
service = request.service
assert str(service).startswith("AnonymousService")
return json({"hi": "hello"})
async def handle_answer(self, answers: Iterable[Answer]) -> None:
"""
Handle Answer objects
:param answers: Answer objects to be sent
"""
# Iterate over answers
for answer in answers:
if not isinstance(answer, Answer):
answer = Answer(str(answer))
sent = await answer._send(self)
self.last_sent = answer, sent
_context.get("history").appendleft(Message(sent))
if answer.callback is not None:
if answer.is_query():
self.query_callback[sent['message_id']] = answer.callback
else:
self.callback = answer.callback
async def get(self, request, *args, **kwargs):
context_user = aiotask_context.get(
settings.TASK_CONTEXT_REQUEST_USER)
request_user = request.user
handlers.jwt_decode_handler(request.auth)
token = UserIPService.service_token
assert token is not None
assert dict(request_user) == context_user
return json({"user": dict(request_user)})
async def request_context_middleware(request: web.Request, handler: Callable) -> web.Response:
"""
A Middleware that sets the current request's ID in the thread-local context, this helps in getting
the unique ID of the current request in handlers.
"""
x_request_id_header = "X-Request-ID"
context.set(x_request_id_header, request.headers.get(x_request_id_header, str(uuid.uuid4())))
response = await handler(request)
response.headers[x_request_id_header] = context.get(x_request_id_header)
return response
def context_user() -> dict:
"""
Retrieves the user from the asyncio task.
:return:
"""
default = dict(AnonymousUser)
try:
user = aiotask_context.get(settings.TASK_CONTEXT_REQUEST_USER, default)
except AttributeError:
user = default
return user
def context_correlation_id() -> str:
"""
Retrives the request/correlation id from the asyncio task.
:return:
"""
try:
correlation_id = aiotask_context.get(
settings.TASK_CONTEXT_CORRELATION_ID, "unknown")
except AttributeError:
correlation_id = "not set"
return correlation_id
async def test_propagates_copy_of_context(self, event_loop):
@asyncio.coroutine
def adds_to_context():
context.set('foo', 'bar')
return True
context.set('key', 'value')
task = context.copying_task_factory(event_loop, adds_to_context())
await task
assert task.context == {'key': 'value', 'foo': 'bar'}
assert context.get('foo') is None
async def get(self, request, *args, **kwargs):
user = aiotask_context.get(settings.TASK_CONTEXT_REQUEST_USER)
assert user is not None
request_user = request.user
assert user == dict(request_user)
payload = handlers.jwt_decode_handler(request.auth)
assert "user" not in payload
assert user == dict(request_user)
return json({"user": user})
async def handle(request):
name = request.match_info.get('name')
await first_call()
await second_call()
logger.info("Received new GET /{} call".format(name))
text = "Hello, {}. Your request id is {}.\n".format(
name, context.get("X-Request-ID"))
return web.Response(body=text.encode('utf-8'))
def filter(self, record):
cur_task = asyncio.Task.current_task()
if not asyncio.Task.current_task():
record.request_id = 'bootstrap'
return True
if not hasattr(cur_task, 'context'):
record.request_id = None
return True
record.request_id = context.get('request_id')
return True
async def _stream_and_cache(self):
"""Stream data from upstream and cache them.
The download and caching is done in the background, to prevent
disconnecting clients from stopping it.
"""
client_session = aiotask_context.get('client_session')
self.log.info('Caching upstream', url=self.url, path=self.path)
queue = asyncio.Queue()
fut_finished = asyncio.Future()
cur_task = asyncio.current_task()
async def _stream_queue():
while queue.qsize() or not fut_finished.done():
data = await queue.get()
try:
yield data
finally:
queue.task_done()
async def _enqueue_upstream():
try:
log.debug('Streaming from upstream into file and queue',
file=self.path_preparing,
url=self.url)
async with aiofiles.open(self.path_preparing, 'xb') as f:
async with client_session.get(self.url) as r:
async for data, _ in r.content.iter_chunks():
await f.write(data)
await queue.put(data)
fut_finished.set_result(True)
self.path_preparing.rename(self.path)
self.log.info('Finished download', path=self.path)
except (asyncio.CancelledError, IOError,
Exception) as ex: # noqa: W0703
cur_task.cancel()
# cleanup broken download
self.log.error('Cleaning broken download',
path=self.path_preparing,
error=ex)
try:
self.path_preparing.unlink()
except FileNotFoundError:
pass
# TODO use aiojobs ??? to cancel this future graceully
# GeneratorExit
asyncio.ensure_future(_enqueue_upstream())
async for data in _stream_queue():
yield data
def get_current_request() -> Optional[str]:
"""
A helper function that returns the ID of the current application request from the
thread-local context.
:return: String or None.
"""
request_id = context.get("X-Request-ID", None)
if request_id:
_logger.debug(f"Current request ID is: `{request_id}`.")
return request_id
_logger.warn("Request ID is missing from the context!")
async def get(self, request, *args, **kwargs):
context_user = aiotask_context.get(
settings.TASK_CONTEXT_REQUEST_USER)
request_user = request.user
payload = handlers.jwt_decode_handler(request.auth)
assert context_user is not None
payload.pop("user_id")
assert context_user == dict(request_user)
service = request.service
assert service is not None
assert service == AnonymousRequestService
return json({"user": dict(request_user)})
def extra_fields(self):
if not self._extra_fields:
self._extra_fields = {
"service": settings.get("SERVICE_NAME", None),
"environment": settings.get("ENVIRONMENT", None),
"insanic_version": __version__,
"application_version": settings.get("APPLICATION_VERSION",
None),
}
try:
correlation_id = aiotask_context.get(
settings.TASK_CONTEXT_CORRELATION_ID, default="unknown")
except ValueError:
correlation_id = "not set"
self._extra_fields.update({"correlation_id": correlation_id})
return self._extra_fields
def get_current_request() -> IRequest:
"""
Return the current request by heuristically looking it up from stack
"""
try:
task_context = aiotask_context.get('request')
if task_context is not None:
return task_context
except (ValueError, AttributeError, RuntimeError):
pass
# fallback
frame = inspect.currentframe()
while frame is not None:
request = getattr(frame.f_locals.get('self'), 'request', None)
if request is not None:
return request
elif isinstance(frame.f_locals.get('request'), Request):
return frame.f_locals['request']
frame = frame.f_back
raise RequestNotFound(RequestNotFound.__doc__)
async def test(request):
log.debug('X-Request-ID: %s', context.get('X-Request-ID'))
log.info('Hello from test!')
return response.json({"test": True})
def filter(self, record):
record.request_id = context.get('X-Request-ID')
return True
def __init__(self, request, func=None):
self.request = request
self.original = aiotask_context.get('request')
self.func = func