class ContextVarsRuntimeContext(RuntimeContext):
"""An implementation of the RuntimeContext interface which wraps ContextVar under
the hood. This is the prefered implementation for usage with Python 3.5+
"""
_CONTEXT_KEY = "current_context"
def __init__(self) -> None:
self._current_context = ContextVar(self._CONTEXT_KEY,
default=Context())
def attach(self, context: Context) -> object:
"""See `opentelemetry.context.RuntimeContext.attach`."""
return self._current_context.set(context)
def get_current(self) -> Context:
"""See `opentelemetry.context.RuntimeContext.get_current`."""
return self._current_context.get()
def detach(self, token: object) -> None:
"""See `opentelemetry.context.RuntimeContext.detach`."""
self._current_context.reset(token) # type: ignore
class ServiceContextClass:
def __init__(self) -> None:
self.request_id_var = ContextVar("request_id_var",
default=t.cast(
"t.Optional[int]", None))
self.component_name_var: ContextVar[str] = ContextVar("component_name",
default="cli")
@property
def sampled(self) -> t.Optional[int]:
span = trace.get_current_span()
if span is None:
return None
return 1 if span.get_span_context().trace_flags.sampled else 0
@property
def trace_id(self) -> t.Optional[int]:
span = trace.get_current_span()
if span is None:
return None
return span.get_span_context().trace_id
@property
def span_id(self) -> t.Optional[int]:
span = trace.get_current_span()
if span is None:
return None
return span.get_span_context().span_id
@property
def request_id(self) -> t.Optional[int]:
return self.request_id_var.get()
@property
def component_name(self) -> t.Optional[str]:
return self.component_name_var.get()
def trace_stack_push(trace_stack_var: ContextVar, node: Any) -> None:
"""Push an element to the top of a trace stack."""
if (trace_stack := trace_stack_var.get()) is None:
trace_stack = []
trace_stack_var.set(trace_stack)
class Resolver(IResolver):
def __init__(
self,
dependency_storage: DependencyStorage,
) -> None:
self._dependency_storage = dependency_storage
self._condition_collections = ConditionCollections(
self,
dependency_storage, [
AnyCondition, CollectionCondition, UnionCondition,
IteratorCondition, ForwardRefCondition, GenericCondition
],
default_condition=DefaultCondition)
self._context_dependencies = ContextVar(f"_context_storage_{id(self)}")
def _add_scoped_dependency(self, wrapper: DependencyWrapper,
resolved_dependency: Any) -> None:
if wrapper.type_arguments:
name_in_store = (wrapper.type_, wrapper.type_arguments)
else:
name_in_store = wrapper.type_
context_dependencies = self._context_dependencies.get(None)
if context_dependencies is None:
new_context_dependencies = {name_in_store: [resolved_dependency]}
self._context_dependencies.set(new_context_dependencies)
else:
dependencies = context_dependencies.get(name_in_store, None)
if dependencies is None:
context_dependencies[name_in_store] = [resolved_dependency]
else:
dependencies.append(resolved_dependency)
def get_implementation_attr(
self, annotations: Tuple[Tuple[str, Any]]) -> Dict[str, Any]:
"""Get resolved signature attributes"""
callable_object_arguments = {}
for name, annotation in annotations:
if name == "self":
continue
callable_object_arguments[name] = self._condition_collections.find(
annotation)
return callable_object_arguments
def _get_implementation(self, dependency_wrapper: DependencyWrapper):
if dependency_wrapper.annotations is None:
if dependency_wrapper.scope == Scope.CONTEXT:
self._add_scoped_dependency(dependency_wrapper.target,
dependency_wrapper.target)
return dependency_wrapper.target
attr = self.get_implementation_attr(dependency_wrapper.annotations)
resolved_dependency = dependency_wrapper.target(**attr)
if dependency_wrapper.scope == Scope.SINGLETON:
dependency_wrapper.cache = resolved_dependency
elif dependency_wrapper.scope == Scope.CONTEXT:
self._add_scoped_dependency(dependency_wrapper,
resolved_dependency)
return resolved_dependency
def get_resolved_dependencies(self, typing: Any) -> Iterator[Any]:
"""Get attributes from container for typing"""
context_dependencies = self._context_dependencies.get(None)
if context_dependencies is not None:
dependencies = context_dependencies.get(typing, None)
if dependencies is not None:
for dependency in dependencies:
yield dependency
return
wrappers = self._dependency_storage.get_dependencies_by_annotation(
typing)
for wrapper in wrappers:
yield self._check_and_get_implementation(wrapper)
for dependency_wrapper in self._dependency_storage.get_dependencies(
ignore_annotation=typing):
if not _check_annotation(typing, dependency_wrapper.type_):
continue
yield self._check_and_get_implementation(dependency_wrapper)
def _check_and_get_implementation(
self, dependency_wrapper: DependencyWrapper) -> Any:
if dependency_wrapper.cache is not None and dependency_wrapper.scope == Scope.SINGLETON:
return dependency_wrapper.cache
else:
return self._get_implementation(dependency_wrapper)
class Kanata(BaseDispatcher):
"彼方."
always = True # 兼容重构版的 bcc.
signature_list: List[Union[NormalMatch, PatternReceiver]]
stop_exec_if_fail: bool = True
parsed_items: ContextVar[Dict[str, MessageChain]]
allow_quote: bool
skip_one_at_in_quote: bool
content_token: Optional[Token] = None
def __init__(
self,
signature_list: List[Union[NormalMatch, PatternReceiver]],
stop_exec_if_fail: bool = True,
allow_quote: bool = True,
skip_one_at_in_quote: bool = False,
) -> None:
"""该魔法方法用于实例化该参数解析器.
Args:
signature_list (List[Union[NormalMatch, PatternReceiver]]): 匹配标识链
stop_exec_if_fail (bool, optional): 是否在无可用匹配时停止监听器执行. Defaults to True.
allow_quote (bool, optional): 是否允许 Kanata 处理回复消息中的用户输入部分. Defaults to True.
skip_one_at_in_quote (bool, optional): 是否允许 Kanata 在处理回复消息中的用户输入部分时自动删除可能\
由 QQ 客户端添加的 At 和一个包含在单独 Plain 元素中的空格. Defaults to False.
"""
self.signature_list = signature_list
self.stop_exec_if_fail = stop_exec_if_fail
self.parsed_items = ContextVar("kanata_parsed_items")
self.allow_quote = allow_quote
self.skip_one_at_in_quote = skip_one_at_in_quote
@staticmethod
def detect_index(
signature_chain: Tuple[Union[NormalMatch, PatternReceiver]],
message_chain: MessageChain,
) -> Optional[Dict[str, Tuple[MessageIndex, MessageIndex]]]:
merged_chain = merge_signature_chain(signature_chain)
message_chain = message_chain.asMerged()
element_num = len(message_chain.__root__)
end_index: MessageIndex = (
element_num - 1,
len(message_chain.__root__[-1].text) if element_num != 0
and message_chain.__root__[-1].__class__ is Plain else None,
)
reached_message_index: MessageIndex = (0, None)
# [0] => real_index
# [1] => text_index(optional)
start_index: MessageIndex = (0, None)
match_result: Dict[Arguments, Tuple[
MessageIndex, MessageIndex], # start(include) # stop(exclude)
] = {}
signature_iterable = InsertGenerator(enumerate(merged_chain))
latest_index = None
matching_recevier: Optional[Arguments] = None
for signature_index, signature in signature_iterable:
if isinstance(signature, (Arguments, PatternReceiver)):
if matching_recevier: # 已经选中了一个...
if isinstance(signature, Arguments):
if latest_index == signature_index:
matching_recevier.content.extend(signature.content)
continue
else:
raise TypeError(
"a unexpected case: match conflict")
if isinstance(signature, PatternReceiver):
matching_recevier.content.append(signature)
continue
else:
if isinstance(signature, PatternReceiver):
signature = Arguments([signature])
matching_recevier = signature
start_index = reached_message_index
elif isinstance(signature, NormalMatch):
if not matching_recevier:
# 如果不要求匹配参数, 从当前位置(reached_message_index)开始匹配FullMatch.
current_chain = message_chain.subchain(
slice(reached_message_index, None, None))
if not current_chain.__root__: # index 越界
return
if not isinstance(current_chain.__root__[0], Plain):
# 切片后第一个 **不是** Plain.
return
re_match_result = re.match(signature.operator(),
current_chain.__root__[0].text)
if not re_match_result:
# 不匹配的
return
# 推进当前进度.
plain_text_length = len(current_chain.__root__[0].text)
pattern_length = re_match_result.end(
) - re_match_result.start()
if (pattern_length + 1) > plain_text_length: # 推进后可能造成错误
# 不推进 text_index 进度, 转而推进 element_index 进度
reached_message_index = (reached_message_index[0] + 1,
None)
else:
# 推进 element_index 进度至已匹配到的地方后.
reached_message_index = (
reached_message_index[0],
origin_or_zero(reached_message_index[1]) +
re_match_result.start() + pattern_length,
)
else:
# 需要匹配参数(是否贪婪模式查找, 即是否从后向前)
greed = matching_recevier.isGreed
for element_index, element in enumerate(
message_chain.subchain(
slice(reached_message_index, None,
None)).__root__):
if isinstance(element, Plain):
current_text: str = element.text
# 完成贪婪判断
text_find_result_list = list(
re.finditer(signature.operator(),
current_text))
if not text_find_result_list:
continue
text_find_result = text_find_result_list[-int(greed
)]
if not text_find_result:
continue
text_find_index = text_find_result.start()
# 找到了! 这里不仅要推进进度, 还要把当前匹配的参数记录结束位置并清理.
stop_index = (
reached_message_index[0] + element_index +
int(element_index == 0),
origin_or_zero(reached_message_index[1]) +
text_find_index,
)
match_result[matching_recevier] = (
copy.copy(start_index),
stop_index,
)
start_index = (0, None)
matching_recevier = None
pattern_length = (text_find_result.end() -
text_find_result.start())
if (current_text == text_find_result.string[slice(
*text_find_result.span())]):
# 此处是如果推进 text_index 就会被爆破....
# 推进 element_index 而不是 text_index
reached_message_index = (
reached_message_index[0] + element_index +
int(element_index != 0),
None,
)
else:
reached_message_index = (
reached_message_index[0] + element_index,
origin_or_zero(reached_message_index[1]) +
text_find_index + pattern_length,
)
break
else:
# 找遍了都没匹配到.
return
latest_index = signature_index
else:
if matching_recevier: # 到达了终点, 却仍然还要做点事的.
# 计算终点坐标.
text_index = None
latest_element = message_chain.__root__[-1]
if isinstance(latest_element, Plain):
text_index = len(latest_element.text)
stop_index = (len(message_chain.__root__), text_index)
match_result[matching_recevier] = (start_index, stop_index)
else: # 如果不需要继续捕获消息作为参数, 但 Signature 已经无法指示 Message 的样式时, 判定本次匹配非法.
if reached_message_index < end_index:
return
return match_result
@staticmethod
def detect_and_mapping(
signature_chain: Tuple[Union[NormalMatch, PatternReceiver]],
message_chain: MessageChain,
) -> Optional[Dict[Arguments, MessageChain]]:
match_result = Kanata.detect_index(signature_chain, message_chain)
if match_result is not None:
return {
k: message_chain[v[0]:(
v[1][0],
(v[1][1] - (origin_or_zero(v[0][1]) if
(v[1][0] <= v[0][0] <= v[1][0]) else 0)
) if v[1][1] is not None else None,
)]
for k, v in match_result.items()
}
@staticmethod
def allocation(
mapping: Dict[Arguments, MessageChain]
) -> Optional[Dict[str, MessageChain]]:
if mapping is None:
return None
result = {}
for arguemnt_set, message_chain in mapping.items():
length = len(arguemnt_set.content)
for index, receiver in enumerate(arguemnt_set.content):
if receiver.name in result:
raise ConflictItem(
"{0} is defined repeatedly".format(receiver))
if isinstance(receiver, RequireParam):
if not message_chain.__root__:
return
result[receiver.name] = message_chain
elif isinstance(receiver, OptionalParam):
if not message_chain.__root__:
result[receiver.name] = None
else:
result[receiver.name] = message_chain
break # 还没来得及做长度匹配...
return result
@lru_cache(None)
async def catch_argument_names(self) -> List[str]:
return [
i.name for i in self.signature_list
if isinstance(i, PatternReceiver)
]
async def beforeDispatch(self, interface: DispatcherInterface):
message_chain: MessageChain = (await interface.lookup_param(
"__kanata_messagechain__", MessageChain, None)).exclude(Source)
if set([i.__class__ for i in message_chain.__root__
]).intersection(BLOCKING_ELEMENTS):
raise ExecutionStop()
if self.allow_quote and message_chain.has(Quote):
# 自动忽略自 Quote 后第一个 At
# 0: Quote, 1: At, 2: Plain(一个空格, 可能会在以后的 mirai 版本后被其处理, 这里先自动处理这个了.)
message_chain = message_chain[(3, None):]
if self.skip_one_at_in_quote and message_chain.__root__:
if message_chain.__root__[0].__class__ is At:
message_chain = message_chain[(
1, 1):] # 利用 MessageIndex 可以非常快捷的实现特性.
mapping_result = self.detect_and_mapping(self.signature_list,
message_chain)
if mapping_result is not None:
self.content_token = self.parsed_items.set(
self.allocation(mapping_result))
else:
if self.stop_exec_if_fail:
raise ExecutionStop()
async def catch(self, interface: DispatcherInterface):
if not self.content_token:
return
random_id = random.random()
current_item = self.parsed_items.get()
if current_item is not None:
result = current_item.get(interface.name, random_id)
return Force(result) if result is not random_id else None
else:
if self.stop_exec_if_fail:
raise ExecutionStop()
async def afterDispatch(
self,
interface: "DispatcherInterface",
exception: Optional[Exception] = None,
tb: Optional[TracebackType] = None,
):
if self.content_token:
self.parsed_items.reset(self.content_token)
self.catch = Kanata.catch
self.content_token = None
class GinoEngine:
"""
Connects a :class:`~.dialects.base.Pool` and
:class:`~sqlalchemy.engine.interfaces.Dialect` together to provide a source
of database connectivity and behavior.
A :class:`.GinoEngine` object is instantiated publicly using the
:func:`gino.create_engine` function or
:func:`db.set_bind() <gino.api.Gino.set_bind>` method.
.. seealso::
:doc:`/explanation/engine`
"""
connection_cls = GinoConnection
"""Customizes the connection class to use, default is
:class:`.GinoConnection`."""
def __init__(self,
dialect,
pool,
loop,
logging_name=None,
echo=None,
execution_options=None):
self._sa_engine = _SAEngine(
dialect,
logging_name=logging_name,
echo=echo,
execution_options=execution_options,
)
self._dialect = dialect
self._pool = pool
self._loop = loop
self._ctx = ContextVar("gino", default=None)
@property
def dialect(self):
"""
Read-only property for the
:class:`~sqlalchemy.engine.interfaces.Dialect` of this engine.
"""
return self._dialect
@property
def raw_pool(self):
"""
Read-only access to the underlying database connection pool instance.
This depends on the actual dialect in use, :class:`~asyncpg.pool.Pool`
of asyncpg for example.
"""
return self._pool.raw_pool
def acquire(self, *, timeout=None, reuse=False, lazy=False, reusable=True):
"""
Acquire a connection from the pool.
There are two ways using this method - as an asynchronous context
manager::
async with engine.acquire() as conn:
# play with the connection
which will guarantee the connection is returned to the pool when
leaving the ``async with`` block; or as a coroutine::
conn = await engine.acquire()
try:
# play with the connection
finally:
await conn.release()
where the connection should be manually returned to the pool with
:meth:`conn.release() <.GinoConnection.release>`.
Within the same context (usually the same :class:`~asyncio.Task`, see
also :doc:`/how-to/transaction`), a nesting acquire by default re
:param timeout: Block up to ``timeout`` seconds until there is one free
connection in the pool. Default is ``None`` - block forever until
succeeded. This has no effect when ``lazy=True``, and depends on the
actual situation when ``reuse=True``.
:param reuse: Reuse the latest reusable acquired connection (before
it's returned to the pool) in current context if there is one, or
borrow a new one if none present. Default is ``False`` for always
borrow a new one. This is useful when you are in a nested method call
series, wishing to use the same connection without passing it around
as parameters. See also: :doc:`/how-to/transaction`. A reusing
connection is not reusable even if ``reusable=True``. If the reused
connection happened to be a lazy one, then the reusing connection is
lazy too.
:param lazy: Don't acquire the actual underlying connection yet - do it
only when needed. Default is ``False`` for always do it immediately.
This is useful before entering a code block which may or may not make
use of a given connection object. Feeding in a lazy connection will
save the borrow-return job if the connection is never used. If
setting ``reuse=True`` at the same time, then the reused connection -
if any - applies the same laziness. For example, reusing a lazy
connection with ``lazy=False`` will cause the reused connection to
acquire an underlying connection immediately.
:param reusable: Mark this connection as reusable or otherwise. This
has no effect if it is a reusing connection. All reusable connections
are placed in a stack, any reusing acquire operation will always
reuse the top (latest) reusable connection. One reusable connection
may be reused by several reusing connections - they all share one
same underlying connection. Acquiring a connection with
``reusable=False`` and ``reusing=False`` makes it a cleanly isolated
connection which is only referenced once here.
:return: A :class:`.GinoConnection` object.
"""
return _AcquireContext(
functools.partial(self._acquire, timeout, reuse, lazy, reusable))
async def _acquire(self, timeout, reuse, lazy, reusable):
stack = _ContextualStack(self._ctx)
if reuse and stack:
dbapi_conn = _ReusingDBAPIConnection(self._dialect.cursor_cls,
stack.top)
reusable = False
else:
dbapi_conn = _DBAPIConnection(self._dialect.cursor_cls, self._pool)
rv = self.connection_cls(
self._dialect,
_SAConnection(self._sa_engine, dbapi_conn),
stack if reusable else None,
)
dbapi_conn.gino_conn = rv
if not lazy:
await dbapi_conn.acquire(timeout=timeout)
if reusable:
stack.push(dbapi_conn)
return rv
@property
def current_connection(self):
"""
Gets the most recently acquired reusable connection in the context.
``None`` if there is no such connection.
:return: :class:`.GinoConnection`
"""
stack = self._ctx.get()
if stack:
return stack[-1].gino_conn
async def close(self):
"""
Close the engine, by closing the underlying pool.
"""
await self._pool.close()
async def all(self, clause, *multiparams, **params):
"""
Acquires a connection with ``reuse=True`` and runs
:meth:`~.GinoConnection.all` on it. ``reuse=True`` means you can safely
do this without borrowing more than one underlying connection::
async with engine.acquire():
await engine.all('SELECT ...')
The same applies for other query methods.
"""
async with self.acquire(reuse=True) as conn:
return await conn.all(clause, *multiparams, **params)
async def first(self, clause, *multiparams, **params):
"""
Runs :meth:`~.GinoConnection.first`, See :meth:`.all`.
"""
async with self.acquire(reuse=True) as conn:
return await conn.first(clause, *multiparams, **params)
async def one_or_none(self, clause, *multiparams, **params):
"""
Runs :meth:`~.GinoConnection.one_or_none`, See :meth:`.all`.
"""
async with self.acquire(reuse=True) as conn:
return await conn.one_or_none(clause, *multiparams, **params)
async def one(self, clause, *multiparams, **params):
"""
Runs :meth:`~.GinoConnection.one`, See :meth:`.all`.
"""
async with self.acquire(reuse=True) as conn:
return await conn.one(clause, *multiparams, **params)
async def scalar(self, clause, *multiparams, **params):
"""
Runs :meth:`~.GinoConnection.scalar`, See :meth:`.all`.
"""
async with self.acquire(reuse=True) as conn:
return await conn.scalar(clause, *multiparams, **params)
async def status(self, clause, *multiparams, **params):
"""
Runs :meth:`~.GinoConnection.status`. See also :meth:`.all`.
"""
async with self.acquire(reuse=True) as conn:
return await conn.status(clause, *multiparams, **params)
def compile(self, clause, *multiparams, **params):
"""
A shortcut for :meth:`~gino.dialects.base.AsyncDialectMixin.compile` on
the dialect, returns raw SQL string and parameters according to the
rules of the dialect.
"""
return self._dialect.compile(clause, *multiparams, **params)
def transaction(self,
*args,
timeout=None,
reuse=True,
reusable=True,
**kwargs):
"""
Borrows a new connection and starts a transaction with it.
Different to :meth:`.GinoConnection.transaction`, transaction on engine
level supports only managed usage::
async with engine.transaction() as tx:
# play with transaction here
Where the implicitly acquired connection is available as
:attr:`tx.connection <gino.transaction.GinoTransaction.connection>`.
By default, :meth:`.transaction` acquires connection with
``reuse=True`` and ``reusable=True``, that means it by default tries to
create a nested transaction instead of a new transaction on a new
connection. You can change the default behavior by setting these two
arguments.
The other arguments are the same as
:meth:`~.GinoConnection.transaction` on connection.
.. seealso::
:meth:`.GinoEngine.acquire`
:meth:`.GinoConnection.transaction`
:class:`~gino.transaction.GinoTransaction`
:return: A asynchronous context manager that yields a
:class:`~gino.transaction.GinoTransaction`
"""
return _TransactionContext(
self.acquire(timeout=timeout, reuse=reuse, reusable=reusable),
(args, kwargs),
)
def iterate(self, clause, *multiparams, **params):
"""
Creates a server-side cursor in database for large query results.
This requires that there is a reusable connection in the current
context, and an active transaction is present. Then its
:meth:`.GinoConnection.iterate` is executed and returned.
"""
connection = self.current_connection
if connection is None:
raise ValueError("No Connection in context, please provide one")
return connection.iterate(clause, *multiparams, **params)
def update_execution_options(self, **opt):
"""Update the default execution_options dictionary
of this :class:`.GinoEngine`.
.. seealso::
:meth:`sqlalchemy.engine.Engine.update_execution_options`
:meth:`.GinoConnection.execution_options`
"""
self._sa_engine.update_execution_options(**opt)
async def _run_visitor(self, *args, **kwargs):
async with self.acquire(reuse=True) as conn:
await getattr(conn, "_run_visitor")(*args, **kwargs)
def repr(self, color=False):
return self._pool.repr(color)
def __repr__(self):
return self.repr()
def trace_stack_pop(trace_stack_var: ContextVar) -> None:
"""Remove the top element from a trace stack."""
trace_stack = trace_stack_var.get()
trace_stack.pop()
class Backendpy:
"""The Backendpy ASGI handler"""
def __new__(cls, *args, **kwargs):
"""Process Backendpy class instance."""
config = get_config(project_path=cls._get_project_path())
cls._add_project_sys_path(config['environment']['project_path'])
return MiddlewareProcessor(paths=parse_list(config['middlewares']['active'])) \
.run_process_application(application=super().__new__(cls))
def __init__(self):
"""Initialize Backendpy class instance."""
self.config = get_config(project_path=self._get_project_path(), error_logs=True)
self.context = dict()
self._request_context_var = ContextVar('request')
self._hook_runner = HookRunner()
self._router = Router()
self._middleware_processor = MiddlewareProcessor(
paths=parse_list(self.config['middlewares']['active']))
self.errors = base_errors
self._project_apps = self._get_project_apps()
for app_data in self._project_apps:
if app_data['app'].routes:
for i in app_data['app'].routes:
self._router.routes.merge(i)
if app_data['app'].hooks:
for i in app_data['app'].hooks:
self._hook_runner.hooks.merge(i)
if app_data['app'].errors:
for i in app_data['app'].errors:
self.errors.merge(i)
if app_data['app'].template_dirs:
Template.template_dirs[app_data['path']] = \
[Path(app_data['path']).joinpath(p) for p in app_data['app'].template_dirs]
self._lifespan_startup = False
async def __call__(self, scope, receive, send):
"""Receive the requests and return the responses."""
if scope['type'] == 'http':
if not self._lifespan_startup:
try:
await self.execute_event('startup')
except Exception as e:
LOGGER.exception(e)
else:
self._lifespan_startup = True
try:
body: bytes = await self._get_request_body(receive)
except Exception as e:
LOGGER.exception(f'Request data receive error: {e}')
response = Error(1000)
await self._send_response(send, *await response(None))
return
try:
request = Request(app=self, scope=scope, body=body)
except Exception as e:
LOGGER.exception(f'Request instance creation error: {e}')
response = Error(1000)
await self._send_response(send, *await response(None))
return
token = self._request_context_var.set(request)
await self.execute_event('request_start')
await self._send_response(send, *await self._get_response(request))
await self.execute_event('request_end')
self._request_context_var.reset(token)
elif scope['type'] == 'websocket':
# TODO
raise NotImplementedError
elif scope['type'] == 'lifespan':
await self._handle_lifespan(receive, send)
def get_current_request(self):
"""Return the current request object."""
return self._request_context_var.get()
def event(self, name: str) -> callable:
"""Register an event hook with python decorator.
.. seealso:: :func:`~backendpy.hook.Hooks.event`
"""
return self._hook_runner.hooks.event(name)
async def execute_event(self, name: str, args: Optional[Mapping[str, Any]] = None) -> None:
"""Trigger all hooks related to the event.
:param name: The name of an event
:param args: A dictionary-like object containing arguments passed to the hook function.
"""
return await self._hook_runner.trigger(name, args)
async def _handle_lifespan(self, receive, send):
while True:
message = await receive()
if message['type'] == 'lifespan.startup':
try:
await self.execute_event('startup')
except Exception as e:
LOGGER.exception(e)
await send({'type': 'lifespan.startup.failed',
'message': str(e)})
else:
self._lifespan_startup = True
await send({'type': 'lifespan.startup.complete'})
elif message['type'] == 'lifespan.shutdown':
try:
await self.execute_event('shutdown')
except Exception as e:
LOGGER.exception(e)
await send({'type': 'lifespan.shutdown.failed',
'message': str(e)})
else:
await send({'type': 'lifespan.shutdown.complete'})
return
async def _get_response(self, request):
try:
request = await self._middleware_processor.run_process_request(request=request)
except ExceptionResponse as e:
return await e(request)
except Exception as e:
LOGGER.exception(f'Middleware error: {e}')
response = Error(1000)
return await response(request)
try:
handler, data_handler_cls, request.url_vars = \
self._router.match(request.path, request.method, request.scheme)
if not handler:
response = Error(1001)
return await response(request)
except Exception as e:
LOGGER.exception(e)
response = Error(1000)
return await response(request)
try:
handler = await self._middleware_processor.run_process_handler(
request=request,
handler=handler)
except ExceptionResponse as e:
return await e(request)
except Exception as e:
LOGGER.exception(f'Middleware error: {e}')
response = Error(1000)
return await response(request)
try:
data_errors = None
if data_handler_cls:
request.cleaned_data, data_errors = \
await data_handler_cls(request=request).get_cleaned_data()
if data_errors:
response = Error(1002, data=data_errors)
return await response(request)
except Exception as e:
LOGGER.exception(f'Data handler error: {e}')
response = Error(1000)
return await response(request)
try:
response = await handler(request=request)
except ExceptionResponse as e:
return await e(request)
except Exception as e:
LOGGER.exception(f'Handler error: {e}')
response = Error(1000)
return await response(request)
try:
response = await self._middleware_processor.run_process_response(
request=request,
response=response)
except ExceptionResponse as e:
return await e(request)
except Exception as e:
LOGGER.exception(f'Middleware error: {e}')
response = Error(1000)
return await response(request)
return await response(request)
@staticmethod
async def _send_response(send, body, status, headers, stream=False):
await send({
'type': 'http.response.start',
'status': status,
'headers': headers})
if stream:
if hasattr(body, '__aiter__'):
async for chunk in body:
await send({
'type': 'http.response.body',
'body': to_bytes(chunk),
'more_body': True})
else:
for chunk in body:
await send({
'type': 'http.response.body',
'body': to_bytes(chunk),
'more_body': True})
await send({
'type': 'http.response.body',
'body': b''})
else:
await send({
'type': 'http.response.body',
'body': to_bytes(body)})
@staticmethod
async def _get_request_body(receive) -> bytes:
# Todo: Problem for a huge body ?
body = b''
more_body = True
while more_body:
message = await receive()
body += message.get('body', b'')
more_body = message.get('more_body', False)
return body
@staticmethod
async def _get_request_body_generator(receive):
more_body = True
while more_body:
message = await receive()
yield message.get('body', b'')
more_body = message.get('more_body', False)
def _get_project_apps(self):
apps: list[dict] = list()
for package_name in parse_list(self.config['apps']['active']):
try:
module = importlib.import_module(f'{package_name}.main')
app = getattr(module, 'app')
if isinstance(app, App):
apps.append(dict(
package_name=package_name,
path=os.path.dirname(os.path.abspath(module.__file__)),
app=app))
else:
LOGGER.error(f'"{package_name}" app instance error')
except (ImportError, AttributeError):
LOGGER.error(f'"{package_name}" app instance import error')
return apps
@staticmethod
def _get_project_path():
return os.path.dirname(os.path.realpath(inspect.stack()[2].filename))
@staticmethod
def _add_project_sys_path(project_path):
sys.path.insert(0, os.path.dirname(project_path))
def increase_counter(contextvar: ContextVar) -> Generator:
token = contextvar.set(contextvar.get() + 1)
try:
yield
finally:
contextvar.reset(token)
"deck": deck,
"auth_token": request.app.get("auth_token"),
},
)
async def send_to_clients(data, serial=None):
clients = CLIENTS.get()
if client_id := data.get("client_id"):
if (client := clients.get(client_id)) is None:
return
clients = {client_id: client}
elif serial:
watched_serials = WATCHED_SERIALS.get()
clients = {client_id: clients[client_id] for client_id in watched_serials.get(serial, set())}
for client in clients.values():
try:
await client["websocket"].send_json(data)
except Exception:
if logger.level <= logging.DEBUG:
logger.error(
f'[WEB] Cannot send websocket message `{data}` to client "{client["id"]}"', exc_info=True
)
async def handle_to_web_queue(queue, loop, server_task):
watched_serials = WATCHED_SERIALS.get()
while True:
class TimeoutIdleChecker(BaseIdleChecker):
"""
Checks the idleness of a session by the elapsed time since last used.
The usage means processing of any computation requests, such as
query/batch-mode code execution and having active service-port connections.
"""
name: ClassVar[str] = "timeout"
_config_iv = t.Dict({
t.Key('threshold', default="10m"): tx.TimeDuration(),
}).allow_extra('*')
idle_timeout: timedelta
_policy_cache: ContextVar[Dict[AccessKey, Optional[Mapping[str, Any]]]]
async def __ainit__(self) -> None:
await super().__ainit__()
self._policy_cache = ContextVar('_policy_cache')
self._evh_session_started = \
self._event_dispatcher.consume("session_started", None, self._session_started_cb)
self._evh_execution_started = \
self._event_dispatcher.consume("execution_started", None, self._execution_started_cb)
self._evh_execution_finished = \
self._event_dispatcher.consume("execution_finished", None, self._execution_exited_cb)
self._evh_execution_timeout = \
self._event_dispatcher.consume("execution_timeout", None, self._execution_exited_cb)
self._evh_execution_cancelled = \
self._event_dispatcher.consume("execution_cancelled", None, self._execution_exited_cb)
async def aclose(self) -> None:
self._event_dispatcher.unconsume("session_started",
self._evh_session_started)
self._event_dispatcher.unconsume("execution_started",
self._evh_execution_started)
self._event_dispatcher.unconsume("execution_finished",
self._evh_execution_finished)
self._event_dispatcher.unconsume("execution_timeout",
self._evh_execution_timeout)
self._event_dispatcher.unconsume("execution_cancelled",
self._evh_execution_cancelled)
await super().aclose()
async def populate_config(self, raw_config: Mapping[str, Any]) -> None:
config = self._config_iv.check(raw_config)
self.idle_timeout = config['threshold']
log.info(
'TimeoutIdleChecker: default idle_timeout = {0:,} seconds',
self.idle_timeout.total_seconds(),
)
async def update_app_streaming_status(
self,
session_id: SessionId,
status: AppStreamingStatus,
) -> None:
if status == AppStreamingStatus.HAS_ACTIVE_CONNECTIONS:
await self._disable_timeout(session_id)
elif status == AppStreamingStatus.NO_ACTIVE_CONNECTIONS:
await self._update_timeout(session_id)
async def _disable_timeout(self, session_id: SessionId) -> None:
log.debug(f"TimeoutIdleChecker._disable_timeout({session_id})")
await self._redis.set(f"session.{session_id}.last_access",
"0",
exist=self._redis.SET_IF_EXIST)
async def _update_timeout(self, session_id: SessionId) -> None:
log.debug(f"TimeoutIdleChecker._update_timeout({session_id})")
t = await self._redis.time()
await self._redis.set(
f"session.{session_id}.last_access",
f"{t:.06f}",
expire=max(86400,
self.idle_timeout.total_seconds() * 2),
)
async def _session_started_cb(
self,
context: Any,
agent_id: AgentId,
event_name: str,
session_id: SessionId,
creation_id: str,
) -> None:
await self._update_timeout(session_id)
async def _execution_started_cb(
self,
context: Any,
agent_id: AgentId,
event_name: str,
session_id: SessionId,
) -> None:
await self._disable_timeout(session_id)
async def _execution_exited_cb(
self,
context: Any,
agent_id: AgentId,
event_name: str,
session_id: SessionId,
) -> None:
await self._update_timeout(session_id)
async def _do_idle_check(self, context: Any, agent_id: AgentId,
event_name: str) -> None:
cache_token = self._policy_cache.set(dict())
try:
return await super()._do_idle_check(context, agent_id, event_name)
finally:
self._policy_cache.reset(cache_token)
async def check_session(self, session: RowProxy,
dbconn: SAConnection) -> bool:
session_id = session['id']
active_streams = await self._redis.zcount(
f"session.{session_id}.active_app_connections")
if active_streams is not None and active_streams > 0:
return True
t = await self._redis.time()
raw_last_access = await self._redis.get(
f"session.{session_id}.last_access")
if raw_last_access is None or raw_last_access == "0":
return True
last_access = float(raw_last_access)
# serves as the default fallback if keypair resource policy's idle_timeout is "undefined"
idle_timeout = self.idle_timeout.total_seconds()
policy_cache = self._policy_cache.get()
policy = policy_cache.get(session['access_key'], None)
if policy is None:
query = (sa.select([keypair_resource_policies]).select_from(
sa.join(
keypairs,
keypair_resource_policies,
(keypair_resource_policies.c.name
== keypairs.c.resource_policy),
)).where(keypairs.c.access_key == session['access_key']))
result = await dbconn.execute(query)
policy = await result.first()
assert policy is not None
policy_cache[session['access_key']] = policy
# setting idle_timeout:
# - zero/inf means "infinite"
# - negative means "undefined"
if policy['idle_timeout'] >= 0:
idle_timeout = float(policy['idle_timeout'])
if ((idle_timeout <= 0)
or (math.isinf(idle_timeout) and idle_timeout > 0)
or (t - last_access <= idle_timeout)):
return True
return False
hosts = ContextVar('hosts')
hosts.set({})
async def scan(host, port):
href = '{}:{}'.format(host, port)
print('scan start {}'.format(href))
try:
reader, writer = await open_connection(host, port)
writer.write('Test \r\n'.encode('utf-8'))
await writer.drain()
result = await reader.read(1000)
r = hosts.get()
r[href] = result
hosts.set(r)
writer.close()
await writer.wait_closed()
except Exception as e:
print('scan failed: {}'.format(e))
host = 'baidu.com'
loop = get_event_loop()
chunk = 100
for j in range(10):
tasks = [scan(host, i + j * chunk) for i in range(1, chunk + 1)]
loop.run_until_complete(wait(tasks))
loop.close()
r = hosts.get()
print(r.keys())
class CoreInterface:
"""Represent the interface of the ImJoy core."""
# pylint: disable=no-self-use, too-many-instance-attributes, too-many-public-methods
def __init__(self, app, imjoy_api=None, app_controller=None):
"""Set up instance."""
self.event_bus = EventBus()
self.current_user = ContextVar("current_user")
self.current_plugin = ContextVar("current_plugin")
self.current_workspace = ContextVar("current_workspace")
self._all_users: Dict[str, UserInfo] = {} # uid:user_info
self._all_workspaces: Dict[str,
WorkspaceInfo] = {} # wid:workspace_info
self._app = app
self.app_controller = app_controller
self.disconnect_delay = 1
imjoy_api = imjoy_api or {}
self._codecs = {}
self._disconnected_plugins = []
self._imjoy_api = dotdict({
"_rintf": True,
"log": self.info,
"info": self.info,
"error": self.error,
"warning": self.warning,
"critical": self.critical,
"registerService": self.register_service,
"register_service": self.register_service,
"listServices": self.list_services,
"list_services": self.list_services,
"getService": self.get_service,
"get_service": self.get_service,
"utils": {},
"listPlugins": self.list_plugins,
"list_plugins": self.list_plugins,
"getPlugin": self.get_plugin,
"get_plugin": self.get_plugin,
"generateToken": self.generate_token,
"generate_token": self.generate_token,
"create_workspace": self.create_workspace,
"createWorkspace": self.create_workspace,
"get_workspace": self.get_workspace_interface,
"getWorkspace": self.get_workspace_interface,
"list_workspaces": self.list_workspaces,
"listWorkspaces": self.list_workspaces,
"disconnect": self.disconnect,
})
self._imjoy_api.update(imjoy_api)
# Add public workspace
self.register_workspace(
WorkspaceInfo.parse_obj({
"name": "public",
"persistent": True,
"owners": ["root"],
"allow_list": [],
"deny_list": [],
"visibility": "public",
}))
# Create root user
self.root_user = UserInfo(
id="root",
is_anonymous=False,
email=None,
parent=None,
roles=[],
scopes=[],
expires_at=None,
)
# Create root workspace
self.root_workspace = WorkspaceInfo(
name="root",
owners=["root"],
visibility=VisibilityEnum.protected,
persistent=True,
)
self.root_workspace.set_global_event_bus(self.event_bus)
self.register_workspace(self.root_workspace)
self.load_extensions()
def get_all_users(self):
"""Get all the users."""
return list(self._all_users.values())
def get_user_info_from_token(self, token):
"""Get user info from token."""
user_info = parse_user(token)
# Note here we only use the newly created user info object
# if the same user id does not exist
if user_info.id in self._all_users:
user_info = self._all_users[user_info.id]
else:
self._all_users[user_info.id] = user_info
return user_info
async def restore_plugin(self, plugin):
"""Restore the plugin."""
if plugin in self._disconnected_plugins:
logger.info("Plugin connection restored")
self._disconnected_plugins.remove(plugin)
else:
logger.warning("Plugin connection is not in the disconnected list")
async def remove_plugin_delayed(self, plugin):
"""Remove the plugin after a delayed period (if not cancelled)."""
await asyncio.sleep(self.disconnect_delay)
# It means the session has been reconnected
if plugin not in self._disconnected_plugins:
return
await self._terminate_plugin(plugin)
def remove_plugin_temp(self, sid):
"""Remove session temporarily."""
plugin = DynamicPlugin.get_plugin_by_session_id(sid)
if plugin is None:
logger.warning(
"Plugin (sid: %s) does not exist or has already been terminated.",
sid)
return
if plugin not in self._disconnected_plugins:
self._disconnected_plugins.append(plugin)
loop = asyncio.get_running_loop()
loop.create_task(self.remove_plugin_delayed(plugin))
async def disconnect(self, ):
"""Disconnect from the workspace."""
plugin = self.current_plugin.get()
await self._terminate_plugin(plugin)
async def _terminate_plugin(self, plugin):
"""Terminate the plugin."""
await plugin.terminate()
user_info = plugin.user_info
# Remove the user completely if no plugins exists
if len(user_info.get_plugins()) <= 0:
del self._all_users[user_info.id]
logger.info(
"Removing user (%s) completely since the user "
"has no other plugin connected.",
user_info.id,
)
workspace = plugin.workspace
# Remove the user completely if no plugins exists
if len(workspace.get_plugins()) <= 0 and not workspace.persistent:
del self._all_workspaces[workspace.name]
logger.info(
"Removing workspace (%s) completely "
"since there is no other plugin connected.",
workspace.name,
)
def check_permission(self, workspace, user_info):
"""Check user permission for a workspace."""
# pylint: disable=too-many-return-statements
if isinstance(workspace, str):
workspace = self.get_workspace(workspace)
if not workspace:
logger.error("Workspace %s not found", workspace)
return False
# Make exceptions for root user, the children of root and test workspace
if (user_info.id == "root" or user_info.parent == "root"
or workspace.name == "public"):
return True
if workspace.name == user_info.id:
return True
if user_info.parent:
parent = self._all_users.get(user_info.parent)
if not parent:
return False
if not self.check_permission(workspace, parent):
return False
# if the parent has access
# and the workspace is in the scopes
# then we allow the access
if workspace.name in user_info.scopes:
return True
_id = user_info.email or user_info.id
if _id in workspace.owners:
return True
if workspace.visibility == VisibilityEnum.public:
if workspace.deny_list and user_info.email not in workspace.deny_list:
return True
elif workspace.visibility == VisibilityEnum.protected:
if workspace.allow_list and user_info.email in workspace.allow_list:
return True
if "admin" in user_info.roles:
logger.info("Allowing access to %s for admin user %s",
workspace.name, user_info.id)
return True
return False
def get_all_workspace(self):
"""Return all workspaces."""
return list(self._all_workspaces.values())
def is_workspace_registered(self, ws):
"""Return true if workspace is registered."""
if ws in self._all_workspaces.values():
return True
return False
def get_workspace(self, name):
"""Return the workspace."""
if name in self._all_workspaces:
return self._all_workspaces[name]
return None
def register_workspace(self, ws):
"""Register the workspace."""
ws.set_global_event_bus(self.event_bus)
if ws.name in self._all_workspaces:
raise Exception(
f"Another workspace with the same name {ws.name} already exist."
)
self._all_workspaces[ws.name] = ws
self.event_bus.emit("workspace_registered", ws)
def unregister_workspace(self, name):
"""Unregister the workspace."""
if name not in self._all_workspaces:
raise Exception(f"Workspace has not been registered: {name}")
ws = self._all_workspaces[name]
del self._all_workspaces[name]
self.event_bus.emit("workspace_unregistered", ws)
def load_extensions(self):
"""Load imjoy engine extensions."""
# Support imjoy engine extensions
# See how it works:
# https://packaging.python.org/guides/creating-and-discovering-plugins/
for entry_point in pkg_resources.iter_entry_points(
"imjoy_engine_extension"):
self.current_user.set(self.root_user)
self.current_workspace.set(self.root_workspace)
try:
setup_extension = entry_point.load()
setup_extension(self)
except Exception:
logger.exception("Failed to setup extension: %s",
entry_point.name)
raise
def register_router(self, router):
"""Register a router."""
self._app.include_router(router)
def register_interface(self, name, func):
"""Register a interface function."""
assert callable(func)
self._imjoy_api[name] = func
def register_service(self, service: dict):
"""Register a service."""
plugin = self.current_plugin.get()
workspace = self.current_workspace.get()
if "name" not in service or "type" not in service:
raise Exception(
"Service should at least contain `name` and `type`")
# TODO: check if it's already exists
service.config = service.get("config", {})
assert isinstance(service.config,
dict), "service.config must be a dictionary"
service.config["workspace"] = workspace.name
formated_service = ServiceInfo.parse_obj(service)
formated_service.set_provider(plugin)
service_dict = formated_service.dict()
if formated_service.config.require_context:
for key in service_dict:
if callable(service_dict[key]):
def wrap_func(func, *args, **kwargs):
user_info = self.current_user.get()
workspace = self.current_workspace.get()
kwargs["context"] = {
"user_id": user_info.id,
"email": user_info.email,
"is_anonymous": user_info.email,
"workspace": workspace.name,
}
return func(*args, **kwargs)
setattr(formated_service, key,
partial(wrap_func, service_dict[key]))
# service["_rintf"] = True
# Note: service can set its `visibility` to `public` or `protected`
workspace.add_service(formated_service)
return formated_service.get_id()
def unregister_service(self, service_id):
"""Unregister an service."""
workspace = self.current_workspace.get()
plugin = self.current_plugin.get()
services = workspace.get_services_by_plugin(plugin)
not_exists = True
for service in services:
if service.get_id() == service_id:
workspace.remove_service(service)
not_exists = False
if not_exists:
raise KeyError(
f"The service {service_id} is not registered in the current workspace."
)
def list_plugins(self):
"""List all plugins in the workspace."""
workspace = self.current_workspace.get()
return list(workspace.get_plugins())
async def get_plugin(self, name):
"""Return a plugin by its name."""
workspace = self.current_workspace.get()
plugin = workspace.get_plugin_by_name(name)
if plugin:
return await plugin.get_api()
raise Exception(f"Plugin {name} not found")
async def get_service(self, service_id):
"""Return a service."""
if isinstance(service_id, str):
query = {"id": service_id}
else:
query = service_id
if "workspace" in query:
workspace = self.get_workspace(query["workspace"])
if not workspace:
raise Exception(
f"Service not found: {service_id} (workspace unavailable)")
else:
workspace = self.current_workspace.get()
if "id" in query:
service = workspace.get_services().get(query["id"])
if not service:
raise Exception(f"Service not found: {query['id']}")
elif "name" in query:
service = workspace.get_service_by_name(query["name"])
if not service:
raise Exception(f"Service not found: {query['name']}")
else:
raise Exception(
"Please specify the service id or name to get the service")
user_info = self.current_user.get()
if (not self.check_permission(workspace, user_info)
and service.config.visibility != VisibilityEnum.public):
raise Exception(f"Permission denied: {service_id}")
return service.dict()
def list_workspaces(self, ):
"""List the workspaces for the user."""
user_info = self.current_user.get()
ret = []
for workspace in self._all_workspaces.values():
if self.check_permission(workspace, user_info):
ret.append({"name": workspace.name})
return ret
def list_services(self, query: Optional[dict] = None):
"""Return a list of services based on the query."""
# if workspace is not set, then it means current workspace
# if workspace = *, it means search globally
# otherwise, it search the specified workspace
user_info = self.current_user.get()
if query is None:
query = {"workspace": "*"}
ws = query.get("workspace")
if ws:
del query["workspace"]
if ws == "*":
ret = []
for workspace in self.get_all_workspace():
can_access_ws = self.check_permission(workspace, user_info)
for service in workspace.get_services().values():
# To access the service, it should be public or owned by the user
if (not can_access_ws and service.config.visibility !=
VisibilityEnum.public):
continue
match = True
for key in query:
if getattr(service, key) != query[key]:
match = False
if match:
ret.append(service.get_summary())
return ret
if ws is not None:
workspace = self.get_workspace(ws)
else:
workspace = self.current_workspace.get()
ret = []
workspace_services = workspace.get_services()
for service in workspace_services.values():
match = True
for key in query:
if getattr(service, key) != query[key]:
match = False
if match:
ret.append(service.get_summary())
if workspace is None:
raise Exception("Workspace not found: {ws}")
return ret
def info(self, msg):
"""Log a plugin message."""
plugin = self.current_plugin.get()
logger.info("%s: %s", plugin.name, msg)
workspace_logger = plugin.workspace.get_logger()
if workspace_logger:
workspace_logger.info("%s: %s", plugin.name, msg)
def warning(self, msg):
"""Log a plugin message (warning)."""
plugin = self.current_plugin.get()
workspace_logger = plugin.workspace.get_logger()
if workspace_logger:
workspace_logger.warning("%s: %s", plugin.name, msg)
def error(self, msg):
"""Log a plugin error message (error)."""
plugin = self.current_plugin.get()
workspace_logger = plugin.workspace.get_logger()
if workspace_logger:
workspace_logger.error("%s: %s", plugin.name, msg)
def critical(self, msg):
"""Log a plugin error message (critical)."""
plugin = self.current_plugin.get()
workspace_logger = plugin.workspace.get_logger()
if workspace_logger:
workspace_logger.critical("%s: %s", plugin.name, msg)
def generate_token(self, config: Optional[dict] = None):
"""Generate a token for the current workspace."""
workspace = self.current_workspace.get()
user_info = self.current_user.get()
config = config or {}
if "scopes" in config and config["scopes"] != [workspace.name]:
raise Exception(
"Scopes must be empty or contains only the workspace name.")
config["scopes"] = [workspace.name]
token_config = TokenConfig.parse_obj(config)
scopes = token_config.scopes
for scope in scopes:
if not self.check_permission(scope, user_info):
raise PermissionError(
f"User has no permission to scope: {scope}")
token = generate_presigned_token(user_info, token_config)
return token
def create_workspace(self, config: dict):
"""Create a new workspace."""
user_info = self.current_user.get()
config["persistent"] = config.get("persistent") or False
if user_info.is_anonymous and config["persistent"]:
raise Exception(
"Only registered user can create persistent workspace.")
workspace = WorkspaceInfo.parse_obj(config)
workspace.set_global_event_bus(self.event_bus)
if self.get_workspace(workspace.name):
raise Exception(f"Workspace {workspace.name} already exists.")
user_info = self.current_user.get()
# make sure we add the user's email to owners
_id = user_info.email or user_info.id
if _id not in workspace.owners:
workspace.owners.append(_id)
workspace.owners = [o.strip() for o in workspace.owners if o.strip()]
user_info.scopes.append(workspace.name)
self.register_workspace(workspace)
return self.get_workspace_interface(workspace.name)
def _update_workspace(self, name, config: dict):
"""Bind the context to the generated workspace."""
if not name:
raise Exception("Workspace name is not specified.")
if not self.get_workspace(name):
raise Exception(f"Workspace {name} not found")
workspace = self.get_workspace(name)
user_info = self.current_user.get()
if not self.check_permission(workspace, user_info):
raise PermissionError(f"Permission denied for workspace {name}")
if "name" in config:
raise Exception("Changing workspace name is not allowed.")
# make sure all the keys are valid
# TODO: verify the type
for key in config:
if key.startswith("_") or not hasattr(workspace, key):
raise KeyError(f"Invalid key: {key}")
for key in config:
if not key.startswith("_") and hasattr(workspace, key):
setattr(workspace, key, config[key])
# make sure we add the user's email to owners
_id = user_info.email or user_info.id
if _id not in workspace.owners:
workspace.owners.append(_id)
workspace.owners = [o.strip() for o in workspace.owners if o.strip()]
def get_workspace_interface(self, name: str):
"""Bind the context to the generated workspace."""
workspace = self.get_workspace(name)
if not workspace:
raise Exception(f"Workspace {name} not found")
user_info = self.current_user.get()
if not self.check_permission(workspace, user_info):
raise PermissionError(f"Permission denied for workspace {name}")
interface = self.get_interface()
bound_interface = {}
for key in interface:
if callable(interface[key]):
async def wrap_func(func, *args, **kwargs):
try:
workspace_bk = self.current_workspace.get()
except LookupError:
workspace_bk = None
ret = None
try:
self.current_workspace.set(workspace)
ret = func(*args, **kwargs)
if inspect.isawaitable(ret):
ret = await ret
except Exception as exp:
raise exp
finally:
self.current_workspace.set(workspace_bk)
return ret
bound_interface[key] = partial(wrap_func, interface[key])
bound_interface[key].__name__ = key # required for imjoy-rpc
else:
bound_interface[key] = interface[key]
bound_interface["config"] = json.loads(workspace.json())
bound_interface["set"] = partial(self._update_workspace, name)
bound_interface["_rintf"] = True
event_bus = workspace.get_event_bus()
bound_interface["on"] = event_bus.on
bound_interface["off"] = event_bus.off
bound_interface["once"] = event_bus.once
bound_interface["emit"] = event_bus.emit
# Remove disconnect, since the plugin can call disconnect()
# from their own workspace
del bound_interface["disconnect"]
self.event_bus.emit("user_entered_workspace", (user_info, workspace))
return bound_interface
def get_workspace_as_root(self, name="root"):
"""Get a workspace api as root user."""
self.current_user.set(self.root_user)
return dotdict(self.get_workspace_interface(name))
async def get_plugin_as_root(self, name, workspace):
"""Get a plugin api as root user."""
self.current_user.set(self.root_user)
workspace = self.get_workspace(workspace)
if not workspace:
raise Exception(f"Workspace {workspace} does not exist.")
self.current_workspace.set(workspace)
return dotdict(await self.get_plugin(name))
def get_interface(self):
"""Return the interface."""
return self._imjoy_api.copy()
def register_codec(self, config):
"""Register a codec."""
assert "name" in config
assert "encoder" in config or "decoder" in config
if "type" in config:
for codec_type, codec in list(self._codecs.items()):
if codec.type == config["type"] or codec_type == config["name"]:
logger.info("Removing duplicated codec: %s", codec_type)
del self._codecs[codec_type]
self._codecs[config["name"]] = dotdict(config)
def get_codecs(self):
"""Return registered codecs for rpc."""
return self._codecs
def mount_app(self, path, app, name=None, priority=-1):
"""Mount an app to fastapi."""
route = Mount(path, app, name=name)
# remove existing path
routes_remove = [
route for route in self._app.routes if route.path == path
]
for rou in routes_remove:
self._app.routes.remove(rou)
# The default priority is -1 which assumes the last one is websocket
self._app.routes.insert(priority, route)
def umount_app(self, path):
"""Unmount an app to fastapi."""
routes_remove = [
route for route in self._app.routes if route.path == path
]
for route in routes_remove:
self._app.routes.remove(route)
class BaseContainerRegistry(metaclass=ABCMeta):
etcd: AsyncEtcd
registry_name: str
registry_info: Mapping[str, Any]
registry_url: yarl.URL
max_concurrency_per_registry: int
base_hdrs: Dict[str, str]
credentials: Dict[str, str]
ssl_verify: bool
sema: ContextVar[asyncio.Semaphore]
reporter: ContextVar[Optional[ProgressReporter]]
all_updates: ContextVar[Dict[str, str]]
def __init__(
self,
etcd: AsyncEtcd,
registry_name: str,
registry_info: Mapping[str, Any],
*,
max_concurrency_per_registry: int = 4,
ssl_verify: bool = True,
) -> None:
self.etcd = etcd
self.registry_name = registry_name
self.registry_info = registry_info
self.registry_url = registry_info['']
self.max_concurrency_per_registry = max_concurrency_per_registry
self.base_hdrs = {
'Accept': 'application/vnd.docker.distribution.manifest.v2+json',
}
self.credentials = {}
self.ssl_verify = ssl_verify
self.sema = ContextVar('sema')
self.reporter = ContextVar('reporter', default=None)
self.all_updates = ContextVar('all_updates')
async def rescan_single_registry(
self,
reporter: ProgressReporter = None,
) -> None:
self.all_updates.set({})
self.sema.set(asyncio.Semaphore(self.max_concurrency_per_registry))
self.reporter.set(reporter)
username = self.registry_info['username']
if username is not None:
self.credentials['username'] = username
password = self.registry_info['password']
if password is not None:
self.credentials['password'] = password
non_kernel_words = (
'common-',
'commons-',
'base-',
'krunner',
'builder',
'backendai',
'geofront',
)
ssl_ctx = None # default
if not self.registry_info['ssl-verify']:
ssl_ctx = False
connector = aiohttp.TCPConnector(ssl=ssl_ctx)
async with aiohttp.ClientSession(connector=connector) as sess:
async with aiotools.TaskGroup() as tg:
async for image in self.fetch_repositories(sess):
if not any((w in image) for w in
non_kernel_words): # skip non-kernel images
tg.create_task(self._scan_image(sess, image))
all_updates = self.all_updates.get()
if not all_updates:
log.info('No images found in registry {0}', self.registry_url)
else:
for kvlist in chunked(sorted(all_updates.items()), 16):
await self.etcd.put_dict(dict(kvlist))
async def _scan_image(
self,
sess: aiohttp.ClientSession,
image: str,
) -> None:
rqst_args = await registry_login(
sess,
self.registry_url,
self.credentials,
f'repository:{image}:pull',
)
rqst_args['headers'].update(**self.base_hdrs)
tags = []
tag_list_url: Optional[yarl.URL]
tag_list_url = (self.registry_url /
f'v2/{image}/tags/list').with_query({'n': '10'}, )
while tag_list_url is not None:
async with sess.get(tag_list_url, **rqst_args) as resp:
data = json.loads(await resp.read())
if 'tags' in data:
# sometimes there are dangling image names in the hub.
tags.extend(data['tags'])
tag_list_url = None
next_page_link = resp.links.get('next')
if next_page_link:
next_page_url = cast(yarl.URL, next_page_link['url'])
tag_list_url = (self.registry_url.with_path(
next_page_url.path).with_query(next_page_url.query))
if (reporter := self.reporter.get()) is not None:
reporter.total_progress += len(tags)
async with aiotools.TaskGroup() as tg:
for tag in tags:
tg.create_task(self._scan_tag(sess, rqst_args, image, tag))
class PurityClient:
endpoint: URL
api_token: str
api_version: str
auth_token: ContextVar[str]
_session: aiohttp.ClientSession
_auth_token_cvtoken: Token
def __init__(
self,
endpoint: str,
api_token: str,
*,
api_version: str = "1.8",
) -> None:
self.endpoint = URL(endpoint)
self.api_token = api_token
self.api_version = api_version
self.auth_token = ContextVar("auth_token")
self._session = aiohttp.ClientSession()
async def aclose(self) -> None:
await self._session.close()
async def __aenter__(self) -> PurityClient:
async with self._session.post(
self.endpoint / "api" / "login",
headers={"api-token": self.api_token},
ssl=False,
raise_for_status=True,
) as resp:
auth_token = resp.headers["x-auth-token"]
self._auth_token_cvtoken = self.auth_token.set(auth_token)
_ = await resp.json()
return self
async def __aexit__(self, *exc_info) -> None:
self.auth_token.reset(self._auth_token_cvtoken)
# For the concrete API reference, check out:
# https://purity-fb.readthedocs.io/en/latest/
async def get_metadata(self) -> Mapping[str, Any]:
if self.auth_token is None:
raise RuntimeError("The auth token for Purity API is not initialized.")
items = []
pagination_token = ""
while True:
async with self._session.get(
(self.endpoint / "api" / self.api_version / "arrays"),
headers={"x-auth-token": self.auth_token.get()},
params={
"items_returned": 10,
"token": pagination_token,
},
ssl=False,
raise_for_status=True,
) as resp:
data = await resp.json()
for item in data["items"]:
items.append(item)
pagination_token = data["pagination_info"]["continuation_token"]
if pagination_token is None:
break
if not items:
return {}
first = items[0]
return {
"id": first["id"],
"name": first["name"],
"os": first["os"],
"revision": first["revision"],
"version": first["version"],
"blade_count": str(len(items)),
"console_url": str(self.endpoint),
}
async def get_nfs_metric(
self,
fs_name: str,
) -> AsyncGenerator[Mapping[str, Any], None]:
if self.auth_token is None:
raise RuntimeError("The auth token for Purity API is not initialized.")
pagination_token = ""
while True:
async with self._session.get(
(
self.endpoint
/ "api"
/ self.api_version
/ "file-systems"
/ "performance"
),
headers={"x-auth-token": self.auth_token.get()},
params={
"names": fs_name,
"protocol": "NFS",
"items_returned": 10,
"token": pagination_token,
},
ssl=False,
raise_for_status=True,
) as resp:
data = await resp.json()
for item in data["items"]:
yield item
pagination_token = data["pagination_info"]["continuation_token"]
if pagination_token is None:
break
async def get_usage(self, fs_name: str) -> Mapping[str, Any]:
if self.auth_token is None:
raise RuntimeError("The auth token for Purity API is not initialized.")
items = []
pagination_token = ""
while True:
async with self._session.get(
(self.endpoint / "api" / self.api_version / "file-systems"),
headers={"x-auth-token": self.auth_token.get()},
params={
"names": fs_name,
"items_returned": 10,
"token": pagination_token,
},
ssl=False,
raise_for_status=True,
) as resp:
data = await resp.json()
for item in data["items"]:
items.append(item)
pagination_token = data["pagination_info"]["continuation_token"]
if pagination_token is None:
break
if not items:
return {}
first = items[0]
return {
"capacity_bytes": data["total"]["provisioned"],
"used_bytes": first["space"]["total_physical"],
}
class LocalStack(Generic[T]):
"""LocalStack.
Manage state per coroutine (also thread safe).
Most famously used probably in Flask to keep track of the current
request object.
"""
_stack: ContextVar[Optional[List[T]]]
def __init__(self) -> None:
self._stack = ContextVar('_stack')
# XXX mypy bug; when fixed type Generator, should be ContextManager.
@contextmanager
def push(self, obj: T) -> Generator[None, None, None]:
"""Push a new item to the stack."""
self.push_without_automatic_cleanup(obj)
try:
yield
finally:
self.pop()
def push_without_automatic_cleanup(self, obj: T) -> None:
stack = self._stack.get(None)
if stack is None:
stack = []
self._stack.set(stack)
stack.append(obj)
def pop(self) -> Optional[T]:
"""Remove the topmost item from the stack.
Note:
Will return the old value or `None` if the stack was already empty.
"""
stack = self._stack.get(None)
if stack is None:
return None
else:
size = len(stack)
if not size:
self._stack.set(None)
return None
elif size == 1:
item = stack[-1]
self._stack.set(None)
else:
item = stack.pop()
return item
def __len__(self) -> int:
stack = self._stack.get(None)
return len(stack) if stack else 0
@property
def stack(self) -> Sequence[T]:
# read-only version, do not modify
return self._stack.get(None) or []
@property
def top(self) -> Optional[T]:
"""Return the topmost item on the stack.
Does not remove it from the stack.
Note:
If the stack is empty, :const:`None` is returned.
"""
stack = self._stack.get(None)
return stack[-1] if stack else None
class PurityClient:
endpoint: URL
api_token: str
api_version: str
auth_token: ContextVar[str]
_session: aiohttp.ClientSession
_auth_token_cvtoken: Token
def __init__(
self,
endpoint: str,
api_token: str,
*,
api_version: str = '1.8',
) -> None:
self.endpoint = URL(endpoint)
self.api_token = api_token
self.api_version = api_version
self.auth_token = ContextVar('auth_token')
self._session = aiohttp.ClientSession()
async def aclose(self) -> None:
await self._session.close()
async def __aenter__(self) -> PurityClient:
async with self._session.post(
self.endpoint / 'api' / 'login',
headers={'api-token': self.api_token},
ssl=False,
raise_for_status=True,
) as resp:
auth_token = resp.headers['x-auth-token']
self._auth_token_cvtoken = self.auth_token.set(auth_token)
_ = await resp.json()
return self
async def __aexit__(self, *exc_info) -> None:
self.auth_token.reset(self._auth_token_cvtoken)
async def get_nfs_metric(
self, fs_name: str) -> AsyncGenerator[Mapping[str, Any], None]:
if self.auth_token is None:
raise RuntimeError(
'The auth token for Purity API is not initialized.')
pagination_token = ''
while True:
async with self._session.get(
(self.endpoint / 'api' / self.api_version / 'file-systems' /
'performance'),
headers={'x-auth-token': self.auth_token.get()},
params={
'names': fs_name,
'protocol': 'NFS',
'items_returned': 10,
'token': pagination_token,
},
ssl=False,
raise_for_status=True,
) as resp:
data = await resp.json()
for item in data['items']:
yield item
pagination_token = data['pagination_info'][
'continuation_token']
if pagination_token is None:
break
class SentryReporter:
"""SentryReporter designed for sending reports to the Sentry server from
a Tribler Client.
"""
def __init__(self):
self.scrubber = None
self.last_event = None
self.ignored_exceptions = [KeyboardInterrupt, SystemExit]
# more info about how SentryReporter choose a strategy see in
# SentryReporter.get_actual_strategy()
self.global_strategy = SentryStrategy.SEND_ALLOWED_WITH_CONFIRMATION
self.thread_strategy = ContextVar('context_strategy', default=None)
self._sentry_logger_name = 'SentryReporter'
self._logger = logging.getLogger(self._sentry_logger_name)
def init(self,
sentry_url='',
release_version='',
scrubber=None,
strategy=SentryStrategy.SEND_ALLOWED_WITH_CONFIRMATION):
"""Initialization.
This method should be called in each process that uses SentryReporter.
Args:
sentry_url: URL for Sentry server. If it is empty then Sentry's
sending mechanism will not be initialized.
scrubber: a class that will be used for scrubbing sending events.
Only a single method should be implemented in the class:
```
def scrub_event(self, event):
pass
```
release_version: string that represents a release version.
See Also: https://docs.sentry.io/platforms/python/configuration/releases/
strategy: a Sentry strategy for sending events (see class Strategy
for more information)
Returns:
Sentry Guard.
"""
self._logger.debug(f"Init: {sentry_url}")
self.scrubber = scrubber
self.global_strategy = strategy
rv = sentry_sdk.init(
sentry_url,
release=release_version,
# https://docs.sentry.io/platforms/python/configuration/integrations/
integrations=[
LoggingIntegration(
level=logging.
INFO, # Capture info and above as breadcrumbs
event_level=None, # Send no errors as events
),
ThreadingIntegration(propagate_hub=True),
],
before_send=self._before_send,
ignore_errors=[KeyboardInterrupt],
)
ignore_logger(self._sentry_logger_name)
return rv
def ignore_logger(self, logger_name: str):
self._logger.debug(f"Ignore logger: {logger_name}")
ignore_logger(logger_name)
def add_breadcrumb(self, message='', category='', level='info', **kwargs):
"""Adds a breadcrumb for current Sentry client.
It is necessary to specify a message, a category and a level to make this
breadcrumb visible in Sentry server.
Args:
**kwargs: named arguments that will be added to Sentry event as well
"""
crumb = {'message': message, 'category': category, 'level': level}
self._logger.debug(f"Add the breadcrumb: {crumb}")
return sentry_sdk.add_breadcrumb(crumb, **kwargs)
def send_event(self,
event: Dict = None,
post_data: Dict = None,
sys_info: Dict = None,
additional_tags: List[str] = None,
retrieve_error_message_from_stacktrace=False):
"""Send the event to the Sentry server
This method
1. Enable Sentry's sending mechanism.
2. Extend sending event by the information from post_data.
3. Send the event.
4. Disables Sentry's sending mechanism.
Scrubbing the information will be performed in the `_before_send` method.
During the execution of this method, all unhandled exceptions that
will be raised, will be sent to Sentry automatically.
Args:
event: event to send. It should be taken from SentryReporter at
post_data: dictionary made by the feedbackdialog.py
previous stages of executing.
sys_info: dictionary made by the feedbackdialog.py
additional_tags: tags that will be added to the event
Returns:
Event that was sent to Sentry server
"""
self._logger.info(f"Send: {post_data}, {event}")
if event is None:
return event
post_data = post_data or dict()
sys_info = sys_info or dict()
additional_tags = additional_tags or dict()
if CONTEXTS not in event:
event[CONTEXTS] = {}
if TAGS not in event:
event[TAGS] = {}
event[CONTEXTS][REPORTER] = {}
# tags
tags = event[TAGS]
tags['version'] = get_value(post_data, 'version')
tags['machine'] = get_value(post_data, 'machine')
tags['os'] = get_value(post_data, 'os')
tags['platform'] = get_first_item(get_value(sys_info, 'platform'))
tags[f'{PLATFORM_DETAILS}'] = get_first_item(
get_value(sys_info, PLATFORM_DETAILS))
tags.update(additional_tags)
# context
context = event[CONTEXTS]
reporter = context[REPORTER]
version = get_value(post_data, 'version')
context['browser'] = {'version': version, 'name': 'Tribler'}
stacktrace_parts = parse_stacktrace(get_value(post_data, 'stack'))
reporter[STACKTRACE] = next(stacktrace_parts, [])
stacktrace_extra = next(stacktrace_parts, [])
reporter[f'{STACKTRACE}_extra'] = stacktrace_extra
reporter[f'{STACKTRACE}_context'] = next(stacktrace_parts, [])
reporter['comments'] = get_value(post_data, 'comments')
reporter[OS_ENVIRON] = parse_os_environ(get_value(
sys_info, OS_ENVIRON))
delete_item(sys_info, OS_ENVIRON)
reporter['events'] = extract_dict(sys_info, r'^(event|request)')
reporter[SYSINFO] = {
key: sys_info[key]
for key in sys_info if key not in reporter['events']
}
# try to retrieve an error from the stacktrace
if retrieve_error_message_from_stacktrace and stacktrace_extra:
exception_value = stacktrace_extra[-1].split(':', maxsplit=1)
exception_values = event.get(EXCEPTION, {}).get(VALUES, [])
if len(exception_value) == 2:
exception_values.append({
'type': exception_value[0],
'value': exception_value[1]
})
with this_sentry_strategy(self, SentryStrategy.SEND_ALLOWED):
sentry_sdk.capture_event(event)
return event
def get_confirmation(self, exception):
"""Get confirmation on sending exception to the Team.
There are two message boxes, that will be triggered:
1. Message box with the error_text
2. Message box with confirmation about sending this report to the Tribler
team.
Args:
exception: exception to be sent.
"""
# pylint: disable=import-outside-toplevel
try:
from PyQt5.QtWidgets import QApplication, QMessageBox
except ImportError:
self._logger.debug(
"PyQt5 is not available. User confirmation is not possible.")
return False
self._logger.debug(f"Get confirmation: {exception}")
_ = QApplication(sys.argv)
messagebox = QMessageBox(icon=QMessageBox.Critical,
text=f'{exception}.')
messagebox.setWindowTitle("Error")
messagebox.exec()
messagebox = QMessageBox(
icon=QMessageBox.Question,
text='Do you want to send this crash report to the Tribler team? '
'We anonymize all your data, who you are and what you downloaded.',
)
messagebox.setWindowTitle("Error")
messagebox.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
return messagebox.exec() == QMessageBox.Yes
def capture_exception(self, exception):
self._logger.info(f"Capture exception: {exception}")
sentry_sdk.capture_exception(exception)
def event_from_exception(self, exception) -> Dict:
"""This function format the exception by passing it through sentry
Args:
exception: an exception that will be passed to `sentry_sdk.capture_exception(exception)`
Returns:
the event that has been saved in `_before_send` method
"""
self._logger.info(f"Event from exception: {exception}")
if not exception:
return {}
with this_sentry_strategy(self, SentryStrategy.SEND_SUPPRESSED):
sentry_sdk.capture_exception(exception)
return self.last_event
def set_user(self, user_id):
"""Set the user to identify the event on a Sentry server
The algorithm is the following:
1. Calculate hash from `user_id`.
2. Generate fake user, based on the hash.
No real `user_id` will be used in Sentry.
Args:
user_id: Real user id.
Returns:
Generated user (dictionary: {id, username}).
"""
# calculate hash to keep real `user_id` in secret
user_id_hash = md5(user_id).hexdigest()
self._logger.debug(f"Set user: {user_id_hash}")
Faker.seed(user_id_hash)
user_name = Faker().name()
user = {'id': user_id_hash, 'username': user_name}
sentry_sdk.set_user(user)
return user
def get_actual_strategy(self):
"""This method is used to determine actual strategy.
Strategy can be global: self.strategy
and local: self._context_strategy.
Returns: the local strategy if it is defined, the global strategy otherwise
"""
strategy = self.thread_strategy.get()
return strategy if strategy else self.global_strategy
@staticmethod
def get_test_sentry_url():
return os.environ.get('TRIBLER_TEST_SENTRY_URL', None)
@staticmethod
def is_in_test_mode():
return bool(SentryReporter.get_test_sentry_url())
def _before_send(self, event: Optional[Dict],
hint: Optional[Dict]) -> Optional[Dict]:
"""The method that is called before each send. Both allowed and
disallowed.
The algorithm:
1. If sending is allowed, then scrub the event and send.
2. If sending is disallowed, then store the event in
`self.last_event`
Args:
event: event that generated by Sentry
hint: root exception (can be used in some cases)
Returns:
The event, prepared for sending, or `None`, if sending is suppressed.
"""
if not event:
return event
# trying to get context-depending strategy first
strategy = self.get_actual_strategy()
self._logger.info(f"Before send strategy: {strategy}")
exc_info = get_value(hint, 'exc_info')
error_type = get_first_item(exc_info)
if error_type in self.ignored_exceptions:
self._logger.debug(f"Exception is in ignored: {hint}. Skipped.")
return None
if strategy == SentryStrategy.SEND_SUPPRESSED:
self._logger.debug("Suppress sending. Storing the event.")
self.last_event = event
return None
if strategy == SentryStrategy.SEND_ALLOWED_WITH_CONFIRMATION:
self._logger.debug("Request confirmation.")
if not self.get_confirmation(hint):
return None
# clean up the event
self._logger.debug(
f"Clean up the event with scrubber: {self.scrubber}")
if self.scrubber:
event = self.scrubber.scrub_event(event)
return event
def run(coro):
loop = asyncio.get_event_loop()
return loop.run_until_complete(with_session(coro))
@contextmanager
def settings(**values):
old_values = SETTINGS.get()
try:
token = SETTINGS.set(merge(old_values, values))
yield
finally:
SETTINGS.reset(token)
settings.get = lambda param, default=None: SETTINGS.get().get(param, default)
def settings_middleware(what, *, after=None, before=None):
assert not (after and before), "Should use either after or before param"
if callable(what):
what = [what]
middleware = settings.get('middleware', [])
i = _get_index(middleware, after or before)
if after:
i += 1
return settings(middleware=middleware[:i] + what + middleware[i:])
settings.middleware = settings_middleware
class ProxyServer:
def __init__(self, port: int = 8080, block_images: bool = False, cfg=None):
self.connection = ContextVar("connection")
self.block_images = block_images
self.port = port
self._spent_data = {}
if cfg is not None:
if isinstance(cfg, dict):
self._cfg = cfg
else:
raise ValueError(f"Config should be {dict.__name__} object")
for rsc in chain(cfg["limited"], cfg["black-list"]):
self._spent_data[rsc] = 0
self.context_token = None
async def run(self):
"""
Launch async proxy-server at specified host and port.
"""
srv = await asyncio.start_server(self._handle_connection, LOCALHOST,
self.port)
addr = srv.sockets[0].getsockname()
LOGGER.info(START_SERVER_MSG.format(app_address=addr))
async with srv:
await srv.serve_forever()
async def _handle_connection(self, client_reader: StreamReader,
client_writer: StreamWriter) -> None:
"""
Handle every client response.
Called whenever a new connection is established.
"""
try:
raw_request = await client_reader.read(CHUNK_SIZE)
print(raw_request)
await client_writer.drain()
if not raw_request:
return
pr = ProxyRequest(raw_request, self._cfg)
LOGGER.info(f"{pr.method:<{len('CONNECT')}} " f"{pr.abs_url}")
try:
server_reader, server_writer = await asyncio.open_connection(
pr.hostname, pr.port)
except OSError:
LOGGER.info(
CONNECTION_REFUSED_MSG.format(method=pr.method,
url=pr.abs_url))
return
client_endpoint = Endpoint(client_reader, client_writer)
server_endpoint = Endpoint(server_reader, server_writer)
conn = Connection(client_endpoint, server_endpoint, pr,
self.block_images)
self.context_token = self.connection.set(conn)
if self.block_images and pr.is_image_request:
await self.connection.get().reset()
return
if pr.scheme is HTTPScheme.HTTPS:
await self._handle_https()
else:
await self._handle_http()
except Exception as e:
if isinstance(e, ConnectionResetError):
LOGGER.info(CONNECTION_CLOSED_MSG.format(url=pr.abs_url))
else:
LOGGER.exception(e)
asyncio.get_event_loop().stop()
if self.context_token is not None:
self.connection.reset(self.context_token)
async def _handle_http(self) -> None:
"""
Send HTTP request and then forwards the following HTTP requests.
"""
conn = self.connection.get()
LOGGER.debug(
HANDLING_HTTP_REQUEST_MSG.format(method=conn.pr.method,
url=conn.pr.abs_url))
await conn.server.write_and_drain(conn.pr.raw)
await asyncio.gather(conn.forward_to_client(self._spent_data),
conn.forward_to_server())
async def _handle_https(self) -> None:
"""
Handles https connection by making HTTP tunnel.
"""
conn = self.connection.get()
hostname = conn.pr.hostname
LOGGER.debug(HANDLING_HTTPS_CONNECTION_MSG.format(url=hostname))
rsc = conn.pr.restriction
if rsc:
if self._spent_data[rsc.initiator] >= rsc.data_limit:
await conn.reset()
return
await conn.client.write_and_drain(CONNECTION_ESTABLISHED_HTTP_MSG)
LOGGER.debug(CONNECTION_ESTABLISHED_MSG.format(url=conn.pr.abs_url))
await asyncio.gather(conn.forward_to_server(),
conn.forward_to_client(self._spent_data))
def trace_stack_top(trace_stack_var: ContextVar) -> Any | None:
"""Return the element at the top of a trace stack."""
trace_stack = trace_stack_var.get()
return trace_stack[-1] if trace_stack else None
# 'var' was set to 'spam' before
# calling 'copy_context()' and 'ctx.run(main)', so:
# var.get() == ctx[var] == 'spam'
var.set('ham')
print('insite:',var.get())
# Now, after setting 'var' to 'ham':
# var.get() == ctx[var] == 'ham'
ctx = copy_context()
# Any changes that the 'main' function makes to 'var'
# will be contained in 'ctx'.
ctx.run(main)
print('outsite:',var.get())
def main1():
var.set('ham')
print('insite2:',var.get())
main1()
print('outsite2:',var.get())
# The 'main()' function was run in the 'ctx' context,
# so changes to 'var' are contained in it:
# ctx[var] == 'ham'
# However, outside of 'ctx', 'var' is still set to 'spam':
# var.get() == 'spam'
class Database(LimitInstances):
"""Represents a database."""
__instances__: dict[str, Database]
def __init__(self, name: str):
self.name = name
self.user: t.Optional[str] = None
self.url: t.Optional[str] = None
self.pool: t.Optional[asyncpg.Pool] = None
self.type = types
self.schemas: t.Set[Schema] = set()
self._mock = False
self._prepared = False
self._tracking = ContextVar(f"stmt_tracking:{name}")
@classmethod
def connect(cls, name: str, user: str, password: str, *, host: str = "localhost", port: int = 5432) -> Database:
"""Establish the connection URL and name for the database, returning the instance representing it."""
if len(cls.__instances__) == 1:
db = cls.__instances__["__default__"]
cls.__instances__[name] = db
db.name = name
else:
db = Database(name)
db.user = user
db.url = f"postgres://{user}:{password}@{host}:{port}/{name}"
return db
@property
def public_schema(self):
return self.Schema("public")
def __call__(self, name: str) -> Database:
"""Return the instance representing the given database name."""
return Database(name)
def __str__(self):
"""Return the URL representation of the given database instance, if set."""
return self.url or self.name
def __repr__(self):
status = " disabled" if self._mock else ""
if self.user:
return f"<Database '{self.name}' user='******'{status}>"
else:
return f"<Database '{self.name}'{status}>"
def __hash__(self):
return hash(str(self))
def __eq__(self, other: t.Any):
if isinstance(other, Database):
return str(self) == str(other)
return False
def __getitem__(self, name: str) -> Database:
"""Retrieve an existing database with the given name."""
return self.__instances__[name]
def __delitem__(self, name: str):
"""Delete a database instance by it's name."""
del self.__instances__[name]
@classmethod
def get_default(cls):
return cls.__instances__["__default__"]
async def create_pool(self):
"""Create the asyncpg connection pool for this database connection to use."""
if self.pool:
self.pool.close()
if not self.url:
raise DBError("Please define a connection with Database.connect.")
self.pool = await asyncpg.create_pool(self.url, init=self._enable_json) # pragma: no cover
@staticmethod
async def _enable_json(conn: asyncpg.Connection): # pragma: no cover
await conn.set_type_codec("jsonb", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
await conn.set_type_codec("json", encoder=json.dumps, decoder=json.loads, schema="pg_catalog")
async def prepare(self):
"""Prepare all child objects for this database."""
for schema in self.schemas:
await schema.prepare()
self._prepared = True
def disable_execution(self):
"""Return generated SQL without executing when Database.execute is used."""
self._mock = True
def enable_execution(self):
"""Sets Database.execute to it's normal execution behaviour."""
self._mock = False
@contextlib.contextmanager
def stmt_tracking(self):
"""Collects raw executed statements until exit when execution is disabled."""
ctx_token = self._tracking.set([])
try:
yield self
finally:
self._tracking.reset(ctx_token)
async def close(self): # pragma: no cover
"""Close the asyncpg connection pool for this database."""
if self.pool:
await self.pool.close()
async def execute(self, sql: str, *args, timeout: t.Optional[float] = None) -> t.Union[str, tuple[str, t.Any]]:
"""Execute an SQL statement."""
if self._mock:
try:
stmt_list = self._tracking.get()
stmt_list.append((sql, args))
except LookupError:
pass
if not args:
return sql
else:
return sql, *args
if not self.pool: # pragma: no cover
await self.create_pool()
return await self.pool.execute(sql, *args, timeout=timeout) # pragma: no cover
def Schema(self, name: str) -> Schema:
"""Return a bound Schema for this database."""
s = Schema(name, self)
self.schemas.add(s)
return s
def Table(self, name: str) -> Table:
"""Return a bound Table for the public schema on this database."""
return Table(name, self)
class Kanata(BaseDispatcher):
"彼方."
always = True # 兼容重构版的 bcc.
signature_list: List[Union[NormalMatch, PatternReceiver]]
stop_exec_if_fail: bool = True
parsed_items: ContextVar[Dict[str, MessageChain]]
def __init__(self,
signature_list: List[Union[NormalMatch, PatternReceiver]],
stop_exec_if_fail: bool = True) -> None:
"""该魔法方法用于实例化该参数解析器.
Args:
signature_list (List[Union[NormalMatch, PatternReceiver]]): 匹配标识链
stop_exec_if_fail (bool, optional): 是否在无可用匹配时停止监听器执行. Defaults to True.
"""
self.signature_list = signature_list
self.stop_exec_if_fail = stop_exec_if_fail
self.parsed_items = ContextVar("kanata_parsed_items")
@staticmethod
def detect_index(
signature_chain: Tuple[Union[NormalMatch, PatternReceiver]],
message_chain: MessageChain
) -> Optional[Dict[str, Tuple[MessageIndex, MessageIndex]]]:
merged_chain = merge_signature_chain(signature_chain)
message_chain = message_chain.asMerged()
reached_message_index: MessageIndex = (0, None)
# [0] => real_index
# [1] => text_index(optional)
start_index: MessageIndex = (0, None)
match_result: Dict[Arguments, Tuple[MessageIndex, # start(include)
MessageIndex # stop(exclude)
]] = {}
signature_iterable = InsertGenerator(enumerate(merged_chain))
latest_index = None
matching_recevier: Optional[Arguments] = None
for signature_index, signature in signature_iterable:
if isinstance(signature, (Arguments, PatternReceiver)):
if matching_recevier: # 已经选中了一个...
if isinstance(signature, Arguments):
if latest_index == signature_index:
matching_recevier.content.extend(signature.content)
continue
else:
raise TypeError(
"a unexpected case: match conflict")
if isinstance(signature, PatternReceiver):
matching_recevier.content.append(signature)
continue
else:
if isinstance(signature, PatternReceiver):
signature = Arguments([signature])
matching_recevier = signature
start_index = reached_message_index
elif isinstance(signature, NormalMatch):
if not matching_recevier:
# 如果不要求匹配参数, 从当前位置(reached_message_index)开始匹配FullMatch.
current_chain = message_chain.subchain(
slice(reached_message_index, None, None))
if not current_chain.__root__: # index 越界
return
if not isinstance(current_chain.__root__[0], Plain):
# 切片后第一个 **不是** Plain.
return
re_match_result = re.match(signature.operator(),
current_chain.__root__[0].text)
if not re_match_result:
# 不匹配的
return
# 推进当前进度.
plain_text_length = len(current_chain.__root__[0].text)
pattern_length = re_match_result.end(
) - re_match_result.start()
if (pattern_length + 1) > plain_text_length:
# 不推进 text_index 进度, 转而推进 element_index 进度
reached_message_index = (reached_message_index[0] + 1,
None)
else:
# 推进 element_index 进度至已匹配到的地方后.
reached_message_index = (
reached_message_index[0],
origin_or_zero(reached_message_index[1]) +
re_match_result.start() + pattern_length)
else:
# 需要匹配参数(是否贪婪模式查找, 即是否从后向前)
greed = matching_recevier.isGreed
for element_index, element in \
enumerate(message_chain.subchain(slice(reached_message_index, None, None)).__root__):
if isinstance(element, Plain):
current_text: str = element.text
# 完成贪婪判断
text_find_result_list = list(
re.finditer(signature.operator(),
current_text))
if not text_find_result_list:
continue
text_find_result = text_find_result_list[-int(greed
)]
if not text_find_result:
continue
text_find_index = text_find_result.start()
# 找到了! 这里不仅要推进进度, 还要把当前匹配的参数记录结束位置并清理.
stop_index = (
reached_message_index[0] + element_index +
int(element_index == 0),
origin_or_zero(reached_message_index[1]) +
text_find_index)
match_result[matching_recevier] = (
copy.copy(start_index), stop_index)
start_index = (0, None)
matching_recevier = None
pattern_length = text_find_result.end(
) - text_find_result.start()
if current_text == text_find_result.string[slice(
*text_find_result.span())]:
# 此处是如果推进 text_index 就会被爆破....
# 推进 element_index 而不是 text_index
reached_message_index = (
reached_message_index[0] + element_index +
int(element_index != 0), None)
else:
reached_message_index = (
reached_message_index[0] + element_index,
origin_or_zero(reached_message_index[1]) +
text_find_index + pattern_length)
break
else:
# 找遍了都没匹配到.
return
latest_index = signature_index
else:
if matching_recevier: # 到达了终点, 却仍然还要做点事的.
# 计算终点坐标.
text_index = None
latest_element = message_chain.__root__[-1]
if isinstance(latest_element, Plain):
text_index = len(latest_element.text)
stop_index = (len(message_chain.__root__), text_index)
match_result[matching_recevier] = (start_index, stop_index)
return match_result
@staticmethod
def detect_and_mapping(
signature_chain: Tuple[Union[NormalMatch, PatternReceiver]],
message_chain: MessageChain
) -> Optional[Dict[Arguments, MessageChain]]:
match_result = Kanata.detect_index(signature_chain, message_chain)
if match_result is not None:
return {
k: message_chain[v[0]:(v[1][0], (
v[1][1] - (origin_or_zero(v[0][1]) if any([
v[0][0] + 1 == v[1][0], v[0][0] == v[1][0], v[0][0] -
1 == v[1][0]
]) else 0)) if v[1][1] is not None else None)]
for k, v in match_result.items()
}
@staticmethod
def allocation(
mapping: Dict[Arguments, MessageChain]
) -> Optional[Dict[str, MessageChain]]:
if mapping is None:
return None
result = {}
for arguemnt_set, message_chain in mapping.items():
length = len(arguemnt_set.content)
for index, receiver in enumerate(arguemnt_set.content):
if receiver.name in result:
raise ConflictItem(
'{0} is defined repeatedly'.format(receiver))
if isinstance(receiver, RequireParam):
if not message_chain.__root__:
return
result[receiver.name] = message_chain
elif isinstance(receiver, OptionalParam):
if not message_chain.__root__:
result[receiver.name] = None
else:
result[receiver.name] = message_chain
break # 还没来得及做长度匹配...
return result
@lru_cache(None)
def catch_argument_names(self) -> List[str]:
return [
i.name for i in self.signature_list
if isinstance(i, PatternReceiver)
]
async def catch(self, interface: DispatcherInterface):
# 因为 Dispatcher 的特性, 要用 yield (自动清理 self.parsed_items)
token = None
if self.parsed_items.get(None) is None:
message_chain = (await interface.execute_with(
"__kanata_messagechain_origin__", MessageChain,
None)).exclude(Source, Quote, Xml, Json, App, Poke)
mapping_result = self.detect_and_mapping(self.signature_list,
message_chain)
if mapping_result is not None:
token = self.parsed_items.set(self.allocation(mapping_result))
else:
if self.stop_exec_if_fail:
raise ExecutionStop()
_i = random.random()
result = self.parsed_items.get({}).get(interface.name, _i)
if result is _i:
yield # 跳过.(另: Executor 应加入对 default 的不可预测行为反制措施.)
else:
yield Force(result)
if token is not None:
self.parsed_items.reset(token)
class Database:
SUPPORTED_BACKENDS = {
"postgresql": "databases.backends.postgres:PostgresBackend",
"mysql": "databases.backends.mysql:MySQLBackend",
"sqlite": "databases.backends.sqlite:SQLiteBackend",
}
def __init__(self,
url: typing.Union[str, "DatabaseURL"],
*,
force_rollback: bool = False):
self._url = DatabaseURL(url)
self._force_rollback = force_rollback
self.is_connected = False
backend_str = self.SUPPORTED_BACKENDS[self._url.dialect]
backend_cls = import_from_string(backend_str)
assert issubclass(backend_cls, DatabaseBackend)
self._backend = backend_cls(self._url)
# Connections are stored as task-local state.
self._connection_context = ContextVar(
"connection_context") # type: ContextVar
# When `force_rollback=True` is used, we use a single global
# connection, within a transaction that always rolls back.
self._global_connection = None # type: typing.Optional[Connection]
self._global_transaction = None # type: typing.Optional[Transaction]
if self._force_rollback:
self._global_connection = Connection(self._backend)
self._global_transaction = self._global_connection.transaction(
force_rollback=True)
async def connect(self) -> None:
"""
Establish the connection pool.
"""
assert not self.is_connected, "Already connected."
await self._backend.connect()
self.is_connected = True
if self._force_rollback:
assert self._global_transaction is not None
await self._global_transaction.__aenter__()
async def disconnect(self) -> None:
"""
Close all connections in the connection pool.
"""
assert self.is_connected, "Already disconnected."
if self._force_rollback:
assert self._global_transaction is not None
await self._global_transaction.__aexit__()
await self._backend.disconnect()
self.is_connected = False
async def __aenter__(self) -> "Database":
await self.connect()
return self
async def __aexit__(
self,
exc_type: typing.Type[BaseException] = None,
exc_value: BaseException = None,
traceback: TracebackType = None,
) -> None:
await self.disconnect()
async def fetch_all(self, query: ClauseElement) -> typing.List[RowProxy]:
async with self.connection() as connection:
return await connection.fetch_all(query=query)
async def fetch_one(self, query: ClauseElement) -> RowProxy:
async with self.connection() as connection:
return await connection.fetch_one(query=query)
async def execute(self,
query: ClauseElement,
values: dict = None) -> typing.Any:
async with self.connection() as connection:
return await connection.execute(query=query, values=values)
async def execute_many(self, query: ClauseElement, values: list) -> None:
async with self.connection() as connection:
return await connection.execute_many(query=query, values=values)
async def iterate(
self,
query: ClauseElement) -> typing.AsyncGenerator[RowProxy, None]:
async with self.connection() as connection:
async for record in connection.iterate(query):
yield record
def connection(self) -> "Connection":
if self._global_connection is not None:
return self._global_connection
try:
return self._connection_context.get()
except LookupError:
connection = Connection(self._backend)
self._connection_context.set(connection)
return connection
def transaction(self, *, force_rollback: bool = False) -> "Transaction":
return self.connection().transaction(force_rollback=force_rollback)
class Broker(dramatiq.Broker): # pylint: disable=abstract-method
"""Dramatiq broker using sqlalchemy to process messages."""
def __init__(self, middleware=None, settings=None):
super().__init__(middleware=middleware)
self.__settings = settings or Settings()
self.__db_engine = db_engine(self.__settings)
self.__shared_session = ContextVar("shared_session")
self.__broker_id = uuid.uuid4()
self.__queue_events = defaultdict(Event)
# We have some actors using this, so it's always enabled.
self.add_middleware(CurrentMessage())
# Enable special handling of actors with 'scheduled=True' in options.
self.add_middleware(
SchedulerMiddleware(self.__settings, self.__db_engine))
self.add_middleware(LocalNotifyMiddleware())
# This is postgres-specific, so...
if "postgresql" in str(self.__db_engine.url):
self.add_middleware(PostgresNotifyMiddleware(self.__db_engine))
def set_session(self, session):
"""Set an sqlalchemy session for use with the broker.
A session should be set during handling of any HTTP requests.
It will ensure that enqueues take place in the same transaction
as other changes made during handling of that request.
In other contexts, e.g. one dramatiq actor invoking another,
the session can be safely left unset and the broker will manage
its own session as needed.
"""
self.__shared_session.set(session)
@property
def session(self):
return self.__shared_session.get(None)
def notify(self):
"""Notify all consumers that something might have changed.
Consumers are called in a loop and they will sleep between iterations.
Calling this method will wake any sleeping consumers so that new messages
can be found earlier.
"""
queues = self.get_declared_queues().union(
self.get_declared_delay_queues())
for queue_name in queues:
self.__queue_events[queue_name].set()
def declare_queue(self, queue_name):
if queue_name not in self.queues:
self.emit_before("declare_queue", queue_name)
self.queues[queue_name] = None
self.emit_after("declare_queue", queue_name)
delayed_name = dq_name(queue_name)
self.queues[delayed_name] = None
self.delay_queues.add(delayed_name)
self.emit_after("declare_delay_queue", delayed_name)
def consume(self, queue_name, prefetch=1, timeout=30000):
consumer_id = "%s-%s" % (queue_name, self.__broker_id)
# We need one (arbitrarily selected) consumer to act as the
# "master" which will take on additional maintenance duties.
# We'll have one consumer per queue and it would be wasteful
# to let all of them do this.
master = queue_name == list(self.queues.keys())[0]
return Consumer(
queue_name,
db_engine=self.__db_engine,
consumer_id=consumer_id,
prefetch=prefetch,
master=master,
queue_event=self.__queue_events[queue_name],
settings=self.__settings,
)
def enqueue_using_session(self, db, message, delay=None):
# Given a dramatiq message, saves it to the queue in the DB.
queue_name = message.queue_name
if delay is not None:
queue_name = dq_name(queue_name)
message.options["eta"] = current_millis() + delay
db_message = DramatiqMessage(id=message.message_id,
actor=message.actor_name,
queue=queue_name)
message_dict = message.asdict()
# Drop these so we're not storing them in two places.
del message_dict["message_id"]
del message_dict["queue_name"]
del message_dict["actor_name"]
db_message.body = message_dict
# Use merge rather than add since the message may already exist;
# for instance this happens in case of retry.
db_message = db.merge(db_message)
# Explicitly wipe out the consumer, since if we've updated an existing
# message it'll have to be consumed again.
db_message.consumer_id = None
def enqueue(self, message, *, delay=None):
self.emit_before("enqueue", message, delay)
if db := self.session:
# We have a shared session, e.g. we're in an http request handler.
# We reuse the session and don't commit ourselves.
self.enqueue_using_session(db, message, delay)
self.emit_after("enqueue", message, delay)
return message
# We don't have a shared session e.g.
# - an automated test not using a real app
# - testing from python CLI
# - one actor invoking another
# Then we use our own short-lived session and commit.
db = Session(bind=self.__db_engine)
try:
self.enqueue_using_session(db, message, delay)
db.commit()
self.emit_after("enqueue", message, delay)
finally:
db.close()
return message
class Database:
SUPPORTED_BACKENDS = {
"postgresql": "databases.backends.postgres:PostgresBackend",
"postgresql+aiopg": "databases.backends.aiopg:AiopgBackend",
"postgres": "databases.backends.postgres:PostgresBackend",
"mysql": "databases.backends.mysql:MySQLBackend",
"sqlite": "databases.backends.sqlite:SQLiteBackend",
}
def __init__(
self,
url: typing.Union[str, "DatabaseURL"],
*,
force_rollback: bool = False,
**options: typing.Any,
):
self.url = DatabaseURL(url)
self.options = options
self.is_connected = False
self._force_rollback = force_rollback
backend_str = self.SUPPORTED_BACKENDS[self.url.scheme]
backend_cls = import_from_string(backend_str)
assert issubclass(backend_cls, DatabaseBackend)
self._backend = backend_cls(self.url, **self.options)
# Connections are stored as task-local state.
self._connection_context = ContextVar(
"connection_context") # type: ContextVar
# When `force_rollback=True` is used, we use a single global
# connection, within a transaction that always rolls back.
self._global_connection = None # type: typing.Optional[Connection]
self._global_transaction = None # type: typing.Optional[Transaction]
async def connect(self) -> None:
"""
Establish the connection pool.
"""
assert not self.is_connected, "Already connected."
await self._backend.connect()
logger.info("Connected to database %s",
self.url.obscure_password,
extra=CONNECT_EXTRA)
self.is_connected = True
if self._force_rollback:
assert self._global_connection is None
assert self._global_transaction is None
self._global_connection = Connection(self._backend)
self._global_transaction = self._global_connection.transaction(
force_rollback=True)
await self._global_transaction.__aenter__()
async def disconnect(self) -> None:
"""
Close all connections in the connection pool.
"""
assert self.is_connected, "Already disconnected."
if self._force_rollback:
assert self._global_connection is not None
assert self._global_transaction is not None
await self._global_transaction.__aexit__()
self._global_transaction = None
self._global_connection = None
await self._backend.disconnect()
logger.info(
"Disconnected from database %s",
self.url.obscure_password,
extra=DISCONNECT_EXTRA,
)
self.is_connected = False
async def __aenter__(self) -> "Database":
await self.connect()
return self
async def __aexit__(
self,
exc_type: typing.Type[BaseException] = None,
exc_value: BaseException = None,
traceback: TracebackType = None,
) -> None:
await self.disconnect()
async def fetch_all(self,
query: typing.Union[ClauseElement, str],
values: dict = None) -> typing.List[typing.Mapping]:
async with self.connection() as connection:
return await connection.fetch_all(query, values)
async def fetch_one(
self,
query: typing.Union[ClauseElement, str],
values: dict = None) -> typing.Optional[typing.Mapping]:
async with self.connection() as connection:
return await connection.fetch_one(query, values)
async def fetch_val(
self,
query: typing.Union[ClauseElement, str],
values: dict = None,
column: typing.Any = 0,
) -> typing.Any:
async with self.connection() as connection:
return await connection.fetch_val(query, values, column=column)
async def execute(self,
query: typing.Union[ClauseElement, str],
values: dict = None) -> typing.Any:
async with self.connection() as connection:
return await connection.execute(query, values)
async def execute_many(self, query: typing.Union[ClauseElement, str],
values: list) -> None:
async with self.connection() as connection:
return await connection.execute_many(query, values)
async def iterate(self,
query: typing.Union[ClauseElement, str],
values: dict = None
) -> typing.AsyncGenerator[typing.Mapping, None]:
async with self.connection() as connection:
async for record in connection.iterate(query, values):
yield record
def connection(self) -> "Connection":
if self._global_connection is not None:
return self._global_connection
try:
return self._connection_context.get()
except LookupError:
connection = Connection(self._backend)
self._connection_context.set(connection)
return connection
def transaction(self,
*,
force_rollback: bool = False,
**kwargs: typing.Any) -> "Transaction":
return Transaction(self.connection,
force_rollback=force_rollback,
**kwargs)
@contextlib.contextmanager
def force_rollback(self) -> typing.Iterator[None]:
initial = self._force_rollback
self._force_rollback = True
try:
yield
finally:
self._force_rollback = initial
class AsyncPostgresSQL:
__slots__ = ('_ctxvar', '_pool', '_pool_kwargs', '_listener')
def __init__(self, dsn=None, min_size=10, max_size=10, on_init_conn=None):
"""
Define settings to establish a connection to a PostgreSQL server.
The connection parameters may be specified either as a connection
URI in *dsn*, or as specific keyword arguments, or both.
If both *dsn* and keyword arguments are specified, the latter
override the corresponding values parsed from the connection URI.
:param dsn:
Connection arguments specified using as a single string in the
`[https://www.postgresql.org/docs/current/libpq-connect.html
#id-1.7.3.8.3.6](libpq connection URI format)`_:
``postgres://user:password@host:port/database?option=value``.
"""
if not on_init_conn:
on_init_conn = _init_connection
self._pool_kwargs = dict(dsn=dsn,
min_size=min_size,
max_size=max_size,
init=on_init_conn)
self._ctxvar = ContextVar('connection')
self._listener = None
def transaction(self, *d_args, renew=False, autocommit=False):
"""Decorate the function to access datasbase.
:param renew: Force the function with a new connection.
:param autocommit: autocommit
"""
def _sqlblk_decorator(func):
async def _sqlblock_wrapper(*args, **kwargs):
ctxvar = self._ctxvar
pool = self._pool
if pool is None:
raise ValueError('pool is none')
block = ctxvar.get(None)
if block is None or renew:
conn = None
try:
conn = await pool.acquire()
if conn is None:
conn_dsn = self._pool_kwargs.get("dsn")
errmsg = (
f"unavailable connection '{conn_dsn}' "
f"to invoke sql block '{func.__module__}.{func.__name__}'"
)
raise UnavailableConnectionException(errmsg)
block = SQLBlock(conn, autocommit=autocommit)
return await _scoped_invoke(ctxvar, block, conn,
autocommit, func, args,
kwargs)
finally:
if pool and conn:
await pool.release(conn)
else:
conn = block._conn
childBlock = SQLBlock(conn,
parent=block,
autocommit=autocommit)
return await _scoped_invoke(ctxvar, childBlock, conn,
autocommit, func, args, kwargs)
return update_func_wrapper(_sqlblock_wrapper, func)
if len(d_args) > 0 and iscoroutinefunction(d_args[0]):
# no argument decorator
return _sqlblk_decorator(d_args[0])
else:
return lambda f: _sqlblk_decorator(f)
async def __aenter__(self):
""" startup the connection pool """
self._pool = LazyConnectionPool(**self._pool_kwargs)
# self._pool = create_pool(**self._pool_kwargs)
await self._pool.__aenter__()
self._listener = Listener(self._pool)
return self
async def __aexit__(self, etyp, exc_val, tb):
""" gracefull shutdown the connection pool """
if self._listener is not None:
await self._listener.close()
self._listener = None
await self._pool.__aexit__()
self._pool = None
# def __call__(self, *sqltexts, **params):
# if not params:
# params = _get_ctx_frame(1).f_locals
# sqlblock = self._sqlblock
# for sqltext in sqltexts:
# sqlblock.join(sqltext, vars=params)
# return self
def sql(self, *sqltexts, **params):
if not params:
params = _get_ctx_frame(1).f_locals
sqlblock = self._sqlblock
for sqltext in sqltexts:
sqlblock.join(sqltext, vars=params)
return self
def __lshift__(self, sqltext):
self._sqlblock.join(sqltext, vars=_get_ctx_frame(1).f_locals)
return self
async def execute(self, **params):
return await self._sqlblock.fetch(**params)
def __await__(self):
return self._sqlblock.fetch().__await__()
async def fetch_first(self, **params):
return await self._sqlblock.fetch_first(**params)
async def first(self, **params):
return await self._sqlblock.fetch_first(**params)
def __aiter__(self):
return self._sqlblock.__aiter__()
async def listen(self, channel):
""" listen for Postgres notifications
The returned value is the payload of notification
:param str channel: Channel to listen on.
"""
return await self._listener.get(channel)
async def notify(self, channel, payload):
await self._sqlblock._conn.execute("NOTIFY $1 $2", channel, payload)
@property
def _sqlblock(self) -> SQLBlock:
"""Get sqlblock in context"""
sqlblock = self._ctxvar.get()
return sqlblock