async def process_callback(self, callback_results):
try:
if isasyncgen(callback_results):
async for callback_result in callback_results:
await self._process_callback(callback_result)
elif isgenerator(callback_results):
for callback_result in callback_results:
await self._process_callback(callback_result)
else:
await self._process_callback(callback_results)
except Exception as e:
self.logger.exception(e)
async def async_transform(self, value, context=None):
if not isinstance(value, AbcMapping):
self._failure("value is not a dict",
value=value,
code=codes.IS_NOT_A_DICT)
collect = {}
errors = {}
touched_names = []
for key in self._keys:
key_run = getattr(key, 'async_call', key)(
value,
context=context,
)
if inspect.isasyncgen(key_run):
async for k, v, names in key_run:
if isinstance(v, DataError):
errors[k] = v
else:
collect[k] = v
touched_names.extend(names)
else:
for k, v, names in key_run:
if isinstance(v, DataError):
errors[k] = v
else:
collect[k] = v
touched_names.extend(names)
if not self.ignore_any:
for key in value:
if key in touched_names:
continue
if key in self.ignore:
continue
if not self.allow_any and key not in self.extras:
if key in collect:
errors[key] = DataError("%s key was shadowed" % key,
code=codes.SHADOWED)
else:
errors[key] = DataError("%s is not allowed key" % key,
code=codes.NOT_ALLOWED)
elif key in collect:
errors[key] = DataError("%s key was shadowed" % key,
code=codes.SHADOWED)
else:
try:
collect[key] = await self.extras_trafaret.async_check(
value[key])
except DataError as de:
errors[key] = de
if errors:
self._failure(error=errors, code=codes.SOME_ELEMENTS_DID_NOT_MATCH)
return collect
async def _build_path_generator(self):
"""
Adapter that takes whatever code was provided for _list_paths and turns it into a
generator so we can load nodes in chunks (rather than all at once).
"""
if inspect.isgeneratorfunction(self._list_paths) or inspect.isasyncgen(
self._list_paths):
self._path_generator = self._list_paths()
elif inspect.iscoroutinefunction(self._list_paths):
self._path_generator = (p for p in await self._list_paths())
else:
self._path_generator = (p for p in self._list_paths())
def _ensure_aiter(
iter_: Union[AsyncGenerator[bytes, None], Iterable],
) -> AsyncGenerator[bytes, None]:
if isasyncgen(iter_):
return iter_ # type: ignore
else:
async def aiter() -> AsyncGenerator[bytes, None]:
for data in iter_: # type: ignore
yield data
return aiter()
async def main(req: APIRequest,
areaType: str = Query(..., max_length=10, title="Area type"),
release: str = Query(...,
regex=r"^\d{4}-\d{2}-\d{2}$",
title="Release date"),
metric: List[str] = Query(...),
format: str = Query("json",
regex=r"^csv|jsonl?|xml$",
title="Response format"),
areaCode: Optional[str] = Query(None,
max_length=10,
title="Area code")):
request = Request(area_type=areaType,
release=release,
format=format,
metric=metric,
area_code=areaCode,
method=req.method,
url=req.url)
try:
response = await get_data(request=request)
except APIException as err:
logging.info(err)
content = dumps({"response": err.message, "status_code": err.code})
response = Response(content=content.encode(), status_code=err.code)
except Exception as err:
# A generic exception may contain sensitive data and must
# never be included in the response.
logger.exception(err)
err = HTTPStatus.INTERNAL_SERVER_ERROR
content = dumps({
"response":
("An internal error occurred whilst processing your request, please "
"try again. If the problem persists, please report as an issue and "
"include your request."),
"status_code":
err,
"status":
getattr(err, 'phrase')
})
response = Response(content=content.encode(), status_code=err)
if request.method == "HEAD" or not isasyncgen(response.content):
return APIResponse(response.content,
status_code=response.status_code,
headers=await response.headers)
return StreamingResponse(response.content,
status_code=response.status_code,
headers=await response.headers)
async def embedded(self) -> T.Dict[str, T.Any]:
result = {}
_links = await self._links()
for key, value in _links.items():
if key in self.embed:
if (inspect.isasyncgen(value) or inspect.isgenerator(value)
or isinstance(value, _view.View)
or isinstance(value, collections.abc.Iterable)):
result[key] = value
else:
_logger.error("Don't know how to embed object: %s", value)
return result
async def test_aiter_chunk(iter_client):
resp = await iter_client.select_all(measurement='cpu_load',
chunked=True,
chunk_size=10,
wrap=True)
assert inspect.isasyncgen(resp.gen)
chunks = []
async for chunk in resp.iterchunks():
chunks.append(chunk)
logger.info(resp)
logger.info(chunks[0])
assert len(chunks) == 10
def ensure_generator(value):
if legacy.is_generator(value):
return True, value
if hasattr(inspect, "isasyncgen") and\
inspect.isasyncgen(value): #@UndefinedVariable
return True, AyncWrapper(value)
if hasattr(inspect, "iscoroutine") and\
inspect.iscoroutine(value): #@UndefinedVariable
return True, CoroutineWrapper(value)
return False, value
def async_to_sync_wrap(*args, **kwargs):
coroutine = function(*args, **kwargs)
try:
loop = asyncio.get_event_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
if threading.current_thread() is threading.main_thread():
if loop.is_running():
return coroutine
else:
if inspect.iscoroutine(coroutine):
return loop.run_until_complete(coroutine)
if inspect.isasyncgen(coroutine):
return loop.run_until_complete(
consume_generator(coroutine))
else:
if inspect.iscoroutine(coroutine):
if loop.is_running():
async def coro_wrapper():
return await asyncio.wrap_future(
asyncio.run_coroutine_threadsafe(
coroutine, main_loop))
return coro_wrapper()
else:
return asyncio.run_coroutine_threadsafe(
coroutine, main_loop).result()
if inspect.isasyncgen(coroutine):
if loop.is_running():
return coroutine
else:
return asyncio.run_coroutine_threadsafe(
consume_generator(coroutine), main_loop).result()
def _run_func(func, *args, **kwargs):
out = None
if callable(func):
out = func(*args, **kwargs)
if is_asyncio_context():
return out
elif inspect.isasyncgen(out):
with _loop_lock:
return iter_over_async(out, _get_loop())
elif inspect.iscoroutinefunction(func):
with _loop_lock:
return _get_loop().run_until_complete(out)
return out
def __init__(
self, iterable: Union[AsyncGenerator[bytes, None],
Iterable]) -> None:
self.iter: AsyncGenerator[bytes, None]
if isasyncgen(iterable):
self.iter = iterable # type: ignore
else:
async def _aiter() -> AsyncGenerator[bytes, None]:
for data in iterable: # type: ignore
yield data
self.iter = _aiter()
async def eval(self, ctx, *, code: str):
env = {
"ctx":
ctx,
"author":
ctx.author,
"message":
ctx.message,
"guild":
ctx.guild,
"bot":
self.bot,
"reference":
ctx.message.reference,
"resolved":
ctx.message.reference.resolved if ctx.message.reference else None,
}
env.update(globals())
imports = "import asyncio\n"
"import discord\nfrom discord.ext import commands\nimport aiohttp\n"
body = "async def func():\n" + textwrap.indent(imports + code, " ")
try:
import_expression.exec(body, env, locals())
except Exception as e:
etype = type(e)
trace = e.__traceback__
lines = traceback.format_exception(etype, e, trace)
paginator = WrappedPaginator(max_size=500,
prefix="A weird error occured```py",
suffix="```")
paginator.add_line("".join(lines))
interface = PaginatorInterface(ctx.bot,
paginator,
owner=ctx.author)
return await interface.send_to(ctx)
try:
maybe_coro = locals()["func"]()
if inspect.isasyncgen(maybe_coro):
async for i in maybe_coro:
await ctx.send(i)
return
thing = await maybe_coro
if thing:
if isinstance(thing, discord.Embed):
return await ctx.send(embed=thing)
if isinstance(thing, discord.File):
return await ctx.send(file=thing)
else:
await ctx.send(thing)
except Exception as e:
await ctx.send(e)
async def _encode(obj: T.Any, stack: T.Set) -> T.Union[str, T.Any]:
if isinstance(obj, URL):
yield _encode_string(str(obj))
elif hasattr(obj, 'to_dict'):
try:
obj = await obj.to_dict()
except web.HTTPException as e:
_logger.error("Unexpected exception", exc_info=e, stack_info=True)
obj = {
'_links': {
'self': {
'href': obj.canonical_rel_url
}
},
'_status': e.status_code
}
if e.text is not None:
obj['description'] = e.text
async for s in _encode_dict(obj, stack):
yield s
elif isinstance(obj, str):
yield _encode_string(obj)
elif obj is None:
yield 'null'
elif obj is True:
yield 'true'
elif obj is False:
yield 'false'
elif isinstance(obj, float):
yield _encode_float(obj)
elif isinstance(obj, int):
yield str(obj)
elif isinstance(obj, collections.abc.Mapping):
async for s in _encode_dict(obj, stack):
yield s
elif isinstance(obj, collections.abc.Iterable):
async for s in _encode_list(obj, stack):
yield s
elif inspect.isasyncgen(obj):
async for s in _encode_async_generator(obj, stack):
yield s
elif hasattr(obj, '__str__'):
message = "Not sure how to serialize object of class %s:\n" \
"%s\nDefaulting to str()."
_logger.warning(message, type(obj), repr(obj))
yield _encode_string(str(obj))
else:
message = "Don't know how to serialize object of class %s:\n" \
"%s"
_logger.error(message, type(obj), repr(obj))
yield 'null'
def whatis(self, arguments):
"""Prints the type of the argument.
Usage:
whatis <name>...
"""
arg = " ".join(arguments["argv"][1:])
try:
value = eval(arg, self._obj.curframe.f_globals, self._obj.curframe.f_locals)
except: # noqa
v = sys.exc_info()[1]
self._ui.printf('*** %R{}%N: {}\n'.format(type(v).__name__, v))
return
if inspect.ismodule(value):
filename = value.__file__ if value.__file__ else "builtin module"
self._ui.print('Module:', filename)
elif inspect.isasyncgenfunction(value):
self._ui.print('Async Gen function:', value.__name__, inspect.signature(value))
elif inspect.isasyncgen(value):
self._ui.print('Async Gen:', value.__name__, inspect.signature(value))
elif inspect.iscoroutine(value):
self._ui.print('Coroutine:', value)
self._ui.print(' state:', inspect.getcoroutinestate(value))
if inspect.isawaitable(value):
self._ui.print(' and awaitable.')
self._ui.print(' stack:', _coroutine_format_stack(value, complete=False))
elif inspect.isgenerator(value):
self._ui.print('Generator:', value)
self._ui.print(' state:', inspect.getgeneratorstate(value))
if inspect.isawaitable(value):
self._ui.print(' and awaitable.')
elif inspect.iscoroutinefunction(value):
self._ui.print('Coroutine function:', value.__name__, inspect.signature(value))
elif inspect.isgeneratorfunction(value):
self._ui.print('Generator function:', value.__name__, inspect.signature(value))
elif inspect.isfunction(value):
self._ui.print('Function:', value.__name__, inspect.signature(value))
elif inspect.ismethod(value):
self._ui.print('Method:', value.__name__, inspect.signature(value))
elif inspect.iscode(value):
self._ui.print('Code object:', value.co_name)
elif inspect.isclass(value):
self._ui.print('Class:', value.__name__)
elif inspect.ismethoddescriptor(value):
self._ui.print('Method descriptor:', value.__name__)
elif inspect.isdatadescriptor(value):
self._ui.print('Data descriptor:', value.__name__)
# None of the above...
else:
self._ui.print("Type of:", type(value))
def initialize_async_fixtures(funcargs, testargs):
"""
Get async generator fixtures first value, and await coroutine fixtures
"""
for name, value in funcargs.items():
if name not in testargs:
continue
if sys.version_info >= (3, 6) and inspect.isasyncgen(value):
try:
testargs[name] = yield from value.__anext__()
except StopAsyncIteration:
raise RuntimeError("async generator didn't yield") from None
elif sys.version_info >= (3, 5) and inspect.iscoroutine(value):
testargs[name] = yield from value
async def send_body(self, response, send):
if isinstance(response.body, str):
await send({
'type': HTTP_RESPONSE_BODY,
'body': response.body.encode('utf-8')
})
elif inspect.isasyncgen(response.body):
async for chunk in response.body:
await send({
'type': HTTP_RESPONSE_BODY,
'body': response.body.encode('utf-8')
})
else:
raise Exception('Cannot handle')
def wrapper(*args, **kwargs):
result = args[0]
if asyncio is not None and hasattr(
result,
'__await__') and asyncio.coroutines.iscoroutine(result):
d = _aio_as_deferred(result)
d.addCallback(iterate_spider_output)
return d
elif hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result):
d = _aio_as_deferred(collect_asyncgen(result))
d.addCallback(iterate_spider_output)
return d
else:
return func(*args, **kwargs)
def __init__(
self, data: typing.Union[typing.Sequence, typing.Generator, typing.Callable[[int], typing.Any]], *,
per_page: int = 10,
formatter: typing.Callable[
['Paginator', typing.Sequence[typing.Any]], typing.Union[str, discord.Embed, dict]
] = None):
"""
Args:
data (typing.Union[typing.Sequence, typing.Generator, typing.Callable[[int], typing.Any]]): The
data that you want to paginate.
If a generator or function is given then the `max_pages` will start as the string "?", and the `per_page`
parameter will be ignored - the formatter will be passed the content of whatever your generator returns.
If a function is given, then you will be passed the page number as an argument - raising
`StopIteration` from this function will cause the `max_pages` attribute to be set,
and the page will go back to what it was previously.
per_page (int, optional): The number of items that appear on each page. This argument only works for sequences
formatter (typing.Callable[['Paginator', typing.Sequence[typing.Any]], typing.Union[str, discord.Embed, dict]], optional): A
function taking the paginator instance and a list of things to display, returning a dictionary of kwargs that get passed
directly into a :func:`discord.Message.edit`.
"""
self.data = data
self.per_page: int = per_page
self.formatter: typing.Callable[['Paginator', typing.Sequence[typing.Any]], typing.Union[str, discord.Embed, dict]]
if formatter is None:
self.formatter = self.default_list_formatter
else:
self.formatter = formatter
self.current_page: int = None
self._page_cache = {}
self.max_pages: int = '?'
self._data_is_generator = any((
inspect.isasyncgenfunction(self.data),
inspect.isasyncgen(self.data),
inspect.isgeneratorfunction(self.data),
inspect.isgenerator(self.data),
))
self._data_is_function = any((
inspect.isfunction(self.data),
inspect.iscoroutine(self.data),
))
self._data_is_iterable = not (self._data_is_generator or self._data_is_function)
if self._data_is_iterable:
pages, left_over = divmod(len(data), self.per_page)
if left_over:
pages += 1
self.max_pages = pages
self._message = None
def compute(context, input_defs) -> Generator[Output, None, None]:
kwargs = {}
for input_name in input_names:
kwargs[input_name] = input_defs[input_name]
result = fn(context, **kwargs) if context_arg_provided else fn(**kwargs)
if inspect.isgenerator(result):
return result
elif inspect.isasyncgen(result):
return result
elif inspect.iscoroutine(result):
return _coerce_async_solid_to_async_gen(result, context, output_defs)
else:
return _coerce_solid_output_to_iterator(result, context, output_defs)
async def async_put_item(self, result):
"""
async put item
async generator/generator/list/item
"""
if isasyncgen(result):
async for item in result:
await self.put_item(item)
elif isawaitable(result):
await self.put_item(await result)
elif isinstance(result, list):
for item in result:
await self.put_item(item)
else:
await self.put_item(result)
def _invoke(self, args, kwargs):
# This runs on a separate thread
res = self._f(*args, **kwargs)
if inspect.iscoroutine(res):
try:
loop = asyncio.new_event_loop()
return loop.run_until_complete(res)
finally:
loop.close()
elif inspect.isasyncgen(res):
raise RuntimeError("Async generators are not supported")
elif inspect.isgenerator(res):
raise RuntimeError("Generators are not supported")
else:
return res
def _get_requests(
self,
**kwargs) -> Union[Iterator['Request'], AsyncIterator['Request']]:
"""Get request in generator"""
_kwargs = vars(self.args)
_kwargs['data'] = self.input_fn
# override by the caller-specific kwargs
_kwargs.update(kwargs)
if inspect.isasyncgen(self.input_fn):
from .request.asyncio import request_generator
return request_generator(**_kwargs)
else:
from .request import request_generator
return request_generator(**_kwargs)
async def async_transform(self, value, context=None):
if not isinstance(value, AbcMapping):
self._failure("value is not a dict", value=value)
collect = {}
errors = {}
touched_names = []
for key in self.keys:
if not callable(key) and not hasattr(key, 'async_call'):
raise ValueError('Non callable Keys are not supported')
key_run = call_with_context_if_support(
getattr(key, 'async_call', key),
value,
context=context,
)
if inspect.isasyncgen(key_run):
async for k, v, names in key_run:
if isinstance(v, DataError):
errors[k] = v
else:
collect[k] = v
touched_names.extend(names)
else:
for k, v, names in key_run:
if isinstance(v, DataError):
errors[k] = v
else:
collect[k] = v
touched_names.extend(names)
if not self.ignore_any:
for key in value:
if key in touched_names:
continue
if key in self.ignore:
continue
if not self.allow_any and key not in self.extras:
errors[key] = DataError("%s is not allowed key" % key)
elif key in collect:
errors[key] = DataError("%s key was shadowed" % key)
else:
try:
collect[key] = await self.extras_trafaret.async_check(
value[key])
except DataError as de:
errors[key] = de
if errors:
raise DataError(error=errors, trafaret=self)
return collect
async def handle_dispatches(self, ctx: EventContext, name: str,
dispatch: dict):
"""
Handles dispatches for the client.
"""
try:
handle_name = name.lower()
handler = getattr(self.state, f"handle_{handle_name}")
except AttributeError:
logger.warning(f"Got unknown dispatch {name}")
return
else:
logger.debug(f"Processing event {name}")
try:
result = handler(ctx.gateway, dispatch)
if inspect.isawaitable(result):
result = await result
elif inspect.isasyncgen(result):
async with multio.finalize_agen(result) as gen:
async for i in gen:
await self.events.fire_event(i[0],
*i[1:],
gateway=ctx.gateway,
client=self)
# no more processing after the async gen
return
if not isinstance(result, tuple):
await self.events.fire_event(result,
gateway=ctx.gateway,
client=self)
else:
await self.events.fire_event(result[0],
*result[1:],
gateway=ctx.gateway,
client=self)
# TODO: Change this from being an exception to being an event.
except ChunkGuilds:
ctx.gateway._get_chunks()
except Exception:
logger.exception(
f"Error decoding event {event} with data {dispatch}!")
await ctx.gateway.close(code=1006, reason="Internal client error")
raise
async def links(self) -> T.Dict[str, T.Any]:
result = {}
_links = await self._links()
for key, value in _links.items():
if key == 'item':
key = 'item'
if isinstance(value, _view.View):
if key not in self.embed:
result[key] = value.to_link
elif inspect.isasyncgen(value):
if key not in self.embed:
async def g1(resources):
async for resource in resources:
yield resource.to_link
result[key] = g1(value)
elif inspect.isgenerator(value):
if key not in self.embed:
def g2(resources):
for resource in resources:
yield resource.to_link
result[key] = g2(value)
elif isinstance(value, collections.Mapping):
if key in self.embed:
_logger.info(
'Client asked to embed unembeddable object: %s', value)
result[key] = value
elif isinstance(value, collections.abc.Iterable):
def g3(p_key, p_value):
for o in p_value:
if not isinstance(o, _view.View):
if p_key in self.embed:
_logger.info(
'Client asked to embed unembeddable object: %s',
o)
yield o
elif p_key not in self.embed:
yield o.to_link
result[key] = g3(key, value)
elif key not in self.embed:
_logger.error("Don't know how to render object as link: %s",
value)
return result
def _method(*args, **kwargs):
# We need to keep the original attributes so that they could be correctly
# bound to the class/instance at runtime.
func = getattr(cls, orig_name)
coro = func(*args, **kwargs)
_api_session = api_session.get()
if _api_session is None:
raise RuntimeError("API functions must be called "
"inside the context of a valid API session")
if isinstance(_api_session, AsyncSession):
return coro
else:
if inspect.isasyncgen(coro):
return _api_session.worker_thread.execute_generator(coro)
else:
return _api_session.worker_thread.execute(coro)
async def whatever_gen_once(any_gen, *args, **kwargs):
if inspect.isasyncgenfunction(any_gen):
# 如果是异步生成器函数
async for i in any_gen(*args, **kwargs):
return i
elif inspect.isgeneratorfunction(
any_gen) and not inspect.iscoroutinefunction(any_gen):
# 同步生成器
for i in any_gen(*args, **kwargs):
return i
elif inspect.isgenerator(any_gen):
for i in any_gen:
return i
elif inspect.isasyncgen(any_gen):
async for i in any_gen:
return i
def wrapper(*args, **kwargs):
memo = _CallMemo(python_func, _localns, args=args, kwargs=kwargs)
check_argument_types(memo)
retval = func(*args, **kwargs)
check_return_type(retval, memo)
# If a generator is returned, wrap it if its yield/send/return types can be checked
if inspect.isgenerator(retval) or isasyncgen(retval):
return_type = memo.type_hints.get('return')
origin = getattr(return_type, '__origin__')
if origin in generator_origin_types:
return TypeCheckedGenerator(retval, memo)
elif origin is not None and origin in asyncgen_origin_types:
return TypeCheckedAsyncGenerator(retval, memo)
return retval
def ensure_generator(value):
if legacy.is_generator(value):
return True, value
if hasattr(value, "_generator"):
return True, value
if hasattr(inspect, "iscoroutine") and\
inspect.iscoroutine(value): #@UndefinedVariable
return True, CoroutineWrapper(value)
if hasattr(inspect, "isasyncgen") and\
inspect.isasyncgen(value): #@UndefinedVariable
return True, AyncgenWrapper(value)
return False, value
def __init__(
self,
content: typing.Any,
status_code: int = 200,
headers: dict = None,
media_type: str = None,
background: BackgroundTask = None,
) -> None:
if inspect.isasyncgen(content):
self.body_iterator = content
else:
self.body_iterator = iterate_in_threadpool(content)
self.status_code = status_code
self.media_type = self.media_type if media_type is None else media_type
self.background = background
self.init_headers(headers)
def _core_generator(
solid_def: "SolidDefinition", context: "DirectSolidExecutionContext", input_dict: Dict[str, Any]
) -> Generator[Any, None, None]:
from dagster.core.execution.plan.compute import gen_from_async_gen
with user_code_error_boundary(
DagsterSolidInvocationError,
control_flow_exceptions=[RetryRequested],
msg_fn=lambda: f'Error occurred while invoking solid "{solid_def.name}":',
):
compute_iterator = solid_def.compute_fn(context, input_dict)
if inspect.isasyncgen(compute_iterator):
compute_iterator = gen_from_async_gen(compute_iterator)
yield from compute_iterator
async def async_transform(self, value, context=None):
if not isinstance(value, AbcMapping):
self._failure("value is not a dict", value=value)
collect = {}
errors = {}
touched_names = []
for key in self._keys:
if not callable(key) and not hasattr(key, 'async_call'):
raise ValueError('Non callable Keys are not supported')
key_run = getattr(key, 'async_call', key)(
value,
context=context,
)
if inspect.isasyncgen(key_run):
async for k, v, names in key_run:
if isinstance(v, DataError):
errors[k] = v
else:
collect[k] = v
touched_names.extend(names)
else:
for k, v, names in key_run:
if isinstance(v, DataError):
errors[k] = v
else:
collect[k] = v
touched_names.extend(names)
if not self.ignore_any:
for key in value:
if key in touched_names:
continue
if key in self.ignore:
continue
if not self.allow_any and key not in self.extras:
errors[key] = DataError("%s is not allowed key" % key)
elif key in collect:
errors[key] = DataError("%s key was shadowed" % key)
else:
try:
collect[key] = await self.extras_trafaret.async_check(value[key])
except DataError as de:
errors[key] = de
if errors:
raise DataError(error=errors, trafaret=self)
return collect
def is_async_generator(value):
if not hasattr(inspect, "isasyncgen"): return False
return inspect.isasyncgen(value)
def isasyncgen(obj):
if hasattr(inspect, "isasyncgen"):
if inspect.isasyncgen(obj):
return True
return isinstance(obj, AsyncGenerator)
