async def process_callback(self, callback_results): try: if isasyncgen(callback_results): async for callback_result in callback_results: await self._process_callback(callback_result) elif isgenerator(callback_results): for callback_result in callback_results: await self._process_callback(callback_result) else: await self._process_callback(callback_results) except Exception as e: self.logger.exception(e)
async def async_transform(self, value, context=None): if not isinstance(value, AbcMapping): self._failure("value is not a dict", value=value, code=codes.IS_NOT_A_DICT) collect = {} errors = {} touched_names = [] for key in self._keys: key_run = getattr(key, 'async_call', key)( value, context=context, ) if inspect.isasyncgen(key_run): async for k, v, names in key_run: if isinstance(v, DataError): errors[k] = v else: collect[k] = v touched_names.extend(names) else: for k, v, names in key_run: if isinstance(v, DataError): errors[k] = v else: collect[k] = v touched_names.extend(names) if not self.ignore_any: for key in value: if key in touched_names: continue if key in self.ignore: continue if not self.allow_any and key not in self.extras: if key in collect: errors[key] = DataError("%s key was shadowed" % key, code=codes.SHADOWED) else: errors[key] = DataError("%s is not allowed key" % key, code=codes.NOT_ALLOWED) elif key in collect: errors[key] = DataError("%s key was shadowed" % key, code=codes.SHADOWED) else: try: collect[key] = await self.extras_trafaret.async_check( value[key]) except DataError as de: errors[key] = de if errors: self._failure(error=errors, code=codes.SOME_ELEMENTS_DID_NOT_MATCH) return collect
async def _build_path_generator(self): """ Adapter that takes whatever code was provided for _list_paths and turns it into a generator so we can load nodes in chunks (rather than all at once). """ if inspect.isgeneratorfunction(self._list_paths) or inspect.isasyncgen( self._list_paths): self._path_generator = self._list_paths() elif inspect.iscoroutinefunction(self._list_paths): self._path_generator = (p for p in await self._list_paths()) else: self._path_generator = (p for p in self._list_paths())
def _ensure_aiter( iter_: Union[AsyncGenerator[bytes, None], Iterable], ) -> AsyncGenerator[bytes, None]: if isasyncgen(iter_): return iter_ # type: ignore else: async def aiter() -> AsyncGenerator[bytes, None]: for data in iter_: # type: ignore yield data return aiter()
async def main(req: APIRequest, areaType: str = Query(..., max_length=10, title="Area type"), release: str = Query(..., regex=r"^\d{4}-\d{2}-\d{2}$", title="Release date"), metric: List[str] = Query(...), format: str = Query("json", regex=r"^csv|jsonl?|xml$", title="Response format"), areaCode: Optional[str] = Query(None, max_length=10, title="Area code")): request = Request(area_type=areaType, release=release, format=format, metric=metric, area_code=areaCode, method=req.method, url=req.url) try: response = await get_data(request=request) except APIException as err: logging.info(err) content = dumps({"response": err.message, "status_code": err.code}) response = Response(content=content.encode(), status_code=err.code) except Exception as err: # A generic exception may contain sensitive data and must # never be included in the response. logger.exception(err) err = HTTPStatus.INTERNAL_SERVER_ERROR content = dumps({ "response": ("An internal error occurred whilst processing your request, please " "try again. If the problem persists, please report as an issue and " "include your request."), "status_code": err, "status": getattr(err, 'phrase') }) response = Response(content=content.encode(), status_code=err) if request.method == "HEAD" or not isasyncgen(response.content): return APIResponse(response.content, status_code=response.status_code, headers=await response.headers) return StreamingResponse(response.content, status_code=response.status_code, headers=await response.headers)
async def embedded(self) -> T.Dict[str, T.Any]: result = {} _links = await self._links() for key, value in _links.items(): if key in self.embed: if (inspect.isasyncgen(value) or inspect.isgenerator(value) or isinstance(value, _view.View) or isinstance(value, collections.abc.Iterable)): result[key] = value else: _logger.error("Don't know how to embed object: %s", value) return result
async def test_aiter_chunk(iter_client): resp = await iter_client.select_all(measurement='cpu_load', chunked=True, chunk_size=10, wrap=True) assert inspect.isasyncgen(resp.gen) chunks = [] async for chunk in resp.iterchunks(): chunks.append(chunk) logger.info(resp) logger.info(chunks[0]) assert len(chunks) == 10
def ensure_generator(value): if legacy.is_generator(value): return True, value if hasattr(inspect, "isasyncgen") and\ inspect.isasyncgen(value): #@UndefinedVariable return True, AyncWrapper(value) if hasattr(inspect, "iscoroutine") and\ inspect.iscoroutine(value): #@UndefinedVariable return True, CoroutineWrapper(value) return False, value
def async_to_sync_wrap(*args, **kwargs): coroutine = function(*args, **kwargs) try: loop = asyncio.get_event_loop() except RuntimeError: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) if threading.current_thread() is threading.main_thread(): if loop.is_running(): return coroutine else: if inspect.iscoroutine(coroutine): return loop.run_until_complete(coroutine) if inspect.isasyncgen(coroutine): return loop.run_until_complete( consume_generator(coroutine)) else: if inspect.iscoroutine(coroutine): if loop.is_running(): async def coro_wrapper(): return await asyncio.wrap_future( asyncio.run_coroutine_threadsafe( coroutine, main_loop)) return coro_wrapper() else: return asyncio.run_coroutine_threadsafe( coroutine, main_loop).result() if inspect.isasyncgen(coroutine): if loop.is_running(): return coroutine else: return asyncio.run_coroutine_threadsafe( consume_generator(coroutine), main_loop).result()
def _run_func(func, *args, **kwargs): out = None if callable(func): out = func(*args, **kwargs) if is_asyncio_context(): return out elif inspect.isasyncgen(out): with _loop_lock: return iter_over_async(out, _get_loop()) elif inspect.iscoroutinefunction(func): with _loop_lock: return _get_loop().run_until_complete(out) return out
def __init__( self, iterable: Union[AsyncGenerator[bytes, None], Iterable]) -> None: self.iter: AsyncGenerator[bytes, None] if isasyncgen(iterable): self.iter = iterable # type: ignore else: async def _aiter() -> AsyncGenerator[bytes, None]: for data in iterable: # type: ignore yield data self.iter = _aiter()
async def eval(self, ctx, *, code: str): env = { "ctx": ctx, "author": ctx.author, "message": ctx.message, "guild": ctx.guild, "bot": self.bot, "reference": ctx.message.reference, "resolved": ctx.message.reference.resolved if ctx.message.reference else None, } env.update(globals()) imports = "import asyncio\n" "import discord\nfrom discord.ext import commands\nimport aiohttp\n" body = "async def func():\n" + textwrap.indent(imports + code, " ") try: import_expression.exec(body, env, locals()) except Exception as e: etype = type(e) trace = e.__traceback__ lines = traceback.format_exception(etype, e, trace) paginator = WrappedPaginator(max_size=500, prefix="A weird error occured```py", suffix="```") paginator.add_line("".join(lines)) interface = PaginatorInterface(ctx.bot, paginator, owner=ctx.author) return await interface.send_to(ctx) try: maybe_coro = locals()["func"]() if inspect.isasyncgen(maybe_coro): async for i in maybe_coro: await ctx.send(i) return thing = await maybe_coro if thing: if isinstance(thing, discord.Embed): return await ctx.send(embed=thing) if isinstance(thing, discord.File): return await ctx.send(file=thing) else: await ctx.send(thing) except Exception as e: await ctx.send(e)
async def _encode(obj: T.Any, stack: T.Set) -> T.Union[str, T.Any]: if isinstance(obj, URL): yield _encode_string(str(obj)) elif hasattr(obj, 'to_dict'): try: obj = await obj.to_dict() except web.HTTPException as e: _logger.error("Unexpected exception", exc_info=e, stack_info=True) obj = { '_links': { 'self': { 'href': obj.canonical_rel_url } }, '_status': e.status_code } if e.text is not None: obj['description'] = e.text async for s in _encode_dict(obj, stack): yield s elif isinstance(obj, str): yield _encode_string(obj) elif obj is None: yield 'null' elif obj is True: yield 'true' elif obj is False: yield 'false' elif isinstance(obj, float): yield _encode_float(obj) elif isinstance(obj, int): yield str(obj) elif isinstance(obj, collections.abc.Mapping): async for s in _encode_dict(obj, stack): yield s elif isinstance(obj, collections.abc.Iterable): async for s in _encode_list(obj, stack): yield s elif inspect.isasyncgen(obj): async for s in _encode_async_generator(obj, stack): yield s elif hasattr(obj, '__str__'): message = "Not sure how to serialize object of class %s:\n" \ "%s\nDefaulting to str()." _logger.warning(message, type(obj), repr(obj)) yield _encode_string(str(obj)) else: message = "Don't know how to serialize object of class %s:\n" \ "%s" _logger.error(message, type(obj), repr(obj)) yield 'null'
def whatis(self, arguments): """Prints the type of the argument. Usage: whatis <name>... """ arg = " ".join(arguments["argv"][1:]) try: value = eval(arg, self._obj.curframe.f_globals, self._obj.curframe.f_locals) except: # noqa v = sys.exc_info()[1] self._ui.printf('*** %R{}%N: {}\n'.format(type(v).__name__, v)) return if inspect.ismodule(value): filename = value.__file__ if value.__file__ else "builtin module" self._ui.print('Module:', filename) elif inspect.isasyncgenfunction(value): self._ui.print('Async Gen function:', value.__name__, inspect.signature(value)) elif inspect.isasyncgen(value): self._ui.print('Async Gen:', value.__name__, inspect.signature(value)) elif inspect.iscoroutine(value): self._ui.print('Coroutine:', value) self._ui.print(' state:', inspect.getcoroutinestate(value)) if inspect.isawaitable(value): self._ui.print(' and awaitable.') self._ui.print(' stack:', _coroutine_format_stack(value, complete=False)) elif inspect.isgenerator(value): self._ui.print('Generator:', value) self._ui.print(' state:', inspect.getgeneratorstate(value)) if inspect.isawaitable(value): self._ui.print(' and awaitable.') elif inspect.iscoroutinefunction(value): self._ui.print('Coroutine function:', value.__name__, inspect.signature(value)) elif inspect.isgeneratorfunction(value): self._ui.print('Generator function:', value.__name__, inspect.signature(value)) elif inspect.isfunction(value): self._ui.print('Function:', value.__name__, inspect.signature(value)) elif inspect.ismethod(value): self._ui.print('Method:', value.__name__, inspect.signature(value)) elif inspect.iscode(value): self._ui.print('Code object:', value.co_name) elif inspect.isclass(value): self._ui.print('Class:', value.__name__) elif inspect.ismethoddescriptor(value): self._ui.print('Method descriptor:', value.__name__) elif inspect.isdatadescriptor(value): self._ui.print('Data descriptor:', value.__name__) # None of the above... else: self._ui.print("Type of:", type(value))
def initialize_async_fixtures(funcargs, testargs): """ Get async generator fixtures first value, and await coroutine fixtures """ for name, value in funcargs.items(): if name not in testargs: continue if sys.version_info >= (3, 6) and inspect.isasyncgen(value): try: testargs[name] = yield from value.__anext__() except StopAsyncIteration: raise RuntimeError("async generator didn't yield") from None elif sys.version_info >= (3, 5) and inspect.iscoroutine(value): testargs[name] = yield from value
async def send_body(self, response, send): if isinstance(response.body, str): await send({ 'type': HTTP_RESPONSE_BODY, 'body': response.body.encode('utf-8') }) elif inspect.isasyncgen(response.body): async for chunk in response.body: await send({ 'type': HTTP_RESPONSE_BODY, 'body': response.body.encode('utf-8') }) else: raise Exception('Cannot handle')
def wrapper(*args, **kwargs): result = args[0] if asyncio is not None and hasattr( result, '__await__') and asyncio.coroutines.iscoroutine(result): d = _aio_as_deferred(result) d.addCallback(iterate_spider_output) return d elif hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result): d = _aio_as_deferred(collect_asyncgen(result)) d.addCallback(iterate_spider_output) return d else: return func(*args, **kwargs)
def __init__( self, data: typing.Union[typing.Sequence, typing.Generator, typing.Callable[[int], typing.Any]], *, per_page: int = 10, formatter: typing.Callable[ ['Paginator', typing.Sequence[typing.Any]], typing.Union[str, discord.Embed, dict] ] = None): """ Args: data (typing.Union[typing.Sequence, typing.Generator, typing.Callable[[int], typing.Any]]): The data that you want to paginate. If a generator or function is given then the `max_pages` will start as the string "?", and the `per_page` parameter will be ignored - the formatter will be passed the content of whatever your generator returns. If a function is given, then you will be passed the page number as an argument - raising `StopIteration` from this function will cause the `max_pages` attribute to be set, and the page will go back to what it was previously. per_page (int, optional): The number of items that appear on each page. This argument only works for sequences formatter (typing.Callable[['Paginator', typing.Sequence[typing.Any]], typing.Union[str, discord.Embed, dict]], optional): A function taking the paginator instance and a list of things to display, returning a dictionary of kwargs that get passed directly into a :func:`discord.Message.edit`. """ self.data = data self.per_page: int = per_page self.formatter: typing.Callable[['Paginator', typing.Sequence[typing.Any]], typing.Union[str, discord.Embed, dict]] if formatter is None: self.formatter = self.default_list_formatter else: self.formatter = formatter self.current_page: int = None self._page_cache = {} self.max_pages: int = '?' self._data_is_generator = any(( inspect.isasyncgenfunction(self.data), inspect.isasyncgen(self.data), inspect.isgeneratorfunction(self.data), inspect.isgenerator(self.data), )) self._data_is_function = any(( inspect.isfunction(self.data), inspect.iscoroutine(self.data), )) self._data_is_iterable = not (self._data_is_generator or self._data_is_function) if self._data_is_iterable: pages, left_over = divmod(len(data), self.per_page) if left_over: pages += 1 self.max_pages = pages self._message = None
def compute(context, input_defs) -> Generator[Output, None, None]: kwargs = {} for input_name in input_names: kwargs[input_name] = input_defs[input_name] result = fn(context, **kwargs) if context_arg_provided else fn(**kwargs) if inspect.isgenerator(result): return result elif inspect.isasyncgen(result): return result elif inspect.iscoroutine(result): return _coerce_async_solid_to_async_gen(result, context, output_defs) else: return _coerce_solid_output_to_iterator(result, context, output_defs)
async def async_put_item(self, result): """ async put item async generator/generator/list/item """ if isasyncgen(result): async for item in result: await self.put_item(item) elif isawaitable(result): await self.put_item(await result) elif isinstance(result, list): for item in result: await self.put_item(item) else: await self.put_item(result)
def _invoke(self, args, kwargs): # This runs on a separate thread res = self._f(*args, **kwargs) if inspect.iscoroutine(res): try: loop = asyncio.new_event_loop() return loop.run_until_complete(res) finally: loop.close() elif inspect.isasyncgen(res): raise RuntimeError("Async generators are not supported") elif inspect.isgenerator(res): raise RuntimeError("Generators are not supported") else: return res
def _get_requests( self, **kwargs) -> Union[Iterator['Request'], AsyncIterator['Request']]: """Get request in generator""" _kwargs = vars(self.args) _kwargs['data'] = self.input_fn # override by the caller-specific kwargs _kwargs.update(kwargs) if inspect.isasyncgen(self.input_fn): from .request.asyncio import request_generator return request_generator(**_kwargs) else: from .request import request_generator return request_generator(**_kwargs)
async def async_transform(self, value, context=None): if not isinstance(value, AbcMapping): self._failure("value is not a dict", value=value) collect = {} errors = {} touched_names = [] for key in self.keys: if not callable(key) and not hasattr(key, 'async_call'): raise ValueError('Non callable Keys are not supported') key_run = call_with_context_if_support( getattr(key, 'async_call', key), value, context=context, ) if inspect.isasyncgen(key_run): async for k, v, names in key_run: if isinstance(v, DataError): errors[k] = v else: collect[k] = v touched_names.extend(names) else: for k, v, names in key_run: if isinstance(v, DataError): errors[k] = v else: collect[k] = v touched_names.extend(names) if not self.ignore_any: for key in value: if key in touched_names: continue if key in self.ignore: continue if not self.allow_any and key not in self.extras: errors[key] = DataError("%s is not allowed key" % key) elif key in collect: errors[key] = DataError("%s key was shadowed" % key) else: try: collect[key] = await self.extras_trafaret.async_check( value[key]) except DataError as de: errors[key] = de if errors: raise DataError(error=errors, trafaret=self) return collect
async def handle_dispatches(self, ctx: EventContext, name: str, dispatch: dict): """ Handles dispatches for the client. """ try: handle_name = name.lower() handler = getattr(self.state, f"handle_{handle_name}") except AttributeError: logger.warning(f"Got unknown dispatch {name}") return else: logger.debug(f"Processing event {name}") try: result = handler(ctx.gateway, dispatch) if inspect.isawaitable(result): result = await result elif inspect.isasyncgen(result): async with multio.finalize_agen(result) as gen: async for i in gen: await self.events.fire_event(i[0], *i[1:], gateway=ctx.gateway, client=self) # no more processing after the async gen return if not isinstance(result, tuple): await self.events.fire_event(result, gateway=ctx.gateway, client=self) else: await self.events.fire_event(result[0], *result[1:], gateway=ctx.gateway, client=self) # TODO: Change this from being an exception to being an event. except ChunkGuilds: ctx.gateway._get_chunks() except Exception: logger.exception( f"Error decoding event {event} with data {dispatch}!") await ctx.gateway.close(code=1006, reason="Internal client error") raise
async def links(self) -> T.Dict[str, T.Any]: result = {} _links = await self._links() for key, value in _links.items(): if key == 'item': key = 'item' if isinstance(value, _view.View): if key not in self.embed: result[key] = value.to_link elif inspect.isasyncgen(value): if key not in self.embed: async def g1(resources): async for resource in resources: yield resource.to_link result[key] = g1(value) elif inspect.isgenerator(value): if key not in self.embed: def g2(resources): for resource in resources: yield resource.to_link result[key] = g2(value) elif isinstance(value, collections.Mapping): if key in self.embed: _logger.info( 'Client asked to embed unembeddable object: %s', value) result[key] = value elif isinstance(value, collections.abc.Iterable): def g3(p_key, p_value): for o in p_value: if not isinstance(o, _view.View): if p_key in self.embed: _logger.info( 'Client asked to embed unembeddable object: %s', o) yield o elif p_key not in self.embed: yield o.to_link result[key] = g3(key, value) elif key not in self.embed: _logger.error("Don't know how to render object as link: %s", value) return result
def _method(*args, **kwargs): # We need to keep the original attributes so that they could be correctly # bound to the class/instance at runtime. func = getattr(cls, orig_name) coro = func(*args, **kwargs) _api_session = api_session.get() if _api_session is None: raise RuntimeError("API functions must be called " "inside the context of a valid API session") if isinstance(_api_session, AsyncSession): return coro else: if inspect.isasyncgen(coro): return _api_session.worker_thread.execute_generator(coro) else: return _api_session.worker_thread.execute(coro)
async def whatever_gen_once(any_gen, *args, **kwargs): if inspect.isasyncgenfunction(any_gen): # 如果是异步生成器函数 async for i in any_gen(*args, **kwargs): return i elif inspect.isgeneratorfunction( any_gen) and not inspect.iscoroutinefunction(any_gen): # 同步生成器 for i in any_gen(*args, **kwargs): return i elif inspect.isgenerator(any_gen): for i in any_gen: return i elif inspect.isasyncgen(any_gen): async for i in any_gen: return i
def wrapper(*args, **kwargs): memo = _CallMemo(python_func, _localns, args=args, kwargs=kwargs) check_argument_types(memo) retval = func(*args, **kwargs) check_return_type(retval, memo) # If a generator is returned, wrap it if its yield/send/return types can be checked if inspect.isgenerator(retval) or isasyncgen(retval): return_type = memo.type_hints.get('return') origin = getattr(return_type, '__origin__') if origin in generator_origin_types: return TypeCheckedGenerator(retval, memo) elif origin is not None and origin in asyncgen_origin_types: return TypeCheckedAsyncGenerator(retval, memo) return retval
def ensure_generator(value): if legacy.is_generator(value): return True, value if hasattr(value, "_generator"): return True, value if hasattr(inspect, "iscoroutine") and\ inspect.iscoroutine(value): #@UndefinedVariable return True, CoroutineWrapper(value) if hasattr(inspect, "isasyncgen") and\ inspect.isasyncgen(value): #@UndefinedVariable return True, AyncgenWrapper(value) return False, value
def __init__( self, content: typing.Any, status_code: int = 200, headers: dict = None, media_type: str = None, background: BackgroundTask = None, ) -> None: if inspect.isasyncgen(content): self.body_iterator = content else: self.body_iterator = iterate_in_threadpool(content) self.status_code = status_code self.media_type = self.media_type if media_type is None else media_type self.background = background self.init_headers(headers)
def _core_generator( solid_def: "SolidDefinition", context: "DirectSolidExecutionContext", input_dict: Dict[str, Any] ) -> Generator[Any, None, None]: from dagster.core.execution.plan.compute import gen_from_async_gen with user_code_error_boundary( DagsterSolidInvocationError, control_flow_exceptions=[RetryRequested], msg_fn=lambda: f'Error occurred while invoking solid "{solid_def.name}":', ): compute_iterator = solid_def.compute_fn(context, input_dict) if inspect.isasyncgen(compute_iterator): compute_iterator = gen_from_async_gen(compute_iterator) yield from compute_iterator
async def async_transform(self, value, context=None): if not isinstance(value, AbcMapping): self._failure("value is not a dict", value=value) collect = {} errors = {} touched_names = [] for key in self._keys: if not callable(key) and not hasattr(key, 'async_call'): raise ValueError('Non callable Keys are not supported') key_run = getattr(key, 'async_call', key)( value, context=context, ) if inspect.isasyncgen(key_run): async for k, v, names in key_run: if isinstance(v, DataError): errors[k] = v else: collect[k] = v touched_names.extend(names) else: for k, v, names in key_run: if isinstance(v, DataError): errors[k] = v else: collect[k] = v touched_names.extend(names) if not self.ignore_any: for key in value: if key in touched_names: continue if key in self.ignore: continue if not self.allow_any and key not in self.extras: errors[key] = DataError("%s is not allowed key" % key) elif key in collect: errors[key] = DataError("%s key was shadowed" % key) else: try: collect[key] = await self.extras_trafaret.async_check(value[key]) except DataError as de: errors[key] = de if errors: raise DataError(error=errors, trafaret=self) return collect
def is_async_generator(value): if not hasattr(inspect, "isasyncgen"): return False return inspect.isasyncgen(value)
def isasyncgen(obj): if hasattr(inspect, "isasyncgen"): if inspect.isasyncgen(obj): return True return isinstance(obj, AsyncGenerator)