def test_get_function_type(self) -> None: """ Test get_function_type returns the right type """ self.assertEqual( get_function_type( FunctionDef( args=arguments( args=[set_arg("something else")], arg=None, ), arguments_args=None, identifier_name=None, stmt=None, ) ), "static", ) self.assertEqual( get_function_type( FunctionDef( args=arguments(args=[], arg=None), arguments_args=None, identifier_name=None, stmt=None, ) ), "static", ) self.assertEqual( get_function_type( FunctionDef( args=arguments( args=[set_arg("self")], arg=None, ), arguments_args=None, identifier_name=None, stmt=None, ) ), "self", ) self.assertEqual( get_function_type( FunctionDef( args=arguments( args=[set_arg("cls")], arg=None, ), arguments_args=None, identifier_name=None, stmt=None, ) ), "cls", )
def _make_call_meth(body, return_type, param_names): """ Construct a `__call__` method from the provided `body` :param body: The body, probably from a FunctionDef.body :type body: ```List[AST]``` :param return_type: The return type of the parent symbol (probably class). Used to fill in `__call__` return. :type return_type: ```Optional[str]``` :param param_names: Container of AST `id`s to match for rename :type param_names: ```Optional[Iterator[str]]``` :return: Internal function for `__call__` :rtype: ```FunctionDef``` """ body_len = len(body) return_ = (ast.fix_missing_locations( RewriteName(param_names).visit( Return(get_value(ast.parse(return_type[3:-3]).body[0]), expr=None))) if return_type is not None and len(return_type) > 6 and return_type.startswith("```") and return_type.endswith("```") else None) if body_len: if isinstance(body[0], Expr): doc_str = get_value(body[0].value) if isinstance(doc_str, str) and body_len > 0: body = (body[1:] if body_len > 1 else ([ set_value(doc_str.replace(":cvar", ":param")) if return_ is None else return_ ] if body_len == 1 else body)) # elif not isinstance(body[0], Return) and return_ is not None: # body.append(return_) # elif return_ is not None: # body = [return_] return FunctionDef(args=arguments( args=[set_arg("self")], defaults=[], kw_defaults=[], kwarg=None, kwonlyargs=[], posonlyargs=[], vararg=None, arg=None, ), body=body, decorator_list=[], name="__call__", returns=None, arguments_args=None, identifier_name=None, stmt=None, lineno=None, **maybe_type_comment)
def test_find_in_ast_no_val(self) -> None: """Tests that `find_in_ast` correctly gives AST node from `def class C(object): def function_name(self,dataset_name: str,…)`""" run_ast_test( self, find_in_ast( "C.function_name.dataset_name".split("."), class_with_optional_arg_method_ast, ), set_arg( annotation=Name( "str", Load(), ), arg="dataset_name", ), )
def test_emit_arg(self) -> None: """ Tests that `arg` is emitted from `emit_arg` """ self.assertIsInstance( class_with_method_and_body_types_ast.body[1].args.args[1], arg) self.assertIsInstance( emit_arg( class_with_method_and_body_types_ast.body[1].args.args[1]), arg) assign = Assign(targets=[Name("yup", Store())], value=set_value("nup"), expr=None, **maybe_type_comment) gen_ast = emit_arg(assign) self.assertIsInstance(gen_ast, arg) run_ast_test( self, gen_ast=gen_ast, gold=set_arg("yup"), )
def test_find_in_ast_with_val(self) -> None: """Tests that `find_in_ast` correctly gives AST node from `def class C(object): def function_name(self,dataset_name: str='foo',…)`""" gen_ast = find_in_ast( "C.function_name.dataset_name".split("."), class_with_method_and_body_types_ast, ) self.assertIsInstance(gen_ast.default, Constant if PY_GTE_3_8 else Str) self.assertEqual(get_value(gen_ast.default), "~/tensorflow_datasets") run_ast_test( self, gen_ast, set_arg( annotation=Name( "str", Load(), ), arg="dataset_name", ), )
"RemoteMonitor", "TensorBoard", "TerminateOnNaN", ), action="append", help="Collection of callables that are run inside the training loop", ) return argument_parser '''.format( _cli_doc_str=_cli_doc_str, header_doc_str=docstring_header_str ) argparse_func_ast = fix_missing_locations( FunctionDef( args=arguments( args=[set_arg("argument_parser")], defaults=[], kw_defaults=[], kwarg=None, kwonlyargs=[], posonlyargs=[], vararg=None, arg=None, ), body=[ _cli_doc_expr, Assign( targets=[ Attribute( Name("argument_parser", Load()), "description",
def argparse_function( intermediate_repr, emit_default_doc=False, emit_default_doc_in_return=False, function_name="set_cli_args", function_type="static", ): """ Convert to an argparse FunctionDef :param intermediate_repr: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :type intermediate_repr: ```dict``` :param emit_default_doc: Whether help/docstring should include 'With default' text :type emit_default_doc: ```bool``` :param emit_default_doc_in_return: Whether help/docstring in return should include 'With default' text :type emit_default_doc_in_return: ```bool``` :param function_name: name of function_def :type function_name: ```str``` :param function_type: Type of function, static is static or global method, others just become first arg :type function_type: ```Literal['self', 'cls', 'static']``` :return: AST node for function definition which constructs argparse :rtype: ```FunctionDef``` """ function_name = function_name or intermediate_repr["name"] function_type = function_type or intermediate_repr["type"] internal_body = get_internal_body( target_name=function_name, target_type=function_type, intermediate_repr=intermediate_repr, ) return FunctionDef( args=arguments( args=[set_arg("argument_parser")], # None if function_type in frozenset((None, "static")) # else set_arg(function_type), defaults=[], kw_defaults=[], kwarg=None, kwonlyargs=[], posonlyargs=[], vararg=None, arg=None, ), body=list( chain.from_iterable(( iter(( Expr( set_value( "\n Set CLI arguments\n\n " ":param argument_parser: argument parser\n " ":type argument_parser: ```ArgumentParser```\n\n " "{return_params}".format(return_params=( lambda returns: ":return: argument_parser{return_doc}\n " "{rtype}".format( return_doc=", {}".format(returns["doc"]) if "doc" in returns else "", rtype=":rtype: ```ArgumentParser```\n " if intermediate_repr["returns"][ "return_type"].get("typ") in (None, "None", NoneStr) else ":rtype: ```Tuple[ArgumentParser, {returns[typ]}]```\n{tab}" "".format(returns=returns, tab=tab), ) )(returns=set_default_doc( next(iter(intermediate_repr["returns"].items()) ), emit_default_doc=emit_default_doc_in_return, )[1]) if "return_type" in ( intermediate_repr.get("returns") or iter(()) ) else ":return: argument_parser\n " ":rtype: ```ArgumentParser```\n " ))), Assign(targets=[ Attribute( Name("argument_parser", Load()), "description", Store(), ) ], value=set_value(intermediate_repr["doc"]), lineno=None, expr=None, **maybe_type_comment), )), filter( None, ( *((map( partial( param2argparse_param, emit_default_doc=emit_default_doc, ), intermediate_repr["params"].items(), )) if "params" in intermediate_repr else ()), *(internal_body[ 2 if len(internal_body) > 1 and isinstance( internal_body[1], Assign) and internal_body[1]. targets[0].id == "argument_parser" else 1:] if internal_body and isinstance(internal_body[0], Expr) and isinstance(get_value(internal_body[0].value), str) else internal_body), None if internal_body and isinstance(internal_body[-1], Return) else (Return( value=Tuple( ctx=Load(), elts=[ Name("argument_parser", Load()), set_value(intermediate_repr["returns"] ["return_type"]["default"]) if intermediate_repr["returns"]["return_type"] ["default"].startswith("```") and intermediate_repr["returns"] ["return_type"]["default"].endswith("```") else ast.parse(intermediate_repr["returns"] ["return_type"] ["default"]).body[0].value, ], expr=None, ), expr=None, ) if "default" in ( intermediate_repr.get("returns") or { "return_type": iter(()) })["return_type"] else Return(value=Name( "argument_parser", Load()), expr=None)), ), ), ))), decorator_list=[], name=function_name, returns=None, lineno=None, arguments_args=None, identifier_name=None, stmt=None, **maybe_type_comment)
def function( intermediate_repr, function_name, function_type, emit_default_doc=False, docstring_format="rest", indent_level=2, emit_separating_tab=PY3_8, inline_types=True, emit_as_kwonlyargs=True, ): """ Construct a function from our IR :param intermediate_repr: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :type intermediate_repr: ```dict``` :param function_name: name of function_def :type function_name: ```Optional[str]``` :param function_type: Type of function, static is static or global method, others just become first arg :type function_type: ```Optional[Literal['self', 'cls', 'static']]``` :param docstring_format: Format of docstring :type docstring_format: ```Literal['rest', 'numpy', 'google']``` :param emit_default_doc: Whether help/docstring should include 'With default' text :type emit_default_doc: ```bool``` :param indent_level: docstring indentation level whence: 0=no_tabs, 1=one tab; 2=two tabs :type indent_level: ```int``` :param emit_separating_tab: docstring decider for whether to put a tab between :param and return and desc :type emit_separating_tab: ```bool``` :param inline_types: Whether the type should be inline or in docstring :type inline_types: ```bool``` :param emit_as_kwonlyargs: Whether argument(s) emitted must be keyword only :type emit_as_kwonlyargs: ```bool``` :return: AST node for function definition :rtype: ```FunctionDef``` """ params_no_kwargs = tuple( filter( lambda param: not param[0].endswith("kwargs"), intermediate_repr["params"].items(), )) function_name = function_name or intermediate_repr["name"] function_type = function_type or intermediate_repr["type"] args = ([] if function_type in frozenset( (None, "static")) else [set_arg(function_type)]) from doctrans.emitter_utils import ast_parse_fix args_from_params = list( map( lambda param: set_arg( annotation=(Name(param[1]["typ"], Load()) if param[1]["typ"] in simple_types else ast_parse_fix(param[1]["typ"])) if inline_types and "typ" in param[1] else None, arg=param[0], ), params_no_kwargs, ), ) defaults_from_params = list( map( lambda param: set_value(None) if param[1].get("default") in (None, "None", NoneStr) else set_value(param[1].get("default")), params_no_kwargs, )) if emit_as_kwonlyargs: kwonlyargs, kw_defaults, defaults = args_from_params, defaults_from_params, [] else: kwonlyargs, kw_defaults, defaults = [], [], defaults_from_params args += args_from_params internal_body = get_internal_body( target_name=function_name, target_type=function_type, intermediate_repr=intermediate_repr, ) return_val = (Return( value=ast.parse(intermediate_repr["returns"]["return_type"] ["default"].strip("`")).body[0].value, expr=None, ) if (intermediate_repr.get("returns") or { "return_type": {} })["return_type"].get("default") else None) return FunctionDef( args=arguments( args=args, defaults=defaults, kw_defaults=kw_defaults, kwarg=next( map( lambda param: set_arg(param[0]), filter( lambda param: param[0].endswith("kwargs"), intermediate_repr["params"].items(), ), ), None, ), kwonlyargs=kwonlyargs, posonlyargs=[], vararg=None, arg=None, ), body=list( filter( None, ( Expr( set_value( to_docstring( intermediate_repr, emit_default_doc=emit_default_doc, docstring_format=docstring_format, emit_types=not inline_types, indent_level=indent_level, emit_separating_tab=emit_separating_tab, ))), *(internal_body[:-1] if internal_body and isinstance(internal_body[-1], Return) and return_val else internal_body), return_val, ), )), decorator_list=[], name=function_name, returns=(ast.parse( intermediate_repr["returns"]["return_type"]["typ"]).body[0].value if inline_types and (intermediate_repr.get("returns") or { "return_type": {} })["return_type"].get("typ") else None), lineno=None, arguments_args=None, identifier_name=None, stmt=None, **maybe_type_comment)
def _make_call_meth(body, return_type, param_names, docstring_format, word_wrap): """ Construct a `__call__` method from the provided `body` :param body: The body, probably from a FunctionDef.body :type body: ```List[AST]``` :param return_type: The return type of the parent symbol (probably class). Used to fill in `__call__` return. :type return_type: ```Optional[str]``` :param param_names: Container of AST `id`s to match for rename :type param_names: ```Optional[Iterator[str]]``` :param docstring_format: Format of docstring :type docstring_format: ```Literal['rest', 'numpydoc', 'google']``` :param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length. :type word_wrap: ```bool``` :returns: Internal function for `__call__` :rtype: ```FunctionDef``` """ body_len = len(body) # return_ = ( # ast.fix_missing_locations( # RewriteName(param_names).visit( # Return(get_value(ast.parse(return_type.strip("`")).body[0]), expr=None) # ) # ) # if return_type is not None and code_quoted(return_type) # else None # ) if body_len: if isinstance(body, dict): body = list( filter( None, ( None if body.get("doc") in none_types else Expr( set_value( emit_param_str( ( "return_type", { "doc": multiline( indent_all_but_first( body["doc"])) }, ), style=docstring_format, word_wrap=word_wrap, ))), RewriteName(param_names).visit( Return( get_value( ast.parse(return_type.strip("`")).body[0]), expr=None, )) if code_quoted(body["default"]) else Return( set_value(body["default"]), expr=None), ), )) # elif isinstance(body[0], Expr): # doc_str = get_value(body[0].value) # if isinstance(doc_str, str) and body_len > 0: # body = ( # body[1:] # if body_len > 1 # else ( # [ # set_value(doc_str.replace(":cvar", ":param")) # if return_ is None # else return_ # ] # if body_len == 1 # else body # ) # ) # elif not isinstance(body[0], Return) and return_ is not None: # body.append(return_) # elif return_ is not None: # body = [return_] return (ast.fix_missing_locations( FunctionDef(args=arguments( args=[set_arg("self")], defaults=[], kw_defaults=[], kwarg=None, kwonlyargs=[], posonlyargs=[], vararg=None, arg=None, ), body=body, decorator_list=[], name="__call__", returns=None, arguments_args=None, identifier_name=None, stmt=None, lineno=None, **maybe_type_comment)) if body else None)
''' class_with_method_and_body_types_ast = fix_missing_locations( ClassDef( name="C", bases=[Name("object", Load())], keywords=[], body=[ Expr(set_value(" C class (mocked!) ")), FunctionDef( name="function_name", args=arguments( posonlyargs=[], vararg=None, args=[ set_arg("self"), set_arg(arg="dataset_name", annotation=Name("str", Load())), set_arg( arg="tfds_dir", annotation=Subscript( Name("Optional", Load()), set_slice(Name("str", Load())), Load(), ), ), set_arg( arg="K", annotation=Subscript( Name("Literal", Load()), set_slice(
expr_annotation=None, ), AnnAssign( annotation=Name("str", Load()), simple=1, target=Name("return_type", Store()), value=set_value( "```K.mean(math_ops.square(math_ops.maximum(1.0 - y_true * y_pred, 0.0)), axis=-1)```" ), expr=None, expr_target=None, expr_annotation=None, ), FunctionDef( args=arguments( args=[set_arg("self")], defaults=[], kw_defaults=[], kwarg=None, kwonlyargs=[], vararg=None, arg=None, posonlyargs=[], ), body=[ Assign( targets=[ Attribute( Name("self", Load()), "y_pred", Load(),
def argparse_function( intermediate_repr, emit_default_doc=False, function_name="set_cli_args", function_type="static", wrap_description=False, word_wrap=True, docstring_format="rest", ): """ Convert to an argparse FunctionDef :param intermediate_repr: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :type intermediate_repr: ```dict``` :param emit_default_doc: Whether help/docstring should include 'With default' text :type emit_default_doc: ```bool``` :param function_name: name of function_def :type function_name: ```str``` :param function_type: Type of function, static is static or global method, others just become first arg :type function_type: ```Literal['self', 'cls', 'static']``` :param wrap_description: Whether to word-wrap the description. Set `DOCTRANS_LINE_LENGTH` to configure length. :type wrap_description: ```bool``` :param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length. :type word_wrap: ```bool``` :param docstring_format: Format of docstring :type docstring_format: ```Literal['rest', 'numpydoc', 'google']``` :returns: AST node for function definition which constructs argparse :rtype: ```FunctionDef``` """ function_name = function_name or intermediate_repr["name"] function_type = function_type or intermediate_repr["type"] internal_body = get_internal_body( target_name=function_name, target_type=function_type, intermediate_repr=intermediate_repr, ) return FunctionDef( args=arguments( args=[set_arg("argument_parser")], # None if function_type in frozenset((None, "static")) # else set_arg(function_type), defaults=[], kw_defaults=[], kwarg=None, kwonlyargs=[], posonlyargs=[], vararg=None, arg=None, ), body=list( chain.from_iterable( ( iter( ( Expr( set_value( indent( docstring( { "doc": "Set CLI arguments", "params": OrderedDict( ( ( "argument_parser", { "doc": "argument parser", "typ": "ArgumentParser", }, ), ) ), "returns": OrderedDict( ( ( "return_type", { "doc": "argument_parser, {}".format( intermediate_repr[ "returns" ]["return_type"][ "doc" ] ) if intermediate_repr[ "returns" ]["return_type"].get( "doc" ) else "argument_parser", "typ": "Tuple[ArgumentParser, {typ}]".format( typ=intermediate_repr[ "returns" ][ "return_type" ][ "typ" ] ), } if "return_type" in ( ( intermediate_repr or {} ).get("returns") or iter(()) ) and intermediate_repr[ "returns" ]["return_type"].get("typ") not in none_types else { "doc": "argument_parser", "typ": "ArgumentParser", }, ), ), ), }, docstring_format=docstring_format, word_wrap=word_wrap, ), tab, ) + tab ) ), Assign( targets=[ Attribute( Name("argument_parser", Load()), "description", Store(), ) ], value=set_value( (fill if wrap_description else identity)( intermediate_repr["doc"] ) ), lineno=None, expr=None, **maybe_type_comment ), ) ), filter( None, ( *( ( map( partial( param2argparse_param, word_wrap=word_wrap, emit_default_doc=emit_default_doc, ), intermediate_repr["params"].items(), ) ) if "params" in intermediate_repr else () ), *( internal_body[ 2 if len(internal_body) > 1 and isinstance(internal_body[1], Assign) and internal_body[1].targets[0].id == "argument_parser" else 1 : ] if internal_body and isinstance(internal_body[0], Expr) and isinstance(get_value(internal_body[0].value), str) else internal_body ), None if internal_body and isinstance(internal_body[-1], Return) else ( Return( value=Tuple( ctx=Load(), elts=[ Name("argument_parser", Load()), set_value( intermediate_repr["returns"][ "return_type" ]["default"] ) if code_quoted( intermediate_repr["returns"][ "return_type" ]["default"] ) else ast.parse( intermediate_repr["returns"][ "return_type" ]["default"] ) .body[0] .value, ], expr=None, ), expr=None, ) if "default" in ( intermediate_repr.get("returns") or {"return_type": iter(())} )["return_type"] else Return( value=Name("argument_parser", Load()), expr=None ) ), ), ), ) ) ), decorator_list=[], name=function_name, returns=None, lineno=None, arguments_args=None, identifier_name=None, stmt=None, **maybe_type_comment )