def sqlalchemy_table(call_or_name): """ Parse out a `sqlalchemy.Table`, or a `name = sqlalchemy.Table`, into the IR :param call_or_name: The call to `sqlalchemy.Table` or an assignment followed by the call :type call_or_name: ```Union[AnnAssign, Assign, Call]``` :returns: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :rtype: ```dict``` """ if isinstance(call_or_name, Assign): name, call_or_name = call_or_name.targets[0].id, call_or_name.value elif isinstance(call_or_name, AnnAssign): name, call_or_name = call_or_name.target.id, call_or_name.value else: if not isinstance(call_or_name, Call): call_or_name = get_value(call_or_name) name = get_value(call_or_name.args[0]) # Binding should be same name as table… I guess? assert_equal(get_value(call_or_name.args[0]), name) comment = next( map( get_value, map(get_value, filter(lambda kw: kw.arg == "comment", call_or_name.keywords)), ), None, ) intermediate_repr = ({ "type": None, "doc": "", "params": OrderedDict() } if comment is None else docstring(comment)) intermediate_repr["name"] = name assert isinstance(call_or_name, Call) assert_equal(call_or_name.func.id.rpartition(".")[2], "Table") assert len(call_or_name.args) > 2 merge_ir = { "params": OrderedDict(map(column_call_to_param, call_or_name.args[2:])), "returns": None, } ir_merge(target=intermediate_repr, other=merge_ir) if intermediate_repr["returns"] and intermediate_repr["returns"].get( "return_type", {}).get("doc"): intermediate_repr["returns"]["return_type"]["doc"] = extract_default( intermediate_repr["returns"]["return_type"]["doc"], emit_default_doc=False)[0] return intermediate_repr
def _make_call_meth(body, return_type, param_names): """ Construct a `__call__` method from the provided `body` :param body: The body, probably from a FunctionDef.body :type body: ```List[AST]``` :param return_type: The return type of the parent symbol (probably class). Used to fill in `__call__` return. :type return_type: ```Optional[str]``` :param param_names: Container of AST `id`s to match for rename :type param_names: ```Optional[Iterator[str]]``` :return: Internal function for `__call__` :rtype: ```FunctionDef``` """ body_len = len(body) return_ = (ast.fix_missing_locations( RewriteName(param_names).visit( Return(get_value(ast.parse(return_type[3:-3]).body[0]), expr=None))) if return_type is not None and len(return_type) > 6 and return_type.startswith("```") and return_type.endswith("```") else None) if body_len: if isinstance(body[0], Expr): doc_str = get_value(body[0].value) if isinstance(doc_str, str) and body_len > 0: body = (body[1:] if body_len > 1 else ([ set_value(doc_str.replace(":cvar", ":param")) if return_ is None else return_ ] if body_len == 1 else body)) # elif not isinstance(body[0], Return) and return_ is not None: # body.append(return_) # elif return_ is not None: # body = [return_] return FunctionDef(args=arguments( args=[set_arg("self")], defaults=[], kw_defaults=[], kwarg=None, kwonlyargs=[], posonlyargs=[], vararg=None, arg=None, ), body=body, decorator_list=[], name="__call__", returns=None, arguments_args=None, identifier_name=None, stmt=None, lineno=None, **maybe_type_comment)
def test_get_value(self) -> None: """ Tests get_value succeeds """ val = "foo" self.assertEqual( get_value(Str(s=val, constant_value=None, string=None)), val) self.assertEqual( get_value(Constant(value=val, constant_value=None, string=None)), val) self.assertIsInstance(get_value(Tuple(expr=None)), Tuple) self.assertIsInstance(get_value(Tuple(expr=None)), Tuple) self.assertIsNone(get_value(Name(None, None)))
def _parse_return(e, intermediate_repr, function_def, emit_default_doc): """ Parse return into a param dict :param e: Return AST node :type e: Return :param intermediate_repr: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :type intermediate_repr: ```dict``` :param function_def: AST node for function definition :type function_def: ```FunctionDef``` :param emit_default_doc: Whether help/docstring should include 'With default' text :type emit_default_doc: ```bool``` :returns: Name, dict with keys: 'typ', 'doc', 'default' :rtype: ```Tuple[str, dict]``` """ assert isinstance(e, Return) return set_default_doc( ( "return_type", { "doc": extract_default( next( line.partition(",")[2].lstrip() for line in get_value( function_def.body[0].value).split("\n") if line.lstrip().startswith(":return")), emit_default_doc=emit_default_doc, )[0], "default": to_code(e.value.elts[1]).rstrip("\n"), "typ": to_code( get_value( ast.parse( intermediate_repr["returns"]["return_type"] ["typ"]).body[0].value.slice).elts[1]).rstrip() # 'Tuple[ArgumentParser, {typ}]'.format(typ=intermediate_repr['returns']['typ']) }, ), emit_default_doc=emit_default_doc, )
def _infer_default(_param, infer_type): """ Internal function to infer the default. Not intended for use by more than [the current] one function. :param _param: dict with keys: 'typ', 'doc', 'default' :type _param: ```dict``` :param infer_type: Whether to try inferring the typ (from the default) :type infer_type: ```bool``` """ if isinstance(_param["default"], (ast.Str, ast.Num, ast.Constant, ast.NameConstant)): _param["default"] = get_value(_param["default"]) if _param.get("default", False) in (None, "None"): _param["default"] = NoneStr if (infer_type and _param.get("typ") is None and _param["default"] not in (None, "None", NoneStr)): _param["typ"] = type(_param["default"]).__name__ if needs_quoting(_param.get("typ")) or isinstance(_param["default"], str): _param["default"] = unquote(_param["default"]) elif isinstance(_param["default"], AST): _param["default"] = "```{default}```".format( default=paren_wrap_code(to_code(_param["default"]).rstrip("\n"))) if _param.get("typ") is None and _param["default"]: _param["typ"] = type(_param["default"]).__name__ if (isinstance(_param["default"], str) and _param["default"].startswith("```") and _param["default"].endswith("```") and "[" not in _param[ "typ"] # Skip if you've actually formed a proper type ): del _param["typ"] # Could make it `object` I suppose…
def test_parse_to_scalar(self) -> None: """ Test various inputs and outputs for `parse_to_scalar` """ for fst, snd in ( (5, 5), ("5", "5"), (set_value(5), 5), (ast.Expr(None), NoneStr), ): self.assertEqual(parse_to_scalar(fst), snd) self.assertEqual( get_value(parse_to_scalar(ast.parse("[5]").body[0]).elts[0]), 5 ) self.assertTrue( cmp_ast( parse_to_scalar(ast.parse("[5]").body[0]), List([set_value(5)], Load()), ) ) self.assertEqual(parse_to_scalar(ast.parse("[5]")), "[5]") parse_to_scalar(ast.parse("[5]").body[0]) self.assertRaises(NotImplementedError, parse_to_scalar, memoryview(b"")) self.assertRaises(NotImplementedError, parse_to_scalar, memoryview(b""))
def param2json_schema_property(param, required): """ Turn a param into a JSON schema property :param param: Name, dict with keys: 'typ', 'doc', 'default' :type param: ```Tuple[str, dict]``` :param required: Required parameters. This function may push to the list. :type required: ```List[str]``` :returns: JSON schema property. Also may push to `required`. :rtype: ```dict``` """ name, _param = param del param if _param.get("doc"): _param["description"] = _param.pop("doc") if _param.get("typ", ast) is not ast: _param["type"] = _param.pop("typ") if _param["type"].startswith("Optional["): _param["type"] = _param["type"][len("Optional["):-1] else: required.append(name) if _param["type"].startswith("Literal["): parsed_typ = get_value(ast.parse(_param["type"]).body[0]) assert ( parsed_typ.value.id == "Literal" ), "Only basic Literal support is implemented, not {}".format( parsed_typ.value.id) _param["enum"] = list( map(get_value, get_value(parsed_typ.slice).elts)) _param["type"] = typ2json_type[type(_param["enum"][0]).__name__] else: _param["type"] = typ2json_type[_param["type"]] if _param.get("default", False) in none_types: del _param["default"] # Will be inferred as `null` from the type return name, _param
def test_param2argparse_param_default_class_str(self) -> None: """ Tests that param2argparse_param works to change the type based on the default whence said default is a proxy for an unexpected type """ self.assertEqual( get_value( param2argparse_param( ( "byo", {"default": 5.5, "typ": "<class 'float'>"}, ), ) .value.keywords[0] .value ), "float", )
def test_from_function(self) -> None: """ Tests that parse.function produces properly """ gen_ir = parse.function(function_default_complex_default_arg_ast) gold_ir = { "name": "call_peril", "params": OrderedDict(( ( "dataset_name", { "default": "mnist", "typ": "str" }, ), ( "writer", { "default": "```{}```".format( paren_wrap_code( get_value( function_default_complex_default_arg_ast. args.defaults[1]))), }, ), )), "returns": None, "type": "static", } del gen_ir["_internal"] # Not needed for this test self.assertDictEqual( gen_ir, gold_ir, )
def test_find_in_ast_with_val(self) -> None: """Tests that `find_in_ast` correctly gives AST node from `def class C(object): def function_name(self,dataset_name: str='foo',…)`""" gen_ast = find_in_ast( "C.function_name.dataset_name".split("."), class_with_method_and_body_types_ast, ) self.assertIsInstance(gen_ast.default, Constant if PY_GTE_3_8 else Str) self.assertEqual(get_value(gen_ast.default), "~/tensorflow_datasets") run_ast_test( self, gen_ast, set_arg( annotation=Name( "str", Load(), ), arg="dataset_name", ), )
def _handle_keyword(keyword, typ): """ Decide which type to wrap the keyword tuples in :param keyword: AST keyword :type keyword: ```ast.keyword``` :param typ: string representation of type :type typ: ```str``` :returns: string representation of type :rtype: ```str``` """ quote_f = identity type_ = "Union" if typ == Any or typ in simple_types: if typ in ("str", Any): def quote_f(s): """ Wrap the input in quotes :param s: Any value :type s: ```Any``` :returns: the input value :rtype: ```Any``` """ return "'{}'".format(s) type_ = "Literal" return "{type}[{types}]".format( type=type_, types=", ".join(quote_f(get_value(elt)) for elt in keyword.value.elts), )
def _infer_default(_param, infer_type): """ Internal function to infer the default. Not intended for use by more than [the current] one function. :param _param: dict with keys: 'typ', 'doc', 'default' :type _param: ```dict``` :param infer_type: Whether to try inferring the typ (from the default) :type infer_type: ```bool``` """ if isinstance(_param["default"], (ast.Str, ast.Num, ast.Constant, ast.NameConstant)): _param["default"] = get_value(_param["default"]) if _param.get("default", False) in none_types: _param["default"] = NoneStr if infer_type and _param.get( "typ") is None and _param["default"] not in none_types: _param["typ"] = type(_param["default"]).__name__ if needs_quoting(_param.get("typ")) or isinstance(_param["default"], str): _param["default"] = unquote(_param["default"]) elif isinstance(_param["default"], AST): try: _param["default"] = ast.literal_eval(_param["default"]) # if _param.get("typ") is None or _param["typ"] == "UnaryOp": # _param["typ"] = type(_param["default"]).__name__ except ValueError: _param["default"] = "```{default}```".format( default=paren_wrap_code( to_code(_param["default"]).rstrip("\n"))) if _param.get("typ") is None and _param["default"] != NoneStr: _param["typ"] = type(_param["default"]).__name__ if (_param["default"] != NoneStr and code_quoted(_param["default"]) and "[" not in _param.get("typ", iter( ())) # Skip if you've actually formed a proper type ): del _param["typ"] # Could make it `object` I suppose…
def argparse_function( intermediate_repr, emit_default_doc=False, function_name="set_cli_args", function_type="static", wrap_description=False, word_wrap=True, docstring_format="rest", ): """ Convert to an argparse FunctionDef :param intermediate_repr: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :type intermediate_repr: ```dict``` :param emit_default_doc: Whether help/docstring should include 'With default' text :type emit_default_doc: ```bool``` :param function_name: name of function_def :type function_name: ```str``` :param function_type: Type of function, static is static or global method, others just become first arg :type function_type: ```Literal['self', 'cls', 'static']``` :param wrap_description: Whether to word-wrap the description. Set `DOCTRANS_LINE_LENGTH` to configure length. :type wrap_description: ```bool``` :param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length. :type word_wrap: ```bool``` :param docstring_format: Format of docstring :type docstring_format: ```Literal['rest', 'numpydoc', 'google']``` :returns: AST node for function definition which constructs argparse :rtype: ```FunctionDef``` """ function_name = function_name or intermediate_repr["name"] function_type = function_type or intermediate_repr["type"] internal_body = get_internal_body( target_name=function_name, target_type=function_type, intermediate_repr=intermediate_repr, ) return FunctionDef( args=arguments( args=[set_arg("argument_parser")], # None if function_type in frozenset((None, "static")) # else set_arg(function_type), defaults=[], kw_defaults=[], kwarg=None, kwonlyargs=[], posonlyargs=[], vararg=None, arg=None, ), body=list( chain.from_iterable( ( iter( ( Expr( set_value( indent( docstring( { "doc": "Set CLI arguments", "params": OrderedDict( ( ( "argument_parser", { "doc": "argument parser", "typ": "ArgumentParser", }, ), ) ), "returns": OrderedDict( ( ( "return_type", { "doc": "argument_parser, {}".format( intermediate_repr[ "returns" ]["return_type"][ "doc" ] ) if intermediate_repr[ "returns" ]["return_type"].get( "doc" ) else "argument_parser", "typ": "Tuple[ArgumentParser, {typ}]".format( typ=intermediate_repr[ "returns" ][ "return_type" ][ "typ" ] ), } if "return_type" in ( ( intermediate_repr or {} ).get("returns") or iter(()) ) and intermediate_repr[ "returns" ]["return_type"].get("typ") not in none_types else { "doc": "argument_parser", "typ": "ArgumentParser", }, ), ), ), }, docstring_format=docstring_format, word_wrap=word_wrap, ), tab, ) + tab ) ), Assign( targets=[ Attribute( Name("argument_parser", Load()), "description", Store(), ) ], value=set_value( (fill if wrap_description else identity)( intermediate_repr["doc"] ) ), lineno=None, expr=None, **maybe_type_comment ), ) ), filter( None, ( *( ( map( partial( param2argparse_param, word_wrap=word_wrap, emit_default_doc=emit_default_doc, ), intermediate_repr["params"].items(), ) ) if "params" in intermediate_repr else () ), *( internal_body[ 2 if len(internal_body) > 1 and isinstance(internal_body[1], Assign) and internal_body[1].targets[0].id == "argument_parser" else 1 : ] if internal_body and isinstance(internal_body[0], Expr) and isinstance(get_value(internal_body[0].value), str) else internal_body ), None if internal_body and isinstance(internal_body[-1], Return) else ( Return( value=Tuple( ctx=Load(), elts=[ Name("argument_parser", Load()), set_value( intermediate_repr["returns"][ "return_type" ]["default"] ) if code_quoted( intermediate_repr["returns"][ "return_type" ]["default"] ) else ast.parse( intermediate_repr["returns"][ "return_type" ]["default"] ) .body[0] .value, ], expr=None, ), expr=None, ) if "default" in ( intermediate_repr.get("returns") or {"return_type": iter(())} )["return_type"] else Return( value=Name("argument_parser", Load()), expr=None ) ), ), ), ) ) ), decorator_list=[], name=function_name, returns=None, lineno=None, arguments_args=None, identifier_name=None, stmt=None, **maybe_type_comment )
def function(function_def, infer_type=False, function_type=None, function_name=None): """ Converts a method to our IR :param function_def: AST node for function definition :type function_def: ```Union[FunctionDef, FunctionType]``` :param infer_type: Whether to try inferring the typ (from the default) :type infer_type: ```bool``` :param function_type: Type of function, static is static or global method, others just become first arg :type function_type: ```Literal['self', 'cls', 'static']``` :param function_name: name of function_def :type function_name: ```str``` :return: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :rtype: ```dict``` """ if isinstance(function_def, FunctionType): # Dynamic function, i.e., this isn't source code; and is in your memory ir = _inspect(function_def, function_name) parsed_source = ast.parse(getsource(function_def).lstrip()).body[0] ir["_internal"] = { "body": list( filterfalse(rpartial(isinstance, AnnAssign), parsed_source.body) ), "from_name": parsed_source.name, "from_type": "cls", } return ir assert isinstance( function_def, FunctionDef ), "Expected 'FunctionDef' got `{!r}`".format(type(function_def).__name__) assert ( function_name is None or function_def.name == function_name ), "Expected {!r} got {!r}".format(function_name, function_def.name) found_type = get_function_type(function_def) # Read docstring intermediate_repr_docstring = ( get_docstring(function_def) if isinstance(function_def, FunctionDef) else None ) if intermediate_repr_docstring is None: intermediate_repr = { "name": function_name or function_def.name, "params": OrderedDict( map( func_arg2param, function_def.args.args if found_type == "static" else function_def.args.args[1:], ) ), "returns": None, } else: intermediate_repr = docstring( intermediate_repr_docstring.replace(":cvar", ":param"), infer_type=infer_type, ) intermediate_repr.update( { "name": function_name or function_def.name, "type": function_type or found_type, } ) if function_def.body: intermediate_repr["_internal"] = { "body": function_def.body if intermediate_repr_docstring is None else function_def.body[1:], "from_name": function_def.name, "from_type": found_type, } params_to_append = OrderedDict() if ( hasattr(function_def.args, "kwarg") and function_def.args.kwarg and function_def.args.kwarg.arg in intermediate_repr["params"] ): _param = intermediate_repr["params"].pop(function_def.args.kwarg.arg) assert "typ" in _param _param["default"] = NoneStr # if "typ" not in _param: # _param["typ"] = ( # "Optional[dict]" # if function_def.args.kwarg.annotation is None # else to_code(function_def.args.kwarg.annotation).rstrip("\n") # ) params_to_append[function_def.args.kwarg.arg] = _param del _param idx = count() # Set defaults if intermediate_repr["params"]: deque( map( lambda args_defaults: deque( map( lambda idxparam_idx_arg: intermediate_repr["params"][ idxparam_idx_arg[2].arg ].update( dict( ( {} if getattr(idxparam_idx_arg[2], "annotation", None) is None else dict( typ=to_code( idxparam_idx_arg[2].annotation ).rstrip("\n") ) ), **( lambda _defaults: dict( default=( lambda v: ( _defaults[idxparam_idx_arg[1]] if isinstance( _defaults[idxparam_idx_arg[1]], (NameConstant, Constant), ) else v ) if v is None else v )(get_value(_defaults[idxparam_idx_arg[1]])) ) if idxparam_idx_arg[1] < len(_defaults) and _defaults[idxparam_idx_arg[1]] is not None else {} )(getattr(function_def.args, args_defaults[1])), ) ), ( lambda _args: map( lambda idx_arg: (next(idx), idx_arg[0], idx_arg[1]), enumerate( filterfalse( lambda _arg: _arg.arg[0] == "*", ( _args if found_type == "static" or args_defaults[0] == "kwonlyargs" else _args[1:] ), ) ), ) )(getattr(function_def.args, args_defaults[0])), ), maxlen=0, ), (("args", "defaults"), ("kwonlyargs", "kw_defaults")), ), maxlen=0, ) intermediate_repr["params"].update(params_to_append) intermediate_repr["params"] = OrderedDict( map( partial(_set_name_and_type, infer_type=infer_type), intermediate_repr["params"].items(), ) ) # Convention - the final top-level `return` is the default intermediate_repr = _interpolate_return(function_def, intermediate_repr) if "return_type" in (intermediate_repr.get("returns") or iter(())): intermediate_repr["returns"] = OrderedDict( map( partial(_set_name_and_type, infer_type=infer_type), intermediate_repr["returns"].items(), ) ) return intermediate_repr
def sqlalchemy(class_def): """ Parse out a `class C(Base): __tablename__= 'tbl'; dataset_name = Column(String, doc="p", primary_key=True)`, as constructed on an SQLalchemy declarative `Base`. :param class_def: A class inheriting from declarative `Base`, where `Base = sqlalchemy.orm.declarative_base()` :type class_def: ```Union[ClassDef]``` :returns: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :rtype: ```dict``` """ assert isinstance(class_def, ClassDef) # Parse into the same format that `sqlalchemy_table` can read, then return with a call to it name = get_value( next( filter( lambda assign: any( filter( partial(eq, "__tablename__"), map(attrgetter("id"), assign.targets), )), filter(rpartial(isinstance, Assign), class_def.body), )).value) doc_string = get_docstring(class_def) def _merge_name_to_column(assign): """ Merge `a = Column()` into `Column("a")` :param assign: Of form `a = Column()` :type assign: ```Assign``` :returns: Unwrapped Call with name prepended :rtype: ```Call``` """ assign.value.args.insert(0, set_value(assign.targets[0].id)) return assign.value return sqlalchemy_table( Call( func=Name("Table", Load()), args=list( chain.from_iterable(( iter((set_value(name), Name("metadata", Load()))), map( _merge_name_to_column, filterfalse( lambda assign: any( map( lambda target: target.id == "__tablename__" or hasattr(target, "value") and isinstance( target.value, Call) and target.func. rpartition(".")[2] == "Column", assign.targets, ), ), filter(rpartial(isinstance, Assign), class_def.body), ), ), ))), keywords=[] if doc_string is None else [ keyword(arg="comment", value=set_value(doc_string), identifier=None) ], expr=None, expr_func=None, ))
def argparse_function( intermediate_repr, emit_default_doc=False, emit_default_doc_in_return=False, function_name="set_cli_args", function_type="static", ): """ Convert to an argparse FunctionDef :param intermediate_repr: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :type intermediate_repr: ```dict``` :param emit_default_doc: Whether help/docstring should include 'With default' text :type emit_default_doc: ```bool``` :param emit_default_doc_in_return: Whether help/docstring in return should include 'With default' text :type emit_default_doc_in_return: ```bool``` :param function_name: name of function_def :type function_name: ```str``` :param function_type: Type of function, static is static or global method, others just become first arg :type function_type: ```Literal['self', 'cls', 'static']``` :return: AST node for function definition which constructs argparse :rtype: ```FunctionDef``` """ function_name = function_name or intermediate_repr["name"] function_type = function_type or intermediate_repr["type"] internal_body = get_internal_body( target_name=function_name, target_type=function_type, intermediate_repr=intermediate_repr, ) return FunctionDef( args=arguments( args=[set_arg("argument_parser")], # None if function_type in frozenset((None, "static")) # else set_arg(function_type), defaults=[], kw_defaults=[], kwarg=None, kwonlyargs=[], posonlyargs=[], vararg=None, arg=None, ), body=list( chain.from_iterable(( iter(( Expr( set_value( "\n Set CLI arguments\n\n " ":param argument_parser: argument parser\n " ":type argument_parser: ```ArgumentParser```\n\n " "{return_params}".format(return_params=( lambda returns: ":return: argument_parser{return_doc}\n " "{rtype}".format( return_doc=", {}".format(returns["doc"]) if "doc" in returns else "", rtype=":rtype: ```ArgumentParser```\n " if intermediate_repr["returns"][ "return_type"].get("typ") in (None, "None", NoneStr) else ":rtype: ```Tuple[ArgumentParser, {returns[typ]}]```\n{tab}" "".format(returns=returns, tab=tab), ) )(returns=set_default_doc( next(iter(intermediate_repr["returns"].items()) ), emit_default_doc=emit_default_doc_in_return, )[1]) if "return_type" in ( intermediate_repr.get("returns") or iter(()) ) else ":return: argument_parser\n " ":rtype: ```ArgumentParser```\n " ))), Assign(targets=[ Attribute( Name("argument_parser", Load()), "description", Store(), ) ], value=set_value(intermediate_repr["doc"]), lineno=None, expr=None, **maybe_type_comment), )), filter( None, ( *((map( partial( param2argparse_param, emit_default_doc=emit_default_doc, ), intermediate_repr["params"].items(), )) if "params" in intermediate_repr else ()), *(internal_body[ 2 if len(internal_body) > 1 and isinstance( internal_body[1], Assign) and internal_body[1]. targets[0].id == "argument_parser" else 1:] if internal_body and isinstance(internal_body[0], Expr) and isinstance(get_value(internal_body[0].value), str) else internal_body), None if internal_body and isinstance(internal_body[-1], Return) else (Return( value=Tuple( ctx=Load(), elts=[ Name("argument_parser", Load()), set_value(intermediate_repr["returns"] ["return_type"]["default"]) if intermediate_repr["returns"]["return_type"] ["default"].startswith("```") and intermediate_repr["returns"] ["return_type"]["default"].endswith("```") else ast.parse(intermediate_repr["returns"] ["return_type"] ["default"]).body[0].value, ], expr=None, ), expr=None, ) if "default" in ( intermediate_repr.get("returns") or { "return_type": iter(()) })["return_type"] else Return(value=Name( "argument_parser", Load()), expr=None)), ), ), ))), decorator_list=[], name=function_name, returns=None, lineno=None, arguments_args=None, identifier_name=None, stmt=None, **maybe_type_comment)
def parse_out_param(expr, require_default=False, emit_default_doc=True): """ Turns the class_def repr of '--dataset_name', type=str, help='name of dataset.', required=True, default='mnist' into Tuple[Literal['dataset_name'], {"typ": Literal["str"], "doc": Literal["name of dataset."], "default": Literal["mnist"]}] :param expr: Expr :type expr: ```Expr``` :param require_default: Whether a default is required, if not found in doc, infer the proper default from type :type require_default: ```bool``` :param emit_default_doc: Whether help/docstring should include 'With default' text :type emit_default_doc: ```bool``` :returns: Name, dict with keys: 'typ', 'doc', 'default' :rtype: ```Tuple[str, dict]``` """ required = get_value( get_value( next( (keyword for keyword in expr.value.keywords if keyword.arg == "required"), set_value(False), ))) typ = next( (_handle_value(get_value(key_word)) for key_word in expr.value.keywords if key_word.arg == "type"), "str", ) name = get_value(expr.value.args[0])[len("--"):] default = next( (get_value(key_word.value) for key_word in expr.value.keywords if key_word.arg == "default"), None, ) doc = (lambda help_: help_ if help_ is None else (help_ if default is None or emit_default_doc is False or (hasattr(default, "__len__") and len(default) == 0 ) or "defaults to" in help_ or "Defaults to" in help_ else "{help} Defaults to {default}".format( help=help_ if help_.endswith(".") else "{}.".format(help_), default=default, )))(next( (get_value(key_word.value) for key_word in expr.value.keywords if key_word.arg == "help" and key_word.value), None, )) if default is None: doc, default = extract_default(doc, emit_default_doc=emit_default_doc) if default is None: if required: # if name.endswith("kwargs"): # default = NoneStr # else: default = simple_types[typ] if typ in simple_types else NoneStr elif require_default or typ.startswith("Optional"): default = NoneStr # nargs = next( # ( # get_value(key_word.value) # for key_word in expr.value.keywords # if key_word.arg == "nargs" # ), # None, # ) action = next( (get_value(key_word.value) for key_word in expr.value.keywords if key_word.arg == "action"), None, ) typ = next( (_handle_keyword(keyword, typ) for keyword in expr.value.keywords if keyword.arg == "choices"), typ, ) if action == "append": typ = "List[{typ}]".format(typ=typ) if not required and "Optional" not in typ: typ = "Optional[{typ}]".format(typ=typ) # if "str" in typ or "Literal" in typ and (typ.count("'") > 1 or typ.count('"') > 1): # default = quote(default) return name, dict(doc=doc, typ=typ, **({} if default is None else { "default": default }))
def class_( class_def, class_name=None, merge_inner_function=None, infer_type=False, word_wrap=True, ): """ Converts an AST to our IR :param class_def: Class AST or Module AST with a ClassDef inside :type class_def: ```Union[Module, ClassDef]``` :param class_name: Name of `class`. If None, gives first found. :type class_name: ```Optional[str]``` :param merge_inner_function: Name of inner function to merge. If None, merge nothing. :type merge_inner_function: ```Optional[str]``` :param infer_type: Whether to try inferring the typ (from the default) :type infer_type: ```bool``` :param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length. :type word_wrap: ```bool``` :returns: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :rtype: ```dict``` """ assert not isinstance(class_def, FunctionDef) is_supported_ast_node = isinstance(class_def, (Module, ClassDef)) if not is_supported_ast_node and isinstance(class_def, type): ir = _inspect(class_def, class_name, word_wrap) parsed_body = ast.parse(getsource(class_def).lstrip()).body[0] parsed_body.body = (parsed_body.body if ast.get_docstring(parsed_body) is None else parsed_body.body[1:]) if merge_inner_function is not None: _merge_inner_function( parsed_body, infer_type=infer_type, intermediate_repr=ir, merge_inner_function=merge_inner_function, ) return ir ir["_internal"] = { "body": list( filterfalse( rpartial(isinstance, AnnAssign), parsed_body.body, )), "from_name": class_name, "from_type": "cls", } body_ir = class_( class_def=parsed_body, class_name=class_name, merge_inner_function=merge_inner_function, ) ir_merge(ir, body_ir) return ir assert (is_supported_ast_node ), "Expected 'Union[Module, ClassDef]' got `{!r}`".format( type(class_def).__name__) class_def = find_ast_type(class_def, class_name) doc_str = get_docstring(class_def) intermediate_repr = ({ "name": class_name, "type": "static", "doc": "", "params": OrderedDict(), "returns": None, } if doc_str is None else docstring(get_docstring(class_def).replace( ":cvar", ":param"), emit_default_doc=False)) if "return_type" in intermediate_repr["params"]: intermediate_repr["returns"] = OrderedDict( (("return_type", intermediate_repr["params"].pop("return_type")), )) body = class_def.body if doc_str is None else class_def.body[1:] for e in body: if isinstance(e, AnnAssign): typ = to_code(e.annotation).rstrip("\n") val = (lambda v: { "default": NoneStr } if v is None else { "default": v if type(v).__name__ in simple_types else (lambda value: { "{}": {} if isinstance(v, Dict) else set(), "[]": [], "()": (), }.get(value, parse_to_scalar(value)))(to_code(v).rstrip("\n")) })(get_value(get_value(e))) # if 'str' in typ and val: val["default"] = val["default"].strip("'") # Unquote? typ_default = dict(typ=typ, **val) for key in "params", "returns": if e.target.id in (intermediate_repr[key] or iter(())): intermediate_repr[key][e.target.id].update(typ_default) typ_default = False break if typ_default: k = "returns" if e.target.id == "return_type" else "params" if intermediate_repr.get(k) is None: intermediate_repr[k] = OrderedDict() intermediate_repr[k][e.target.id] = typ_default elif isinstance(e, Assign): val = get_value(e) if val is not None: val = get_value(val) deque( map( lambda target: setitem(*( (intermediate_repr["params"][target.id], "default", val) if target.id in intermediate_repr["params"] else ( intermediate_repr["params"], target.id, { "default": val }, ))), e.targets, ), maxlen=0, ) intermediate_repr.update({ "params": OrderedDict( map( partial(_set_name_and_type, infer_type=infer_type, word_wrap=word_wrap), intermediate_repr["params"].items(), )), "_internal": { "body": list(filterfalse(rpartial(isinstance, (AnnAssign, Assign)), body)), "from_name": class_def.name, "from_type": "cls", }, }) if merge_inner_function is not None: assert isinstance(class_def, ClassDef) _merge_inner_function( class_def, infer_type=infer_type, intermediate_repr=intermediate_repr, merge_inner_function=merge_inner_function, ) # intermediate_repr['_internal']["body"]= list(filterfalse(rpartial(isinstance,(AnnAssign,Assign)),class_def.body)) return intermediate_repr
def argparse_ast(function_def, function_type=None, function_name=None): """ Converts an argparse AST to our IR :param function_def: AST of argparse function_def :type function_def: ```FunctionDef``` :param function_type: Type of function, static is static or global method, others just become first arg :type function_type: ```Literal['self', 'cls', 'static']``` :param function_name: name of function_def :type function_name: ```str``` :returns: a dictionary of form { "name": Optional[str], "type": Optional[str], "doc": Optional[str], "params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}] "returns": Optional[OrderedDict[Literal['return_type'], {'typ': str, 'doc': Optional[str], 'default': Any}),)]] } :rtype: ```dict``` """ assert isinstance(function_def, FunctionDef), "Expected 'FunctionDef' got `{!r}`".format( type(function_def).__name__) doc_string = get_docstring(function_def) intermediate_repr = { "name": function_name, "type": function_type or get_function_type(function_def), "doc": "", "params": OrderedDict(), } ir = parse_docstring(doc_string, emit_default_doc=True) # Whether a default is required, if not found in doc, infer the proper default from type require_default = False # Parse all relevant nodes from function body body = function_def.body if doc_string is None else function_def.body[1:] for node in body: if is_argparse_add_argument(node): name, _param = parse_out_param(node, emit_default_doc=False, require_default=require_default) (intermediate_repr["params"][name].update if name in intermediate_repr["params"] else partial( setitem, intermediate_repr["params"], name))(_param) if not require_default and _param.get("default") is not None: require_default = True elif isinstance(node, Assign) and is_argparse_description(node): intermediate_repr["doc"] = get_value(node.value) elif isinstance(node, Return) and isinstance(node.value, Tuple): intermediate_repr["returns"] = OrderedDict((_parse_return( node, intermediate_repr=ir, function_def=function_def, emit_default_doc=False, ), )) inner_body = list( filterfalse( is_argparse_description, filterfalse(is_argparse_add_argument, body), )) if inner_body: intermediate_repr["_internal"] = { "body": inner_body, "from_name": function_def.name, "from_type": "static", } # if "return_type" in intermediate_repr.get("returns", {}): # pp({'intermediate_repr["returns"]["return_type"]': intermediate_repr["returns"]["return_type"]}) # intermediate_repr["returns"]["return_type"].update = dict( # interpolate_defaults(intermediate_repr["returns"]["return_type"]) # ) return intermediate_repr
def param_to_sqlalchemy_column_call(param, include_name): """ Turn a param into a `Column(…)` :param param: Name, dict with keys: 'typ', 'doc', 'default' :type param: ```Tuple[str, dict]``` :param include_name: Whether to include the name (exclude in declarative base) :type include_name: ```bool``` :returns: Form of: `Column(…)` :rtype: ```Call``` """ print("param_to_sqlalchemy_column_call::include_name:", include_name, ";") name, _param = param del param args, keywords, nullable = [], [], None if _param["typ"].startswith("Optional["): _param["typ"] = _param["typ"][len("Optional["):-1] nullable = True if include_name: args.append(set_value(name)) if "Literal[" in _param["typ"]: parsed_typ = get_value(ast.parse(_param["typ"]).body[0]) assert (parsed_typ.value.id == "Literal" ), "Only basic Literal support is implemented, not {}".format( parsed_typ.value.id) args.append( Call( func=Name("Enum", Load()), args=get_value(parsed_typ.slice).elts, keywords=[ keyword(arg="name", value=set_value(name), identifier=None) ], expr=None, expr_func=None, )) else: args.append(Name(typ2column_type[_param["typ"]], Load())) has_default = _param.get("default", ast) is not ast pk = _param.get("doc", "").startswith("[PK]") if pk: _param["doc"] = _param["doc"][4:].lstrip() elif has_default and _param["default"] not in none_types: nullable = False keywords.append( keyword(arg="doc", value=set_value(_param["doc"].rstrip(".")), identifier=None)) if has_default: if _param["default"] == NoneStr: _param["default"] = None keywords.append( keyword( arg="default", value=set_value(_param["default"]), identifier=None, )) # Sorting :\ if pk: keywords.append( keyword(arg="primary_key", value=set_value(True), identifier=None), ) if isinstance(nullable, bool): keywords.append( keyword(arg="nullable", value=set_value(nullable), identifier=None)) return Call( func=Name("Column", Load()), args=args, keywords=keywords, expr=None, expr_func=None, )
def _make_call_meth(body, return_type, param_names, docstring_format, word_wrap): """ Construct a `__call__` method from the provided `body` :param body: The body, probably from a FunctionDef.body :type body: ```List[AST]``` :param return_type: The return type of the parent symbol (probably class). Used to fill in `__call__` return. :type return_type: ```Optional[str]``` :param param_names: Container of AST `id`s to match for rename :type param_names: ```Optional[Iterator[str]]``` :param docstring_format: Format of docstring :type docstring_format: ```Literal['rest', 'numpydoc', 'google']``` :param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length. :type word_wrap: ```bool``` :returns: Internal function for `__call__` :rtype: ```FunctionDef``` """ body_len = len(body) # return_ = ( # ast.fix_missing_locations( # RewriteName(param_names).visit( # Return(get_value(ast.parse(return_type.strip("`")).body[0]), expr=None) # ) # ) # if return_type is not None and code_quoted(return_type) # else None # ) if body_len: if isinstance(body, dict): body = list( filter( None, ( None if body.get("doc") in none_types else Expr( set_value( emit_param_str( ( "return_type", { "doc": multiline( indent_all_but_first( body["doc"])) }, ), style=docstring_format, word_wrap=word_wrap, ))), RewriteName(param_names).visit( Return( get_value( ast.parse(return_type.strip("`")).body[0]), expr=None, )) if code_quoted(body["default"]) else Return( set_value(body["default"]), expr=None), ), )) # elif isinstance(body[0], Expr): # doc_str = get_value(body[0].value) # if isinstance(doc_str, str) and body_len > 0: # body = ( # body[1:] # if body_len > 1 # else ( # [ # set_value(doc_str.replace(":cvar", ":param")) # if return_ is None # else return_ # ] # if body_len == 1 # else body # ) # ) # elif not isinstance(body[0], Return) and return_ is not None: # body.append(return_) # elif return_ is not None: # body = [return_] return (ast.fix_missing_locations( FunctionDef(args=arguments( args=[set_arg("self")], defaults=[], kw_defaults=[], kwarg=None, kwonlyargs=[], posonlyargs=[], vararg=None, arg=None, ), body=body, decorator_list=[], name="__call__", returns=None, arguments_args=None, identifier_name=None, stmt=None, lineno=None, **maybe_type_comment)) if body else None)