def sqlalchemy_table(call_or_name):
"""
Parse out a `sqlalchemy.Table`, or a `name = sqlalchemy.Table`, into the IR
:param call_or_name: The call to `sqlalchemy.Table` or an assignment followed by the call
:type call_or_name: ```Union[AnnAssign, Assign, Call]```
:returns: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:rtype: ```dict```
"""
if isinstance(call_or_name, Assign):
name, call_or_name = call_or_name.targets[0].id, call_or_name.value
elif isinstance(call_or_name, AnnAssign):
name, call_or_name = call_or_name.target.id, call_or_name.value
else:
if not isinstance(call_or_name, Call):
call_or_name = get_value(call_or_name)
name = get_value(call_or_name.args[0])
# Binding should be same name as table… I guess?
assert_equal(get_value(call_or_name.args[0]), name)
comment = next(
map(
get_value,
map(get_value,
filter(lambda kw: kw.arg == "comment", call_or_name.keywords)),
),
None,
)
intermediate_repr = ({
"type": None,
"doc": "",
"params": OrderedDict()
} if comment is None else docstring(comment))
intermediate_repr["name"] = name
assert isinstance(call_or_name, Call)
assert_equal(call_or_name.func.id.rpartition(".")[2], "Table")
assert len(call_or_name.args) > 2
merge_ir = {
"params": OrderedDict(map(column_call_to_param,
call_or_name.args[2:])),
"returns": None,
}
ir_merge(target=intermediate_repr, other=merge_ir)
if intermediate_repr["returns"] and intermediate_repr["returns"].get(
"return_type", {}).get("doc"):
intermediate_repr["returns"]["return_type"]["doc"] = extract_default(
intermediate_repr["returns"]["return_type"]["doc"],
emit_default_doc=False)[0]
return intermediate_repr
def _make_call_meth(body, return_type, param_names):
"""
Construct a `__call__` method from the provided `body`
:param body: The body, probably from a FunctionDef.body
:type body: ```List[AST]```
:param return_type: The return type of the parent symbol (probably class). Used to fill in `__call__` return.
:type return_type: ```Optional[str]```
:param param_names: Container of AST `id`s to match for rename
:type param_names: ```Optional[Iterator[str]]```
:return: Internal function for `__call__`
:rtype: ```FunctionDef```
"""
body_len = len(body)
return_ = (ast.fix_missing_locations(
RewriteName(param_names).visit(
Return(get_value(ast.parse(return_type[3:-3]).body[0]),
expr=None))) if return_type is not None
and len(return_type) > 6 and return_type.startswith("```")
and return_type.endswith("```") else None)
if body_len:
if isinstance(body[0], Expr):
doc_str = get_value(body[0].value)
if isinstance(doc_str, str) and body_len > 0:
body = (body[1:] if body_len > 1 else ([
set_value(doc_str.replace(":cvar", ":param"))
if return_ is None else return_
] if body_len == 1 else body))
# elif not isinstance(body[0], Return) and return_ is not None:
# body.append(return_)
# elif return_ is not None:
# body = [return_]
return FunctionDef(args=arguments(
args=[set_arg("self")],
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
posonlyargs=[],
vararg=None,
arg=None,
),
body=body,
decorator_list=[],
name="__call__",
returns=None,
arguments_args=None,
identifier_name=None,
stmt=None,
lineno=None,
**maybe_type_comment)
def test_get_value(self) -> None:
""" Tests get_value succeeds """
val = "foo"
self.assertEqual(
get_value(Str(s=val, constant_value=None, string=None)), val)
self.assertEqual(
get_value(Constant(value=val, constant_value=None, string=None)),
val)
self.assertIsInstance(get_value(Tuple(expr=None)), Tuple)
self.assertIsInstance(get_value(Tuple(expr=None)), Tuple)
self.assertIsNone(get_value(Name(None, None)))
def _parse_return(e, intermediate_repr, function_def, emit_default_doc):
"""
Parse return into a param dict
:param e: Return AST node
:type e: Return
:param intermediate_repr: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:type intermediate_repr: ```dict```
:param function_def: AST node for function definition
:type function_def: ```FunctionDef```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:returns: Name, dict with keys: 'typ', 'doc', 'default'
:rtype: ```Tuple[str, dict]```
"""
assert isinstance(e, Return)
return set_default_doc(
(
"return_type",
{
"doc":
extract_default(
next(
line.partition(",")[2].lstrip() for line in get_value(
function_def.body[0].value).split("\n")
if line.lstrip().startswith(":return")),
emit_default_doc=emit_default_doc,
)[0],
"default":
to_code(e.value.elts[1]).rstrip("\n"),
"typ":
to_code(
get_value(
ast.parse(
intermediate_repr["returns"]["return_type"]
["typ"]).body[0].value.slice).elts[1]).rstrip()
# 'Tuple[ArgumentParser, {typ}]'.format(typ=intermediate_repr['returns']['typ'])
},
),
emit_default_doc=emit_default_doc,
)
def _infer_default(_param, infer_type):
"""
Internal function to infer the default. Not intended for use by more than [the current] one function.
:param _param: dict with keys: 'typ', 'doc', 'default'
:type _param: ```dict```
:param infer_type: Whether to try inferring the typ (from the default)
:type infer_type: ```bool```
"""
if isinstance(_param["default"],
(ast.Str, ast.Num, ast.Constant, ast.NameConstant)):
_param["default"] = get_value(_param["default"])
if _param.get("default", False) in (None, "None"):
_param["default"] = NoneStr
if (infer_type and _param.get("typ") is None
and _param["default"] not in (None, "None", NoneStr)):
_param["typ"] = type(_param["default"]).__name__
if needs_quoting(_param.get("typ")) or isinstance(_param["default"], str):
_param["default"] = unquote(_param["default"])
elif isinstance(_param["default"], AST):
_param["default"] = "```{default}```".format(
default=paren_wrap_code(to_code(_param["default"]).rstrip("\n")))
if _param.get("typ") is None and _param["default"]:
_param["typ"] = type(_param["default"]).__name__
if (isinstance(_param["default"], str)
and _param["default"].startswith("```")
and _param["default"].endswith("```") and "[" not in _param[
"typ"] # Skip if you've actually formed a proper type
):
del _param["typ"] # Could make it `object` I suppose…
def test_parse_to_scalar(self) -> None:
""" Test various inputs and outputs for `parse_to_scalar` """
for fst, snd in (
(5, 5),
("5", "5"),
(set_value(5), 5),
(ast.Expr(None), NoneStr),
):
self.assertEqual(parse_to_scalar(fst), snd)
self.assertEqual(
get_value(parse_to_scalar(ast.parse("[5]").body[0]).elts[0]), 5
)
self.assertTrue(
cmp_ast(
parse_to_scalar(ast.parse("[5]").body[0]),
List([set_value(5)], Load()),
)
)
self.assertEqual(parse_to_scalar(ast.parse("[5]")), "[5]")
parse_to_scalar(ast.parse("[5]").body[0])
self.assertRaises(NotImplementedError, parse_to_scalar, memoryview(b""))
self.assertRaises(NotImplementedError, parse_to_scalar, memoryview(b""))
def param2json_schema_property(param, required):
"""
Turn a param into a JSON schema property
:param param: Name, dict with keys: 'typ', 'doc', 'default'
:type param: ```Tuple[str, dict]```
:param required: Required parameters. This function may push to the list.
:type required: ```List[str]```
:returns: JSON schema property. Also may push to `required`.
:rtype: ```dict```
"""
name, _param = param
del param
if _param.get("doc"):
_param["description"] = _param.pop("doc")
if _param.get("typ", ast) is not ast:
_param["type"] = _param.pop("typ")
if _param["type"].startswith("Optional["):
_param["type"] = _param["type"][len("Optional["):-1]
else:
required.append(name)
if _param["type"].startswith("Literal["):
parsed_typ = get_value(ast.parse(_param["type"]).body[0])
assert (
parsed_typ.value.id == "Literal"
), "Only basic Literal support is implemented, not {}".format(
parsed_typ.value.id)
_param["enum"] = list(
map(get_value,
get_value(parsed_typ.slice).elts))
_param["type"] = typ2json_type[type(_param["enum"][0]).__name__]
else:
_param["type"] = typ2json_type[_param["type"]]
if _param.get("default", False) in none_types:
del _param["default"] # Will be inferred as `null` from the type
return name, _param
def test_param2argparse_param_default_class_str(self) -> None:
"""
Tests that param2argparse_param works to change the type based on the default
whence said default is a proxy for an unexpected type
"""
self.assertEqual(
get_value(
param2argparse_param(
(
"byo",
{"default": 5.5, "typ": "<class 'float'>"},
),
)
.value.keywords[0]
.value
),
"float",
)
def test_from_function(self) -> None:
"""
Tests that parse.function produces properly
"""
gen_ir = parse.function(function_default_complex_default_arg_ast)
gold_ir = {
"name":
"call_peril",
"params":
OrderedDict((
(
"dataset_name",
{
"default": "mnist",
"typ": "str"
},
),
(
"writer",
{
"default":
"```{}```".format(
paren_wrap_code(
get_value(
function_default_complex_default_arg_ast.
args.defaults[1]))),
},
),
)),
"returns":
None,
"type":
"static",
}
del gen_ir["_internal"] # Not needed for this test
self.assertDictEqual(
gen_ir,
gold_ir,
)
def test_find_in_ast_with_val(self) -> None:
"""Tests that `find_in_ast` correctly gives AST node from
`def class C(object): def function_name(self,dataset_name: str='foo',…)`"""
gen_ast = find_in_ast(
"C.function_name.dataset_name".split("."),
class_with_method_and_body_types_ast,
)
self.assertIsInstance(gen_ast.default, Constant if PY_GTE_3_8 else Str)
self.assertEqual(get_value(gen_ast.default), "~/tensorflow_datasets")
run_ast_test(
self,
gen_ast,
set_arg(
annotation=Name(
"str",
Load(),
),
arg="dataset_name",
),
)
def _handle_keyword(keyword, typ):
"""
Decide which type to wrap the keyword tuples in
:param keyword: AST keyword
:type keyword: ```ast.keyword```
:param typ: string representation of type
:type typ: ```str```
:returns: string representation of type
:rtype: ```str```
"""
quote_f = identity
type_ = "Union"
if typ == Any or typ in simple_types:
if typ in ("str", Any):
def quote_f(s):
"""
Wrap the input in quotes
:param s: Any value
:type s: ```Any```
:returns: the input value
:rtype: ```Any```
"""
return "'{}'".format(s)
type_ = "Literal"
return "{type}[{types}]".format(
type=type_,
types=", ".join(quote_f(get_value(elt)) for elt in keyword.value.elts),
)
def _infer_default(_param, infer_type):
"""
Internal function to infer the default. Not intended for use by more than [the current] one function.
:param _param: dict with keys: 'typ', 'doc', 'default'
:type _param: ```dict```
:param infer_type: Whether to try inferring the typ (from the default)
:type infer_type: ```bool```
"""
if isinstance(_param["default"],
(ast.Str, ast.Num, ast.Constant, ast.NameConstant)):
_param["default"] = get_value(_param["default"])
if _param.get("default", False) in none_types:
_param["default"] = NoneStr
if infer_type and _param.get(
"typ") is None and _param["default"] not in none_types:
_param["typ"] = type(_param["default"]).__name__
if needs_quoting(_param.get("typ")) or isinstance(_param["default"], str):
_param["default"] = unquote(_param["default"])
elif isinstance(_param["default"], AST):
try:
_param["default"] = ast.literal_eval(_param["default"])
# if _param.get("typ") is None or _param["typ"] == "UnaryOp":
# _param["typ"] = type(_param["default"]).__name__
except ValueError:
_param["default"] = "```{default}```".format(
default=paren_wrap_code(
to_code(_param["default"]).rstrip("\n")))
if _param.get("typ") is None and _param["default"] != NoneStr:
_param["typ"] = type(_param["default"]).__name__
if (_param["default"] != NoneStr and code_quoted(_param["default"])
and "[" not in _param.get("typ", iter(
())) # Skip if you've actually formed a proper type
):
del _param["typ"] # Could make it `object` I suppose…
def argparse_function(
intermediate_repr,
emit_default_doc=False,
function_name="set_cli_args",
function_type="static",
wrap_description=False,
word_wrap=True,
docstring_format="rest",
):
"""
Convert to an argparse FunctionDef
:param intermediate_repr: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:type intermediate_repr: ```dict```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:param function_name: name of function_def
:type function_name: ```str```
:param function_type: Type of function, static is static or global method, others just become first arg
:type function_type: ```Literal['self', 'cls', 'static']```
:param wrap_description: Whether to word-wrap the description. Set `DOCTRANS_LINE_LENGTH` to configure length.
:type wrap_description: ```bool```
:param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length.
:type word_wrap: ```bool```
:param docstring_format: Format of docstring
:type docstring_format: ```Literal['rest', 'numpydoc', 'google']```
:returns: AST node for function definition which constructs argparse
:rtype: ```FunctionDef```
"""
function_name = function_name or intermediate_repr["name"]
function_type = function_type or intermediate_repr["type"]
internal_body = get_internal_body(
target_name=function_name,
target_type=function_type,
intermediate_repr=intermediate_repr,
)
return FunctionDef(
args=arguments(
args=[set_arg("argument_parser")],
# None if function_type in frozenset((None, "static"))
# else set_arg(function_type),
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
posonlyargs=[],
vararg=None,
arg=None,
),
body=list(
chain.from_iterable(
(
iter(
(
Expr(
set_value(
indent(
docstring(
{
"doc": "Set CLI arguments",
"params": OrderedDict(
(
(
"argument_parser",
{
"doc": "argument parser",
"typ": "ArgumentParser",
},
),
)
),
"returns": OrderedDict(
(
(
"return_type",
{
"doc": "argument_parser, {}".format(
intermediate_repr[
"returns"
]["return_type"][
"doc"
]
)
if intermediate_repr[
"returns"
]["return_type"].get(
"doc"
)
else "argument_parser",
"typ": "Tuple[ArgumentParser, {typ}]".format(
typ=intermediate_repr[
"returns"
][
"return_type"
][
"typ"
]
),
}
if "return_type"
in (
(
intermediate_repr
or {}
).get("returns")
or iter(())
)
and intermediate_repr[
"returns"
]["return_type"].get("typ")
not in none_types
else {
"doc": "argument_parser",
"typ": "ArgumentParser",
},
),
),
),
},
docstring_format=docstring_format,
word_wrap=word_wrap,
),
tab,
)
+ tab
)
),
Assign(
targets=[
Attribute(
Name("argument_parser", Load()),
"description",
Store(),
)
],
value=set_value(
(fill if wrap_description else identity)(
intermediate_repr["doc"]
)
),
lineno=None,
expr=None,
**maybe_type_comment
),
)
),
filter(
None,
(
*(
(
map(
partial(
param2argparse_param,
word_wrap=word_wrap,
emit_default_doc=emit_default_doc,
),
intermediate_repr["params"].items(),
)
)
if "params" in intermediate_repr
else ()
),
*(
internal_body[
2
if len(internal_body) > 1
and isinstance(internal_body[1], Assign)
and internal_body[1].targets[0].id
== "argument_parser"
else 1 :
]
if internal_body
and isinstance(internal_body[0], Expr)
and isinstance(get_value(internal_body[0].value), str)
else internal_body
),
None
if internal_body and isinstance(internal_body[-1], Return)
else (
Return(
value=Tuple(
ctx=Load(),
elts=[
Name("argument_parser", Load()),
set_value(
intermediate_repr["returns"][
"return_type"
]["default"]
)
if code_quoted(
intermediate_repr["returns"][
"return_type"
]["default"]
)
else ast.parse(
intermediate_repr["returns"][
"return_type"
]["default"]
)
.body[0]
.value,
],
expr=None,
),
expr=None,
)
if "default"
in (
intermediate_repr.get("returns")
or {"return_type": iter(())}
)["return_type"]
else Return(
value=Name("argument_parser", Load()), expr=None
)
),
),
),
)
)
),
decorator_list=[],
name=function_name,
returns=None,
lineno=None,
arguments_args=None,
identifier_name=None,
stmt=None,
**maybe_type_comment
)
def function(function_def, infer_type=False, function_type=None, function_name=None):
"""
Converts a method to our IR
:param function_def: AST node for function definition
:type function_def: ```Union[FunctionDef, FunctionType]```
:param infer_type: Whether to try inferring the typ (from the default)
:type infer_type: ```bool```
:param function_type: Type of function, static is static or global method, others just become first arg
:type function_type: ```Literal['self', 'cls', 'static']```
:param function_name: name of function_def
:type function_name: ```str```
:return: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:rtype: ```dict```
"""
if isinstance(function_def, FunctionType):
# Dynamic function, i.e., this isn't source code; and is in your memory
ir = _inspect(function_def, function_name)
parsed_source = ast.parse(getsource(function_def).lstrip()).body[0]
ir["_internal"] = {
"body": list(
filterfalse(rpartial(isinstance, AnnAssign), parsed_source.body)
),
"from_name": parsed_source.name,
"from_type": "cls",
}
return ir
assert isinstance(
function_def, FunctionDef
), "Expected 'FunctionDef' got `{!r}`".format(type(function_def).__name__)
assert (
function_name is None or function_def.name == function_name
), "Expected {!r} got {!r}".format(function_name, function_def.name)
found_type = get_function_type(function_def)
# Read docstring
intermediate_repr_docstring = (
get_docstring(function_def) if isinstance(function_def, FunctionDef) else None
)
if intermediate_repr_docstring is None:
intermediate_repr = {
"name": function_name or function_def.name,
"params": OrderedDict(
map(
func_arg2param,
function_def.args.args
if found_type == "static"
else function_def.args.args[1:],
)
),
"returns": None,
}
else:
intermediate_repr = docstring(
intermediate_repr_docstring.replace(":cvar", ":param"),
infer_type=infer_type,
)
intermediate_repr.update(
{
"name": function_name or function_def.name,
"type": function_type or found_type,
}
)
if function_def.body:
intermediate_repr["_internal"] = {
"body": function_def.body
if intermediate_repr_docstring is None
else function_def.body[1:],
"from_name": function_def.name,
"from_type": found_type,
}
params_to_append = OrderedDict()
if (
hasattr(function_def.args, "kwarg")
and function_def.args.kwarg
and function_def.args.kwarg.arg in intermediate_repr["params"]
):
_param = intermediate_repr["params"].pop(function_def.args.kwarg.arg)
assert "typ" in _param
_param["default"] = NoneStr
# if "typ" not in _param:
# _param["typ"] = (
# "Optional[dict]"
# if function_def.args.kwarg.annotation is None
# else to_code(function_def.args.kwarg.annotation).rstrip("\n")
# )
params_to_append[function_def.args.kwarg.arg] = _param
del _param
idx = count()
# Set defaults
if intermediate_repr["params"]:
deque(
map(
lambda args_defaults: deque(
map(
lambda idxparam_idx_arg: intermediate_repr["params"][
idxparam_idx_arg[2].arg
].update(
dict(
(
{}
if getattr(idxparam_idx_arg[2], "annotation", None)
is None
else dict(
typ=to_code(
idxparam_idx_arg[2].annotation
).rstrip("\n")
)
),
**(
lambda _defaults: dict(
default=(
lambda v: (
_defaults[idxparam_idx_arg[1]]
if isinstance(
_defaults[idxparam_idx_arg[1]],
(NameConstant, Constant),
)
else v
)
if v is None
else v
)(get_value(_defaults[idxparam_idx_arg[1]]))
)
if idxparam_idx_arg[1] < len(_defaults)
and _defaults[idxparam_idx_arg[1]] is not None
else {}
)(getattr(function_def.args, args_defaults[1])),
)
),
(
lambda _args: map(
lambda idx_arg: (next(idx), idx_arg[0], idx_arg[1]),
enumerate(
filterfalse(
lambda _arg: _arg.arg[0] == "*",
(
_args
if found_type == "static"
or args_defaults[0] == "kwonlyargs"
else _args[1:]
),
)
),
)
)(getattr(function_def.args, args_defaults[0])),
),
maxlen=0,
),
(("args", "defaults"), ("kwonlyargs", "kw_defaults")),
),
maxlen=0,
)
intermediate_repr["params"].update(params_to_append)
intermediate_repr["params"] = OrderedDict(
map(
partial(_set_name_and_type, infer_type=infer_type),
intermediate_repr["params"].items(),
)
)
# Convention - the final top-level `return` is the default
intermediate_repr = _interpolate_return(function_def, intermediate_repr)
if "return_type" in (intermediate_repr.get("returns") or iter(())):
intermediate_repr["returns"] = OrderedDict(
map(
partial(_set_name_and_type, infer_type=infer_type),
intermediate_repr["returns"].items(),
)
)
return intermediate_repr
def sqlalchemy(class_def):
"""
Parse out a `class C(Base): __tablename__= 'tbl'; dataset_name = Column(String, doc="p", primary_key=True)`,
as constructed on an SQLalchemy declarative `Base`.
:param class_def: A class inheriting from declarative `Base`, where `Base = sqlalchemy.orm.declarative_base()`
:type class_def: ```Union[ClassDef]```
:returns: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:rtype: ```dict```
"""
assert isinstance(class_def, ClassDef)
# Parse into the same format that `sqlalchemy_table` can read, then return with a call to it
name = get_value(
next(
filter(
lambda assign: any(
filter(
partial(eq, "__tablename__"),
map(attrgetter("id"), assign.targets),
)),
filter(rpartial(isinstance, Assign), class_def.body),
)).value)
doc_string = get_docstring(class_def)
def _merge_name_to_column(assign):
"""
Merge `a = Column()` into `Column("a")`
:param assign: Of form `a = Column()`
:type assign: ```Assign```
:returns: Unwrapped Call with name prepended
:rtype: ```Call```
"""
assign.value.args.insert(0, set_value(assign.targets[0].id))
return assign.value
return sqlalchemy_table(
Call(
func=Name("Table", Load()),
args=list(
chain.from_iterable((
iter((set_value(name), Name("metadata", Load()))),
map(
_merge_name_to_column,
filterfalse(
lambda assign: any(
map(
lambda target: target.id == "__tablename__"
or hasattr(target, "value") and isinstance(
target.value, Call) and target.func.
rpartition(".")[2] == "Column",
assign.targets,
), ),
filter(rpartial(isinstance, Assign),
class_def.body),
),
),
))),
keywords=[] if doc_string is None else [
keyword(arg="comment",
value=set_value(doc_string),
identifier=None)
],
expr=None,
expr_func=None,
))
def argparse_function(
intermediate_repr,
emit_default_doc=False,
emit_default_doc_in_return=False,
function_name="set_cli_args",
function_type="static",
):
"""
Convert to an argparse FunctionDef
:param intermediate_repr: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:type intermediate_repr: ```dict```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:param emit_default_doc_in_return: Whether help/docstring in return should include 'With default' text
:type emit_default_doc_in_return: ```bool```
:param function_name: name of function_def
:type function_name: ```str```
:param function_type: Type of function, static is static or global method, others just become first arg
:type function_type: ```Literal['self', 'cls', 'static']```
:return: AST node for function definition which constructs argparse
:rtype: ```FunctionDef```
"""
function_name = function_name or intermediate_repr["name"]
function_type = function_type or intermediate_repr["type"]
internal_body = get_internal_body(
target_name=function_name,
target_type=function_type,
intermediate_repr=intermediate_repr,
)
return FunctionDef(
args=arguments(
args=[set_arg("argument_parser")],
# None if function_type in frozenset((None, "static"))
# else set_arg(function_type),
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
posonlyargs=[],
vararg=None,
arg=None,
),
body=list(
chain.from_iterable((
iter((
Expr(
set_value(
"\n Set CLI arguments\n\n "
":param argument_parser: argument parser\n "
":type argument_parser: ```ArgumentParser```\n\n "
"{return_params}".format(return_params=(
lambda returns:
":return: argument_parser{return_doc}\n "
"{rtype}".format(
return_doc=", {}".format(returns["doc"])
if "doc" in returns else "",
rtype=":rtype: ```ArgumentParser```\n "
if intermediate_repr["returns"][
"return_type"].get("typ") in
(None, "None", NoneStr) else
":rtype: ```Tuple[ArgumentParser, {returns[typ]}]```\n{tab}"
"".format(returns=returns, tab=tab),
)
)(returns=set_default_doc(
next(iter(intermediate_repr["returns"].items())
),
emit_default_doc=emit_default_doc_in_return,
)[1]) if "return_type" in (
intermediate_repr.get("returns") or iter(())
) else ":return: argument_parser\n "
":rtype: ```ArgumentParser```\n "
))),
Assign(targets=[
Attribute(
Name("argument_parser", Load()),
"description",
Store(),
)
],
value=set_value(intermediate_repr["doc"]),
lineno=None,
expr=None,
**maybe_type_comment),
)),
filter(
None,
(
*((map(
partial(
param2argparse_param,
emit_default_doc=emit_default_doc,
),
intermediate_repr["params"].items(),
)) if "params" in intermediate_repr else ()),
*(internal_body[
2 if len(internal_body) > 1 and isinstance(
internal_body[1], Assign) and internal_body[1].
targets[0].id == "argument_parser" else 1:] if
internal_body and isinstance(internal_body[0], Expr)
and isinstance(get_value(internal_body[0].value),
str) else internal_body),
None if internal_body
and isinstance(internal_body[-1], Return) else (Return(
value=Tuple(
ctx=Load(),
elts=[
Name("argument_parser", Load()),
set_value(intermediate_repr["returns"]
["return_type"]["default"]) if
intermediate_repr["returns"]["return_type"]
["default"].startswith("```")
and intermediate_repr["returns"]
["return_type"]["default"].endswith("```")
else ast.parse(intermediate_repr["returns"]
["return_type"]
["default"]).body[0].value,
],
expr=None,
),
expr=None,
) if "default" in (
intermediate_repr.get("returns") or {
"return_type": iter(())
})["return_type"] else Return(value=Name(
"argument_parser", Load()),
expr=None)),
),
),
))),
decorator_list=[],
name=function_name,
returns=None,
lineno=None,
arguments_args=None,
identifier_name=None,
stmt=None,
**maybe_type_comment)
def parse_out_param(expr, require_default=False, emit_default_doc=True):
"""
Turns the class_def repr of '--dataset_name', type=str, help='name of dataset.', required=True, default='mnist'
into
Tuple[Literal['dataset_name'], {"typ": Literal["str"], "doc": Literal["name of dataset."],
"default": Literal["mnist"]}]
:param expr: Expr
:type expr: ```Expr```
:param require_default: Whether a default is required, if not found in doc, infer the proper default from type
:type require_default: ```bool```
:param emit_default_doc: Whether help/docstring should include 'With default' text
:type emit_default_doc: ```bool```
:returns: Name, dict with keys: 'typ', 'doc', 'default'
:rtype: ```Tuple[str, dict]```
"""
required = get_value(
get_value(
next(
(keyword for keyword in expr.value.keywords
if keyword.arg == "required"),
set_value(False),
)))
typ = next(
(_handle_value(get_value(key_word))
for key_word in expr.value.keywords if key_word.arg == "type"),
"str",
)
name = get_value(expr.value.args[0])[len("--"):]
default = next(
(get_value(key_word.value)
for key_word in expr.value.keywords if key_word.arg == "default"),
None,
)
doc = (lambda help_: help_ if help_ is None else
(help_ if default is None or emit_default_doc is False or
(hasattr(default, "__len__") and len(default) == 0
) or "defaults to" in help_ or "Defaults to" in help_ else
"{help} Defaults to {default}".format(
help=help_ if help_.endswith(".") else "{}.".format(help_),
default=default,
)))(next(
(get_value(key_word.value) for key_word in expr.value.keywords
if key_word.arg == "help" and key_word.value),
None,
))
if default is None:
doc, default = extract_default(doc, emit_default_doc=emit_default_doc)
if default is None:
if required:
# if name.endswith("kwargs"):
# default = NoneStr
# else:
default = simple_types[typ] if typ in simple_types else NoneStr
elif require_default or typ.startswith("Optional"):
default = NoneStr
# nargs = next(
# (
# get_value(key_word.value)
# for key_word in expr.value.keywords
# if key_word.arg == "nargs"
# ),
# None,
# )
action = next(
(get_value(key_word.value)
for key_word in expr.value.keywords if key_word.arg == "action"),
None,
)
typ = next(
(_handle_keyword(keyword, typ)
for keyword in expr.value.keywords if keyword.arg == "choices"),
typ,
)
if action == "append":
typ = "List[{typ}]".format(typ=typ)
if not required and "Optional" not in typ:
typ = "Optional[{typ}]".format(typ=typ)
# if "str" in typ or "Literal" in typ and (typ.count("'") > 1 or typ.count('"') > 1):
# default = quote(default)
return name, dict(doc=doc,
typ=typ,
**({} if default is None else {
"default": default
}))
def class_(
class_def,
class_name=None,
merge_inner_function=None,
infer_type=False,
word_wrap=True,
):
"""
Converts an AST to our IR
:param class_def: Class AST or Module AST with a ClassDef inside
:type class_def: ```Union[Module, ClassDef]```
:param class_name: Name of `class`. If None, gives first found.
:type class_name: ```Optional[str]```
:param merge_inner_function: Name of inner function to merge. If None, merge nothing.
:type merge_inner_function: ```Optional[str]```
:param infer_type: Whether to try inferring the typ (from the default)
:type infer_type: ```bool```
:param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length.
:type word_wrap: ```bool```
:returns: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:rtype: ```dict```
"""
assert not isinstance(class_def, FunctionDef)
is_supported_ast_node = isinstance(class_def, (Module, ClassDef))
if not is_supported_ast_node and isinstance(class_def, type):
ir = _inspect(class_def, class_name, word_wrap)
parsed_body = ast.parse(getsource(class_def).lstrip()).body[0]
parsed_body.body = (parsed_body.body
if ast.get_docstring(parsed_body) is None else
parsed_body.body[1:])
if merge_inner_function is not None:
_merge_inner_function(
parsed_body,
infer_type=infer_type,
intermediate_repr=ir,
merge_inner_function=merge_inner_function,
)
return ir
ir["_internal"] = {
"body":
list(
filterfalse(
rpartial(isinstance, AnnAssign),
parsed_body.body,
)),
"from_name":
class_name,
"from_type":
"cls",
}
body_ir = class_(
class_def=parsed_body,
class_name=class_name,
merge_inner_function=merge_inner_function,
)
ir_merge(ir, body_ir)
return ir
assert (is_supported_ast_node
), "Expected 'Union[Module, ClassDef]' got `{!r}`".format(
type(class_def).__name__)
class_def = find_ast_type(class_def, class_name)
doc_str = get_docstring(class_def)
intermediate_repr = ({
"name": class_name,
"type": "static",
"doc": "",
"params": OrderedDict(),
"returns": None,
} if doc_str is None else docstring(get_docstring(class_def).replace(
":cvar", ":param"),
emit_default_doc=False))
if "return_type" in intermediate_repr["params"]:
intermediate_repr["returns"] = OrderedDict(
(("return_type",
intermediate_repr["params"].pop("return_type")), ))
body = class_def.body if doc_str is None else class_def.body[1:]
for e in body:
if isinstance(e, AnnAssign):
typ = to_code(e.annotation).rstrip("\n")
val = (lambda v: {
"default": NoneStr
} if v is None else {
"default":
v if type(v).__name__ in simple_types else (lambda value: {
"{}": {} if isinstance(v, Dict) else set(),
"[]": [],
"()": (),
}.get(value, parse_to_scalar(value)))(to_code(v).rstrip("\n"))
})(get_value(get_value(e)))
# if 'str' in typ and val: val["default"] = val["default"].strip("'") # Unquote?
typ_default = dict(typ=typ, **val)
for key in "params", "returns":
if e.target.id in (intermediate_repr[key] or iter(())):
intermediate_repr[key][e.target.id].update(typ_default)
typ_default = False
break
if typ_default:
k = "returns" if e.target.id == "return_type" else "params"
if intermediate_repr.get(k) is None:
intermediate_repr[k] = OrderedDict()
intermediate_repr[k][e.target.id] = typ_default
elif isinstance(e, Assign):
val = get_value(e)
if val is not None:
val = get_value(val)
deque(
map(
lambda target: setitem(*(
(intermediate_repr["params"][target.id], "default",
val)
if target.id in intermediate_repr["params"] else (
intermediate_repr["params"],
target.id,
{
"default": val
},
))),
e.targets,
),
maxlen=0,
)
intermediate_repr.update({
"params":
OrderedDict(
map(
partial(_set_name_and_type,
infer_type=infer_type,
word_wrap=word_wrap),
intermediate_repr["params"].items(),
)),
"_internal": {
"body":
list(filterfalse(rpartial(isinstance, (AnnAssign, Assign)), body)),
"from_name":
class_def.name,
"from_type":
"cls",
},
})
if merge_inner_function is not None:
assert isinstance(class_def, ClassDef)
_merge_inner_function(
class_def,
infer_type=infer_type,
intermediate_repr=intermediate_repr,
merge_inner_function=merge_inner_function,
)
# intermediate_repr['_internal']["body"]= list(filterfalse(rpartial(isinstance,(AnnAssign,Assign)),class_def.body))
return intermediate_repr
def argparse_ast(function_def, function_type=None, function_name=None):
"""
Converts an argparse AST to our IR
:param function_def: AST of argparse function_def
:type function_def: ```FunctionDef```
:param function_type: Type of function, static is static or global method, others just become first arg
:type function_type: ```Literal['self', 'cls', 'static']```
:param function_name: name of function_def
:type function_name: ```str```
:returns: a dictionary of form
{ "name": Optional[str],
"type": Optional[str],
"doc": Optional[str],
"params": OrderedDict[str, {'typ': str, 'doc': Optional[str], 'default': Any}]
"returns": Optional[OrderedDict[Literal['return_type'],
{'typ': str, 'doc': Optional[str], 'default': Any}),)]] }
:rtype: ```dict```
"""
assert isinstance(function_def,
FunctionDef), "Expected 'FunctionDef' got `{!r}`".format(
type(function_def).__name__)
doc_string = get_docstring(function_def)
intermediate_repr = {
"name": function_name,
"type": function_type or get_function_type(function_def),
"doc": "",
"params": OrderedDict(),
}
ir = parse_docstring(doc_string, emit_default_doc=True)
# Whether a default is required, if not found in doc, infer the proper default from type
require_default = False
# Parse all relevant nodes from function body
body = function_def.body if doc_string is None else function_def.body[1:]
for node in body:
if is_argparse_add_argument(node):
name, _param = parse_out_param(node,
emit_default_doc=False,
require_default=require_default)
(intermediate_repr["params"][name].update
if name in intermediate_repr["params"] else partial(
setitem, intermediate_repr["params"], name))(_param)
if not require_default and _param.get("default") is not None:
require_default = True
elif isinstance(node, Assign) and is_argparse_description(node):
intermediate_repr["doc"] = get_value(node.value)
elif isinstance(node, Return) and isinstance(node.value, Tuple):
intermediate_repr["returns"] = OrderedDict((_parse_return(
node,
intermediate_repr=ir,
function_def=function_def,
emit_default_doc=False,
), ))
inner_body = list(
filterfalse(
is_argparse_description,
filterfalse(is_argparse_add_argument, body),
))
if inner_body:
intermediate_repr["_internal"] = {
"body": inner_body,
"from_name": function_def.name,
"from_type": "static",
}
# if "return_type" in intermediate_repr.get("returns", {}):
# pp({'intermediate_repr["returns"]["return_type"]': intermediate_repr["returns"]["return_type"]})
# intermediate_repr["returns"]["return_type"].update = dict(
# interpolate_defaults(intermediate_repr["returns"]["return_type"])
# )
return intermediate_repr
def param_to_sqlalchemy_column_call(param, include_name):
"""
Turn a param into a `Column(…)`
:param param: Name, dict with keys: 'typ', 'doc', 'default'
:type param: ```Tuple[str, dict]```
:param include_name: Whether to include the name (exclude in declarative base)
:type include_name: ```bool```
:returns: Form of: `Column(…)`
:rtype: ```Call```
"""
print("param_to_sqlalchemy_column_call::include_name:", include_name, ";")
name, _param = param
del param
args, keywords, nullable = [], [], None
if _param["typ"].startswith("Optional["):
_param["typ"] = _param["typ"][len("Optional["):-1]
nullable = True
if include_name:
args.append(set_value(name))
if "Literal[" in _param["typ"]:
parsed_typ = get_value(ast.parse(_param["typ"]).body[0])
assert (parsed_typ.value.id == "Literal"
), "Only basic Literal support is implemented, not {}".format(
parsed_typ.value.id)
args.append(
Call(
func=Name("Enum", Load()),
args=get_value(parsed_typ.slice).elts,
keywords=[
keyword(arg="name", value=set_value(name), identifier=None)
],
expr=None,
expr_func=None,
))
else:
args.append(Name(typ2column_type[_param["typ"]], Load()))
has_default = _param.get("default", ast) is not ast
pk = _param.get("doc", "").startswith("[PK]")
if pk:
_param["doc"] = _param["doc"][4:].lstrip()
elif has_default and _param["default"] not in none_types:
nullable = False
keywords.append(
keyword(arg="doc",
value=set_value(_param["doc"].rstrip(".")),
identifier=None))
if has_default:
if _param["default"] == NoneStr:
_param["default"] = None
keywords.append(
keyword(
arg="default",
value=set_value(_param["default"]),
identifier=None,
))
# Sorting :\
if pk:
keywords.append(
keyword(arg="primary_key", value=set_value(True),
identifier=None), )
if isinstance(nullable, bool):
keywords.append(
keyword(arg="nullable", value=set_value(nullable),
identifier=None))
return Call(
func=Name("Column", Load()),
args=args,
keywords=keywords,
expr=None,
expr_func=None,
)
def _make_call_meth(body, return_type, param_names, docstring_format,
word_wrap):
"""
Construct a `__call__` method from the provided `body`
:param body: The body, probably from a FunctionDef.body
:type body: ```List[AST]```
:param return_type: The return type of the parent symbol (probably class). Used to fill in `__call__` return.
:type return_type: ```Optional[str]```
:param param_names: Container of AST `id`s to match for rename
:type param_names: ```Optional[Iterator[str]]```
:param docstring_format: Format of docstring
:type docstring_format: ```Literal['rest', 'numpydoc', 'google']```
:param word_wrap: Whether to word-wrap. Set `DOCTRANS_LINE_LENGTH` to configure length.
:type word_wrap: ```bool```
:returns: Internal function for `__call__`
:rtype: ```FunctionDef```
"""
body_len = len(body)
# return_ = (
# ast.fix_missing_locations(
# RewriteName(param_names).visit(
# Return(get_value(ast.parse(return_type.strip("`")).body[0]), expr=None)
# )
# )
# if return_type is not None and code_quoted(return_type)
# else None
# )
if body_len:
if isinstance(body, dict):
body = list(
filter(
None,
(
None if body.get("doc") in none_types else Expr(
set_value(
emit_param_str(
(
"return_type",
{
"doc":
multiline(
indent_all_but_first(
body["doc"]))
},
),
style=docstring_format,
word_wrap=word_wrap,
))),
RewriteName(param_names).visit(
Return(
get_value(
ast.parse(return_type.strip("`")).body[0]),
expr=None,
)) if code_quoted(body["default"]) else Return(
set_value(body["default"]), expr=None),
),
))
# elif isinstance(body[0], Expr):
# doc_str = get_value(body[0].value)
# if isinstance(doc_str, str) and body_len > 0:
# body = (
# body[1:]
# if body_len > 1
# else (
# [
# set_value(doc_str.replace(":cvar", ":param"))
# if return_ is None
# else return_
# ]
# if body_len == 1
# else body
# )
# )
# elif not isinstance(body[0], Return) and return_ is not None:
# body.append(return_)
# elif return_ is not None:
# body = [return_]
return (ast.fix_missing_locations(
FunctionDef(args=arguments(
args=[set_arg("self")],
defaults=[],
kw_defaults=[],
kwarg=None,
kwonlyargs=[],
posonlyargs=[],
vararg=None,
arg=None,
),
body=body,
decorator_list=[],
name="__call__",
returns=None,
arguments_args=None,
identifier_name=None,
stmt=None,
lineno=None,
**maybe_type_comment)) if body else None)
