def test_to_file(self) -> None:
"""
Tests whether `file` constructs a file, and fills it with the right content
"""
with TemporaryDirectory() as tempdir:
filename = os.path.join(
tempdir, "delete_me{extsep}py".format(extsep=extsep))
try:
emit.file(class_ast, filename, skip_black=True)
with open(filename, "rt") as f:
ugly = f.read()
os.remove(filename)
emit.file(class_ast, filename, skip_black=False)
with open(filename, "rt") as f:
blacked = f.read()
self.assertNotEqual(ugly + "" if "black" in modules else "\t",
blacked)
# if PY3_8:
self.assertTrue(
cmp_ast(ast.parse(ugly), ast.parse(blacked)),
"Ugly AST doesn't match blacked AST",
)
finally:
if os.path.isfile(filename):
os.remove(filename)
def test__conform_filename_filled(self) -> None:
"""Tests that _conform_filename returns the right result"""
with TemporaryDirectory() as tempdir:
argparse_function_filename = os.path.realpath(
os.path.join(
tempdir, "correct_contents{extsep}py".format(extsep=extsep)
)
)
emit.file(
argparse_func_ast,
argparse_function_filename,
mode="wt",
)
self.assertTupleEqual(
_conform_filename(
filename=argparse_function_filename,
search=["impossibru"],
emit_func=emit.argparse_function,
replacement_node_ir=deepcopy(intermediate_repr),
type_wanted=FunctionDef,
),
(argparse_function_filename, True),
)
def doctrans(filename, docstring_format, type_annotations):
"""
Transform the docstrings found within provided filename to intended docstring_format
:param filename: Python file to convert docstrings within. Edited in place.
:type filename: ```str```
:param docstring_format: Format of docstring
:type docstring_format: ```Literal['rest', 'numpydoc', 'google']```
:param type_annotations: True to have type annotations (3.6+), False to place in docstring
:type type_annotations: ```bool```
"""
with open(filename, "rt") as f:
node = ast_parse(f.read(), skip_docstring_remit=False)
orig_node = deepcopy(node)
node = DocTrans(
docstring_format=docstring_format,
type_annotations=type_annotations,
existing_type_annotations=has_type_annotations(node),
whole_ast=orig_node,
).visit(node)
if not cmp_ast(node, orig_node):
emit.file(node, filename, mode="wt", skip_black=True)
def sync_properties(
input_eval,
input_filename,
input_params,
output_filename,
output_params,
output_param_wrap=None,
):
"""
Sync one property, inline to a file
:param input_eval: Whether to evaluate the `param`, or just leave it
:type input_eval: ```bool```
:param input_filename: Filename to find `param` from
:type input_filename: ```str```
:param input_params: Locations within file of properties.
Can be top level like `['a']` for `a=5` or with the `.` syntax as in `output_params`.
:type input_params: ```List[str]```
:param output_filename: Filename that will be edited in place, the property within this file (to update)
is selected by `output_param`
:type output_filename: ```str```
:param output_params: Parameters to update. E.g., `['A.F']` for `class A: F = None`, `['f.g']` for `def f(g): pass`
:type output_params: ```List[str]```
:param output_param_wrap: Wrap all input_str params with this. E.g., `Optional[Union[{output_param}, str]]`
:param output_param_wrap: ```Optional[str]```
"""
with open(path.realpath(path.expanduser(input_filename)), "rt") as f:
input_ast = ast_parse(f.read(), filename=input_filename)
with open(path.realpath(path.expanduser(output_filename)), "rt") as f:
output_ast = ast_parse(f.read(), filename=output_filename)
assert len(input_params) == len(output_params)
for (input_param, output_param) in zip(input_params, output_params):
output_ast = sync_property(
input_eval,
input_param,
input_ast,
input_filename,
output_param,
output_param_wrap,
output_ast,
)
emit.file(output_ast, output_filename, mode="wt", skip_black=False)
def test__conform_filename_unchanged(self) -> None:
"""Tests that _conform_filename returns the right result"""
with TemporaryDirectory() as tempdir:
argparse_function_filename = os.path.realpath(
os.path.join(
tempdir, "do_not_touch_this{extsep}py".format(extsep=extsep)
)
)
emit.file(argparse_func_ast, argparse_function_filename, mode="wt")
with patch("sys.stdout", new_callable=StringIO):
self.assertTupleEqual(
_conform_filename(
filename=argparse_function_filename,
search=["set_cli_args"],
emit_func=emit.argparse_function,
replacement_node_ir=deepcopy(intermediate_repr),
type_wanted=FunctionDef,
),
(argparse_function_filename, False),
)
def ground_truth_tester(
tempdir,
_argparse_func_ast=argparse_func_ast,
_class_ast=class_ast_no_default_doc,
_class_with_method_ast=class_with_method_types_ast,
):
"""
Helper for ground_truth tests
:param tempdir: temporary directory
:type tempdir: ```str```
:param _argparse_func_ast: AST node
:type _argparse_func_ast: ```FunctionDef```
:param _class_ast: AST node
:type _class_ast: ```ClassDef```
:param _class_with_method_ast: AST node
:type _class_with_method_ast: ```ClassDef```
:returns: OrderedDict of filenames and whether they were changed, Args
:rtype: ```Tuple[OrderedDict, Namespace]```
"""
argparse_function = os.path.join(
tempdir, "argparse{extsep}py".format(extsep=extsep)
)
emit.file(_argparse_func_ast, argparse_function, mode="wt")
class_ = os.path.join(tempdir, "classes{extsep}py".format(extsep=extsep))
emit.file(_class_ast, class_, mode="wt")
function = os.path.join(tempdir, "methods{extsep}py".format(extsep=extsep))
emit.file(_class_with_method_ast, function, mode="wt")
args = Namespace(
**{
"argparse_functions": (argparse_function,),
"argparse_function_names": ("set_cli_args",),
"classes": (class_,),
"class_names": ("ConfigClass",),
"functions": (function,),
"function_names": ("C.function_name",),
"truth": "argparse_function",
}
)
with patch("sys.stdout", new_callable=StringIO), patch(
"sys.stderr", new_callable=StringIO
):
return (
ground_truth(
args,
argparse_function,
),
args,
)
def _conform_filename(
filename,
search,
emit_func,
replacement_node_ir,
type_wanted,
):
"""
Conform the given file to the `intermediate_repr`
:param filename: Location of file
:type filename: ```str```
:param search: Search query, e.g., ['node_name', 'function_name', 'arg_name']
:type search: ```List[str]```
:param replacement_node_ir: Replace what is found with the contents of this param
:type replacement_node_ir: ```dict```
:param type_wanted: AST instance
:type type_wanted: ```AST```
:returns: filename, whether the file was modified
:rtype: ```Tuple[str, bool]```
"""
filename = path.realpath(path.expanduser(filename))
if not path.isfile(filename):
emit.file(
emit_func(
replacement_node_ir,
emit_default_doc=False, # emit_func.__name__ == "class_"
),
filename=filename,
mode="wt",
skip_black=False,
)
return filename, True
with open(filename, "rt") as f:
parsed_ast = ast_parse(f.read(), filename=filename)
assert isinstance(parsed_ast, Module)
original_node = find_in_ast(search, parsed_ast)
replacement_node = emit_func(
replacement_node_ir,
**_default_options(node=original_node,
search=search,
type_wanted=type_wanted)())
if original_node is None:
emit.file(replacement_node,
filename=filename,
mode="a",
skip_black=False)
return filename, True
assert len(search) > 0
assert (type(replacement_node) == type_wanted
), "Expected {type_wanted!r} got {type_replacement_node!r}".format(
type_wanted=type_wanted,
type_replacement_node=type(replacement_node).__name__)
replaced = False
if not cmp_ast(original_node, replacement_node):
rewrite_at_query = RewriteAtQuery(
search=search,
replacement_node=replacement_node,
)
rewrite_at_query.visit(parsed_ast)
print("modified" if rewrite_at_query.replaced else "unchanged",
filename,
sep="\t")
if rewrite_at_query.replaced:
emit.file(parsed_ast, filename, mode="wt", skip_black=False)
replaced = rewrite_at_query.replaced
return filename, replaced
def exmod(
module,
emit_name,
blacklist,
whitelist,
output_directory,
dry_run,
filesystem_layout="as_input",
):
"""
Expose module as `emit` types into `output_directory`
:param module: Module name or path
:type module: ```str```
:param emit_name: What type(s) to generate.
:type emit_name: ```List[Literal["argparse", "class", "function", "sqlalchemy", "sqlalchemy_table"]]```
:param blacklist: Modules/FQN to omit. If unspecified will emit all (unless whitelist).
:type blacklist: ```List[str]```
:param whitelist: Modules/FQN to emit. If unspecified will emit all (minus blacklist).
:type whitelist: ```List[str]```
:param output_directory: Where to place the generated exposed interfaces to the given `--module`.
:type output_directory: ```str```
:param dry_run: Show what would be created; don't actually write to the filesystem
:type dry_run: ```bool```
:param filesystem_layout: Hierarchy of folder and file names generated. "java" is file per package per name.
:type filesystem_layout: ```Literal["java", "as_input"]```
"""
if dry_run:
print("mkdir\t{output_directory!r}".format(
output_directory=output_directory))
elif not path.isdir(output_directory):
makedirs(output_directory)
if blacklist:
raise NotImplementedError("blacklist")
elif whitelist:
raise NotImplementedError("whitelist")
module_name, new_module_name = map(path.basename,
(module, output_directory))
module = (partial(module_from_file, module_name=module_name)
if path.isdir(module) else import_module)(module)
module_root_dir = path.dirname(module.__file__) + path.sep
_mkdir_and_emit_file = partial(
mkdir_and_emit_file,
emit_name=emit_name,
module_name=module_name,
new_module_name=new_module_name,
filesystem_layout=filesystem_layout,
output_directory=output_directory,
dry_run=dry_run,
)
# Might need some `groupby` in case multiple files are in the one project; same for `get_module_contents`
imports = list(
map(
_mkdir_and_emit_file,
map(
lambda name_source: (
name_source[0],
(lambda filename: filename[len(module_name) + 1:]
if filename.startswith(module_name) else filename)
(relative_filename(getfile(name_source[1]))),
{
"params": OrderedDict(),
"returns": OrderedDict()
} if dry_run else parse.class_(name_source[1]),
),
# sorted(
map(
lambda name_source: (
name_source[0][len(module_name) + 1:],
name_source[1],
),
get_module_contents(
module, module_root_dir=module_root_dir).items(),
),
# key=itemgetter(0),
# ),
),
), )
assert len(imports), "Module contents are empty"
modules_names = tuple(
map(
lambda name_module: (
name_module[0],
tuple(map(itemgetter(1), name_module[1])),
),
groupby(
map(
lambda node_mod: (
node_mod[0],
node_mod[2].module,
),
imports,
),
itemgetter(0),
),
))
init_filepath = path.join(output_directory, new_module_name,
"__init__{extsep}py".format(extsep=extsep))
if dry_run:
print("write\t{init_filepath!r}".format(init_filepath=init_filepath))
else:
emit.file(
Module(
body=list(
chain.from_iterable((
(Expr(set_value("\nExport internal imports\n")), ),
map(
lambda module_names: ImportFrom(
module=module_names[0],
names=list(
map(
lambda names: alias(
names,
None,
identifier=None,
identifier_name=None,
),
module_names[1],
)),
level=1,
identifier=None,
),
modules_names,
),
(Assign(targets=[Name("__all__", Store())],
value=List(
ctx=Load(),
elts=list(
map(
set_value,
sorted(
frozenset(
chain.from_iterable(
map(
itemgetter(1),
modules_names,
)), )),
)),
expr=None,
),
expr=None,
lineno=None,
**maybe_type_comment), ),
))),
stmt=None,
type_ignores=[],
),
init_filepath,
mode="wt",
)
def test_ground_truths(self) -> None:
"""My truth is being tested."""
with TemporaryDirectory() as tempdir:
tempdir_join = partial(path.join, tempdir)
argparse_functions = list(
map(
lambda i: (
lambda argparse_function: emit.file(
argparse_func_ast, argparse_function, mode="wt"
)
or argparse_function
)(tempdir_join("argparse{i}{extsep}py".format(i=i, extsep=extsep))),
range(10),
)
)
# Test if can create missing file
argparse_functions.append(
tempdir_join("argparse_missing{extsep}py".format(extsep=extsep))
)
# Test if can fill in empty file
argparse_functions.append(
tempdir_join("argparse_empty{extsep}py".format(extsep=extsep))
)
class_ = tempdir_join("classes{extsep}py".format(extsep=extsep))
emit.file(class_ast_no_default_doc, class_, mode="wt")
function = tempdir_join("methods{extsep}py".format(extsep=extsep))
emit.file(class_with_method_ast, function, mode="wt")
args = Namespace(
**{
"argparse_functions": argparse_functions,
"argparse_function_names": ("set_cli_args",),
"classes": (class_,),
"class_names": ("ConfigClass",),
"functions": (function,),
"function_names": ("C.function_name",),
"truth": "argparse_function",
}
)
with patch("sys.stdout", new_callable=StringIO), patch(
"sys.stderr", new_callable=StringIO
):
res = ground_truth(
args,
argparse_functions[0],
)
self.assertTupleEqual(
tuple(
map(
lambda filename_unmodified: (
os.path.basename(filename_unmodified[0]),
filename_unmodified[1],
),
res.items(),
)
),
(
("argparse0{extsep}py".format(extsep=extsep), False),
("argparse1{extsep}py".format(extsep=extsep), False),
("argparse2{extsep}py".format(extsep=extsep), False),
("argparse3{extsep}py".format(extsep=extsep), False),
("argparse4{extsep}py".format(extsep=extsep), False),
("argparse5{extsep}py".format(extsep=extsep), False),
("argparse6{extsep}py".format(extsep=extsep), False),
("argparse7{extsep}py".format(extsep=extsep), False),
("argparse8{extsep}py".format(extsep=extsep), False),
("argparse9{extsep}py".format(extsep=extsep), False),
("argparse_missing{extsep}py".format(extsep=extsep), True),
("argparse_empty{extsep}py".format(extsep=extsep), True),
("classes{extsep}py".format(extsep=extsep), False),
("methods{extsep}py".format(extsep=extsep), False),
),
)
def mkdir_and_emit_file(
name_orig_ir,
emit_name,
module_name,
new_module_name,
filesystem_layout,
output_directory,
dry_run,
):
"""
Generate Java-package—or match input—style file hierarchy from fully-qualified module name
:param name_orig_ir: FQ module name, original filename path, IR
:type name_orig_ir: ```Tuple[str, str, dict]```
:param emit_name: What type(s) to generate.
:type emit_name: ```List[Literal["argparse", "class", "function", "sqlalchemy", "sqlalchemy_table"]]```
:param module_name: Name of [original] module
:type module_name: ```str```
:param new_module_name: Name of [new] module
:type new_module_name: ```str```
:param filesystem_layout: Hierarchy of folder and file names generated. "java" is file per package per name.
:type filesystem_layout: ```Literal["java", "as_input"]```
:param output_directory: Where to place the generated exposed interfaces to the given `--module`.
:type output_directory: ```str```
:param dry_run: Show what would be created; don't actually write to the filesystem
:type dry_run: ```bool```
:returns: Import to generated module
:rtype: ```ImportFrom```
"""
mod_name, _, name = name_orig_ir[0].rpartition(".")
original_relative_filename_path, ir = name_orig_ir[1], name_orig_ir[2]
mod_path = path.join(
output_directory,
new_module_name,
mod_name.replace(".", path.sep),
)
if not path.isdir(mod_path):
if dry_run:
print("mkdir\t{mod_path!r}".format(mod_path=mod_path))
else:
makedirs(mod_path)
init_filepath = path.join(
path.dirname(mod_path), "__init__{extsep}py".format(extsep=extsep)
)
if dry_run:
print("touch\t{init_filepath!r}".format(init_filepath=init_filepath))
else:
open(init_filepath, "a").close()
gen_node = getattr(emit, emit_name.replace("class", "class_"))(
ir,
**dict(
**{"{emit_name}_name".format(emit_name=emit_name): name},
**{} if emit_name == "class" else {"function_type": "static"}
)
)
__all___node = Assign(
targets=[Name("__all__", Store())],
value=List(
ctx=Load(),
elts=[set_value(name)],
expr=None,
),
expr=None,
lineno=None,
**maybe_type_comment
)
if not isinstance(gen_node, Module):
gen_node = Module(
body=list(
chain.from_iterable(
(
(
Expr(
set_value(
"\nGenerated from {module_name}.{name}\n".format(
module_name=module_name,
name=name_orig_ir[0],
)
)
),
),
ast.parse(imports_header).body,
(gen_node, __all___node),
)
)
),
stmt=None,
type_ignores=[],
)
emit_filename, init_filepath = (
map(
partial(path.join, output_directory, new_module_name),
(
original_relative_filename_path,
path.join(
path.dirname(original_relative_filename_path),
"__init__{extsep}py".format(extsep=extsep),
),
),
)
if filesystem_layout == "as_input"
else map(
partial(path.join, mod_path),
(
"{name}{extsep}py".format(name=name, extsep=extsep),
"__init__{extsep}py".format(extsep=extsep),
),
)
)
if path.isfile(emit_filename):
with open(emit_filename, "rt") as f:
mod = ast.parse(f.read())
gen_node = merge_modules(mod, gen_node)
merge_assignment_lists(gen_node, "__all__")
if dry_run:
print("write\t{emit_filename!r}".format(emit_filename=emit_filename))
else:
emit.file(gen_node, filename=emit_filename, mode="wt")
if name != "__init__" and not path.isfile(init_filepath):
if dry_run:
print("write\t{emit_filename!r}".format(emit_filename=emit_filename))
else:
emit.file(
Module(
body=[
Expr(
set_value("\n__init__ to expose internals of this module\n")
),
ImportFrom(
module=name,
names=[
alias(
name=name,
asname=None,
identifier=None,
identifier_name=None,
),
],
level=1,
identifier=None,
),
__all___node,
],
stmt=None,
type_ignores=[],
),
filename=init_filepath,
mode="wt",
)
return (
mod_name,
original_relative_filename_path,
ImportFrom(
module=name,
names=[
alias(
name=name,
asname=None,
identifier=None,
identifier_name=None,
),
],
level=1,
identifier=None,
),
)