def test_to_file(self) -> None: """ Tests whether `file` constructs a file, and fills it with the right content """ with TemporaryDirectory() as tempdir: filename = os.path.join( tempdir, "delete_me{extsep}py".format(extsep=extsep)) try: emit.file(class_ast, filename, skip_black=True) with open(filename, "rt") as f: ugly = f.read() os.remove(filename) emit.file(class_ast, filename, skip_black=False) with open(filename, "rt") as f: blacked = f.read() self.assertNotEqual(ugly + "" if "black" in modules else "\t", blacked) # if PY3_8: self.assertTrue( cmp_ast(ast.parse(ugly), ast.parse(blacked)), "Ugly AST doesn't match blacked AST", ) finally: if os.path.isfile(filename): os.remove(filename)
def test__conform_filename_filled(self) -> None: """Tests that _conform_filename returns the right result""" with TemporaryDirectory() as tempdir: argparse_function_filename = os.path.realpath( os.path.join( tempdir, "correct_contents{extsep}py".format(extsep=extsep) ) ) emit.file( argparse_func_ast, argparse_function_filename, mode="wt", ) self.assertTupleEqual( _conform_filename( filename=argparse_function_filename, search=["impossibru"], emit_func=emit.argparse_function, replacement_node_ir=deepcopy(intermediate_repr), type_wanted=FunctionDef, ), (argparse_function_filename, True), )
def doctrans(filename, docstring_format, type_annotations): """ Transform the docstrings found within provided filename to intended docstring_format :param filename: Python file to convert docstrings within. Edited in place. :type filename: ```str``` :param docstring_format: Format of docstring :type docstring_format: ```Literal['rest', 'numpydoc', 'google']``` :param type_annotations: True to have type annotations (3.6+), False to place in docstring :type type_annotations: ```bool``` """ with open(filename, "rt") as f: node = ast_parse(f.read(), skip_docstring_remit=False) orig_node = deepcopy(node) node = DocTrans( docstring_format=docstring_format, type_annotations=type_annotations, existing_type_annotations=has_type_annotations(node), whole_ast=orig_node, ).visit(node) if not cmp_ast(node, orig_node): emit.file(node, filename, mode="wt", skip_black=True)
def sync_properties( input_eval, input_filename, input_params, output_filename, output_params, output_param_wrap=None, ): """ Sync one property, inline to a file :param input_eval: Whether to evaluate the `param`, or just leave it :type input_eval: ```bool``` :param input_filename: Filename to find `param` from :type input_filename: ```str``` :param input_params: Locations within file of properties. Can be top level like `['a']` for `a=5` or with the `.` syntax as in `output_params`. :type input_params: ```List[str]``` :param output_filename: Filename that will be edited in place, the property within this file (to update) is selected by `output_param` :type output_filename: ```str``` :param output_params: Parameters to update. E.g., `['A.F']` for `class A: F = None`, `['f.g']` for `def f(g): pass` :type output_params: ```List[str]``` :param output_param_wrap: Wrap all input_str params with this. E.g., `Optional[Union[{output_param}, str]]` :param output_param_wrap: ```Optional[str]``` """ with open(path.realpath(path.expanduser(input_filename)), "rt") as f: input_ast = ast_parse(f.read(), filename=input_filename) with open(path.realpath(path.expanduser(output_filename)), "rt") as f: output_ast = ast_parse(f.read(), filename=output_filename) assert len(input_params) == len(output_params) for (input_param, output_param) in zip(input_params, output_params): output_ast = sync_property( input_eval, input_param, input_ast, input_filename, output_param, output_param_wrap, output_ast, ) emit.file(output_ast, output_filename, mode="wt", skip_black=False)
def test__conform_filename_unchanged(self) -> None: """Tests that _conform_filename returns the right result""" with TemporaryDirectory() as tempdir: argparse_function_filename = os.path.realpath( os.path.join( tempdir, "do_not_touch_this{extsep}py".format(extsep=extsep) ) ) emit.file(argparse_func_ast, argparse_function_filename, mode="wt") with patch("sys.stdout", new_callable=StringIO): self.assertTupleEqual( _conform_filename( filename=argparse_function_filename, search=["set_cli_args"], emit_func=emit.argparse_function, replacement_node_ir=deepcopy(intermediate_repr), type_wanted=FunctionDef, ), (argparse_function_filename, False), )
def ground_truth_tester( tempdir, _argparse_func_ast=argparse_func_ast, _class_ast=class_ast_no_default_doc, _class_with_method_ast=class_with_method_types_ast, ): """ Helper for ground_truth tests :param tempdir: temporary directory :type tempdir: ```str``` :param _argparse_func_ast: AST node :type _argparse_func_ast: ```FunctionDef``` :param _class_ast: AST node :type _class_ast: ```ClassDef``` :param _class_with_method_ast: AST node :type _class_with_method_ast: ```ClassDef``` :returns: OrderedDict of filenames and whether they were changed, Args :rtype: ```Tuple[OrderedDict, Namespace]``` """ argparse_function = os.path.join( tempdir, "argparse{extsep}py".format(extsep=extsep) ) emit.file(_argparse_func_ast, argparse_function, mode="wt") class_ = os.path.join(tempdir, "classes{extsep}py".format(extsep=extsep)) emit.file(_class_ast, class_, mode="wt") function = os.path.join(tempdir, "methods{extsep}py".format(extsep=extsep)) emit.file(_class_with_method_ast, function, mode="wt") args = Namespace( **{ "argparse_functions": (argparse_function,), "argparse_function_names": ("set_cli_args",), "classes": (class_,), "class_names": ("ConfigClass",), "functions": (function,), "function_names": ("C.function_name",), "truth": "argparse_function", } ) with patch("sys.stdout", new_callable=StringIO), patch( "sys.stderr", new_callable=StringIO ): return ( ground_truth( args, argparse_function, ), args, )
def _conform_filename( filename, search, emit_func, replacement_node_ir, type_wanted, ): """ Conform the given file to the `intermediate_repr` :param filename: Location of file :type filename: ```str``` :param search: Search query, e.g., ['node_name', 'function_name', 'arg_name'] :type search: ```List[str]``` :param replacement_node_ir: Replace what is found with the contents of this param :type replacement_node_ir: ```dict``` :param type_wanted: AST instance :type type_wanted: ```AST``` :returns: filename, whether the file was modified :rtype: ```Tuple[str, bool]``` """ filename = path.realpath(path.expanduser(filename)) if not path.isfile(filename): emit.file( emit_func( replacement_node_ir, emit_default_doc=False, # emit_func.__name__ == "class_" ), filename=filename, mode="wt", skip_black=False, ) return filename, True with open(filename, "rt") as f: parsed_ast = ast_parse(f.read(), filename=filename) assert isinstance(parsed_ast, Module) original_node = find_in_ast(search, parsed_ast) replacement_node = emit_func( replacement_node_ir, **_default_options(node=original_node, search=search, type_wanted=type_wanted)()) if original_node is None: emit.file(replacement_node, filename=filename, mode="a", skip_black=False) return filename, True assert len(search) > 0 assert (type(replacement_node) == type_wanted ), "Expected {type_wanted!r} got {type_replacement_node!r}".format( type_wanted=type_wanted, type_replacement_node=type(replacement_node).__name__) replaced = False if not cmp_ast(original_node, replacement_node): rewrite_at_query = RewriteAtQuery( search=search, replacement_node=replacement_node, ) rewrite_at_query.visit(parsed_ast) print("modified" if rewrite_at_query.replaced else "unchanged", filename, sep="\t") if rewrite_at_query.replaced: emit.file(parsed_ast, filename, mode="wt", skip_black=False) replaced = rewrite_at_query.replaced return filename, replaced
def exmod( module, emit_name, blacklist, whitelist, output_directory, dry_run, filesystem_layout="as_input", ): """ Expose module as `emit` types into `output_directory` :param module: Module name or path :type module: ```str``` :param emit_name: What type(s) to generate. :type emit_name: ```List[Literal["argparse", "class", "function", "sqlalchemy", "sqlalchemy_table"]]``` :param blacklist: Modules/FQN to omit. If unspecified will emit all (unless whitelist). :type blacklist: ```List[str]``` :param whitelist: Modules/FQN to emit. If unspecified will emit all (minus blacklist). :type whitelist: ```List[str]``` :param output_directory: Where to place the generated exposed interfaces to the given `--module`. :type output_directory: ```str``` :param dry_run: Show what would be created; don't actually write to the filesystem :type dry_run: ```bool``` :param filesystem_layout: Hierarchy of folder and file names generated. "java" is file per package per name. :type filesystem_layout: ```Literal["java", "as_input"]``` """ if dry_run: print("mkdir\t{output_directory!r}".format( output_directory=output_directory)) elif not path.isdir(output_directory): makedirs(output_directory) if blacklist: raise NotImplementedError("blacklist") elif whitelist: raise NotImplementedError("whitelist") module_name, new_module_name = map(path.basename, (module, output_directory)) module = (partial(module_from_file, module_name=module_name) if path.isdir(module) else import_module)(module) module_root_dir = path.dirname(module.__file__) + path.sep _mkdir_and_emit_file = partial( mkdir_and_emit_file, emit_name=emit_name, module_name=module_name, new_module_name=new_module_name, filesystem_layout=filesystem_layout, output_directory=output_directory, dry_run=dry_run, ) # Might need some `groupby` in case multiple files are in the one project; same for `get_module_contents` imports = list( map( _mkdir_and_emit_file, map( lambda name_source: ( name_source[0], (lambda filename: filename[len(module_name) + 1:] if filename.startswith(module_name) else filename) (relative_filename(getfile(name_source[1]))), { "params": OrderedDict(), "returns": OrderedDict() } if dry_run else parse.class_(name_source[1]), ), # sorted( map( lambda name_source: ( name_source[0][len(module_name) + 1:], name_source[1], ), get_module_contents( module, module_root_dir=module_root_dir).items(), ), # key=itemgetter(0), # ), ), ), ) assert len(imports), "Module contents are empty" modules_names = tuple( map( lambda name_module: ( name_module[0], tuple(map(itemgetter(1), name_module[1])), ), groupby( map( lambda node_mod: ( node_mod[0], node_mod[2].module, ), imports, ), itemgetter(0), ), )) init_filepath = path.join(output_directory, new_module_name, "__init__{extsep}py".format(extsep=extsep)) if dry_run: print("write\t{init_filepath!r}".format(init_filepath=init_filepath)) else: emit.file( Module( body=list( chain.from_iterable(( (Expr(set_value("\nExport internal imports\n")), ), map( lambda module_names: ImportFrom( module=module_names[0], names=list( map( lambda names: alias( names, None, identifier=None, identifier_name=None, ), module_names[1], )), level=1, identifier=None, ), modules_names, ), (Assign(targets=[Name("__all__", Store())], value=List( ctx=Load(), elts=list( map( set_value, sorted( frozenset( chain.from_iterable( map( itemgetter(1), modules_names, )), )), )), expr=None, ), expr=None, lineno=None, **maybe_type_comment), ), ))), stmt=None, type_ignores=[], ), init_filepath, mode="wt", )
def test_ground_truths(self) -> None: """My truth is being tested.""" with TemporaryDirectory() as tempdir: tempdir_join = partial(path.join, tempdir) argparse_functions = list( map( lambda i: ( lambda argparse_function: emit.file( argparse_func_ast, argparse_function, mode="wt" ) or argparse_function )(tempdir_join("argparse{i}{extsep}py".format(i=i, extsep=extsep))), range(10), ) ) # Test if can create missing file argparse_functions.append( tempdir_join("argparse_missing{extsep}py".format(extsep=extsep)) ) # Test if can fill in empty file argparse_functions.append( tempdir_join("argparse_empty{extsep}py".format(extsep=extsep)) ) class_ = tempdir_join("classes{extsep}py".format(extsep=extsep)) emit.file(class_ast_no_default_doc, class_, mode="wt") function = tempdir_join("methods{extsep}py".format(extsep=extsep)) emit.file(class_with_method_ast, function, mode="wt") args = Namespace( **{ "argparse_functions": argparse_functions, "argparse_function_names": ("set_cli_args",), "classes": (class_,), "class_names": ("ConfigClass",), "functions": (function,), "function_names": ("C.function_name",), "truth": "argparse_function", } ) with patch("sys.stdout", new_callable=StringIO), patch( "sys.stderr", new_callable=StringIO ): res = ground_truth( args, argparse_functions[0], ) self.assertTupleEqual( tuple( map( lambda filename_unmodified: ( os.path.basename(filename_unmodified[0]), filename_unmodified[1], ), res.items(), ) ), ( ("argparse0{extsep}py".format(extsep=extsep), False), ("argparse1{extsep}py".format(extsep=extsep), False), ("argparse2{extsep}py".format(extsep=extsep), False), ("argparse3{extsep}py".format(extsep=extsep), False), ("argparse4{extsep}py".format(extsep=extsep), False), ("argparse5{extsep}py".format(extsep=extsep), False), ("argparse6{extsep}py".format(extsep=extsep), False), ("argparse7{extsep}py".format(extsep=extsep), False), ("argparse8{extsep}py".format(extsep=extsep), False), ("argparse9{extsep}py".format(extsep=extsep), False), ("argparse_missing{extsep}py".format(extsep=extsep), True), ("argparse_empty{extsep}py".format(extsep=extsep), True), ("classes{extsep}py".format(extsep=extsep), False), ("methods{extsep}py".format(extsep=extsep), False), ), )
def mkdir_and_emit_file( name_orig_ir, emit_name, module_name, new_module_name, filesystem_layout, output_directory, dry_run, ): """ Generate Java-package—or match input—style file hierarchy from fully-qualified module name :param name_orig_ir: FQ module name, original filename path, IR :type name_orig_ir: ```Tuple[str, str, dict]``` :param emit_name: What type(s) to generate. :type emit_name: ```List[Literal["argparse", "class", "function", "sqlalchemy", "sqlalchemy_table"]]``` :param module_name: Name of [original] module :type module_name: ```str``` :param new_module_name: Name of [new] module :type new_module_name: ```str``` :param filesystem_layout: Hierarchy of folder and file names generated. "java" is file per package per name. :type filesystem_layout: ```Literal["java", "as_input"]``` :param output_directory: Where to place the generated exposed interfaces to the given `--module`. :type output_directory: ```str``` :param dry_run: Show what would be created; don't actually write to the filesystem :type dry_run: ```bool``` :returns: Import to generated module :rtype: ```ImportFrom``` """ mod_name, _, name = name_orig_ir[0].rpartition(".") original_relative_filename_path, ir = name_orig_ir[1], name_orig_ir[2] mod_path = path.join( output_directory, new_module_name, mod_name.replace(".", path.sep), ) if not path.isdir(mod_path): if dry_run: print("mkdir\t{mod_path!r}".format(mod_path=mod_path)) else: makedirs(mod_path) init_filepath = path.join( path.dirname(mod_path), "__init__{extsep}py".format(extsep=extsep) ) if dry_run: print("touch\t{init_filepath!r}".format(init_filepath=init_filepath)) else: open(init_filepath, "a").close() gen_node = getattr(emit, emit_name.replace("class", "class_"))( ir, **dict( **{"{emit_name}_name".format(emit_name=emit_name): name}, **{} if emit_name == "class" else {"function_type": "static"} ) ) __all___node = Assign( targets=[Name("__all__", Store())], value=List( ctx=Load(), elts=[set_value(name)], expr=None, ), expr=None, lineno=None, **maybe_type_comment ) if not isinstance(gen_node, Module): gen_node = Module( body=list( chain.from_iterable( ( ( Expr( set_value( "\nGenerated from {module_name}.{name}\n".format( module_name=module_name, name=name_orig_ir[0], ) ) ), ), ast.parse(imports_header).body, (gen_node, __all___node), ) ) ), stmt=None, type_ignores=[], ) emit_filename, init_filepath = ( map( partial(path.join, output_directory, new_module_name), ( original_relative_filename_path, path.join( path.dirname(original_relative_filename_path), "__init__{extsep}py".format(extsep=extsep), ), ), ) if filesystem_layout == "as_input" else map( partial(path.join, mod_path), ( "{name}{extsep}py".format(name=name, extsep=extsep), "__init__{extsep}py".format(extsep=extsep), ), ) ) if path.isfile(emit_filename): with open(emit_filename, "rt") as f: mod = ast.parse(f.read()) gen_node = merge_modules(mod, gen_node) merge_assignment_lists(gen_node, "__all__") if dry_run: print("write\t{emit_filename!r}".format(emit_filename=emit_filename)) else: emit.file(gen_node, filename=emit_filename, mode="wt") if name != "__init__" and not path.isfile(init_filepath): if dry_run: print("write\t{emit_filename!r}".format(emit_filename=emit_filename)) else: emit.file( Module( body=[ Expr( set_value("\n__init__ to expose internals of this module\n") ), ImportFrom( module=name, names=[ alias( name=name, asname=None, identifier=None, identifier_name=None, ), ], level=1, identifier=None, ), __all___node, ], stmt=None, type_ignores=[], ), filename=init_filepath, mode="wt", ) return ( mod_name, original_relative_filename_path, ImportFrom( module=name, names=[ alias( name=name, asname=None, identifier=None, identifier_name=None, ), ], level=1, identifier=None, ), )