def test_source_scope_modification(self): source_code = pyparam_parser.read_source_code(self.sample_path / "template7.py") pyparams = pyparam_parser.get_all_pyparams_from_source_code( source_code) named_nodes, source_code_module = pyparam_parser.get_source_params_assignments( source_code) scoped_pyparams = pyparam_parser.add_scope("test", pyparams) node_to_config_param = {} for named_param, new_param in zip(pyparams, scoped_pyparams): node_to_config_param[named_nodes[ named_param.full_name]] = new_param transformer = pyparam_parser.get_render_as_ast_node_transformer( node_to_config_param) new_root_module = transformer.visit(source_code_module) new_source = astor.to_source( new_root_module, indent_with=pyparam_parser.COMPILED_SOURCE_INDENTATION, pretty_source=pyparam_parser.astor_pretty_source_formatter) new_scoped_pyparams = pyparam_parser.get_all_pyparams_from_source_code( new_source) self.assertEqual(new_scoped_pyparams, scoped_pyparams)
def derive_module(source_code: str, search_folders: List[Path]) -> str: """Looks for DeriveModule declarations in the code and include the source code of the derived module. Source code may contain only single `DeriveModule` declaration Args: source_code: a source code which potentially contains `DeriveModule` declaration search_folders: a folders to search for the derived module name. Returns: a new source code which includes the source of the derived module """ if DeriveModule.__name__ not in source_code: return source_code if re.split("[()\\s]+", source_code).count(DeriveModule.__name__) != 1: raise ValueError("DeriveModule can be used only once in the code") derive_module_name = None for line in source_code.split("\n"): matcher = REMatcher(line) if matcher.match(fr'.*{DeriveModule.__name__}.*\([ "]+(.*)[ "]+\).*'): derive_module_name = matcher.group(1) if derive_module_name is None: raise ValueError(f"Cannot parse {DeriveModule.__name__} module name. " f"Check module source code!") print(f"PyParams: deriving module: {derive_module_name}") modules_to_derive = pyparam_parser.get_all_pyparams_from_source_code( source_code, assigment_op_name=ReplaceModule.__name__, ast_parser=PyParamModule) new_source = PyParamModule( "derived", derive_module_name).find_module_source(search_folders) new_source = include_source_from_nodes(source_code=new_source, search_folders=search_folders) parsed_modules_list = pyparam_parser.get_all_pyparams_from_source_code( new_source, assigment_op_name=IncludeModule.__name__, ast_parser=PyParamModule) modules_list = [] for module in parsed_modules_list: for module_to_use in modules_to_derive: if module.name == module_to_use.name: print( f"PyParams: deriving module `{module.name}`: {module.path} => {module_to_use.path}" ) module = module_to_use break modules_list.append(module) new_source = update_modules_pyparams(new_source, modules_list) return new_source
def test_update_pyparams(self): source_code = pyparam_parser.read_source_code(self.sample_path / "template7.py") pyparams = pyparam_parser.get_all_pyparams_from_source_code( source_code) scoped_pyparams = pyparam_parser.add_scope("test", pyparams) new_source = pyparam_parser.update_source_pyparams( source_code, scoped_pyparams) new_scoped_pyparams = pyparam_parser.get_all_pyparams_from_source_code( new_source) self.assertEqual(new_scoped_pyparams, scoped_pyparams)
def test_save_and_load_check_descriptions(self): source_code = pyparam_parser.read_source_code(self.sample_path / "template7.py") pyparams = pyparam_parser.get_all_pyparams_from_source_code( source_code) pyparam_parser.source_to_yaml_config(source_code, self.config_tmp_path) config = pyparam_parser.read_yaml_file(self.config_tmp_path) loaded_pyparams = pyparam_parser.read_params_from_config(config) self.assertEqual(pyparams, loaded_pyparams) pyparam_parser.compile_source_code(source_code, config)
def test_get_all_pyparams_from_source_code(self): source_code = pyparam_parser.read_source_code(self.sample_path / "template1.py") pyparams = pyparam_parser.get_all_pyparams_from_source_code( source_code) exp_params = [ NamedPyParam( "start_index", PyParam(1, int, scope="loop", desc="summation start index"), ), NamedPyParam("max_iters", PyParam(6, int, "loop", "max number of iterations")), ] self.assertEqual(pyparams, exp_params)
def update_modules_pyparams(source_code: str, new_includes: List[PyParamModule]) -> str: """Replace existing pyparams in the source code with new ones. This function can be used to e.g. replace scope parameter of each variable in source Args: source_code: a string source code with pyparams new_includes: a list of PyParamModule in with new values Returns: new source code with pyparams parameters adapted from new_params list """ includes = pyparam_parser.get_all_pyparams_from_source_code( source_code, assigment_op_name=IncludeModule.__name__, ast_parser=PyParamModule) named_nodes, source_code_module = pyparam_parser.get_source_params_assignments( source_code, assigment_op_name=IncludeModule.__name__, ast_parser=PyParamModule) node_to_config_param = {} for include, new_include in zip(includes, new_includes): node_to_config_param[named_nodes[include.full_name]] = new_include # transforming nodes with values from config class ASTTransformer(ast.NodeTransformer): def visit_AnnAssign(self, node: ast.AnnAssign) -> ast.AST: """ For nodes: replace with static value """ if node in node_to_config_param: node.value = node_to_config_param[node].render_as_ast_node( lineno=node.value.lineno, col_offset=node.value.col_offset) return node else: return node new_root_module = ASTTransformer().visit(source_code_module) new_source = astor.to_source( new_root_module, indent_with=pyparam_parser.COMPILED_SOURCE_INDENTATION, pretty_source=pyparam_parser.astor_pretty_source_formatter, ) return new_source
def test_params_wo_annotations_in_functions_def(self): source_code = pyparam_parser.read_source_code(self.sample_path / "template9.py") pyparams = pyparam_parser.get_all_pyparams_from_source_code( source_code) pyparam_parser.source_to_yaml_config(source_code, self.config_tmp_path) config = pyparam_parser.read_yaml_file(self.config_tmp_path) loaded_pyparams = pyparam_parser.read_params_from_config(config) self.assertEqual(pyparams, loaded_pyparams) compiled_source = pyparam_parser.compile_source_code( source_code, config) self.assertTrue( "some_function(x, y, param2: int=2, param3: float=3, " "param4: int=4, param5=5, param6=6)" in compiled_source) self.assertTrue("self, arg1: float=1.1, arg2=2.2" in compiled_source) self.assertTrue("result = some_function(0, 1, param2=12, param3=13)" in compiled_source) self.assertTrue(" param2 = 2" in compiled_source) self.assertTrue(" param3: int = 3" in compiled_source) self.assertTrue( " def nested_function2(x, y, np2: int=2)" in compiled_source)
def include_modules(source_code: str, search_folders: List[Path]) -> str: """Parse source code for possible PyParams modules declarations and generate single source which will contain all imported modules Args: source_code: a source code of file which should be parsed search_folders: a list of folder in which imported modules can be found. For example ["path/to/models", "path/to/modules"] Returns: a new source code which will contain all imported modules. """ source_code = derive_module(source_code=source_code, search_folders=search_folders) source_code = "from pyparams import Module\n" + source_code source_code = include_source_from_nodes(source_code=source_code, search_folders=search_folders) modules_list = pyparam_parser.get_all_pyparams_from_source_code( source_code, assigment_op_name=IncludeModule.__name__, ast_parser=PyParamModule) if len(modules_list) == 0: return source_code imported_modules = "" for module_param in modules_list: module_param: PyParamModule source = module_param.find_module_source(search_folders=search_folders) if module_param.scope != "": pyparams = pyparam_parser.get_all_pyparams_from_source_code(source) scoped_pyparams = pyparam_parser.add_scope(module_param.scope, pyparams) source = pyparam_parser.update_source_pyparams( source, scoped_pyparams) source = module_param.encapsulate_source_with_class(source) debug_line = ( f"PyParams: importing `{module_param.path}` as `{module_param.name}`" ) print(debug_line) comment_line = f"PyParams:\n\tauto import of `{module_param.path}`\n\tused by: `{module_param.name}`" imported_modules += '\n"""\n' + "-" * 80 + "\n" imported_modules += f"{comment_line}`\n" imported_modules += "-" * 80 + '\n"""\n' imported_modules += source imported_modules += '\n"""\n' + "-" * 80 + '\n"""\n' compiled_source_code = pyparam_parser.compile_source_code_from_configs_list( source_code, modules_list, assigment_op_name=IncludeModule.__name__, ast_parser=PyParamModule, ) code = imported_modules + compiled_source_code return code