def test_source_scope_modification(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template7.py")
pyparams = pyparam_parser.get_all_pyparams_from_source_code(
source_code)
named_nodes, source_code_module = pyparam_parser.get_source_params_assignments(
source_code)
scoped_pyparams = pyparam_parser.add_scope("test", pyparams)
node_to_config_param = {}
for named_param, new_param in zip(pyparams, scoped_pyparams):
node_to_config_param[named_nodes[
named_param.full_name]] = new_param
transformer = pyparam_parser.get_render_as_ast_node_transformer(
node_to_config_param)
new_root_module = transformer.visit(source_code_module)
new_source = astor.to_source(
new_root_module,
indent_with=pyparam_parser.COMPILED_SOURCE_INDENTATION,
pretty_source=pyparam_parser.astor_pretty_source_formatter)
new_scoped_pyparams = pyparam_parser.get_all_pyparams_from_source_code(
new_source)
self.assertEqual(new_scoped_pyparams, scoped_pyparams)
def derive_module(source_code: str, search_folders: List[Path]) -> str:
"""Looks for DeriveModule declarations in the code and include the source
code of the derived module. Source code may contain only single `DeriveModule`
declaration
Args:
source_code: a source code which potentially contains `DeriveModule` declaration
search_folders: a folders to search for the derived module name.
Returns:
a new source code which includes the source of the derived module
"""
if DeriveModule.__name__ not in source_code:
return source_code
if re.split("[()\\s]+", source_code).count(DeriveModule.__name__) != 1:
raise ValueError("DeriveModule can be used only once in the code")
derive_module_name = None
for line in source_code.split("\n"):
matcher = REMatcher(line)
if matcher.match(fr'.*{DeriveModule.__name__}.*\([ "]+(.*)[ "]+\).*'):
derive_module_name = matcher.group(1)
if derive_module_name is None:
raise ValueError(f"Cannot parse {DeriveModule.__name__} module name. "
f"Check module source code!")
print(f"PyParams: deriving module: {derive_module_name}")
modules_to_derive = pyparam_parser.get_all_pyparams_from_source_code(
source_code,
assigment_op_name=ReplaceModule.__name__,
ast_parser=PyParamModule)
new_source = PyParamModule(
"derived", derive_module_name).find_module_source(search_folders)
new_source = include_source_from_nodes(source_code=new_source,
search_folders=search_folders)
parsed_modules_list = pyparam_parser.get_all_pyparams_from_source_code(
new_source,
assigment_op_name=IncludeModule.__name__,
ast_parser=PyParamModule)
modules_list = []
for module in parsed_modules_list:
for module_to_use in modules_to_derive:
if module.name == module_to_use.name:
print(
f"PyParams: deriving module `{module.name}`: {module.path} => {module_to_use.path}"
)
module = module_to_use
break
modules_list.append(module)
new_source = update_modules_pyparams(new_source, modules_list)
return new_source
def test_update_pyparams(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template7.py")
pyparams = pyparam_parser.get_all_pyparams_from_source_code(
source_code)
scoped_pyparams = pyparam_parser.add_scope("test", pyparams)
new_source = pyparam_parser.update_source_pyparams(
source_code, scoped_pyparams)
new_scoped_pyparams = pyparam_parser.get_all_pyparams_from_source_code(
new_source)
self.assertEqual(new_scoped_pyparams, scoped_pyparams)
def test_save_and_load_check_descriptions(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template7.py")
pyparams = pyparam_parser.get_all_pyparams_from_source_code(
source_code)
pyparam_parser.source_to_yaml_config(source_code, self.config_tmp_path)
config = pyparam_parser.read_yaml_file(self.config_tmp_path)
loaded_pyparams = pyparam_parser.read_params_from_config(config)
self.assertEqual(pyparams, loaded_pyparams)
pyparam_parser.compile_source_code(source_code, config)
def test_get_all_pyparams_from_source_code(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template1.py")
pyparams = pyparam_parser.get_all_pyparams_from_source_code(
source_code)
exp_params = [
NamedPyParam(
"start_index",
PyParam(1, int, scope="loop", desc="summation start index"),
),
NamedPyParam("max_iters",
PyParam(6, int, "loop", "max number of iterations")),
]
self.assertEqual(pyparams, exp_params)
def update_modules_pyparams(source_code: str,
new_includes: List[PyParamModule]) -> str:
"""Replace existing pyparams in the source code with new ones. This function
can be used to e.g. replace scope parameter of each variable in source
Args:
source_code: a string source code with pyparams
new_includes: a list of PyParamModule in with new values
Returns:
new source code with pyparams parameters adapted from new_params list
"""
includes = pyparam_parser.get_all_pyparams_from_source_code(
source_code,
assigment_op_name=IncludeModule.__name__,
ast_parser=PyParamModule)
named_nodes, source_code_module = pyparam_parser.get_source_params_assignments(
source_code,
assigment_op_name=IncludeModule.__name__,
ast_parser=PyParamModule)
node_to_config_param = {}
for include, new_include in zip(includes, new_includes):
node_to_config_param[named_nodes[include.full_name]] = new_include
# transforming nodes with values from config
class ASTTransformer(ast.NodeTransformer):
def visit_AnnAssign(self, node: ast.AnnAssign) -> ast.AST:
"""
For nodes: replace with static value
"""
if node in node_to_config_param:
node.value = node_to_config_param[node].render_as_ast_node(
lineno=node.value.lineno, col_offset=node.value.col_offset)
return node
else:
return node
new_root_module = ASTTransformer().visit(source_code_module)
new_source = astor.to_source(
new_root_module,
indent_with=pyparam_parser.COMPILED_SOURCE_INDENTATION,
pretty_source=pyparam_parser.astor_pretty_source_formatter,
)
return new_source
def test_params_wo_annotations_in_functions_def(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template9.py")
pyparams = pyparam_parser.get_all_pyparams_from_source_code(
source_code)
pyparam_parser.source_to_yaml_config(source_code, self.config_tmp_path)
config = pyparam_parser.read_yaml_file(self.config_tmp_path)
loaded_pyparams = pyparam_parser.read_params_from_config(config)
self.assertEqual(pyparams, loaded_pyparams)
compiled_source = pyparam_parser.compile_source_code(
source_code, config)
self.assertTrue(
"some_function(x, y, param2: int=2, param3: float=3, "
"param4: int=4, param5=5, param6=6)" in compiled_source)
self.assertTrue("self, arg1: float=1.1, arg2=2.2" in compiled_source)
self.assertTrue("result = some_function(0, 1, param2=12, param3=13)" in
compiled_source)
self.assertTrue(" param2 = 2" in compiled_source)
self.assertTrue(" param3: int = 3" in compiled_source)
self.assertTrue(
" def nested_function2(x, y, np2: int=2)" in compiled_source)
def include_modules(source_code: str, search_folders: List[Path]) -> str:
"""Parse source code for possible PyParams modules declarations and generate
single source which will contain all imported modules
Args:
source_code: a source code of file which should be parsed
search_folders: a list of folder in which imported modules can be found.
For example ["path/to/models", "path/to/modules"]
Returns:
a new source code which will contain all imported modules.
"""
source_code = derive_module(source_code=source_code,
search_folders=search_folders)
source_code = "from pyparams import Module\n" + source_code
source_code = include_source_from_nodes(source_code=source_code,
search_folders=search_folders)
modules_list = pyparam_parser.get_all_pyparams_from_source_code(
source_code,
assigment_op_name=IncludeModule.__name__,
ast_parser=PyParamModule)
if len(modules_list) == 0:
return source_code
imported_modules = ""
for module_param in modules_list:
module_param: PyParamModule
source = module_param.find_module_source(search_folders=search_folders)
if module_param.scope != "":
pyparams = pyparam_parser.get_all_pyparams_from_source_code(source)
scoped_pyparams = pyparam_parser.add_scope(module_param.scope,
pyparams)
source = pyparam_parser.update_source_pyparams(
source, scoped_pyparams)
source = module_param.encapsulate_source_with_class(source)
debug_line = (
f"PyParams: importing `{module_param.path}` as `{module_param.name}`"
)
print(debug_line)
comment_line = f"PyParams:\n\tauto import of `{module_param.path}`\n\tused by: `{module_param.name}`"
imported_modules += '\n"""\n' + "-" * 80 + "\n"
imported_modules += f"{comment_line}`\n"
imported_modules += "-" * 80 + '\n"""\n'
imported_modules += source
imported_modules += '\n"""\n' + "-" * 80 + '\n"""\n'
compiled_source_code = pyparam_parser.compile_source_code_from_configs_list(
source_code,
modules_list,
assigment_op_name=IncludeModule.__name__,
ast_parser=PyParamModule,
)
code = imported_modules + compiled_source_code
return code
