def test_source_scope_modification(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template7.py")
pyparams = pyparam_parser.get_all_pyparams_from_source_code(
source_code)
named_nodes, source_code_module = pyparam_parser.get_source_params_assignments(
source_code)
scoped_pyparams = pyparam_parser.add_scope("test", pyparams)
node_to_config_param = {}
for named_param, new_param in zip(pyparams, scoped_pyparams):
node_to_config_param[named_nodes[
named_param.full_name]] = new_param
transformer = pyparam_parser.get_render_as_ast_node_transformer(
node_to_config_param)
new_root_module = transformer.visit(source_code_module)
new_source = astor.to_source(
new_root_module,
indent_with=pyparam_parser.COMPILED_SOURCE_INDENTATION,
pretty_source=pyparam_parser.astor_pretty_source_formatter)
new_scoped_pyparams = pyparam_parser.get_all_pyparams_from_source_code(
new_source)
self.assertEqual(new_scoped_pyparams, scoped_pyparams)
def test_version_check(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template3.py")
config = pyparam_parser.read_yaml_file(self.sample_path /
"template3_config.yml")
new_source_code = pyparam_parser.compile_source_code(
source_code=source_code, config=config, validate_version=True)
with open(self.source_code_tmp_path, "w") as file:
file.write(new_source_code)
with self.assertRaises(ValueError):
del config["version"]
pyparam_parser.compile_source_code(source_code=source_code,
config=config,
validate_version=True)
with self.assertRaises(ValueError):
config["version"] = {}
config["version"]["value"] = "x"
config["version"]["dtype"] = "str"
pyparam_parser.compile_source_code(source_code=source_code,
config=config,
validate_version=True)
def test_include_from_derived_module(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"derive_module.py")
code = mods.include_modules(source_code, [self.sample_path])
self.assertTrue('bias: float' in code)
self.assertTrue('beta: float' in code)
self.assertTrue("scope='c/matmul'" in code)
self.assertTrue('matmul2: Module = _pyparam_module__matmul2()' in code)
def module_source(self, base_path: Path) -> str:
for folder in base_path.rglob("*"):
path = folder.parent / Path(self.module_path)
if path.exists():
return parser.read_source_code(path)
raise FileNotFoundError(
f"Cannot find module: {self.module_path}, search path: {base_path}"
)
def test_derive_module(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"derive_module.py")
code = mods.derive_module(source_code, [self.sample_path])
self.assertTrue(
"matmul1: Module = IncludeModule(path='fun_module', scope='a')" in
code)
self.assertTrue(
"matmul2: Module = IncludeModule(path='fun2_module', scope='c')" in
code)
def test_source_code_compilation(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template3.py")
pyparam_parser.source_to_yaml_config(source_code, self.config_tmp_path)
config = pyparam_parser.read_yaml_file(Path(self.config_tmp_path))
new_source_code = pyparam_parser.compile_source_code(
source_code=source_code, config=config)
with open(self.source_code_tmp_path, "w") as file:
file.write(new_source_code)
def test_include_source(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"base_module_test.py")
code = mods.include_modules(source_code, [self.sample_path])
self.assertTrue("bias: float = PyParam(1.1, float, 'matmul')" in code)
self.assertTrue(
"PyParams: auto include source of `fun2_module`" in code)
self.assertTrue("INCLUDE END OF `fun_module`" in code)
self.assertTrue(" return alpha * matrix @ x + offset" in code)
self.assertTrue(
" alpha: float = PyParam(1.0, float, 'matmul')" in code)
def test_include_module(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"fun_module_import.py")
code = mods.include_modules(source_code, [self.sample_path])
self.assertTrue("class _pyparam_module__matmul2():" in code)
self.assertTrue("matmul2: Module = _pyparam_module__matmul2()" in code)
self.assertTrue(" self.matmul = matmul" in code)
self.assertTrue("offset: float = PyParam(value=1.0, dtype='float', "
"scope='b/matmul', desc='')" in code)
self.assertTrue("offset: float = PyParam(value=1.0, dtype='float', "
"scope='a/matmul', desc='')" in code)
def test_update_pyparams(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template7.py")
pyparams = pyparam_parser.get_all_pyparams_from_source_code(
source_code)
scoped_pyparams = pyparam_parser.add_scope("test", pyparams)
new_source = pyparam_parser.update_source_pyparams(
source_code, scoped_pyparams)
new_scoped_pyparams = pyparam_parser.get_all_pyparams_from_source_code(
new_source)
self.assertEqual(new_scoped_pyparams, scoped_pyparams)
def test_save_and_load_check_descriptions(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template7.py")
pyparams = pyparam_parser.get_all_pyparams_from_source_code(
source_code)
pyparam_parser.source_to_yaml_config(source_code, self.config_tmp_path)
config = pyparam_parser.read_yaml_file(self.config_tmp_path)
loaded_pyparams = pyparam_parser.read_params_from_config(config)
self.assertEqual(pyparams, loaded_pyparams)
pyparam_parser.compile_source_code(source_code, config)
def test_get_all_pyparams_from_source_code(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template1.py")
pyparams = pyparam_parser.get_all_pyparams_from_source_code(
source_code)
exp_params = [
NamedPyParam(
"start_index",
PyParam(1, int, scope="loop", desc="summation start index"),
),
NamedPyParam("max_iters",
PyParam(6, int, "loop", "max number of iterations")),
]
self.assertEqual(pyparams, exp_params)
def test_ast_assign_to_pyparam(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template1.py")
root = ast.parse(source=source_code)
pyparam_nodes = pyparam_parser.find_pyparams_assignments_nodes(root)
pyparam = NamedPyParam.from_ast_node(pyparam_nodes[0])
exp_param = NamedPyParam(
"start_index",
PyParam(1, int, scope="loop", desc="summation start index"))
self.assertEqual(pyparam, exp_param)
pyparam = NamedPyParam.from_ast_node(pyparam_nodes[1])
exp_param = NamedPyParam(
"max_iters", PyParam(6, int, "loop", "max number of iterations"))
self.assertEqual(pyparam, exp_param)
def test_params_wo_annotations_in_functions_def(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template9.py")
pyparams = pyparam_parser.get_all_pyparams_from_source_code(
source_code)
pyparam_parser.source_to_yaml_config(source_code, self.config_tmp_path)
config = pyparam_parser.read_yaml_file(self.config_tmp_path)
loaded_pyparams = pyparam_parser.read_params_from_config(config)
self.assertEqual(pyparams, loaded_pyparams)
compiled_source = pyparam_parser.compile_source_code(
source_code, config)
self.assertTrue(
"some_function(x, y, param2: int=2, param3: float=3, "
"param4: int=4, param5=5, param6=6)" in compiled_source)
self.assertTrue("self, arg1: float=1.1, arg2=2.2" in compiled_source)
self.assertTrue("result = some_function(0, 1, param2=12, param3=13)" in
compiled_source)
self.assertTrue(" param2 = 2" in compiled_source)
self.assertTrue(" param3: int = 3" in compiled_source)
self.assertTrue(
" def nested_function2(x, y, np2: int=2)" in compiled_source)
def test_include_source_module_decorators(self):
source_code = pyparam_parser.read_source_code(
self.sample_path / "fun_module_import_decorators.py")
code = mods.include_modules(source_code, [self.sample_path])
def test_loading_yaml(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template3.py")
pyparam_parser.source_to_yaml_config(source_code, self.config_tmp_path)
config = pyparam_parser.read_yaml_file(Path(self.config_tmp_path))
params = pyparam_parser.read_params_from_config(config)
exp_params = [
NamedPyParam(
name="version",
param=PyParam(value="1.0",
dtype=str,
scope="",
desc="model version"),
),
NamedPyParam(
name="base_num_filters",
param=PyParam(value=4,
dtype=int,
scope="feature_extractor",
desc=""),
),
NamedPyParam(
name="include_root",
param=PyParam(value=False,
dtype=bool,
scope="feature_extractor",
desc=""),
),
NamedPyParam(
name="regularize_depthwise",
param=PyParam(value=False,
dtype=bool,
scope="feature_extractor",
desc=""),
),
NamedPyParam(
name="activation_fn_in_separable_conv",
param=PyParam(value=False,
dtype=bool,
scope="feature_extractor",
desc=""),
),
NamedPyParam(
name="entry_flow_blocks",
param=PyParam(
value=(1, 1, 1),
dtype=tuple,
scope="feature_extractor",
desc="Number of units in each bock in the entry flow.",
),
),
NamedPyParam(
name="middle_flow_blocks",
param=PyParam(
value=(1, ),
dtype=tuple,
scope="feature_extractor",
desc="Number of units in the middle flow.",
),
),
]
self.assertEqual(params, exp_params)
def test_find_pyparams_assignments(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template1.py")
root = ast.parse(source=source_code)
pyparam_nodes = pyparam_parser.find_pyparams_assignments_nodes(root)
self.assertEqual(len(pyparam_nodes), 2)
def test_to_yaml(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template3.py")
pyparam_parser.source_to_yaml_config(source_code, self.config_tmp_path)
def test_dict_dtype_pyparam(self):
source_code = pyparam_parser.read_source_code(self.sample_path /
"template6.py")
root = ast.parse(source=source_code)
pyparam_nodes = pyparam_parser.find_pyparams_assignments_nodes(root)
pyparam = NamedPyParam.from_ast_node(pyparam_nodes[0])
exp_param = NamedPyParam(
name="foo1_dict",
param=PyParam(value={
"a": 1,
"b": 2
},
dtype=dict,
scope="model",
desc="foo1"),
)
self.assertEqual(pyparam, exp_param)
pyparam = NamedPyParam.from_ast_node(pyparam_nodes[1])
exp_param = NamedPyParam(
name="foo2_dict",
param=PyParam(value={"a": [1, 1, 2]},
dtype=dict,
scope="model",
desc="foo2"),
)
self.assertEqual(pyparam, exp_param)
pyparam = NamedPyParam.from_ast_node(pyparam_nodes[2])
exp_param = NamedPyParam(
name="foo3_dict",
param=PyParam(
value={
"a": {
"aa": 3,
"ab": [1, 3]
},
"b": [1, 2, 3],
"c": "test"
},
dtype=dict,
scope="model",
desc="foo2",
),
)
self.assertEqual(pyparam, exp_param)
pyparam = NamedPyParam.from_ast_node(pyparam_nodes[3])
exp_param = NamedPyParam(
name="foo4_dict",
param=PyParam(
value=[
{
"a": {
"aa": 3,
"ab": [1, 3]
},
"b": [1, 2, 3],
"c": "test"
},
{
"A": {
"AA": 15.,
"AB": [1]
},
"B": [2, 3],
"C": "TEST"
},
],
dtype=list,
scope="model",
desc="foo4 nested dict in list",
),
)
self.assertEqual(pyparam, exp_param)
source_code = pyparam_parser.read_source_code(self.sample_path /
"template6.py")
pyparam_parser.source_to_yaml_config(source_code, self.config_tmp_path)
config = pyparam_parser.read_yaml_file(Path(self.config_tmp_path))
pyparam_parser.compile_source_code(source_code,
config,
validate_version=False)
