def test_param_group_markdown_is_order_independent():
"""Markdown output of ParameterGroup.markdown() is independent of initial order"""
PARAMETERS_3C_INITIAL1 = f"""{PARAMETERS_3C_BASE}\n{PARAMETERS_3C_KINETIC}"""
PARAMETERS_3C_INITIAL2 = f"""{PARAMETERS_3C_KINETIC}\n{PARAMETERS_3C_BASE}"""
initial_parameters_ref = ParameterGroup.from_dict(
{
"j": [["1", 1, {"vary": False, "non-negative": False}]],
"kinetic": [
["1", 0.3],
["2", 500e-4, {"standard-error": 0.000012345678}],
["3", 700e-5, {"expr": "$kinetic.1 + $kinetic.2"}],
],
"irf": [["center", 1.3], ["width", 7.8]],
}
)
initial_parameters1 = load_parameters(PARAMETERS_3C_INITIAL1, format_name="yml_str")
initial_parameters2 = load_parameters(PARAMETERS_3C_INITIAL2, format_name="yml_str")
assert str(initial_parameters1.markdown()) == RENDERED_MARKDOWN
assert str(initial_parameters2.markdown()) == RENDERED_MARKDOWN
assert str(initial_parameters_ref.markdown()) == RENDERED_MARKDOWN
minimal_params = ParameterGroup.from_dict(
{"irf": [["center", 1.3, {"standard-error": 0.000012345678}]]}
)
assert str(minimal_params.markdown(float_format=".5e")) == RENDERED_MARKDOWN_E5_PRECISION
def test_parameter_to_csv(tmpdir):
csv_path = tmpdir.join("parameters.csv")
params = load_parameters(
"""
b:
- ["1", 0.25, {vary: false, min: 0, max: 8}]
- ["2", 0.75, {expr: '1 - $b.1', non-negative: true}]
rates:
- ["total", 2]
- ["branch1", {expr: '$rates.total * $b.1'}]
- ["branch2", {expr: '$rates.total * $b.2'}]
""",
format_name="yml_str",
)
save_parameters(csv_path, params, "csv")
with open(csv_path) as f:
print(f.read())
params_from_csv = load_parameters(csv_path)
for label, p in params.all():
assert params_from_csv.has(label)
p_from_csv = params_from_csv.get(label)
assert p.label == p_from_csv.label
assert p.value == p_from_csv.value
assert p.minimum == p_from_csv.minimum
assert p.maximum == p_from_csv.maximum
assert p.vary == p_from_csv.vary
assert p.non_negative == p_from_csv.non_negative
assert p.expression == p_from_csv.expression
def test_markdown_is_order_independent():
"""Markdown output of ParameterGroup.markdown() is independent of initial order"""
PARAMETERS_3C_INITIAL1 = f"""{PARAMETERS_3C_BASE}\n{PARAMETERS_3C_KINETIC}"""
PARAMETERS_3C_INITIAL2 = f"""{PARAMETERS_3C_KINETIC}\n{PARAMETERS_3C_BASE}"""
initial_parameters_ref = ParameterGroup.from_dict({
"j": [["1", 1, {
"vary": False,
"non-negative": False
}]],
"kinetic": [
["1", 300e-3],
["2", 500e-4],
["3", 700e-5],
],
"irf": [["center", 1.3], ["width", 7.8]],
})
initial_parameters1 = load_parameters(PARAMETERS_3C_INITIAL1,
format_name="yml_str")
initial_parameters2 = load_parameters(PARAMETERS_3C_INITIAL2,
format_name="yml_str")
assert initial_parameters1.markdown() == RENDERED_MARKDOWN
assert initial_parameters2.markdown() == RENDERED_MARKDOWN
assert initial_parameters_ref.markdown() == RENDERED_MARKDOWN
def test_roundtrips(yaml_reference: ParameterGroup, tmp_path: Path,
format_name: str, reference_path: Path):
"""Roundtrip via save and load have the same data."""
format_reference = load_parameters(reference_path)
parameter_path = tmp_path / f"test_parameters.{format_name}"
save_parameters(file_name=parameter_path,
format_name=format_name,
parameters=yaml_reference)
parameters_roundtrip = load_parameters(parameter_path)
assert parameters_roundtrip == yaml_reference
assert parameters_roundtrip == format_reference
if format_name in {"csv", "tsv"}:
assert parameter_path.read_text() == reference_path.read_text()
first_data_line = parameter_path.read_text().splitlines()[1]
sep = "," if format_name == "csv" else "\t"
assert f"{sep}-inf" not in first_data_line
assert f"{sep}inf" not in first_data_line
else:
assert_frame_equal(
pd.read_excel(parameter_path, na_values=["None", "none"]),
pd.read_excel(reference_path, na_values=["None", "none"]),
)
class OneComponentOneChannelGaussianIrf:
model = load_model(MODEL_1C_GAUSSIAN_IRF, format_name="yml_str")
initial_parameters = load_parameters(PARAMETERS_1C_GAUSSIAN_IRF_INITIAL,
format_name="yml_str")
wanted_parameters = load_parameters(PARAMETERS_1C_GAUSSIAN_WANTED,
format_name="yml_str")
time = np.asarray(np.arange(-10, 50, 1.5))
spectral = np.asarray([0])
axis = {"time": time, "spectral": spectral}
class ThreeComponentParallel:
model = load_model(MODEL_3C_PARALLEL, format_name="yml_str")
initial_parameters = load_parameters(PARAMETERS_3C_INITIAL_PARALLEL,
format_name="yml_str")
wanted_parameters = load_parameters(PARAMETERS_3C_PARALLEL_WANTED,
format_name="yml_str")
time = np.arange(-10, 100, 1.5)
spectral = np.arange(600, 750, 10)
axis = {"time": time, "spectral": spectral}
class ThreeComponentNoIrf:
model = load_model(MODEL_3C_NO_IRF, format_name="yml_str")
initial_parameters = load_parameters(PARAMETERS_3C_NO_IRF_INITIAL,
format_name="yml_str")
wanted_parameters = load_parameters(PARAMETERS_3C_NO_IRF_WANTED,
format_name="yml_str")
time = np.arange(0, 100, 1.5)
spectral = np.arange(600, 750, 10)
axis = {"time": time, "spectral": spectral}
class ThreeComponentSequential:
model = load_model(MODEL_3C_SEQUENTIAL, format_name="yml_str")
initial_parameters = load_parameters(PARAMETERS_3C_INITIAL_SEQUENTIAL,
format_name="yml_str")
wanted_parameters = load_parameters(PARAMETERS_3C_SIM_SEQUENTIAL,
format_name="yml_str")
time = np.arange(-10, 50, 1.0)
spectral = np.arange(600, 750, 5.0)
axis = {"time": time, "spectral": spectral}
def test_parameter_group_to_from_parameter_dict_list():
parameter_group = load_parameters(
"""
b:
- ["1", 0.25, {vary: false, min: 0, max: 8}]
- ["2", 0.75, {expr: '1 - $b.1', non-negative: true}]
rates:
- ["total", 2]
- ["branch1", {expr: '$rates.total * $b.1'}]
- ["branch2", {expr: '$rates.total * $b.2'}]
""",
format_name="yml_str",
)
parameter_dict_list = parameter_group.to_parameter_dict_list()
parameter_group_from_dict_list = ParameterGroup.from_parameter_dict_list(parameter_dict_list)
for label, wanted in parameter_group.all():
got = parameter_group_from_dict_list.get(label)
assert got.label == wanted.label
assert got.full_label == wanted.full_label
assert got.expression == wanted.expression
assert got.maximum == wanted.maximum
assert got.minimum == wanted.minimum
assert got.non_negative == wanted.non_negative
assert got.value == wanted.value
assert got.vary == wanted.vary
def test_param_options():
params = """
- ["5", 1, {non-negative: false, min: -1, max: 1, vary: false}]
- ["6", 4e2, {non-negative: true, min: -7e2, max: 8e2, vary: true}]
- ["7", 2e4]
"""
params = load_parameters(params, format_name="yml_str")
assert params.get("5").value == 1.0
assert not params.get("5").non_negative
assert params.get("5").minimum == -1
assert params.get("5").maximum == 1
assert not params.get("5").vary
assert params.get("6").value == 4e2
assert params.get("6").non_negative
assert params.get("6").minimum == -7e2
assert params.get("6").maximum == 8e2
assert params.get("6").vary
assert params.get("7").value == 2e4
assert not params.get("7").non_negative
assert params.get("7").minimum == float("-inf")
assert params.get("7").maximum == float("inf")
assert params.get("7").vary
def setup():
# %% Load in data, model and parameters
dataset1 = load_dataset(script_folder.joinpath(DATA_PATH1))
dataset2 = load_dataset(script_folder.joinpath(DATA_PATH2))
model = load_model(script_folder.joinpath(MODEL_PATH))
parameters = load_parameters(script_folder.joinpath(PARAMETERS_FILE_PATH))
return model, parameters, dataset1, dataset2
def test_parameter_group_to_csv(tmpdir):
csv_path = tmpdir.join("parameters.csv")
params = load_parameters(
"""
b:
- ["1", 0.25, {vary: false, min: 0, max: 8}]
- ["2", 0.75, {expr: '1 - $b.1', non-negative: true}]
rates:
- ["total", 2]
- ["branch1", {expr: '$rates.total * $b.1'}]
- ["branch2", {expr: '$rates.total * $b.2'}]
""",
format_name="yml_str",
)
for _, p in params.all():
p.standard_error = 42
save_parameters(params, csv_path, "csv")
wanted = dedent(
"""\
label,value,expression,minimum,maximum,non-negative,vary,standard-error
b.1,0.25,None,0.0,8.0,False,False,42
b.2,0.75,1 - $b.1,,,False,False,42
rates.total,2.0,None,,,False,True,42
rates.branch1,0.5,$rates.total * $b.1,,,False,False,42
rates.branch2,1.5,$rates.total * $b.2,,,False,False,42
"""
)
with open(csv_path) as f:
got = f.read()
print(got)
assert got == wanted
params_from_csv = load_parameters(csv_path)
for label, p in params.all():
assert params_from_csv.has(label)
p_from_csv = params_from_csv.get(label)
assert p.label == p_from_csv.label
assert p.value == p_from_csv.value
assert p.minimum == p_from_csv.minimum
assert p.maximum == p_from_csv.maximum
assert p.vary == p_from_csv.vary
assert p.non_negative == p_from_csv.non_negative
assert p.expression == p_from_csv.expression
class SimpleIrfDispersion:
model = load_model(MODEL_SIMPLE_IRF_DISPERSION, format_name="yml_str")
parameters = load_parameters(PARAMETERS_SIMPLE_IRF_DISPERSION, format_name="yml_str")
time_p1 = np.linspace(-1, 2, 50, endpoint=False)
time_p2 = np.linspace(2, 5, 30, endpoint=False)
time_p3 = np.geomspace(5, 10, num=20)
time = np.concatenate([time_p1, time_p2, time_p3])
spectral = np.arange(300, 500, 100)
axis = {"time": time, "spectral": spectral}
def test_parameter_expressions():
params = """
- ["1", 2]
- ["2", 5]
- ["3", {expr: '$1 * exp($2)'}]
- ["4", {expr: '2'}]
"""
params = load_parameters(params, format_name="yml_str")
assert params.get("3").expression is not None
assert not params.get("3").vary
assert params.get("3").value == params.get("1") * np.exp(params.get("2"))
assert params.get("4").value == 2
with pytest.raises(ValueError):
params_bad_expr = """
- ["3", {expr: 'None'}]
"""
load_parameters(params_bad_expr, format_name="yml_str")
def test_param_block_options():
params = """
block:
- 1.0
- [3.4, {vary: true}]
- {vary: false}
"""
params = load_parameters(params, format_name="yml_str")
assert not params.get("block.1").vary
assert params.get("block.2").vary
def test_param_label():
params = """
- ["5", 1]
- ["4", 2]
- ["3", 3]
"""
params = load_parameters(params, format_name="yml_str")
assert len(list(params.all())) == 3
assert [p.label for _, p in params.all()] == [f"{i}" for i in range(5, 2, -1)]
assert [p.value for _, p in params.all()] == list(range(1, 4))
def test_param_scientific():
values = [5e3, -4.2e-4, 3e-2, -2e6]
params = """
- ["1", 5e3]
- ["2", -4.2e-4]
- ["3", 3e-2]
- ["4", -2e6]
"""
params = load_parameters(params, format_name="yml_str")
assert [p.value for _, p in params.all()] == values
def test_param_group_from_csv(tmpdir):
csv_path = tmpdir.join("parameters.csv")
with open(csv_path, "w") as f:
f.write(TEST_CSV)
params = load_parameters(csv_path)
assert "rates" in params
assert params.has("rates.k1")
p = params.get("rates.k1")
assert p.label == "k1"
assert p.value == 0.05
assert p.minimum == 0
assert p.maximum == 5
assert p.vary
assert p.non_negative
assert p.expression is None
assert params.has("rates.k2")
p = params.get("rates.k2")
assert p.label == "k2"
assert p.value == params.get("rates.k1") * 2
assert p.minimum == -np.inf
assert p.maximum == np.inf
assert not p.vary
assert not p.non_negative
assert p.expression == "$rates.k1 * 2"
assert params.has("rates.k3")
p = params.get("rates.k3")
assert p.label == "k3"
assert p.value == 2.311
assert p.minimum == -np.inf
assert p.maximum == np.inf
assert p.vary
assert p.non_negative
assert p.expression is None
assert "pen" in params
assert "eq" in params["pen"]
assert params.has("pen.eq.1")
p = params.get("pen.eq.1")
assert p.label == "1"
assert p.value == 1.0
assert p.minimum == -np.inf
assert p.maximum == np.inf
assert not p.vary
assert not p.non_negative
assert p.expression is None
def load_parameters(self, file_name: str) -> ParameterGroup:
"""Load parameters from TSV file.
Parameters
----------
file_name : str
Name of file to be loaded.
Returns
-------
:class:`ParameterGroup
"""
return load_parameters(file_name, format_name="csv", sep="\t")
def test_param_array():
params = """
- 5
- 4
- 3
- 2
- 1
"""
params = load_parameters(params, format_name="yml_str")
assert len(list(params.all())) == 5
assert [p.label for _, p in params.all()] == [f"{i}" for i in range(1, 6)]
assert [p.value for _, p in params.all()] == list(range(1, 6))[::-1]
def test_parameter_set_from_group():
"""Parameter extracted from group has correct values"""
group = load_parameters(
"foo:\n - [\"1\", 123,{non-negative: true, min: 10, max: 8e2, vary: true, expr:'2'}]",
format_name="yml_str",
)
parameter = Parameter(full_label="foo.1")
parameter.set_from_group(group=group)
assert parameter.value == 123
assert parameter.non_negative is True
assert np.allclose(parameter.minimum, 10)
assert np.allclose(parameter.maximum, 800)
assert parameter.vary is True
# Set to None since value and expr were provided?
assert parameter.expression is None
def setup(self):
dataset1 = load_dataset(SCRIPT_DIR / "data/data1.ascii")
dataset2 = load_dataset(SCRIPT_DIR / "data/data2.ascii")
model = load_model(str(SCRIPT_DIR / "models/model.yml"))
parameters = load_parameters(str(SCRIPT_DIR / "models/parameters.yml"))
self.scheme = Scheme(
model,
parameters,
{
"dataset1": dataset1,
"dataset2": dataset2
},
maximum_number_function_evaluations=11,
non_negative_least_squares=True,
optimization_method="TrustRegionReflection",
)
def test_parameter_group_to_from_df():
parameter_group = load_parameters(
"""
b:
- ["1", 0.25, {vary: false, min: 0, max: 8}]
- ["2", 0.75, {expr: '1 - $b.1', non-negative: true}]
rates:
- ["total", 2]
- ["branch1", {expr: '$rates.total * $b.1'}]
- ["branch2", {expr: '$rates.total * $b.2'}]
""",
format_name="yml_str",
)
for _, p in parameter_group.all():
p.standard_error = 42
parameter_df = parameter_group.to_dataframe()
for column in [
"label",
"value",
"standard-error",
"expression",
"minimum",
"maximum",
"non-negative",
"vary",
]:
assert column in parameter_df
assert all(parameter_df["standard-error"] == 42)
parameter_group_from_df = ParameterGroup.from_dataframe(parameter_df)
for label, wanted in parameter_group.all():
got = parameter_group_from_df.get(label)
assert got.label == wanted.label
assert got.full_label == wanted.full_label
assert got.expression == wanted.expression
assert got.maximum == wanted.maximum
assert got.minimum == wanted.minimum
assert got.non_negative == wanted.non_negative
assert got.value == wanted.value
assert got.vary == wanted.vary
def test_nested_param_group():
params = """
kinetic:
j:
- 7
- 8
- 9
"""
params = load_parameters(params, format_name="yml_str")
assert len(list(params.all())) == 3
group = params["kinetic"]
assert len(list(group.all())) == 3
group = group["j"]
assert len(list(group.all())) == 3
assert [p.label for _, p in group.all()] == [f"{i}" for i in range(1, 4)]
assert [p.value for _, p in group.all()] == list(range(7, 10))
def test_parameter_expressions_groups():
params_vary_explicit = """
b:
- [0.25, {vary: True}]
- [0.75, {expr: '1 - $b.1', vary: False}]
rates:
- ["total", 2, {vary: True}]
- ["branch1", {expr: '$rates.total * $b.1', vary: False}]
- ["branch2", {expr: '$rates.total * $b.2', vary: False}]
"""
params_vary_implicit = """
b:
- [0.25]
- [0.75, {expr: '1 - $b.1'}]
rates:
- ["total", 2]
- ["branch1", {expr: '$rates.total * $b.1'}]
- ["branch2", {expr: '$rates.total * $b.2'}]
"""
params_label_explicit = """
b:
- ["1", 0.25]
- ["2", 0.75, {expr: '1 - $b.1'}]
rates:
- ["total", 2]
- ["branch1", {expr: '$rates.total * $b.1'}]
- ["branch2", {expr: '$rates.total * $b.2'}]
"""
for params in [
params_vary_explicit, params_vary_implicit, params_label_explicit
]:
params = load_parameters(params, format_name="yml_str")
assert params.get("b.1").expression is None
assert params.get("b.1").vary
assert not params.get("b.2").vary
assert params.get("b.2").expression is not None
assert params.get("rates.branch1").value == params.get(
"rates.total") * params.get("b.1")
assert params.get("rates.branch2").value == params.get(
"rates.total") * params.get("b.2")
assert params.get("rates.total").vary
assert not params.get("rates.branch1").vary
assert not params.get("rates.branch2").vary
def read_parameters_from_csv_file(parameters_file: str) -> ParameterGroup:
"""Parse ``parameters_file`` to :class:`ParameterGroup`.
Warning
-------
Deprecated use ``glotaran.io.load_parameters(parameters_file)`` instead.
Parameters
----------
parameters_file : str
File with parameters in csv.
Returns
-------
ParameterGroup
ParameterGroup described in ``parameters_file``.
"""
return load_parameters(parameters_file)
def test_update_parameter_group_from_array():
params = """
- ["1", 1, {non-negative: false, min: -1, max: 1, vary: false}]
- ["2", 4e2, {non-negative: true, min: 10, max: 8e2, vary: true}]
- ["3", 2e4]
"""
params = load_parameters(params, format_name="yml_str")
labels = ["1", "2", "3"]
values = [0, np.log(6e2), 42]
params.set_from_label_and_value_arrays(labels, values)
values[1] = np.exp(values[1])
for i in range(3):
assert params.get(f"{i+1}").value == values[i]
def test_replace_infinfinity(yaml_reference: ParameterGroup, tmp_path: Path,
format_name: str, sep: str):
"""Column 'standard-error' is missing if as_optimized==False"""
parameter_path = tmp_path / f"test_parameters.{format_name}"
save_parameters(
file_name=parameter_path,
format_name=format_name,
parameters=yaml_reference,
replace_infinfinity=False,
)
df = pd.read_csv(parameter_path, sep=sep)
assert all(df["minimum"] == -np.inf)
assert all(df["maximum"] == np.inf)
first_data_line = parameter_path.read_text().splitlines()[1]
assert f"{sep}-inf" in first_data_line
assert f"{sep}inf" in first_data_line
assert load_parameters(parameter_path) == yaml_reference
def read_parameters_from_yaml(parameters_yml_str: str) -> ParameterGroup:
"""Parse ``yaml`` string to :class:`ParameterGroup`.
Warning
-------
Deprecated use ``glotaran.io.load_parameters(parameters_yml_str, format_name="yml_str")``
instead.
Parameters
----------
parameters_yml_str : str
PArameter spec description in yaml.
Returns
-------
ParameterGroup
ParameterGroup described in ``parameters_yml_str``.
"""
return load_parameters(parameters_yml_str, format_name="yml_str")
def test_param_group_copy():
params = """
kinetic:
- ["5", 1, {non-negative: true, min: -1, max: 1, vary: false}]
- 4
- 5
j:
- 7
- 8
"""
params = load_parameters(params, format_name="yml_str")
copy = params.copy()
for label, parameter in params.all():
assert copy.has(label)
copied_parameter = copy.get(label)
assert parameter.value == copied_parameter.value
assert parameter.non_negative == copied_parameter.non_negative
assert parameter.minimum == copied_parameter.minimum
assert parameter.maximum == copied_parameter.maximum
assert parameter.vary == copied_parameter.vary
