def test_experiment_do_not_exist(self):
"""Tests that an error is returned when the experiment doesn't exist"""
with pytest.raises(NoConfigurationError) as exception:
get_experiment("a")
assert (
"No experiment with given name 'a' and version '*' inside database, "
"no view can be created." == str(exception.value))
def test_read_write_mode(self):
"""Tests that experiment can be created in write mode"""
experiment = create_experiment("a", space={"x": "uniform(0, 10)"})
assert experiment.mode == "x"
experiment = get_experiment("a", 2, mode="r")
assert experiment.mode == "r"
with pytest.raises(UnsupportedOperation) as exc:
experiment.insert({"x": 0})
assert exc.match(
"ExperimentClient must have write rights to execute `insert()")
experiment = get_experiment("a", 2, mode="w")
assert experiment.mode == "w"
trial = experiment.insert({"x": 0})
with pytest.raises(UnsupportedOperation) as exc:
experiment.reserve(trial)
assert exc.match(
"ExperimentClient must have execution rights to execute `reserve()"
)
def test_duplicate_only_once(storage, monkeypatch):
"""Test that trials may not be duplicated twice"""
with disable_duplication(monkeypatch):
build_evc_tree(list(range(5)))
for exp in ["root", "parent", "experiment", "child", "grand-child"]:
assert len(get_experiment(name=exp).fetch_trials(with_evc_tree=False)) == len(
Trial.allowed_stati
)
experiment = build_experiment(name="experiment")
experiment._experiment.duplicate_pending_trials()
for exp in ["root", "parent", "child", "grand-child"]:
assert len(get_experiment(name=exp).fetch_trials(with_evc_tree=False)) == len(
Trial.allowed_stati
)
assert (
len(experiment.fetch_trials(with_evc_tree=False))
== len(Trial.allowed_stati) + N_PENDING * 4
)
experiment._experiment.duplicate_pending_trials()
for exp in ["root", "parent", "child", "grand-child"]:
assert len(get_experiment(name=exp).fetch_trials(with_evc_tree=False)) == len(
Trial.allowed_stati
)
assert (
len(experiment.fetch_trials(with_evc_tree=False))
== len(Trial.allowed_stati) + N_PENDING * 4
)
def _check_enable(self, name, command, enabled):
command += " --cli-change "
experiment = get_experiment(name)
if enabled:
assert orion.core.cli.main(command.split(" ")) == 0
assert get_experiment(name).version == experiment.version + 1
else:
assert orion.core.cli.main(command.split(" ")) == 0
assert get_experiment(name).version == experiment.version
def _check_non_monitored_arguments(self, name, command,
non_monitored_arguments):
for argument in non_monitored_arguments:
command += f" --{argument} "
experiment = get_experiment(name)
# Test that cli change with non-monitored args do not cause branching
assert orion.core.cli.main(command.split(" ")) == 0
assert get_experiment(name).version == experiment.version
def _check_cli_change(self, name, command, change_type):
command += " --cli-change "
experiment = get_experiment(name)
# Test that manual_resolution is False and it branches when changing cli
assert orion.core.cli.main(command.split(" ")) == 0
new_experiment = get_experiment(name)
assert new_experiment.version == experiment.version + 1
assert new_experiment.refers["adapter"].configuration[0] == {
"of_type": "commandlinechange",
"change_type": change_type,
}
def check_local_config(self, tmp_path, conf_file, monkeypatch):
"""Check that local configuration overrides global/envvars configuration"""
command = f"hunt --worker-max-trials 0 -c {conf_file}"
orion.core.cli.main(command.split(" "))
storage = get_storage()
experiment = get_experiment("test-name")
self._compare(self.local["experiment"], experiment.configuration)
def _check_script_config_change(self, tmp_path, name, command,
change_type):
experiment = get_experiment(name)
# Test that config change is handled with 'break'
with self.setup_user_script_config(tmp_path) as user_script_config:
command += f" --config {user_script_config}"
assert orion.core.cli.main(command.split(" ")) == 0
new_experiment = get_experiment(name)
assert new_experiment.version == experiment.version + 1
assert len(new_experiment.refers["adapter"].configuration) == 2
assert new_experiment.refers["adapter"].configuration[1] == {
"of_type": "scriptconfigchange",
"change_type": change_type,
}
def test_version_do_not_exist(self, caplog):
"""Tests that a warning is printed when the experiment exist but the version doesn't"""
create_experiment("a", space={"x": "uniform(0, 10)"})
experiment = get_experiment("a", 2)
assert experiment.version == 1
assert (
"Version 2 was specified but most recent version is only 1. Using 1."
in caplog.text)
def test_fix_lost_trials_in_evc(storage, monkeypatch):
"""Test that lost trials from parents can be fixed as well.
`fix_lost_trials` is tested more carefully in experiment's unit-tests (without the EVC).
"""
with disable_duplication(monkeypatch), mocked_datetime(monkeypatch):
build_evc_tree(list(range(5)))
for exp_name in ["root", "parent", "experiment", "child", "grand-child"]:
exp = get_experiment(name=exp_name)
assert len(exp.fetch_trials(with_evc_tree=False)) == len(Trial.allowed_stati)
assert len(exp.fetch_trials_by_status("reserved", with_evc_tree=False)) == 1
experiment = build_experiment(name="experiment")
experiment._experiment.fix_lost_trials()
for exp_name in ["root", "parent", "experiment", "child", "grand-child"]:
exp = get_experiment(name=exp_name)
assert len(exp.fetch_trials(with_evc_tree=False)) == len(Trial.allowed_stati)
assert len(exp.fetch_trials_by_status("reserved", with_evc_tree=False)) == 0
def test_experiment_exist(self):
"""
Tests that an instance of :class:`orion.client.experiment.ExperimentClient` is
returned representing the latest version when none is given.
"""
experiment = create_experiment("a", space={"x": "uniform(0, 10)"})
experiment = get_experiment("a")
assert experiment
assert isinstance(experiment, ExperimentClient)
assert experiment.mode == "r"
def check_db_config(self):
"""Check that db config overrides global/envvar config"""
name = "test-name"
command = f"hunt --worker-max-trials 0 -n {name}"
orion.core.cli.main(command.split(" "))
storage = get_storage()
experiment = get_experiment(name)
self._compare(self.database,
experiment.configuration,
ignore=["worker_trials"])
def test_duplicate_closest_duplicated_pending_trials(storage, monkeypatch):
"""Test that only closest duplicated pending trials are duplicated"""
with disable_duplication(monkeypatch):
build_evc_tree([0, 0, 1, 2, 2])
for exp in ["root", "parent", "experiment", "child", "grand-child"]:
assert len(get_experiment(name=exp).fetch_trials(with_evc_tree=False)) == len(
Trial.allowed_stati
)
experiment = build_experiment(name="experiment")
experiment._experiment.duplicate_pending_trials()
for exp in ["root", "parent", "child", "grand-child"]:
assert len(get_experiment(name=exp).fetch_trials(with_evc_tree=False)) == len(
Trial.allowed_stati
)
assert (
len(experiment.fetch_trials(with_evc_tree=False))
== len(Trial.allowed_stati) + N_PENDING * 2
)
def _check_code_change(
self,
monkeypatch,
name,
command,
version,
mock_ignore_code_changes,
ignore_code_changes,
change_type,
):
# Test that code change is handled with 'no-effect'
def fixed_dictionary(user_script):
"""Create VCS"""
vcs = {}
vcs["type"] = "git"
vcs["is_dirty"] = False
vcs["HEAD_sha"] = "test " + str(random.random())
vcs["active_branch"] = None
vcs["diff_sha"] = "diff"
return vcs
monkeypatch.setattr(orion.core.io.resolve_config,
"infer_versioning_metadata", fixed_dictionary)
self._mock_consumer(monkeypatch)
detect = orion.core.evc.conflicts.CodeConflict.detect
def mock_detect(old_config, new_config, branching_config=None):
if branching_config and "ignore_code_changes" in branching_config:
assert (branching_config["ignore_code_changes"] is
mock_ignore_code_changes)
branching_config["ignore_code_changes"] = False
return detect(old_config, new_config, branching_config)
monkeypatch.setattr(orion.core.evc.conflicts.CodeConflict, "detect",
mock_detect)
assert orion.core.cli.main(command.split(" ")) == 0
self._check_consumer({"ignore_code_changes": ignore_code_changes})
experiment = get_experiment(name, version=version + 1)
assert experiment.version == version + 1
assert experiment.refers["adapter"].configuration[0] == {
"of_type": "codechange",
"change_type": change_type,
}
monkeypatch.undo()
def check_global_config(self, tmp_path, monkeypatch):
"""Check that global configuration is set properly"""
self._compare(self.config["experiment"],
orion.core.config.to_dict()["experiment"])
command = f"hunt --init-only -n test python {script} -x~uniform(0,1)"
orion.core.cli.main(command.split(" "))
storage = get_storage()
experiment = get_experiment("test")
self._compare(
self.config["experiment"],
experiment.configuration,
ignore=["worker_trials"],
)
def check_env_var_config(self, tmp_path, monkeypatch):
"""Check that env vars overrides global configuration"""
assert (orion.core.config.experiment.max_trials ==
self.env_vars["ORION_EXP_MAX_TRIALS"])
assert (orion.core.config.experiment.max_broken ==
self.env_vars["ORION_EXP_MAX_BROKEN"])
assert (orion.core.config.experiment.working_dir ==
self.env_vars["ORION_WORKING_DIR"])
command = f"hunt --init-only -n test python {script} -x~uniform(0,1)"
orion.core.cli.main(command.split(" "))
experiment = get_experiment("test")
assert experiment.max_trials == self.env_vars["ORION_EXP_MAX_TRIALS"]
assert experiment.max_broken == self.env_vars["ORION_EXP_MAX_BROKEN"]
assert experiment.working_dir == self.env_vars["ORION_WORKING_DIR"]
def check_cmd_args_config(self, tmp_path, conf_file, monkeypatch):
"""Check that cmdargs configuration overrides global/envvars/local configuration"""
command = f"hunt --worker-max-trials 0 -c {conf_file} --branch-from test-name"
command += " " + " ".join("--{} {}".format(name, value)
for name, value in self.cmdargs.items())
orion.core.cli.main(command.split(" "))
storage = get_storage()
experiment = get_experiment("exp-name")
assert experiment.name == "exp-name"
assert experiment.node.parent.name == "test-name"
assert experiment.version == 1
assert experiment.metadata["user"] == self.cmdargs["user"]
assert experiment.max_trials == self.cmdargs["exp-max-trials"]
assert experiment.max_broken == self.cmdargs["exp-max-broken"]
assert experiment.working_dir == self.cmdargs["working-dir"]
def plot_exps(experiment_names, host=TMP_DB_HOST):
print("Plotting experiments from", host)
storage = setup_tmp_storage(host)
# Plot exps
for experiment_name, version in experiment_names:
print(f" {experiment_name}-v{version}")
experiment = get_experiment(experiment_name, version=version)
for plot in [
"regret", "lpi", "partial_dependencies", "parallel_coordinates"
]:
experiment.plot(
kind=plot).write_html(f"_static/{experiment.name}_{plot}.html")
if experiment_name in custom_plots:
custom_plot = custom_plots[experiment_name]
kwargs = custom_plot["kwargs"]
name = (
f"_static/{experiment.name}_{kwargs['kind']}_{custom_plot['name']}.html"
)
experiment.plot(**kwargs).write_html(name)
def build_child_experiment(space=None,
trials=None,
name="child",
parent="root"):
"""Build a child experiment by branching from `parent` and generate trials."""
if trials is None:
trials = [None for i in range(6)]
max_trials = get_experiment(parent).max_trials + len(trials)
child = build_experiment(
name=name,
space=space,
max_trials=max_trials,
branching={
"branch_from": parent,
"enable": True
},
)
assert child.name == name
assert child.version == 1
generate_trials(child, trials)
.. autofunction:: orion.plotting.base.lpi
:noindex:
The local parameter importance plot can be executed directly from the ``experiment`` with
``plot.lpi()`` as shown in the example below.
"""
from orion.client import get_experiment
# Specify the database where the experiments are stored. We use a local PickleDB here.
storage = dict(type="legacy",
database=dict(type="pickleddb", host="../db.pkl"))
# Load the data for the specified experiment
experiment = get_experiment("2-dim-exp", storage=storage)
fig = experiment.plot.lpi()
fig
#%%
# On this plot the x-axis shows the different hyperparameters while the y-axis gives the
# local parameter importance.
# The error bars represent the standard deviation of the LPI based on 10 runs. Remember
# that the LPI requires the training of a regression model. The initial state of this model can
# greatly influence the results. This is why the computation of the LPI is done multiple times
# (10 times by default) using different random seeds. Here is an example
# setting the number of points for the grids, the number of runs and the initial random seed
# for the regression model.
experiment.plot.lpi(n_points=10, n_runs=5, model_kwargs=dict(random_state=1))
def _check_code_change(
self,
monkeypatch,
name,
command,
mock_ignore_code_changes,
ignore_code_changes,
change_type,
enable_evc,
):
"""Check if code changes are correctly ignored during experiment build and by consumer
between two trial executions.
"""
# Test that code change is handled with 'no-effect'
def fixed_dictionary(user_script):
"""Create VCS"""
vcs = {}
vcs["type"] = "git"
vcs["is_dirty"] = False
vcs["HEAD_sha"] = "test " + str(random.random())
vcs["active_branch"] = None
vcs["diff_sha"] = "diff"
return vcs
monkeypatch.setattr(orion.core.io.resolve_config,
"infer_versioning_metadata", fixed_dictionary)
self._mock_consumer(monkeypatch)
detect = orion.core.evc.conflicts.CodeConflict.detect
def mock_detect(old_config, new_config, branching_config=None):
if branching_config and "ignore_code_changes" in branching_config:
assert (branching_config["ignore_code_changes"] is
mock_ignore_code_changes)
# branching_config["ignore_code_changes"] = False
return detect(old_config, new_config, branching_config)
monkeypatch.setattr(orion.core.evc.conflicts.CodeConflict, "detect",
mock_detect)
experiment = get_experiment(name)
assert orion.core.cli.main(command.split(" ")) == 0
self._check_consumer({
"ignore_code_changes": ((enable_evc and ignore_code_changes)
or not enable_evc)
})
new_experiment = get_experiment(name)
if enable_evc and not ignore_code_changes:
assert new_experiment.version == experiment.version + 1
assert new_experiment.refers["adapter"].configuration[0] == {
"of_type": "codechange",
"change_type": change_type,
}
elif enable_evc: # But code change ignored, so no branching event.
assert new_experiment.version == experiment.version
else:
assert new_experiment.version == experiment.version
def generic_tree_test(
experiment_name,
parent_name=None,
grand_parent_name=None,
children_names=tuple(),
grand_children_names=tuple(),
node_trials=0,
parent_trials=0,
grand_parent_trials=0,
children_trials=tuple(),
grand_children_trials=tuple(),
total_trials=0,
):
"""Test fetching of trials from experiments in the EVC tree.
Parameters
----------
experiment_name: str
The name of the experiment that will be the main node for the tests.
parent_name: str or None
The name of the parent experiment, this will be used to fetch the trials from the parent
experiment directly (not in EVC) for comparison.
grand_parent_name: str or None
The name of the grand parent experiment, this will be used to fetch the trials from the
grand parent experiment directly (not in EVC) for comparison.
children_names: list or str
The names of the children experiments, this will be used to fetch the trials from the
children experiments directly (not in EVC) for comparison.
grand_children_names: list or str
The names of the grand children experiments, this will be used to fetch the trials from the
grand children experiments directly (not in EVC) for comparison. All grand children names
may be included in the list even though they are associated to different children.
node_trials: int,
The number of trials that should be fetched from current node experiment.
parent_trials: int,
The number of trials that should be fetched from parent experiment (not using EVC tree).
grand_parent_trials: int,
The number of trials that should be fetched from grand parent experiment (not using EVC tree).
children_trials: list of int,
The number of trials that should be fetched from each children experiment (not using EVC tree).
grand_children_trials: list of int,
The number of trials that should be fetched from each grand children experiment (not using EVC tree).
total_trials: int,
The number of trials that should be fetched from current node experiment when fetching
recursively from the EVC tree. This may not be equal to the sum of all trials in parent and
children experiments depending on the adapters.
"""
experiment = get_experiment(experiment_name)
exp_node = experiment.node
assert exp_node.item.name == experiment_name
num_nodes = 1
if parent_name:
assert exp_node.parent.item.name == parent_name
num_nodes += 1
if grand_parent_name:
assert exp_node.parent.parent.item.name == grand_parent_name
num_nodes += 1
assert len(exp_node.children) == len(children_names)
if children_names:
assert [child.item.name for child in exp_node.children] == children_names
num_nodes += len(children_names)
if grand_children_names:
grand_children = sum([child.children for child in exp_node.children], [])
assert [child.item.name for child in grand_children] == grand_children_names
num_nodes += len(grand_children_names)
assert len(list(exp_node.root)) == num_nodes
print("In node")
for trial in experiment.fetch_trials():
print(trial)
assert len(experiment.fetch_trials()) == node_trials
if parent_name:
print("In parent")
for trial in exp_node.parent.item.fetch_trials():
print(trial)
assert len(exp_node.parent.item.fetch_trials()) == parent_trials
if grand_parent_name:
print("In grand-parent")
for trial in exp_node.parent.parent.item.fetch_trials():
print(trial)
assert len(exp_node.parent.parent.item.fetch_trials()) == grand_parent_trials
if children_names:
print("In children")
for trial in exp_node.children[0].item.fetch_trials():
print(trial)
assert [
len(child.item.fetch_trials()) for child in exp_node.children
] == children_trials
if grand_children_names:
grand_children = sum([child.children for child in exp_node.children], [])
all_trials = sum(
[child_node.item.fetch_trials() for child_node in grand_children], []
)
print("In grand-children")
for trial in all_trials:
print(trial)
assert [
len(child.item.fetch_trials()) for child in grand_children
] == grand_children_trials
print("with evc")
for trial in experiment.fetch_trials(with_evc_tree=True):
print(trial)
assert len(experiment.fetch_trials(with_evc_tree=True)) == total_trials
all_ids = [trial.id for trial in experiment.fetch_trials(with_evc_tree=True)]
exp_ids = [trial.id for trial in experiment.fetch_trials(with_evc_tree=False)]
# Ensure all trials of experiment are fetched when fetching from all EVC
# It could happen that some trials are missing if duplicates are incorrectly filtered out
# from current node instead of from parent or child.
assert set(exp_ids) - set(all_ids) == set()
from the y-axis.
.. autofunction:: orion.plotting.base.regret
:noindex:
The regret plot can be executed directly from the ``experiment`` with ``plot.regret()`` as
shown in the example below.
"""
from orion.client import get_experiment
# Specify the database where the experiments are stored. We use a local PickleDB here.
storage = dict(type="legacy", database=dict(type="pickleddb", host="../db.pkl"))
# Load the data for the specified experiment
experiment = get_experiment("random-rosenbrock", storage=storage)
fig = experiment.plot.regret()
fig
#%%
# The objective of the trials is overlayed as a scatter plot under the regret curve.
# Thanks to this we can see whether the algorithm focused its optimization close to the
# optimum (if all points are close to the regret curve near the end) or if it explored far
# from it (if many points are far from the regret curve near the end). We can see in this example
# with random search that the algorithm unsurpringly randomly explored the space.
# If we plot the results from the algorithm TPE applied on the same task, we will see a very
# different pattern
# (see how the results were generated in tutorial :ref:`sphx_glr_auto_tutorials_code_1_python_api.py`).
experiment = get_experiment("tpe-rosenbrock", storage=storage)
fig = experiment.plot.regret()
Image
-----
Plots can be saved to multiple image formats using ``plotly``'s
`fig.write_image() <https://plotly.github.io/plotly.py-docs/generated/plotly.io.write_image.html>`_.
"""
from orion.client import get_experiment
# Specify the database where the experiments are stored. We use a local PickleDB here.
storage = dict(type="legacy",
database=dict(type="pickleddb", host="../../db.pkl"))
# Load the data for the specified experiment
experiment = get_experiment("2-dim-exp", storage=storage)
fig = experiment.plot.regret()
fig.write_image("regret.png")
#%%
# See `plotly`'s `short tutorial <https://plotly.com/python/static-image-export/>`_
# for more examples.
#
# HTML
# ----
#
# Using ``write_image`` is convenient for static images that can be included in PDF articles
# but it looses all advantages of plotly's interactive plots.
# You can save an HTML version instead using
# `fig.write_html() <https://plotly.github.io/plotly.py-docs/generated/plotly.io.write_html.html>`_
# if you wish to keep an interactive version of the plot.
