def test_new_config(init_full_x_new_config, monkeypatch):
"""Test experiment branching with new config"""
experiment = experiment_builder.load(name="full_x_new_config")
assert experiment.refers["adapter"].configuration == [
{
"change_type": "noeffect",
"of_type": "commandlinechange"
},
{
"of_type": "dimensionaddition",
"param": {
"name": "/y",
"type": "real",
"value": 0
},
},
{
"change_type": "unsure",
"of_type": "scriptconfigchange"
},
]
assert len(experiment.fetch_trials(with_evc_tree=True)) == 3
assert len(experiment.fetch_trials()) == 2
def test_no_update_id_from_parent(
clean_db, three_experiments_same_name_with_trials, capsys
):
"""Test trials of the parent experiment are not updated"""
exp1 = experiment_builder.load(name="test_single_exp", version=1)
exp2 = experiment_builder.load(name="test_single_exp", version=2)
exp3 = experiment_builder.load(name="test_single_exp", version=3)
trial = exp1.fetch_trials_by_status("broken")[0]
execute(f"db set -f test_single_exp id={trial.id} status=shouldnothappen")
assert len(exp1.fetch_trials_by_status("shouldnothappen")) == 0
assert len(exp2.fetch_trials_by_status("shouldnothappen")) == 0
assert len(exp3.fetch_trials_by_status("shouldnothappen")) == 0
captured = capsys.readouterr()
assert captured.out.endswith("0 trials modified\n")
def test_full_x_rename_half_y_half_z(init_full_x_rename_half_y_half_z):
"""Test if full x half z is properly initialized and can fetch from its 3 parents"""
experiment = experiment_builder.load(name="full_x_rename_half_y_half_z")
assert experiment.refers["adapter"].configuration == [
{
"of_type": "commandlinechange",
"change_type": "noeffect"
},
{
"of_type": "dimensionrenaming",
"old_name": "/y",
"new_name": "/z"
},
]
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((("/x", 5), ("/z", 5)), (("/x", -5), ("/z", 5)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == (
(("/x", 0), ("/z", 1)),
(("/x", 1), ("/z", 1)),
(("/x", -1), ("/z", 1)),
(("/x", 3), ("/z", 3)),
(("/x", -3), ("/z", 3)),
(("/x", 5), ("/z", 5)),
(("/x", -5), ("/z", 5)),
)
def main(args):
"""List all experiments inside database."""
config = experiment_builder.get_cmd_config(args)
experiment_builder.setup_storage(config.get("storage"))
query = {}
if args["name"]:
query["name"] = args["name"]
experiments = get_storage().fetch_experiments(query)
if args["name"]:
root_experiments = experiments
else:
root_experiments = [
exp for exp in experiments
if exp["refers"].get("root_id", exp["_id"]) == exp["_id"]
]
if not root_experiments:
print("No experiment found")
return
for root_experiment in root_experiments:
root = experiment_builder.load(
name=root_experiment["name"],
version=root_experiment.get("version")).node
print_tree(root, nameattr="tree_name")
def test_full_x_remove_y(init_full_x_remove_y):
"""Test if full x removed y is properly initialized and can fetch from its 2 parents"""
experiment = experiment_builder.load(name="full_x_remove_y")
assert experiment.refers["adapter"].configuration == [
{
"of_type": "commandlinechange",
"change_type": "noeffect"
},
{
"of_type": "dimensiondeletion",
"param": {
"name": "/y",
"type": "real",
"value": 1
},
},
]
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((("/x", 7), ), (("/x", -7), ))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == (
(("/x", 0), ),
(("/x", 1), ),
(("/x", -1), ),
(("/x", 7), ),
(("/x", -7), ),
)
def get_experiment(name, version=None, mode="r", storage=None):
"""
Retrieve an existing experiment as :class:`orion.client.experiment.ExperimentClient`.
Parameters
----------
name: str
The name of the experiment.
version: int, optional
Version to select. If None, last version will be selected. If version given is larger than
largest version available, the largest version will be selected.
mode: str, optional
The access rights of the experiment on the database.
'r': read access only
'w': can read and write to database
Default is 'r'
storage: dict, optional
Configuration of the storage backend.
Returns
-------
An instance of :class:`orion.client.experiment.ExperimentClient` representing the experiment.
Raises
------
`orion.core.utils.exceptions.NoConfigurationError`
The experiment is not in the database provided by the user.
"""
setup_storage(storage)
assert mode in set("rw")
experiment = experiment_builder.load(name, version, mode)
return ExperimentClient(experiment)
def test_full_x_full_y_add_z_remove_y(init_full_x_full_y_add_z_remove_y):
"""Test that if z is added and y removed at the same time, both are correctly detected"""
experiment = experiment_builder.load(name="full_x_full_z_remove_y")
assert experiment.refers["adapter"].configuration == [
{"of_type": "commandlinechange", "change_type": "noeffect"},
{
"of_type": "dimensiondeletion",
"param": {"name": "/y", "type": "real", "value": 1},
},
{
"of_type": "dimensionaddition",
"param": {"name": "/z", "type": "real", "value": 0},
},
]
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((("/x", 7), ("/z", 2)), (("/x", -7), ("/z", 2)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == (
(("/x", 0), ("/z", 0)),
(("/x", 1), ("/z", 0)),
(("/x", -1), ("/z", 0)),
(("/x", 7), ("/z", 2)),
(("/x", -7), ("/z", 2)),
)
def test_run_entire_full_x_full_y(init_entire):
"""Test if branched experiment can be executed without triggering a branching event again"""
experiment = experiment_builder.load(name="full_x_full_y")
assert experiment.refers["adapter"].configuration == [
{"change_type": "noeffect", "of_type": "commandlinechange"},
{
"of_type": "dimensionaddition",
"param": {"name": "/y", "type": "real", "value": 1},
},
]
assert len(experiment.fetch_trials(with_evc_tree=True)) == 23
assert len(experiment.fetch_trials()) == 4
orion.core.cli.main(
(
"-vv hunt --max-trials 20 --pool-size 1 -n full_x_full_y "
"./black_box_with_y.py "
"-x~uniform(-10,10) "
"-y~uniform(-10,10,default_value=1)"
).split(" ")
)
assert len(experiment.fetch_trials(with_evc_tree=True)) == 39
assert len(experiment.fetch_trials()) == 20
def main(args):
"""Remove the experiment(s) or trial(s)."""
config = experiment_builder.get_cmd_config(args)
experiment_builder.setup_storage(config.get("storage"))
# Find root experiment
root = experiment_builder.load(
name=args["name"], version=args.get("version", None)
).node
try:
query = build_query(args["query"])
update = build_update(args["update"])
except ValueError as e:
print(f"Error: {e}", file=sys.stderr)
return 1
# List all experiments with children
print_tree(root, nameattr="tree_name")
confirmed = confirm_name(
CONFIRM_MESSAGE.format(query=args["query"], update=args["update"]),
args["name"],
args["force"],
)
if not confirmed:
print("Confirmation failed, aborting operation.")
return 1
storage = get_storage()
process_updates(storage, root, query, update)
return 0
def test_full_x_half_y(init_full_x_half_y):
"""Test if full x half y is properly initialized and can fetch from its 2 parents"""
experiment = experiment_builder.load(name="full_x_half_y")
assert experiment.refers["adapter"].configuration == [{
"of_type":
"dimensionpriorchange",
"name":
"/y",
"old_prior":
"uniform(-10, 10, default_value=1)",
"new_prior":
"uniform(0, 10, default_value=1)",
}]
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((("/x", 3), ("/y", 3)), (("/x", -3), ("/y", 3)))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert pairs == (
(("/x", 0), ("/y", 1)),
(("/x", 1), ("/y", 1)),
(("/x", -1), ("/y", 1)),
(("/x", 3), ("/y", 3)),
(("/x", -3), ("/y", 3)),
)
def test_full_x_remove_z_default_4(init_full_x_remove_z_default_4):
"""Test if full x removed z (default 4) is properly initialized and can fetch
from 1 of its 3 parents
"""
experiment = experiment_builder.load(name="full_x_remove_z_default_4")
assert experiment.refers["adapter"].configuration == [
{
"change_type": "noeffect",
"of_type": "commandlinechange"
},
{
"of_type": "dimensiondeletion",
"param": {
"name": "/z",
"type": "real",
"value": 4.0
},
},
]
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((("/x", 9), ), (("/x", -9), ))
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
# Note that full_x and full_x_full_y are filtered out because default_value=4
assert pairs == ((("/x", 4), ), (("/x", -4), ), (("/x", 9), ), (("/x",
-9), ))
def test_init(init_full_x):
"""Test if original experiment contains trial 0"""
experiment = experiment_builder.load(name="full_x")
assert experiment.refers["adapter"].configuration == []
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == ((("/x", 0),),)
def test_build_without_config_hit(python_api_config):
"""Try building experiment without commandline config when in db (no branch)"""
name = "supernaekei"
with OrionState(experiments=[python_api_config], trials=[]):
# Test that experiment already exists (this should fail otherwise)
experiment_builder.load(name=name)
exp = experiment_builder.build(name=name)
assert exp._id == python_api_config["_id"]
assert exp.name == python_api_config["name"]
assert exp.configuration["refers"] == python_api_config["refers"]
assert exp.metadata == python_api_config["metadata"]
assert exp.max_trials == python_api_config["max_trials"]
assert exp.max_broken == python_api_config["max_broken"]
assert exp.algorithms.configuration == python_api_config["algorithms"]
def test_entire_full_x_full_y(init_entire):
"""Test if full x full y can fetch from its parent and all children"""
experiment = experiment_builder.load(name="full_x_full_y")
assert experiment.refers["adapter"].configuration == [
{"change_type": "noeffect", "of_type": "commandlinechange"},
{
"of_type": "dimensionaddition",
"param": {"name": "/y", "type": "real", "value": 1},
},
]
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == (
(("/x", 1), ("/y", 1)),
(("/x", -1), ("/y", 1)),
(("/x", 1), ("/y", -1)),
(("/x", -1), ("/y", -1)),
)
pairs = get_name_value_pairs(experiment.fetch_trials(with_evc_tree=True))
assert set(pairs) == set(
(
(("/x", 0), ("/y", 1)),
# full_x_full_y
(("/x", 1), ("/y", 1)),
(("/x", -1), ("/y", 1)),
(("/x", 1), ("/y", -1)),
(("/x", -1), ("/y", -1)),
# half_x_full_y
(("/x", 2), ("/y", 2)),
(("/x", 2), ("/y", -2)),
# full_x_half_y
(("/x", 3), ("/y", 3)),
(("/x", -3), ("/y", 3)),
# full_x_rename_y_z
(("/x", 4), ("/y", 4)),
(("/x", -4), ("/y", 4)),
(("/x", 4), ("/y", -4)),
(("/x", -4), ("/y", -4)),
# full_x_rename_half_y_half_z
(("/x", 5), ("/y", 5)),
(("/x", -5), ("/y", 5)),
# full_x_rename_half_y_full_z
(("/x", 6), ("/y", 6)),
(("/x", -6), ("/y", 6)),
# full_x_remove_y
(("/x", 7), ("/y", 1)),
(("/x", -7), ("/y", 1)),
# full_x_remove_z
(("/x", 8), ("/y", 1)),
(("/x", -8), ("/y", 1)),
# full_x_remove_z_default_4
(("/x", 9), ("/y", 4)),
(("/x", -9), ("/y", 4)),
)
)
def test_existing_experiment_read_mode(self, new_config):
"""Hit exp_name + user's name in the db, support read only."""
with OrionState(experiments=[new_config], trials=[]):
exp = experiment_builder.load(name="supernaekei", mode="r")
assert exp.mode == "r"
exp.fetch_trials() # Should pass
with pytest.raises(UnsupportedOperation) as exc:
exp.fix_lost_trials()
assert exc.match("to execute `fix_lost_trials()")
def test_load_unavailable_algo(algo_unavailable_config, capsys):
with OrionState(experiments=[algo_unavailable_config]):
experiment = experiment_builder.load("supernaekei", mode="r")
assert experiment.algorithms == algo_unavailable_config["algorithms"]
assert (
experiment.configuration["algorithms"]
== algo_unavailable_config["algorithms"]
)
experiment = experiment_builder.load("supernaekei", mode="w")
assert experiment.algorithms == algo_unavailable_config["algorithms"]
assert (
experiment.configuration["algorithms"]
== algo_unavailable_config["algorithms"]
)
with pytest.raises(NotImplementedError) as exc:
experiment_builder.build("supernaekei")
exc.match("Could not find implementation of BaseAlgorithm")
def test_run_entire_full_x_full_y_no_args(init_entire):
"""Test if branched experiment can be executed without script arguments"""
experiment = experiment_builder.load(name="full_x_full_y")
assert len(experiment.fetch_trials(with_evc_tree=True)) == 23
assert len(experiment.fetch_trials()) == 4
orion.core.cli.main(
("-vv hunt --max-trials 30 --pool-size 1 -n full_x_full_y").split(" "))
assert len(experiment.fetch_trials(with_evc_tree=True)) == 30
assert len(experiment.fetch_trials(with_evc_tree=False)) == 30
def item(self):
"""Get the experiment associated to the node
Note that accessing `item` may trigger the lazy initialization of the experiment if it was
not done already.
"""
if self._item is None:
# TODO: Find another way around the circular import
import orion.core.io.experiment_builder as experiment_builder
self._item = experiment_builder.load(name=self.name,
version=self.version)
self._item._node = self
return self._item
def main(args):
"""Remove the experiment(s) or trial(s)."""
config = experiment_builder.get_cmd_config(args)
experiment_builder.setup_storage(config.get("storage"))
# Find root experiment
root = experiment_builder.load(name=args["name"],
version=args.get("version", None)).node
# List all experiments with children
print_tree(root, nameattr="tree_name")
storage = get_storage()
release_locks(storage, root, args["name"], args["force"])
def test_new_cli(init_full_x_new_cli):
"""Test that new cli conflict is automatically resolved"""
experiment = experiment_builder.load(name="full_x_new_cli")
assert experiment.refers["adapter"].configuration == [
{"change_type": "noeffect", "of_type": "commandlinechange"}
]
assert len(experiment.fetch_trials(with_evc_tree=True)) == 3
assert len(experiment.fetch_trials()) == 2
orion.core.cli.main(
("-vv hunt --max-trials 20 --pool-size 1 -n full_x_new_cli").split(" ")
)
assert len(experiment.fetch_trials(with_evc_tree=True)) == 21
assert len(experiment.fetch_trials()) == 20
def test_new_algo(init_full_x_new_algo):
"""Test that new algo conflict is automatically resolved"""
experiment = experiment_builder.load(name="full_x_new_algo")
assert experiment.refers["adapter"].configuration == [{
"of_type":
"algorithmchange"
}]
assert len(experiment.fetch_trials(with_evc_tree=True)) == 3
assert len(experiment.fetch_trials()) == 2
orion.core.cli.main(
("-vv hunt --max-trials 20 --pool-size 1 -n full_x_new_algo"
).split(" "))
assert len(experiment.fetch_trials(with_evc_tree=True)) == 20
assert len(experiment.fetch_trials(with_evc_tree=False)) == 20
def test_no_evc_overwrite(setup_pickleddb_database, init_no_evc):
"""Test that the experiment config is overwritten if --enable-evc is not passed"""
storage = get_storage()
assert len(get_storage().fetch_experiments({})) == 1
experiment = experiment_builder.load(name="full_x")
assert experiment.refers["adapter"].configuration == []
assert experiment.space.configuration == {
"/x": "uniform(-10, 10)",
"/y": "uniform(-10, 10, default_value=1)",
}
pairs = get_name_value_pairs(experiment.fetch_trials())
assert pairs == (
(("/x", 1), ("/y", 1)),
(("/x", -1), ("/y", 1)),
(("/x", 1), ("/y", -1)),
(("/x", -1), ("/y", -1)),
)
def test_auto_resolution_with_fidelity(init_full_x_full_y, monkeypatch):
"""Test that auto-resolution does resolve all conflicts including new fidelity"""
# Patch cmdloop to avoid autoresolution's prompt
monkeypatch.setattr("sys.__stdin__.isatty", lambda: True)
name = "full_x_full_y"
branch = "half_x_no_y_new_w"
# If autoresolution was not successful, this to fail with a sys.exit without registering the
# experiment
orion.core.cli.main(("hunt --init-only -n {branch} --branch-from {name} "
"--enable-evc "
"./black_box_with_y.py "
"-x~uniform(0,10) "
"-w~fidelity(1,10)").format(name=name,
branch=branch).split(" "))
experiment = experiment_builder.load(name=branch)
assert experiment.refers["adapter"].configuration == [
{
"of_type": "dimensionpriorchange",
"name": "/x",
"new_prior": "uniform(0, 10)",
"old_prior": "uniform(-10, 10)",
},
{
"of_type": "dimensiondeletion",
"param": {
"name": "/y",
"type": "real",
"value": 1
},
},
{
"of_type": "dimensionaddition",
"param": {
"name": "/w",
"type": "fidelity",
"value": 10
},
},
]
def test_experiment_overwritten_evc_disabled(self, parent_version_config,
caplog):
"""Build an existing experiment with different config, overwriting previous config."""
parent_version_config.pop("version")
# Note: this is a bit of a hack, but we need to make sure the config is also compatible with gridsearch.
parent_version_config = copy.deepcopy(parent_version_config)
parent_version_config["space"] = {"/x": "uniform(0, 1)"}
with OrionState(experiments=[parent_version_config]):
with caplog.at_level(logging.WARNING):
exp = experiment_builder.build(
name=parent_version_config["name"])
assert "Running experiment in a different state" not in caplog.text
assert exp.version == 1
assert exp.configuration["algorithms"] == {
"random": {
"seed": None
}
}
new_algo = "gridsearch"
with caplog.at_level(logging.WARNING):
exp = experiment_builder.build(
name=parent_version_config["name"], algorithms=new_algo)
assert "Running experiment in a different state" in caplog.text
assert exp.version == 1
assert list(exp.configuration["algorithms"].keys())[0] == new_algo
caplog.clear()
with caplog.at_level(logging.WARNING):
exp = experiment_builder.load(
name=parent_version_config["name"])
assert "Running experiment in a different state" not in caplog.text
assert exp.version == 1
assert list(exp.configuration["algorithms"].keys())[0] == new_algo
