def three_experiments_same_name_with_trials(two_experiments_same_name,
storage):
"""Create three experiments with the same name but different versions."""
orion.core.cli.main([
"hunt",
"--init-only",
"-n",
"test_single_exp",
"./black_box.py",
"--x~uniform(0,1)",
"--y~normal(0,1)",
"--z~+normal(0,1)",
])
ensure_deterministic_id("test_single_exp", storage, version=3)
exp = experiment_builder.build(name="test_single_exp", version=1)
exp2 = experiment_builder.build(name="test_single_exp", version=2)
exp3 = experiment_builder.build(name="test_single_exp", version=3)
x = {"name": "/x", "type": "real"}
y = {"name": "/y", "type": "real"}
z = {"name": "/z", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
y["value"] = x_value * 10
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x], status=status)
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
trial3 = Trial(experiment=exp3.id, params=[x, y, z], status=status)
Database().write("trials", trial.to_dict())
Database().write("trials", trial2.to_dict())
Database().write("trials", trial3.to_dict())
x_value += 1
def family_with_trials(two_experiments):
"""Create two related experiments with all types of trials."""
exp = experiment_builder.build(name='test_double_exp')
exp2 = experiment_builder.build(name='test_double_exp_child')
x = {'name': '/x', 'type': 'real'}
y = {'name': '/y', 'type': 'real'}
x_value = 0
for status in Trial.allowed_stati:
x['value'] = x_value
y['value'] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x['value'] = x_value
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
x_value += 1
Database().write('trials', trial.to_dict())
Database().write('trials', trial2.to_dict())
def family_with_trials(two_experiments):
"""Create two related experiments with all types of trials."""
exp = experiment_builder.build(name="test_double_exp")
exp2 = experiment_builder.build(name="test_double_exp_child")
x = {"name": "/x", "type": "real"}
y = {"name": "/y", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
y["value"] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x["value"] = x_value
trial2 = Trial(experiment=exp2.id, params=[x, y], status=status)
x_value += 1
Database().write("trials", trial.to_dict())
Database().write("trials", trial2.to_dict())
def single_with_trials(single_without_success):
"""Create an experiment with all types of trials."""
exp = ExperimentBuilder().build_from({'name': 'test_single_exp'})
x = {'name': '/x', 'type': 'real', 'value': 100}
results = {"name": "obj", "type": "objective", "value": 0}
trial = Trial(experiment=exp.id, params=[x], status='completed', results=[results])
Database().write('trials', trial.to_dict())
def single_with_trials(single_without_success):
"""Create an experiment with all types of trials."""
exp = experiment_builder.build(name="test_single_exp")
x = {"name": "/x", "type": "real", "value": 100}
results = {"name": "obj", "type": "objective", "value": 0}
trial = Trial(experiment=exp.id, params=[x], status="completed", results=[results])
database_factory.create().write("trials", trial.to_dict())
return exp.configuration
def three_family_with_trials(three_experiments_family, family_with_trials):
"""Create three experiments, all related, two direct children, with all types of trials."""
exp = experiment_builder.build(name='test_double_exp_child2')
x = {'name': '/x', 'type': 'real'}
z = {'name': '/z', 'type': 'real'}
x_value = 0
for status in Trial.allowed_stati:
x['value'] = x_value
z['value'] = x_value * 100
trial = Trial(experiment=exp.id, params=[x, z], status=status)
x_value += 1
Database().write('trials', trial.to_dict())
def three_family_with_trials(three_experiments_family, family_with_trials):
"""Create three experiments, all related, two direct children, with all types of trials."""
exp = experiment_builder.build(name="test_double_exp_child2")
x = {"name": "/x", "type": "real"}
z = {"name": "/z", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x, z], status=status)
x_value += 1
Database().write("trials", trial.to_dict())
def single_without_success(one_experiment):
"""Create an experiment without a succesful trial."""
statuses = list(Trial.allowed_stati)
statuses.remove('completed')
exp = experiment_builder.build(name='test_single_exp')
x = {'name': '/x', 'type': 'real'}
x_value = 0
for status in statuses:
x['value'] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x_value += 1
Database().write('trials', trial.to_dict())
def single_without_success(one_experiment):
"""Create an experiment without a succesful trial."""
statuses = list(Trial.allowed_stati)
statuses.remove("completed")
exp = experiment_builder.build(name="test_single_exp")
x = {"name": "/x", "type": "real"}
x_value = 0
for status in statuses:
x["value"] = x_value
trial = Trial(experiment=exp.id, params=[x], status=status)
x_value += 1
Database().write("trials", trial.to_dict())
def three_family_branch_with_trials(three_experiments_family_branch, family_with_trials):
"""Create three experiments, all related, one child and one grandchild,
with all types of trials.
"""
exp = ExperimentBuilder().build_from({'name': 'test_double_exp_grand_child'})
x = {'name': '/x', 'type': 'real'}
y = {'name': '/y', 'type': 'real'}
z = {'name': '/z', 'type': 'real'}
x_value = 0
for status in Trial.allowed_stati:
x['value'] = x_value
y['value'] = x_value * 10
z['value'] = x_value * 100
trial = Trial(experiment=exp.id, params=[x, y, z], status=status)
x_value += 1
Database().write('trials', trial.to_dict())
def three_family_branch_with_trials(three_experiments_family_branch,
family_with_trials):
"""Create three experiments, all related, one child and one grandchild,
with all types of trials.
"""
exp = experiment_builder.build(name="test_double_exp_grand_child")
x = {"name": "/x", "type": "real"}
y = {"name": "/y", "type": "real"}
z = {"name": "/z", "type": "real"}
x_value = 0
for status in Trial.allowed_stati:
x["value"] = x_value
y["value"] = x_value * 10
z["value"] = x_value * 100
trial = Trial(experiment=exp.id, params=[x, y, z], status=status)
x_value += 1
Database().write("trials", trial.to_dict())
def test_convertion_to_dict(self, exp_config):
"""Convert to dictionary form for database using ``dict``."""
t = Trial(**exp_config[1][1])
assert t.to_dict() == exp_config[1][1]
def test_higher_shape_id_is_same(self):
"""Check if a Trial with a shape > 1 has the same id once it has been through the DB."""
x = {'name': '/x', 'value': [1, 2], 'type': 'real'}
trial = Trial(params=[x])
assert trial.id == Trial(
**bson.BSON.decode(bson.BSON.encode(trial.to_dict()))).id
def test_higher_shape_id_is_same(self):
"""Check if a Trial with a shape > 1 has the same id once it has been through the DB."""
x = {"name": "/x", "value": [1, 2], "type": "real"}
trial = Trial(params=[x])
assert (trial.id == Trial(
**bson.BSON.decode(bson.BSON.encode(trial.to_dict()))).id)
def test_conversion_to_dict(self, trial_config):
"""Convert to dictionary form for database using ``dict``."""
t = Trial(**trial_config)
assert t.to_dict() == trial_config
def compare_registered_trial(registered_trial: tuple[float | None, Trial],
trial: Trial) -> None:
assert trial.objective is not None
assert registered_trial[0] == trial.objective.value
assert registered_trial[1].to_dict() == trial.to_dict()
