def hash_params(self):
"""See `~orion.core.worker.trial.Trial`"""
return OrionTrial.compute_trial_hash(self, ignore_fidelity=True)
def incomplete_trial():
"""Return a single trial without results"""
return Trial(params=[{'name': 'a', 'type': 'integer', 'value': 6}])
def test_register_lie(self, storage):
"""Test register lie"""
with OrionState(experiments=[base_experiment], storage=storage) as cfg:
storage = cfg.storage()
storage.register_lie(Trial(**base_trial))
def result(val):
return OrionTrial.Result(name=self.objective_key,
value=val,
type="objective")
def test_bad_access(self):
"""Other than `Trial.__slots__` are not allowed."""
t = Trial()
with pytest.raises(AttributeError):
t.asdfa = 3
def _add_result(trial: Trial, y: float) -> Trial:
trial = copy.deepcopy(trial)
trial.results = [_result(y)]
trial.status = "completed"
return trial
def test_conversion_to_dict(self, exp_config):
"""Convert to dictionary form for database using ``dict``."""
t = Trial(**exp_config[1][1])
assert t.to_dict() == exp_config[1][1]
def test_str_trial(self, exp_config):
"""Test representation of `Trial`."""
t = Trial(**exp_config[1][2])
assert str(t) == "Trial(experiment='supernaedo2', status='completed',\n"\
" params=/encoding_layer:gru\n"\
" /decoding_layer:lstm_with_attention)"
def dummy_param():
"""Give dummy param integer param with value 1"""
return Trial.Param(name='dummy', type='integer', value=1)
def _add_trial(**kwargs):
"""Add trials to the dummy orion instance"""
base_trial.update(copy.deepcopy(kwargs))
get_storage().register_trial(Trial(**base_trial))
def test_format_commandline_and_config(parser, commandline, json_config,
tmpdir, json_converter):
"""Format the commandline and a configuration file."""
cmd_args = json_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
trial = Trial(params=[{
'name': '/lr',
'type': 'real',
'value': -2.4
}, {
'name': '/prior',
'type': 'categorical',
'value': 'sgd'
}, {
'name': '/layers/1/width',
'type': 'integer',
'value': 100
}, {
'name': '/layers/1/type',
'type': 'categorical',
'value': 'relu'
}, {
'name': '/layers/2/type',
'type': 'categorical',
'value': 'sigmoid'
}, {
'name': '/training/lr0',
'type': 'real',
'value': 0.032
}, {
'name': '/training/mbs',
'type': 'integer',
'value': 64
}, {
'name': '/something-same',
'type': 'categorical',
'value': '3'
}])
output_file = str(tmpdir.join("output.json"))
cmd_inst = parser.format(output_file, trial)
assert cmd_inst == [
'--config', output_file, "--seed", "555", "--lr", "-2.4",
"--non-prior", "choices({'sgd': 0.2, 'adam': 0.8})", "--prior", "sgd"
]
output_data = json_converter.parse(output_file)
assert output_data == {
'yo':
5,
'training': {
'lr0': 0.032,
'mbs': 64
},
'layers': [{
'width': 64,
'type': 'relu'
}, {
'width': 100,
'type': 'relu'
}, {
'width': 16,
'type': 'sigmoid'
}],
'something-same':
'3'
}
def test_full_process(self, monkeypatch, hyperband):
"""Test Hyperband full process."""
sample_trials = [
create_trial_for_hb(("fidelity", i)) for i in range(100)
]
hyperband._refresh_brackets()
mock_samples(hyperband, copy.deepcopy(sample_trials))
# Fill all brackets' first rung
first_rung = hyperband.suggest(100)
first_bracket_first_rung = first_rung[6:]
second_bracket_first_rung = first_rung[3:6]
third_bracket_first_rung = first_rung[:3]
compare_trials(
first_bracket_first_rung,
[create_trial_for_hb((1, i)) for i in range(6, 15)],
)
compare_trials(
second_bracket_first_rung,
[create_trial_for_hb((3, i)) for i in range(3, 6)],
)
compare_trials(third_bracket_first_rung,
[create_trial_for_hb((9, i)) for i in range(3)])
assert hyperband.brackets[0].has_rung_filled(0)
assert not hyperband.brackets[0].is_ready()
assert hyperband.suggest(100) == []
assert hyperband.suggest(100) == []
# Observe first bracket first rung
for i, trial in enumerate(first_bracket_first_rung):
trial.status = "completed"
trial._results.append(
Trial.Result(name="objective", type="objective", value=16 - i))
hyperband.observe(first_bracket_first_rung)
assert hyperband.brackets[0].is_ready()
assert not hyperband.brackets[1].is_ready()
assert not hyperband.brackets[2].is_ready()
# Promote first bracket first rung
first_bracket_second_rung = hyperband.suggest(100)
compare_trials(
first_bracket_second_rung,
[create_trial_for_hb((3, 3 + 3 + 9 - 1 - i)) for i in range(3)],
)
assert hyperband.brackets[0].has_rung_filled(1)
assert not hyperband.brackets[0].is_ready()
assert not hyperband.brackets[1].is_ready()
assert not hyperband.brackets[2].is_ready()
# Observe first bracket second rung
for i, trial in enumerate(first_bracket_second_rung):
trial.status = "completed"
trial._results.append(
Trial.Result(name="objective", type="objective", value=8 - i))
hyperband.observe(first_bracket_second_rung)
assert hyperband.brackets[0].is_ready()
assert not hyperband.brackets[1].is_ready()
assert not hyperband.brackets[2].is_ready()
# Promote first bracket second rung
first_bracket_third_rung = hyperband.suggest(100)
compare_trials(first_bracket_third_rung,
[create_trial_for_hb((9, 12))])
assert hyperband.brackets[0].has_rung_filled(2)
assert not hyperband.brackets[0].is_ready()
assert not hyperband.brackets[1].is_ready()
assert not hyperband.brackets[2].is_ready()
# Observe second bracket first rung
for i, trial in enumerate(second_bracket_first_rung):
trial.status = "completed"
trial._results.append(
Trial.Result(name="objective", type="objective", value=8 - i))
hyperband.observe(second_bracket_first_rung)
assert not hyperband.brackets[0].is_ready()
assert hyperband.brackets[1].is_ready()
assert not hyperband.brackets[2].is_ready()
# Promote second bracket first rung
second_bracket_second_rung = hyperband.suggest(100)
compare_trials(second_bracket_second_rung,
[create_trial_for_hb((9, 5))])
assert not hyperband.brackets[0].is_ready()
assert hyperband.brackets[1].has_rung_filled(1)
assert not hyperband.brackets[1].is_ready()
assert not hyperband.brackets[2].is_ready()
# Observe third bracket first rung
for i, trial in enumerate(third_bracket_first_rung):
trial.status = "completed"
trial._results.append(
Trial.Result(name="objective", type="objective", value=3 - i))
hyperband.observe(third_bracket_first_rung)
assert not hyperband.brackets[0].is_ready(2)
assert not hyperband.brackets[1].is_ready(1)
assert hyperband.brackets[2].is_ready(0)
assert hyperband.brackets[2].is_done
# Observe second bracket second rung
for i, trial in enumerate(second_bracket_second_rung):
trial.status = "completed"
trial._results.append(
Trial.Result(name="objective", type="objective", value=5 - i))
hyperband.observe(second_bracket_second_rung)
assert not hyperband.brackets[0].is_ready(2)
assert hyperband.brackets[1].is_ready(1)
assert hyperband.brackets[1].is_done
hyperband.observe(first_bracket_third_rung)
assert hyperband.is_done
assert hyperband.brackets[0].is_done
assert hyperband.suggest(100) == []
# Refresh repeat and execution times property
monkeypatch.setattr(hyperband, "repetitions", 2)
# monkeypatch.setattr(hyperband.brackets[0], "repetition_id", 0)
# hyperband.observe([(9, 12)], [{"objective": 3 - i}])
assert len(hyperband.brackets) == 3
hyperband._refresh_brackets()
assert len(hyperband.brackets) == 6
mock_samples(hyperband,
copy.deepcopy(sample_trials[:3] + sample_trials))
trials = hyperband.suggest(100)
assert not hyperband.is_done
assert not hyperband.brackets[3].is_ready(2)
assert not hyperband.brackets[3].is_done
compare_trials(trials[:3],
map(create_trial_for_hb, [(9, 3), (9, 4), (9, 6)]))
compare_trials(trials[3:6],
map(create_trial_for_hb, [(3, 7), (3, 8), (3, 9)]))
compare_trials(trials[6:],
[create_trial_for_hb((1, i)) for i in range(15, 24)])
def test_set_state_dict(parser, commandline, json_config, tmpdir,
json_converter):
"""Test that set_state_dict sets state properly to generate new config."""
cmd_args = json_config
cmd_args.extend(commandline)
parser.parse(cmd_args)
state = parser.get_state_dict()
parser = None
blank_parser = OrionCmdlineParser(allow_non_existing_user_script=True)
blank_parser.set_state_dict(state)
trial = Trial(params=[{
'name': '/lr',
'type': 'real',
'value': -2.4
}, {
'name': '/prior',
'type': 'categorical',
'value': 'sgd'
}, {
'name': '/layers/1/width',
'type': 'integer',
'value': 100
}, {
'name': '/layers/1/type',
'type': 'categorical',
'value': 'relu'
}, {
'name': '/layers/2/type',
'type': 'categorical',
'value': 'sigmoid'
}, {
'name': '/training/lr0',
'type': 'real',
'value': 0.032
}, {
'name': '/training/mbs',
'type': 'integer',
'value': 64
}, {
'name': '/something-same',
'type': 'categorical',
'value': '3'
}])
output_file = str(tmpdir.join("output.json"))
cmd_inst = blank_parser.format(output_file, trial)
assert cmd_inst == [
'--config', output_file, "--seed", "555", "--lr", "-2.4",
"--non-prior", "choices({'sgd': 0.2, 'adam': 0.8})", "--prior", "sgd"
]
output_data = json_converter.parse(output_file)
assert output_data == {
'yo':
5,
'training': {
'lr0': 0.032,
'mbs': 64
},
'layers': [{
'width': 64,
'type': 'relu'
}, {
'width': 100,
'type': 'relu'
}, {
'width': 16,
'type': 'sigmoid'
}],
'something-same':
'3'
}
"working_dir",
]
read_write_only_methods = [
"fix_lost_trials",
"register_lie",
"register_trial",
"set_trial_status",
"update_completed_trial",
]
execute_only_methods = [
"reserve_trial",
]
ignore = ["non_branching_attrs", "mode", "node"]
dummy_trial = Trial(**_generate(base_trial, "status", value="reserved"))
stati = (["completed"] * 3) + (["reserved"] * 2)
trials = generate_trials(stati)
running_trial = Trial(**trials[-1])
kwargs = {
"fetch_trials_by_status": {
"status": "completed"
},
"get_trial": {
"uid": 0
},
"retrieve_result": {
"trial": dummy_trial
},
def test_build_trials(self, exp_config):
"""Convert to objects form using `Trial.build`."""
trials = Trial.build(exp_config[1])
assert list(map(lambda x: x.to_dict(), trials)) == exp_config[1]
def fetch_lost_trials(self, query):
trial_object = Trial(**trial)
trial_object.status = "reserved"
return [trial_object]
def test_invalid_result(self, exp_config):
"""Test that invalid objectives cannot be set"""
t = Trial(**exp_config[1][1])
# Make sure valid ones pass
t.results = [
Trial.Result(name="asfda", type="constraint", value=None),
Trial.Result(name="asfdb", type="objective", value=1e-5),
]
# No results at all
with pytest.raises(ValueError) as exc:
t.results = []
assert "No objective found" in str(exc.value)
# No objectives
with pytest.raises(ValueError) as exc:
t.results = [
Trial.Result(name="asfda", type="constraint", value=None)
]
assert "No objective found" in str(exc.value)
# Bad objective type
with pytest.raises(ValueError) as exc:
t.results = [
Trial.Result(name="asfda", type="constraint", value=None),
Trial.Result(name="asfdb", type="objective", value=None),
]
assert "Results must contain" in str(exc.value)
def test_str_value(self, exp_config):
"""Test representation of `Trial.Value`."""
t = Trial(**exp_config[1][2])
assert str(t.params[0]) == "Param(name='/encoding_layer', "\
"type='categorical', value='gru')"
def test_higher_shape_id_is_same(self):
"""Check if a Trial with a shape > 1 has the same id once it has been through the DB."""
x = {"name": "/x", "value": [1, 2], "type": "real"}
trial = Trial(params=[x])
assert (trial.id == Trial(
**bson.BSON.decode(bson.BSON.encode(trial.to_dict()))).id)
def test_bad_init(self):
"""Other than `Trial.__slots__` are not allowed in __init__ too."""
with pytest.raises(AttributeError):
Trial(ispii='iela')
def _result(y: float | np.ndarray) -> Trial.Result:
return Trial.Result(name="objective", type="objective", value=float(y))
