def hash_params(self): """See `~orion.core.worker.trial.Trial`""" return OrionTrial.compute_trial_hash(self, ignore_fidelity=True)
def incomplete_trial(): """Return a single trial without results""" return Trial(params=[{'name': 'a', 'type': 'integer', 'value': 6}])
def test_register_lie(self, storage): """Test register lie""" with OrionState(experiments=[base_experiment], storage=storage) as cfg: storage = cfg.storage() storage.register_lie(Trial(**base_trial))
def result(val): return OrionTrial.Result(name=self.objective_key, value=val, type="objective")
def test_bad_access(self): """Other than `Trial.__slots__` are not allowed.""" t = Trial() with pytest.raises(AttributeError): t.asdfa = 3
def _add_result(trial: Trial, y: float) -> Trial: trial = copy.deepcopy(trial) trial.results = [_result(y)] trial.status = "completed" return trial
def test_conversion_to_dict(self, exp_config): """Convert to dictionary form for database using ``dict``.""" t = Trial(**exp_config[1][1]) assert t.to_dict() == exp_config[1][1]
def test_str_trial(self, exp_config): """Test representation of `Trial`.""" t = Trial(**exp_config[1][2]) assert str(t) == "Trial(experiment='supernaedo2', status='completed',\n"\ " params=/encoding_layer:gru\n"\ " /decoding_layer:lstm_with_attention)"
def dummy_param(): """Give dummy param integer param with value 1""" return Trial.Param(name='dummy', type='integer', value=1)
def _add_trial(**kwargs): """Add trials to the dummy orion instance""" base_trial.update(copy.deepcopy(kwargs)) get_storage().register_trial(Trial(**base_trial))
def test_format_commandline_and_config(parser, commandline, json_config, tmpdir, json_converter): """Format the commandline and a configuration file.""" cmd_args = json_config cmd_args.extend(commandline) parser.parse(cmd_args) trial = Trial(params=[{ 'name': '/lr', 'type': 'real', 'value': -2.4 }, { 'name': '/prior', 'type': 'categorical', 'value': 'sgd' }, { 'name': '/layers/1/width', 'type': 'integer', 'value': 100 }, { 'name': '/layers/1/type', 'type': 'categorical', 'value': 'relu' }, { 'name': '/layers/2/type', 'type': 'categorical', 'value': 'sigmoid' }, { 'name': '/training/lr0', 'type': 'real', 'value': 0.032 }, { 'name': '/training/mbs', 'type': 'integer', 'value': 64 }, { 'name': '/something-same', 'type': 'categorical', 'value': '3' }]) output_file = str(tmpdir.join("output.json")) cmd_inst = parser.format(output_file, trial) assert cmd_inst == [ '--config', output_file, "--seed", "555", "--lr", "-2.4", "--non-prior", "choices({'sgd': 0.2, 'adam': 0.8})", "--prior", "sgd" ] output_data = json_converter.parse(output_file) assert output_data == { 'yo': 5, 'training': { 'lr0': 0.032, 'mbs': 64 }, 'layers': [{ 'width': 64, 'type': 'relu' }, { 'width': 100, 'type': 'relu' }, { 'width': 16, 'type': 'sigmoid' }], 'something-same': '3' }
def test_full_process(self, monkeypatch, hyperband): """Test Hyperband full process.""" sample_trials = [ create_trial_for_hb(("fidelity", i)) for i in range(100) ] hyperband._refresh_brackets() mock_samples(hyperband, copy.deepcopy(sample_trials)) # Fill all brackets' first rung first_rung = hyperband.suggest(100) first_bracket_first_rung = first_rung[6:] second_bracket_first_rung = first_rung[3:6] third_bracket_first_rung = first_rung[:3] compare_trials( first_bracket_first_rung, [create_trial_for_hb((1, i)) for i in range(6, 15)], ) compare_trials( second_bracket_first_rung, [create_trial_for_hb((3, i)) for i in range(3, 6)], ) compare_trials(third_bracket_first_rung, [create_trial_for_hb((9, i)) for i in range(3)]) assert hyperband.brackets[0].has_rung_filled(0) assert not hyperband.brackets[0].is_ready() assert hyperband.suggest(100) == [] assert hyperband.suggest(100) == [] # Observe first bracket first rung for i, trial in enumerate(first_bracket_first_rung): trial.status = "completed" trial._results.append( Trial.Result(name="objective", type="objective", value=16 - i)) hyperband.observe(first_bracket_first_rung) assert hyperband.brackets[0].is_ready() assert not hyperband.brackets[1].is_ready() assert not hyperband.brackets[2].is_ready() # Promote first bracket first rung first_bracket_second_rung = hyperband.suggest(100) compare_trials( first_bracket_second_rung, [create_trial_for_hb((3, 3 + 3 + 9 - 1 - i)) for i in range(3)], ) assert hyperband.brackets[0].has_rung_filled(1) assert not hyperband.brackets[0].is_ready() assert not hyperband.brackets[1].is_ready() assert not hyperband.brackets[2].is_ready() # Observe first bracket second rung for i, trial in enumerate(first_bracket_second_rung): trial.status = "completed" trial._results.append( Trial.Result(name="objective", type="objective", value=8 - i)) hyperband.observe(first_bracket_second_rung) assert hyperband.brackets[0].is_ready() assert not hyperband.brackets[1].is_ready() assert not hyperband.brackets[2].is_ready() # Promote first bracket second rung first_bracket_third_rung = hyperband.suggest(100) compare_trials(first_bracket_third_rung, [create_trial_for_hb((9, 12))]) assert hyperband.brackets[0].has_rung_filled(2) assert not hyperband.brackets[0].is_ready() assert not hyperband.brackets[1].is_ready() assert not hyperband.brackets[2].is_ready() # Observe second bracket first rung for i, trial in enumerate(second_bracket_first_rung): trial.status = "completed" trial._results.append( Trial.Result(name="objective", type="objective", value=8 - i)) hyperband.observe(second_bracket_first_rung) assert not hyperband.brackets[0].is_ready() assert hyperband.brackets[1].is_ready() assert not hyperband.brackets[2].is_ready() # Promote second bracket first rung second_bracket_second_rung = hyperband.suggest(100) compare_trials(second_bracket_second_rung, [create_trial_for_hb((9, 5))]) assert not hyperband.brackets[0].is_ready() assert hyperband.brackets[1].has_rung_filled(1) assert not hyperband.brackets[1].is_ready() assert not hyperband.brackets[2].is_ready() # Observe third bracket first rung for i, trial in enumerate(third_bracket_first_rung): trial.status = "completed" trial._results.append( Trial.Result(name="objective", type="objective", value=3 - i)) hyperband.observe(third_bracket_first_rung) assert not hyperband.brackets[0].is_ready(2) assert not hyperband.brackets[1].is_ready(1) assert hyperband.brackets[2].is_ready(0) assert hyperband.brackets[2].is_done # Observe second bracket second rung for i, trial in enumerate(second_bracket_second_rung): trial.status = "completed" trial._results.append( Trial.Result(name="objective", type="objective", value=5 - i)) hyperband.observe(second_bracket_second_rung) assert not hyperband.brackets[0].is_ready(2) assert hyperband.brackets[1].is_ready(1) assert hyperband.brackets[1].is_done hyperband.observe(first_bracket_third_rung) assert hyperband.is_done assert hyperband.brackets[0].is_done assert hyperband.suggest(100) == [] # Refresh repeat and execution times property monkeypatch.setattr(hyperband, "repetitions", 2) # monkeypatch.setattr(hyperband.brackets[0], "repetition_id", 0) # hyperband.observe([(9, 12)], [{"objective": 3 - i}]) assert len(hyperband.brackets) == 3 hyperband._refresh_brackets() assert len(hyperband.brackets) == 6 mock_samples(hyperband, copy.deepcopy(sample_trials[:3] + sample_trials)) trials = hyperband.suggest(100) assert not hyperband.is_done assert not hyperband.brackets[3].is_ready(2) assert not hyperband.brackets[3].is_done compare_trials(trials[:3], map(create_trial_for_hb, [(9, 3), (9, 4), (9, 6)])) compare_trials(trials[3:6], map(create_trial_for_hb, [(3, 7), (3, 8), (3, 9)])) compare_trials(trials[6:], [create_trial_for_hb((1, i)) for i in range(15, 24)])
def test_set_state_dict(parser, commandline, json_config, tmpdir, json_converter): """Test that set_state_dict sets state properly to generate new config.""" cmd_args = json_config cmd_args.extend(commandline) parser.parse(cmd_args) state = parser.get_state_dict() parser = None blank_parser = OrionCmdlineParser(allow_non_existing_user_script=True) blank_parser.set_state_dict(state) trial = Trial(params=[{ 'name': '/lr', 'type': 'real', 'value': -2.4 }, { 'name': '/prior', 'type': 'categorical', 'value': 'sgd' }, { 'name': '/layers/1/width', 'type': 'integer', 'value': 100 }, { 'name': '/layers/1/type', 'type': 'categorical', 'value': 'relu' }, { 'name': '/layers/2/type', 'type': 'categorical', 'value': 'sigmoid' }, { 'name': '/training/lr0', 'type': 'real', 'value': 0.032 }, { 'name': '/training/mbs', 'type': 'integer', 'value': 64 }, { 'name': '/something-same', 'type': 'categorical', 'value': '3' }]) output_file = str(tmpdir.join("output.json")) cmd_inst = blank_parser.format(output_file, trial) assert cmd_inst == [ '--config', output_file, "--seed", "555", "--lr", "-2.4", "--non-prior", "choices({'sgd': 0.2, 'adam': 0.8})", "--prior", "sgd" ] output_data = json_converter.parse(output_file) assert output_data == { 'yo': 5, 'training': { 'lr0': 0.032, 'mbs': 64 }, 'layers': [{ 'width': 64, 'type': 'relu' }, { 'width': 100, 'type': 'relu' }, { 'width': 16, 'type': 'sigmoid' }], 'something-same': '3' }
"working_dir", ] read_write_only_methods = [ "fix_lost_trials", "register_lie", "register_trial", "set_trial_status", "update_completed_trial", ] execute_only_methods = [ "reserve_trial", ] ignore = ["non_branching_attrs", "mode", "node"] dummy_trial = Trial(**_generate(base_trial, "status", value="reserved")) stati = (["completed"] * 3) + (["reserved"] * 2) trials = generate_trials(stati) running_trial = Trial(**trials[-1]) kwargs = { "fetch_trials_by_status": { "status": "completed" }, "get_trial": { "uid": 0 }, "retrieve_result": { "trial": dummy_trial },
def test_build_trials(self, exp_config): """Convert to objects form using `Trial.build`.""" trials = Trial.build(exp_config[1]) assert list(map(lambda x: x.to_dict(), trials)) == exp_config[1]
def fetch_lost_trials(self, query): trial_object = Trial(**trial) trial_object.status = "reserved" return [trial_object]
def test_invalid_result(self, exp_config): """Test that invalid objectives cannot be set""" t = Trial(**exp_config[1][1]) # Make sure valid ones pass t.results = [ Trial.Result(name="asfda", type="constraint", value=None), Trial.Result(name="asfdb", type="objective", value=1e-5), ] # No results at all with pytest.raises(ValueError) as exc: t.results = [] assert "No objective found" in str(exc.value) # No objectives with pytest.raises(ValueError) as exc: t.results = [ Trial.Result(name="asfda", type="constraint", value=None) ] assert "No objective found" in str(exc.value) # Bad objective type with pytest.raises(ValueError) as exc: t.results = [ Trial.Result(name="asfda", type="constraint", value=None), Trial.Result(name="asfdb", type="objective", value=None), ] assert "Results must contain" in str(exc.value)
def test_str_value(self, exp_config): """Test representation of `Trial.Value`.""" t = Trial(**exp_config[1][2]) assert str(t.params[0]) == "Param(name='/encoding_layer', "\ "type='categorical', value='gru')"
def test_higher_shape_id_is_same(self): """Check if a Trial with a shape > 1 has the same id once it has been through the DB.""" x = {"name": "/x", "value": [1, 2], "type": "real"} trial = Trial(params=[x]) assert (trial.id == Trial( **bson.BSON.decode(bson.BSON.encode(trial.to_dict()))).id)
def test_bad_init(self): """Other than `Trial.__slots__` are not allowed in __init__ too.""" with pytest.raises(AttributeError): Trial(ispii='iela')
def _result(y: float | np.ndarray) -> Trial.Result: return Trial.Result(name="objective", type="objective", value=float(y))