def test_suggest_promote_identic_objectives(self, hyperband: Hyperband, bracket: HyperbandBracket): """Test that identic objectives are handled properly""" hyperband.brackets = [bracket] assert bracket.owner is hyperband n_trials = 9 resources = 1 results = {} for param in np.linspace(0, 8, 9): trial = create_trial_for_hb((resources, param), objective=0) trial_hash = trial.compute_trial_hash( trial, ignore_fidelity=True, ignore_experiment=True, ) assert trial.objective is not None results[trial_hash] = (trial.objective.value, trial) bracket.rungs[0] = RungDict(n_trials=n_trials, resources=resources, results=results) candidates = hyperband.suggest(2) assert candidates is not None assert len(candidates) == 2 assert (sum(1 for trial in candidates if trial.params[hyperband.fidelity_index] == 3) == 2)
def test_suggest_duplicates_between_calls(self, monkeypatch, hyperband: Hyperband, bracket: HyperbandBracket): """Test that same trials are not allowed in different suggest call of the same hyperband execution. """ hyperband.brackets = [bracket] assert bracket.owner is hyperband fidelity = 1 duplicate_trial = create_trial_for_hb((fidelity, 0.0)) new_trial = create_trial_for_hb((fidelity, 0.5)) duplicate_id = hyperband.get_id(duplicate_trial, ignore_fidelity=True) bracket.rungs[0]["results"] = { duplicate_id: (0.0, duplicate_trial) } # type: ignore hyperband.trial_to_brackets[duplicate_id] = 0 trials = [duplicate_trial, new_trial] mock_samples( hyperband, trials + [create_trial_for_hb((fidelity, i)) for i in range(10 - 2)], ) trials = hyperband.suggest(100) assert trials is not None assert trials[0].params == new_trial.params
def test_suggest_duplicates_between_execution(self, monkeypatch, hyperband: Hyperband, budgets: list[BudgetTuple]): """Test that sampling collisions are handled between different hyperband execution.""" hyperband.repetitions = 2 bracket = HyperbandBracket(hyperband, budgets, 1) hyperband.brackets = [bracket] assert bracket.owner is hyperband for i in range(9): force_observe(hyperband, create_trial_for_hb((1, i), objective=i)) for i in range(3): force_observe(hyperband, create_trial_for_hb((3, i), objective=i)) force_observe(hyperband, create_trial_for_hb((9, 0), objective=0)) assert not hyperband.is_done # lr:7 and lr:8 are already sampled in first repetition, they should not be present # in second repetition. Samples with lr:7 and lr:8 will be ignored. # (9, 0) already exists candidates_for_epoch_9_bracket = [(9, 0), (9, 2), (9, 3), (9, 10)] # (9, 1) -> (3, 1) already promoted in last repetition # (9, 3) sampled for previous bracket candidates_for_epoch_3_bracket = [(9, 1), (9, 3), (9, 4), (9, 5), (9, 11)] # (9, 0) -> (1, 0) already sampled in last repetition # (9, 8) -> (1, 8) already sampled in last repetition candidates_for_epoch_1_bracket = [(9, 0), (9, 8), (9, 12), (9, 13)] zhe_point = list( map( create_trial_for_hb, candidates_for_epoch_9_bracket + candidates_for_epoch_3_bracket + candidates_for_epoch_1_bracket, )) hyperband._refresh_brackets() mock_samples(hyperband, zhe_point) zhe_samples = hyperband.suggest(100) assert zhe_samples is not None assert len(zhe_samples) == 8 assert zhe_samples[0].params == {"epoch": 9, "lr": 2} assert zhe_samples[1].params == {"epoch": 9, "lr": 3} assert zhe_samples[2].params == {"epoch": 9, "lr": 10} assert zhe_samples[3].params == {"epoch": 3, "lr": 4} assert zhe_samples[4].params == {"epoch": 3, "lr": 5} assert zhe_samples[5].params == {"epoch": 3, "lr": 11} assert zhe_samples[6].params == {"epoch": 1, "lr": 12} assert zhe_samples[7].params == {"epoch": 1, "lr": 13}
def test_suggest_promote(self, hyperband: Hyperband, bracket: HyperbandBracket, rung_0: RungDict): """Test that correct point is promoted and returned.""" hyperband.brackets = [bracket] assert bracket.owner is hyperband bracket.rungs[0] = rung_0 points = hyperband.suggest(100) assert points is not None assert len(points) == 3 assert points[0].params == {"epoch": 3, "lr": 0} assert points[1].params == {"epoch": 3, "lr": 1} assert points[2].params == {"epoch": 3, "lr": 2}
def test_suggest_duplicates_one_call(self, monkeypatch, hyperband: Hyperband, bracket: HyperbandBracket): """Test that same points are not allowed in the same suggest call ofxs the same hyperband execution. """ hyperband.brackets = [bracket] assert bracket.owner is hyperband zhe_point = list( map(create_trial_for_hb, [(1, 0.0), (1, 1.0), (1, 1.0), (1, 2.0)])) mock_samples(hyperband, zhe_point * 2) zhe_samples = hyperband.suggest(100) assert zhe_samples is not None assert zhe_samples[0].params["lr"] == 0.0 assert zhe_samples[1].params["lr"] == 1.0 assert zhe_samples[2].params["lr"] == 2.0 # zhe_point = mock_samples( hyperband, list( map( create_trial_for_hb, [ (3, 0.0), (3, 1.0), (3, 1.0), (3, 2.0), (3, 5.0), (3, 4.0), ], )), ) hyperband.trial_to_brackets[hyperband.get_id(create_trial_for_hb( (1, 0.0)), ignore_fidelity=True)] = 0 hyperband.trial_to_brackets[hyperband.get_id(create_trial_for_hb( (1, 0.0)), ignore_fidelity=True)] = 0 zhe_samples = hyperband.suggest(100) assert zhe_samples is not None assert zhe_samples[0].params["lr"] == 5.0 assert zhe_samples[1].params["lr"] == 4.0
def test_suggest_opt_out( self, hyperband: Hyperband, bracket: HyperbandBracket, rung_0: RungDict, rung_1: RungDict, rung_2: RungDict, ): """Test that Hyperband opts out when rungs are not ready.""" hyperband.brackets = [bracket] assert bracket.owner is hyperband bracket.rungs[0] = rung_0 trial_id = next(iter(rung_0["results"].keys())) objective, point = rung_0["results"][trial_id] rung_0["results"][trial_id] = (None, point) assert hyperband.suggest(100) == []
def test_suggest_inf_duplicates( self, monkeypatch, hyperband: Hyperband, bracket: HyperbandBracket, rung_0: RungDict, rung_1: RungDict, rung_2: RungDict, ): """Test that sampling inf collisions will return None.""" hyperband.brackets = [bracket] assert bracket.owner is hyperband zhe_trial = create_trial_for_hb(("fidelity", 0.0)) hyperband.trial_to_brackets[hyperband.get_id(zhe_trial, ignore_fidelity=True)] = 0 mock_samples(hyperband, [zhe_trial] * 2) assert hyperband.suggest(100) == []
def test_suggest_new( self, monkeypatch, hyperband: Hyperband, bracket: HyperbandBracket, rung_0: RungDict, rung_1: RungDict, rung_2: RungDict, ): """Test that a new point is sampled.""" hyperband.brackets = [bracket] assert bracket.owner is hyperband mock_samples(hyperband, [create_trial_for_hb(("fidelity", i)) for i in range(10)]) trials = hyperband.suggest(100) assert trials is not None assert trials[0].params == {"epoch": 1.0, "lr": 0} assert trials[1].params == {"epoch": 1.0, "lr": 1}
def test_register( self, hyperband: Hyperband, bracket: HyperbandBracket, rung_0: RungDict, rung_1: RungDict, ): """Check that a point is registered inside the bracket.""" hyperband.brackets = [bracket] assert bracket.owner is hyperband bracket.rungs = [rung_0, rung_1] trial = create_trial_for_hb((1, 0.0), 0.0) trial_id = hyperband.get_id(trial, ignore_fidelity=True) hyperband.observe([trial]) assert len(bracket.rungs[0]) assert trial_id in bracket.rungs[0]["results"] assert bracket.rungs[0]["results"][trial_id][0] == 0.0 assert bracket.rungs[0]["results"][trial_id][1].params == trial.params
def test_is_ready( self, hyperband: Hyperband, bracket: HyperbandBracket, rung_0: RungDict, rung_1: RungDict, rung_2: RungDict, ): """Test that Hyperband bracket detects when rung is ready.""" hyperband.brackets = [bracket] assert bracket.owner is hyperband bracket.rungs[0] = rung_0 rung = bracket.rungs[0]["results"] trial_id = next(iter(rung.keys())) objective, point = rung[trial_id] rung[trial_id] = (None, point) assert not bracket.is_ready() assert not bracket.is_ready(0) rung[trial_id] = (objective, point) assert bracket.is_ready() assert bracket.is_ready(0) assert not bracket.is_ready(1) assert not bracket.is_ready(2) bracket.rungs[1] = rung_1 rung = bracket.rungs[1]["results"] trial_id = next(iter(rung.keys())) objective, point = rung[trial_id] rung[trial_id] = (None, point) assert not bracket.is_ready( ) # Should depend on last rung that contains trials assert bracket.is_ready(0) assert not bracket.is_ready(1) assert not bracket.is_ready(2) rung[trial_id] = (objective, point) assert bracket.is_ready( ) # Should depend on last rung that contains trials assert bracket.is_ready(0) assert bracket.is_ready(1) assert not bracket.is_ready(2) bracket.rungs[2] = rung_2 rung = bracket.rungs[2]["results"] trial_id = next(iter(rung.keys())) objective, point = rung[trial_id] rung[trial_id] = (None, point) assert not bracket.is_ready( ) # Should depend on last rung that contains trials assert bracket.is_ready(0) assert bracket.is_ready(1) assert not bracket.is_ready(2) rung[trial_id] = (objective, point) assert bracket.is_ready( ) # Should depend on last rung that contains trials assert bracket.is_ready(0) assert bracket.is_ready(1) assert bracket.is_ready(2)
def test_is_filled( self, hyperband: Hyperband, bracket: HyperbandBracket, rung_0: RungDict, rung_1: RungDict, rung_2: RungDict, ): """Test that Hyperband bracket detects when rung is filled.""" hyperband.brackets = [bracket] assert bracket.owner is hyperband bracket.rungs[0] = rung_0 rung = bracket.rungs[0]["results"] trial_id = next(iter(rung.keys())) objective, point = rung.pop(trial_id) assert not bracket.is_filled assert not bracket.has_rung_filled(0) rung[trial_id] = (objective, point) assert bracket.is_filled assert bracket.has_rung_filled(0) assert not bracket.has_rung_filled(1) assert not bracket.has_rung_filled(2) bracket.rungs[1] = rung_1 rung = bracket.rungs[1]["results"] trial_id = next(iter(rung.keys())) objective, point = rung.pop(trial_id) assert bracket.is_filled # Should depend first rung only assert bracket.has_rung_filled(0) assert not bracket.has_rung_filled(1) rung[trial_id] = (objective, point) assert bracket.is_filled # Should depend first rung only assert bracket.has_rung_filled(0) assert bracket.has_rung_filled(1) assert not bracket.has_rung_filled(2) bracket.rungs[2] = rung_2 rung = bracket.rungs[2]["results"] trial_id = next(iter(rung.keys())) objective, point = rung.pop(trial_id) assert bracket.is_filled # Should depend first rung only assert bracket.has_rung_filled(0) assert bracket.has_rung_filled(1) assert not bracket.has_rung_filled(2) rung[trial_id] = (objective, point) assert bracket.is_filled # Should depend first rung only assert bracket.has_rung_filled(0) assert bracket.has_rung_filled(1) assert bracket.has_rung_filled(2)