def test_suggest_promote_identic_objectives( self, asha: ASHA, bracket: ASHABracket, big_rung_0: RungDict, big_rung_1: RungDict, ): """Test that identic objectives are handled properly""" asha.brackets = [bracket] bracket.owner = asha n_trials = 9 resources = 1 results = {} for param in np.linspace(0, 8, 9): trial = create_trial_for_hb((resources, param), objective=0) trial_hash = trial.compute_trial_hash( trial, ignore_fidelity=True, ignore_experiment=True, ) assert trial.objective is not None results[trial_hash] = (trial.objective.value, trial) bracket.rungs[0] = RungDict(n_trials=n_trials, resources=resources, results=results) candidates = asha.suggest(2) assert len(candidates) == 2 assert (sum(1 for trial in candidates if trial.params[asha.fidelity_index] == 3) == 2)
def test_suggest_in_finite_cardinality(self): """Test that suggest None when search space is empty""" space = Space() space.register(Integer("yolo1", "uniform", 0, 5)) space.register(Fidelity("epoch", 1, 9, 3)) asha = ASHA(space) for i in range(6): force_observe( asha, create_trial( (1, i), names=("epoch", "yolo1"), types=("fidelity", "integer"), results={"objective": i}, ), ) for i in range(2): force_observe( asha, create_trial( (3, i), names=("epoch", "yolo1"), types=("fidelity", "integer"), results={"objective": i}, ), ) assert asha.suggest(1) == []
def test_suggest_promote(self, asha: ASHA, bracket: ASHABracket, rung_0: RungDict): """Test that correct point is promoted and returned.""" asha.brackets = [bracket] assert bracket.owner is asha bracket.rungs[0] = rung_0 trials = asha.suggest(1) assert trials[0].params == {"epoch": 3, "lr": 0.0}
def test_suggest_in_finite_cardinality(self): """Test that suggest None when search space is empty""" space = Space() space.register(Integer("yolo1", "uniform", 0, 5)) space.register(Fidelity("epoch", 1, 9, 3)) asha = ASHA(space) for i in range(6): force_observe(asha, (1, i), {"objective": i}) for i in range(2): force_observe(asha, (3, i), {"objective": i}) assert asha.suggest() is None
def test_suggest_in_finite_cardinality(self): """Test that suggest None when search space is empty""" space = Space() space.register(Integer('yolo1', 'uniform', 0, 6)) space.register(Fidelity('epoch', 1, 9, 3)) asha = ASHA(space) for i in range(6): asha.observe([(1, i)], [{'objective': i}]) for i in range(2): asha.observe([(3, i)], [{'objective': i}]) assert asha.suggest() is None
def test_suggest_promote_many( self, asha: ASHA, bracket: ASHABracket, big_rung_0: RungDict, big_rung_1: RungDict, ): """Test that correct points are promoted and returned.""" asha.brackets = [bracket] assert bracket.owner is asha bracket.rungs[0] = big_rung_0 bracket.rungs[1] = big_rung_1 candidates = asha.suggest(3) assert len(candidates) == 2 + 1 assert (sum(1 for trial in candidates if trial.params[asha.fidelity_index] == 9) == 2) assert (sum(1 for trial in candidates if trial.params[asha.fidelity_index] == 3) == 1)
def test_suggest_new( self, monkeypatch, asha: ASHA, bracket: ASHABracket, rung_0: RungDict, rung_1: RungDict, rung_2: RungDict, ): """Test that a new point is sampled.""" asha.brackets = [bracket] assert bracket.owner is asha def sample(num=1, seed=None): return [create_trial_for_hb(("fidelity", 0.5))] monkeypatch.setattr(asha.space, "sample", sample) trials = asha.suggest(1) assert trials[0].params == {"epoch": 1, "lr": 0.5}
def test_suggest_inf_duplicates( self, monkeypatch, asha: ASHA, bracket: ASHABracket, rung_0: RungDict, rung_1: RungDict, rung_2: RungDict, ): """Test that sampling inf collisions returns None.""" asha.brackets = [bracket] assert bracket.owner is asha fidelity = 1 zhe_trial = create_trial_for_hb((fidelity, 0.0)) asha.trial_to_brackets[asha.get_id(zhe_trial, ignore_fidelity=True)] = 0 def sample(num=1, seed=None): return [zhe_trial] monkeypatch.setattr(asha.space, "sample", sample) assert asha.suggest(1) == []
def test_suggest_duplicates( self, monkeypatch, asha: ASHA, bracket: ASHABracket, rung_0: RungDict, rung_1: RungDict, rung_2: RungDict, ): """Test that sampling collisions are handled.""" asha.brackets = [bracket] assert bracket.owner is asha fidelity = 1 duplicate_trial = create_trial_for_hb((fidelity, 0.0)) new_trial = create_trial_for_hb((fidelity, 0.5)) duplicate_id_wo_fidelity = asha.get_id(duplicate_trial, ignore_fidelity=True) bracket.rungs[0] = dict( n_trials=2, resources=1, results={duplicate_id_wo_fidelity: (0.0, duplicate_trial)}, ) asha.trial_to_brackets[duplicate_id_wo_fidelity] = 0 asha.register(duplicate_trial) trials = [duplicate_trial, new_trial] def sample(num=1, seed=None): return trials monkeypatch.setattr(asha.space, "sample", sample) assert asha.suggest(1)[0].params == new_trial.params