def test_get_teams(self, bracket: BracketEVES, rung_3: RungDict):
"""Test that correct team is promoted."""
bracket.rungs[0] = rung_3
rung, population_range, red_team, blue_team = bracket._get_teams(0)
assert len(list(rung.values())) == 4
assert bracket.search_space_without_fidelity == [1, 2]
assert population_range == 4
assert set(red_team).union(set(blue_team)) == {0, 1, 2, 3}
assert set(red_team).intersection(set(blue_team)) == set()
def test_mutate_population(self, bracket: BracketEVES, rung_3: RungDict):
"""Verify mutated candidates is generated correctly."""
red_team = [0, 2]
blue_team = [1, 3]
population_range = 4
rung_trials = list(rung_3["results"].values())
for trial_index in range(4):
objective, trial = rung_trials[trial_index]
bracket.owner.performance[trial_index] = objective
for ith_dim in [1, 2]:
bracket.owner.population[ith_dim][trial_index] = trial.params[
bracket.owner.space[ith_dim].name
]
org_data = np.stack(
(
list(bracket.owner.population.values())[0],
list(bracket.owner.population.values())[1],
),
axis=0,
).T
org_data = copy.deepcopy(org_data)
bracket._mutate_population(
red_team, blue_team, rung_3["results"], population_range, fidelity=2
)
mutated_data = np.stack(
(
list(bracket.owner.population.values())[0],
list(bracket.owner.population.values())[1],
),
axis=0,
).T
# Winner team will be [0, 2], so [0, 2] will be remained, [1, 3] will be mutated.
assert org_data.shape == mutated_data.shape
assert (mutated_data[0] == org_data[0]).all()
assert (mutated_data[2] == org_data[2]).all()
assert (mutated_data[1] != org_data[1]).any()
assert (mutated_data[3] != org_data[3]).any()
assert (mutated_data[1] != org_data[0]).any()
assert (mutated_data[3] != org_data[2]).any()
# For each individual, mutation occurs in only one dimension chosen from two.
if mutated_data[1][0] != org_data[0][0]:
assert mutated_data[1][1] == org_data[0][1]
else:
assert mutated_data[1][1] != org_data[0][1]
if mutated_data[3][0] != org_data[2][0]:
assert mutated_data[3][1] == org_data[2][1]
else:
assert mutated_data[3][1] != org_data[2][1]
def test_duplicated_mutated_population(
self, bracket: BracketEVES, rung_4: RungDict
):
"""Verify duplicated candidates can be found and processed correctly."""
red_team = [0, 2]
blue_team = [0, 2] # no mutate occur at first.
population_range = 4
rung_trials = list(rung_4["results"].values())
# Duplicate second item
rung_trials.insert(2, rung_trials[1])
for trial_index in range(4):
objective, trial = rung_trials[trial_index]
# bracket.eves.performance[trial_index] = objective
for ith_dim in [1, 2]:
bracket.owner.population[ith_dim][trial_index] = trial.params[
bracket.owner.space[ith_dim].name
]
trials, nums_all_equal = bracket._mutate_population(
red_team, blue_team, rung_4["results"], population_range, fidelity=2
)
# In this case, duplication will occur, and we can make it mutate one more time.
# The trials 1 and 2 should be different, while one of nums_all_equal should be 1.
if trials[1].params["lr"] != trials[2].params["lr"]:
assert trials[1].params["weight_decay"] == trials[2].params["weight_decay"]
else:
assert trials[1].params["weight_decay"] != trials[2].params["weight_decay"]
assert nums_all_equal[0] == 0
assert nums_all_equal[1] == 0
assert nums_all_equal[2] == 1
assert nums_all_equal[3] == 0
def test_register(
self,
evolution: EvolutionES,
bracket: BracketEVES,
rung_0: RungDict,
rung_1: RungDict,
):
"""Check that a point is registered inside the bracket."""
evolution.brackets = [bracket]
bracket.owner = evolution
bracket.owner = evolution
bracket.rungs = [rung_0, rung_1]
trial = create_trial_for_hb((1, 0.0), objective=0.0)
trial_id = evolution.get_id(trial, ignore_fidelity=True)
evolution.observe([trial])
assert len(bracket.rungs[0])
assert trial_id in bracket.rungs[0]["results"]
assert bracket.rungs[0]["results"][trial_id][0] == 0.0
assert bracket.rungs[0]["results"][trial_id][1].params == trial.params
def test_mutate_trials(self, bracket: BracketEVES, rung_3: RungDict):
"""Test that correct trial is promoted."""
red_team = [0, 2]
blue_team = [0, 2]
population_range = 4
rung_trials = list(rung_3["results"].values())
for trial_index in range(4):
objective, trial = rung_trials[trial_index]
# bracket.eves.performance[trial_index] = objective
for ith_dim in [1, 2]:
bracket.owner.population[ith_dim][trial_index] = trial.params[
bracket.owner.space[ith_dim].name
]
trials, nums_all_equal = bracket._mutate_population(
red_team, blue_team, rung_3["results"], population_range, fidelity=2
)
assert trials[0].params == {"epoch": 2, "lr": 1.0, "weight_decay": 1.0}
assert trials[1].params == {"epoch": 2, "lr": 1.0 / 2, "weight_decay": 1.0 / 4}
assert (nums_all_equal == 0).all()
def bracket(budgets, evolution, space1):
"""Return a `Bracket` instance configured with `b_config`."""
return BracketEVES(evolution, budgets, 1, space1)
def test_customized_mutate_population(space1, rung_3, budgets):
"""Verify customized mutated candidates is generated correctly."""
customerized_dict = {
"function": "orion.testing.state.customized_mutate_example",
"multiply_factor": 2.0,
"add_factor": 1,
}
algo = EvolutionES(
space1, repetitions=1, nums_population=4, mutate=customerized_dict
)
algo.brackets[0] = BracketEVES(algo, budgets, 1, space1)
red_team = [0, 2]
blue_team = [1, 3]
population_range = 4
for i in range(4):
for j in [1, 2]:
algo.brackets[0].eves.population[j][i] = list(rung_3["results"].values())[
i
][1][j]
algo.brackets[0].eves.performance[i] = list(rung_3["results"].values())[i][0]
org_data = np.stack(
(
list(algo.brackets[0].eves.population.values())[0],
list(algo.brackets[0].eves.population.values())[1],
),
axis=0,
).T
org_data = copy.deepcopy(org_data)
algo.brackets[0]._mutate_population(
red_team, blue_team, rung_3["results"], population_range
)
mutated_data = np.stack(
(
list(algo.brackets[0].eves.population.values())[0],
list(algo.brackets[0].eves.population.values())[1],
),
axis=0,
).T
# Winner team will be [0, 2], so [0, 2] will be remained, [1, 3] will be mutated.
assert org_data.shape == mutated_data.shape
assert (mutated_data[0] == org_data[0]).all()
assert (mutated_data[2] == org_data[2]).all()
assert (mutated_data[1] != org_data[1]).any()
assert (mutated_data[3] != org_data[3]).any()
assert (mutated_data[1] != org_data[0]).any()
assert (mutated_data[3] != org_data[2]).any()
# For each individual, mutation occurs in only one dimension chosen from two.
# Customized test mutation function is divided by 2 for real type.
if mutated_data[1][0] == org_data[0][0] / customerized_dict["multiply_factor"]:
assert mutated_data[1][1] == org_data[0][1]
else:
assert (
mutated_data[1][1] == org_data[0][1] / customerized_dict["multiply_factor"]
)
if mutated_data[3][0] == org_data[2][0] / customerized_dict["multiply_factor"]:
assert mutated_data[3][1] == org_data[2][1]
else:
assert (
mutated_data[3][1] == org_data[2][1] / customerized_dict["multiply_factor"]
)
def bracket(budgets: list[BudgetTuple], evolution: EvolutionES, space1: Space):
"""Return a `Bracket` instance configured with `b_config`."""
return BracketEVES(evolution, budgets, 1)
def test_customized_mutate_population(
space1: Space, rung_3: RungDict, budgets: list[BudgetTuple]
):
"""Verify customized mutated candidates is generated correctly."""
customerized_dict = {
"function": "orion.testing.algo.customized_mutate_example",
"multiply_factor": 2.0,
"add_factor": 1,
}
population_range = len(rung_3["results"])
algo = EvolutionES(
space1,
repetitions=1,
nums_population=population_range,
mutate=customerized_dict,
)
algo.brackets[0] = BracketEVES(algo, budgets, 1)
red_team = [0, 2]
blue_team = [1, 3]
rung_trials = list(rung_3["results"].values())
for trial_index in range(population_range):
objective, trial = rung_trials[trial_index]
algo.performance[trial_index] = objective
for ith_dim in [1, 2]:
algo.population[ith_dim][trial_index] = trial.params[
algo.space[ith_dim].name
]
org_data = np.stack(
(
list(algo.brackets[0].owner.population.values())[0],
list(algo.brackets[0].owner.population.values())[1],
),
axis=0,
).T
org_data = copy.deepcopy(org_data)
algo.brackets[0]._mutate_population(
red_team, blue_team, rung_3["results"], population_range, fidelity=2
)
mutated_data = np.stack(
(
list(algo.brackets[0].owner.population.values())[0],
list(algo.brackets[0].owner.population.values())[1],
),
axis=0,
).T
# Winner team will be [0, 2], so [0, 2] will be remained, [1, 3] will be mutated.
assert org_data.shape == mutated_data.shape
assert (mutated_data[0] == org_data[0]).all()
assert (mutated_data[2] == org_data[2]).all()
assert (mutated_data[1] != org_data[1]).any()
assert (mutated_data[3] != org_data[3]).any()
assert (mutated_data[1] != org_data[0]).any()
assert (mutated_data[3] != org_data[2]).any()
# For each individual, mutation occurs in only one dimension chosen from two.
# Customized test mutation function is divided by 2 for real type.
if mutated_data[1][0] == org_data[0][0] / customerized_dict["multiply_factor"]:
assert mutated_data[1][1] == org_data[0][1]
else:
assert (
mutated_data[1][1] == org_data[0][1] / customerized_dict["multiply_factor"]
)
if mutated_data[3][0] == org_data[2][0] / customerized_dict["multiply_factor"]:
assert mutated_data[3][1] == org_data[2][1]
else:
assert (
mutated_data[3][1] == org_data[2][1] / customerized_dict["multiply_factor"]
)