def test_fork_non_existing_trial(self):
lineages = Lineages()
trial = TrialStub(id="stub")
new_trial = TrialStub(id="fork")
with pytest.raises(KeyError):
new_lineage = lineages.fork(trial, new_trial)
def test_fork_identical_new_trial(self):
lineage = LineageNode(TrialStub(id="my-id", working_dir="same_folder"))
with pytest.raises(
RuntimeError, match="The new trial new-id has the same working directory"
):
lineage.fork(TrialStub(id="new-id", working_dir="same_folder"))
assert lineage.children == []
def test_add_duplicate(self):
lineages = Lineages()
assert len(lineages) == 0
lineage = lineages.add(TrialStub(id="stub"))
assert len(lineages) == 1
new_lineage = lineages.add(TrialStub(id="stub"))
assert new_lineage is lineage
assert len(lineages) == 1
def test_register_existing_trial(self):
lineages = Lineages()
trial = TrialStub(id="my-id")
lineage = lineages.add(trial)
assert lineages._lineage_roots == [lineage]
assert lineage.item.objective is None
trial.objective = ObjectiveStub(1)
assert lineages.register(trial) is lineage
assert lineages._lineage_roots == [lineage]
assert lineage.item.objective.value == 1
def test_fork_existing_trial(self, tmp_path):
lineages = Lineages()
trial = TrialStub(id="stub", working_dir=os.path.join(tmp_path, "stub"))
os.makedirs(trial.working_dir)
lineage = lineages.add(trial)
assert len(lineages) == 1
new_trial = TrialStub(id="fork", working_dir=os.path.join(tmp_path, "fork"))
new_lineage = lineages.fork(trial, new_trial)
assert len(lineages) == 1
assert lineages._lineage_roots[0].children[0] is new_lineage
assert lineages._trial_to_lineages["fork"] is new_lineage
def test_fork_to_existing_path(self, tmp_path):
trial = TrialStub(id="stub", working_dir=os.path.join(tmp_path, "stub"))
os.makedirs(trial.working_dir)
lineage = LineageNode(trial)
new_trial = TrialStub(id="fork", working_dir=os.path.join(tmp_path, "fork"))
os.makedirs(new_trial.working_dir)
with pytest.raises(
FileExistsError, match="Folder already exists for trial fork."
):
lineage.fork(new_trial)
assert lineage.children == []
def test_fork(self, mocker):
path = "/some_path"
trial = TrialStub(path)
lineage = LineageNode(trial)
new_path = "/another_path"
new_trial = TrialStub(new_path)
mocker.patch("shutil.copytree")
new_lineage = lineage.fork(new_trial)
shutil.copytree.assert_called_once_with(path, new_path)
assert new_lineage.item.working_dir == new_trial.working_dir
assert new_lineage.parent is lineage
assert lineage.children[0] is new_lineage
def test_add_new_trial(self):
lineages = Lineages()
assert len(lineages) == 0
lineage = lineages.add(TrialStub(id="stub"))
assert len(lineages) == 1
assert lineages._lineage_roots[0] is lineage
assert lineages._trial_to_lineages["stub"] is lineage
def test_set_jump_existing_trial(self):
lineages = Lineages()
root_1 = TrialStub(id="root-1")
lineage_1 = lineages.add(root_1)
root_2 = TrialStub(id="root-2")
lineage_2 = lineages.add(root_2)
child_trial = TrialStub(id="child")
child_lineage = lineages.fork(root_1, child_trial)
lineages.set_jump(root_2, child_trial)
assert child_lineage.base is lineage_2
assert lineage_2.jumps == [child_lineage]
assert child_lineage.jumps == []
assert lineage_2.base is None
assert lineage_1.jumps == []
assert lineage_1.base is None
def test_exploit_otherwise_next(self):
for i in range(4):
exploit = PipelineExploit([
dict(of_type="exploitstub",
rval=None if j < i else i,
some="args") for j in range(4)
])
assert exploit(RNGStub(), TrialStub(), None) == i
def test_get_lineage_existing_node_trial(self):
lineages = Lineages()
for root_index in range(2):
trial = TrialStub(id=f"lineage-{root_index}-0")
lineage = lineages.add(trial)
for depth in range(1, 10):
new_trial = TrialStub(id=f"lineage-{root_index}-{depth}")
lineage = lineages.fork(trial, new_trial)
trial = new_trial
lineage = lineages.get_lineage(TrialStub(id="lineage-0-2"))
assert lineage.root is lineages._lineage_roots[0]
assert lineage.node_depth == 2
lineage = lineages.get_lineage(TrialStub(id="lineage-1-5"))
assert lineage.root is lineages._lineage_roots[1]
assert lineage.node_depth == 5
def test_get_trials_at_depth_given_existing_trial(self):
population_size = 5
generations = 10
lineages = build_population([
list(range(population_size)) for generation in range(generations)
])
for depth in [0, 1, 5, 9]:
lineage_index = random.choice(range(population_size))
trial = TrialStub(id=f"lineage-{lineage_index}-{depth}")
compare_generations(lineages.get_trials_at_depth(trial),
population_size, depth)
def test_fetch_trials_properly(self, space, monkeypatch):
lineages = build_lineages_for_exploit(space, monkeypatch)
exploit = self.constructor()
def test_truncate_args(rng, trial, trials):
assert trials == self.get_trials(lineages, trial)
monkeypatch.setattr(exploit, "_truncate", test_truncate_args)
exploit(RNGStub(), TrialStub(id="selected-trial"), lineages)
def test_truncate(self, truncation_quantile, space, monkeypatch):
"""Test threshold at which is needed based on truncation_quantile"""
# Test that trial within threshold is not replaced
lineages = build_lineages_for_exploit(space, monkeypatch)
trials = self.get_trials(lineages, TrialStub(objective=50))
trials = sorted(trials, key=lambda trial: trial.objective.value)
threshold_index = int(truncation_quantile * len(trials))
good_trial = trials[threshold_index - 1]
selected_trial = trials[-1]
# Add non completed trials and shuffle the list to test it is filtered and sorted properly
lots_of_trials = trials + space.sample(20, seed=2)
numpy.random.shuffle(lots_of_trials)
exploit = self.constructor(truncation_quantile=truncation_quantile,
candidate_pool_ratio=0.2)
if truncation_quantile > 0.0:
def mocked_choice(choices, *args, **kwargs):
raise RuntimeError("Should not be called")
rng = RNGStub()
rng.choice = mocked_choice
trial = exploit._truncate(
rng,
good_trial,
lots_of_trials,
)
assert trial is good_trial
if truncation_quantile < 1.0:
bad_trial = trials[threshold_index]
def mocked_choice(choices, *args, **kwargs):
return -1
rng = RNGStub()
rng.choice = mocked_choice
trial = exploit._truncate(
rng,
bad_trial,
lots_of_trials,
)
assert trial is selected_trial
def test_truncate_valid_choice(self, candidate_pool_ratio, space,
monkeypatch):
"""Test the pool of available trials based on candidate_pool_ratio"""
lineages = build_lineages_for_exploit(space, monkeypatch)
trials = self.get_trials(lineages, TrialStub(objective=50))
trials = sorted(trials, key=lambda trial: trial.objective.value)
num_completed_trials = len(trials)
valid_choices = numpy.arange(
int(candidate_pool_ratio * num_completed_trials)).tolist()
selected_trial = trials[valid_choices[-1]]
def mocked_choice(choices, *args, **kwargs):
assert choices.tolist() == valid_choices
return valid_choices[-1]
rng = RNGStub()
rng.choice = mocked_choice
completed_trial_index = numpy.random.choice(range(len(trials)))
completed_trial = trials[completed_trial_index]
# Add non completed trials and shuffle the list to test it is filtered and sorted properly
trials += space.sample(20, seed=2)
numpy.random.shuffle(trials)
exploit = self.constructor(truncation_quantile=0,
candidate_pool_ratio=candidate_pool_ratio)
trial = exploit._truncate(
rng,
completed_trial,
trials,
)
assert trial is selected_trial
def test_get_best_trial_non_completed_root(self):
lineage = LineageNode(TrialStub(id="my-id"))
assert lineage.get_best_trial() is None
def test_get_lineage_existing_root_trial(self):
lineages = Lineages()
trial = TrialStub(id="stub")
lineage = lineages.add(trial)
assert lineages.get_lineage(trial) is lineage
def test_get_trials_at_depth_given_non_existing_trial(self):
lineages = Lineages()
with pytest.raises(KeyError, match="idontexist"):
lineages.get_trials_at_depth(TrialStub(id="idontexist"))
def test_get_elites_none_completed(self):
lineages = Lineages()
lineages.add(TrialStub(id="1"))
lineages.add(TrialStub(id="2"))
lineages.add(TrialStub(id="3"))
assert lineages.get_elites() == []
def test_get_lineage_non_existing_trial(self):
lineages = Lineages()
with pytest.raises(KeyError):
lineages.get_lineage(TrialStub(id="id"))
def test_register_new_trial(self):
lineages = Lineages()
new_trial = TrialStub(id="new")
lineage = lineages.register(new_trial)
assert lineages._lineage_roots == [lineage]
def test_get_best_trial_empty(self):
trial = TrialStub(id="id-1", objective=1)
lineage = LineageNode(trial)
assert lineage.get_best_trial().id == "id-1"
def test_no_exploit(self):
trial = TrialStub()
assert PipelineExploit([])(RNGStub(), trial, None) is trial
def test_set_jump_non_existing_base_trial(self):
lineages = Lineages()
with pytest.raises(KeyError, match="'dontexist'"):
lineages.set_jump(
TrialStub(id="dontexist"), TrialStub(id="dontexistbutdoesntmatter")
)
def test_truncate_not_enough_trials(self, space, monkeypatch):
lineages = build_lineages_for_exploit(space, monkeypatch, num=4)
exploit = self.constructor(min_forking_population=5)
assert exploit(RNGStub(), TrialStub(), lineages) is None
def test_set_jump_non_existing_new_trial(self):
lineages = Lineages()
trial = TrialStub(id="exists")
lineages.add(trial)
with pytest.raises(KeyError, match="'newtrialdontexist'"):
lineages.set_jump(trial, TrialStub(id="newtrialdontexist"))