def test_fork_lineages_branch_duplicates(self, space):
num = 10
pbt = SpaceTransformAlgoWrapper(
PBT,
space,
exploit=ExploitStub(rval="toset").configuration,
explore=ExploreStub(rval="toset").configuration,
fork_timeout=0.05,
).algorithm
trials = sample_trials(pbt.space,
num + 1,
status="completed",
objective=1)
new_params_expected = trials[-1].params
pbt.exploit_func.rval = trials[-1]
pbt.explore_func.rval = new_params_expected
for trial in trials:
pbt.register(trial)
pbt._queue = trials[:-1]
with pytest.raises(RuntimeError):
pbt._fork_lineages(num)
# First queue.pop is fine, fails on second queue.pop.
assert len(pbt._queue) == num - 2
def test_fork_lineages_num_larger_than_queue(self, space):
num = 10
pbt = SpaceTransformAlgoWrapper(
PBT,
space,
exploit=ExploitStub(rval=None).configuration,
).algorithm
trials = sample_trials(pbt.space, num, status="completed", objective=1)
for trial in trials:
pbt.register(trial)
pbt._queue = trials[:]
num_fork = 4
branched_trials = pbt._fork_lineages(num_fork)
assert len(branched_trials) == num_fork
assert len(pbt._queue) == num - num_fork
trial_ids = [trial.id for trial in trials]
assert [trial.parent
for trial in branched_trials] == trial_ids[:num_fork]
assert [trial.id for trial in pbt._queue] == trial_ids[num_fork:]
def test_generate_offspring_exploit_branch(self, space):
pbt = SpaceTransformAlgoWrapper(
PBT,
space,
exploit=ExploitStub(rval="toset").configuration,
explore=ExploreStub(rval="toset").configuration,
).algorithm
trials = sample_trials(pbt.space, 3, status="completed", objective=1)
trial_to_promote = trials[0]
exploited_trial = trials[1]
new_params_expected = trials[2].params
pbt.exploit_func.rval = exploited_trial
pbt.explore_func.rval = new_params_expected
# Make sure they are different
assert new_params_expected != trial_to_promote.params
assert new_params_expected != exploited_trial.params
pbt.register(trials[0])
pbt.register(trials[1])
trial_to_branch, new_trial = pbt._generate_offspring(trial_to_promote)
new_params_expected["f"] = 10.9
assert trial_to_branch is exploited_trial
assert new_trial.params["f"] == new_params_expected["f"]
assert new_trial.params == new_params_expected
def test_generate_offspring_exploit_skip(self, space):
pbt = SpaceTransformAlgoWrapper(
PBT, space, exploit=ExploitStub(skip=True).configuration).algorithm
trial = sample_trials(pbt.space, 1, status="completed", objective=1)[0]
pbt.register(trial)
trial_to_branch, new_trial = pbt._generate_offspring(trial)
assert trial_to_branch is None
assert new_trial is None
def test_fork_lineages_skip_and_requeue_trials(self, space: Space):
num = 10
pbt = _create_algo(
space,
exploit=ExploitStub(skip=True).configuration,
).algorithm
trials = sample_trials(pbt.space, num, status="completed", objective=1)
for trial in trials:
pbt.register(trial)
pbt._queue = trials[:]
assert pbt._fork_lineages(num) == []
assert len(pbt._queue) == num
assert pbt._queue == trials
def test_generate_offspring_timeout(self, space: Space):
pbt = _create_algo(
space,
exploit=ExploitStub(rval=None).configuration,
explore=ExploreStub(rval="toset").configuration,
fork_timeout=0.05,
).algorithm
trial = sample_trials(pbt.space, 1, status="completed", objective=1)[0]
pbt.explore_func.rval = trial.params
pbt.register(trial)
parent = trial.branch(params={"f": pbt.fidelities[space["f"].low]})
pbt.register(parent)
with pytest.raises(RuntimeError):
pbt._generate_offspring(trial)
def test_suggest_num_population_size_fork_completed(self, space, mocker):
population_size = 10
pbt = SpaceTransformAlgoWrapper(
PBT,
space,
population_size=population_size,
exploit=ExploitStub(rval=None).configuration,
).algorithm
pbt_sample_mock = mocker.spy(pbt, "_sample")
pbt_fork_mock = mocker.spy(pbt, "_fork_lineages")
num = 4
trials = pbt.suggest(num)
assert len(trials) == num
pbt_sample_mock.assert_called_with(num)
pbt_fork_mock.assert_called_with(0)
n_completed = 3
for trial in trials[:n_completed]:
trial.exp_working_dir = "/nothing"
trial.status = "completed"
trial._results.append(
Trial.Result(name="objective", type="objective", value=1))
pbt.observe(trials)
assert len(pbt._queue) == n_completed
# There are 4 trials sampled, out of which 3 are completed. Still missing 6 trials
# for base population.
assert len(pbt.suggest(num)) == num
pbt_sample_mock.assert_called_with(num)
pbt_fork_mock.assert_called_with(0)
# There are 8 trials sampled, out of which 3 are completed. Still missing 2 trials
# for base population.
assert len(pbt.suggest(num)) == num
pbt_sample_mock.assert_called_with(2)
pbt_fork_mock.assert_called_with(2)
def test_generate_offspring_exploit_promote(self, space):
pbt = SpaceTransformAlgoWrapper(
PBT,
space,
exploit=ExploitStub().configuration,
explore=ExploreStub(no_call=True).configuration,
).algorithm
trial = sample_trials(pbt.space, 1, status="completed", objective=1)[0]
# Apply the transformation and revert it to have lossy effect (like small precision)
trial = pbt.space.transform(
pbt.space.reverse(pbt.space.transform(trial)))
pbt.register(trial)
new_params_expected = trial.params
new_params_expected["f"] = 10.9
trial_to_branch, new_trial = pbt._generate_offspring(trial)
assert trial_to_branch is trial
assert new_trial.params == new_params_expected
def test_fork_lineages_num_smaller_than_queue(self, space: Space):
num = 4
pbt = _create_algo(
space,
exploit=ExploitStub(rval=None).configuration,
).algorithm
trials = sample_trials(pbt.space, num, status="completed", objective=1)
for trial in trials:
pbt.register(trial)
pbt._queue = trials[:]
num_fork = 10
branched_trials = pbt._fork_lineages(num_fork)
assert len(branched_trials) == num
assert len(pbt._queue) == 0
trial_ids = [trial.id for trial in trials]
assert [trial.parent for trial in branched_trials] == trial_ids
def test_fork_lineages_branch_trials(self, space):
num = 10
pbt = SpaceTransformAlgoWrapper(
PBT,
space,
exploit=ExploitStub(rval="toset").configuration,
fork_timeout=0.05,
).algorithm
trials = sample_trials(pbt.space,
num + 1,
status="completed",
objective=1)
trial_to_branch = trials[-1]
pbt.exploit_func.rval = trial_to_branch
for trial in trials:
pbt.register(trial)
pbt._queue = trials[:-1]
branched_trials = pbt._fork_lineages(num)
assert len(trials) == num + 1
assert len(branched_trials) == num
assert pbt._queue == []
for trial, branched_trial in zip(trials, branched_trials):
# Check if parent is correct
assert branched_trial.parent == trial_to_branch.id
# Check in lineage if jump is set from correct base trial
assert pbt.lineages.get_lineage(
branched_trial).base.item.id == trial.id
# Check if params are not duplicated
should_not_be_params = trial_to_branch.params
should_not_be_params["f"] = 10.9
assert branched_trial.params["f"] == should_not_be_params["f"]
assert branched_trial.params != should_not_be_params
def test_fork_lineages_promote_trial(self, space: Space):
num = 10
pbt = _create_algo(
space,
exploit=ExploitStub(skip=None).configuration,
).algorithm
trials = sample_trials(pbt.space, num, status="completed", objective=1)
for trial in trials:
pbt.register(trial)
pbt._queue = trials[:]
branched_trials = pbt._fork_lineages(num)
assert len(trials) == num
assert len(branched_trials) == num
assert pbt._queue == []
for trial, branched_trial in zip(trials, branched_trials):
expected_params = trial.params
expected_params["f"] = 10.9
assert branched_trial.params == expected_params
def test_generate_offspring_retry_using_same_trial(self, space,
monkeypatch):
"""Test that when exploit returns another trial, the base one is reused and case of
duplicate samples
"""
pbt = SpaceTransformAlgoWrapper(
PBT,
space,
exploit=ExploitStub(rval="toset",
should_receive="toset").configuration,
explore=ExploreStub(rval="toset").configuration,
fork_timeout=0.0001,
).algorithm
trials = sample_trials(pbt.space, 3, status="completed", objective=1)
parent_trial = trials[0]
base_trial = trials[1]
sample_params = trials[2].params
pbt.exploit_func.rval = parent_trial
pbt.exploit_func.should_receive = base_trial
pbt.explore_func.rval = sample_params
pbt.register(parent_trial)
pbt.register(base_trial)
# The trial sampled will already be registered
sample_params["f"] = pbt.fidelities[space["f"].low]
child = parent_trial.branch(params=sample_params)
pbt.register(child)
# Exploit will return parent_trial, but Explore will return params of child, sampling
# a duplite, since child is already registered. ExploitStub.should_receive will
# test that base_trial is passed as expected to exploit when attempting more attemps
# of exploit and explore.
with pytest.raises(RuntimeError):
pbt._generate_offspring(base_trial)