Пример #1
0
    def test_fork_lineages_branch_duplicates(self, space):
        num = 10
        pbt = SpaceTransformAlgoWrapper(
            PBT,
            space,
            exploit=ExploitStub(rval="toset").configuration,
            explore=ExploreStub(rval="toset").configuration,
            fork_timeout=0.05,
        ).algorithm

        trials = sample_trials(pbt.space,
                               num + 1,
                               status="completed",
                               objective=1)
        new_params_expected = trials[-1].params
        pbt.exploit_func.rval = trials[-1]
        pbt.explore_func.rval = new_params_expected
        for trial in trials:
            pbt.register(trial)

        pbt._queue = trials[:-1]

        with pytest.raises(RuntimeError):
            pbt._fork_lineages(num)

        # First queue.pop is fine, fails on second queue.pop.
        assert len(pbt._queue) == num - 2
Пример #2
0
    def test_fork_lineages_num_larger_than_queue(self, space):
        num = 10
        pbt = SpaceTransformAlgoWrapper(
            PBT,
            space,
            exploit=ExploitStub(rval=None).configuration,
        ).algorithm

        trials = sample_trials(pbt.space, num, status="completed", objective=1)
        for trial in trials:
            pbt.register(trial)

        pbt._queue = trials[:]

        num_fork = 4
        branched_trials = pbt._fork_lineages(num_fork)

        assert len(branched_trials) == num_fork
        assert len(pbt._queue) == num - num_fork

        trial_ids = [trial.id for trial in trials]

        assert [trial.parent
                for trial in branched_trials] == trial_ids[:num_fork]
        assert [trial.id for trial in pbt._queue] == trial_ids[num_fork:]
Пример #3
0
    def test_generate_offspring_exploit_branch(self, space):
        pbt = SpaceTransformAlgoWrapper(
            PBT,
            space,
            exploit=ExploitStub(rval="toset").configuration,
            explore=ExploreStub(rval="toset").configuration,
        ).algorithm

        trials = sample_trials(pbt.space, 3, status="completed", objective=1)

        trial_to_promote = trials[0]
        exploited_trial = trials[1]
        new_params_expected = trials[2].params

        pbt.exploit_func.rval = exploited_trial
        pbt.explore_func.rval = new_params_expected

        # Make sure they are different
        assert new_params_expected != trial_to_promote.params
        assert new_params_expected != exploited_trial.params

        pbt.register(trials[0])
        pbt.register(trials[1])

        trial_to_branch, new_trial = pbt._generate_offspring(trial_to_promote)

        new_params_expected["f"] = 10.9

        assert trial_to_branch is exploited_trial
        assert new_trial.params["f"] == new_params_expected["f"]
        assert new_trial.params == new_params_expected
Пример #4
0
    def test_generate_offspring_exploit_skip(self, space):

        pbt = SpaceTransformAlgoWrapper(
            PBT, space, exploit=ExploitStub(skip=True).configuration).algorithm
        trial = sample_trials(pbt.space, 1, status="completed", objective=1)[0]
        pbt.register(trial)

        trial_to_branch, new_trial = pbt._generate_offspring(trial)
        assert trial_to_branch is None
        assert new_trial is None
Пример #5
0
    def test_fork_lineages_skip_and_requeue_trials(self, space: Space):
        num = 10
        pbt = _create_algo(
            space,
            exploit=ExploitStub(skip=True).configuration,
        ).algorithm

        trials = sample_trials(pbt.space, num, status="completed", objective=1)

        for trial in trials:
            pbt.register(trial)

        pbt._queue = trials[:]

        assert pbt._fork_lineages(num) == []
        assert len(pbt._queue) == num
        assert pbt._queue == trials
Пример #6
0
    def test_generate_offspring_timeout(self, space: Space):

        pbt = _create_algo(
            space,
            exploit=ExploitStub(rval=None).configuration,
            explore=ExploreStub(rval="toset").configuration,
            fork_timeout=0.05,
        ).algorithm
        trial = sample_trials(pbt.space, 1, status="completed", objective=1)[0]
        pbt.explore_func.rval = trial.params

        pbt.register(trial)
        parent = trial.branch(params={"f": pbt.fidelities[space["f"].low]})
        pbt.register(parent)

        with pytest.raises(RuntimeError):
            pbt._generate_offspring(trial)
Пример #7
0
    def test_suggest_num_population_size_fork_completed(self, space, mocker):
        population_size = 10
        pbt = SpaceTransformAlgoWrapper(
            PBT,
            space,
            population_size=population_size,
            exploit=ExploitStub(rval=None).configuration,
        ).algorithm

        pbt_sample_mock = mocker.spy(pbt, "_sample")
        pbt_fork_mock = mocker.spy(pbt, "_fork_lineages")

        num = 4
        trials = pbt.suggest(num)
        assert len(trials) == num

        pbt_sample_mock.assert_called_with(num)
        pbt_fork_mock.assert_called_with(0)

        n_completed = 3
        for trial in trials[:n_completed]:
            trial.exp_working_dir = "/nothing"
            trial.status = "completed"
            trial._results.append(
                Trial.Result(name="objective", type="objective", value=1))

        pbt.observe(trials)
        assert len(pbt._queue) == n_completed

        # There are 4 trials sampled, out of which 3 are completed. Still missing 6 trials
        # for base population.
        assert len(pbt.suggest(num)) == num
        pbt_sample_mock.assert_called_with(num)
        pbt_fork_mock.assert_called_with(0)

        # There are 8 trials sampled, out of which 3 are completed. Still missing 2 trials
        # for base population.
        assert len(pbt.suggest(num)) == num
        pbt_sample_mock.assert_called_with(2)
        pbt_fork_mock.assert_called_with(2)
Пример #8
0
    def test_generate_offspring_exploit_promote(self, space):

        pbt = SpaceTransformAlgoWrapper(
            PBT,
            space,
            exploit=ExploitStub().configuration,
            explore=ExploreStub(no_call=True).configuration,
        ).algorithm
        trial = sample_trials(pbt.space, 1, status="completed", objective=1)[0]

        # Apply the transformation and revert it to have lossy effect (like small precision)
        trial = pbt.space.transform(
            pbt.space.reverse(pbt.space.transform(trial)))

        pbt.register(trial)

        new_params_expected = trial.params
        new_params_expected["f"] = 10.9

        trial_to_branch, new_trial = pbt._generate_offspring(trial)
        assert trial_to_branch is trial
        assert new_trial.params == new_params_expected
Пример #9
0
    def test_fork_lineages_num_smaller_than_queue(self, space: Space):
        num = 4
        pbt = _create_algo(
            space,
            exploit=ExploitStub(rval=None).configuration,
        ).algorithm

        trials = sample_trials(pbt.space, num, status="completed", objective=1)
        for trial in trials:
            pbt.register(trial)

        pbt._queue = trials[:]

        num_fork = 10
        branched_trials = pbt._fork_lineages(num_fork)

        assert len(branched_trials) == num
        assert len(pbt._queue) == 0

        trial_ids = [trial.id for trial in trials]

        assert [trial.parent for trial in branched_trials] == trial_ids
Пример #10
0
    def test_fork_lineages_branch_trials(self, space):
        num = 10

        pbt = SpaceTransformAlgoWrapper(
            PBT,
            space,
            exploit=ExploitStub(rval="toset").configuration,
            fork_timeout=0.05,
        ).algorithm

        trials = sample_trials(pbt.space,
                               num + 1,
                               status="completed",
                               objective=1)
        trial_to_branch = trials[-1]
        pbt.exploit_func.rval = trial_to_branch
        for trial in trials:
            pbt.register(trial)

        pbt._queue = trials[:-1]

        branched_trials = pbt._fork_lineages(num)

        assert len(trials) == num + 1
        assert len(branched_trials) == num
        assert pbt._queue == []

        for trial, branched_trial in zip(trials, branched_trials):
            # Check if parent is correct
            assert branched_trial.parent == trial_to_branch.id
            # Check in lineage if jump is set from correct base trial
            assert pbt.lineages.get_lineage(
                branched_trial).base.item.id == trial.id
            # Check if params are not duplicated
            should_not_be_params = trial_to_branch.params
            should_not_be_params["f"] = 10.9
            assert branched_trial.params["f"] == should_not_be_params["f"]
            assert branched_trial.params != should_not_be_params
Пример #11
0
    def test_fork_lineages_promote_trial(self, space: Space):
        num = 10
        pbt = _create_algo(
            space,
            exploit=ExploitStub(skip=None).configuration,
        ).algorithm

        trials = sample_trials(pbt.space, num, status="completed", objective=1)

        for trial in trials:
            pbt.register(trial)

        pbt._queue = trials[:]

        branched_trials = pbt._fork_lineages(num)
        assert len(trials) == num
        assert len(branched_trials) == num
        assert pbt._queue == []

        for trial, branched_trial in zip(trials, branched_trials):
            expected_params = trial.params
            expected_params["f"] = 10.9
            assert branched_trial.params == expected_params
Пример #12
0
    def test_generate_offspring_retry_using_same_trial(self, space,
                                                       monkeypatch):
        """Test that when exploit returns another trial, the base one is reused and case of
        duplicate samples
        """
        pbt = SpaceTransformAlgoWrapper(
            PBT,
            space,
            exploit=ExploitStub(rval="toset",
                                should_receive="toset").configuration,
            explore=ExploreStub(rval="toset").configuration,
            fork_timeout=0.0001,
        ).algorithm

        trials = sample_trials(pbt.space, 3, status="completed", objective=1)
        parent_trial = trials[0]
        base_trial = trials[1]
        sample_params = trials[2].params

        pbt.exploit_func.rval = parent_trial
        pbt.exploit_func.should_receive = base_trial
        pbt.explore_func.rval = sample_params

        pbt.register(parent_trial)
        pbt.register(base_trial)

        # The trial sampled will already be registered
        sample_params["f"] = pbt.fidelities[space["f"].low]
        child = parent_trial.branch(params=sample_params)
        pbt.register(child)

        # Exploit will return parent_trial, but Explore will return params of child, sampling
        # a duplite, since child is already registered. ExploitStub.should_receive will
        # test that base_trial is passed as expected to exploit when attempting more attemps
        # of exploit and explore.
        with pytest.raises(RuntimeError):
            pbt._generate_offspring(base_trial)