def test_perturb_int_no_duplicate_below(self):
explore = PerturbExplore(factor=0.75)
rng = RNGStub()
rng.random = lambda: 1.0
assert explore.perturb_int(rng, 1, (0, 10)) == 0
def test_perturb_int_duplicate_equal(self):
explore = PerturbExplore(factor=1.0)
rng = RNGStub()
rng.random = lambda: 1.0
assert explore.perturb_int(rng, 1, (0, 10)) == 1
def test_perturb_real_volatility_above(self, volatility):
explore = PerturbExplore(factor=1.0, volatility=volatility)
rng = RNGStub()
rng.random = lambda: 1.0
rng.normal = lambda mean, variance: variance
assert explore.perturb_real(rng, 3.0, (1.0, 2.0)) == 2.0 - volatility
def test_perturb_cat(self):
explore = PerturbExplore()
rng = RNGStub()
rng.randint = lambda low, high, size: [1]
rng.choice = lambda choices: choices[0]
dim = Categorical("name", ["one", "two", 3, 4.0])
assert explore.perturb_cat(rng, "whatever", dim) in dim
def test_perturb_real_factor(self, factor):
explore = PerturbExplore(factor=factor)
rng = RNGStub()
rng.random = lambda: 1.0
assert explore.perturb_real(rng, 1.0, (0.1, 2.0)) == factor
rng.random = lambda: 0.0
assert explore.perturb_real(rng, 1.0, (0.1, 2.0)) == 1.0 / factor
def test_perturb_real_above_interval_cap(self):
explore = PerturbExplore(factor=1.0, volatility=0)
rng = RNGStub()
rng.random = lambda: 1.0
rng.normal = lambda mean, variance: variance
assert explore.perturb_real(rng, 3.0, (1.0, 2.0)) == 2.0
explore.volatility = 1000
assert explore.perturb_real(rng, 3.0, (1.0, 2.0)) == 1.0
def test_perturb_int_factor(self, factor):
explore = PerturbExplore(factor=factor)
rng = RNGStub()
rng.random = lambda: 1.0
assert explore.perturb_int(rng, 5,
(0, 10)) == int(numpy.round(5 * factor))
rng.random = lambda: 0.0
assert explore.perturb_int(rng, 5,
(0, 10)) == int(numpy.round(5 / factor))
def test_resample_probability(self, space):
explore = ResampleExplore(probability=0.5)
rng = RNGStub()
rng.randint = lambda low, high, size: [1]
rng.random = lambda: 0.5
params = {"x": 1.0, "y": 2, "z": 0, "f": 10}
assert explore(rng, space, params) is params
rng.random = lambda: 0.4
assert explore(rng, space, params) is not params
def test_exploit_otherwise_next(self):
for i in range(4):
exploit = PipelineExploit([
dict(of_type="exploitstub",
rval=None if j < i else i,
some="args") for j in range(4)
])
assert exploit(RNGStub(), TrialStub(), None) == i
def test_perturb_with_invalid_dim(self, space, monkeypatch):
explore = PerturbExplore()
monkeypatch.setattr(Dimension, "type", "type_that_dont_exist")
with pytest.raises(
ValueError,
match="Unsupported dimension type type_that_dont_exist"):
explore(RNGStub(), space, {"x": 1.0, "y": 2, "z": 0, "f": 10})
def test_fetch_trials_properly(self, space, monkeypatch):
lineages = build_lineages_for_exploit(space, monkeypatch)
exploit = self.constructor()
def test_truncate_args(rng, trial, trials):
assert trials == self.get_trials(lineages, trial)
monkeypatch.setattr(exploit, "_truncate", test_truncate_args)
exploit(RNGStub(), TrialStub(id="selected-trial"), lineages)
def test_truncate_valid_choice(self, candidate_pool_ratio, space,
monkeypatch):
"""Test the pool of available trials based on candidate_pool_ratio"""
lineages = build_lineages_for_exploit(space, monkeypatch)
trials = self.get_trials(lineages, TrialStub(objective=50))
trials = sorted(trials, key=lambda trial: trial.objective.value)
num_completed_trials = len(trials)
valid_choices = numpy.arange(
int(candidate_pool_ratio * num_completed_trials)).tolist()
selected_trial = trials[valid_choices[-1]]
def mocked_choice(choices, *args, **kwargs):
assert choices.tolist() == valid_choices
return valid_choices[-1]
rng = RNGStub()
rng.choice = mocked_choice
completed_trial_index = numpy.random.choice(range(len(trials)))
completed_trial = trials[completed_trial_index]
# Add non completed trials and shuffle the list to test it is filtered and sorted properly
trials += space.sample(20, seed=2)
numpy.random.shuffle(trials)
exploit = self.constructor(truncation_quantile=0,
candidate_pool_ratio=candidate_pool_ratio)
trial = exploit._truncate(
rng,
completed_trial,
trials,
)
assert trial is selected_trial
def test_perturb(self, space):
explore = PerturbExplore()
rng = RNGStub()
rng.randint = lambda low, high, size: [1]
rng.random = lambda: 1.0
rng.normal = lambda mean, variance: 0.0
rng.choice = lambda choices: choices[0]
params = {"x": 1.0, "y": 2, "z": 0, "f": 10}
new_params = explore(rng, space, params)
for key in space.keys():
assert new_params[key] in space[key]
def test_truncate(self, truncation_quantile, space, monkeypatch):
"""Test threshold at which is needed based on truncation_quantile"""
# Test that trial within threshold is not replaced
lineages = build_lineages_for_exploit(space, monkeypatch)
trials = self.get_trials(lineages, TrialStub(objective=50))
trials = sorted(trials, key=lambda trial: trial.objective.value)
threshold_index = int(truncation_quantile * len(trials))
good_trial = trials[threshold_index - 1]
selected_trial = trials[-1]
# Add non completed trials and shuffle the list to test it is filtered and sorted properly
lots_of_trials = trials + space.sample(20, seed=2)
numpy.random.shuffle(lots_of_trials)
exploit = self.constructor(truncation_quantile=truncation_quantile,
candidate_pool_ratio=0.2)
if truncation_quantile > 0.0:
def mocked_choice(choices, *args, **kwargs):
raise RuntimeError("Should not be called")
rng = RNGStub()
rng.choice = mocked_choice
trial = exploit._truncate(
rng,
good_trial,
lots_of_trials,
)
assert trial is good_trial
if truncation_quantile < 1.0:
bad_trial = trials[threshold_index]
def mocked_choice(choices, *args, **kwargs):
return -1
rng = RNGStub()
rng.choice = mocked_choice
trial = exploit._truncate(
rng,
bad_trial,
lots_of_trials,
)
assert trial is selected_trial
def test_perturb_int_no_out_of_bounds(self):
explore = PerturbExplore(factor=0.75, volatility=0)
rng = RNGStub()
rng.random = lambda: 1.0
rng.normal = lambda mean, variance: variance
assert explore.perturb_int(rng, 0, (0, 10)) == 0
rng.random = lambda: 0.0
rng.normal = lambda mean, variance: variance
assert explore.perturb_int(rng, 10, (0, 10)) == 10
def test_perturb_hierarchical_params(self, hspace):
explore = PerturbExplore()
rng = RNGStub()
rng.randint = lambda low, high, size: [1]
rng.random = lambda: 1.0
rng.normal = lambda mean, variance: 0.0
rng.choice = lambda choices: choices[0]
params = {"numerical": {"x": 1.0, "y": 2, "f": 10}, "z": 0}
new_params = explore(rng, hspace, params)
assert "numerical" in new_params
assert "x" in new_params["numerical"]
for key in hspace.keys():
assert flatten(new_params)[key] in hspace[key]
def test_truncate_not_enough_trials(self, space, monkeypatch):
lineages = build_lineages_for_exploit(space, monkeypatch, num=4)
exploit = self.constructor(min_forking_population=5)
assert exploit(RNGStub(), TrialStub(), lineages) is None
def test_no_exploit(self):
trial = TrialStub()
assert PipelineExploit([])(RNGStub(), trial, None) is trial
def test_no_explore(self):
params = object()
assert PipelineExplore([])(RNGStub(), None, params) is params