def test_update():
domain = Domain({"x": {-5., 6.}})
gs = exhaustive.GridSearch(domain)
gs.update([domain.sample() for _ in range(10)], list(range(10)))
gs.update(domain.sample(), {"score": 23.0})
gs.update(domain.sample(), 2.0)
assert len(gs.history) == 12
def test_update():
domain = Domain({"x": [-5., 6.]})
rs = random.RandomSearch(domain)
rs.update([domain.sample() for _ in range(4)], list(range(4)))
rs.update(domain.sample(), {"score": 23.0})
rs.update(domain.sample(), 2.0)
assert len(rs.history) == 6
rs.reset()
assert len(rs.history) == 0
def test_slurm_from_script():
domain = Domain({
"x": {0, 1, 2, 3},
"y": [-1., 1.],
"z": {"123", "abc"}
},
seed=7)
jobs, dirs = [], []
n_jobs = 4
for i in range(n_jobs):
sample = domain.sample()
# NOTE: this test might fail if /tmp is not shared in the slurm cluster.
# Adding the argument dir="/path/to/shared/dir" can fix that
dirs.append(tempfile.TemporaryDirectory())
jobs.append(
SlurmJob(
task="hypertunity/scheduling/tests/script.py",
args=(*sample.as_namedtuple(), ),
output_file=f"{os.path.join(dirs[-1].name, 'results.pkl')}",
meta={
"binary": "python",
"resources": {
"cpu": 1
}
}))
results = run_jobs(jobs)
assert all([r.data == script.main(*j.args) for r, j in zip(results, jobs)])
# clean-up the temporary dirs
for d in dirs:
d.cleanup()
def test_bo_set_history():
n_samples = 10
domain = Domain({"a": {"b": [2, 3]}, "c": [0, 0.1]})
history = [
base.HistoryPoint(domain.sample(),
{"score": base.EvaluationScore(float(i))})
for i in range(n_samples)
]
bayes_opt = bo.BayesianOptimisation(domain, seed=7)
bayes_opt.history = history
assert bayes_opt.history == history
assert len(bayes_opt._data_x) == len(bayes_opt._data_fx) == len(history)
def test_local_from_script_and_function():
domain = Domain({
"x": {0, 1, 2, 3},
"y": [-1., 1.],
"z": {"123", "abc"}
},
seed=7)
jobs = [
Job(task="hypertunity/scheduling/tests/script.py::main",
args=(*domain.sample().as_namedtuple(), )) for _ in range(10)
]
results = run_jobs(jobs)
assert all([r.data == script.main(*j.args) for r, j in zip(results, jobs)])
def generated_history():
domain = Domain({
"x": [-5., 6.],
"y": {"sin", "sqr"},
"z": set(range(4))
},
seed=7)
n_samples = 10
history = [
HistoryPoint(sample=domain.sample(),
metrics={
"metric_1": EvaluationScore(float(i)),
"metric_2": EvaluationScore(i * 2.)
}) for i in range(n_samples)
]
if len(history) == 1:
history = history[0]
return history, domain
def test_local_from_script_and_cmdline_named_args():
domain = Domain(
{
"--x": {0, 1, 2, 3},
"--y": [-1., 1.],
"--z": {"acb123", "abc"}
},
seed=7)
jobs = [
Job(task="hypertunity/scheduling/tests/script.py",
args=domain.sample().as_dict(),
meta={"binary": "python"}) for _ in range(10)
]
results = run_jobs(jobs)
assert all([
r.data == script.main(**{k.lstrip("-"): v
for k, v in j.args.items()})
for r, j in zip(results, jobs)
])
def test_local_from_fn():
domain = Domain({"x": [0., 1.]}, seed=7)
jobs = [Job(task=square, args=(domain.sample(), )) for _ in range(10)]
results = run_jobs(jobs)
assert all(
[r.data.value == square(*j.args).value for r, j in zip(results, jobs)])
def test_sampling():
domain = Domain({"a": {"b": {2, 3, 4}}, "c": [0, 0.1]})
for i in range(10):
sample = domain.sample()
assert sample["a"]["b"] in {2, 3, 4} and 0. <= sample["c"] <= 0.1
class GridSearch(Optimiser):
"""Grid search pseudo-optimiser."""
def __init__(self,
domain: Domain,
sample_continuous: bool = False,
seed: int = None):
"""Initialise the :class:`GridSearch` optimiser from a discrete domain.
If the domain contains continuous subspaces, then they could be sampled
if `sample_continuous` is enabled.
Args:
domain: :class:`Domain`. The domain to iterate over.
sample_continuous: (optional) :obj:`bool`. Whether to sample the
continuous subspaces of the domain.
seed: (optional) :obj:`int`. Seed for the sampling of the continuous
subspace if necessary.
"""
if domain.is_continuous and not sample_continuous:
raise DomainNotIterableError(
"Cannot perform grid search on (partially) continuous domain. "
"To enable grid search in this case, set the argument "
"'sample_continuous' to True.")
super(GridSearch, self).__init__(domain)
(discrete_domain, categorical_domain,
continuous_domain) = domain.split_by_type()
# unify the discrete and the categorical into one,
# as they can be iterated:
self.discrete_domain = discrete_domain + categorical_domain
if seed is not None:
self.continuous_domain = Domain(continuous_domain.as_dict(),
seed=seed)
else:
self.continuous_domain = continuous_domain
self._discrete_domain_iter = iter(self.discrete_domain)
self._is_exhausted = len(self.discrete_domain) == 0
self.__exhausted_err = ExhaustedSearchSpaceError(
"The domain has been exhausted. Reset the optimiser to start again."
)
def run_step(self, batch_size: int = 1, **kwargs) -> List[Sample]:
"""Get the next `batch_size` samples from the Cartesian-product walk
over the domain.
Args:
batch_size: (optional) :obj:`int`. The number of samples to suggest
at once.
Returns:
A list of :class:`Sample` instances from the domain.
Raises:
:class:`ExhaustedSearchSpaceError`: if the (discrete part of the)
domain is fully exhausted and no samples can be generated.
Notes:
This method does not guarantee that the returned list of
:class:`Samples` will be of length `batch_size`. This is due to the
size of the domain and the fact that samples will not be repeated.
"""
if self._is_exhausted:
raise self.__exhausted_err
samples = []
for i in range(batch_size):
try:
discrete = next(self._discrete_domain_iter)
except StopIteration:
self._is_exhausted = True
break
if self.continuous_domain:
continuous = self.continuous_domain.sample()
samples.append(discrete + continuous)
else:
samples.append(discrete)
if samples:
return samples
raise self.__exhausted_err
def reset(self):
"""Reset the optimiser to the beginning of the Cartesian-product walk."""
super(GridSearch, self).reset()
self._discrete_domain_iter = iter(self.discrete_domain)
self._is_exhausted = len(self.discrete_domain) == 0
