def test_update(): domain = Domain({"x": {-5., 6.}}) gs = exhaustive.GridSearch(domain) gs.update([domain.sample() for _ in range(10)], list(range(10))) gs.update(domain.sample(), {"score": 23.0}) gs.update(domain.sample(), 2.0) assert len(gs.history) == 12
def test_update(): domain = Domain({"x": [-5., 6.]}) rs = random.RandomSearch(domain) rs.update([domain.sample() for _ in range(4)], list(range(4))) rs.update(domain.sample(), {"score": 23.0}) rs.update(domain.sample(), 2.0) assert len(rs.history) == 6 rs.reset() assert len(rs.history) == 0
def test_slurm_from_script(): domain = Domain({ "x": {0, 1, 2, 3}, "y": [-1., 1.], "z": {"123", "abc"} }, seed=7) jobs, dirs = [], [] n_jobs = 4 for i in range(n_jobs): sample = domain.sample() # NOTE: this test might fail if /tmp is not shared in the slurm cluster. # Adding the argument dir="/path/to/shared/dir" can fix that dirs.append(tempfile.TemporaryDirectory()) jobs.append( SlurmJob( task="hypertunity/scheduling/tests/script.py", args=(*sample.as_namedtuple(), ), output_file=f"{os.path.join(dirs[-1].name, 'results.pkl')}", meta={ "binary": "python", "resources": { "cpu": 1 } })) results = run_jobs(jobs) assert all([r.data == script.main(*j.args) for r, j in zip(results, jobs)]) # clean-up the temporary dirs for d in dirs: d.cleanup()
def test_bo_set_history(): n_samples = 10 domain = Domain({"a": {"b": [2, 3]}, "c": [0, 0.1]}) history = [ base.HistoryPoint(domain.sample(), {"score": base.EvaluationScore(float(i))}) for i in range(n_samples) ] bayes_opt = bo.BayesianOptimisation(domain, seed=7) bayes_opt.history = history assert bayes_opt.history == history assert len(bayes_opt._data_x) == len(bayes_opt._data_fx) == len(history)
def test_local_from_script_and_function(): domain = Domain({ "x": {0, 1, 2, 3}, "y": [-1., 1.], "z": {"123", "abc"} }, seed=7) jobs = [ Job(task="hypertunity/scheduling/tests/script.py::main", args=(*domain.sample().as_namedtuple(), )) for _ in range(10) ] results = run_jobs(jobs) assert all([r.data == script.main(*j.args) for r, j in zip(results, jobs)])
def generated_history(): domain = Domain({ "x": [-5., 6.], "y": {"sin", "sqr"}, "z": set(range(4)) }, seed=7) n_samples = 10 history = [ HistoryPoint(sample=domain.sample(), metrics={ "metric_1": EvaluationScore(float(i)), "metric_2": EvaluationScore(i * 2.) }) for i in range(n_samples) ] if len(history) == 1: history = history[0] return history, domain
def test_local_from_script_and_cmdline_named_args(): domain = Domain( { "--x": {0, 1, 2, 3}, "--y": [-1., 1.], "--z": {"acb123", "abc"} }, seed=7) jobs = [ Job(task="hypertunity/scheduling/tests/script.py", args=domain.sample().as_dict(), meta={"binary": "python"}) for _ in range(10) ] results = run_jobs(jobs) assert all([ r.data == script.main(**{k.lstrip("-"): v for k, v in j.args.items()}) for r, j in zip(results, jobs) ])
def test_local_from_fn(): domain = Domain({"x": [0., 1.]}, seed=7) jobs = [Job(task=square, args=(domain.sample(), )) for _ in range(10)] results = run_jobs(jobs) assert all( [r.data.value == square(*j.args).value for r, j in zip(results, jobs)])
def test_sampling(): domain = Domain({"a": {"b": {2, 3, 4}}, "c": [0, 0.1]}) for i in range(10): sample = domain.sample() assert sample["a"]["b"] in {2, 3, 4} and 0. <= sample["c"] <= 0.1
class GridSearch(Optimiser): """Grid search pseudo-optimiser.""" def __init__(self, domain: Domain, sample_continuous: bool = False, seed: int = None): """Initialise the :class:`GridSearch` optimiser from a discrete domain. If the domain contains continuous subspaces, then they could be sampled if `sample_continuous` is enabled. Args: domain: :class:`Domain`. The domain to iterate over. sample_continuous: (optional) :obj:`bool`. Whether to sample the continuous subspaces of the domain. seed: (optional) :obj:`int`. Seed for the sampling of the continuous subspace if necessary. """ if domain.is_continuous and not sample_continuous: raise DomainNotIterableError( "Cannot perform grid search on (partially) continuous domain. " "To enable grid search in this case, set the argument " "'sample_continuous' to True.") super(GridSearch, self).__init__(domain) (discrete_domain, categorical_domain, continuous_domain) = domain.split_by_type() # unify the discrete and the categorical into one, # as they can be iterated: self.discrete_domain = discrete_domain + categorical_domain if seed is not None: self.continuous_domain = Domain(continuous_domain.as_dict(), seed=seed) else: self.continuous_domain = continuous_domain self._discrete_domain_iter = iter(self.discrete_domain) self._is_exhausted = len(self.discrete_domain) == 0 self.__exhausted_err = ExhaustedSearchSpaceError( "The domain has been exhausted. Reset the optimiser to start again." ) def run_step(self, batch_size: int = 1, **kwargs) -> List[Sample]: """Get the next `batch_size` samples from the Cartesian-product walk over the domain. Args: batch_size: (optional) :obj:`int`. The number of samples to suggest at once. Returns: A list of :class:`Sample` instances from the domain. Raises: :class:`ExhaustedSearchSpaceError`: if the (discrete part of the) domain is fully exhausted and no samples can be generated. Notes: This method does not guarantee that the returned list of :class:`Samples` will be of length `batch_size`. This is due to the size of the domain and the fact that samples will not be repeated. """ if self._is_exhausted: raise self.__exhausted_err samples = [] for i in range(batch_size): try: discrete = next(self._discrete_domain_iter) except StopIteration: self._is_exhausted = True break if self.continuous_domain: continuous = self.continuous_domain.sample() samples.append(discrete + continuous) else: samples.append(discrete) if samples: return samples raise self.__exhausted_err def reset(self): """Reset the optimiser to the beginning of the Cartesian-product walk.""" super(GridSearch, self).reset() self._discrete_domain_iter = iter(self.discrete_domain) self._is_exhausted = len(self.discrete_domain) == 0