def from_json(cls, obj): """Recreate a SearchSpace from its JSON representation. Parameters ---------- obj : dict JSON encoding of the SearchSpace Returns ------- searchspace : `pyrameter.searchspace.SearchSpace` """ domains = [Domain.from_json(d) for d in obj['domains']] searchspace = cls(domains, exp_key=obj['exp_key']) trials = [] for t in obj['trials']: trial = Trial(searchspace, hyperparameters=t['hyperparameters'], results=t['results'], objective=t['objective'], errmsg=t['errmsg']) trial.dirty = False trials.append(trial) searchspace.trials = trials searchspace._complexity = obj['complexity'] searchspace._uncertainty = obj['uncertainty'] return searchspace
def __call__(self, method=None, to_dict=False): """Generate a new trial for this search space. Parameters ---------- to_dict : bool Convert the hyperparameter values to a nested dictionary on return. Returns ------- trial : ``pyrameter.trial.Trial`` or dict Trial data, including hyperparameter values and metadata for a database. If ``to_dict`` is ``True``, instead return only the nested dictionary of hyperparameter values matching the structure of the original specification. """ self.ready = self.population is None or all( [t.objective is not None for t in self.population]) if self.ready: if method is None: method = RandomSearch() population = method(self) if not isinstance(population, list) or isinstance( population, np.ndarray) and population.ndim == 1: population = [population] self.population = [ Trial(self, hyperparameters=h) for h in population ] self.trials.extend(self.population) self.generations += 1 return [t.parameter_dict for t in self.population] if to_dict else self.population else: return []
def __call__(self, method=None, to_dict=False): """Generate a new trial for this search space if ready. Parameters ---------- to_dict : bool Convert the hyperparameter values to a nested dictionary on return. Returns ------- trial : ``pyrameter.trial.Trial`` or dict or None Trial data, including hyperparameter values and metadata for a database. If ``to_dict`` is ``True``, instead return only the nested dictionary of hyperparameter values matching the structure of the original specification. If ``None`` is returned, then the space requires more in-progress trials to return before more hyperparameters may be generated. This last case is to ensure proper warm-up sampling for model-based methods. """ if method is None: method = RandomSearch() hyperparameters = method(self) for i, d in enumerate(self.domains): d.current = hyperparameters[i] trial = Trial(self, hyperparameters=hyperparameters) self.trials.append(trial) return trial.parameter_dict if to_dict else trial
def from_json(cls, obj): """Recreate a SearchSpace from its JSON representation. Parameters ---------- obj : dict JSON encoding of the SearchSpace Returns ------- searchspace : `pyrameter.searchspace.SearchSpace` """ domains = [Domain.from_json(d) for d in obj['domains']] searchspace = cls(domains, exp_key=obj['exp_key']) trials = [] for t in obj['trials']: t['searchspace'] = searchspace trial = Trial.from_json(t) trial.dirty = False trials.append(trial) searchspace.trials = trials searchspace._complexity = obj['complexity'] searchspace._uncertainty = obj['uncertainty'] print(obj) searchspace.id = obj['id'] return searchspace
def test_parameter_dict(): s = SearchSpace([ConstantDomain('A', 8)]) t = Trial(s, hyperparameters=[8]) assert t.parameter_dict == {'A': 8} s = SearchSpace([ConstantDomain('/A', 8), ConstantDomain('/B/a/b', 2)]) t = Trial(s, hyperparameters=[8, 2]) assert t.parameter_dict == {'A': 8, 'B': {'a': {'b': 2}}} s = SearchSpace([ ConstantDomain('/A', 8), ConstantDomain('/B/a/b', 2), ConstantDomain('/B/a/c', 4) ]) t = Trial(s, hyperparameters=[8, 2, 4]) assert t.parameter_dict == {'A': 8, 'B': {'a': {'b': 2, 'c': 4}}}
def __call__(self, method=None, to_dict=False): try: hyperparameters = np.array(list(next(self._iterator))) trial = Trial(self, hyperparameters=hyperparameters) self.trials.append(trial) except StopIteration: self.done = True return None
def __call__(self, method=None, to_dict=False): """Generate a new trial for this search space. Parameters ---------- to_dict : bool Convert the hyperparameter values to a nested dictionary on return. Returns ------- trial : ``pyrameter.trial.Trial`` or dict Trial data, including hyperparameter values and metadata for a database. If ``to_dict`` is ``True``, instead return only the nested dictionary of hyperparameter values matching the structure of the original specification. """ if method is None: method = random_search hyperparameters = method(self) trial = Trial(self, hyperparameters=hyperparameters) self.trials.append(trial) return trial.parameter_dict if to_dict else trial
def tpe(space, best_split=0.2, n_samples=10, warm_up=10, **gmm_kws): """Tree-structured Parzen Enstimators for generating hyperparameters. Parameters ---------- space : pyrameter.domains.SearchSpace The space to generate values from. best_split : float in [0, 1] The percentage of results to use for the top-k mixture model. n_samples : int The number of candidate samples to generate. warm_up : int The number of random search iterations to use to seed TPE. Other Parameters ---------------- **gmm_kws Additional keyword arguments to parameterize the Gaussian Mixture Models. Returns ------- values : array-like The array of hyperparameter values with the highest expected improvement from among the candidate ``n_samples``. """ # Warm up with random search and inject new random search # hyperparameters at an interval. This attempts to prevent TPE from # converging too quickly. if len(space.objective) < warm_up or len(space.objective) % warm_up == 0: params = random_search(space) else: params = [] # Collect all of the evaluated hyperparameter values and their # associated objective function value into a feature vector. features = space.to_array().T losses = np.array(space.objective) # Sort the hyperparameters by their performance and split into # the "best" and "rest" performers. idx = np.argsort(losses, axis=0) split = int(np.ceil(idx.shape[0] * best_split)) losses = np.reshape(losses, (-1, 1)) # Model the objective function based on each feature. for j in range(features.shape[0]): l = GaussianMixture(**gmm_kws) g = GaussianMixture(**gmm_kws) l.fit(np.reshape(features[j, idx[:split]], (-1, 1)), losses[idx[:split]]) l.fit(np.reshape(features[j, idx[split:]], (-1, 1)), losses[idx[split:]]) # Sample hyperparameter values from the "best" model and score # the samples with each model. samples, _ = l.sample(n_samples=n_samples) score_l = l.score(samples) score_g = g.score(samples) # Compute the expected improvement; i.e. maximize the l score # while minimizing the g score. Higher values are better. ei = score_l / score_g best = samples[np.argmax(np.squeeze(ei).ravel())] # Add the value with the best expected improvement domain = space.nodes[j] params.append(domain.map_to_domain(best[0]), bound=True) params = Trial(space, hyperparameters=params) space.results.append(params) return params
def test_to_json(): s = SearchSpace([ConstantDomain('A', 8)]) t = Trial(s) assert t.to_json() == { 'searchspace': s.id, 'status': TrialStatus.INIT.value, 'hyperparameters': None, 'results': None, 'objective': None, 'errmsg': None } s = SearchSpace([ConstantDomain('A', 8)]) t = Trial(s, hyperparameters=[8]) assert t.to_json() == { 'searchspace': s.id, 'status': TrialStatus.READY.value, 'hyperparameters': [8], 'results': None, 'objective': None, 'errmsg': None } s = SearchSpace([ConstantDomain('A', 8)]) t = Trial(s, hyperparameters=[8], objective=0.374) assert t.to_json() == { 'searchspace': s.id, 'status': TrialStatus.READY.value, 'hyperparameters': [8], 'results': None, 'objective': 0.374, 'errmsg': None } s = SearchSpace([ConstantDomain('A', 8)]) t = Trial(s, hyperparameters=[8], objective=0.374, errmsg='hi') assert t.to_json() == { 'searchspace': s.id, 'status': TrialStatus.ERROR.value, 'hyperparameters': [8], 'results': None, 'objective': 0.374, 'errmsg': 'hi' }
def test_init(): s = SearchSpace([ConstantDomain('A', 8)]) t = Trial(s) assert t.searchspace() is s assert t.hyperparameters is None assert t.results is None assert t.objective is None assert t.errmsg is None assert t.dirty assert t.status == TrialStatus.INIT s = SearchSpace([ConstantDomain('A', 8)]) t = Trial(s, hyperparameters=[8]) assert t.searchspace() is s assert t.hyperparameters == [8] assert t.results is None assert t.objective is None assert t.errmsg is None assert t.dirty assert t.status == TrialStatus.READY s = SearchSpace([ConstantDomain('A', 8)]) t = Trial(s, hyperparameters=[8], results={'loss': 10}, objective=10) assert t.searchspace() is s assert t.hyperparameters == [8] assert t.results == {'loss': 10} assert t.objective == 10 assert t.errmsg is None assert t.dirty assert t.status == TrialStatus.DONE s = SearchSpace([ConstantDomain('A', 8)]) t = Trial(s, hyperparameters=[8], results={'loss': 10}, objective=10, errmsg='HI!') assert t.searchspace() is s assert t.hyperparameters == [8] assert t.results == {'loss': 10} assert t.objective == 10 assert t.errmsg == 'HI!' assert t.dirty assert t.status == TrialStatus.ERROR