def __init__(self, dimensions, n_random_starts=10, mutation_rate=0.05): self.Xi = [] self.yi = [] self.space = Space(dimensions) self.n_random_starts = n_random_starts self.n_random_starts_ = n_random_starts self.mutation_rate = mutation_rate
def test_space_from_yaml(): with NamedTemporaryFile(delete=False) as tmp: tmp.write(b""" Space: - Real: low: 0.0 high: 1.0 - Integer: low: -5 high: 5 - Categorical: categories: - a - b - c - Real: low: 1.0 high: 5.0 prior: log-uniform - Categorical: categories: - e - f """) tmp.flush() space = Space([(0.0, 1.0), (-5, 5), ("a", "b", "c"), (1.0, 5.0, "log-uniform"), ("e", "f")]) space2 = Space.from_yaml(tmp.name) assert_equal(space, space2) tmp.close() os.unlink(tmp.name)
def test_space_api(): space = Space([(0.0, 1.0), (-5, 5), ("a", "b", "c"), (1.0, 5.0, "log-uniform")]) assert_equal(len(space.dimensions), 4) assert_true(isinstance(space.dimensions[0], Real)) assert_true(isinstance(space.dimensions[1], Integer)) assert_true(isinstance(space.dimensions[2], Categorical)) assert_true(isinstance(space.dimensions[3], Real)) samples = space.rvs(n_samples=10, random_state=0) assert_equal(len(samples), 10) assert_equal(len(samples[0]), 4) samples_transformed = space.transform(samples) assert_equal(samples_transformed.shape[0], len(samples)) assert_equal(samples_transformed.shape[1], 1 + 1 + 3 + 1) assert_array_equal(samples, space.inverse_transform(samples_transformed)) for b1, b2 in zip(space.bounds, [(0.0, 1.0), (-5, 5), np.asarray(["a", "b", "c"]), (1.0, 5.0)]): assert_array_equal(b1, b2) for b1, b2 in zip(space.transformed_bounds, [(0.0, 1.0), (-5, 5), (0.0, 1.0), (0.0, 1.0), (0.0, 1.0), (np.log10(1.0), np.log10(5.0))]): assert_array_equal(b1, b2)
def cook_estimator(base_estimator, space=None, **kwargs): """Cook a default estimator. For the special base_estimator called "DUMMY" the return value is None. This corresponds to sampling points at random, hence there is no need for an estimator. Parameters ---------- base_estimator : "GP", "RF", "ET", "GBRT", "DUMMY" or sklearn regressor Should inherit from `sklearn.base.RegressorMixin`. In addition the `predict` method should have an optional `return_std` argument, which returns `std(Y | x)`` along with `E[Y | x]`. If base_estimator is one of ["GP", "RF", "ET", "GBRT", "DUMMY"], a surrogate model corresponding to the relevant `X_minimize` function is created. space : Space instance Has to be provided if the base_estimator is a gaussian process. Ignored otherwise. kwargs : dict Extra parameters provided to the base_estimator at init time. """ if isinstance(base_estimator, str): base_estimator = base_estimator.upper() if base_estimator not in ["GP", "ET", "RF", "GBRT", "DUMMY"]: raise ValueError("Valid strings for the base_estimator parameter " " are: 'RF', 'ET', 'GP', 'GBRT' or 'DUMMY' not " "%s." % base_estimator) elif not is_regressor(base_estimator): raise ValueError("base_estimator has to be a regressor.") if base_estimator == "GP": if space is not None: space = Space(space) space = Space(normalize_dimensions(space.dimensions)) n_dims = space.transformed_n_dims is_cat = space.is_categorical else: raise ValueError("Expected a Space instance, not None.") cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) # only special if *all* dimensions are categorical if is_cat: other_kernel = HammingKernel(length_scale=np.ones(n_dims)) else: other_kernel = Matern(length_scale=np.ones(n_dims), length_scale_bounds=[(0.01, 100)] * n_dims, nu=2.5) base_estimator = GaussianProcessRegressor(kernel=cov_amplitude * other_kernel, normalize_y=True, noise="gaussian", n_restarts_optimizer=2) if ('n_jobs' in kwargs.keys()) and not hasattr(base_estimator, 'n_jobs'): del kwargs['n_jobs'] base_estimator.set_params(**kwargs) return base_estimator
def test_space_from_space(): # can you pass a Space instance to the Space constructor? space = Space([(0.0, 1.0), (-5, 5), ("a", "b", "c"), (1.0, 5.0, "log-uniform"), ("e", "f")]) space2 = Space(space) assert_equal(space, space2)
def test_partly_categorical_space(): dims = Space([Categorical(['a', 'b', 'c']), Categorical(['A', 'B', 'C'])]) assert dims.is_partly_categorical dims = Space([Categorical(['a', 'b', 'c']), Integer(1, 2)]) assert dims.is_partly_categorical assert not dims.is_categorical dims = Space([Integer(1, 2), Integer(1, 2)]) assert not dims.is_partly_categorical
def hyperband(objective, space, max_iter=100, eta=3, random_state=0, verbose=True, n_evaluations=None, rank=0): """ Hyperband algorithm as defined by Kevin Jamieson. Parameters: ---------- * `objective`: [function] Objective function to be minimized. * `space`: [hyperspace.space] Hyperparameter search space bounds. * `max_iter`: [int] Maximum number of iterations. * `eta`: [int] Returns: ------- * `result` [`OptimizeResult`, scipy object] Reference: --------- http://people.eecs.berkeley.edu/~kjamieson/hyperband.html """ logeta = lambda x: log(x)/log(eta) s_max = int(logeta(max_iter)) # number of unique executions of Successive Halving (minus one) B = (s_max+1)*max_iter # total number of iterations (without reuse) per execution of Succesive Halving (n,r) # If space is the original list of tuples, convert to Space() if isinstance(space, list): space = Space(space) #### Begin Finite Horizon Hyperband outlerloop. Repeat indefinetely. yi = [] Xi = [] for s in reversed(range(s_max+1)): n = int(ceil(int(B/max_iter/(s+1))*eta**s)) # initial number of configurations r = max_iter*eta**(-s) # initial number of iterations to run configurations for # Begin Finite Horizon Successive Halving with (n,r) for i in range(s+1): # Run each of the n_i configs for r_i iterations and keep best n_i/eta n_i = ceil(n*eta**(-i)) r_i = ceil(r*eta**(i)) T = space.rvs(ceil(n_i), random_state) iter_result = [objective(t, r_i) for t in T] yi.append(iter_result) Xi.append(T) # Get next hyperparameter configurations T = [ T[i] for i in np.argsort(iter_result)[0:int( n_i/eta )] ] if verbose and rank == 0: print(f'Iteration number: {i}, Epochs per config: {r_i}, Num configs: {n_i}, Incumbent: {min(iter_result)}') result = create_result(Xi, yi, n_evaluations=n_evaluations, space=space, rng=random_state) # End Finite Horizon Successive Halving with (n,r) return result
def __init__(self, dimensions, n_random_starts=10, tournament_fraction=0.2, mutation_rate=0.05): self.Xi = [] self.yi = [] self.space = Space(dimensions) self.n_random_starts = n_random_starts self.n_random_starts_ = n_random_starts self.tournament_fraction = tournament_fraction self.mutation_rate = mutation_rate
def test_normalize_types(): # can you pass a Space instance to the Space constructor? space = Space([(0.0, 1.0), Integer(-5, 5, dtype=int), (True, False)]) space.set_transformer("normalize") X = [[0., -5, False]] Xt = np.zeros((1, 3)) assert_array_equal(space.transform(X), Xt) assert_array_equal(space.inverse_transform(Xt), X) assert_array_equal(space.inverse_transform(space.transform(X)), X) assert isinstance(space.inverse_transform(Xt)[0][0], float) assert isinstance(space.inverse_transform(Xt)[0][1], int) assert isinstance(space.inverse_transform(Xt)[0][2], (np.bool_, bool))
def normalize_dimensions(dimensions): """Create a ``Space`` where all dimensions are normalized to unit range. This is particularly useful for Gaussian process based regressors and is used internally by ``gp_minimize``. Parameters ---------- * `dimensions` [list, shape=(n_dims,)]: List of search space dimensions. Each search dimension can be defined either as - a `(lower_bound, upper_bound)` tuple (for `Real` or `Integer` dimensions), - a `(lower_bound, upper_bound, "prior")` tuple (for `Real` dimensions), - as a list of categories (for `Categorical` dimensions), or - an instance of a `Dimension` object (`Real`, `Integer` or `Categorical`). NOTE: The upper and lower bounds are inclusive for `Integer` dimensions. """ space = Space(dimensions) transformed_dimensions = [] if space.is_categorical: # recreate the space and explicitly set transform to "identity" # this is a special case for GP based regressors for dimension in space: transformed_dimensions.append(Categorical(dimension.categories, dimension.prior, name=dimension.name, transform="identity")) else: for dimension in space.dimensions: if isinstance(dimension, Categorical): transformed_dimensions.append(dimension) # To make sure that GP operates in the [0, 1] space elif isinstance(dimension, Real): transformed_dimensions.append( Real(dimension.low, dimension.high, dimension.prior, name=dimension.name, transform="normalize") ) elif isinstance(dimension, Integer): transformed_dimensions.append( Integer(dimension.low, dimension.high, name=dimension.name, transform="normalize") ) else: raise RuntimeError("Unknown dimension type " "(%s)" % type(dimension)) return Space(transformed_dimensions)
def test_create_hyperspace( integer=(0, 10), real=(20.0, 30), cat=['a', 'b', 'c', 'd']): """ Tests for correctly formatted combinations of Scikit-Optimize spaces. """ hparams0 = [integer, real] hparams1 = [real, cat] hparams2 = [integer, cat] # Integer hparam hyper_int = HyperInteger(low=integer[0], high=integer[1]) int_low, int_high = hyper_int.get_hyperspace() # Real hparam hyper_real = HyperReal(low=real[0], high=real[1]) real_low, real_high = hyper_real.get_hyperspace() # Categorical hparam hyper_cat = HyperCategorical(cat) cat_low, cat_high = hyper_cat.get_hyperspace() # Case 0 hyperspace0 = create_hyperspace(hparams0) test_combinations0 = [[int_high, real_high], [int_low, real_high], [int_high, real_low], [int_low, real_low]] test_hyperspace0 = [] for space in test_combinations0: test_hyperspace0.append(Space(space)) # Case 1: [real, cat] hyperspace1 = create_hyperspace(hparams1) test_combinations1 = [[real_high, cat_high], [real_low, cat_high], [real_high, cat_low], [real_low, cat_low]] test_hyperspace1 = [] for space in test_combinations1: test_hyperspace1.append(Space(space)) # Case 3: [int, cat] hyperspace2 = create_hyperspace(hparams2) test_combinations2 = [[int_high, cat_high], [int_low, cat_high], [int_high, cat_low], [int_low, cat_low]] test_hyperspace2 = [] for space in test_combinations2: test_hyperspace2.append(Space(space)) assert_equal(hyperspace0, test_hyperspace0) assert_equal(hyperspace1, test_hyperspace1) assert_equal(hyperspace2, test_hyperspace2)
def lhs_sample(n_samples): """ Takes random n_samples with the lhs method. Returns array x and y. """ x = np.array([]) y = np.array([]) # Makes the space of points which van be chosen from space = Space([(-2., 1.), (-1.5, 1.5)]) # Chooses which kind oh lhs will be used lhs = Lhs(lhs_type="classic", criterion=None) coordinates = 0 # Generates n_samples withhi the chosen space coordinates = lhs.generate(space.dimensions, n_samples) # appends all x and y values to array for coordinate in coordinates: a = coordinate[0] x = np.append(x, a) b = coordinate[1] y = np.append(y, b) return x, y
def create_hyperspace(hyperparameters): """ Converts hyperparameter lists to Scikit-Optimize Space instances. Parameters ---------- * `hyperparameters` [list, shape=(n_hyperparameters,)] Returns ------- * `hyperspace` [list of lists, shape(n_spaces, n_hyperparameters)] - All combinations of hyperspaces. Each list within hyperspace is a search space to be distributed across 2**n_spaces nodes. """ hparams_low = [] hparams_high = [] for hparam in hyperparameters: low, high = check_dimension(hparam) hparams_low.append(low) hparams_high.append(high) all_spaces = fold_spaces(hparams_low, hparams_high) hyperspace = [] for space in all_spaces: hyperspace.append(Space(space)) return hyperspace
def test_minimizer_with_space(minimizer): # check we can pass a Space instance as dimensions argument and get same # result n_calls = 7 n_random_starts = 4 space = Space([(-5.0, 10.0), (0.0, 15.0)]) space_result = minimizer(branin, space, n_calls=n_calls, n_random_starts=n_random_starts, random_state=1) check_minimizer_api(space_result, n_calls) check_minimizer_bounds(space_result, n_calls) dimensions = [(-5.0, 10.0), (0.0, 15.0)] result = minimizer(branin, dimensions, n_calls=n_calls, n_random_starts=n_random_starts, random_state=1) assert_array_almost_equal(space_result.x_iters, result.x_iters) assert_array_almost_equal(space_result.func_vals, result.func_vals)
def solve(self): """ Wrapper function for scipy.optimize.differential_evolution """ progress = [] def cb(xk, convergence): progress.append(self.fun(xk)) # initialize number of points = popsize space = Space([(0.,1.)]*len(self.bounds)) lhs = Lhs() pop = np.asarray(lhs.generate(space.dimensions, self.popsize)) min_b, max_b = np.asarray(self.bounds).T diff = max_b - min_b pop = min_b + pop * diff progress.append(np.min(np.apply_along_axis(self.fun, 1, pop))) result = scipy_de(self.fun, self.bounds, popsize=1, maxiter = self.maxiter, tol=0.0001, disp=self.disp, callback=cb, init=pop) self.x = result.x self.fx = result.fun f_calls = (np.arange(1,len(progress)+1)) * self.popsize self.converge_data = np.vstack((f_calls, np.asarray(progress))) self.solved = True
def test_space_api(): space = Space([(0.0, 1.0), (-5, 5), ("a", "b", "c"), (1.0, 5.0, "log-uniform"), ("e", "f")]) cat_space = Space([(1, "r"), (1.0, "r")]) assert isinstance(cat_space.dimensions[0], Categorical) assert isinstance(cat_space.dimensions[1], Categorical) assert_equal(len(space.dimensions), 5) assert_true(isinstance(space.dimensions[0], Real)) assert_true(isinstance(space.dimensions[1], Integer)) assert_true(isinstance(space.dimensions[2], Categorical)) assert_true(isinstance(space.dimensions[3], Real)) assert_true(isinstance(space.dimensions[4], Categorical)) samples = space.rvs(n_samples=10, random_state=0) assert_equal(len(samples), 10) assert_equal(len(samples[0]), 5) assert_true(isinstance(samples, list)) for n in range(4): assert_true(isinstance(samples[n], list)) assert_true(isinstance(samples[0][0], numbers.Real)) assert_true(isinstance(samples[0][1], numbers.Integral)) assert_true(isinstance(samples[0][2], str)) assert_true(isinstance(samples[0][3], numbers.Real)) assert_true(isinstance(samples[0][4], str)) samples_transformed = space.transform(samples) assert_equal(samples_transformed.shape[0], len(samples)) assert_equal(samples_transformed.shape[1], 1 + 1 + 3 + 1 + 1) # our space contains mixed types, this means we can't use # `array_allclose` or similar to check points are close after a round-trip # of transformations for orig, round_trip in zip(samples, space.inverse_transform(samples_transformed)): assert space.distance(orig, round_trip) < 1.e-8 samples = space.inverse_transform(samples_transformed) assert_true(isinstance(samples[0][0], numbers.Real)) assert_true(isinstance(samples[0][1], numbers.Integral)) assert_true(isinstance(samples[0][2], str)) assert_true(isinstance(samples[0][3], numbers.Real)) assert_true(isinstance(samples[0][4], str)) for b1, b2 in zip(space.bounds, [(0.0, 1.0), (-5, 5), np.asarray(["a", "b", "c"]), (1.0, 5.0), np.asarray(["e", "f"])]): assert_array_equal(b1, b2) for b1, b2 in zip(space.transformed_bounds, [(0.0, 1.0), (-5, 5), (0.0, 1.0), (0.0, 1.0), (0.0, 1.0), (np.log10(1.0), np.log10(5.0)), (0.0, 1.0)]): assert_array_equal(b1, b2)
def test_fold_spaces( integer=(0, 10), real=(20.0, 30), cat=['a', 'b', 'c', 'd']): """ Test all combinations of hyperparameter spaces. """ hparams = [integer, real, cat] hparams_low = [] hparams_high = [] for hparam in hparams: low, high = check_dimension(hparam) hparams_low.append(low) hparams_high.append(high) all_spaces = fold_spaces(hparams_low, hparams_high) # Integer hparam hyper_int = HyperInteger(low=integer[0], high=integer[1]) int_low, int_high = hyper_int.get_hyperspace() # Real hparam hyper_real = HyperReal(low=real[0], high=real[1]) real_low, real_high = hyper_real.get_hyperspace() # Categorical hparam hyper_cat = HyperCategorical(cat) cat_low, cat_high = hyper_cat.get_hyperspace() # All combinations: test_combinations = [[int_high, real_high, cat_high], [int_low, real_high, cat_high], [int_high, real_low, cat_high], [int_low, real_low, cat_high], [int_high, real_high, cat_low], [int_low, real_high, cat_low], [int_high, real_low, cat_low], [int_low, real_low, cat_low]] hyperspace = [] for space in all_spaces: hyperspace.append(Space(space)) test_case = [] for space in test_combinations: test_case.append(Space(space)) assert_equal(hyperspace, test_case)
def test_constant_property(): space = Space([(0.0, 1.0), (1,), ("a", "b", "c"), (1.0, 5.0, "log-uniform"), ("e",)]) assert space.n_constant_dimensions == 2 for i in [1, 4]: assert space.dimensions[i].is_constant for i in [0, 2, 3]: assert not space.dimensions[i].is_constant
def test_normalize(): # can you pass a Space instance to the Space constructor? space = Space([(0.0, 1.0), (-5, 5), ("a", "b", "c"), (1.0, 5.0, "log-uniform"), ("e", "f")]) space.set_transformer("normalize") X = [[0., -5, 'a', 1., 'e']] Xt = np.zeros((1, 5)) assert_array_equal(space.transform(X), Xt) assert_array_equal(space.inverse_transform(Xt), X) assert_array_equal(space.inverse_transform(space.transform(X)), X)
def main(): logging.basicConfig(format="[%(asctime)s] %(levelname)s %(threadName)-23s: %(message)s", level=logging.INFO) cli_parser = ArgumentParser() cli_parser.add_argument("-c", "--config", type=str, required=True) cli_parser.add_argument("-d", "--dataset", type=str, required=True) cli_parser.add_argument("-r", "--repeats", type=int, default=15) cli_parser.add_argument("-a", "--attempts", type=int, default=30) cli_parser.add_argument("-p", "--parallel", type=int, default=1) cli_parser.add_argument("--cache", type=str, default="cache") cli_parser.add_argument("--iterations", type=int, default=1_000_000) args = cli_parser.parse_args() # Search parameters parser = ArgumentParser() parser.add_argument("--strategy", type=str, default='epsgreedy') parser.add_argument("--cold", action='store_true') parser.add_argument("--x0_min", type=float, default=1e-10) parser.add_argument("--x0_max", type=float, default=10.0) parser.add_argument("--x0_prior", type=str, default='log-uniform') parser.add_argument("--x1_min", type=float, default=1e-10) parser.add_argument("--x1_max", type=float, default=1000.0) parser.add_argument("--x1_prior", type=str, default='log-uniform') parser.add_argument("--lr", type=float, default=0.01) parser.add_argument("--l2", type=float, default=1.0) parser.add_argument("--eps", type=float, default=0.1) parser.add_argument("--tau", type=float, default=1.0) parser.add_argument("--alpha", type=float, default=1.0) parser.add_argument("--cap", type=float, default=0.1) # Read experiment configuration with open(args.config, 'rt') as f: lines = f.readlines() configs = [parser.parse_args(line.strip().split(" ")) for line in lines] # Run experiments in task executor with MultiThreadScheduler(args.parallel) as scheduler: targets = [] for config in configs: space = Space([ Real(low=config.x0_min, high=config.x0_max, prior=config.x0_prior), Real(low=config.x1_min, high=config.x1_max, prior=config.x1_prior) ]) maximize = True if config.strategy == 'ips' else False hyperopt = LogGridOptimizer(target_fn, space, maximize=maximize, max_parallel=5, kwargs={ "config": config, "data": args.dataset, "repeats": args.repeats, "iterations": args.iterations, "seed_base": 4200 }, bases=[1, 3]) targets.append(hyperopt) results = [target.optimize(args.attempts if args.attempts != -1 else hyperopt.nr_max_attempts) for target in targets] scheduler.block_until_tasks_finish() results = [r.result.value for r in results] for target, result in zip(targets, results): logging.info(f"{target.kwargs['config'].strategy} == LR:{result[0][0]}, L2:{result[0][1]} -> performance (95% CI) = {result[1]}")
def test_skopt(): space = Space([(0.0, 1.0)]) for i in range(TESTS): prime_idx = randint(0, MAX_BASE) hammer = Hammersley(prime_idx) skhammer = SkoptHammersley(0, 0, [PRIME_VECTOR[prime_idx]]) h1 = hammer.get_array(SIZE) h2 = np.squeeze(skhammer.generate(space.dimensions, SIZE)) assert np.allclose(h1, h2)
def generate(self, dimensions, n_samples, random_state=None): """Creates latin hypercube samples with maxpro criterion. Parameters ---------- dimensions : list, shape (n_dims,) List of search space dimensions. Each search dimension can be defined either as - a `(lower_bound, upper_bound)` tuple (for `Real` or `Integer` dimensions), - a `(lower_bound, upper_bound, "prior")` tuple (for `Real` dimensions), - as a list of categories (for `Categorical` dimensions), or - an instance of a `Dimension` object (`Real`, `Integer` or `Categorical`). n_samples : int The order of the LHS sequence. Defines the number of samples. random_state : int, RandomState instance, or None (default) Set random state to something other than None for reproducible results. Returns ------- np.array, shape=(n_dim, n_samples) LHS set """ rng = check_random_state(random_state) space = Space(dimensions) transformer = space.get_transformer() n_dim = space.n_dims space.set_transformer("normalize") h = self._lhs_normalized(n_dim, n_samples, rng) self.num_pts = n_samples self.dim = n_dim if self.use_gradient: print('Using gradient descent') bounds = [(0, 1)] * len(dimensions) * self.num_pts h_opt = minimize(self.maxpro_criter, h, jac=self.maxpro_grad, bounds=bounds) h_opt = h_opt['x'].reshape(n_samples, n_dim) else: print('Using naive method') best = 1e+6 for i in range(self.iterations): h = self._lhs_normalized(n_dim, n_samples, i * rng) criter = self.maxpro_criter(h) if best > criter: best = criter h_opt = h.copy() h_opt = space.inverse_transform(h_opt) space.set_transformer(transformer) return h_opt
def test_acquisition_gradient_cookbook(): rng = np.random.RandomState(0) X = rng.randn(20, 5) y = rng.randn(20) X_new = rng.randn(5) gpr = cook_estimator("GP", Space(((-5.0, 5.0), )), random_state=0) gpr.fit(X, y) for acq_func in ["LCB", "PI", "EI"]: check_gradient_correctness(X_new, gpr, acq_func, np.max(y))
def test_acquisition_per_second_gradient(acq_func): rng = np.random.RandomState(0) X = rng.randn(20, 10) # Make the second component large, so that mean_grad and std_grad # do not become zero. y = np.vstack((X[:, 0], np.abs(X[:, 0])**3)).T for X_new in [rng.randn(10), rng.randn(10)]: gpr = cook_estimator("GP", Space(((-5.0, 5.0), )), random_state=0) mor = MultiOutputRegressor(gpr) mor.fit(X, y) check_gradient_correctness(X_new, mor, acq_func, 1.5)
def optimize(self, num_vars, objective_function, gradient_function=None, variable_bounds=None, initial_point=None): callbacks = [] def alt_obj_fn(pars): fn = objective_function(pars) callbacks.append({'point': pars, 'fn': fn}) return fn result = super().optimize(num_vars, alt_obj_fn, gradient_function, variable_bounds, initial_point) if self._num_restarts is not None: for i in range(self._num_restarts - 1): if variable_bounds is not None: init_pt = [ np.random.uniform(dn, up) for dn, up in variable_bounds ] else: init_pt = [ np.random.uniform(-np.pi, +np.pi) for _ in range(num_vars) ] result_new = super().optimize(num_vars, alt_obj_fn, gradient_function, variable_bounds, init_pt) if result_new[1] < result[1]: result = result_new X = [step['point'] for step in callbacks] y = [step['fn'] for step in callbacks] if self._make_model and (len(callbacks) < self._max_model_points): model = GaussianProcessRegressor() model.fit(X, y) else: model = None if variable_bounds is not None: space = Space([Real(low, high) for low, high in variable_bounds]) else: space = None self.optimization_result = create_result( X, y, space=space, models=[model] if model is not None else None) return result
def __init__(self, dimensions_file: str, min_num_results_to_fit: int=8, lease_timout='2 days'): self.__all_experiments = pd.DataFrame() self.__all_experiments['status'] = [self.WAITING] * len(self.__all_experiments) self.__all_experiments['last_update'] = pd.Series(pd.Timestamp(float('NaN'))) self.__all_experiments['client'] = [""] * len(self.__all_experiments) self.__lease_duration = pd.to_timedelta(lease_timout) self.__leased_experiments = [] dims = self.__load_dimensions(dimensions_file) self.__dimension_names = list(dims.keys()) self.__dimensions = list(dims.values()) self.__min_num_results_to_fit = min_num_results_to_fit # Initialize dim_types = [check_dimension(d) for d in self.__dimensions] is_cat = all([isinstance(check_dimension(d), Categorical) for d in dim_types]) if is_cat: transformed_dims = [check_dimension(d, transform="identity") for d in self.__dimensions] else: transformed_dims = [] for dim_type, dim in zip(dim_types, self.__dimensions): if isinstance(dim_type, Categorical): transformed_dims.append(check_dimension(dim, transform="onehot")) # To make sure that GP operates in the [0, 1] space else: transformed_dims.append(check_dimension(dim, transform="normalize")) space = Space(transformed_dims) # Default GP cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) if is_cat: other_kernel = HammingKernel(length_scale=np.ones(space.transformed_n_dims)) acq_optimizer = "lbfgs" else: other_kernel = Matern( length_scale=np.ones(space.transformed_n_dims), length_scale_bounds=[(0.01, 100)] * space.transformed_n_dims, nu=2.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, random_state=None, alpha=0.0, noise='gaussian', n_restarts_optimizer=2) self.__opt = Optimizer(self.__dimensions, base_estimator, acq_optimizer="lbfgs", n_random_starts=100, acq_optimizer_kwargs=dict(n_points=10000))
def test_space_api(): space = Space([(0.0, 1.0), (-5, 5), ("a", "b", "c"), (1.0, 5.0, "log-uniform"), ("e", "f")]) cat_space = Space([(1, "r"), (1.0, "r")]) assert isinstance(cat_space.dimensions[0], Categorical) assert isinstance(cat_space.dimensions[1], Categorical) assert_equal(len(space.dimensions), 5) assert_true(isinstance(space.dimensions[0], Real)) assert_true(isinstance(space.dimensions[1], Integer)) assert_true(isinstance(space.dimensions[2], Categorical)) assert_true(isinstance(space.dimensions[3], Real)) assert_true(isinstance(space.dimensions[4], Categorical)) samples = space.rvs(n_samples=10, random_state=0) assert_equal(len(samples), 10) assert_equal(len(samples[0]), 5) assert_true(isinstance(samples, list)) for n in range(4): assert_true(isinstance(samples[n], list)) assert_true(isinstance(samples[0][0], numbers.Real)) assert_true(isinstance(samples[0][1], numbers.Integral)) assert_true(isinstance(samples[0][2], str)) assert_true(isinstance(samples[0][3], numbers.Real)) assert_true(isinstance(samples[0][4], str)) samples_transformed = space.transform(samples) assert_equal(samples_transformed.shape[0], len(samples)) assert_equal(samples_transformed.shape[1], 1 + 1 + 3 + 1 + 1) assert_array_equal(samples, space.inverse_transform(samples_transformed)) samples = space.inverse_transform(samples_transformed) assert_true(isinstance(samples[0][0], numbers.Real)) assert_true(isinstance(samples[0][1], numbers.Integral)) assert_true(isinstance(samples[0][2], str)) assert_true(isinstance(samples[0][3], numbers.Real)) assert_true(isinstance(samples[0][4], str)) for b1, b2 in zip(space.bounds, [(0.0, 1.0), (-5, 5), np.asarray(["a", "b", "c"]), (1.0, 5.0), np.asarray(["e", "f"])]): assert_array_equal(b1, b2) for b1, b2 in zip(space.transformed_bounds, [(0.0, 1.0), (-5, 5), (0.0, 1.0), (0.0, 1.0), (0.0, 1.0), (np.log10(1.0), np.log10(5.0)), (0.0, 1.0)]): assert_array_equal(b1, b2)
def test_acquisition_per_second(acq_func): X = np.reshape(np.linspace(4.0, 8.0, 10), (-1, 1)) y = np.vstack((np.ones(10), np.ravel(np.log(X)))).T cgpr = ConstantGPRSurrogate(Space(((1.0, 9.0),))) cgpr.fit(X, y) X_pred = np.reshape(np.linspace(1.0, 11.0, 20), (-1, 1)) indices = np.arange(6) vals = _gaussian_acquisition(X_pred, cgpr, y_opt=1.0, acq_func=acq_func) for fast, slow in zip(indices[:-1], indices[1:]): assert vals[slow] > vals[fast] acq_wo_time = _gaussian_acquisition( X, cgpr.estimators_[0], y_opt=1.2, acq_func=acq_func[:2]) acq_with_time = _gaussian_acquisition( X, cgpr, y_opt=1.2, acq_func=acq_func) assert_array_almost_equal(acq_wo_time / acq_with_time, np.ravel(X), 2)
def test_dimension_name(): notnames = [1, 1., True] for n in notnames: with pytest.raises(ValueError) as exc: real = Real(1, 2, name=n) assert("Dimension's name must be either string or" "None." == exc.value.args[0]) s = Space([Real(1, 2, name="a"), Integer(1, 100, name="b"), Categorical(["red, blue"], name="c")]) assert s["a"] == (0, s.dimensions[0]) assert s["a", "c"] == [(0, s.dimensions[0]), (2, s.dimensions[2])] assert s[["a", "c"]] == [(0, s.dimensions[0]), (2, s.dimensions[2])] assert s[("a", "c")] == [(0, s.dimensions[0]), (2, s.dimensions[2])] assert s[0] == (0, s.dimensions[0]) assert s[0, "c"] == [(0, s.dimensions[0]), (2, s.dimensions[2])] assert s[0, 2] == [(0, s.dimensions[0]), (2, s.dimensions[2])]
def test_normalize_bounds(): bounds = [(-999, 189000), Categorical((True, False))] space = Space(normalize_dimensions(bounds)) for a in np.linspace(1e-9, 0.4999, 1000): x = space.inverse_transform([[a, a]]) check_limits(x[0][0], -999, 189000) y = space.transform(x) check_limits(y, 0., 1.) for a in np.linspace(0.50001, 1e-9 + 1., 1000): x = space.inverse_transform([[a, a]]) check_limits(x[0][0], -999, 189000) y = space.transform(x) check_limits(y, 0., 1.)
def __init__(self, bounds, method='latin'): self.dimensions = len(bounds) self.position = np.empty(self.dimensions) self.velocity = np.zeros(self.dimensions) self.pbest_position = np.empty(self.dimensions) self.pbest_value = None self.lowerbounds, self.upperbounds = np.asarray(bounds).T # initialize positions if method == 'random': position = np.random.rand(self.dimensions) elif method == 'latin': space = Space([(0.,1.)]*self.dimensions) lhs = Lhs() position = np.asarray(lhs.generate(space.dimensions,1))[0] min_b, max_b = np.asarray(bounds).T diff = max_b - min_b self.position = min_b + position * diff