def test_same_set_of_points_ask(strategy, surrogate): """ For n_points not None, tests whether two consecutive calls to ask return the same sets of points. Parameters ---------- * `strategy` [string]: Name of the strategy to use during optimization. * `surrogate` [scikit-optimize surrogate class]: A class of the scikit-optimize surrogate used in Optimizer. """ optimizer = Optimizer( base_estimator=surrogate(), dimensions=[Real(-5.0, 10.0), Real(0.0, 15.0)], acq_optimizer='sampling', random_state=2 ) for i in range(n_steps): xa = optimizer.ask(n_points, strategy) xb = optimizer.ask(n_points, strategy) optimizer.tell(xa, [branin(v) for v in xa]) assert_equal(xa, xb) # check if the sets of points generated are equal
def test_constant_liar_runs(strategy, surrogate): """ Tests whether the optimizer runs properly during the random initialization phase and beyond Parameters ---------- * `strategy` [string]: Name of the strategy to use during optimization. * `surrogate` [scikit-optimize surrogate class]: A class of the scikit-optimize surrogate used in Optimizer. """ optimizer = Optimizer( base_estimator=surrogate(), dimensions=[Real(-5.0, 10.0), Real(0.0, 15.0)], acq_optimizer='sampling', random_state=0 ) # test arguments check assert_raises(ValueError, optimizer.ask, {"strategy": "cl_maen"}) assert_raises(ValueError, optimizer.ask, {"n_points": "0"}) assert_raises(ValueError, optimizer.ask, {"n_points": 0}) for i in range(n_steps): x = optimizer.ask(n_points=n_points, strategy=strategy) # check if actually n_points was generated assert_equal(len(x), n_points) optimizer.tell(x, [branin(v) for v in x])
def test_dict_list_space_representation(): """ Tests whether the conversion of the dictionary and list representation of a point from a search space works properly. """ chef_space = { 'Cooking time': (0, 1200), # in minutes 'Main ingredient': [ 'cheese', 'cherimoya', 'chicken', 'chard', 'chocolate', 'chicory' ], 'Secondary ingredient': [ 'love', 'passion', 'dedication' ], 'Cooking temperature': (-273.16, 10000.0) # in Celsius } opt = Optimizer(dimensions=dimensions_aslist(chef_space)) point = opt.ask() # check if the back transformed point and original one are equivalent assert_equal( point, point_aslist(chef_space, point_asdict(chef_space, point)) )
def test_all_points_different(strategy, surrogate): """ Tests whether the parallel optimizer always generates different points to evaluate. Parameters ---------- * `strategy` [string]: Name of the strategy to use during optimization. * `surrogate` [scikit-optimize surrogate class]: A class of the scikit-optimize surrogate used in Optimizer. """ optimizer = Optimizer( base_estimator=surrogate(), dimensions=[Real(-5.0, 10.0), Real(0.0, 15.0)], acq_optimizer='sampling', random_state=1 ) tolerance = 1e-3 # distance above which points are assumed same for i in range(n_steps): x = optimizer.ask(n_points, strategy) optimizer.tell(x, [branin(v) for v in x]) distances = pdist(x) assert all(distances > tolerance)
def test_dump_and_load_optimizer(): base_estimator = ExtraTreesRegressor(random_state=2) opt = Optimizer([(-2.0, 2.0)], base_estimator, n_random_starts=1, acq_optimizer="sampling") opt.run(bench1, n_iter=3) with tempfile.TemporaryFile() as f: dump(opt, f) f.seek(0) load(f)
def test_reproducible_runs(strategy, surrogate): # two runs of the optimizer should yield exactly the same results optimizer = Optimizer( base_estimator=surrogate(random_state=1), dimensions=[Real(-5.0, 10.0), Real(0.0, 15.0)], acq_optimizer='sampling', random_state=1 ) points = [] for i in range(n_steps): x = optimizer.ask(n_points, strategy) points.append(x) optimizer.tell(x, [branin(v) for v in x]) # the x's should be exaclty as they are in `points` optimizer = Optimizer( base_estimator=surrogate(random_state=1), dimensions=[Real(-5.0, 10.0), Real(0.0, 15.0)], acq_optimizer='sampling', random_state=1 ) for i in range(n_steps): x = optimizer.ask(n_points, strategy) assert points[i] == x optimizer.tell(x, [branin(v) for v in x])
def __init__(self, dimensions_file: str, min_num_results_to_fit: int=8, lease_timout='2 days'): self.__all_experiments = pd.DataFrame() self.__all_experiments['status'] = [self.WAITING] * len(self.__all_experiments) self.__all_experiments['last_update'] = pd.Series(pd.Timestamp(float('NaN'))) self.__all_experiments['client'] = [""] * len(self.__all_experiments) self.__lease_duration = pd.to_timedelta(lease_timout) self.__leased_experiments = [] dims = self.__load_dimensions(dimensions_file) self.__dimension_names = list(dims.keys()) self.__dimensions = list(dims.values()) self.__min_num_results_to_fit = min_num_results_to_fit # Initialize dim_types = [check_dimension(d) for d in self.__dimensions] is_cat = all([isinstance(check_dimension(d), Categorical) for d in dim_types]) if is_cat: transformed_dims = [check_dimension(d, transform="identity") for d in self.__dimensions] else: transformed_dims = [] for dim_type, dim in zip(dim_types, self.__dimensions): if isinstance(dim_type, Categorical): transformed_dims.append(check_dimension(dim, transform="onehot")) # To make sure that GP operates in the [0, 1] space else: transformed_dims.append(check_dimension(dim, transform="normalize")) space = Space(transformed_dims) # Default GP cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) if is_cat: other_kernel = HammingKernel(length_scale=np.ones(space.transformed_n_dims)) acq_optimizer = "lbfgs" else: other_kernel = Matern( length_scale=np.ones(space.transformed_n_dims), length_scale_bounds=[(0.01, 100)] * space.transformed_n_dims, nu=2.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, random_state=None, alpha=0.0, noise='gaussian', n_restarts_optimizer=2) self.__opt = Optimizer(self.__dimensions, base_estimator, acq_optimizer="lbfgs", n_random_starts=100, acq_optimizer_kwargs=dict(n_points=10000))
def get_optimizer(config, features): return Optimizer( [ Categorical(categories=[0, 1], name=f, prior=[0.75, 0.25]) for f in features ], #[Integer(low=0, high=1, name=f) for f in features], base_estimator=config.get('base_estimator', 'ET'), acq_optimizer=config.get('acq_optimizer', 'sampling'), n_random_starts=config.get('n_random_starts', None), n_initial_points=config.get('n_initial_points', 10), acq_func=config.get('acq_func', 'gp_hedge'), random_state=config.get('random_state', None), acq_func_kwargs=config.get('acq_func_kwargs', None), acq_optimizer_kwargs=config.get('acq_optimizer_kwargs', None))
def optimizer(self): """Return skopt's optimizer.""" if self._optimizer is None: self._optimizer = Optimizer( base_estimator=GaussianProcessRegressor( alpha=self.alpha, n_restarts_optimizer=self.n_restarts_optimizer, noise=self.noise, normalize_y=self.normalize_y), dimensions=orion_space_to_skopt_space(self.space), n_initial_points=self.n_initial_points, acq_func=self.acq_func) self.seed_rng(self.seed) return self._optimizer
def setup_optz(): logger.debug("setup() START") from skopt import Optimizer # Set up problem = Problem() starting_point = problem.starting_point seed = 42 # Start the optimizer parDict = { 'kappa' : 1.96 } space = [problem.space[key] for key in problem.params] optimizer = Optimizer(space, base_estimator='RF', acq_optimizer='sampling', acq_func='LCB', acq_func_kwargs={}, random_state=seed) logger.debug("setup() STOP") return (problem, optimizer)
def set_optimizer(self, n_iter, opt_seed, acq_func, gp_seed, **kwargs): self.logger.info('Retrieving model stored at: {}'.format(self.optimizer_path)) try: optimizer = load(self.optimizer_path) self.logger.info('Loading model stored at: {}'.format(self.optimizer_path)) finished_iter = np.array(optimizer.yi).shape[0] if finished_iter == 0: optimizer = None self.logger.info('Optimizer did not finish any iterations so setting optimizer to null') except KeyError: self.logger.error('Cannot open the file {}'.format(self.optimizer_path)) optimizer = None except ValueError: self.logger.error('Cannot open the file {}'.format(self.optimizer_path)) optimizer = None except FileNotFoundError: self.logger.error('No such file or directory: {}'.format(self.optimizer_path)) optimizer = None if optimizer is not None: n_iter = n_iter - finished_iter if n_iter < 0: n_iter = 0 self.logger.info('Iterations already done: {} and running iterations {}'.format(finished_iter, n_iter)) self.opt = optimizer self.logger.debug('Setting the provided optimizer') self.log_best_params() else: transformed = [] for param in self.parameter_ranges: transformed.append(check_dimension(param)) self.logger.info("Parameter Space: {}".format(transformed)) norm_space = normalize_dimensions(transformed) self.logger.info("Parameter Space after transformation: {}".format(norm_space)) categorical_space = np.array([isinstance(s, Categorical) for s in norm_space]) self.logger.info("categorical_space: {}".format(categorical_space)) if np.all(categorical_space): base_estimator = cook_estimator("RF", space=norm_space, random_state=gp_seed) else: base_estimator = cook_estimator("GP", space=norm_space, random_state=gp_seed, noise="gaussian") self.opt = Optimizer(dimensions=self.parameter_ranges, random_state=opt_seed, base_estimator=base_estimator, acq_func=acq_func, **kwargs) return n_iter
def create_opt(lines, ranker_name): gp_seed, opt_seed = get_seed(lines) _ranker_class = object_rankers[ranker_name] _ranker_class._use_early_stopping = True param_ranges = _ranker_class.set_tunable_parameter_ranges({}) transformed = [] for param in param_ranges: transformed.append(check_dimension(param)) space = normalize_dimensions(transformed) base_estimator = cook_estimator("GP", space=space, random_state=gp_seed, noise="gaussian") optimizer = Optimizer(dimensions=param_ranges, random_state=opt_seed, base_estimator=base_estimator) return optimizer
def run_batch(s): f = open( "/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/tmp/hyperres_pickled_args", "rb") args = pickle.load(f) f = open( "/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/tmp/hyperres_pickled_spaces", "rb") hyperspaces = pickle.load(f) for i in s: current_space = hyperspaces[i] optimizer = Optimizer(current_space) if args.model == "segmentation_cityscapes" or args.model == "segmentation_gis": search_algo = SkOptSearch( optimizer, ['learning_rate', 'epochs', 'batch_size', 'adam_epsilon'], metric='average_res', mode='max') else: search_algo = SkOptSearch(optimizer, [ 'learning_rate', 'dropout', 'epochs', 'batch_size', 'adam_epsilon' ], metric='average_res', mode='max') analysis = tune.run( multi_train, search_alg=search_algo, num_samples=int(args.trials), resources_per_trial={ 'cpu': 25, 'gpu': 1 }, local_dir="/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/ray_results" ) df = analysis.results_df df_name = "/lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/hyper_resilient_results/" + args.out + "/" df_name += "space_" df_name += str(i) df_name += ".csv" df.to_csv(df_name) print("Finished space " + args.space) print( "Finished all spaces. Files writtten to /lus/theta-fs0/projects/CVD-Mol-AI/mzvyagin/hyper_resilient_results/" + args.out)
def __init__( self, problem, run, evaluator, population_size=100, sample_size=10, n_jobs=1, **kwargs, ): super().__init__( problem=problem, run=run, evaluator=evaluator, population_size=population_size, sample_size=sample_size, **kwargs, ) self.n_jobs = int(n_jobs) # Initialize Hyperaparameter space self.hp_space = [] # add the 'learning_rate' space to the HPO search space self.hp_space.append(self.problem.space["hyperparameters"]["learning_rate"]) # add the 'batch_size' space to the HPO search space self.hp_space.append(self.problem.space["hyperparameters"]["batch_size"]) # Initialize opitmizer of hyperparameter space acq_func_kwargs = {"xi": 0.000001, "kappa": 0.001} # tiny exploration # self.free_workers = 128 #! TODO: test self.n_initial_points = self.free_workers self.hp_opt = SkOptimizer( dimensions=self.hp_space, base_estimator=RandomForestRegressor(n_jobs=32), # base_estimator=RandomForestRegressor(n_jobs=4), acq_func="LCB", acq_optimizer="sampling", acq_func_kwargs=acq_func_kwargs, n_initial_points=self.n_initial_points, # model_queue_size=100, )
def __init__(self, problem, num_workers, surrogate_model='RF', acq_func='gp_hedge', acq_kappa=1.96, liar_strategy='cl_max', n_jobs=1, **kwargs): assert surrogate_model in [ "RF", "ET", "GBRT", "GP", "DUMMY" ], f"Unknown scikit-optimize base_estimator: {surrogate_model}" if surrogate_model == "RF": base_estimator = RandomForestRegressor(n_jobs=n_jobs) elif surrogate_model == "ET": base_estimator = ExtraTreesRegressor(n_jobs=n_jobs) elif surrogate_model == "GBRT": base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs) else: base_estimator = surrogate_model self.space = problem.space # queue of remaining starting points self.starting_points = problem.starting_point n_init = inf if surrogate_model == 'DUMMY' else max( num_workers, len(self.starting_points)) self._optimizer = SkOptimizer(self.space.values(), base_estimator=base_estimator, acq_optimizer='sampling', acq_func=acq_func, acq_func_kwargs={'kappa': acq_kappa}, random_state=self.SEED, n_initial_points=n_init) assert liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = liar_strategy self.evals = {} self.counter = 0 logger.info( f"Using skopt.Optimizer with {surrogate_model} base_estimator")
def __init__(self, api_config, base_estimator="GP", acq_func="gp_hedge", n_initial_points=5, **kwargs): """Build wrapper class to use an optimizer in benchmark. Parameters ---------- api_config : dict-like of dict-like Configuration of the optimization variables. See API description. base_estimator : {'GP', 'RF', 'ET', 'GBRT'} How to estimate the objective function. acq_func : {'LCB', 'EI', 'PI', 'gp_hedge', 'EIps', 'PIps'} Acquisition objective to decide next suggestion. n_initial_points : int Number of points to sample randomly before actual Bayes opt. """ AbstractOptimizer.__init__(self, api_config) dimensions, self.round_to_values = ScikitOptimizer.get_sk_dimensions( api_config) # Older versions of skopt don't copy over the dimensions names during # normalization and hence the names are missing in # self.skopt.space.dimensions. Therefore, we save our own copy of # dimensions list to be safe. If we can commit to using the newer # versions of skopt we can delete self.dimensions. self.dimensions_list = tuple(dd.name for dd in dimensions) # Undecided where we want to pass the kwargs, so for now just make sure # they are blank assert len(kwargs) == 0 self.skopt = SkOpt( dimensions, n_initial_points=n_initial_points, base_estimator=base_estimator, acq_func=acq_func, acq_optimizer="auto", acq_func_kwargs={}, acq_optimizer_kwargs={}, )
def __init__( self, problem, run, evaluator, population_size=100, sample_size=10, n_jobs=1, kappa=0.001, xi=0.000001, acq_func="LCB", **kwargs, ): super().__init__( problem=problem, run=run, evaluator=evaluator, population_size=population_size, sample_size=sample_size, **kwargs, ) self.n_jobs = int( n_jobs) # parallelism of BO surrogate model estimator # Initialize Hyperaparameter space self.hp_space = self.problem._hp_space # Initialize opitmizer of hyperparameter space acq_func_kwargs = { "xi": float(xi), "kappa": float(kappa) } # tiny exploration self.n_initial_points = self.free_workers self.hp_opt = SkOptimizer( dimensions=self.hp_space._space, base_estimator=RandomForestRegressor(n_jobs=self.n_jobs), acq_func=acq_func, acq_optimizer="sampling", acq_func_kwargs=acq_func_kwargs, n_initial_points=self.n_initial_points, )
def get_optimizer(self, dimensions: List[Dimension], cpu_count) -> Optimizer: estimator = self.custom_hyperopt.generate_estimator() acq_optimizer = "sampling" if isinstance(estimator, str): if estimator not in ("GP", "RF", "ET", "GBRT"): raise OperationalException(f"Estimator {estimator} not supported.") else: acq_optimizer = "auto" logger.info(f"Using estimator {estimator}.") return Optimizer( dimensions, base_estimator=estimator, acq_optimizer=acq_optimizer, n_initial_points=INITIAL_POINTS, acq_optimizer_kwargs={'n_jobs': cpu_count}, random_state=self.random_state, model_queue_size=SKOPT_MODEL_QUEUE_SIZE, )
def __init__(self, problem, num_workers, args): assert args.learner in ["RF", "ET", "GBRT", "GP", "DUMMY"], f"Unknown scikit-optimize base_estimator: {args.learner}" self.space = problem.space n_init = inf if args.learner=='DUMMY' else num_workers self._optimizer = SkOptimizer( self.space.values(), base_estimator=args.learner, acq_optimizer='sampling', acq_func=args.acq_func, acq_func_kwargs={'kappa':self.KAPPA}, random_state=self.SEED, n_initial_points=n_init ) assert args.liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = args.liar_strategy self.evals = {} self.counter = 0 logger.info("Using skopt.Optimizer with %s base_estimator" % args.learner)
def search(self, data: Dataset, metrics: List[str], cv: Any, n_jobs: int, verbose: int = 0) -> ResultGroup: """ Perform a bayesian search over the specified hyperparameters Parameters ---------- data: Dataset Instance of data to train on metrics: List of str List of metrics to calculate results for cv: Any Either a CV object from sklearn or an int to specify number of folds n_jobs: int Number of jobs to calculate in parallel verbose: int Verbosity level of the method Returns ------- ResultGroup """ optimizer = Optimizer(dimensions_aslist(self.param_grid)) logger.info("Starting Bayesian search...") results = [ self._step(optimizer, data, metrics, cv, n_jobs, verbose) for _ in range(self.n_iter) ] logger.info("Finished Bayesian search...") return ResultGroup(results).sort()
def _make_optimizer(self, params_space): """Instantiate skopt Optimizer class. Parameters ---------- params_space : dict Represents parameter search space. The keys are parameter names (strings) and values are skopt.space.Dimension instances, one of Real, Integer or Categorical. Returns ------- optimizer: Instance of the `Optimizer` class used for for search in some parameter space. """ kwargs = self.optimizer_kwargs_.copy() kwargs['dimensions'] = dimensions_aslist(params_space) optimizer = Optimizer(**kwargs) return optimizer
def submit(n, optimizer: Optimizer, opt_param_names, current_configs, param_space: ParamSpace, queue: Queue): """ Generate and submit n new configurations to a queue. Asks the optimizer for n new values to explore, creates configurations for those points and puts them in the given queue. Args: n: the number of configurations to be generated optimizer: the optimiser object from skopt with the model used for the suggested points to explore opt_param_names: the names for the parameters using the same order of the dimensions in the optimizer current_configs: current list of configurations (updated with the newly generated ones) param_space: parameter space which we can use to convert optimizer points to fully specified configurations queue: que multiprocessing queue in which we put the new configurations """ dims = opt_param_names xs = optimizer.ask(n_points=n) cfgs = [values_to_params(dict(zip(dims, x)), param_space) for x in xs] for i, c in enumerate(cfgs): c["id"] = i + len(current_configs) queue.put(c) current_configs += cfgs
def create_model(self, bounds, modeltag=''): """Updates the model by pickling and uploading (overwriting) the opt model to a special async bucket generated by the stack :param bounds: UUID of the model :type bounds: list of tuples, like [(-1.0,1.0),(-5,5),(-10.5,10.5)...] :param modeltag: Friendly name of the optimizer :type modeltag: string, optional """ self.modelname = str(uuid4()) if all(isinstance(item, tuple) for item in bounds): logging.info('Creating model with bounds: ' + str(bounds)) opt = Optimizer(bounds, "GP", acq_func="EI", acq_optimizer="sampling", initial_point_generator="lhs") self.update_model(self.modelname, opt) logging.info('created model with name ' + self.modelname) if modeltag != '': s3_client = boto3.client('s3') response = s3_client.put_object_tagging( Bucket=self.asyncbucket, Key=self.modelname + '/model.pkl', Tagging={'TagSet': [ { 'Key': 'tag', 'Value': modeltag }, ]}) return self.modelname else: logging.error( 'Input bounds as a list of tuples, like [(-2.0, 2.0), ...]')
class Optimizer: SEED = 12345 KAPPA = 1.96 def __init__(self, num_workers: int, space, learner, acq_func, liar_strategy, **kwargs): assert learner in [ "RF", "ET", "GBRT", "GP", "DUMMY" ], f"Unknown scikit-optimize base_estimator: {learner}" assert liar_strategy in "cl_min cl_mean cl_max".split() self.space = space self.learner = learner self.acq_func = acq_func self.liar_strategy = liar_strategy n_init = inf if learner == 'DUMMY' else num_workers self._optimizer = SkOptimizer(dimensions=self.space.dimensions, base_estimator=self.learner, acq_optimizer='sampling', acq_func=self.acq_func, acq_func_kwargs={'kappa': self.KAPPA}, random_state=self.SEED, n_initial_points=n_init) self.evals = {} self.counter = 0 logger.info("Using skopt.Optimizer with %s base_estimator" % self.learner) def _get_lie(self): if self.liar_strategy == "cl_min": return min(self._optimizer.yi) if self._optimizer.yi else 0.0 elif self.liar_strategy == "cl_mean": return np.mean(self._optimizer.yi) if self._optimizer.yi else 0.0 else: return max(self._optimizer.yi) if self._optimizer.yi else 0.0 def _xy_from_dict(self): XX = list(self.evals.keys()) YY = [self.evals[x] for x in XX] return XX, YY def to_dict(self, x: list) -> dict: return self.space.to_dict(x) def _ask(self): x = self._optimizer.ask() y = self._get_lie() key = tuple(x) if key not in self.evals: self.counter += 1 self._optimizer.tell(x, y) self.evals[key] = y logger.debug(f'_ask: {x} lie: {y}') else: logger.debug(f'Duplicate _ask: {x} lie: {y}') return self.to_dict(x) def ask(self, n_points=None, batch_size=20): if n_points is None: return self._ask() else: batch = [] for _ in range(n_points): batch.append(self._ask()) if len(batch) == batch_size: yield batch batch = [] if batch: yield batch def ask_initial(self, n_points): XX = self._optimizer.ask(n_points=n_points) for x in XX: y = self._get_lie() key = tuple(x) if key not in self.evals: self.counter += 1 self._optimizer.tell(x, y) self.evals[key] = y return [self.to_dict(x) for x in XX] def tell(self, xy_data): assert isinstance(xy_data, list), f"where type(xy_data)=={type(xy_data)}" maxval = max(self._optimizer.yi) if self._optimizer.yi else 0.0 for x, y in xy_data: key = tuple(x.values()) # * tuple(x[k] for k in self.space) assert key in self.evals, f"where key=={key} and self.evals=={self.evals}" logger.debug(f'tell: {x} --> {key}: evaluated objective: {y}') self.evals[key] = (y if y < float_info.max else maxval) self._optimizer.Xi = [] self._optimizer.yi = [] XX, YY = self._xy_from_dict() assert len(XX) == len(YY) == self.counter, ( f"where len(XX)=={len(XX)}," f"len(YY)=={len(YY)}, self.counter=={self.counter}") self._optimizer.tell(XX, YY) assert len(self._optimizer.Xi) == len( self._optimizer.yi) == self.counter, ( f"where len(self._optimizer.Xi)=={len(self._optimizer.Xi)}, " f"len(self._optimizer.yi)=={len(self._optimizer.yi)}," f"self.counter=={self.counter}")
class BayesianOptimizedExperimentQueue(ExperimentQueue): def __init__(self, dimensions_file: str, min_num_results_to_fit: int=8, lease_timout='2 days'): self.__all_experiments = pd.DataFrame() self.__all_experiments['status'] = [self.WAITING] * len(self.__all_experiments) self.__all_experiments['last_update'] = pd.Series(pd.Timestamp(float('NaN'))) self.__all_experiments['client'] = [""] * len(self.__all_experiments) self.__lease_duration = pd.to_timedelta(lease_timout) self.__leased_experiments = [] dims = self.__load_dimensions(dimensions_file) self.__dimension_names = list(dims.keys()) self.__dimensions = list(dims.values()) self.__min_num_results_to_fit = min_num_results_to_fit # Initialize dim_types = [check_dimension(d) for d in self.__dimensions] is_cat = all([isinstance(check_dimension(d), Categorical) for d in dim_types]) if is_cat: transformed_dims = [check_dimension(d, transform="identity") for d in self.__dimensions] else: transformed_dims = [] for dim_type, dim in zip(dim_types, self.__dimensions): if isinstance(dim_type, Categorical): transformed_dims.append(check_dimension(dim, transform="onehot")) # To make sure that GP operates in the [0, 1] space else: transformed_dims.append(check_dimension(dim, transform="normalize")) space = Space(transformed_dims) # Default GP cov_amplitude = ConstantKernel(1.0, (0.01, 1000.0)) if is_cat: other_kernel = HammingKernel(length_scale=np.ones(space.transformed_n_dims)) acq_optimizer = "lbfgs" else: other_kernel = Matern( length_scale=np.ones(space.transformed_n_dims), length_scale_bounds=[(0.01, 100)] * space.transformed_n_dims, nu=2.5) base_estimator = GaussianProcessRegressor( kernel=cov_amplitude * other_kernel, normalize_y=True, random_state=None, alpha=0.0, noise='gaussian', n_restarts_optimizer=2) self.__opt = Optimizer(self.__dimensions, base_estimator, acq_optimizer="lbfgs", n_random_starts=100, acq_optimizer_kwargs=dict(n_points=10000)) @property def all_experiments(self) -> pd.DataFrame: """ :return: The PandasFrame containing the details for all the experiments in the queue. """ return self.__all_experiments @property def completed_percent(self) -> float: return 0. @property def leased_percent(self) -> float: return 0 @property def experiment_parameters(self) -> List: return self.__dimension_names def lease_new(self, client_name: str) -> Tuple[int, Dict]: """ Lease a new experiment lock. Select first any waiting experiments and then re-lease expired ones :param client_name: The name of the leasing client :return: a tuple (id, parameters) or None if nothing is available """ experiment_params = self.__opt.ask() if experiment_params in self.__leased_experiments: experiment_params = self.__compute_alternative_params() self.__leased_experiments.append(experiment_params) # TODO: Add to all experiments, use Ids def parse_dim_val(value, dim_type): if type(dim_type) is Real: return float(value) elif type(dim_type) is Integer: return int(value) return value return {name: parse_dim_val(value, dim_type) for name, dim_type, value in zip(self.__dimension_names, self.__dimensions, experiment_params)}, -1 def __compute_alternative_params(self): # Copied directly from skopt transformed_bounds = np.array(self.__opt.space.transformed_bounds) est = clone(self.__opt.base_estimator) with warnings.catch_warnings(): warnings.simplefilter("ignore") est.fit(self.__opt.space.transform(self.__opt.Xi), self.__opt.yi) X = self.__opt.space.transform(self.__opt.space.rvs( n_samples=self.__opt.n_points, random_state=self.__opt.rng)) values = _gaussian_acquisition(X=X, model=est, y_opt=np.min(self.__opt.yi), acq_func='EI', acq_func_kwargs=dict(n_points=10000)) print('original point ei: %s' % np.min(values)) discount_width = .5 values = self.__discount_leased_params(X, values, discount_width) while np.min(values) > -1e-5 and discount_width > 1e-2: discount_width *= .9 values = _gaussian_acquisition(X=X, model=est, y_opt=np.min(self.__opt.yi), acq_func='EI', acq_func_kwargs=dict(n_points=10000)) values = self.__discount_leased_params(X, values, discount_width) next_x = X[np.argmin(values)] print('new point ei: %s' % np.min(values)) if not self.__opt.space.is_categorical: next_x = np.clip(next_x, transformed_bounds[:, 0], transformed_bounds[:, 1]) return self.__opt.space.inverse_transform(next_x.reshape((1, -1)))[0] @staticmethod def leased_discount(center, width, x_values): """Triangular (cone) discount""" distance_from_center = np.linalg.norm(x_values - center, 2, axis=1) discount = -distance_from_center / width + 1 discount[discount < 0] = 0 return discount def __discount_leased_params(self, X, values, discount_width_size): transformed_leased_params = self.__opt.space.transform(np.array(self.__leased_experiments)) discount_factor = reduce(lambda x, y: x * y, (self.leased_discount(p, discount_width_size, X) for p in self.__leased_experiments), np.ones(values.shape[0])) out_vals = values * (1. - discount_factor) return out_vals def complete(self, experiment_id: int, parameters: Dict, client: str, result: float = 0) -> None: """ Declare an experiment to be completed. :param experiment_id: the id of the experiment or -1 if unknown :param client: the client id :param result: the output results of the experiment. This may be used in optimizing queues. """ parameters = [parameters[n] for n in self.__dimension_names] if parameters in self.__leased_experiments: self.__leased_experiments.remove(parameters) do_fit_model = len(self.__opt.yi) >= self.__min_num_results_to_fit # Unfortunate hack: this depends on the internals. if do_fit_model: self.__opt._n_random_starts = 0 # Since we have adequately many results, stop using random self.__opt.tell(parameters, result, fit=do_fit_model) def __load_dimensions(self, dimensions_file:str)->Dict: with open(dimensions_file) as f: dimensions = json.load(f) def parse_dimension(specs: Dict[str, Any]): if specs['type'] == 'Real': return specs['name'], Real(specs['low'], specs['high']) elif specs['type'] == 'Integer': return specs['name'], Integer(specs['low'], specs['high']) elif specs['type'] == 'Categorical': return specs['name'], Categorical(specs['categories']) else: raise Exception('Unrecognized dimension type %s' % specs['type']) return OrderedDict([parse_dimension(d) for d in dimensions])
def init_opt(self, bounds): return Optimizer(dimensions=bounds, random_state=1, base_estimator='gp', n_initial_points=2 * self.n_points)
def main_rule_utils(config, main_loop_seed=MAIN_LOOP_INITIAL_SEED): rule_utils.store_config_on_first_run(config) experience_buffer = utils.ExperienceBuffer(config['max_experience_buffer']) config_keys = list(config['initial_config_ranges'].keys()) if deterministic_games: utils.set_seed(main_loop_seed) fixed_pool_mode = config['play_fixed_pool_only'] if fixed_pool_mode: fixed_opp_repeats = config['fixed_opponents_num_repeat_first_configs'] config_has_range = isinstance( list(config['initial_config_ranges'].values())[0], tuple) # Prepare the Bayesian optimizer if config_has_range: opt_range = [config['initial_config_ranges'][k][0] for k in config_keys] opt = Optimizer(opt_range) if config['play_fixed_pool_fit_prev_data']: fixed_pool_experience_path = rule_utils.get_self_play_experience_path( config['pool_name']) if os.path.exists(fixed_pool_experience_path): print('\nBayesian fit to earlier experiments') this_folder = os.path.dirname(__file__) agents_folder = os.path.join( this_folder, '../Rule agents/' + config['pool_name']) config_settings_path = os.path.join( agents_folder, CONFIG_SETTINGS_EXTENSION) if os.path.exists(config_settings_path): config_results = pd.read_csv(config_settings_path) suggested = config_results.iloc[:, :-1].values.tolist() target_scores = (-config_results.iloc[:, -1].values).tolist() opt.tell(suggested, target_scores) # import pdb; pdb.set_trace() # print(opt.get_result().x, opt.get_result().fun) # WRONG! else: opt = None next_fixed_opponent_suggested = None iteration_config_rewards = None experience_features_rewards_path = None while True: if deterministic_games: utils.set_seed(main_loop_seed) # Section 1: play games against agents of N previous pools if config['num_games_previous_pools'] and not fixed_pool_mode: print('\nPlay vs other rule based agents from the last {} pools'.format( config['max_pool_size'])) (self_play_experience, rules_config_path, avg_reward_sp, _) = rule_experience.play_games( pool_name=config['pool_name'], num_games=config['num_games_previous_pools'], max_pool_size=config['max_pool_size'], num_agents=config['num_agents_per_game'], exclude_current_from_opponents=False, record_videos_new_iteration=config['record_videos_new_iteration'], initial_config_ranges=config['initial_config_ranges'], use_multiprocessing=config['use_multiprocessing'], episode_steps_override=config['episode_steps_override'], early_episode_termination=config['early_episode_termination_steps'], ) experience_buffer.add(self_play_experience) # Section 2: play games against agents of the previous pool if config['num_games_evaluation'] and not fixed_pool_mode: print('\nPlay vs previous iteration') (evaluation_experience, rules_config_path, avg_reward_eval, _) = rule_experience.play_games( pool_name=config['pool_name'], num_games=config['num_games_evaluation'], max_pool_size=2, num_agents=config['num_agents_per_game'], exclude_current_from_opponents=True, use_multiprocessing=config['use_multiprocessing'], episode_steps_override=config['episode_steps_override'], early_episode_termination=config['early_episode_termination_steps'], ) # experience_buffer.add(evaluation_experience) if fixed_pool_mode: if iteration_config_rewards is not None: # Update the optimizer using the most recent fixed opponent pool # results target_scores = np.reshape(-iteration_config_rewards[ 'episode_reward'].values, [-1, fixed_opp_repeats]).mean(1).tolist() if config_has_range: opt.tell(next_fixed_opponent_suggested, target_scores) # Append the tried settings to the settings-scores file config_rewards = rule_utils.append_config_scores( next_fixed_opponent_suggested, target_scores, config_keys, config['pool_name'], CONFIG_SETTINGS_EXTENSION) # Update the plot of the tried settings and obtained scores rule_utils.plot_reward_versus_features( experience_features_rewards_path, config_rewards, target_col="Average win rate", include_all_targets=True, plot_name_suffix="config setting average win rate", all_scatter=True) # Select the next hyperparameters to try try: next_fixed_opponent_suggested, next_fixed_opponent_configs = ( rule_utils.get_next_config_settings( opt, config_keys, config['num_games_fixed_opponents_pool'], fixed_opp_repeats, config['initial_config_ranges']) ) except: import pdb; pdb.set_trace() # Section 3: play games against a fixed opponents pool if config['num_games_fixed_opponents_pool']: print('\nPlay vs the fixed opponents pool') (fixed_opponents_experience, rules_config_path, avg_reward_fixed_opponents, opponent_rewards) = ( rule_experience.play_games( pool_name=config['pool_name'], num_games=config['num_games_fixed_opponents_pool'], max_pool_size=1, # Any positive integer is fine num_agents=config['num_agents_per_game'], exclude_current_from_opponents=False, fixed_opponent_pool=True, initial_config_ranges=config['initial_config_ranges'], use_multiprocessing=config['use_multiprocessing'], num_repeat_first_configs=fixed_opp_repeats, first_config_overrides=next_fixed_opponent_configs, episode_steps_override=config['episode_steps_override'], early_episode_termination=config['early_episode_termination_steps'], ) ) experience_buffer.add(fixed_opponents_experience) # import pdb; pdb.set_trace() # Select the values that will be used to determine if a next iteration file # will be created serialized_raw_experience = fixed_opponents_experience if ( fixed_pool_mode) else self_play_experience # Optionally append the experience of interest to disk iteration_config_rewards = ( rule_utils.serialize_game_experience_for_learning( serialized_raw_experience, fixed_pool_mode, config_keys)) if config['save_experience_data_to_disk']: experience_features_rewards_path = rule_utils.write_experience_data( config['pool_name'], iteration_config_rewards) # Section 4: Update the iteration, store videos and record learning # progress. if fixed_pool_mode: update_config = {'Time stamp': str(datetime.now())} for i in range(len(opponent_rewards)): update_config['Reward ' + opponent_rewards[i][2]] = np.round( opponent_rewards[i][1]/(1e-10+opponent_rewards[i][0]), 2) rule_utils.update_learning_progress(config['pool_name'], update_config) config_override_agents = ( fixed_opponents_experience[0].config_game_agents) # import pdb; pdb.set_trace() rule_utils.record_videos( rules_config_path, config['num_agents_per_game'], extension_override=str(datetime.now())[:19], config_override_agents=config_override_agents, env_seed_deterministic=fixed_opponents_experience[0].env_random_seed, rng_action_seeds=fixed_opponents_experience[0].act_random_seeds, first_game_recording=fixed_opponents_experience[0].game_recording, deterministic_games=config['deterministic_games'], deterministic_extension=f" - Seed {main_loop_seed}") else: # Save a new iteration if it has significantly improved data_rules_path = rules_config_path if min(avg_reward_sp, avg_reward_eval) >= config[ 'min_new_iteration_win_rate']: original_rules_config_path = rules_config_path incremented_rules_path = utils.increment_iteration_id( rules_config_path, extension='.json') copyfile(rules_config_path, incremented_rules_path) rules_config_path = incremented_rules_path if config['record_videos_new_iteration']: rule_utils.record_videos( original_rules_config_path, config['num_agents_per_game']) elif config['record_videos_each_main_loop']: rule_utils.record_videos( rules_config_path, config['num_agents_per_game'], str(datetime.now())[:19]) # Record learning progress # import pdb; pdb.set_trace() rule_utils.update_learning_progress(config['pool_name'], { 'Time stamp': str(datetime.now()), 'Average reward self play': avg_reward_sp, 'Average evaluation reward': avg_reward_eval, 'Experience buffer size': experience_buffer.size(), 'Data rules path': data_rules_path, }) # Section 5: Update the latest config range using the data in the # experience buffer if rules_config_path is not None: if not fixed_pool_mode: # Evolve the config ranges in a very simple gradient free way. rule_utils.evolve_config( rules_config_path, iteration_config_rewards, config['initial_config_ranges']) # Create plot(s) of the terminal reward as a function of all serialized # features if config['save_experience_data_to_disk']: rule_utils.plot_reward_versus_features( experience_features_rewards_path, iteration_config_rewards, plot_name_suffix=str(datetime.now())[:19]) main_loop_seed += 1
class SkOptOptimizer(PhotonBaseOptimizer): def __init__( self, n_configurations: int = 20, acq_func: str = "gp_hedge", acq_func_kwargs: dict = None, ): self.optimizer = None self.hyperparameter_list = [] self.metric_to_optimize = "" self.ask = self.ask_generator() self.n_configurations = n_configurations self.acq_func = acq_func self.acq_func_kwargs = acq_func_kwargs self.maximize_metric = True self.constant_dictionary = {} def prepare(self, pipeline_elements: list, maximize_metric: bool): self.hyperparameter_list = [] self.maximize_metric = maximize_metric # build space space = [] for pipe_element in pipeline_elements: if hasattr(pipe_element, "hyperparameters"): for name, value in pipe_element.hyperparameters.items(): # if we only have one value we do not need to optimize if isinstance(value, list) and len(value) < 2: self.constant_dictionary[name] = value[0] continue if isinstance(value, PhotonCategorical) and len(value.values) < 2: self.constant_dictionary[name] = value.values[0] continue skopt_param = self._convert_PHOTON_to_skopt_space( value, name) if skopt_param is not None: space.append(skopt_param) if len(space) == 0: logger.warn( "Did not find any hyperparameters to convert into skopt space") self.optimizer = None else: self.optimizer = Optimizer( space, "ET", acq_func=self.acq_func, acq_func_kwargs=self.acq_func_kwargs, ) self.ask = self.ask_generator() def _convert_PHOTON_to_skopt_space(self, hyperparam: object, name: str): if not hyperparam: return None self.hyperparameter_list.append(name) if isinstance(hyperparam, PhotonCategorical): return skoptCategorical(hyperparam.values, name=name) elif isinstance(hyperparam, list): return skoptCategorical(hyperparam, name=name) elif isinstance(hyperparam, FloatRange): if hyperparam.range_type == "linspace": return Real(hyperparam.start, hyperparam.stop, name=name, prior="uniform") elif hyperparam.range_type == "logspace": return Real(hyperparam.start, hyperparam.stop, name=name, prior="log-uniform") else: return Real(hyperparam.start, hyperparam.stop, name=name) elif isinstance(hyperparam, IntegerRange): return Integer(hyperparam.start, hyperparam.stop, name=name) def ask_generator(self): if self.optimizer is None: yield {} else: for i in range(self.n_configurations): next_config_list = self.optimizer.ask() next_config_dict = { self.hyperparameter_list[number]: self._convert_to_native(value) for number, value in enumerate(next_config_list) } yield next_config_dict def _convert_to_native(self, obj): # check if we have a numpy object, if so convert it to python native if type(obj).__module__ == np.__name__: return np.asscalar(obj) else: return obj def tell(self, config, performance): # convert dictionary to list in correct order if self.optimizer is not None: config_values = [config[name] for name in self.hyperparameter_list] best_config_metric_performance = performance[1] if self.maximize_metric: if isinstance(best_config_metric_performance, list): print("BEST CONFIG METRIC PERFORMANCE: " + str(best_config_metric_performance)) best_config_metric_performance = best_config_metric_performance[ 0] best_config_metric_performance = -best_config_metric_performance # random_accuracy = np.random.randn(1)[0] self.optimizer.tell(config_values, best_config_metric_performance) def plot_evaluations(self): results = SkoptResults() results.space = self.optimizer.space results.x_iters = self.optimizer.Xi results = self._convert_categorical_hyperparameters(results) results.x = results.x_iters[np.argmin(self.optimizer.yi)] plt.figure(figsize=(10, 10)) return plot_evaluations(results) def plot_objective(self): results = SkoptResults() results.space = self.optimizer.space results.x_iters = self.optimizer.Xi results = self._convert_categorical_hyperparameters(results) results.x = results.x_iters[np.argmin(self.optimizer.yi)] results.models = self.optimizer.models plt.figure(figsize=(10, 10)) return plot_objective(results) def _convert_categorical_hyperparameters(self, results): parameter_types = list() for i, dim in enumerate(results.space.dimensions): if isinstance(dim, skoptCategorical): parameter_types.append(dim.transformer) setattr(results.space.dimensions[i], "categories", dim.transformed_bounds) else: parameter_types.append(False) for i, xs in enumerate(results.x_iters): for k, xsk in enumerate(xs): if parameter_types[k]: results.x_iters[i][k] = parameter_types[k].transform([xsk]) return results
func=use_func, dimensions=dim, n_calls = run_for, ) print "GPM best value",res.fun,"at",res.x #print res print "took",time.mktime(time.gmtime())-start,"[s]" o = Optimizer( n_initial_points =5, acq_func = 'gp_hedge', acq_optimizer='auto', base_estimator=GaussianProcessRegressor(alpha=0.0, copy_X_train=True, n_restarts_optimizer=2, noise='gaussian', normalize_y=True, optimizer='fmin_l_bfgs_b'), dimensions=dim, ) m = manager(n = 4, skobj = o, iterations = run_for, func = use_func, wait= 0 ) start = time.mktime(time.gmtime()) m.run() import numpy as np best = np.argmin( m.sk.yi)
def init_optimization(optim_config_file, ctx): with optim_config_file.open('r') as f: optim_config = yaml.load(f, Loader=yaml.SafeLoader) # default settings if "objective" not in optim_config: raise ValueError( 'ERROR: the configuration must provide an `objective`.') if "evaluations" not in optim_config: raise ValueError( 'ERROR: the configuration must project a `evaluations` budget.') optim_config = { "solver": {}, "search_space": {}, "preset_params": {}, **optim_config, } optim_config['solver'] = { "name": "scikit-optimize", "random_state": 42, **optim_config.get('solver', {}), } from skopt.utils import Space space = Space.from_yaml(optim_config_file, namespace='search_space') preset_params = optim_config.get('preset_params', {}) click.secho("Search space:", fg="blue", err=True) click.secho(str(space), fg="blue", dim=True, err=True) click.secho("Preset parameters:", fg="blue", err=True) click.secho(str(preset_params), fg="blue", dim=True, err=True) # we use the iteration step in the objective function, to store results at the right place from skopt.utils import Integer dim_iteration = Integer(name='iteration', low=0, high=2 ^ 16) dims = [*space, dim_iteration] from skopt.utils import use_named_args @use_named_args(dims) def objective(**opt_params): params = {**preset_params, **opt_params} # From the UI we will want to see the iteration as a metric del params["iteration"] batch_label = f"{ctx.obj['batch_label']}|iter{opt_params['iteration']+1}" command = ' '.join([ 'qa', f"--label '{batch_label}'", f"--platform '{ctx.obj['platform']}'", f"--configuration '{ctx.obj['configuration']}'", f"--tuning '{json.dumps(params, sort_keys=True, cls=NumpyEncoder)}'", 'batch', ' '.join( [f'--batches-file "{b}"' for b in ctx.obj["batches_files"]]), ' '.join([f'--batch "{b}"' for b in ctx.obj["batches"]]), # we notably forward --batch ' '.join(ctx.obj["forwarded_args"]), ]) click.secho(command, fg="blue") import re command = re.sub( '^qa', 'python -m qaboard', command) # helps make sure we run the right thing when testing if str(subproject) != '.': command = f"cd {subproject} && {command}" if not ctx.obj['dryrun']: p = subprocess.run( command, shell=True, encoding="utf-8", ) if p.returncode != 0: click.secho( f'[ERROR ({p.returncode})] Check the logs in QA-Board to know what output failed', fg='red', bold=True) # Now that we finished computing all the results, we will download the results and # compute the objective function: return batch_objective(batch_label, optim_config['objective']) # For the full list of options, refer to: # https://scikit-optimize.github.io/stable/modules/generated/skopt.optimizer.Optimizer.html#skopt.optimizer.Optimizer from skopt import Optimizer del optim_config['solver']['name'] optimizer = Optimizer(space, **optim_config['solver']) # in the optimization loop, `ask` gives us an array of values # this wrapper converts it back to the actual named parameters @use_named_args([*space]) def dim_mapping(**opt_params): return {**preset_params, **opt_params} return objective, optimizer, optim_config, dim_mapping
def run(): start_time = time.time() print("run() start: {}".format(str(datetime.datetime.now()))) comm = MPI.COMM_WORLD # get MPI communicator object size = comm.size # total number of processes rank = comm.rank # rank of this process status = MPI.Status() # get MPI status object print("ME rank is {}".format(rank)) instance = problem.Problem() spaceDict = instance.space params = instance.params global problem_params problem_params = params starting_point = instance.starting_point # handshake to ensure working eqpy.OUT_put("Params") # initial parameter set telling us the number of times to run the loop initparams = eqpy.IN_get() (init_size, max_evals, num_workers, num_buffer, seed, max_threshold, n_jobs) = eval('{}'.format(initparams)) space = [spaceDict[key] for key in params] print(space) parDict = {} resultsList = [] parDict['kappa'] = 1.96 # can set to num cores parDict['n_jobs'] = n_jobs init_x = [] opt = Optimizer(space, base_estimator='RF', acq_optimizer='sampling', acq_func='LCB', acq_func_kwargs=parDict, random_state=seed) eval_counter = 0 askedDict = {} print( "Master starting with {} init_size, {} max_evals, {} num_workers, {} num_buffer, {} max_threshold" .format(init_size, max_evals, num_workers, num_buffer, max_threshold)) x = opt.ask(n_points=init_size) res, resstring = create_list_of_json_strings(x) print("Initial design is {}".format(resstring)) for r, xx in zip(res, x): askedDict[r] = xx eqpy.OUT_put(resstring) currently_out = init_size total_out = init_size results = [] group = comm.Get_group() # Assumes only one adlb_server # num_workers + 1 = num_turbine_workers newgroup = group.Excl([num_workers + 1]) #print("ME newgroup size is {}".format(newgroup.size)) newcomm = comm.Create_group(newgroup, 1) nrank = newcomm.rank #print("ME nrank is {}".format(nrank)) counter_threshold = 1 counter = 0 end_iter_time = 0 while eval_counter < max_evals: start_iter_time = time.time() print("\neval_counter = {}".format(eval_counter)) data = newcomm.recv(source=MPI.ANY_SOURCE, status=status) counter = counter + 1 xstring = data['x'] x = askedDict[xstring] y = data['cost'] if math.isnan(y): y = sys.float_info.max opt.tell(x, y) #source = status.Get_source() #tag = status.Get_tag() elapsed_time = float(time.time() - start_time) print('elapsed_time:%1.3f' % elapsed_time) results.append(str(data)) eval_counter = eval_counter + 1 currently_out = currently_out - 1 # if jobs are finishing within 16 seconds of # each other, then batch the point production if start_iter_time - end_iter_time < 16: counter_threshold = max_threshold if max_evals - eval_counter < counter_threshold: counter_threshold = max_evals - eval_counter if counter_threshold > currently_out: counter_threshold = currently_out else: counter_threshold = 1 print("counter_threshold: {}".format(counter_threshold)) print("currently_out:{}, total_out:{}".format(currently_out, total_out)) if currently_out < num_workers + num_buffer and total_out < max_evals and counter >= counter_threshold: n_points = counter if n_points + total_out > max_evals: n_points = max_evals - total_out ts = time.time() x = opt.ask(n_points=n_points) res, resstring = create_list_of_json_strings(x) for r, xx in zip(res, x): askedDict[r] = xx eqpy.OUT_put(resstring) print('point production elapsed_time:%1.3f' % float(time.time() - ts)) currently_out = currently_out + n_points total_out = total_out + n_points counter = 0 end_iter_time = start_iter_time print('Search finishing') eqpy.OUT_put("DONE") eqpy.OUT_put(";".join(results))
try: with open('my_optimizer_old.pkl', 'rb') as f: opt = pickle.load(f) except: # opt = Optimizer([(-2.0, 2.0)], "GP", acq_func = 'LCB', acq_func_kwargs = ) #### high k means explore, low kappa means exploit # bounds = [(0., 1.),] * 8 opt = Optimizer( base_estimator='GP', acq_func='LCB', acq_optimizer='auto', dimensions=[ (-5.0, 5.0), # range for param 1 (eg trajectory final height?) (-5.0, 5.0), # range for param 2 (eg trajectory final pitch?) (-5.0, 5.0) ], # range for param 4 (eg vibrational state duration?) acq_func_kwargs={ 'kappa': 10 }, # we should prefer explore (high kappa). howver, with higher dim it will naturally tend to diversify, so kappa could be decreased n_initial_points=10) """ kappa [float, default=1.96]: Controls how much of the variance in the predicted values should be taken into account. Used when the acquisition is "LCB" (lower confidence bound). If set high, then we are favouring exploration over exploitation and vice versa. xi [float, default=0.01]: Controls how much improvement one wants over the previous best values. Used when the acquisition is either "EI" or "PI". to use this, i think i need a way to scale xi by the variance of the signal, which howver will depend on the participant and will have to be adapted online... """
help="Specify the out csv filename.", required=True) args = parser.parse_args() # Defining the hyperspace hyperparameters = [ (0.00001, 0.1), # learning_rate (0.2, 0.9), # dropout (10, 100), # epochs (10, 1000) ] # batch size space = create_hyperspace(hyperparameters) # Perform runs and aggregate results results = [] for section in tqdm(space): # create a skopt gp minimize object optimizer = Optimizer(section) search_algo = SkOptSearch( optimizer, ['learning_rate', 'dropout', 'epochs', 'batch_size'], metric='average_res', mode='max') analysis = tune.run(multi_train, search_alg=search_algo, num_samples=50, resources_per_trial={'gpu': 1}) results.append(analysis) all_pt_results = results[0].results_df for i in range(1, len(results)): all_pt_results = all_pt_results.append(results[i].results_df) all_pt_results.to_csv(args.out)
def fit(self, X, Y, total_duration=6e7, n_iter=100, cv_iter=None, optimizer=None, acq_func='gp_hedge', **kwargs): start = datetime.now() def splitter(itr): for train_idx, test_idx in itr: yield X[train_idx], Y[train_idx], X[test_idx], Y[test_idx] def splitter_dict(itr_dict): n_splits = len(list(itr_dict.values())[0]) for i in range(n_splits): X_train = dict() Y_train = dict() X_test = dict() Y_test = dict() for n_obj, itr in itr_dict.items(): train_idx = itr[i][0] test_idx = itr[i][1] X_train[n_obj] = np.copy(X[n_obj][train_idx]) X_test[n_obj] = np.copy(X[n_obj][test_idx]) Y_train[n_obj] = np.copy(Y[n_obj][train_idx]) Y_test[n_obj] = np.copy(Y[n_obj][test_idx]) yield X_train, Y_train, X_test, Y_test if cv_iter is None: cv_iter = ShuffleSplit(n_splits=3, test_size=0.1, random_state=self.random_state) if isinstance(X, dict): splits = dict() for n_obj, arr in X.items(): if arr.shape[0] == 1: splits[n_obj] = [([0], [0]) for i in range(cv_iter.n_splits)] else: splits[n_obj] = list(cv_iter.split(arr)) else: splits = list(cv_iter.split(X)) # Pre-compute splits for reuse # Here we fix a random seed for all simulations to correlate the random # streams: seed = self.random_state.randint(2**32, dtype='uint32') self.logger.debug( 'Random seed for the ranking algorithm: {}'.format(seed)) opt_seed = self.random_state.randint(2**32, dtype='uint32') self.logger.debug('Random seed for the optimizer: {}'.format(opt_seed)) gp_seed = self.random_state.randint(2**32, dtype='uint32') self.logger.debug( 'Random seed for the GP surrogate: {}'.format(gp_seed)) if optimizer is not None: opt = optimizer self.logger.debug('Setting the provided optimizer') self.log_best_params(opt) else: transformed = [] for param in self.parameter_ranges: transformed.append(check_dimension(param)) self.logger.info("Parameter Space: {}".format(transformed)) space = normalize_dimensions(transformed) self.logger.info( "Parameter Space after transformation: {}".format(space)) # Todo: Make this passable base_estimator = cook_estimator("GP", space=space, random_state=gp_seed, noise="gaussian") opt = Optimizer(dimensions=self.parameter_ranges, random_state=opt_seed, base_estimator=base_estimator, acq_func=acq_func, **kwargs) self._callbacks_set_optimizer(opt) self._callbacks_on_optimization_begin() time_taken = duration_tillnow(start) total_duration -= time_taken max_fit_duration = -10000 self.logger.info('Time left for {} iterations is {}'.format( n_iter, microsec_to_time(total_duration))) try: for t in range(n_iter): start = datetime.now() self._callbacks_on_iteration_begin(t) self.logger.info( 'Starting optimization iteration: {}'.format(t)) if t > 0: self.log_best_params(opt) next_point = opt.ask() self.logger.info('Next parameters:\n{}'.format(next_point)) results = [] running_times = [] if isinstance(X, dict): for X_train, Y_train, X_test, Y_test in splitter_dict( splits): result, time_taken = self._fit_ranker( X_train, Y_train, X_test, Y_test, next_point) running_times.append(time_taken) results.append(result) else: for X_train, Y_train, X_test, Y_test in splitter(splits): result, time_taken = self._fit_ranker( X_train, Y_train, X_test, Y_test, next_point) running_times.append(time_taken) results.append(result) results = np.array(results) running_times = np.array(running_times) mean_result = np.mean(results) mean_fitting_duration = np.mean(running_times) # Storing the maximum time to run the splitting model and adding the time for out of sample evaluation if max_fit_duration < np.sum(running_times): max_fit_duration = np.sum(running_times) self.logger.info( 'Validation error for the parameters is {:.4f}'.format( mean_result)) self.logger.info('Time taken for the parameters is {}'.format( microsec_to_time(np.sum(running_times)))) if "ps" in opt.acq_func: opt.tell(next_point, [mean_result, mean_fitting_duration]) else: opt.tell(next_point, mean_result) self._callbacks_on_iteration_end(t) self.logger.info( "Main optimizer iterations done {} and saving the model". format(np.array(opt.yi).shape[0])) dump(opt, self.optimizer_path) time_taken = duration_tillnow(start) total_duration -= time_taken self.logger.info('Time left for simulations is {} '.format( microsec_to_time(total_duration))) if (total_duration - max_fit_duration) < 0: self.logger.info( 'At iteration {} maximum time required by model to validate a parameter values' .format(microsec_to_time(max_fit_duration))) self.logger.info( 'At iteration {} simulation stops, due to time deficiency' .format(t)) break except KeyboardInterrupt: self.logger.debug( 'Optimizer interrupted saving the model at {}'.format( self.optimizer_path)) self.log_best_params(opt) else: self.logger.debug( 'Finally, fit a model on the complete training set and storing the model at {}' .format(self.optimizer_path)) self._fit_params["epochs"] = self._fit_params.get("epochs", 1000) if "ps" in opt.acq_func: best_point = opt.Xi[np.argmin(np.array(opt.yi)[:, 0])] else: best_point = opt.Xi[np.argmin(opt.yi)] self._set_new_parameters(best_point) self.model = copy.copy(self.ranker) self.model.fit(X, Y, **self._fit_params) finally: self._callbacks_on_optimization_end() self.optimizer = opt if np.array(opt.yi).shape[0] != 0: dump(opt, self.optimizer_path)
def test_n_random_starts_Optimizer(): # n_random_starts got renamed in v0.4 et = ExtraTreesRegressor(random_state=2) with pytest.deprecated_call(): Optimizer([(0, 1.)], et, n_random_starts=10, acq_optimizer='sampling')
class Optimizer: SEED = 12345 def __init__( self, problem, num_workers, surrogate_model="RF", acq_func="gp_hedge", acq_kappa=1.96, liar_strategy="cl_max", n_jobs=1, **kwargs, ): assert surrogate_model in [ "RF", "ET", "GBRT", "GP", "DUMMY", ], f"Unknown scikit-optimize base_estimator: {surrogate_model}" if surrogate_model == "RF": base_estimator = RandomForestRegressor(n_jobs=n_jobs) elif surrogate_model == "ET": base_estimator = ExtraTreesRegressor(n_jobs=n_jobs) elif surrogate_model == "GBRT": base_estimator = GradientBoostingQuantileRegressor(n_jobs=n_jobs) else: base_estimator = surrogate_model self.space = problem.space # queue of remaining starting points self.starting_points = problem.starting_point n_init = (inf if surrogate_model == "DUMMY" else max( num_workers, len(self.starting_points))) self._optimizer = SkOptimizer( dimensions=self.space, base_estimator=base_estimator, acq_optimizer="sampling", acq_func=acq_func, acq_func_kwargs={"kappa": acq_kappa}, random_state=self.SEED, n_initial_points=n_init, ) assert liar_strategy in "cl_min cl_mean cl_max".split() self.strategy = liar_strategy self.evals = {} self.counter = 0 logger.info( f"Using skopt.Optimizer with {surrogate_model} base_estimator") def _get_lie(self): if self.strategy == "cl_min": return min(self._optimizer.yi) if self._optimizer.yi else 0.0 elif self.strategy == "cl_mean": return np.mean(self._optimizer.yi) if self._optimizer.yi else 0.0 else: return max(self._optimizer.yi) if self._optimizer.yi else 0.0 def _xy_from_dict(self): XX = [] for key in self.evals.keys(): x = tuple(convert2np(k) for k in key) XX.append(x) YY = [-self.evals[x] for x in self.evals.keys()] # ! "-" maximizing return XX, YY def dict_to_xy(self, xy_dict: dict): XX = [] for key in xy_dict.keys(): x = [convert2np(k) for k in key] XX.append(x) YY = [-xy_dict[x] for x in xy_dict.keys()] # ! "-" maximizing return XX, YY def to_dict(self, x: list) -> dict: res = {} hps_names = self.space.get_hyperparameter_names() for i in range(len(x)): res[hps_names[i]] = "nan" if isnan(x[i]) else x[i] return res def _ask(self): if len(self.starting_points) > 0: x = self.starting_points.pop() else: x = self._optimizer.ask() y = self._get_lie() key = tuple(self.to_dict(x).values()) if key not in self.evals: self.counter += 1 self._optimizer.tell(x, y) self.evals[key] = y logger.debug(f"_ask: {x} lie: {y}") else: logger.debug(f"Duplicate _ask: {x} lie: {y}") return self.to_dict(x) def ask(self, n_points=None, batch_size=20): if n_points is None: return self._ask() else: batch = [] for _ in range(n_points): batch.append(self._ask()) if len(batch) == batch_size: yield batch batch = [] if batch: yield batch def ask_initial(self, n_points): if len(self.starting_points) > 0: XX = [ self.starting_points.pop() for i in range(min(n_points, len(self.starting_points))) ] if len(XX) < n_points: XX += self._optimizer.ask(n_points=n_points - len(XX)) else: XX = self._optimizer.ask(n_points=n_points) for x in XX: y = self._get_lie() x = [convert2np(xi) for xi in x] key = tuple(self.to_dict(x).values()) if key not in self.evals: self.counter += 1 self._optimizer.tell(x, y) self.evals[key] = y return [self.to_dict(x) for x in XX] def tell(self, xy_data): assert isinstance(xy_data, list), f"where type(xy_data)=={type(xy_data)}" minval = min(self._optimizer.yi) if self._optimizer.yi else 0.0 xy_dict = {} for x, y in xy_data: key = tuple(x[k] for k in self.space) assert key in self.evals, f"where key=={key} and self.evals=={self.evals}" logger.debug(f"tell: {x} --> {key}: evaluated objective: {y}") self.evals[key] = y if y > np.finfo(np.float32).min else minval xy_dict[key] = y if y > np.finfo(np.float32).min else minval XX, YY = self.dict_to_xy(xy_dict) selection = [ (xi, yi) for xi, yi in zip(self._optimizer.Xi, self._optimizer.yi) if not (any([equal_list(xi, x) for x in XX])) # all([diff(xi, x) for x in XX]) ] new_Xi, new_yi = list(zip(*selection)) if len(selection) > 0 else ([], []) new_Xi, new_yi = list(new_Xi), list(new_yi) self._optimizer.Xi = new_Xi self._optimizer.yi = new_yi self._optimizer.tell(XX, YY) assert len(self._optimizer.Xi) == len( self._optimizer.yi) == self.counter, ( f"where len(self._optimizer.Xi)=={len(self._optimizer.Xi)}, " f"len(self._optimizer.yi)=={len(self._optimizer.yi)}," f"self.counter=={self.counter}")
class SkOptOptimizer(PhotonSlaveOptimizer): def __init__(self, n_configurations: int = 20, acq_func: str = 'gp_hedge', acq_func_kwargs: dict = None): self.optimizer = None self.hyperparameter_list = [] self.metric_to_optimize = '' self.ask = self.ask_generator() self.n_configurations = n_configurations self.acq_func = acq_func self.acq_func_kwargs = acq_func_kwargs self.maximize_metric = True self.constant_dictionary = {} def prepare(self, pipeline_elements: list, maximize_metric: bool): self.hyperparameter_list = [] self.maximize_metric = maximize_metric # build space space = [] for pipe_element in pipeline_elements: if pipe_element.__class__.__name__ == 'Switch': error_msg = 'Scikit-Optimize cannot operate in the specified hyperparameter space with a Switch ' \ 'element. We recommend the use of SMAC.' logger.error(error_msg) raise ValueError(error_msg) if hasattr(pipe_element, 'hyperparameters'): for name, value in pipe_element.hyperparameters.items(): # if we only have one value we do not need to optimize if isinstance(value, list) and len(value) < 2: self.constant_dictionary[name] = value[0] continue if isinstance(value, PhotonCategorical) and len(value.values) < 2: self.constant_dictionary[name] = value.values[0] continue skopt_param = self._convert_PHOTON_to_skopt_space( value, name) if skopt_param is not None: space.append(skopt_param) if len(space) == 0: logger.warning( "Did not find any hyperparameters to convert into skopt space") self.optimizer = None else: self.optimizer = Optimizer(space, "ET", acq_func=self.acq_func, acq_func_kwargs=self.acq_func_kwargs) self.ask = self.ask_generator() def _convert_PHOTON_to_skopt_space(self, hyperparam: object, name: str): if not hyperparam: return None self.hyperparameter_list.append(name) if isinstance(hyperparam, PhotonCategorical): return skoptCategorical(hyperparam.values, name=name) elif isinstance(hyperparam, list): return skoptCategorical(hyperparam, name=name) elif isinstance(hyperparam, FloatRange): if hyperparam.range_type == 'linspace': return Real(hyperparam.start, hyperparam.stop, name=name, prior='uniform') elif hyperparam.range_type == 'logspace': return Real(hyperparam.start, hyperparam.stop, name=name, prior='log-uniform') else: return Real(hyperparam.start, hyperparam.stop, name=name) elif isinstance(hyperparam, IntegerRange): return Integer(hyperparam.start, hyperparam.stop, name=name) def ask_generator(self): if self.optimizer is None: yield {} else: for i in range(self.n_configurations): next_config_list = self.optimizer.ask() next_config_dict = { self.hyperparameter_list[number]: self._convert_to_native(value) for number, value in enumerate(next_config_list) } yield next_config_dict def _convert_to_native(self, obj): # check if we have a numpy object, if so convert it to python native if type(obj).__module__ == np.__name__: return obj.item() else: return obj def tell(self, config, performance): # convert dictionary to list in correct order if self.optimizer is not None: config_values = [config[name] for name in self.hyperparameter_list] best_config_metric_performance = performance[1] if self.maximize_metric: best_config_metric_performance = -best_config_metric_performance # random_accuracy = np.random.randn(1)[0] self.optimizer.tell(config_values, best_config_metric_performance)