class HyperoptBackend(StandardBackend): """The hyperopt backend uses hyperopt for black box optimization.""" backend_name = "hyperopt" implemented_funcs = ("choice", "randrange", "uniform", "normalvariate") @override def setup_backend(self, params, algo=tpe.suggest, rstate=np.random.RandomState(), show_progressbar=False, **options): """Special method to initialize the backend from params.""" self.params = params space = (as_apply)(dict( ((name), (create_space(name, func, *args))) for name, (func, args, kwargs) in sorted_items(params))) domain = Domain(self.set_current_values, space) self.trials = Trials() self.fmin_iter = FMinIter(algo, domain, self.trials, rstate, show_progressbar=show_progressbar, **options) @override def tell_examples(self, new_examples): """Special method that allows fast updating of the backend with new examples.""" trial_list = examples_to_trials(new_examples, self.params) self.trials.insert_trial_docs(trial_list) self.trials.refresh() # run one iteration of hyperparameter optimization, with values saved # to the self.set_current_values callback passed to Domain next(self.fmin_iter) assert self.current_values is not None, self.current_values assert set(self.current_values.keys()) == set( self.params), self.current_values def set_current_values(self, values): """Callback to set the values for this run.""" assert isinstance(values, dict), values self.current_values = values return {"status": STATUS_RUNNING}
def nu_simple_fmin(hpo_project_key, objective, rseed=1337, full_model_string=None, notebook_name=None, verbose=True, stack=3, keep_temp=False, data_args=None): # db에서 가져오기 db_info = asyncio.run(Requests().get_action(parameter1 = hpo_project_key, parameter2 = "null", url = hpo_url))[0] hpo_project_id = db_info["hpoProjectId"] algo, space = __transform_db_to_function(method = db_info["method"], config = db_info["config"]) trials = Trials() best = fmin(objective, space, algo=algo, max_evals=50, trials=trials, rstate=np.random.RandomState(rseed), return_argmin=True) importances = calculate_importance(trials) # 저장 api all_info = dict() all_info["best_result"] = trials.best_trial['result'] all_info["best_hp"] = best all_info["trial_result"] = trials.results all_info["trial_hp"] = trials.vals # json int64 때문에 작업 all_info = __to_int(all_info) all_info["hpo_project_key"] = hpo_project_key tmp_importance = list() for i in range(len(importances)): for key1, value1 in importances[i].items(): for key2, value2 in value1.items(): tmp_importance.append(value2) break all_info["importances"] = tmp_importance asyncio.run(Requests().post_action(request_datas = all_info, url = hpo_url)) return best, trials
def setup_backend(self, params, algo=tpe.suggest, rstate=None, show_progressbar=False, **options): """Special method to initialize the backend from params.""" if rstate is None: try: rstate = np.random.default_rng() except AttributeError: rstate = np.random.RandomState() self.params = params space = (as_apply)(dict(((name), (create_space(name, func, *args))) for name, (func, args, kwargs) in sorted_items(params))) domain = Domain(self.set_current_values, space) self.trials = Trials() self.fmin_iter = FMinIter(algo, domain, self.trials, rstate, show_progressbar=show_progressbar, **options)
def foo(self): self.bandit = bandit = Bandit(self.expr) self.algo = algo = Random(bandit) if hasattr(self, 'n_randints'): n_randints = len(filter( lambda x: x.name == 'randint', algo.vh.params.values())) assert n_randints == self.n_randints self.trials = Trials() self.experiment = Experiment(self.trials, algo, async=False) self.experiment.run(5) self.output = output = [] for trial in self.trials._trials: print '' tmp = [] for nid in trial['misc']['idxs']: thing = ( nid, trial['misc']['idxs'][nid], trial['misc']['vals'][nid]) print thing tmp.append(thing) tmp.sort() output.append(tmp) print repr(output) print repr(self.wanted) # -- think of a more robust way to test these things # or, if the sampling style is to be nailed down, # put it in and be sure of it. raise nose.SkipTest() assert output == self.wanted
def main(): search_space = {} search_space["problem_size"] = hp.quniform("problem_size", 1, 64, 1) search_space["num_ranks"] = hp.quniform("num_ranks", 1, 8, 1) trials = Trials() best = fmin(fn=run_hpcg, space=search_space, algo=tpe.suggest, max_evals = 50, trials=trials) print("------------------------------------------------") for trial in trials: print("{1}x({2}, {3}, {4}) = {0} GF".format(-1.0*trial["result"]["loss"], trial["misc"]["vals"]["num_ranks"][0], trial["misc"]["vals"]["problem_size"][0], trial["misc"]["vals"]["problem_size"][0], trial["misc"]["vals"]["problem_size"][0])) print("Saving pkl....") with open('trials.pkl', 'wb') as output: pickle.dump(trials, output) print("... done")
def __init__(self, api_config): """Build wrapper class to use an optimizer in benchmark. Parameters ---------- api_config : dict-like of dict-like Configuration of the optimization variables. See API description. """ AbstractOptimizer.__init__(self, api_config) self.space_x = JointSpace(api_config) self.bounds = self.space_x.get_bounds() self.create_opt_prob( ) # Sets up the optimization problem (needs self.bounds) self.max_evals = np.iinfo(np.int32).max # NOTE: Largest possible int self.turbo_batch_size = None self.pysot_batch_size = None self.history = [] self.proposals = [] self.lb, self.ub = self.bounds[:, 0], self.bounds[:, 1] self.dim = len(self.bounds) self.turbo = Turbo1( f=None, lb=self.bounds[:, 0], ub=self.bounds[:, 1], n_init=2 * self.dim + 1, max_evals=self.max_evals, batch_size=4, # We need to update this later verbose=False, ) # hyperopt self.random = np_random space, self.round_to_values = tuSOTOptimizer.get_hyperopt_dimensions( api_config) self.domain = Domain(dummy_f, space, pass_expr_memo_ctrl=None) self.trials = Trials() # Some book keeping like opentuner wrapper self.trial_id_lookup = {} # Store just for data validation self.param_set_chk = frozenset(api_config.keys())
def __init__(self, est, X, y, params=None, iters=500, time_to_search=None, cv=5, cv_times=1, scorer="f1", verbose=1, random=False, foldtype="Kfold"): # check and get skiperopt style parameters if params == None and not hasattr(est, "param_grid"): raise ValueError("No parameters supplied") else: params = est.param_grid self.params = params self.best_params = None self.__space = create_hyper(params) self.__initparams = est.get_params() # set hyper settings self.__algo = tpe.suggest self.__trial = Trials() # set run settings self.verbose = verbose self.iters = iters self.cv = cv self.cv_times = cv_times self.scorer = scorer self.__run = 0 self.est = est self.stats = {} self.__init_score = cross_validation(self.est, X, y, cv=self.cv, cv_times=self.cv_times, scorer=self.scorer) self.best_score = self.__init_score self.__X = X self.__y = y self.__start_now = None self.__runok = True self.time_to_search = time_to_search self.random = random self.foldtype = foldtype
def __init__(self, api_config, random=np_random): """Build wrapper class to use hyperopt optimizer in benchmark. Parameters ---------- api_config : dict-like of dict-like Configuration of the optimization variables. See API description. """ AbstractOptimizer.__init__(self, api_config) self.random = random space, self.round_to_values = HyperoptOptimizer.get_hyperopt_dimensions(api_config) self.domain = Domain(dummy_f, space, pass_expr_memo_ctrl=None) self.trials = Trials() # Some book keeping like opentuner wrapper self.trial_id_lookup = {} # Store just for data validation self.param_set_chk = frozenset(api_config.keys())
def test_seeding(self): # -- assert that the seeding works a particular way domain = coin_flip() docs = rand.suggest(list(range(10)), domain, Trials(), seed=123) trials = trials_from_docs(docs) idxs, vals = miscs_to_idxs_vals(trials.miscs) # Passes Nov 8 / 2013 self.assertEqual(list(idxs["flip"]), list(range(10))) self.assertEqual(list(vals["flip"]), [0, 1, 0, 0, 0, 0, 0, 1, 1, 0])
def test_suggest_1(self): print 'EXPR', self.bandit.expr docs = self.algo.suggest([0], Trials()) assert len(docs) == 1 print 'DOCS', docs # -- assert validity of docs trials = trials_from_docs(docs) print 'TRIALS', trials assert docs[0]['misc']['idxs']['flip'] == [0] idxs, vals = miscs_to_idxs_vals(trials.miscs) assert idxs['flip'] == [0]
def test_arbitrary_range(self): new_ids = [-2, 0, 7, 'a', '007'] docs = self.algo.suggest(new_ids, Trials()) # -- assert validity of docs trials = trials_from_docs(docs) idxs, vals = miscs_to_idxs_vals(trials.miscs) assert len(docs) == 5 assert len(idxs) == 1 assert len(vals) == 1 print vals assert idxs['flip'] == new_ids assert np.all(vals['flip'] == [0, 1, 0, 1, 1])
def test_failure(): #XXX also test the Bandit.exceptions mechanism that actually catches them class BanditE(Exception): pass class DummyBandit(Bandit): param_gen = {"loss": 10} def __init__(self): super(DummyBandit, self).__init__(self.param_gen) def evaluate(self, config, ctrl): raise BanditE() trials = Trials() bandit_algo = Random(DummyBandit()) exp = Experiment(trials, bandit_algo, async=False) exp.run(1) trials.refresh() assert len(trials) == 0 assert len(trials._dynamic_trials) == 1 assert trials._dynamic_trials[0]['state'] == JOB_STATE_ERROR assert trials._dynamic_trials[0]['misc']['error'] != None exp.catch_bandit_exceptions = False nose.tools.assert_raises(BanditE, exp.run, 1) trials.refresh() assert len(trials) == 0 assert len(trials._dynamic_trials) == 2 assert trials._dynamic_trials[1]['state'] == JOB_STATE_ERROR assert trials._dynamic_trials[1]['misc']['error'] != None
def test_suggest_5(self): docs = self.algo.suggest(range(5), Trials()) print docs assert len(docs) == 5 # -- assert validity of docs trials = trials_from_docs(docs) idxs, vals = miscs_to_idxs_vals(trials.miscs) print idxs print vals assert len(idxs) == 1 assert len(vals) == 1 assert idxs['flip'] == range(5) assert np.all(vals['flip'] == [1, 1, 0, 1, 0])
def __init__(self, examples, params, algo=tpe.suggest, rstate=np.random.RandomState(), show_progressbar=False, **options): self.init_fallback_backend() if not examples: self.current_values = {} return space = (as_apply)(dict(((name), (create_space(name, func, *args))) for name, (func, args, kwargs) in sorted_items(params))) domain = Domain(self.set_current_values, space) trial_list = examples_to_trials(examples, params) trials = Trials() trials.insert_trial_docs(trial_list) # run one iteration of hyperparameter optimization, with values saved # to the self.set_current_values callback passed to Domain (next)(FMinIter(algo, domain, trials, rstate, show_progressbar=show_progressbar, **options)) assert self.current_values is not None, self.current_values assert set(self.current_values.keys()) == set(params), self.current_values
def main(): space = { 'ltr': hp.choice('ltr', [True]), 'shuffle': hp.choice('shuffle', [False]), 'num_leaves': hp.choice('num_leaves', list(np.arange(8, 256, 2, dtype=int))), 'max_depth': hp.choice('max_depth', list(np.arange(4, 64, 2, dtype=int))), 'max_bin': hp.choice('max_bin', list(np.arange(255, 255 * 4, 5, dtype=int))), 'min_data_in_leaf': hp.choice('min_data_in_leaf', list(np.arange(5, 100, 5, dtype=int))), 'learning_rate': hp.uniform('learning_rate', 0.01, 0.3), 'bagging_fraction': hp.uniform('bagging_fraction', 0.2, 1.0), 'feature_fraction': hp.uniform('feature_fraction', 0.2, 1.0), 'early_stopping': hp.uniform('test_size', 100, 1000), } trials_step = 1 # how many additional trials to do after loading saved trials. 1 = save after iteration max_trials = 1 # initial max_trials. put something small to not have to wait try: # try to load an already saved trials object, and increase the max trials = pickle.load( open(BASE_PATH + SET + TRAILKEY + '.hyperopt', "rb")) print("Found saved Trials! Loading...") max_trials = len(trials.trials) + trials_step print("Rerunning from {} trials to {} (+{}) trials".format( len(trials.trials), max_trials, trials_step)) except: # create a new trials object and start searching trials = Trials() best = fmin(fn=objective, space=space, algo=tpe.suggest, trials=trials, max_evals=max_trials) print("Best:", best) print("Num:", max_trials) # save the trials object with open(BASE_PATH + SET + TRAILKEY + ".hyperopt", "wb") as f: pickle.dump(trials, f)
def test_arbitrary_range(self, N=10): assert N <= 10 new_ids = [-2, 0, 7, 'a', '007', 66, 'a3', '899', 23, 2333][:N] docs = self.algo.suggest(new_ids, Trials()) # -- assert validity of docs trials = trials_from_docs(docs) idxs, vals = miscs_to_idxs_vals(trials.miscs) assert len(docs) == N assert len(idxs) == 1 assert len(vals) == 1 print vals assert idxs['flip'] == new_ids # -- assert that the random seed matches that of Jan 8/2013 assert np.all(vals['flip'] == [0, 1, 0, 0, 0, 0, 0, 1, 1, 0][:N])
def test_suggest_N(self, N=10): assert N <= 10 docs = self.algo.suggest(range(N), Trials()) print 'docs', docs assert len(docs) == N # -- assert validity of docs trials = trials_from_docs(docs) idxs, vals = miscs_to_idxs_vals(trials.miscs) print 'idxs', idxs print 'vals', vals assert len(idxs) == 1 assert len(vals) == 1 assert idxs['flip'] == range(N) # -- only works when N == 5 assert np.all(vals['flip'] == [0, 1, 0, 0, 0, 0, 0, 1, 1, 0][:N])
class HyperoptOptimizer(AbstractOptimizer): primary_import = "hyperopt" def __init__(self, api_config, random=np_random): """Build wrapper class to use hyperopt optimizer in benchmark. Parameters ---------- api_config : dict-like of dict-like Configuration of the optimization variables. See API description. """ AbstractOptimizer.__init__(self, api_config) self.random = random space, self.round_to_values = HyperoptOptimizer.get_hyperopt_dimensions( api_config) self.domain = Domain(dummy_f, space, pass_expr_memo_ctrl=None) self.trials = Trials() # Some book keeping like opentuner wrapper self.trial_id_lookup = {} # Store just for data validation self.param_set_chk = frozenset(api_config.keys()) @staticmethod def hashable_dict(d): """A custom function for hashing dictionaries. Parameters ---------- d : dict or dict-like The dictionary to be converted to immutable/hashable type. Returns ------- hashable_object : frozenset of tuple pairs Bijective equivalent to dict that can be hashed. """ hashable_object = frozenset(d.items()) return hashable_object @staticmethod def get_hyperopt_dimensions(api_config): """Help routine to setup hyperopt search space in constructor. Take api_config as argument so this can be static. """ # The ordering of iteration prob makes no difference, but just to be # safe and consistnent with space.py, I will make sorted. param_list = sorted(api_config.keys()) space = {} round_to_values = {} for param_name in param_list: param_config = api_config[param_name] param_type = param_config["type"] param_space = param_config.get("space", None) param_range = param_config.get("range", None) param_values = param_config.get("values", None) # Some setup for case that whitelist of values is provided: values_only_type = param_type in ("cat", "ordinal") if (param_values is not None) and (not values_only_type): assert param_range is None param_values = np.unique(param_values) param_range = (param_values[0], param_values[-1]) round_to_values[param_name] = interp1d( param_values, param_values, kind="nearest", fill_value="extrapolate") if param_type == "int": low, high = param_range if param_space in ("log", "logit"): space[param_name] = hp.qloguniform(param_name, np.log(low), np.log(high), 1) else: space[param_name] = hp.quniform(param_name, low, high, 1) elif param_type == "bool": assert param_range is None assert param_values is None space[param_name] = hp.choice(param_name, (False, True)) elif param_type in ("cat", "ordinal"): assert param_range is None space[param_name] = hp.choice(param_name, param_values) elif param_type == "real": low, high = param_range if param_space in ("log", "logit"): space[param_name] = hp.loguniform(param_name, np.log(low), np.log(high)) else: space[param_name] = hp.uniform(param_name, low, high) else: assert False, "type %s not handled in API" % param_type return space, round_to_values def get_trial(self, trial_id): for trial in self.trials._dynamic_trials: if trial["tid"] == trial_id: assert isinstance(trial, dict) # Make sure right kind of dict assert "state" in trial and "result" in trial assert trial["state"] == JOB_STATE_NEW return trial assert False, "No matching trial ID" def cleanup_guess(self, x_guess): assert isinstance(x_guess, dict) # Also, check the keys are only the vars we are searching over: assert frozenset(x_guess.keys()) == self.param_set_chk # Do the rounding # Make a copy to be safe, and also unpack singletons # We may also need to consider clip_chk at some point like opentuner x_guess = {k: only(x_guess[k]) for k in x_guess} for param_name, round_f in self.round_to_values.items(): x_guess[param_name] = round_f(x_guess[param_name]) # Also ensure this is correct dtype so sklearn is happy x_guess = { k: DTYPE_MAP[self.api_config[k]["type"]](x_guess[k]) for k in x_guess } return x_guess def _suggest(self): """Helper function to `suggest` that does the work of calling `hyperopt` via its dumb API. """ new_ids = self.trials.new_trial_ids(1) assert len(new_ids) == 1 self.trials.refresh() seed = random_seed(self.random) new_trials = tpe.suggest(new_ids, self.domain, self.trials, seed) assert len(new_trials) == 1 self.trials.insert_trial_docs(new_trials) self.trials.refresh() new_trial, = new_trials # extract singleton return new_trial def suggest(self, n_suggestions=1): """Make `n_suggestions` suggestions for what to evaluate next. This requires the user observe all previous suggestions before calling again. Parameters ---------- n_suggestions : int The number of suggestions to return. Returns ------- next_guess : list of dict List of `n_suggestions` suggestions to evaluate the objective function. Each suggestion is a dictionary where each key corresponds to a parameter being optimized. """ assert n_suggestions >= 1, "invalid value for n_suggestions" # Get the new trials, it seems hyperopt either uses random search or # guesses one at a time anyway, so we might as welll call serially. new_trials = [self._suggest() for _ in range(n_suggestions)] X = [] for trial in new_trials: x_guess = self.cleanup_guess(trial["misc"]["vals"]) X.append(x_guess) # Build lookup to get original trial object x_guess_ = HyperoptOptimizer.hashable_dict(x_guess) assert x_guess_ not in self.trial_id_lookup, "the suggestions should not already be in the trial dict" self.trial_id_lookup[x_guess_] = trial["tid"] assert len(X) == n_suggestions return X def observe(self, X, y): """Feed the observations back to hyperopt. Parameters ---------- X : list of dict-like Places where the objective function has already been evaluated. Each suggestion is a dictionary where each key corresponds to a parameter being optimized. y : array-like, shape (n,) Corresponding values where objective has been evaluated. """ assert len(X) == len(y) for x_guess, y_ in zip(X, y): x_guess_ = HyperoptOptimizer.hashable_dict(x_guess) assert x_guess_ in self.trial_id_lookup, "Appears to be guess that did not originate from suggest" assert x_guess_ in self.trial_id_lookup, "trial object not available in trial dict" trial_id = self.trial_id_lookup.pop(x_guess_) trial = self.get_trial(trial_id) assert self.cleanup_guess( trial["misc"]["vals"] ) == x_guess, "trial ID not consistent with x values stored" # Cast to float to ensure native type result = {"loss": float(y_), "status": STATUS_OK} trial["state"] = JOB_STATE_DONE trial["result"] = result # hyperopt.fmin.FMinIter.serial_evaluate only does one refresh at end # of loop of a bunch of evals, so we will do the same thing here. self.trials.refresh()
def setUp(self): self.trials = Trials()
def setUp(self): self.bandit = coin_flip() self.algo = RandomStop(5, self.bandit) self.trials = Trials() self.experiment = Experiment(self.trials, self.algo, async=False)
class tuSOTOptimizer(AbstractOptimizer): primary_import = "pysot" def __init__(self, api_config): """Build wrapper class to use an optimizer in benchmark. Parameters ---------- api_config : dict-like of dict-like Configuration of the optimization variables. See API description. """ AbstractOptimizer.__init__(self, api_config) self.space_x = JointSpace(api_config) self.bounds = self.space_x.get_bounds() self.create_opt_prob( ) # Sets up the optimization problem (needs self.bounds) self.max_evals = np.iinfo(np.int32).max # NOTE: Largest possible int self.turbo_batch_size = None self.pysot_batch_size = None self.history = [] self.proposals = [] self.lb, self.ub = self.bounds[:, 0], self.bounds[:, 1] self.dim = len(self.bounds) self.turbo = Turbo1( f=None, lb=self.bounds[:, 0], ub=self.bounds[:, 1], n_init=2 * self.dim + 1, max_evals=self.max_evals, batch_size=4, # We need to update this later verbose=False, ) # hyperopt self.random = np_random space, self.round_to_values = tuSOTOptimizer.get_hyperopt_dimensions( api_config) self.domain = Domain(dummy_f, space, pass_expr_memo_ctrl=None) self.trials = Trials() # Some book keeping like opentuner wrapper self.trial_id_lookup = {} # Store just for data validation self.param_set_chk = frozenset(api_config.keys()) def restart(self): self.turbo._restart() self.turbo._X = np.zeros((0, self.turbo.dim)) self.turbo._fX = np.zeros((0, 1)) X_init = latin_hypercube(self.turbo.n_init, self.dim) self.X_init = from_unit_cube(X_init, self.lb, self.ub) def create_opt_prob(self): """Create an optimization problem object.""" opt = OptimizationProblem() opt.lb = self.bounds[:, 0] # In warped space opt.ub = self.bounds[:, 1] # In warped space opt.dim = len(self.bounds) opt.cont_var = np.arange(len(self.bounds)) opt.int_var = [] assert len(opt.cont_var) + len(opt.int_var) == opt.dim opt.objfun = None self.opt = opt def start(self): """Starts a new pySOT run.""" self.history = [] self.proposals = [] # Symmetric Latin hypercube design des_pts = max([self.pysot_batch_size, 2 * (self.opt.dim + 1)]) slhd = SymmetricLatinHypercube(dim=self.opt.dim, num_pts=des_pts) # Warped RBF interpolant rbf = RBFInterpolant(dim=self.opt.dim, kernel=CubicKernel(), tail=LinearTail(self.opt.dim), eta=1e-4) rbf = SurrogateUnitBox(rbf, lb=self.opt.lb, ub=self.opt.ub) # Optimization strategy self.strategy = SRBFStrategy( max_evals=self.max_evals, opt_prob=self.opt, exp_design=slhd, surrogate=rbf, asynchronous=True, batch_size=1, use_restarts=True, ) @staticmethod def hashable_dict(d): hashable_object = frozenset(d.items()) return hashable_object @staticmethod def get_hyperopt_dimensions(api_config): param_list = sorted(api_config.keys()) space = {} round_to_values = {} for param_name in param_list: param_config = api_config[param_name] param_type = param_config["type"] param_space = param_config.get("space", None) param_range = param_config.get("range", None) param_values = param_config.get("values", None) # Some setup for case that whitelist of values is provided: values_only_type = param_type in ("cat", "ordinal") if (param_values is not None) and (not values_only_type): assert param_range is None param_values = np.unique(param_values) param_range = (param_values[0], param_values[-1]) round_to_values[param_name] = interp1d( param_values, param_values, kind="nearest", fill_value="extrapolate") if param_type == "int": low, high = param_range if param_space in ("log", "logit"): space[param_name] = hp.qloguniform(param_name, np.log(low), np.log(high), 1) else: space[param_name] = hp.quniform(param_name, low, high, 1) elif param_type == "bool": assert param_range is None assert param_values is None space[param_name] = hp.choice(param_name, (False, True)) elif param_type in ("cat", "ordinal"): assert param_range is None space[param_name] = hp.choice(param_name, param_values) elif param_type == "real": low, high = param_range if param_space in ("log", "logit"): space[param_name] = hp.loguniform(param_name, np.log(low), np.log(high)) else: space[param_name] = hp.uniform(param_name, low, high) else: assert False, "type %s not handled in API" % param_type return space, round_to_values def get_trial(self, trial_id): for trial in self.trials._dynamic_trials: if trial["tid"] == trial_id: assert isinstance(trial, dict) # Make sure right kind of dict assert "state" in trial and "result" in trial assert trial["state"] == JOB_STATE_NEW return trial assert False, "No matching trial ID" def cleanup_guess(self, x_guess): assert isinstance(x_guess, dict) # Also, check the keys are only the vars we are searching over: assert frozenset(x_guess.keys()) == self.param_set_chk # Do the rounding # Make a copy to be safe, and also unpack singletons # We may also need to consider clip_chk at some point like opentuner x_guess = {k: only(x_guess[k]) for k in x_guess} for param_name, round_f in self.round_to_values.items(): x_guess[param_name] = round_f(x_guess[param_name]) # Also ensure this is correct dtype so sklearn is happy x_guess = { k: DTYPE_MAP[self.api_config[k]["type"]](x_guess[k]) for k in x_guess } return x_guess def pysot_suggest(self, n_suggestions=1): if self.pysot_batch_size is None: # First call to suggest self.pysot_batch_size = n_suggestions self.start() # Set the tolerances pretending like we are running batch d, p = float(self.opt.dim), float(n_suggestions) self.strategy.failtol = p * int(max(np.ceil(d / p), np.ceil(4 / p))) # Now we can make suggestions x_w = [] self.proposals = [] for _ in range(n_suggestions): proposal = self.strategy.propose_action() record = EvalRecord(proposal.args, status="pending") proposal.record = record proposal.accept() # This triggers all the callbacks # It is possible that pySOT proposes a previously evaluated point # when all variables are integers, so we just abort in this case # since we have likely converged anyway. See PySOT issue #30. x = list(proposal.record.params) # From tuple to list x_unwarped, = self.space_x.unwarp(x) if x_unwarped in self.history: warnings.warn("pySOT proposed the same point twice") self.start() return self.suggest(n_suggestions=n_suggestions) # NOTE: Append unwarped to avoid rounding issues self.history.append(copy(x_unwarped)) self.proposals.append(proposal) x_w.append(copy(x_unwarped)) return x_w def pysot_get_suggest(self, suggests): turbo_suggest_warps = self.space_x.warp(suggests) for i, warps in enumerate(turbo_suggest_warps): proposal = self.strategy.make_proposal(warps) proposal.add_callback(self.strategy.on_initial_proposal) record = EvalRecord(proposal.args, status="pending") proposal.record = record proposal.accept() self.history.append(copy(suggests[i])) self.proposals.append(proposal) def turbo_suggest(self, n_suggestions=1): if self.turbo_batch_size is None: # Remember the batch size on the first call to suggest self.turbo_batch_size = n_suggestions self.turbo.batch_size = n_suggestions self.turbo.failtol = np.ceil( np.max([ 4.0 / self.turbo_batch_size, self.dim / self.turbo_batch_size ])) self.turbo.n_init = max([self.turbo.n_init, self.turbo_batch_size]) self.restart() X_next = np.zeros((n_suggestions, self.dim)) # Pick from the initial points n_init = min(len(self.X_init), n_suggestions) if n_init > 0: X_next[:n_init] = deepcopy(self.X_init[:n_init, :]) self.X_init = self.X_init[ n_init:, :] # Remove these pending points # Get remaining points from TuRBO n_adapt = n_suggestions - n_init if n_adapt > 0: if len(self.turbo._X ) > 0: # Use random points if we can't fit a GP X = to_unit_cube(deepcopy(self.turbo._X), self.lb, self.ub) fX = copula_standardize(deepcopy( self.turbo._fX).ravel()) # Use Copula X_cand, y_cand, _ = self.turbo._create_candidates( X, fX, length=self.turbo.length, n_training_steps=100, hypers={}) X_next[-n_adapt:, :] = self.turbo._select_candidates( X_cand, y_cand)[:n_adapt, :] X_next[-n_adapt:, :] = from_unit_cube(X_next[-n_adapt:, :], self.lb, self.ub) # Unwarp the suggestions suggestions = self.space_x.unwarp(X_next) return suggestions def _hyperopt_suggest(self): new_ids = self.trials.new_trial_ids(1) assert len(new_ids) == 1 self.trials.refresh() seed = random_seed(self.random) new_trials = tpe.suggest(new_ids, self.domain, self.trials, seed) assert len(new_trials) == 1 self.trials.insert_trial_docs(new_trials) self.trials.refresh() new_trial, = new_trials # extract singleton return new_trial def _hyperopt_transform(self, x): new_id = self.trials.new_trial_ids(1)[0] domain = self.domain rng = np.random.RandomState(1) idxs, vals = pyll.rec_eval(domain.s_idxs_vals, memo={ domain.s_new_ids: [new_id], domain.s_rng: rng, }) rval_miscs = [dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)] rval_results = domain.new_result() for (k, _) in vals.items(): vals[k][0] = x[k] miscs_update_idxs_vals(rval_miscs, idxs, vals) rval_docs = self.trials.new_trial_docs([new_id], [None], rval_results, rval_miscs) return rval_docs[0] def hyperopt_suggest(self, n_suggestions=1): assert n_suggestions >= 1, "invalid value for n_suggestions" # Get the new trials, it seems hyperopt either uses random search or # guesses one at a time anyway, so we might as welll call serially. new_trials = [self._hyperopt_suggest() for _ in range(n_suggestions)] X = [] for trial in new_trials: x_guess = self.cleanup_guess(trial["misc"]["vals"]) X.append(x_guess) # Build lookup to get original trial object x_guess_ = tuSOTOptimizer.hashable_dict(x_guess) assert x_guess_ not in self.trial_id_lookup, "the suggestions should not already be in the trial dict" self.trial_id_lookup[x_guess_] = trial["tid"] assert len(X) == n_suggestions return X def hyperopt_get_suggest(self, suggests): trials = [self._hyperopt_transform(x) for x in suggests] for trial in trials: x_guess = self.cleanup_guess(trial["misc"]["vals"]) x_guess_ = tuSOTOptimizer.hashable_dict(x_guess) assert x_guess_ not in self.trial_id_lookup, "the suggestions should not already be in the trial dict" self.trial_id_lookup[x_guess_] = trial["tid"] self.trials.insert_trial_docs(trials) self.trials.refresh() def suggest(self, n_suggestions=1): if n_suggestions == 1: return self.turbo_suggest(n_suggestions) else: t_suggestion = n_suggestions // 2 # p_suggestion = int((n_suggestions - t_suggestion) * 3/4) h_suggestion = n_suggestions - t_suggestion turbo_suggest = self.turbo_suggest(t_suggestion) # pysot_suggest = self.pysot_suggest(p_suggestion) hyperopt_suggest = self.hyperopt_suggest(h_suggestion) self.hyperopt_get_suggest(turbo_suggest) # self.pysot_get_suggest(turbo_suggest + hyperopt_suggest) return turbo_suggest + hyperopt_suggest def _observe(self, x, y): # Find the matching proposal and execute its callbacks idx = [x == xx for xx in self.history] if np.any(idx): i = np.argwhere( idx)[0].item() # Pick the first index if there are ties proposal = self.proposals[i] proposal.record.complete(y) self.proposals.pop(i) self.history.pop(i) def observe(self, X, y): """Send an observation of a suggestion back to the optimizer. Parameters ---------- X : list of dict-like Places where the objective function has already been evaluated. Each suggestion is a dictionary where each key corresponds to a parameter being optimized. y : array-like, shape (n,) Corresponding values where objective has been evaluated """ assert len(X) == len(y) # # pysot observe # for x_, y_ in zip(X, y): # # Just ignore, any inf observations we got, unclear if right thing # if np.isfinite(y_): # self._observe(x_, y_) # turbo observe XX, yy = self.space_x.warp(X), np.array(y)[:, None] if len(self.turbo._fX) >= self.turbo.n_init: self.turbo._adjust_length(yy) self.turbo.n_evals += self.turbo_batch_size self.turbo._X = np.vstack((self.turbo._X, deepcopy(XX))) self.turbo._fX = np.vstack((self.turbo._fX, deepcopy(yy))) self.turbo.X = np.vstack((self.turbo.X, deepcopy(XX))) self.turbo.fX = np.vstack((self.turbo.fX, deepcopy(yy))) # Check for a restart if self.turbo.length < self.turbo.length_min: self.restart() # hyperopt observe for x_guess, y_ in zip(X, y): x_guess_ = tuSOTOptimizer.hashable_dict(x_guess) assert x_guess_ in self.trial_id_lookup, "Appears to be guess that did not originate from suggest" assert x_guess_ in self.trial_id_lookup, "trial object not available in trial dict" trial_id = self.trial_id_lookup.pop(x_guess_) trial = self.get_trial(trial_id) assert self.cleanup_guess( trial["misc"]["vals"] ) == x_guess, "trial ID not consistent with x values stored" # Cast to float to ensure native type result = {"loss": float(y_), "status": STATUS_OK} trial["state"] = JOB_STATE_DONE trial["result"] = result self.trials.refresh()
def idxs_vals_from_ids(self, ids, seed): docs = self.suggest(ids, self.domain, Trials(), seed) trials = trials_from_docs(docs) idxs, vals = miscs_to_idxs_vals(trials.miscs) return idxs, vals
def __init__(self, client): Trials.__init__(self) self._client = client
#print dir(tt['ar']) #print dir(tt['ar'].metadata) #print 'sent', tt['ar'].sent #print 'elapsed', tt['ar'].elapsed #print 'prog', tt['ar'].progress #print 'succ', tt['ar'].successful #print tt['ar'].metadata if tt['ar'].sent: tt['state'] = JOB_STATE_RUNNING #elif (tt['state'] != JOB_STATE_RUNNING): #and tt['ar'].metadata['started']): #if tt['state'] == JOB_STATE_NEW: #print id(tt['ar']), tt['ar'], tt['ar'].metadata #['status'] # XXX mark errors Trials.refresh(self) def fmin(self, fn, space, algo, max_evals, random_state=0, verbose=0, ): lb_view = self._client.load_balanced_view() random_state = check_random_state(random_state) domain = Domain(fn, space, rseed=int(random_state.randint(2^20))) while len(self.trials) < max_evals: if lb_view.queue_status()['unassigned']: sleep(1e-3) continue