def suggest( new_ids, domain, trials, seed, warmup_cutoff=15, # -- enough for mean & var stats n_buckshots=10000, n_finetunes=50, stop_at=None, plot_contours=None, gp_fit_method='ml2', failure_loss=None, max_ei_thresh=None, ): """ Parameters ---------- """ # XXX would like to cache on domain, but # fmin(fn, space) always rebuilds a new domain for given fn and space key = domain.expr try: dgp = _suggest_domain_cache[key] except KeyError: print 'CREATING GP_EI for', domain dgp = _suggest_domain_cache[key] = DomainGP_EI(domain) if len(trials.trials): X, y, var_y = dgp._X_y_var_y(trials, failure_loss=failure_loss) if len(trials.trials) <= warmup_cutoff: if len(trials.trials): dgp.gpr.prior_mean = np.mean(y) dgp.gpr.prior_var = np.var(y) return rand.suggest(new_ids, domain, trials, seed) if stop_at is not None and min(trials.losses()) < stop_at: return [] dgp.fit_gpr(X, y, var_y, method=gp_fit_method) dgp.set_thresholds(y, var_y, max_ei_thresh=max_ei_thresh) dgp._cost_deriv_reuse_cholesky = 0 print 'EI: Best after %i trials: %f' % (len(y), np.min(y)) #dgp.gpr._params_list[0][:] = 0 rng = np.random.RandomState(seed) t0 = time.time() best_pt = dgp.optimize_over_X( n_buckshots=n_buckshots, n_finetunes=n_finetunes, rng=rng, ) t1 = time.time() print 'optimizing surrogate took', (t1 - t0) if plot_contours: plot_contours(dgp, 2, dgp._lbound, best_pt) new_id, = new_ids #print 'REI: Best pt', best_pt return SuggestBest(domain, trials, seed, best_pt)(new_id)
def suggest(new_ids, domain, trials, seed, warmup_cutoff=15, # -- enough for mean & var stats n_buckshots=10000, n_finetunes=50, stop_at=None, plot_contours=None, gp_fit_method='ml2', failure_loss=None, max_ei_thresh=None, ): """ Parameters ---------- """ # XXX would like to cache on domain, but # fmin(fn, space) always rebuilds a new domain for given fn and space key = domain.expr try: dgp = _suggest_domain_cache[key] except KeyError: print 'CREATING GP_EI for', domain dgp = _suggest_domain_cache[key] = DomainGP_EI(domain) if len(trials.trials): X, y, var_y = dgp._X_y_var_y(trials, failure_loss=failure_loss) if len(trials.trials) <= warmup_cutoff: if len(trials.trials): dgp.gpr.prior_mean = np.mean(y) dgp.gpr.prior_var = np.var(y) return rand.suggest(new_ids, domain, trials, seed) if stop_at is not None and min(trials.losses()) < stop_at: return [] dgp.fit_gpr(X, y, var_y, method=gp_fit_method) dgp.set_thresholds(y, var_y, max_ei_thresh=max_ei_thresh) dgp._cost_deriv_reuse_cholesky = 0 print 'EI: Best after %i trials: %f' % ( len(y), np.min(y)) #dgp.gpr._params_list[0][:] = 0 rng = np.random.RandomState(seed) t0 = time.time() best_pt = dgp.optimize_over_X( n_buckshots=n_buckshots, n_finetunes=n_finetunes, rng=rng, ) t1 = time.time() print 'optimizing surrogate took', (t1 - t0) if plot_contours: # plot_contours(dgp, 2, dgp._lbound, best_pt) plot_contours(dgp, 2, best_pt) new_id, = new_ids #print 'REI: Best pt', best_pt return SuggestBest(domain, trials, seed, best_pt)(new_id)
def suggest(new_ids, domain, trials, seed, warmup_cutoff=1, n_buckshots=10000, n_finetunes=50, best_possible=-np.inf, #best_headroom=1.0, stop_at=None, plot_contours=None, ): """ Parameters ---------- """ if stop_at is not None and stop_at < best_possible: raise ValueError( ('If stop_at is specified' ', it (%f) must be greater than best_possible (%f)') % ( stop_at, best_possible)) if len(trials.trials) <= warmup_cutoff: return rand.suggest(new_ids, domain, trials, seed) try: dgp = _suggest_domain_cache[domain] except KeyError: dgp = _suggest_domain_cache[domain] = DomainGP_LUCB(domain) if stop_at is not None and min(trials.losses()) < stop_at: return [] X, y, var_y = dgp._X_y_var_y(trials) dgp.fit_gpr(X, y, var_y) dgp._optimism = 1.0 #0.5 * dgp._optimism dgp._ubound = np.min(y) dgp._lbound = best_possible #yy = y + np.sqrt(np.maximum(var_y, dgp.gpr.s_var_min.eval())) #dgp._ubound = np.min(yy) #max(opt_lbound, - best_headroom) #print 'LUCB interval:', dgp._lbound, dgp._ubound print 'LUCB: Best after %i trials: %f' % ( len(y), np.min(y)) #dgp.gpr._params_list[0][:] = 0 rng = np.random.RandomState(seed) best_pt = dgp.optimize_over_X( n_buckshots=n_buckshots, n_finetunes=n_finetunes, rng=rng, ) if plot_contours: plot_contours(dgp, 2, dgp._lbound, best_pt) new_id, = new_ids #print 'LUCB: Best pt', best_pt return SuggestBest(domain, trials, seed, best_pt)(new_id)
def test_seeding(self): # -- assert that the seeding works a particular way domain = coin_flip() docs = rand.suggest(list(range(10)), domain, Trials(), seed=123) trials = trials_from_docs(docs) idxs, vals = miscs_to_idxs_vals(trials.miscs) # Passes Nov 8 / 2013 self.assertEqual(list(idxs["flip"]), list(range(10))) self.assertEqual(list(vals["flip"]), [0, 1, 0, 0, 0, 0, 0, 1, 1, 0])
def test_seeding(self): # -- assert that the seeding works a particular way domain = coin_flip() docs = rand.suggest(range(10), domain, Trials(), seed=123) trials = trials_from_docs(docs) idxs, vals = miscs_to_idxs_vals(trials.miscs) # Passes Nov 8 / 2013 self.assertEqual(list(idxs['flip']), range(10)) self.assertEqual(list(vals['flip']), [0, 1, 0, 0, 0, 0, 0, 1, 1, 0])
def suggest( new_ids, domain, trials, seed, warmup_cutoff=15, n_buckshots=10000, n_finetunes=50, stop_at=None, plot_contours=None, ): """ Parameters ---------- """ if len(trials.trials) <= warmup_cutoff: return rand.suggest(new_ids, domain, trials, seed) # XXX would like to cache on domain, but # fmin(fn, space) always rebuilds a new domain for given fn and space key = domain.expr try: dgp = _suggest_domain_cache[key] except KeyError: dgp = _suggest_domain_cache[key] = DomainGP_UCB(domain) if stop_at is not None and min(trials.losses()) < stop_at: return [] X, y, var_y = dgp._X_y_var_y(trials) dgp.fit_gpr(X, y, var_y) print 'Fit ->', dgp.gpr._params_list[0] dgp._ucb_z = 0.2 # XXX: radius should depend on dimensionality? # 1e-8 worked for branin in case current one doesn't dgp._K_thresh = (1 - 1e-5) # / (1000 + len(y) ** 2)) print 'UCB: Best after %i trials: %f' % (len(y), np.min(y)) #dgp.gpr._params_list[0][:] = 0 rng = np.random.RandomState(seed) best_pt = dgp.optimize_over_X( n_buckshots=n_buckshots, n_finetunes=n_finetunes, rng=rng, ) if plot_contours: plot_contours(dgp, 2, dgp._lbound, best_pt) new_id, = new_ids #print 'REI: Best pt', best_pt return SuggestBest(domain, trials, seed, best_pt)(new_id)
def suggest(new_ids, domain, trials, seed, warmup_cutoff=15, n_buckshots=10000, n_finetunes=50, stop_at=None, plot_contours=None, ): """ Parameters ---------- """ if len(trials.trials) <= warmup_cutoff: return rand.suggest(new_ids, domain, trials, seed) # XXX would like to cache on domain, but # fmin(fn, space) always rebuilds a new domain for given fn and space key = domain.expr try: dgp = _suggest_domain_cache[key] except KeyError: dgp = _suggest_domain_cache[key] = DomainGP_UCB(domain) if stop_at is not None and min(trials.losses()) < stop_at: return [] X, y, var_y = dgp._X_y_var_y(trials) dgp.fit_gpr(X, y, var_y) print 'Fit ->', dgp.gpr._params_list[0] dgp._ucb_z = 0.2 # XXX: radius should depend on dimensionality? # 1e-8 worked for branin in case current one doesn't dgp._K_thresh = (1 - 1e-5) # / (1000 + len(y) ** 2)) print 'UCB: Best after %i trials: %f' % ( len(y), np.min(y)) #dgp.gpr._params_list[0][:] = 0 rng = np.random.RandomState(seed) best_pt = dgp.optimize_over_X( n_buckshots=n_buckshots, n_finetunes=n_finetunes, rng=rng, ) if plot_contours: plot_contours(dgp, 2, dgp._lbound, best_pt) new_id, = new_ids #print 'REI: Best pt', best_pt return SuggestBest(domain, trials, seed, best_pt)(new_id)
def suggest(new_ids, domain, trials, seed, samples_count=200, maxlog=8, rand_iters=3, create_regressor=lambda: GaussianProcessRegressor(alpha=1e-6)): rand_iters = max(1, rand_iters) if len(trials) < rand_iters: return rand.suggest(new_ids, domain, trials, seed) rng = np.random.RandomState(seed) rval = [] for ii, new_id in enumerate(new_ids): # -- sample new specs, idxs, vals idxs, vals = get_best_eval(new_id, domain, trials, rng, samples_count, maxlog, create_regressor) new_result = domain.new_result() new_misc = dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir) miscs_update_idxs_vals([new_misc], idxs, vals) rval.extend(trials.new_trial_docs([new_id], [None], [new_result], [new_misc])) return rval
def suggest(a, b, c): seed = int(time.time()) return rand.suggest(a, b, c, seed=seed)
def search(self, run_name, store, context, hp_records, runs): def make_trial(tid, arg_dict, loss_value): trial = {"book_time": None, "exp_key": None, "owner": None, "refresh_time": None, "spec": None, "state": 0, "tid": tid, "version": 0} #trial["result"] = {"status": "New"} misc = {} trial["misc"] = misc misc["cmd"] = ("domain_attachment", "FMinIter_Domain") misc["idxs"] = {key: [tid] for key in arg_dict.keys()} misc["tid"] = tid misc["vals"] = arg_dict trial["state"] = 2 # done trial["result"] = {"loss": loss_value, "status": "ok"} #trial["refresh_time"] = coarse_utcnow() return trial dummy_loss = lambda x: None param_space = {r["name"]: r["space_func"] for r in hp_records} domain = base.Domain(dummy_loss, param_space) rstate = np.random.RandomState() # convert runs to Trials trial_list = [] for run in runs: # don't trip over inappropriate runs if (not "run_name" in run) or (not "hparams" in run) or (not context.primary_metric in run): continue run_name = run["run_name"] arg_dict = run["hparams"] loss_value = run[context.primary_metric] if context.maximize_metric: loss_value = -loss_value # extract a unique int from run_name (parent.childnum) tid = run_helper.get_int_from_run_name(run_name) trial = make_trial(tid, arg_dict, loss_value) trial_list.append(trial) # finally, add our trial_list to trials trials = Trials() trials.insert_trial_docs(trial_list) trials.refresh() # get next suggested hyperparameter values from TPE algorithm tid = run_helper.get_int_from_run_name(run_name) min_trials = 3 # before this, just do rand sampling seed = rstate.randint(2 ** 31 - 1) if len(trials) < min_trials: new_trials = rand.suggest([tid], domain, trials, seed) else: new_trials = tpe.suggest([tid], domain, trials, seed) # apply the suggested hparam values trial = new_trials[0] arg_dict = trial["misc"]["vals"] arg_dict = self.fixup_hyperopt_hparams(param_space, arg_dict) return arg_dict