def suggest(
new_ids,
domain,
trials,
seed,
warmup_cutoff=15, # -- enough for mean & var stats
n_buckshots=10000,
n_finetunes=50,
stop_at=None,
plot_contours=None,
gp_fit_method='ml2',
failure_loss=None,
max_ei_thresh=None,
):
"""
Parameters
----------
"""
# XXX would like to cache on domain, but
# fmin(fn, space) always rebuilds a new domain for given fn and space
key = domain.expr
try:
dgp = _suggest_domain_cache[key]
except KeyError:
print 'CREATING GP_EI for', domain
dgp = _suggest_domain_cache[key] = DomainGP_EI(domain)
if len(trials.trials):
X, y, var_y = dgp._X_y_var_y(trials, failure_loss=failure_loss)
if len(trials.trials) <= warmup_cutoff:
if len(trials.trials):
dgp.gpr.prior_mean = np.mean(y)
dgp.gpr.prior_var = np.var(y)
return rand.suggest(new_ids, domain, trials, seed)
if stop_at is not None and min(trials.losses()) < stop_at:
return []
dgp.fit_gpr(X, y, var_y, method=gp_fit_method)
dgp.set_thresholds(y, var_y, max_ei_thresh=max_ei_thresh)
dgp._cost_deriv_reuse_cholesky = 0
print 'EI: Best after %i trials: %f' % (len(y), np.min(y))
#dgp.gpr._params_list[0][:] = 0
rng = np.random.RandomState(seed)
t0 = time.time()
best_pt = dgp.optimize_over_X(
n_buckshots=n_buckshots,
n_finetunes=n_finetunes,
rng=rng,
)
t1 = time.time()
print 'optimizing surrogate took', (t1 - t0)
if plot_contours:
plot_contours(dgp, 2, dgp._lbound, best_pt)
new_id, = new_ids
#print 'REI: Best pt', best_pt
return SuggestBest(domain, trials, seed, best_pt)(new_id)
def suggest(new_ids, domain, trials, seed,
warmup_cutoff=15, # -- enough for mean & var stats
n_buckshots=10000,
n_finetunes=50,
stop_at=None,
plot_contours=None,
gp_fit_method='ml2',
failure_loss=None,
max_ei_thresh=None,
):
"""
Parameters
----------
"""
# XXX would like to cache on domain, but
# fmin(fn, space) always rebuilds a new domain for given fn and space
key = domain.expr
try:
dgp = _suggest_domain_cache[key]
except KeyError:
print 'CREATING GP_EI for', domain
dgp = _suggest_domain_cache[key] = DomainGP_EI(domain)
if len(trials.trials):
X, y, var_y = dgp._X_y_var_y(trials, failure_loss=failure_loss)
if len(trials.trials) <= warmup_cutoff:
if len(trials.trials):
dgp.gpr.prior_mean = np.mean(y)
dgp.gpr.prior_var = np.var(y)
return rand.suggest(new_ids, domain, trials, seed)
if stop_at is not None and min(trials.losses()) < stop_at:
return []
dgp.fit_gpr(X, y, var_y, method=gp_fit_method)
dgp.set_thresholds(y, var_y, max_ei_thresh=max_ei_thresh)
dgp._cost_deriv_reuse_cholesky = 0
print 'EI: Best after %i trials: %f' % ( len(y), np.min(y))
#dgp.gpr._params_list[0][:] = 0
rng = np.random.RandomState(seed)
t0 = time.time()
best_pt = dgp.optimize_over_X(
n_buckshots=n_buckshots,
n_finetunes=n_finetunes,
rng=rng,
)
t1 = time.time()
print 'optimizing surrogate took', (t1 - t0)
if plot_contours:
# plot_contours(dgp, 2, dgp._lbound, best_pt)
plot_contours(dgp, 2, best_pt)
new_id, = new_ids
#print 'REI: Best pt', best_pt
return SuggestBest(domain, trials, seed, best_pt)(new_id)
def suggest(new_ids, domain, trials, seed,
warmup_cutoff=1,
n_buckshots=10000,
n_finetunes=50,
best_possible=-np.inf,
#best_headroom=1.0,
stop_at=None,
plot_contours=None,
):
"""
Parameters
----------
"""
if stop_at is not None and stop_at < best_possible:
raise ValueError(
('If stop_at is specified'
', it (%f) must be greater than best_possible (%f)') % (
stop_at, best_possible))
if len(trials.trials) <= warmup_cutoff:
return rand.suggest(new_ids, domain, trials, seed)
try:
dgp = _suggest_domain_cache[domain]
except KeyError:
dgp = _suggest_domain_cache[domain] = DomainGP_LUCB(domain)
if stop_at is not None and min(trials.losses()) < stop_at:
return []
X, y, var_y = dgp._X_y_var_y(trials)
dgp.fit_gpr(X, y, var_y)
dgp._optimism = 1.0 #0.5 * dgp._optimism
dgp._ubound = np.min(y)
dgp._lbound = best_possible
#yy = y + np.sqrt(np.maximum(var_y, dgp.gpr.s_var_min.eval()))
#dgp._ubound = np.min(yy)
#max(opt_lbound, - best_headroom)
#print 'LUCB interval:', dgp._lbound, dgp._ubound
print 'LUCB: Best after %i trials: %f' % ( len(y), np.min(y))
#dgp.gpr._params_list[0][:] = 0
rng = np.random.RandomState(seed)
best_pt = dgp.optimize_over_X(
n_buckshots=n_buckshots,
n_finetunes=n_finetunes,
rng=rng,
)
if plot_contours:
plot_contours(dgp, 2, dgp._lbound, best_pt)
new_id, = new_ids
#print 'LUCB: Best pt', best_pt
return SuggestBest(domain, trials, seed, best_pt)(new_id)
def test_seeding(self):
# -- assert that the seeding works a particular way
domain = coin_flip()
docs = rand.suggest(list(range(10)), domain, Trials(), seed=123)
trials = trials_from_docs(docs)
idxs, vals = miscs_to_idxs_vals(trials.miscs)
# Passes Nov 8 / 2013
self.assertEqual(list(idxs["flip"]), list(range(10)))
self.assertEqual(list(vals["flip"]), [0, 1, 0, 0, 0, 0, 0, 1, 1, 0])
def test_seeding(self):
# -- assert that the seeding works a particular way
domain = coin_flip()
docs = rand.suggest(range(10), domain, Trials(), seed=123)
trials = trials_from_docs(docs)
idxs, vals = miscs_to_idxs_vals(trials.miscs)
# Passes Nov 8 / 2013
self.assertEqual(list(idxs['flip']), range(10))
self.assertEqual(list(vals['flip']), [0, 1, 0, 0, 0, 0, 0, 1, 1, 0])
def suggest(
new_ids,
domain,
trials,
seed,
warmup_cutoff=15,
n_buckshots=10000,
n_finetunes=50,
stop_at=None,
plot_contours=None,
):
"""
Parameters
----------
"""
if len(trials.trials) <= warmup_cutoff:
return rand.suggest(new_ids, domain, trials, seed)
# XXX would like to cache on domain, but
# fmin(fn, space) always rebuilds a new domain for given fn and space
key = domain.expr
try:
dgp = _suggest_domain_cache[key]
except KeyError:
dgp = _suggest_domain_cache[key] = DomainGP_UCB(domain)
if stop_at is not None and min(trials.losses()) < stop_at:
return []
X, y, var_y = dgp._X_y_var_y(trials)
dgp.fit_gpr(X, y, var_y)
print 'Fit ->', dgp.gpr._params_list[0]
dgp._ucb_z = 0.2
# XXX: radius should depend on dimensionality?
# 1e-8 worked for branin in case current one doesn't
dgp._K_thresh = (1 - 1e-5) # / (1000 + len(y) ** 2))
print 'UCB: Best after %i trials: %f' % (len(y), np.min(y))
#dgp.gpr._params_list[0][:] = 0
rng = np.random.RandomState(seed)
best_pt = dgp.optimize_over_X(
n_buckshots=n_buckshots,
n_finetunes=n_finetunes,
rng=rng,
)
if plot_contours:
plot_contours(dgp, 2, dgp._lbound, best_pt)
new_id, = new_ids
#print 'REI: Best pt', best_pt
return SuggestBest(domain, trials, seed, best_pt)(new_id)
def suggest(new_ids, domain, trials, seed,
warmup_cutoff=15,
n_buckshots=10000,
n_finetunes=50,
stop_at=None,
plot_contours=None,
):
"""
Parameters
----------
"""
if len(trials.trials) <= warmup_cutoff:
return rand.suggest(new_ids, domain, trials, seed)
# XXX would like to cache on domain, but
# fmin(fn, space) always rebuilds a new domain for given fn and space
key = domain.expr
try:
dgp = _suggest_domain_cache[key]
except KeyError:
dgp = _suggest_domain_cache[key] = DomainGP_UCB(domain)
if stop_at is not None and min(trials.losses()) < stop_at:
return []
X, y, var_y = dgp._X_y_var_y(trials)
dgp.fit_gpr(X, y, var_y)
print 'Fit ->', dgp.gpr._params_list[0]
dgp._ucb_z = 0.2
# XXX: radius should depend on dimensionality?
# 1e-8 worked for branin in case current one doesn't
dgp._K_thresh = (1 - 1e-5) # / (1000 + len(y) ** 2))
print 'UCB: Best after %i trials: %f' % ( len(y), np.min(y))
#dgp.gpr._params_list[0][:] = 0
rng = np.random.RandomState(seed)
best_pt = dgp.optimize_over_X(
n_buckshots=n_buckshots,
n_finetunes=n_finetunes,
rng=rng,
)
if plot_contours:
plot_contours(dgp, 2, dgp._lbound, best_pt)
new_id, = new_ids
#print 'REI: Best pt', best_pt
return SuggestBest(domain, trials, seed, best_pt)(new_id)
def suggest(new_ids, domain, trials, seed, samples_count=200, maxlog=8, rand_iters=3,
create_regressor=lambda: GaussianProcessRegressor(alpha=1e-6)):
rand_iters = max(1, rand_iters)
if len(trials) < rand_iters:
return rand.suggest(new_ids, domain, trials, seed)
rng = np.random.RandomState(seed)
rval = []
for ii, new_id in enumerate(new_ids):
# -- sample new specs, idxs, vals
idxs, vals = get_best_eval(new_id, domain, trials, rng, samples_count, maxlog, create_regressor)
new_result = domain.new_result()
new_misc = dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)
miscs_update_idxs_vals([new_misc], idxs, vals)
rval.extend(trials.new_trial_docs([new_id],
[None], [new_result], [new_misc]))
return rval
def suggest(a, b, c):
seed = int(time.time())
return rand.suggest(a, b, c, seed=seed)
def search(self, run_name, store, context, hp_records, runs):
def make_trial(tid, arg_dict, loss_value):
trial = {"book_time": None, "exp_key": None, "owner": None, "refresh_time": None, "spec": None, "state": 0, "tid": tid, "version": 0}
#trial["result"] = {"status": "New"}
misc = {}
trial["misc"] = misc
misc["cmd"] = ("domain_attachment", "FMinIter_Domain")
misc["idxs"] = {key: [tid] for key in arg_dict.keys()}
misc["tid"] = tid
misc["vals"] = arg_dict
trial["state"] = 2 # done
trial["result"] = {"loss": loss_value, "status": "ok"}
#trial["refresh_time"] = coarse_utcnow()
return trial
dummy_loss = lambda x: None
param_space = {r["name"]: r["space_func"] for r in hp_records}
domain = base.Domain(dummy_loss, param_space)
rstate = np.random.RandomState()
# convert runs to Trials
trial_list = []
for run in runs:
# don't trip over inappropriate runs
if (not "run_name" in run) or (not "hparams" in run) or (not context.primary_metric in run):
continue
run_name = run["run_name"]
arg_dict = run["hparams"]
loss_value = run[context.primary_metric]
if context.maximize_metric:
loss_value = -loss_value
# extract a unique int from run_name (parent.childnum)
tid = run_helper.get_int_from_run_name(run_name)
trial = make_trial(tid, arg_dict, loss_value)
trial_list.append(trial)
# finally, add our trial_list to trials
trials = Trials()
trials.insert_trial_docs(trial_list)
trials.refresh()
# get next suggested hyperparameter values from TPE algorithm
tid = run_helper.get_int_from_run_name(run_name)
min_trials = 3 # before this, just do rand sampling
seed = rstate.randint(2 ** 31 - 1)
if len(trials) < min_trials:
new_trials = rand.suggest([tid], domain, trials, seed)
else:
new_trials = tpe.suggest([tid], domain, trials, seed)
# apply the suggested hparam values
trial = new_trials[0]
arg_dict = trial["misc"]["vals"]
arg_dict = self.fixup_hyperopt_hparams(param_space, arg_dict)
return arg_dict