def run():
train_inputs, train_targets,\
tests_inputs, tests_targets, unscale_y = load_boston_housing(train_frac)
N_train = train_inputs.shape[0]
batch_size = N_train
alpha = alpha_un / N_train
parser, pred_fun, nllfun, rmse = make_regression_nn_funs(layer_sizes)
N_param = len(parser.vect)
def indexed_loss_fun(w, i_iter):
rs = RandomState((seed, i, i_iter))
idxs = rs.randint(N_train, size=batch_size)
nll = nllfun(w, train_inputs[idxs], train_targets[idxs]) * N_train
return nll
gradfun = grad(indexed_loss_fun)
def callback(x, t, entropy):
results["entropy_per_dpt"].append(entropy / N_train)
results["x_rms"].append(np.sqrt(np.mean(x * x)))
results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
results["log_prior_per_dpt"].append(-neg_log_prior(x) / N_train)
if t % thin != 0 and t != N_iter and t != 0: return
results["iterations"].append(t)
results["train_likelihood"].append(
-nllfun(x, train_inputs, train_targets))
results["tests_likelihood"].append(
-nllfun(x, tests_inputs, tests_targets))
results["train_rmse"].append(
unscale_y(rmse(x, train_inputs, train_targets)))
results["tests_rmse"].append(
unscale_y(rmse(x, tests_inputs, tests_targets)))
results["marg_likelihood"].append(
estimate_marginal_likelihood(results["train_likelihood"][-1],
results["entropy_per_dpt"][-1]))
print "Iteration {0:5} Train lik {1:2.4f} Test lik {2:2.4f}" \
" Marg lik {3:2.4f} Test RMSE {4:2.4f}".format(
t, results["train_likelihood"][-1],
results["tests_likelihood"][-1],
results["marg_likelihood" ][-1],
results["tests_rmse" ][-1])
all_results = []
for i in xrange(N_samples):
results = defaultdict(list)
rs = RandomState((seed, i))
sgd_entropic_damped(gradfun,
np.full(N_param, init_scale),
N_iter,
alpha,
rs,
callback,
width=1)
all_results.append(results)
return all_results
def run():
all_results = []
for width in widths:
init_scale = np.min(0.5 / np.sqrt(width), 0.5 / np.sqrt(784))
def neg_log_prior(w):
D = len(w)
return 0.5 * D * np.log(2 * np.pi) + 0.5 * np.dot(w, w) / init_scale ** 2 + D * np.log(init_scale)
(train_images, train_labels), (tests_images, tests_labels) = load_data_subset(N_train, N_tests)
layer_sizes = [784, 100, 10]
parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
N_param = len(parser.vect)
print "Number of parameters in model: ", N_param
def indexed_loss_fun(w, i_iter):
rs = RandomState((seed, i_iter))
idxs = rs.randint(N_train, size=batch_size)
nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
# nlp = neg_log_prior(w)
return nll # + nlp
gradfun = grad(indexed_loss_fun)
def callback(x, t, entropy):
results["entropy_per_dpt"].append(entropy / N_train)
results["x_rms"].append(np.sqrt(np.mean(x * x)))
results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
results["log_prior_per_dpt"].append(-neg_log_prior(x) / N_train)
if t % thin != 0 and t != N_iter and t != 0:
return
results["iterations"].append(t)
results["train_likelihood"].append(-nllfun(x, train_images, train_labels))
results["tests_likelihood"].append(-nllfun(x, tests_images, tests_labels))
results["tests_error"].append(frac_err(x, tests_images, tests_labels))
results["marg_likelihood"].append(
estimate_marginal_likelihood(
results["log_prior_per_dpt"][-1], results["train_likelihood"][-1], results["entropy_per_dpt"][-1]
)
)
print "Iteration {0:5} Train lik {1:2.4f} Test lik {2:2.4f}" " Marg lik {3:2.4f} Test err {4:2.4f}".format(
t,
results["train_likelihood"][-1],
results["tests_likelihood"][-1],
results["marg_likelihood"][-1],
results["tests_error"][-1],
)
results = defaultdict(list)
rs = RandomState((seed))
sgd_entropic_damped(gradfun, np.full(N_param, init_scale), N_iter, alpha, rs, callback, width=width)
all_results.append(results)
return all_results
def run():
(train_images, train_labels),\
(tests_images, tests_labels) = load_data_subset(N_train, N_tests)
parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
N_param = len(parser.vect)
def indexed_loss_fun(w, i_iter):
rs = RandomState((seed, i, i_iter))
idxs = rs.randint(N_train, size=batch_size)
nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
nlp = neg_log_prior(w)
return nll + nlp
gradfun = grad(indexed_loss_fun)
def callback(x, t, entropy):
results["entropy_per_dpt" ].append(entropy / N_train)
results["x_rms" ].append(np.sqrt(np.mean(x * x)))
results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
results["log_prior_per_dpt" ].append(-neg_log_prior(x) / N_train)
if t % thin != 0 and t != N_iter and t != 0: return
results["iterations" ].append(t)
results["train_likelihood"].append(-nllfun(x, train_images, train_labels))
results["tests_likelihood"].append(-nllfun(x, tests_images, tests_labels))
results["tests_error" ].append(frac_err(x, tests_images, tests_labels))
results["marg_likelihood" ].append(estimate_marginal_likelihood(
results["train_likelihood"][-1], results["entropy_per_dpt"][-1]))
preds[i].append(pred_fun(x, tests_images))
print "Iteration {0:5} Train lik {1:2.4f} Test lik {2:2.4f}" \
" Marg lik {3:2.4f} Test err {4:2.4f}".format(
t, results["train_likelihood"][-1],
results["tests_likelihood"][-1],
results["marg_likelihood" ][-1],
results["tests_error" ][-1])
all_results = []
preds = defaultdict(list)
for i in xrange(N_samples):
results = defaultdict(list)
rs = RandomState((seed, i))
sgd_entropic_damped(gradfun, np.full(N_param, init_scale), N_iter, alpha, rs, callback, width=100)
all_results.append(results)
# Make ensemble prediction by averaging predicted class-conditional probabilities.
ensemble_frac_err = []
ensemble_loglike = []
for t in xrange(len(all_results[0]["iterations"])):
cur_probs = [preds[i][t] for i in xrange(N_samples)]
avg_probs_unn = np.mean(np.exp(cur_probs), axis=0)
avg_probs = avg_probs_unn / np.sum(avg_probs_unn, axis=1, keepdims=True)
ensemble_preds = np.argmax(avg_probs, axis=1)
ensemble_frac_err.append(np.mean(np.argmax(tests_labels, axis=1) != ensemble_preds))
ensemble_loglike.append(np.sum(np.log(avg_probs) * tests_labels)/tests_images.shape[0])
return all_results, ensemble_frac_err, ensemble_loglike
def run():
(train_images, train_labels),\
(tests_images, tests_labels) = load_data_subset(N_train, N_tests)
parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
N_param = len(parser.vect)
print "Number of parameters in model: ", N_param
def indexed_loss_fun(w, i_iter):
rs = RandomState((seed, i_iter))
idxs = rs.randint(N_train, size=batch_size)
nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
nlp = neg_log_prior(w)
return nll + nlp
gradfun = grad(indexed_loss_fun)
def callback(x, t, entropy):
results["entropy_per_dpt"].append(entropy / N_train)
results["x_rms"].append(np.sqrt(np.mean(x * x)))
results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
results["log_prior_per_dpt"].append(-neg_log_prior(x) / N_train)
if t % thin != 0 and t != N_iter and t != 0: return
results["iterations"].append(t)
results["train_likelihood"].append(
-nllfun(x, train_images, train_labels))
results["tests_likelihood"].append(
-nllfun(x, tests_images, tests_labels))
results["tests_error"].append(frac_err(x, tests_images, tests_labels))
results["marg_likelihood"].append(
estimate_marginal_likelihood(results["train_likelihood"][-1],
results["entropy_per_dpt"][-1]))
print "Iteration {0:5} Train lik {1:2.4f} Test lik {2:2.4f}" \
" Marg lik {3:2.4f} Test err {4:2.4f}".format(
t, results["train_likelihood"][-1],
results["tests_likelihood"][-1],
results["marg_likelihood" ][-1],
results["tests_error" ][-1])
all_results = []
preds = defaultdict(list)
for width in widths:
results = defaultdict(list)
rs = RandomState((seed))
sgd_entropic_damped(gradfun,
np.full(N_param, init_scale),
N_iter,
alpha,
rs,
callback,
width=width)
all_results.append(results)
return all_results
def run():
train_inputs, train_targets, tests_inputs, tests_targets, unscale_y = load_boston_housing(train_frac)
N_train = train_inputs.shape[0]
batch_size = N_train
alpha = alpha_un / N_train
parser, pred_fun, nllfun, rmse = make_regression_nn_funs(layer_sizes)
N_param = len(parser.vect)
def indexed_loss_fun(w, i_iter):
rs = RandomState((seed, i, i_iter))
idxs = rs.randint(N_train, size=batch_size)
nll = nllfun(w, train_inputs[idxs], train_targets[idxs]) * N_train
return nll
gradfun = grad(indexed_loss_fun)
def callback(x, t, entropy):
results["entropy_per_dpt"].append(entropy / N_train)
results["x_rms"].append(np.sqrt(np.mean(x * x)))
results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
results["log_prior_per_dpt"].append(-neg_log_prior(x) / N_train)
if t % thin != 0 and t != N_iter and t != 0:
return
results["iterations"].append(t)
results["train_likelihood"].append(-nllfun(x, train_inputs, train_targets))
results["tests_likelihood"].append(-nllfun(x, tests_inputs, tests_targets))
results["train_rmse"].append(unscale_y(rmse(x, train_inputs, train_targets)))
results["tests_rmse"].append(unscale_y(rmse(x, tests_inputs, tests_targets)))
results["marg_likelihood"].append(
estimate_marginal_likelihood(results["train_likelihood"][-1], results["entropy_per_dpt"][-1])
)
print "Iteration {0:5} Train lik {1:2.4f} Test lik {2:2.4f}" " Marg lik {3:2.4f} Test RMSE {4:2.4f}".format(
t,
results["train_likelihood"][-1],
results["tests_likelihood"][-1],
results["marg_likelihood"][-1],
results["tests_rmse"][-1],
)
all_results = []
for i in xrange(N_samples):
results = defaultdict(list)
rs = RandomState((seed, i))
sgd_entropic_damped(gradfun, np.full(N_param, init_scale), N_iter, alpha, rs, callback, width=1)
all_results.append(results)
return all_results
def run():
(train_images, train_labels),\
(tests_images, tests_labels) = load_data_subset(N_train, N_tests)
parser, pred_fun, nllfun, frac_err = make_nn_funs(layer_sizes)
N_param = len(parser.vect)
def indexed_loss_fun(w, i_iter):
rs = RandomState((seed, i, i_iter))
idxs = rs.randint(N_train, size=batch_size)
nll = nllfun(w, train_images[idxs], train_labels[idxs]) * N_train
nlp = neg_log_prior(w)
return nll + nlp
gradfun = grad(indexed_loss_fun)
def callback(x, t, entropy):
results["entropy_per_dpt"].append(entropy / N_train)
results["x_rms"].append(np.sqrt(np.mean(x * x)))
results["minibatch_likelihood"].append(-indexed_loss_fun(x, t))
results["log_prior_per_dpt"].append(-neg_log_prior(x) / N_train)
if t % thin != 0 and t != N_iter and t != 0: return
results["iterations"].append(t)
results["train_likelihood"].append(
-nllfun(x, train_images, train_labels))
results["tests_likelihood"].append(
-nllfun(x, tests_images, tests_labels))
results["tests_error"].append(frac_err(x, tests_images, tests_labels))
results["marg_likelihood"].append(
estimate_marginal_likelihood(results["train_likelihood"][-1],
results["entropy_per_dpt"][-1]))
preds[i].append(pred_fun(x, tests_images))
print "Iteration {0:5} Train lik {1:2.4f} Test lik {2:2.4f}" \
" Marg lik {3:2.4f} Test err {4:2.4f}".format(
t, results["train_likelihood"][-1],
results["tests_likelihood"][-1],
results["marg_likelihood" ][-1],
results["tests_error" ][-1])
all_results = []
preds = defaultdict(list)
for i in xrange(N_samples):
results = defaultdict(list)
rs = RandomState((seed, i))
sgd_entropic_damped(gradfun,
np.full(N_param, init_scale),
N_iter,
alpha,
rs,
callback,
width=100)
all_results.append(results)
# Make ensemble prediction by averaging predicted class-conditional probabilities.
ensemble_frac_err = []
ensemble_loglike = []
for t in xrange(len(all_results[0]["iterations"])):
cur_probs = [preds[i][t] for i in xrange(N_samples)]
avg_probs_unn = np.mean(np.exp(cur_probs), axis=0)
avg_probs = avg_probs_unn / np.sum(
avg_probs_unn, axis=1, keepdims=True)
ensemble_preds = np.argmax(avg_probs, axis=1)
ensemble_frac_err.append(
np.mean(np.argmax(tests_labels, axis=1) != ensemble_preds))
ensemble_loglike.append(
np.sum(np.log(avg_probs) * tests_labels) / tests_images.shape[0])
return all_results, ensemble_frac_err, ensemble_loglike
