def callback(bnn_params, dsc_params, iter, gen_gradient, dsc_gradient):
# Sample functions from priors f ~ p(f)
n_samples = 3
plot_inputs = np.linspace(-8, 8, num=100).reshape(100, 1)
std_norm_param = init_var_params(bnn_layer_sizes,
scale_mean=0,
scale=1)
f_bnn_gpp = sample_bnn(bnn_params, plot_inputs, n_samples,
bnn_layer_sizes)
f_gp = sample_gpp(plot_inputs, n_samples)
f_bnn = sample_bnn(std_norm_param, plot_inputs, n_samples,
bnn_layer_sizes)
# Plot samples of functions from the bnn and gp priors.
if plot_during:
for axes in ax:
axes.cla()
# ax.plot(x.ravel(), y.ravel(), 'ko')
ax[0].plot(plot_inputs, f_gp, color='green')
ax[1].plot(plot_inputs, f_bnn_gpp, color='red')
ax[2].plot(plot_inputs, f_bnn, color='blue')
plt.draw()
plt.pause(1.0 / 40.0)
print("Iteration {} ".format(iter))
def callback(bnn_params, dsc_params, iter, gen_gradient, dsc_gradient):
# Sample functions from priors f ~ p(f)
n_samples, ndata = 3, 500
plot_inputs = np.linspace(-8, 8, num=ndata).reshape(ndata, 1)
std_norm_param = init_var_params(bnn_arch, scale_mean=0, scale=1)
f_bnn_gpp = sample_bnn(bnn_params, plot_inputs, n_samples, bnn_arch,
act)
f_gp = sample_gpp(plot_inputs, n_samples, ker)
f_bnn = sample_bnn(std_norm_param, plot_inputs, n_samples, bnn_arch,
act)
if plot_during:
for axes in ax:
axes.cla()
# ax.plot(x.ravel(), y.ravel(), 'ko')
ax[0].plot(plot_inputs, f_gp.T, color='green')
ax[1].plot(plot_inputs, f_bnn_gpp.T, color='red')
ax[2].plot(plot_inputs, f_bnn.T, color='blue')
#ax[0].set_ylim([-3,3])
#ax[1].set_ylim([-3,3])
#ax[2].set_ylim([-3,3])
plt.draw()
plt.pause(1.0 / 40.0)
print("Iteration {} ".format(iter))
def callback_kl(prior_params, iter, g):
n_samples, n_data = 3, 500
plot_inputs = np.linspace(-8, 8, num=n_data).reshape(1, n_data)
f_bnn_gpp = sample_bnn(plot_inputs, n_samples, arch, act, prior_params)
f_bnn = sample_bnn(plot_inputs, arch, act, n_samples)
f_gp = sample_gpp(plot_inputs, n_samples)
# Plot samples of functions from the bnn and gp priors.
if plot_during:
for axes in ax:
axes.cla() # clear plots
# ax.plot(x.ravel(), y.ravel(), 'ko')
ax[0].plot(plot_inputs, f_gp, color='green')
ax[1].plot(plot_inputs, f_bnn_gpp, color='red')
ax[2].plot(plot_inputs, f_bnn, color='blue')
#ax[0].set_ylim([-5, 5])
#ax[1].set_ylim([-5, 5])
#ax[2].set_ylim([-5, 5])
plt.draw()
plt.pause(1.0 / 40.0)
fs = (f_gp, f_bnn, f_bnn_gpp)
kl_val = kl(prior_params, iter)
if save_during:
title = " iter {} kl {:5}".format(iter, kl_val)
plotting.plot_priors(plot_inputs, fs,
os.path.join(save_dir, title))
print("Iteration {} KL {} ".format(iter, kl_val))
def gan_objective(prior_params,
d_params,
n_data,
n_samples,
bnn_layer_sizes,
act,
d_act='tanh'):
'''estimates V(G, D) = E_p_gp[D(f)] - E_pbnn[D(f)]]'''
x = sample_inputs('uniform', n_data, (-10, 10))
fbnns = sample_bnn(prior_params, x, n_samples, bnn_layer_sizes,
act) # [nf, nd]
fgps = sample_gpp(x, n_samples, 'rbf') # sample f ~ P_gp(f)
D_fbnns = nn_predict(d_params, fbnns, d_act)
D_fgps = nn_predict(d_params, fgps, d_act)
print(D_fbnns.shape)
eps = np.random.uniform()
f = eps * fgps + (1 - eps) * fbnns
def D(function):
return nn_predict(d_params, function, 'tanh')
J = jacobian(D)(f)
print(J.shape)
g = elementwise_grad(D)(f)
print(g.shape)
pen = 10 * (norm(g, ord=2, axis=1) - 1)**2
return np.mean(D_fgps - D_fbnns + pen)
def plot_save_priors_fdensity(bnn_arch=[1, 20, 1], bnn_act='rbf'):
plot_inputs = np.linspace(-10, 10, num=500)[:, None]
std_norm_param = init_var_params(bnn_arch, scale_mean=0, scale=1)
f_bnn = sample_bnn(std_norm_param, plot_inputs, 25, bnn_arch, bnn_act)
f_gps = sample_gpp(plot_inputs, 25)
plot_density(f_bnn)
plot_density(f_gps, plot="gpp")
pass
def plot_save_priors_functions(bnn_arch=[1, 20, 1], bnn_act='rbf'):
plot_inputs = np.linspace(-10, 10, num=500)[:, None]
std_norm_param = init_var_params(bnn_arch, scale_mean=0, scale=1)
f_bnn = sample_bnn(std_norm_param, plot_inputs, 3, bnn_arch, bnn_act)
f_gps = sample_gpp(plot_inputs, 3)
plot_samples(plot_inputs, f_bnn.T)
plot_samples(plot_inputs, f_gps.T, plot="gpp")
pass
def callback(params, t, g):
y = sample_gpp(x, 1, ker)
#y=sample_function(x,1)
preds = hyper_predict(params, x, y, nn_arch, act) #[1,nd]
if plot: p.plot_iter(ax, x, x, y, preds)
cd = np.cov(y.ravel()) - np.cov(preds.ravel())
print("ITER {} | OBJ {} COV DIFF {}".format(t, objective(params, t),
cd))
def plot_samples(params, x, n_samples, layer_sizes, act, ker, save=None):
" plots samples of "
ws = sample_normal(params, n_samples)
fnn = bnn_predict(ws, x, layer_sizes, act)[:, :, 0] # [ns, nd]
fgp = sample_gpp(x, n_samples, ker)
# functions from a standard normal bnn prior
wz = sample_normal((np.zeros(ws.shape[1]), np.ones(ws.shape[1])), n_samples)
f = bnn_predict(wz, x, layer_sizes, act)[:, :, 0]
p.plot_priors(x, (fgp.T, f.T, fnn.T), save)
def plot_samples(params, x, layer_sizes, n_samples=1, act='tanh', save=None):
#ws = sample_normal(params, n_samples)
ws = sample_full_normal(params, n_samples)
fnn = bnn_predict(ws, x, layer_sizes, act)[:, :, 0] # [ns, nd]
fgp = sample_gpp(x, n_samples)
fig = plt.figure(figsize=(12, 8), facecolor='white')
ax = fig.add_subplot(111, frameon=False)
bx = fig.add_subplot(111, frameon=False)
bx.plot(x.ravel(), fgp.T, color='green')
ax.plot(x.ravel(), fnn.T, color='red')
if save is not None: plt.savefig(save)
plt.show()
def callback(params, iter, g):
n_samples = 3
plot_inputs = np.linspace(-5, 5, num=100)
f_bnn = sample_bnn(params, plot_inputs[:, None], n_samples, arch, act)
fgp = sample_gpp(plot_inputs[:, None], n_samples, kernel=ker)
for axes in ax: axes.cla()
# ax.plot(x.ravel(), y.ravel(), 'ko')
ax[0].plot(plot_inputs, fgp.T, color='green')
ax[1].plot(plot_inputs, f_bnn.T, color='red')
#ax[0].set_ylim([-5, 5])
#ax[1].set_ylim([-5, 5])
plt.draw()
plt.pause(1.0/40.0)
print("Iteration {} KL {} ".format(iter, kl(params, iter)))
def outer_objective(params_prior, n_data, n_samples, layer_sizes, act=rbf):
# x = np.random.uniform(low=0, high=10, size=(n_data, 1))
x = np.linspace(0, 10, num=n_data).reshape(n_data, 1)
fgp = sample_gpp(x, n_samples=n_samples, kernel='rbf')
def objective_inner(params_q, t):
return -np.mean(NN_likelihood(x, fgp, params_q, layer_sizes, act))
op_params_q = adam(grad(objective_inner), init_var_params(layer_sizes),
step_size=0.1, num_iters=200)
wq = sample_weights(op_params_q, n_samples)
log_pnn = NN_likelihood(x, fgp, op_params_q, layer_sizes, act)
log_p = diag_gaussian_log_density(wq, params_prior[0], params_prior[1])
log_q = diag_gaussian_log_density(wq, op_params_q[0], op_params_q[1])
return -np.mean(log_pnn+log_p-log_q)
def outer_objective(params_prior, n_data, n_samples, layer_sizes, act, ker='rbf'):
# x = np.random.uniform(low=0, high=10, size=(n_data, 1))
x = np.linspace(0, 10, num=n_data).reshape(n_data, 1)
fgp = sample_gpp(x, n_samples=n_samples, kernel=ker)
def objective_inner(params_q, t):
return -np.mean(NN_likelihood(x, fgp, params_q, layer_sizes, act, n_samples))
op_params_q = adam(grad(objective_inner), init_var_params(layer_sizes),
step_size=0.1, num_iters=100)
wq = sample_weights(op_params_q, n_samples) # [ns, nw]
pnn = NN_likelihood(x, fgp, op_params_q, layer_sizes, act, n_samples)
p = diag_gaussian_density(wq, params_prior[0], params_prior[1])
q = diag_gaussian_density(wq, op_params_q[0], op_params_q[1])
iwae = p*pnn/q
# print(iwae.shape)
return np.mean(np.log(iwae))
def sample_gps(nfuncs, ndata, ker):
xs = sample_inputs(nfuncs, ndata)
fgp = [sample_gpp(x, 1, kernel=ker) for x in xs]
return xs, np.concatenate(fgp, axis=0)
if __name__ == '__main__':
n_data, n_samples, arch = 10, 1, [1, 20, 20, 1]
f, ax = plt.subplots(2, sharex=True)
plt.ion()
plt.show(block=False)
def kl(prior_params,t):
return outer_objective(prior_params, n_data, n_samples, arch)
def callback(params, iter, g):
n_samples = 3
f_bnn_gpp = sample_bnn(params, plot_inputs[:, None], n_samples, arch, rbf)
f_gp = sample_gpp(plot_inputs[:, None], n_samples)
_min
for axes in ax: axes.cla()
# ax.plot(x.ravel(), y.ravel(), 'ko')
ax[0].plot(plot_inputs, f_gp.T, color='green')
ax[1].plot(plot_inputs, f_bnn_gpp.T, color='red')
#ax[0].set_ylim([-5, 5])
#ax[1].set_ylim([-5, 5])
plt.draw()
plt.pause(1.0/40.0)
print("Iteration {} KL {} ".format(iter, kl(params, iter)))
prior_params = adam(grad(kl), init_var_params(arch),
step_size=0.05, num_iters=100, callback=callback)
n_functions = 500
nn_arch = [1, 50, 1]
_, num_weights = shapes_and_num(nn_arch)
hyper_arch = [n_data, 20, 20, num_weights]
act = 'rbf'
ker = 'rbf'
save_name = 'exp' + str(n_data) + 'nf-' + str(
n_functions) + "-" + act + ker
x = np.linspace(-10, 10, n_data).reshape(n_data, 1)
x = np.random.uniform(-10, 10, n_data)[:, None]
x = np.sort(x, 0)
xt = np.linspace(-10, 10, n_data_test).reshape(n_data_test, 1)
ys = sample_gpp(x, n_samples=n_functions, kernel=ker) # [ns, nd]
#ys = sample_function(x, n_functions, n_data)
plot = True
if plot: fig, ax = setup_plot()
def objective(params, t):
return hyper_loss(params, x, ys, nn_arch, act)
def callback(params, t, g):
y = sample_gpp(x, 1, ker)
#y=sample_function(x,1)
preds = hyper_predict(params, x, y, nn_arch, act) #[1,nd]
if plot: p.plot_iter(ax, x, x, y, preds)
cd = np.cov(y.ravel()) - np.cov(preds.ravel())
print("ITER {} | OBJ {} COV DIFF {}".format(t, objective(params, t),