def callback(params):
print("Log likelihood {}".format(-objective(params)))
plt.cla()
# Show posterior marginals.
plot_xs = np.reshape(np.linspace(-7, 7, 300), (300,1))
pred_mean, pred_cov = predict(params, X, y, plot_xs)
marg_std = np.sqrt(np.diag(pred_cov))
ax.plot(plot_xs, pred_mean, 'b')
ax.fill(np.concatenate([plot_xs, plot_xs[::-1]]),
np.concatenate([pred_mean - 1.96 * marg_std,
(pred_mean + 1.96 * marg_std)[::-1]]),
alpha=.15, fc='Blue', ec='None')
# Show samples from posterior.
rs = npr.RandomState(0)
sampled_funcs = rs.multivariate_normal(pred_mean, pred_cov, size=10)
ax.plot(plot_xs, sampled_funcs.T)
ax.plot(X, y, 'kx')
ax.set_ylim([-1.5, 1.5])
ax.set_xticks([])
ax.set_yticks([])
plt.draw()
plt.pause(1.0/60.0)
def make_pinwheel_data(num_spokes=5, points_per_spoke=40, rate=1.0, noise_std=0.005):
"""Make synthetic data in the shape of a pinwheel."""
spoke_angles = np.linspace(0, 2 * np.pi, num_spokes + 1)[:-1]
rs = npr.RandomState(0)
x = np.linspace(0.1, 1, points_per_spoke)
xs = np.concatenate([x * np.cos(angle + x * rate) + noise_std * rs.randn(len(x))
for angle in spoke_angles])
ys = np.concatenate([x * np.sin(angle + x * rate) + noise_std * rs.randn(len(x))
for angle in spoke_angles])
return np.concatenate([np.expand_dims(xs, 1), np.expand_dims(ys,1)], axis=1)
def make_pinwheel_data(num_classes, num_per_class, rate=2.0, noise_std=0.001):
spoke_angles = np.linspace(0, 2*np.pi, num_classes+1)[:-1]
rs = npr.RandomState(0)
x = np.linspace(0.1, 1, num_per_class)
xs = np.concatenate([rate *x * np.cos(angle + x * rate) + noise_std * rs.randn(num_per_class)
for angle in spoke_angles])
ys = np.concatenate([rate *x * np.sin(angle + x * rate) + noise_std * rs.randn(num_per_class)
for angle in spoke_angles])
return np.concatenate([np.expand_dims(xs, 1), np.expand_dims(ys,1)], axis=1)
def build_toy_dataset(n_data=40, noise_std=0.1):
D = 1
rs = npr.RandomState(0)
inputs = np.concatenate([np.linspace(0, 2, num=n_data/2),
np.linspace(6, 8, num=n_data/2)])
inputs1 = np.concatenate([np.linspace(0, 2, num=n_data/2),
np.linspace(6, 8, num=n_data/2)])
targets = np.cos(inputs) + rs.randn(n_data) * noise_std
inputs = (inputs - 4.0) / 4.0
inputs1 = (inputs1 - 4.0) / 4.0
inputs = inputs.reshape((len(inputs), D))
inputs1 = inputs.reshape((len(inputs), D))
imputfull = np.concatenate((inputs , inputs1),axis=1)
targets = targets.reshape((len(targets), D))
return inputs, targets
def plot_gmm(params, ax, num_points=100):
angles = np.expand_dims(np.linspace(0, 2*np.pi, num_points), 1)
xs, ys = np.cos(angles), np.sin(angles)
circle_pts = np.concatenate([xs, ys], axis=1) * 2.0
for log_proportion, mean, chol in zip(*unpack_params(params)):
cur_pts = mean + np.dot(circle_pts, chol)
ax.plot(cur_pts[:, 0], cur_pts[:, 1], '-')
def build_toy_dataset(D=1, n_data=20, noise_std=0.1):
rs = npr.RandomState(0)
inputs = np.concatenate([np.linspace(0, 3, num=n_data/2),
np.linspace(6, 8, num=n_data/2)])
targets = (np.cos(inputs) + rs.randn(n_data) * noise_std) / 2.0
inputs = (inputs - 4.0) / 2.0
inputs = inputs.reshape((len(inputs), D))
return inputs, targets
def log_marginal_likelihood(params, data):
cluster_lls = []
for log_proportion, mean, chol in zip(*unpack_params(params)):
cov = np.dot(chol.T, chol) + 0.000001 * np.eye(D)
cluster_log_likelihood = log_proportion + mvn.logpdf(data, mean, cov)
cluster_lls.append(np.expand_dims(cluster_log_likelihood, axis=0))
cluster_lls = np.concatenate(cluster_lls, axis=0)
return np.sum(logsumexp(cluster_lls, axis=0))
def plot_isocontours(ax, func, xlimits=[-2, 2], ylimits=[-4, 2], numticks=101):
x = np.linspace(*xlimits, num=numticks)
y = np.linspace(*ylimits, num=numticks)
X, Y = np.meshgrid(x, y)
zs = func(np.concatenate([np.atleast_2d(X.ravel()), np.atleast_2d(Y.ravel())]).T)
Z = zs.reshape(X.shape)
plt.contour(X, Y, Z)
ax.set_yticks([])
ax.set_xticks([])
def plot_matrix(ax, r, g, b, t, render=False):
if ax:
plt.cla()
ax.imshow(np.concatenate((r[...,np.newaxis], g[...,np.newaxis], b[...,np.newaxis]), axis=2))
ax.set_xticks([])
ax.set_yticks([])
plt.draw()
if render:
plt.savefig('step{0:03d}.png'.format(t), bbox_inches='tight')
plt.pause(0.001)
def unpack_params(params):
"""Unpacks parameter vector into the proportions, means and covariances
of each mixture component. The covariance matrices are parametrized by
their Cholesky decompositions."""
log_proportions = parser.get(params, 'log proportions')
normalized_log_proportions = log_proportions - logsumexp(log_proportions)
means = parser.get(params, 'means')
lower_tris = np.tril(parser.get(params, 'lower triangles'), k=-1)
diag_chols = np.exp( parser.get(params, 'log diagonals'))
chols = []
for lower_tri, diag in zip(lower_tris, diag_chols):
chols.append(np.expand_dims(lower_tri + np.diag(diag), 0))
chols = np.concatenate(chols, axis=0)
return normalized_log_proportions, means, chols
zs = func(np.concatenate([np.atleast_2d(X.ravel()), np.atleast_2d(Y.ravel())]).T)
Z = zs.reshape(X.shape)
plt.contour(X, Y, Z)
ax.set_yticks([])
ax.set_xticks([])
# Set up figure.
fig = plt.figure(figsize=(8, 8), facecolor="white")
ax = fig.add_subplot(111, frameon=False)
plt.ion()
plt.show(block=False)
def callback(params, t, g):
print("Iteration {} lower bound {}".format(t, -objective(params, t)))
plt.cla()
target_distribution = lambda x: np.exp(log_posterior(x, t))
plot_isocontours(ax, target_distribution)
mean, log_std = unpack_params(params)
variational_contour = lambda x: mvn.pdf(x, mean, np.diag(np.exp(2 * log_std)))
plot_isocontours(ax, variational_contour)
plt.draw()
plt.pause(1.0 / 30.0)
print("Optimizing variational parameters...")
init_mean = -1 * np.ones(D)
init_log_std = -5 * np.ones(D)
init_var_params = np.concatenate([init_mean, init_log_std])
variational_params = adam(gradient, init_var_params, step_size=0.1, num_iters=2000, callback=callback)
def activations(weights, *args):
cat_state = np.concatenate(args + (np.ones((args[0].shape[0],1)),), axis=1)
return np.dot(cat_state, weights)