def test_svgd_mean_double(self):
n_dims = 50
n_samples = 20
fn_grad, mu, precision_matrix = get_normal(n_dims=n_dims, seed=111)
mu[:4] = np.array([-0.6871, 3.8010, 13.0, 3.0])
sigma = inv(precision_matrix).diagonal()
np.random.seed(17)
theta0 = np.random.normal(0, 1, [n_samples, n_dims])
theta1 = SVGD(objective_grad=fn_grad, eta=0.05,
bandwidth_heuristic='mean').run(theta0, n_iter=1000)
for n_iter in [500, 500]:
theta1 = np.vstack([theta1,
theta1 + np.random.normal(scale=0.1, size=theta1.shape),
theta1 + np.random.normal(scale=0.1, size=theta1.shape)
])
theta1 = SVGD(objective_grad=fn_grad, eta=0.05,
bandwidth_heuristic='mean').run(theta1, n_iter=n_iter)
theta_mu = np.mean(theta1, axis=0)
theta_var = np.var(theta1, axis=0)
print('\nerror:', norm(mu - theta_mu) / mu.shape[0], norm(theta_var - sigma) / sigma.shape[0])
print('n_samples:', theta1.shape[0])
print('svgd_adam:\n', theta_mu, theta_var)
print('ground truth:\n', mu, sigma)
def GMM_test(stepsize=0.01):
A = np.array([[[1.0,0.0],[0.0,1.0]], [[1.0,0.0],[0.0,1.0]]])
mu = np.array([[-5.0,-5.0], [5.0,5.0]])
gmprob = np.array([0.2, 0.8])
model = GMM(mu, A, gmprob)
tik = time()
x0 = np.random.uniform(-4.0, 4.0, [1000, 2])
theta = SVGD().update(x0, model.dlnprob, n_iter=1000, stepsize=stepsize)
tok = time()
print("ground truth: ", mu)
print("svgd: ", np.mean(theta,axis=0))
print("time: ", tok-tik)
plt.scatter(theta.T[0], theta.T[1], s=1)
plt.show()
bias_initializer=tf.keras.initializers.RandomNormal(
mean=0.0, stddev=0.8)))
# train
for _ in range(num_iterations):
grads_list, vars_list, prob_1_x_w_list = [], [], []
for i in range(num_particles):
grads, variables, prob_1_x_w = inference(X_train, y_train,
models[i])
grads_list.append(grads)
vars_list.append(variables)
prob_1_x_w_list.append(prob_1_x_w)
svgd = SVGD(grads_list=grads_list,
vars_list=vars_list,
optimizer=grad_optimizer)
svgd.run()
# evaluation
# train
prob_train_x = tf.reduce_mean(tf.stack(prob_1_x_w_list), axis=0)
classification = prob_train_x.numpy() > 0.5
accuracy = np.sum(classification == y_train) / y_train.shape[0]
print("train accuracy score: {:.2f}".format(accuracy))
# test
prob_test_list = []
for i in range(num_particles):
_, prob_test = predict(X_test, models[i])
prob_test_list.append(prob_test)
return np.array(return_matrix)
if __name__ == '__main__':
filtered_means = []
filtered_covs = []
total_thetas = []
n_iter = 1000
time_series = np.round(np.power(np.sin(np.arange(10)+1),2)*10 + 10)
model = StateSpaceModel()
num_particles = 10
x0 = np.random.normal(0,10,[num_particles,2]).astype(float)
theta = SVGD().update(x0,0,x0,time_series, model.grad_overall, n_iter=n_iter, stepsize=0.01)
total_thetas.append(theta)
#theta = p(x_0|y_0)
filtered_means.append(np.mean(theta,axis=0)[0])
filtered_covs.append(np.var(theta,axis=0)[0])
for t in range(1,len(time_series)):
theta = SVGD().update(theta,t,theta, time_series, model.grad_overall, n_iter=n_iter, stepsize=0.01)
total_thetas.append(theta)
filtered_means.append(np.mean(theta,axis=0)[0])
filtered_covs.append(np.var(theta,axis=0)[0])
return_list = filtered_means + filtered_covs
M = 20 # number of particles
alpha = 1
inner_iteration = 100
outer_iteration = 20
n_iter = inner_iteration * outer_iteration
repeat = 20
results_vanilla_svgd = np.zeros(shape=(repeat, outer_iteration, 2))
for rep in range(repeat):
X_train, X_test, y_train, y_test = train_test_split(X_input,
y_input,
test_size=0.2)
theta0 = np.random.normal(0, np.sqrt(alpha), (M, D))
# EVI
model = BayesianLR(X_train, y_train, 256, alpha)
results_vanilla_svgd[rep, :, :] = SVGD().svgd_updates(
x0=theta0,
lnprob=model.dlnprob,
inner_iteration=inner_iteration,
outer_iteration=outer_iteration,
stepsize=1,
X_test=X_test,
y_test=y_test,
evaluation=model.evaluation)
print('one trial ends')
np.savez('lr_vanilla_svgd',
results_mean=np.mean(results_vanilla_svgd, axis=0),
results_var=np.std(results_vanilla_svgd, axis=0))
import numpy as np
import numpy.matlib as nm
from svgd import SVGD
class MVN:
def __init__(self, mu, A):
self.mu = mu
self.A = A
def dlnprob(self, theta):
return -1 * np.matmul(theta - nm.repmat(self.mu, theta.shape[0], 1),
self.A)
if __name__ == '__main__':
A = np.array([[0.2260, 0.1652], [0.1652, 0.6779]])
mu = np.array([-0.6871, 0.8010])
model = MVN(mu, A)
x0 = np.random.normal(0, 1, [10, 2])
theta = SVGD().update(x0, model.dlnprob, n_iter=500, stepsize=0.01)
#theta = SVGD().update(x0, model.dlnprob,n_iter=2000, method = 'subparticles',m=5, cf=False,stepsize=0.01)
#theta = SVGD().update(x0, model.dlnprob,n_iter=2000, method = 'subparticles',m=5, cf=True,stepsize=0.01)
#theta = SVGD().update(x0, model.dlnprob, n_iter=2000,method = 'inducedPoints',adver=False,stepsize=0.01)
#theta = SVGD().update(x0, model.dlnprob,n_iter=500, method = 'inducedPoints',adver=True, m=5, adverMaxIter=5,stepsize=0.01)
print("ground truth: ", mu)
print("svgd: ", np.mean(theta, axis=0))
if __name__ == '__main__':
num_iterations = 5000
num_samples = 100
weights = [1]
mus = [np.array([0, 0])]
sigmas = [np.eye(2)]
init_particles = sample_gmm(num_samples, [1.], [np.array([-7, -7])],
[np.eye(2)])
true_samples = sample_gmm(num_samples, weights, mus, sigmas)
plotfunc = get_plotfunc(true_samples)
gld = gmm_gld(weights, mus, sigmas)
svgd = SVGD(gld=gld)
log_lik_func = log_lik_gmm(weights, mus, sigmas)
report_metrics = metrics(log_lik_func)
particles = svgd.do_svgd_iterations_optimized(
init_particles=init_particles,
num_iterations=num_iterations,
learning_rate=1e-2,
plotfunc=plotfunc,
metrics=report_metrics)
np.array([2])]
sigmas = [np.eye(1),
np.eye(1)]
toy_init_particles = sample_gmm(num_samples,
[1.],
[np.array([-10])],
[np.eye(1)])
true_samples = sample_gmm(num_samples, weights, mus, sigmas)
plotfunc = get_plotfunc(true_samples)
toy_score = gmm_gld(weights, mus, sigmas)
svgd = SVGD(gld=toy_score)
particles= svgd.do_svgd_iterations_optimized(init_particles=toy_init_particles,
num_iterations=num_iterations,
learning_rate=0.1,
plotfunc=plotfunc)
x = np.linspace(-15, 10, 1000)
fig, ax = plt.subplots(1,1)
sns.kdeplot(true_samples.flatten(), ax=ax, label='Sampler')
sns.kdeplot(particles.flatten(), ax=ax, label='SVGD')
ax.legend()
animate,
int(theta_hist.shape[1] / 25),
repeat=False,
blit=True)
plt.show()
if __name__ == '__main__':
mu1 = -4
var1 = 0.5
mu2 = 2
var2 = 4
model = Bimodal(mu1, var1, mu2, var2)
x0 = np.random.normal(0, 1, [200, 1])
theta, theta_hist = SVGD().update(x0,
model.dlnprob,
n_iter=3000,
stepsize=0.01,
debug=True)
# print("ground truth: mu = {} var = {}".format(mu, var))
mu = np.mean(theta)
var = np.std(theta)**2
print("svgd: mu = {} var = {}".format(round(mu, 2), round(var, 2)))
# plot_results(mu1, var1, mu2, var2, theta)
animate_results(mu1, var1, mu2, var2, theta_hist, n_bins=30)
D = X_train.shape[1]
# initialization
M = 100 # number of particles
alpha = 1e-2
inner_iteration = 25
outer_iteration = 20
n_iter = inner_iteration * outer_iteration
repeat = 5
results_SVGD = np.zeros(shape=(repeat, outer_iteration, 2))
for rep in range(repeat):
theta0 = np.random.normal(0, np.sqrt(alpha), (M, D))
# SVGD
model = BayesianLR(X_train, y_train, 256, alpha) # batchsize = 32
results_SVGD[rep, :, :] = SVGD().update(
x0=theta0,
lnprob=model.dlnprob,
inner_iteration=inner_iteration,
outer_iteration=outer_iteration,
stepsize=0.01,
X_test=X_test,
y_test=y_test,
evaluation=model.evaluation,
debug=True)
np.savez('lr_svgd',
results_mean=np.mean(results_SVGD, axis=0),
results_var=np.std(results_SVGD, axis=0))
# split the dataset into training and testing
X_train, X_test, y_train, y_test = train_test_split(X_input,
y_input,
test_size=0.2,
random_state=42)
a0, b0 = 1, 0.01 #hyper-parameters
model = BayesianLR(X_train, y_train, 100, a0, b0) # batchsize = 100
# initialization
M = 100 # number of particles
theta0 = np.zeros([M, D])
alpha0 = np.random.gamma(a0, b0, M)
for i in range(M):
theta0[i, :] = np.hstack([
np.random.normal(0, np.sqrt(1 / alpha0[i]), d),
np.log(alpha0[i])
])
theta = SVGD().update(x0=theta0,
lnprob=model.dlnprob,
bandwidth=-1,
n_iter=6000,
stepsize=0.05,
alpha=0.9,
debug=True)
print '[accuracy, log-likelihood]'
print model.evaluation(theta, X_test, y_test)
y_test = data[dataName]['t'][0,0][ data[dataName]['test'][0,0][splitIdx-1,:]-1, : ]
y_train = y_train.ravel(); y_test = y_test.ravel();
y_train[y_train == -1] = 0; y_test[y_test == -1] = 0;
######################
X_train = np.hstack([X_train, np.ones([X_train.shape[0], 1])])
X_test = np.hstack([X_test, np.ones([X_test.shape[0], 1])])
D = X_train.shape[1]
app = 0
while True:
app += 1; filename = 'res_%s_lrSimp_svgd_%s_%d.txt'%(dataName, optType, app)
try: fid = open(filename, 'r')
except IOError: break
fid.close()
with open(filename, 'w') as fid:
fid.write('alpha=%.2e; '%alpha + 'M=%d; '%M + 'batchsize=%d; '%batchsize + 'stepsize=%.2e; '%stepsize + 'optAlpha=%.2e; '%optAlpha + 'bandwidth=%s;'%str(bandwidth) + '\n')
fid.write('iter accuracy log-llh\n')
# initialization
theta = np.random.normal(0, np.sqrt(alpha), [M, D])
model = BayesianLR_simp(X_train, y_train, batchsize=batchsize, alpha=alpha) # batchsize = 100
svgdobj = SVGD(theta, model.dlnprob, bandwidth, stepsize, optAlpha, optType)
with open(filename, 'a') as fid:
for it in range(n_round):
theta = svgdobj.update(n_iter) #, debug=True)
res = model.evaluation(theta, X_test, y_test)
fid.write('%4d %.6f %.6e\n'%((it+1)*n_iter, res[0], res[1]))
# hyper-parameters
num_particles = 100 # number of ensembles (SVGD particles)
num_iterations = 2000 # number of training iterations
learning_rate = 0.01
seed = 0
# random seeds
np.random.seed(seed)
model = GaussianMixtureModel()
with Time("Get initial particles"):
initial_xs = np.array(np.random.normal(-10, 1, (100, 1)), dtype=np.float32)
with Time("training & Get last particles"):
final_xs = SVGD().update(initial_xs, model.dlnprob, n_iter=num_iterations, stepsize=learning_rate)
initial_xs, final_xs = initial_xs.reshape(-1), final_xs.reshape(-1)
def plot():
fig = plt.figure(figsize=(5, 5))
ax = fig.add_subplot(111)
x_grid = np.linspace(-15, 15, 200)
initial_density = gaussian_kde(initial_xs)
ax.plot(x_grid, initial_density(x_grid), color='green', label='0th iteration')
ax.scatter(initial_xs, np.zeros_like(initial_xs), color='green')
final_density = gaussian_kde(final_xs)
ax.plot(x_grid, final_density(x_grid), color='red', label='{}th iteration'.format(num_iterations))
ax.scatter(final_xs, np.zeros_like(final_xs), color='red')
import numpy as np
import numpy.matlib as nm
from svgd import SVGD
class MVN:
def __init__(self, mu, A):
self.mu = mu
self.A = A
def dlnprob(self, theta):
return -1 * np.matmul(theta - nm.repmat(self.mu, theta.shape[0], 1),
self.A)
if __name__ == '__main__':
A = np.array([[0.2260, 0.1652], [0.1652, 0.6779]])
mu = np.array([-0.6871, 0.8010])
model = MVN(mu, A)
x0 = np.random.normal(0, 1, [10, 2])
theta = SVGD().update(x0,
model.dlnprob,
n_iter=10000,
stepsize=0.01,
debug=True)
print("ground truth: ", mu)
print("svgd: ", np.mean(theta, axis=0))
def __init__(self, mu, A):
self.mu = mu
self.A = A
def dlnprob(self, theta,t):
x= (time_series[t]-theta)/self.A
return x
if __name__ == '__main__':
filtered_means = []
filtered_covs = []
A = np.array([1])
mu = np.array([1])
model = MVN(mu, A)
time_series = time_series[:100]
x0 = np.random.normal(0,1, [100,1]);
theta = SVGD().update(x0,0, model.dlnprob, n_iter=1000, stepsize=0.01)
filtered_means.append(np.mean(theta,axis=0)[0])
filtered_covs.append(np.var(theta,axis=0)[0])
for t in range(1,len(time_series)):
theta = transition(theta)
theta = SVGD().update(theta,t, model.dlnprob, n_iter=1000, stepsize=0.01)
filtered_means.append(np.mean(theta,axis=0)[0])
filtered_covs.append(np.var(theta,axis=0)[0])
print "svgd: ", filtered_means
print "\n"
print "svgd var", filtered_covs
print "\n"
print "time series", time_series
plt.plot(range(len(time_series)),time_series,color='blue')
plt.fill_between(range(len(time_series)), filtered_means + np.sqrt(filtered_covs),filtered_means - np.sqrt(filtered_covs))