def Bound2(phi_0, phi_1, phi_2, sigma_noise, K_mm, mean_y):
# Preliminary Bound
beta = 1 / tf.square(sigma_noise)
bound = 0
N = h.get_dim(mean_y, 0)
M = h.get_dim(K_mm, 0)
W_inv_part = beta * phi_2 + K_mm
global phi_200
phi_200 = tf.matrix_solve(W_inv_part, tf.transpose(phi_1))
W = beta * np.eye(N) - tf.square(beta) * h.Mul(
phi_1, tf.matrix_solve(W_inv_part, tf.transpose(phi_1)))
# Computations
bound += N * tf.log(beta)
bound += h.log_det(K_mm + 1e-3 * np.eye(M))
bound -= h.Mul(tf.transpose(mean_y), W, mean_y)
global matrix_determinant
matrix_determinant = tf.ones(
1
) #h.log_det(W_inv_part+1e2*np.eye(M))#-1e-40*tf.exp(h.log_det(W_inv_part))
bound -= h.log_det(W_inv_part +
1e-3 * tf.reduce_mean(W_inv_part) * np.eye(M))
bound -= beta * phi_0
bound += beta * tf.trace(tf.cholesky_solve(tf.cholesky(K_mm), phi_2))
bound = bound * 0.5
return bound
def pred(X, X_m_1, mu, len_sc_1, noise_1):
Kmm = h.tf_SE_K(X_m_1, X_m_1, len_sc_1, noise_1)
Knm = h.tf_SE_K(X, X_m_1, len_sc_1, noise_1)
posterior_mean = h.Mul(Knm, tf.matrix_solve(Kmm, mu))
K_nn = h.tf_SE_K(X, X, len_sc_1, noise_1)
full_cov = K_nn - h.Mul(Knm, tf.matrix_solve(Kmm, tf.transpose(Knm)))
posterior_cov = tf.diag_part(full_cov)
return posterior_mean, tf.reshape(posterior_cov, [N, 1]), full_cov
def predict2():
# predicitions
cov=h.Mul(K_mm_2,tf.matrix_inverse(K_mm_2+K_mnnm_2/tf.square(sigma_2)),K_mm_2)
cov_chol=tf.cholesky(cov)
mu=h.Mul(K_mm_2,tf.cholesky_solve(cov_chol,K_mn_2),Ytr)/tf.square(sigma_2)
mean=h.Mul(K_nm_2,tf.matrix_solve(K_mm_1,mu))
variance=K_nn_2-h.Mul(K_nm_2,h.safe_chol(K_mm_2,tf.transpose(K_nm_2)))
var_terms=2*tf.sqrt(tf.reshape(tf.diag_part(variance)+tf.square(sigma_2),[N,1]))
return mean, var_terms
def predict(K_mn,sigma,K_mm,K_nn):
# predicitions
K_nm=tf.transpose(K_mn)
Sig_Inv=1e-1*np.eye(M)+K_mm+K_mnnm_2/tf.square(sigma)
mu_post=h.Mul(tf.matrix_solve(Sig_Inv,K_mn),Ytr)/tf.square(sigma)
mean=h.Mul(K_nm,mu_post)
variance=K_nn-h.Mul(K_nm,h.safe_chol(K_mm,K_mn))+h.Mul(K_nm,tf.matrix_solve(Sig_Inv,K_mn))
var_terms=2*tf.sqrt(tf.reshape(tf.diag_part(variance)+tf.square(sigma),[N,1]))
return mean, var_terms
def test_h_Mul(self):
with self.test_session():
A1=np.array([[1,2],[3,4]]); A=tf.constant(A1,shape=[2,2],dtype=tf.float32)
B1=np.array([[1,-2],[-5,4]]); B=tf.constant(B1,shape=[2,2],dtype=tf.float32)
C1=np.array([[7,1],[5,4]]); C=tf.constant(C1,shape=[2,2],dtype=tf.float32)
Z1=np.matmul(np.matmul(A1,B1),np.matmul(B1,C1));
print(Z1)
self.assertAllEqual(h.Mul(A,B,B,C).eval(),Z1)
def Bound1(y, S, Kmm, Knm, Tr_Knn, sigma):
#matrices to be used
Kmm_chol = tf.cholesky(Kmm)
sig_2 = tf.square(sigma)
N = h.get_dim(y, 0)
Q_nn = h.Mul(Knm, tf.cholesky_solve(Kmm_chol, tf.transpose(Knm)))
Q_I_chol = tf.cholesky(sig_2 * np.eye(N) + Q_nn)
bound = -0.5 * (Tr_Knn - Q_nn) / sig_2
bound += h.multivariate_normal(y, tf.zeros([N, 1], dtype=tf.float32),
Q_I_chol)
bound -= 0.5 * tf.reduce_sum(S) / sig_2 + 0.1 * 0.5 * tf.reduce_sum(
tf.log(S))
return bound
X_m = tf.Variable(tf.random_uniform([M, 1], minval=0, maxval=15), name='X_m')
sigma = tf.Variable(tf.ones([1, 1]), dtype=tf.float32, name='sigma')
noise = tf.Variable(tf.ones([1, 1]), dtype=tf.float32, name='sigma')
len_sc = tf.Variable(tf.ones([1, 1]), dtype=tf.float32, name='len_sc')
s.run(tf.initialize_all_variables())
print(s.run(X_m))
X_j_m = h.jitter(X_m)
K_nm = h.tf_SE_K(Xtr, X_m, len_sc, noise)
K_mm = h.tf_SE_K(X_m, X_m, len_sc, noise) + h.tol * np.eye(M, M)
K_nn = h.tf_SE_K(Xtr, Xtr, len_sc, noise)
K_mn = h.tf.transpose(K_nm)
Sig_Inv = K_mm + h.Mul(K_mn, K_nm) / (tf.square(sigma) + h.tol)
mu_post = h.Mul(h.safe_chol(Sig_Inv, K_mn), Ytr) / (tf.square(sigma) + h.tol)
mean = h.Mul(K_nm, mu_post)
variance = K_nn - h.Mul(K_nm, h.safe_chol(K_mm, K_mn)) + h.Mul(
K_nm, h.safe_chol(Sig_Inv, K_mn))
var_terms = 2 * tf.sqrt(
tf.reshape(tf.diag_part(variance) + tf.square(sigma), [N, 1]))
F_v = h.F_bound_v1(Ytr, K_mm, K_nm, K_nn, sigma)
tf.scalar_summary("F_v", F_v)
opt = tf.train.AdamOptimizer(0.5)
train = opt.minimize(-1 * F_v)
init = tf.initialize_all_variables()
s.run(init)
Tr_Knn, K_nm_2, K_mnnm_2 = KS.build_psi_stats_rbf(X_m_2,tf.square(noise_2), len_sc_2, h_mu, h_S) K_mm_2=h.tf_SE_K(X_m_2,X_m_2,len_sc_2,noise_2) K_nn_2=h.tf_SE_K(h_mu,h_mu,len_sc_2,noise_2) K_mn_2=h.tf.transpose(K_nm_2) ''' opti_mu1= opti_Sig1= opti_mu2= opti_sig2= ''' a_mn=tf.matrix_solve(K_mm_1,K_mn_1) a_nm=tf.transpose(a_mn) sig_inv=tf.matrix_inverse(K_mm_1)+h.Mul(a_mn,a_nm)/tf.square(sigma_1) mu1=h.Mul(tf.matrix_inverse(sig_inv),a_mn,Ytr)/tf.square(sigma_1) F_v_1=GPLVM.Bound1(h_mu,h_S,K_mm_1,K_nm_1,Tr_Knn_1,sigma_1) s.run(tf.initialize_all_variables()) F_v_2=GPLVM.Bound2(Tr_Knn,K_nm_2,K_mnnm_2,sigma_2,K_mm_2,Ytr) mean,var_terms=predict(K_mn_2,sigma_2,K_mm_2,K_nn_2) mean=tf.reshape(mean,[-1]) var_terms=tf.reshape(var_terms,[-1]) global_step = tf.Variable(0, trainable=False) starter_learning_rate = 0.1 lrate=tf.train.exponential_decay(starter_learning_rate, global_step,
def mean_cov(K_mm, sigma, K_mn, K_mnnm, Y):
beta = 1 / tf.square(sigma)
A_I = beta * K_mnnm + K_mm
Sig = h.Mul(K_mm, tf.matrix_solve(A_I, K_mm))
mu = beta * h.Mul(K_mm, tf.matrix_solve(A_I, K_mn), Y)
return mu, Sig
def sample(mu, Sigma):
N = h.get_dim(mu, 0)
rand = h.Mul(tf.cholesky(Sigma + 1e-1 * np.eye(N)), tf.random_normal(
[N, 1])) + mu
return rand
h_S_std = tf.Variable(np.ones((N, 1)), dtype=tf.float32)
h_S = tf.square(h_S_std)
X_m_2 = tf.Variable(tf.random_uniform([M, 1], minval=-5, maxval=5),
dtype=tf.float32)
sigma_2 = tf.Variable(tf.ones([1, 1]), dtype=tf.float32)
noise_2 = tf.Variable(tf.ones([1, 1]), dtype=tf.float32)
len_sc_2 = tf.Variable(tf.ones([1, 1]), dtype=tf.float32)
Tr_Knn, K_nm_2, K_mnnm_2 = KS.build_psi_stats_rbf(X_m_2, tf.square(noise_2),
len_sc_2, h_mu, h_S)
K_mm_2 = h.tf_SE_K(X_m_2, X_m_2, len_sc_2, noise_2)
K_nn_2 = h.tf_SE_K(h_mu, h_mu, len_sc_2, noise_2)
K_mn_2 = h.tf.transpose(K_nm_2)
opti_mu1, opti_Sig1 = mean_cov(K_mm_1, sigma_1, K_mn_1, h.Mul(K_mn_1, K_nm_1),
h_mu)
opti_mu2, opti_Sig2 = mean_cov(K_mm_2, sigma_2, K_mn_2, K_mnnm_2, Ytr)
X_mean1, X_cov1, X_real_cov1 = pred(Xtr, X_m_1, opti_mu1, len_sc_1, noise_1)
X_mean2, X_cov2, X_real_cov2 = pred(X_mean1, X_m_2, opti_mu2, len_sc_2,
noise_2)
F_v_1 = GPLVM.Bound1(h_mu, h_S, K_mm_1, K_nm_1, Tr_Knn_1, sigma_1)
F_v_2 = GPLVM.Bound2(Tr_Knn, K_nm_2, K_mnnm_2, sigma_2, K_mm_2, Ytr)
mean, var_terms = predict(K_mn_2, sigma_2, K_mm_2, K_nn_2)
mean = tf.reshape(mean, [-1])
var_terms = tf.reshape(var_terms, [-1])
mu=h.Mul(K_mm_2,tf.cholesky_solve(cov_chol,K_mn_2),Ytr)/tf.square(sigma_2)
mean=h.Mul(K_nm_2,tf.matrix_solve(K_mm_1,mu))
variance=K_nn_2-h.Mul(K_nm_2,h.safe_chol(K_mm_2,tf.transpose(K_nm_2)))
var_terms=2*tf.sqrt(tf.reshape(tf.diag_part(variance)+tf.square(sigma_2),[N,1]))
return mean, var_terms
# layer 1
X_m_1=tf.Variable(tf.random_uniform([M,1],minval=0,maxval=15),name='X_m',dtype=tf.float32)
sigma_1=tf.Variable(tf.ones([1,1]),dtype=tf.float32,name='sigma')
noise_1=tf.Variable(tf.ones([1,1]),dtype=tf.float32,name='sigma')
len_sc_1=tf.square(tf.Variable(tf.ones([1,1]),dtype=tf.float32))+0.3
K_nm_1=h.tf_SE_K(Xtr,X_m_1,len_sc_1,noise_1)
K_mm_1=h.tf_SE_K(X_m_1,X_m_1,len_sc_1,noise_1)+h.tol*np.eye(M,M)
K_nn_1=h.tf_SE_K(Xtr,Xtr,len_sc_1,noise_1)
K_mn_1=h.tf.transpose(K_nm_1)
K_mnnm_1=h.Mul(K_mn_1,K_nm_1)
Tr_Knn_1=tf.trace(K_nn_1)
#mean1,var1=predict(K_mn_1,sigma_1,K_mm_1,K_nn_1)
# layer 2
h_mu=tf.Variable(Ytr)
h_S_std=tf.Variable(np.ones((N,1)),dtype=tf.float32)
h_S=tf.square(h_S_std)
X_m_2=tf.Variable(tf.random_uniform([M,1],minval=-20,maxval=20),dtype=tf.float32)
sigma_2=tf.Variable(tf.ones([1,1]),dtype=tf.float32)
noise_2=tf.Variable(tf.ones([1,1]),dtype=tf.float32)
len_sc_2=tf.square(tf.Variable(tf.ones([1,1]),dtype=tf.float32))+0.3
'''
K_nm_2=h.tf_SE_K(h_mu,X_m_2,len_sc_2,noise_2)
K_mm_2=h.tf_SE_K(X_m_2,X_m_2,len_sc_2,noise_2)+h.tol*np.eye(M,M)