def collapsed_gibbs(self, wordIds, S, T): K = self.K V = self.V D = self.D N = self.N latent_vars = {} training_data = {} qbeta = Empirical(tf.Variable(tf.zeros([S, K, V]) + 0.01)) latent_vars[self.beta] = qbeta qtheta = [None] * D qz = [None] * D for d in range(D): qtheta[d] = Empirical(tf.Variable(tf.zeros([S, K]) + 0.1)) latent_vars[self.theta[d]] = qtheta[d] qz[d] = Empirical(tf.Variable(tf.zeros([S, N[d]], dtype=tf.int32))) latent_vars[self.z[d]] = qz[d] training_data[self.w[d]] = wordIds[d] self.latent_vars = latent_vars proposal_vars = {} proposal_vars[self.beta] = ed.complete_conditional(self.beta) cond_set = set(self.w + self.z) for d in range(D): proposal_vars[self.theta[d]] = \ ed.complete_conditional(self.theta[d]) proposal_vars[self.z[d]] = \ ed.complete_conditional(self.z[d], cond_set) self.inference = ed.Gibbs(latent_vars, proposal_vars, training_data) print("collapsed gibbs setup finished") self.inference.initialize(n_iter=T, n_print=1) print("initialize finished") self.__run_inference__(T) self.qbeta_sample = qbeta.eval()
def test_mog(self): x_val = np.array([1.1, 1.2, 2.1, 4.4, 5.5, 7.3, 6.8], np.float32) z_val = np.array([0, 0, 0, 1, 1, 2, 2], np.int32) pi_val = np.array([0.2, 0.3, 0.5], np.float32) mu_val = np.array([1.0, 5.0, 7.0], np.float32) N = x_val.shape[0] K = z_val.max() + 1 pi_alpha = 1.3 + np.zeros(K, dtype=np.float32) mu_sigma = 4.0 sigmasq = 2.0**2 pi = rvs.Dirichlet(pi_alpha) mu = rvs.Normal(0.0, mu_sigma, sample_shape=[K]) x = rvs.ParamMixture(pi, { 'loc': mu, 'scale': tf.sqrt(sigmasq) }, rvs.Normal, sample_shape=N) z = x.cat mu_cond = ed.complete_conditional(mu) pi_cond = ed.complete_conditional(pi) z_cond = ed.complete_conditional(z) with self.test_session() as sess: pi_cond_alpha, mu_cond_mu, mu_cond_sigma, z_cond_p = (sess.run( [ pi_cond.concentration, mu_cond.loc, mu_cond.scale, z_cond.probs ], { z: z_val, x: x_val, pi: pi_val, mu: mu_val })) true_pi = pi_alpha + np.unique(z_val, return_counts=True)[1] self.assertAllClose(pi_cond_alpha, true_pi) for k in range(K): sigmasq_true = (1.0 / 4**2 + 1.0 / sigmasq * (z_val == k).sum())**-1 mu_true = sigmasq_true * (1.0 / sigmasq * x_val[z_val == k].sum()) self.assertAllClose(np.sqrt(sigmasq_true), mu_cond_sigma[k]) self.assertAllClose(mu_true, mu_cond_mu[k]) true_log_p_z = np.log(pi_val) - 0.5 / sigmasq * (x_val[:, np.newaxis] - mu_val)**2 true_log_p_z -= true_log_p_z.max(1, keepdims=True) true_p_z = np.exp(true_log_p_z) true_p_z /= true_p_z.sum(1, keepdims=True) self.assertAllClose(z_cond_p, true_p_z)
def test_blanket_changes(self): pi = rvs.Dirichlet(tf.ones(3)) mu = rvs.Normal(0.0, 1.0) z = rvs.Categorical(p=pi) pi1_cond = ed.complete_conditional(pi, [z, pi]) pi2_cond = ed.complete_conditional(pi, [z, mu, pi]) self.assertIsInstance(pi1_cond, rvs.Dirichlet) self.assertIsInstance(pi2_cond, rvs.Dirichlet) with self.test_session() as sess: alpha1_val, alpha2_val = sess.run([pi1_cond.alpha, pi2_cond.alpha]) self.assertAllClose(alpha1_val, alpha2_val)
def main(_): ed.set_seed(42) # DATA x_data = np.array([0, 1, 0, 0, 0, 0, 0, 0, 0, 1]) # MODEL p = Beta(1.0, 1.0) x = Bernoulli(probs=p, sample_shape=10) # COMPLETE CONDITIONAL p_cond = ed.complete_conditional(p) sess = ed.get_session() print('p(probs | x) type:', p_cond.parameters['name']) param_vals = sess.run( { key: val for key, val in six.iteritems(p_cond.parameters) if isinstance(val, tf.Tensor) }, {x: x_data}) print('parameters:') for key, val in six.iteritems(param_vals): print('%s:\t%.3f' % (key, val))
def test_blanket_changes(self): pi = rvs.Dirichlet(tf.ones(3)) mu = rvs.Normal(0.0, 1.0) z = rvs.Categorical(probs=pi) pi1_cond = ed.complete_conditional(pi, [z, pi]) pi2_cond = ed.complete_conditional(pi, [z, mu, pi]) self.assertIsInstance(pi1_cond, rvs.Dirichlet) self.assertIsInstance(pi2_cond, rvs.Dirichlet) with self.test_session() as sess: conc1_val, conc2_val = sess.run([pi1_cond.concentration, pi2_cond.concentration]) self.assertAllClose(conc1_val, conc2_val)
def test_missing_blanket(self): N = 10 z = rvs.Bernoulli(probs=0.75, sample_shape=N) z_cond = ed.complete_conditional(z) self.assertIsInstance(z_cond, rvs.Bernoulli) with self.test_session() as sess: p_val = sess.run(z_cond.probs) self.assertAllClose(p_val, 0.75 + np.zeros(N, np.float32))
def test_basic_bernoulli(self): N = 10 z = rvs.Bernoulli(probs=0.75, sample_shape=N) z_cond = ed.complete_conditional(z, [z]) self.assertIsInstance(z_cond, rvs.Bernoulli) with self.test_session() as sess: p_val = sess.run(z_cond.probs) self.assertAllClose(p_val, 0.75 + np.zeros(N, np.float32))
def test_missing_blanket(self): N = 10 z = rvs.Bernoulli(p=0.75, sample_shape=N) z_cond = ed.complete_conditional(z) self.assertIsInstance(z_cond, rvs.Bernoulli) with self.test_session() as sess: p_val = sess.run(z_cond.p) self.assertAllClose(p_val, 0.75 + np.zeros(N, np.float32))
def test_mog(self): x_val = np.array([1.1, 1.2, 2.1, 4.4, 5.5, 7.3, 6.8], np.float32) z_val = np.array([0, 0, 0, 1, 1, 2, 2], np.int32) pi_val = np.array([0.2, 0.3, 0.5], np.float32) mu_val = np.array([1.0, 5.0, 7.0], np.float32) N = x_val.shape[0] K = z_val.max() + 1 pi_alpha = 1.3 + np.zeros(K, dtype=np.float32) mu_sigma = 4.0 sigmasq = 2.0**2 pi = rvs.Dirichlet(pi_alpha) mu = rvs.Normal(0.0, mu_sigma, sample_shape=[K]) x = rvs.ParamMixture(pi, {'loc': mu, 'scale': tf.sqrt(sigmasq)}, rvs.Normal, sample_shape=N) z = x.cat mu_cond = ed.complete_conditional(mu) pi_cond = ed.complete_conditional(pi) z_cond = ed.complete_conditional(z) with self.test_session() as sess: pi_cond_alpha, mu_cond_mu, mu_cond_sigma, z_cond_p = ( sess.run([pi_cond.concentration, mu_cond.loc, mu_cond.scale, z_cond.probs], {z: z_val, x: x_val, pi: pi_val, mu: mu_val})) true_pi = pi_alpha + np.unique(z_val, return_counts=True)[1] self.assertAllClose(pi_cond_alpha, true_pi) for k in range(K): sigmasq_true = (1.0 / 4**2 + 1.0 / sigmasq * (z_val == k).sum())**-1 mu_true = sigmasq_true * (1.0 / sigmasq * x_val[z_val == k].sum()) self.assertAllClose(np.sqrt(sigmasq_true), mu_cond_sigma[k]) self.assertAllClose(mu_true, mu_cond_mu[k]) true_log_p_z = np.log(pi_val) - 0.5 / sigmasq * (x_val[:, np.newaxis] - mu_val)**2 true_log_p_z -= true_log_p_z.max(1, keepdims=True) true_p_z = np.exp(true_log_p_z) true_p_z /= true_p_z.sum(1, keepdims=True) self.assertAllClose(z_cond_p, true_p_z)
def test_dirichlet_multinomial(self): x_data = np.array([4, 3, 2, 1], np.int32) N = x_data.sum() D = x_data.shape[0] alpha = np.zeros(D).astype(np.float32) + 2.0 theta = rvs.Dirichlet(alpha) x = rvs.Multinomial(total_count=tf.cast(N, tf.float32), probs=theta) theta_cond = ed.complete_conditional(theta, [theta, x]) with self.test_session() as sess: alpha_val = sess.run(theta_cond.concentration, {x: x_data}) self.assertAllClose(alpha_val, np.array([6.0, 5.0, 4.0, 3.0], np.float32))
def test_dirichlet_categorical(self): x_data = np.array([0, 0, 0, 0, 1, 1, 1, 2, 2, 3], np.int32) N = x_data.shape[0] D = x_data.max() + 1 alpha = np.zeros(D).astype(np.float32) + 2.0 theta = rvs.Dirichlet(alpha) x = rvs.Categorical(probs=theta, sample_shape=N) theta_cond = ed.complete_conditional(theta, [theta, x]) with self.test_session() as sess: alpha_val = sess.run(theta_cond.concentration, {x: x_data}) self.assertAllClose(alpha_val, np.array([6.0, 5.0, 4.0, 3.0], np.float32))
def test_dirichlet_categorical(self): x_data = np.array([0, 0, 0, 0, 1, 1, 1, 2, 2, 3], np.int32) N = x_data.shape[0] D = x_data.max() + 1 alpha = np.zeros(D).astype(np.float32) + 2.0 theta = rvs.Dirichlet(alpha) x = rvs.Categorical(p=theta, sample_shape=N) theta_cond = ed.complete_conditional(theta, [theta, x]) with self.test_session() as sess: alpha_val = sess.run(theta_cond.alpha, {x: x_data}) self.assertAllClose(alpha_val, np.array([6.0, 5.0, 4.0, 3.0], np.float32))
def test_dirichlet_multinomial(self): x_data = np.array([4, 3, 2, 1], np.int32) N = x_data.sum() D = x_data.shape[0] alpha = np.zeros(D).astype(np.float32) + 2.0 theta = rvs.Dirichlet(alpha) x = rvs.Multinomial(n=tf.cast(N, tf.float32), p=theta) theta_cond = ed.complete_conditional(theta, [theta, x]) with self.test_session() as sess: alpha_val = sess.run(theta_cond.alpha, {x: x_data}) self.assertAllClose(alpha_val, np.array([6.0, 5.0, 4.0, 3.0], np.float32))
def test_beta_bernoulli(self): x_data = np.array([0, 1, 0, 0, 0, 0, 0, 0, 0, 1]) a0 = 0.5 b0 = 1.5 pi = rvs.Beta(a=a0, b=b0) x = rvs.Bernoulli(p=pi, sample_shape=10) pi_cond = ed.complete_conditional(pi, [pi, x]) self.assertIsInstance(pi_cond, rvs.Beta) with self.test_session() as sess: a_val, b_val = sess.run([pi_cond.a, pi_cond.b], {x: x_data}) self.assertAllClose(a_val, a0 + x_data.sum()) self.assertAllClose(b_val, b0 + (1 - x_data).sum())
def test_gamma_gamma(self): x_data = np.array([0.1, 0.5, 3.3, 2.7]) alpha0 = 0.5 beta0 = 1.75 alpha_likelihood = 2.3 beta = rvs.Gamma(alpha0, beta0) x = rvs.Gamma(alpha_likelihood, beta, sample_shape=4) beta_cond = ed.complete_conditional(beta, [beta, x]) self.assertIsInstance(beta_cond, rvs.Gamma) with self.test_session() as sess: alpha_val, beta_val = sess.run( [beta_cond.concentration, beta_cond.rate], {x: x_data}) self.assertAllClose(alpha_val, alpha0 + alpha_likelihood * len(x_data)) self.assertAllClose(beta_val, beta0 + x_data.sum())
def test_gamma_exponential(self): x_data = np.array([0.1, 0.5, 3.3, 2.7]) alpha0 = 0.5 beta0 = 1.75 lam = rvs.Gamma(alpha=alpha0, beta=beta0) x = rvs.Exponential(lam=lam, sample_shape=4) lam_cond = ed.complete_conditional(lam, [lam, x]) self.assertIsInstance(lam_cond, rvs.Gamma) with self.test_session() as sess: alpha_val, beta_val = sess.run( [lam_cond.alpha, lam_cond.beta], {x: x_data}) self.assertAllClose(alpha_val, alpha0 + len(x_data)) self.assertAllClose(beta_val, beta0 + x_data.sum())
def test_gamma_exponential(self): x_data = np.array([0.1, 0.5, 3.3, 2.7]) alpha0 = 0.5 beta0 = 1.75 lam = rvs.Gamma(alpha0, beta0) x = rvs.Exponential(lam, sample_shape=4) lam_cond = ed.complete_conditional(lam, [lam, x]) self.assertIsInstance(lam_cond, rvs.Gamma) with self.test_session() as sess: alpha_val, beta_val = sess.run( [lam_cond.concentration, lam_cond.rate], {x: x_data}) self.assertAllClose(alpha_val, alpha0 + len(x_data)) self.assertAllClose(beta_val, beta0 + x_data.sum())
def test_mul_rate_gamma(self): x_data = np.array([0.1, 0.5, 3.3, 2.7]) alpha0 = 0.5 beta0 = 1.75 alpha_likelihood = 2.3 beta = rvs.Gamma(alpha0, beta0) x = rvs.Gamma(alpha_likelihood, alpha_likelihood * beta, sample_shape=4) beta_cond = ed.complete_conditional(beta, [beta, x]) self.assertIsInstance(beta_cond, rvs.Gamma) with self.test_session() as sess: alpha_val, beta_val = sess.run([beta_cond.concentration, beta_cond.rate], {x: x_data}) self.assertAllClose(alpha_val, alpha0 + alpha_likelihood * len(x_data)) self.assertAllClose(beta_val, beta0 + alpha_likelihood * x_data.sum())
def test_beta_bernoulli(self): x_data = np.array([0, 1, 0, 0, 0, 0, 0, 0, 0, 1]) a0 = 0.5 b0 = 1.5 pi = rvs.Beta(a0, b0) x = rvs.Bernoulli(probs=pi, sample_shape=10) pi_cond = ed.complete_conditional(pi, [pi, x]) self.assertIsInstance(pi_cond, rvs.Beta) with self.test_session() as sess: a_val, b_val = sess.run([pi_cond.concentration1, pi_cond.concentration0], {x: x_data}) self.assertAllClose(a_val, a0 + x_data.sum()) self.assertAllClose(b_val, b0 + (1 - x_data).sum())
def test_gamma_poisson(self): x_data = np.array([0, 1, 0, 7, 0, 0, 2, 0, 0, 1]) alpha0 = 0.5 beta0 = 1.75 lam = rvs.Gamma(alpha=alpha0, beta=beta0) # use value since cannot sample x = rvs.Poisson(lam=lam, value=tf.zeros(10), sample_shape=10) lam_cond = ed.complete_conditional(lam, [lam, x]) self.assertIsInstance(lam_cond, rvs.Gamma) with self.test_session() as sess: alpha_val, beta_val = sess.run( [lam_cond.alpha, lam_cond.beta], {x: x_data}) self.assertAllClose(alpha_val, alpha0 + x_data.sum()) self.assertAllClose(beta_val, beta0 + len(x_data))
def test_beta_binomial(self): n_data = 10 x_data = 2 a0 = 0.5 b0 = 1.5 pi = rvs.Beta(a=a0, b=b0) # use value since cannot sample x = rvs.Binomial(n=n_data, p=pi, value=0.0) pi_cond = ed.complete_conditional(pi, [pi, x]) self.assertIsInstance(pi_cond, rvs.Beta) with self.test_session() as sess: a_val, b_val = sess.run([pi_cond.a, pi_cond.b], {x: x_data}) self.assertAllClose(a_val, a0 + x_data) self.assertAllClose(b_val, b0 + n_data - x_data)
def test_gamma_poisson(self): x_data = np.array([0, 1, 0, 7, 0, 0, 2, 0, 0, 1]) alpha0 = 0.5 beta0 = 1.75 lam = rvs.Gamma(alpha0, beta0) # use value since cannot sample x = rvs.Poisson(lam, value=tf.zeros(10), sample_shape=10) lam_cond = ed.complete_conditional(lam, [lam, x]) self.assertIsInstance(lam_cond, rvs.Gamma) with self.test_session() as sess: alpha_val, beta_val = sess.run( [lam_cond.concentration, lam_cond.rate], {x: x_data}) self.assertAllClose(alpha_val, alpha0 + x_data.sum()) self.assertAllClose(beta_val, beta0 + len(x_data))
def test_inverse_gamma_normal(self): x_data = np.array([0.1, 0.5, 3.3, 2.7]) sigmasq_conc = 1.3 sigmasq_rate = 2.1 x_loc = 0.3 sigmasq = rvs.InverseGamma(sigmasq_conc, sigmasq_rate) x = rvs.Normal(x_loc, tf.sqrt(sigmasq), sample_shape=len(x_data)) sigmasq_cond = ed.complete_conditional(sigmasq, [sigmasq, x]) self.assertIsInstance(sigmasq_cond, rvs.InverseGamma) with self.test_session() as sess: conc_val, rate_val = sess.run( [sigmasq_cond.concentration, sigmasq_cond.rate], {x: x_data}) self.assertAllClose(conc_val, sigmasq_conc + 0.5 * len(x_data)) self.assertAllClose(rate_val, sigmasq_rate + 0.5 * np.sum( (x_data - x_loc)**2))
def test_beta_binomial(self): n_data = 10 x_data = 2 a0 = 0.5 b0 = 1.5 pi = rvs.Beta(a0, b0) # use value since cannot sample x = rvs.Binomial(total_count=n_data, probs=pi, value=0.0) pi_cond = ed.complete_conditional(pi, [pi, x]) self.assertIsInstance(pi_cond, rvs.Beta) with self.test_session() as sess: a_val, b_val = sess.run([pi_cond.concentration1, pi_cond.concentration0], {x: x_data}) self.assertAllClose(a_val, a0 + x_data) self.assertAllClose(b_val, b0 + n_data - x_data)
def test_inverse_gamma_normal(self): x_data = np.array([0.1, 0.5, 3.3, 2.7]) sigmasq_conc = 1.3 sigmasq_rate = 2.1 x_loc = 0.3 sigmasq = rvs.InverseGamma(sigmasq_conc, sigmasq_rate) x = rvs.Normal(x_loc, tf.sqrt(sigmasq), sample_shape=len(x_data)) sigmasq_cond = ed.complete_conditional(sigmasq, [sigmasq, x]) self.assertIsInstance(sigmasq_cond, rvs.InverseGamma) with self.test_session() as sess: conc_val, rate_val = sess.run( [sigmasq_cond.concentration, sigmasq_cond.rate], {x: x_data}) self.assertAllClose(conc_val, sigmasq_conc + 0.5 * len(x_data)) self.assertAllClose(rate_val, sigmasq_rate + 0.5 * np.sum((x_data - x_loc)**2))
def test_normal_normal(self): x_data = np.array([0.1, 0.5, 3.3, 2.7]) mu0 = 0.3 sigma0 = 2.1 sigma_likelihood = 1.2 mu = rvs.Normal(mu0, sigma0) x = rvs.Normal(mu, sigma_likelihood, sample_shape=len(x_data)) mu_cond = ed.complete_conditional(mu, [mu, x]) self.assertIsInstance(mu_cond, rvs.Normal) with self.test_session() as sess: mu_val, sigma_val = sess.run([mu_cond.mu, mu_cond.sigma], {x: x_data}) self.assertAllClose(sigma_val, (1.0 / sigma0**2 + len(x_data) / sigma_likelihood**2) ** -0.5) self.assertAllClose(mu_val, sigma_val**2 * (mu0 / sigma0**2 + (1.0 / sigma_likelihood**2 * x_data.sum())))
def test_normal_normal(self): x_data = np.array([0.1, 0.5, 3.3, 2.7]) mu0 = 0.3 sigma0 = 2.1 sigma_likelihood = 1.2 mu = rvs.Normal(mu0, sigma0) x = rvs.Normal(mu, sigma_likelihood, sample_shape=len(x_data)) mu_cond = ed.complete_conditional(mu, [mu, x]) self.assertIsInstance(mu_cond, rvs.Normal) with self.test_session() as sess: mu_val, sigma_val = sess.run([mu_cond.loc, mu_cond.scale], {x: x_data}) self.assertAllClose(sigma_val, (1.0 / sigma0**2 + len(x_data) / sigma_likelihood**2) ** -0.5) self.assertAllClose(mu_val, sigma_val**2 * (mu0 / sigma0**2 + (1.0 / sigma_likelihood**2 * x_data.sum())))
def main(_): ed.set_seed(42) # DATA x_data = np.array([0, 1, 0, 0, 0, 0, 0, 0, 0, 1]) # MODEL p = Beta(1.0, 1.0) x = Bernoulli(probs=p, sample_shape=10) # COMPLETE CONDITIONAL p_cond = ed.complete_conditional(p) sess = ed.get_session() print('p(probs | x) type:', p_cond.parameters['name']) param_vals = sess.run({key: val for key, val in six.iteritems(p_cond.parameters) if isinstance(val, tf.Tensor)}, {x: x_data}) print('parameters:') for key, val in six.iteritems(param_vals): print('%s:\t%.3f' % (key, val))
G2 = ed.models.Bernoulli(probs=p_nextg2) G3 = ed.models.Bernoulli(probs=p_nextg3) mean_x2 = tf.where(tf.cast(G2,tf.bool),60.,50.) mean_x3 = tf.where(tf.cast(G3,tf.bool),60.,50.) sig = np.float32(np.sqrt(10)) X2 = ed.models.Normal(loc=mean_x2,scale=sig) X3 = ed.models.Normal(loc=mean_x3,scale=sig) ### run for prob(g1=2|x2=50) # I treaat 2==1 and 1==0 #part1 = ed.models.Bernoulli(probs=tf.nn.sigmoid(tf.Variable(tf.random_normal([])))) #ed.get_session() #inf = ed.KLpq({G1:part1},data={X2:tf.constant(50,dtype=tf.float32)}) #inf.run(n_samples=200) #print(part1.probs.eval()) ### run for prob(x3=50|x2=50) ed.get_session() cond = ed.complete_conditional(X3,{X2:tf.constant(50,dtype=tf.float32)}) probs = 0 for _ in range(100000): s = round(cond.eval()) if s==50: probs += 1 probs = probs/100000 print(probs)
# Prior hyperparameters pi_alpha = np.ones(K, dtype=np.float32) mu_sigma = np.std(true_mu) sigmasq_alpha = 1.0 sigmasq_beta = 2.0 # Model pi = Dirichlet(pi_alpha) mu = Normal(0.0, mu_sigma, sample_shape=K) sigmasq = InverseGamma(sigmasq_alpha, sigmasq_beta, sample_shape=K) x = ParamMixture(pi, {'loc': mu, 'scale': tf.sqrt(sigmasq)}, Normal, sample_shape=N) z = x.cat # Conditionals mu_cond = ed.complete_conditional(mu) sigmasq_cond = ed.complete_conditional(sigmasq) pi_cond = ed.complete_conditional(pi) z_cond = ed.complete_conditional(z) sess = ed.get_session() # Initialize randomly pi_est, mu_est, sigmasq_est, z_est = sess.run([pi, mu, sigmasq, z]) print('Initial parameters:') print('pi:', pi_est) print('mu:', mu_est) print('sigmasq:', sigmasq_est) print()
sigmasq_beta = 2.0 # Model pi = Dirichlet(pi_alpha) mu = Normal(0.0, mu_sigma, sample_shape=K) sigmasq = InverseGamma(sigmasq_alpha, sigmasq_beta, sample_shape=K) x = ParamMixture(pi, { 'mu': mu, 'sigma': tf.sqrt(sigmasq) }, Normal, sample_shape=N) z = x.cat # Conditionals mu_cond = ed.complete_conditional(mu) sigmasq_cond = ed.complete_conditional(sigmasq) pi_cond = ed.complete_conditional(pi) z_cond = ed.complete_conditional(z) sess = ed.get_session() # Initialize randomly pi_est, mu_est, sigmasq_est, z_est = sess.run([pi, mu, sigmasq, z]) print('Initial parameters:') print('pi:', pi_est) print('mu:', mu_est) print('sigmasq:', sigmasq_est) print()
import edward as ed import numpy as np import six import tensorflow as tf from edward.models import Bernoulli, Beta ed.set_seed(42) # DATA x_data = np.array([0, 1, 0, 0, 0, 0, 0, 0, 0, 1]) # MODEL pi = Beta(1.0, 1.0) x = Bernoulli(probs=pi, sample_shape=10) # COMPLETE CONDITIONAL pi_cond = ed.complete_conditional(pi) sess = ed.get_session() tf.global_variables_initializer().run() print('p(pi | x) type:', pi_cond.parameters['name']) param_vals = sess.run({key: val for key, val in six.iteritems(pi_cond.parameters) if isinstance(val, tf.Tensor)}, {x: x_data}) print('parameters:') for key, val in six.iteritems(param_vals): print('%s:\t%.3f' % (key, val))
def main(_): # Generate data true_mu = np.array([-1.0, 0.0, 1.0], np.float32) * 10 true_sigmasq = np.array([1.0**2, 2.0**2, 3.0**2], np.float32) true_pi = np.array([0.2, 0.3, 0.5], np.float32) N = 10000 K = len(true_mu) true_z = np.random.choice(np.arange(K), size=N, p=true_pi) x_data = true_mu[true_z] + np.random.randn(N) * np.sqrt( true_sigmasq[true_z]) # Prior hyperparameters pi_alpha = np.ones(K, dtype=np.float32) mu_sigma = np.std(true_mu) sigmasq_alpha = 1.0 sigmasq_beta = 2.0 # Model pi = Dirichlet(pi_alpha) mu = Normal(0.0, mu_sigma, sample_shape=K) sigmasq = InverseGamma(sigmasq_alpha, sigmasq_beta, sample_shape=K) x = ParamMixture(pi, { 'loc': mu, 'scale': tf.sqrt(sigmasq) }, Normal, sample_shape=N) z = x.cat # Conditionals mu_cond = ed.complete_conditional(mu) sigmasq_cond = ed.complete_conditional(sigmasq) pi_cond = ed.complete_conditional(pi) z_cond = ed.complete_conditional(z) sess = ed.get_session() # Initialize randomly pi_est, mu_est, sigmasq_est, z_est = sess.run([pi, mu, sigmasq, z]) print('Initial parameters:') print('pi:', pi_est) print('mu:', mu_est) print('sigmasq:', sigmasq_est) print() # Gibbs sampler cond_dict = { pi: pi_est, mu: mu_est, sigmasq: sigmasq_est, z: z_est, x: x_data } t0 = time() T = 500 for t in range(T): z_est = sess.run(z_cond, cond_dict) cond_dict[z] = z_est pi_est, mu_est = sess.run([pi_cond, mu_cond], cond_dict) cond_dict[pi] = pi_est cond_dict[mu] = mu_est sigmasq_est = sess.run(sigmasq_cond, cond_dict) cond_dict[sigmasq] = sigmasq_est print('took %.3f seconds to run %d iterations' % (time() - t0, T)) print() print('Final sample for parameters::') print('pi:', pi_est) print('mu:', mu_est) print('sigmasq:', sigmasq_est) print() print() print('True parameters:') print('pi:', true_pi) print('mu:', true_mu) print('sigmasq:', true_sigmasq) print() plt.figure(figsize=[10, 10]) plt.subplot(2, 1, 1) plt.hist(x_data, 50) plt.title('Empirical Distribution of $x$') plt.xlabel('$x$') plt.ylabel('frequency') xl = plt.xlim() plt.subplot(2, 1, 2) plt.hist(sess.run(x, {pi: pi_est, mu: mu_est, sigmasq: sigmasq_est}), 50) plt.title("Predictive distribution $p(x \mid \mathrm{inferred }\ " "\pi, \mu, \sigma^2)$") plt.xlabel('$x$') plt.ylabel('frequency') plt.xlim(xl) plt.show()
# 基本模型 # p(x, z, beta) = Normal(x | beta, I) Categorical(z | pi) Normal(beta | 0, I) # 变分推断 # beta的后验用一组参数正态逼近,z的后验用参数Categorical逼近 from edward.models import Categorical, Normal qbeta = Normal(loc=tf.Variable(tf.zeros([K, D])), scale=tf.exp(tf.Variable(tf.zeros[K, D]))) # 定义后验分布 qz = Categorical(logits=tf.Variable(tf.zeros[N, K])) inference = ed.VariationalInference({beta: qbeta, z: qz}, data={x: x_train}) # 用MAP方法做推断,推断方法都是继承VariationalInference from edward.models import PointMass qbeta = PointMass(params=tf.Variable(tf.zeros([K, D]))) qz = PointMass(params=tf.Variable(tf.zeros(N))) inference = ed.MAP({beta: qbeta, z: qz}, data={x: x_train}) # MonteCarlo推断 # 用beta和z的采样分布,作为后验的逼近 from edward.models import Empirical T = 10000 # number of samples qbeta = Empirical(params=tf.Variable(tf.zeros([T, K, D]))) # Empirical为经验分布 qz = Empirical(params=tf.Variable(tf.zeros([T, N]))) inference = ed.MonteCarlo({beta: qbeta, z: qz}, data={x: x_train}) # 精确推断 from edward.models import Bernoulli, Beta pi = Beta(1.0, 1.0) x = Bernoulli(probs=pi, sample_shape=10) pi_cond = ed.complete_conditional(pi) # 计算pi的后验分布的精确表达式 sess.run(pi_cond, {x: np.array([0, 1, 0, 0, 0, 0, 0, 0, 0, 1])})
import edward as ed from deepx import T import tensorflow as tf from edward.models import Normal, Bernoulli mu = Normal(tf.constant(0.0), tf.constant(0.00001)) x = Normal(mu, tf.constant(0.0001)) mu_x = ed.complete_conditional(mu) sess = T.interactive_session()
def main(_): # Generate data true_mu = np.array([-1.0, 0.0, 1.0], np.float32) * 10 true_sigmasq = np.array([1.0**2, 2.0**2, 3.0**2], np.float32) true_pi = np.array([0.2, 0.3, 0.5], np.float32) N = 10000 K = len(true_mu) true_z = np.random.choice(np.arange(K), size=N, p=true_pi) x_data = true_mu[true_z] + np.random.randn(N) * np.sqrt(true_sigmasq[true_z]) # Prior hyperparameters pi_alpha = np.ones(K, dtype=np.float32) mu_sigma = np.std(true_mu) sigmasq_alpha = 1.0 sigmasq_beta = 2.0 # Model pi = Dirichlet(pi_alpha) mu = Normal(0.0, mu_sigma, sample_shape=K) sigmasq = InverseGamma(sigmasq_alpha, sigmasq_beta, sample_shape=K) x = ParamMixture(pi, {'loc': mu, 'scale': tf.sqrt(sigmasq)}, Normal, sample_shape=N) z = x.cat # Conditionals mu_cond = ed.complete_conditional(mu) sigmasq_cond = ed.complete_conditional(sigmasq) pi_cond = ed.complete_conditional(pi) z_cond = ed.complete_conditional(z) sess = ed.get_session() # Initialize randomly pi_est, mu_est, sigmasq_est, z_est = sess.run([pi, mu, sigmasq, z]) print('Initial parameters:') print('pi:', pi_est) print('mu:', mu_est) print('sigmasq:', sigmasq_est) print() # Gibbs sampler cond_dict = {pi: pi_est, mu: mu_est, sigmasq: sigmasq_est, z: z_est, x: x_data} t0 = time() T = 500 for t in range(T): z_est = sess.run(z_cond, cond_dict) cond_dict[z] = z_est pi_est, mu_est = sess.run([pi_cond, mu_cond], cond_dict) cond_dict[pi] = pi_est cond_dict[mu] = mu_est sigmasq_est = sess.run(sigmasq_cond, cond_dict) cond_dict[sigmasq] = sigmasq_est print('took %.3f seconds to run %d iterations' % (time() - t0, T)) print() print('Final sample for parameters::') print('pi:', pi_est) print('mu:', mu_est) print('sigmasq:', sigmasq_est) print() print() print('True parameters:') print('pi:', true_pi) print('mu:', true_mu) print('sigmasq:', true_sigmasq) print() plt.figure(figsize=[10, 10]) plt.subplot(2, 1, 1) plt.hist(x_data, 50) plt.title('Empirical Distribution of $x$') plt.xlabel('$x$') plt.ylabel('frequency') xl = plt.xlim() plt.subplot(2, 1, 2) plt.hist(sess.run(x, {pi: pi_est, mu: mu_est, sigmasq: sigmasq_est}), 50) plt.title("Predictive distribution $p(x \mid \mathrm{inferred }\ " "\pi, \mu, \sigma^2)$") plt.xlabel('$x$') plt.ylabel('frequency') plt.xlim(xl) plt.show()