def test_truncated_scalar_gaussian_lb(): tn0_test = TruncatedScalarGaussian(lb=0) tn0_true = truncnorm(0, np.Inf) print "E[TN(0,inf)]:\t", tn0_test.expected_x() print "E[TN(0,inf)]:\t", tn0_true.mean() assert np.allclose(tn0_test.expected_x(), tn0_true.mean()) print "Var[TN(0,inf)]:\t", tn0_test.variance_x()
def test_truncated_gaussian_entropy(mu=0.0, sigma=1.0): """ Consider Z ~ N(mu,1) versus A ~ Bern(p) for p = 1-Phi(-mu) Z ~ A * TN(mu, 1, 0, Inf) + (1-A) * TN(mu, 1, -Inf, 0) We have simply separated the Gaussian sampling into two steps: 1. Sample the sign of Z 2. Sample Z from a truncated normal. Thus, E_Z[LN p(Z)] should equal E_A[ E_{Z|A} [LN p (Z | A)]] using iterated expectations. :return: """ p = 1.0 - normal_cdf(0, mu, sigma) print "p: ", p # Compute the entropy of the scalar Gaussian H0 = ScalarGaussian(mu=mu, sigmasq=sigma**2).negentropy() # Compute the entropy of the two stage process # H1 = p * TruncatedScalarGaussian(lb=0, mu=mu, sigmasq=sigma).negentropy() + \ # (1-p) * TruncatedScalarGaussian(ub=0, mu=mu, sigmasq=sigma).negentropy() TN1 = TruncatedScalarGaussian(lb=0, mu=mu, sigmasq=sigma) EZ1 = TN1.expected_x() EZ1sq = TN1.expected_xsq() H1 = p * ScalarGaussian(mu=mu, sigmasq=sigma).negentropy(E_x=EZ1, E_xsq=EZ1sq) TN0 = TruncatedScalarGaussian(ub=0, mu=mu, sigmasq=sigma) EZ0 = TN0.expected_x() EZ0sq = TN0.expected_xsq() H1 += (1 - p) * ScalarGaussian(mu=mu, sigmasq=sigma).negentropy( E_x=EZ0, E_xsq=EZ0sq) H2 = -norm(loc=mu, scale=sigma).entropy() # H3 = -p * truncnorm((0-mu)/sigma, np.inf, loc=mu).entropy() + \ # -(1-p) * truncnorm(-np.inf, (0-mu)/sigma, loc=mu).entropy() print "H0: ", H0 print "H1: ", H1 print "H2: ", H2
def test_truncated_scalar_gaussian(): lb = 0 ub = 10 mu = 3 sigma = 1 tn0_test = TruncatedScalarGaussian(mu=mu, sigmasq=np.sqrt(sigma), lb=lb, ub=ub) tn0_true = truncnorm((lb - mu) / sigma, (ub - mu) / sigma, loc=mu) print "E[TN(0,10)]:\t", tn0_test.expected_x() print "E[TN(0,10)]:\t", tn0_true.mean() assert np.allclose(tn0_test.expected_x(), tn0_true.mean()) print "Var[TN(0,10)]:\t", tn0_test.variance_x() print "Var[TN(0,10)]:\t", tn0_true.var() assert np.allclose(tn0_test.variance_x(), tn0_true.var()) print "E[-LN{TN(0,10)}]:\t", -tn0_test.negentropy() print "E[-LN{TN(0,10)}]:\t", tn0_true.entropy() assert np.allclose(-tn0_test.negentropy(), tn0_true.entropy())
def test_truncated_scalar_gaussian(): lb = 0 ub = 10 mu = 3 sigma = 1 tn0_test = TruncatedScalarGaussian(mu=mu, sigmasq=np.sqrt(sigma), lb=lb, ub=ub) tn0_true = truncnorm((lb-mu)/sigma, (ub-mu)/sigma, loc=mu) print "E[TN(0,10)]:\t", tn0_test.expected_x() print "E[TN(0,10)]:\t", tn0_true.mean() assert np.allclose(tn0_test.expected_x(), tn0_true.mean()) print "Var[TN(0,10)]:\t", tn0_test.variance_x() print "Var[TN(0,10)]:\t", tn0_true.var() assert np.allclose(tn0_test.variance_x(), tn0_true.var()) print "E[-LN{TN(0,10)}]:\t", -tn0_test.negentropy() print "E[-LN{TN(0,10)}]:\t", tn0_true.entropy() assert np.allclose(-tn0_test.negentropy(), tn0_true.entropy())
def test_truncated_gaussian_entropy(mu=0.0, sigma=1.0): """ Consider Z ~ N(mu,1) versus A ~ Bern(p) for p = 1-Phi(-mu) Z ~ A * TN(mu, 1, 0, Inf) + (1-A) * TN(mu, 1, -Inf, 0) We have simply separated the Gaussian sampling into two steps: 1. Sample the sign of Z 2. Sample Z from a truncated normal. Thus, E_Z[LN p(Z)] should equal E_A[ E_{Z|A} [LN p (Z | A)]] using iterated expectations. :return: """ p = 1.0 - normal_cdf(0, mu, sigma) print "p: ", p # Compute the entropy of the scalar Gaussian H0 = ScalarGaussian(mu=mu, sigmasq=sigma**2).negentropy() # Compute the entropy of the two stage process # H1 = p * TruncatedScalarGaussian(lb=0, mu=mu, sigmasq=sigma).negentropy() + \ # (1-p) * TruncatedScalarGaussian(ub=0, mu=mu, sigmasq=sigma).negentropy() TN1 = TruncatedScalarGaussian(lb=0, mu=mu, sigmasq=sigma) EZ1 = TN1.expected_x() EZ1sq = TN1.expected_xsq() H1 = p * ScalarGaussian(mu=mu, sigmasq=sigma).negentropy(E_x=EZ1, E_xsq=EZ1sq) TN0 = TruncatedScalarGaussian(ub=0, mu=mu, sigmasq=sigma) EZ0 = TN0.expected_x() EZ0sq = TN0.expected_xsq() H1 += (1-p) * ScalarGaussian(mu=mu, sigmasq=sigma).negentropy(E_x=EZ0, E_xsq=EZ0sq) H2 = -norm(loc=mu, scale=sigma).entropy() # H3 = -p * truncnorm((0-mu)/sigma, np.inf, loc=mu).entropy() + \ # -(1-p) * truncnorm(-np.inf, (0-mu)/sigma, loc=mu).entropy() print "H0: ", H0 print "H1: ", H1 print "H2: ", H2
def get_vlb(self): """ Compute the variational lower bound. :return: """ vlb = 0 # p(A=1) [E_{Z | A} [ln N(z | mu, sigma=1)], # where z ~ TN(mu, 1, lb=0) E_mu = self.mf_expected_mu() E_musq = self.mf_expected_musq() qZ1 = TruncatedScalarGaussian(lb=0, mu=self.mf_mu_Z, sigmasq=1.0) tmp = ScalarGaussian().negentropy(E_x=qZ1.expected_x(), E_xsq=qZ1.expected_xsq(), E_mu=E_mu, E_musq=E_musq) vlb += (self.mf_A * tmp).sum() # p(A=0) [E_{Z | A} [ln N(z | mu, sigma=1)], # where z ~ TN(mu, 1, ub=0) qZ0 = TruncatedScalarGaussian(ub=0, mu=self.mf_mu_Z, sigmasq=1.0) tmp = ScalarGaussian().negentropy(E_x=qZ0.expected_x(), E_xsq=qZ0.expected_xsq(), E_mu=E_mu, E_musq=E_musq) vlb += ((1-self.mf_A) * tmp).sum() # -E[ln q(z | A, mf_mu_z, 1)] for A=1 tmp = TruncatedScalarGaussian(lb=0, mu=self.mf_mu_Z, sigmasq=1.0).negentropy() vlb -= ( self.mf_A * tmp).sum() # -E[ln q(z | A, mf_mu_z, 1)] for A=0 tmp = TruncatedScalarGaussian(ub=0, mu=self.mf_mu_Z, sigmasq=1.0).negentropy() vlb -= ((1-self.mf_A) * tmp).sum() # E[ln p(mu0 | mu_{mu0}, sigma_{mu0})] vlb += ScalarGaussian(mu=self.mu_mu_0, sigmasq=self.sigma_mu0)\ .negentropy(E_x=self.mf_mu_mu0, E_xsq=self.mf_expected_mu0sq()) # -E[ln q(mu0 | mf_mu_mu0, mf_sigma_mu0)] vlb -= ScalarGaussian(mu=self.mf_mu_mu0, sigmasq=self.mf_sigma_mu0).negentropy() # E[ln p(F | 0, sigma_{F})] # -E[ln q(F | mf_mu_F, mf_sigma_F)] for n in xrange(self.N): vlb += Gaussian(mu=np.zeros(self.D), Sigma=self.sigma_F *np.eye(self.D))\ .negentropy(E_x=self.mf_mu_F[n,:], E_xxT=self.mf_expected_ffT(n)) vlb -= Gaussian(mu=self.mf_mu_F[n,:], Sigma=self.mf_Sigma_F[n,:,:]).negentropy() # E[ln p(lmbda | mu_lmbda, sigma_{lmbda})] if not self.lmbda_given: vlb += Gaussian(mu=self.mu_lmbda, Sigma=self.sigma_lmbda * np.eye(self.D))\ .negentropy(E_x=self.mf_mu_lmbda, E_xxT=self.mf_expected_llT()) # -E[ln q(lmbda | mf_mu_lmbda, mf_sigma_lmbda)] vlb -= Gaussian(mu=self.mf_mu_lmbda, Sigma=self.mf_Sigma_lmbda).negentropy() return vlb