def test_probs(self):
rbm = RBM(9, 11)
rbm.W = np.matrix(np.random.randn(rbm.X.shape[0], rbm.Y.shape[0]))
rbm.b = np.matrix(np.random.rand(rbm.X.shape[0], 1))
rbm.c = np.matrix(np.random.randn(rbm.Y.shape[0], 1))
examples_vis = np.matrix(np.random.rand(rbm.X.shape[0], 100) < 0.5)
examples_hid = np.matrix(np.random.rand(rbm.Y.shape[0], 100) < 0.5)
states_vis = utils.binary_numbers(rbm.X.shape[0])
states_hid = utils.binary_numbers(rbm.Y.shape[0])
# check that conditional probabilities are normalized
logprobs = rbm._clogprob_vis_hid(states_vis,
examples_hid,
all_pairs=True)
self.assertTrue(np.all(utils.logsumexp(logprobs, 0) < 1E-10))
logprobs = rbm._clogprob_hid_vis(examples_vis,
states_hid,
all_pairs=True)
self.assertTrue(np.all(utils.logsumexp(logprobs, 1) < 1E-10))
# test for consistency
logprobs1 = rbm._ulogprob(examples_vis, examples_hid, all_pairs=True)
logprobs2 = rbm._clogprob_vis_hid(examples_vis, examples_hid, all_pairs=True) \
+ rbm._ulogprob_hid(examples_hid)
logprobs3 = rbm._clogprob_hid_vis(examples_vis, examples_hid, all_pairs=True) \
+ rbm._ulogprob_vis(examples_vis).T
self.assertTrue(np.all(np.abs(logprobs1 - logprobs2) < 1E-10))
self.assertTrue(np.all(np.abs(logprobs1 - logprobs3) < 1E-10))
def test_probs(self): rbm = RBM(9, 11) rbm.W = np.matrix(np.random.randn(rbm.X.shape[0], rbm.Y.shape[0])) rbm.b = np.matrix(np.random.rand(rbm.X.shape[0], 1)) rbm.c = np.matrix(np.random.randn(rbm.Y.shape[0], 1)) examples_vis = np.matrix(np.random.rand(rbm.X.shape[0], 100) < 0.5) examples_hid = np.matrix(np.random.rand(rbm.Y.shape[0], 100) < 0.5) states_vis = utils.binary_numbers(rbm.X.shape[0]) states_hid = utils.binary_numbers(rbm.Y.shape[0]) # check that conditional probabilities are normalized logprobs = rbm._clogprob_vis_hid(states_vis, examples_hid, all_pairs=True) self.assertTrue(np.all(utils.logsumexp(logprobs, 0) < 1E-10)) logprobs = rbm._clogprob_hid_vis(examples_vis, states_hid, all_pairs=True) self.assertTrue(np.all(utils.logsumexp(logprobs, 1) < 1E-10)) # test for consistency logprobs1 = rbm._ulogprob(examples_vis, examples_hid, all_pairs=True) logprobs2 = rbm._clogprob_vis_hid(examples_vis, examples_hid, all_pairs=True) \ + rbm._ulogprob_hid(examples_hid) logprobs3 = rbm._clogprob_hid_vis(examples_vis, examples_hid, all_pairs=True) \ + rbm._ulogprob_vis(examples_vis).T self.assertTrue(np.all(np.abs(logprobs1 - logprobs2) < 1E-10)) self.assertTrue(np.all(np.abs(logprobs1 - logprobs3) < 1E-10))
def test_ais_with_semirbm_dbn(self):
dbn = DBN(RBM(5, 5))
dbn.add_layer(SemiRBM(5, 5))
ais = Estimator(dbn)
ais.estimate_log_partition_function(100,
np.arange(0, 1, 1E-3),
layer=0)
ais.estimate_log_partition_function(10, np.arange(0, 1, 1E-3), layer=1)
dbn[0]._brf_logz = utils.logsumexp(dbn[0]._ulogprob_vis(
utils.binary_numbers(dbn[0].X.shape[0])))
dbn[1]._brf_logz = utils.logsumexp(dbn[1]._ulogprob_vis(
utils.binary_numbers(dbn[1].X.shape[0])))
samples = np.concatenate(
[dbn.sample(25, 100, 20),
np.matrix(np.random.rand(5, 25) > 0.5)], 1)
Y = utils.binary_numbers(dbn[0].Y.shape[0])
X = utils.binary_numbers(dbn[0].X.shape[0])
logRy = dbn[1]._ulogprob_vis(Y)
logQy = utils.logsumexp(dbn[0]._ulogprob(X, Y, all_pairs=True), 0)
log_sum = utils.logsumexp(
dbn[0]._clogprob_hid_vis(samples, Y, all_pairs=True) - logQy +
logRy, 1)
logPx = log_sum + dbn[0]._ulogprob_vis(samples) - dbn[1]._brf_logz
logPx_ = ais.estimate_log_probability(samples)[0]
self.assertTrue(np.abs(logPx_.mean() - logPx.mean()) < 0.1)
def test_probabilities(self): grbm = GaussianRBM(7, 13) grbm.W = np.asmatrix(np.random.randn(grbm.X.shape[0], grbm.Y.shape[0])) grbm.b = np.asmatrix(np.random.rand(grbm.X.shape[0], 1)) grbm.c = np.asmatrix(np.random.randn(grbm.Y.shape[0], 1)) grbm.sigma = np.random.rand() * 0.5 + 0.5 grbm.sigma = 1. examples_vis = np.asmatrix(np.random.randn(grbm.X.shape[0], 1000) * 2.) examples_hid = np.matrix(np.random.rand(grbm.Y.shape[0], 100) < 0.5) states_hid = utils.binary_numbers(grbm.Y.shape[0]) # check that conditional probabilities are normalized logprobs = grbm._clogprob_hid_vis(examples_vis, states_hid, all_pairs=True) self.assertTrue(np.all(utils.logsumexp(logprobs, 1) < 1E-8)) # test for consistency logprobs1 = grbm._ulogprob(examples_vis, examples_hid, all_pairs=True) logprobs2 = grbm._clogprob_vis_hid(examples_vis, examples_hid, all_pairs=True) \ + grbm._ulogprob_hid(examples_hid) logprobs3 = grbm._clogprob_hid_vis(examples_vis, examples_hid, all_pairs=True) \ + grbm._ulogprob_vis(examples_vis).T self.assertTrue(np.all(np.abs(logprobs1 - logprobs2) < 1E-10)) self.assertTrue(np.all(np.abs(logprobs1 - logprobs3) < 1E-3))
def test_ais_with_dbn(self): dbn = DBN(RBM(10, 10)) dbn.add_layer(RBM(10, 20)) hidden_states = utils.binary_numbers(dbn[0].Y.shape[0]) ais = Estimator(dbn) ais_logz = ais.estimate_log_partition_function(100, np.sqrt(np.arange(0, 1, 0.001))) brf_logz = utils.logsumexp(dbn[1]._ulogprob_vis(hidden_states)) brf_probs = [] dbn_probs = [] dbn_bound = [] for i in range(50): sample = np.matrix(np.random.rand(10, 1)) > 0.5 prob, bound = ais.estimate_log_probability(sample, 200) brf_probs.append(utils.logsumexp(dbn[1]._ulogprob_vis(hidden_states) + dbn[0]._clogprob_vis_hid(sample, hidden_states), 1) - brf_logz) dbn_probs.append(prob) dbn_bound.append(bound) self.assertTrue(np.mean(dbn_bound) < np.mean(dbn_probs)) self.assertTrue(np.abs(np.mean(dbn_probs) - np.mean(brf_probs)) < 0.1)
def test_probabilities(self):
grbm = GaussianRBM(7, 13)
grbm.W = np.asmatrix(np.random.randn(grbm.X.shape[0], grbm.Y.shape[0]))
grbm.b = np.asmatrix(np.random.rand(grbm.X.shape[0], 1))
grbm.c = np.asmatrix(np.random.randn(grbm.Y.shape[0], 1))
grbm.sigma = np.random.rand() * 0.5 + 0.5
grbm.sigma = 1.
examples_vis = np.asmatrix(np.random.randn(grbm.X.shape[0], 1000) * 2.)
examples_hid = np.matrix(np.random.rand(grbm.Y.shape[0], 100) < 0.5)
states_hid = utils.binary_numbers(grbm.Y.shape[0])
# check that conditional probabilities are normalized
logprobs = grbm._clogprob_hid_vis(examples_vis,
states_hid,
all_pairs=True)
self.assertTrue(np.all(utils.logsumexp(logprobs, 1) < 1E-8))
# test for consistency
logprobs1 = grbm._ulogprob(examples_vis, examples_hid, all_pairs=True)
logprobs2 = grbm._clogprob_vis_hid(examples_vis, examples_hid, all_pairs=True) \
+ grbm._ulogprob_hid(examples_hid)
logprobs3 = grbm._clogprob_hid_vis(examples_vis, examples_hid, all_pairs=True) \
+ grbm._ulogprob_vis(examples_vis).T
self.assertTrue(np.all(np.abs(logprobs1 - logprobs2) < 1E-10))
self.assertTrue(np.all(np.abs(logprobs1 - logprobs3) < 1E-3))
def test_ais_with_dbn(self):
dbn = DBN(RBM(10, 10))
dbn.add_layer(RBM(10, 20))
hidden_states = utils.binary_numbers(dbn[0].Y.shape[0])
ais = Estimator(dbn)
ais_logz = ais.estimate_log_partition_function(
100, np.sqrt(np.arange(0, 1, 0.001)))
brf_logz = utils.logsumexp(dbn[1]._ulogprob_vis(hidden_states))
brf_probs = []
dbn_probs = []
dbn_bound = []
for i in range(50):
sample = np.matrix(np.random.rand(10, 1)) > 0.5
prob, bound = ais.estimate_log_probability(sample, 200)
brf_probs.append(
utils.logsumexp(
dbn[1]._ulogprob_vis(hidden_states) +
dbn[0]._clogprob_vis_hid(sample, hidden_states), 1) -
brf_logz)
dbn_probs.append(prob)
dbn_bound.append(bound)
self.assertTrue(np.mean(dbn_bound) < np.mean(dbn_probs))
self.assertTrue(np.abs(np.mean(dbn_probs) - np.mean(brf_probs)) < 0.1)
def test_probabilities(self):
srbm = SemiRBM(9, 12)
srbm.W = np.matrix(np.random.randn(srbm.X.shape[0], srbm.Y.shape[0]))
srbm.b = np.matrix(np.random.rand(srbm.X.shape[0], 1))
srbm.c = np.matrix(np.random.randn(srbm.Y.shape[0], 1))
srbm.L = np.matrix(np.random.randn(srbm.X.shape[0],
srbm.X.shape[0])) / 2.
srbm.L = np.triu(srbm.L) + np.triu(
srbm.L).T - 2. * np.diag(np.diag(srbm.L))
examples_vis = np.matrix(np.random.rand(srbm.X.shape[0], 100) < 0.5)
examples_hid = np.matrix(np.random.rand(srbm.Y.shape[0], 100) < 0.5)
states_vis = utils.binary_numbers(srbm.X.shape[0])
states_hid = utils.binary_numbers(srbm.Y.shape[0])
# check that conditional probabilities are normalized
logprobs = srbm._clogprob_hid_vis(examples_vis,
states_hid,
all_pairs=True)
self.assertTrue(np.all(utils.logsumexp(logprobs, 1) < 1E-10))
# test for consistency
logprobs1 = srbm._ulogprob(examples_vis, examples_hid, all_pairs=True)
logprobs3 = srbm._clogprob_hid_vis(examples_vis, examples_hid, all_pairs=True) \
+ srbm._ulogprob_vis(examples_vis).T
self.assertTrue(np.all(np.abs(logprobs1 - logprobs3) < 1E-10))
rbm = RBM(srbm.X.shape[0], srbm.Y.shape[0])
rbm.W = srbm.W
rbm.b = srbm.b
rbm.c = srbm.c
srbm.L *= 0
logprobs1 = rbm._ulogprob_vis(examples_vis)
logprobs2 = srbm._ulogprob_vis(examples_vis)
self.assertTrue(np.all(np.abs(logprobs1 - logprobs2) < 1E-10))
logprobs1 = rbm._clogprob_hid_vis(examples_vis, examples_hid)
logprobs2 = srbm._clogprob_hid_vis(examples_vis, examples_hid)
self.assertTrue(np.all(np.abs(logprobs1 - logprobs2) < 1E-10))
def test_probabilities(self): srbm = SemiRBM(9, 12) srbm.W = np.matrix(np.random.randn(srbm.X.shape[0], srbm.Y.shape[0])) srbm.b = np.matrix(np.random.rand(srbm.X.shape[0], 1)) srbm.c = np.matrix(np.random.randn(srbm.Y.shape[0], 1)) srbm.L = np.matrix(np.random.randn(srbm.X.shape[0], srbm.X.shape[0])) / 2. srbm.L = np.triu(srbm.L) + np.triu(srbm.L).T - 2. * np.diag(np.diag(srbm.L)) examples_vis = np.matrix(np.random.rand(srbm.X.shape[0], 100) < 0.5) examples_hid = np.matrix(np.random.rand(srbm.Y.shape[0], 100) < 0.5) states_vis = utils.binary_numbers(srbm.X.shape[0]) states_hid = utils.binary_numbers(srbm.Y.shape[0]) # check that conditional probabilities are normalized logprobs = srbm._clogprob_hid_vis(examples_vis, states_hid, all_pairs=True) self.assertTrue(np.all(utils.logsumexp(logprobs, 1) < 1E-10)) # test for consistency logprobs1 = srbm._ulogprob(examples_vis, examples_hid, all_pairs=True) logprobs3 = srbm._clogprob_hid_vis(examples_vis, examples_hid, all_pairs=True) \ + srbm._ulogprob_vis(examples_vis).T self.assertTrue(np.all(np.abs(logprobs1 - logprobs3) < 1E-10)) rbm = RBM(srbm.X.shape[0], srbm.Y.shape[0]) rbm.W = srbm.W rbm.b = srbm.b rbm.c = srbm.c srbm.L *= 0 logprobs1 = rbm._ulogprob_vis(examples_vis) logprobs2 = srbm._ulogprob_vis(examples_vis) self.assertTrue(np.all(np.abs(logprobs1 - logprobs2) < 1E-10)) logprobs1 = rbm._clogprob_hid_vis(examples_vis, examples_hid) logprobs2 = srbm._clogprob_hid_vis(examples_vis, examples_hid) self.assertTrue(np.all(np.abs(logprobs1 - logprobs2) < 1E-10))
def test_ais_with_semirbm_dbn(self): dbn = DBN(RBM(5, 5)) dbn.add_layer(SemiRBM(5, 5)) ais = Estimator(dbn) ais.estimate_log_partition_function(100, np.arange(0, 1, 1E-3), layer=0) ais.estimate_log_partition_function(10, np.arange(0, 1, 1E-3), layer=1) dbn[0]._brf_logz = utils.logsumexp(dbn[0]._ulogprob_vis(utils.binary_numbers(dbn[0].X.shape[0]))) dbn[1]._brf_logz = utils.logsumexp(dbn[1]._ulogprob_vis(utils.binary_numbers(dbn[1].X.shape[0]))) samples = np.concatenate([dbn.sample(25, 100, 20), np.matrix(np.random.rand(5, 25) > 0.5)], 1) Y = utils.binary_numbers(dbn[0].Y.shape[0]) X = utils.binary_numbers(dbn[0].X.shape[0]) logRy = dbn[1]._ulogprob_vis(Y) logQy = utils.logsumexp(dbn[0]._ulogprob(X, Y, all_pairs=True), 0) log_sum = utils.logsumexp(dbn[0]._clogprob_hid_vis(samples, Y, all_pairs=True) - logQy + logRy, 1) logPx = log_sum + dbn[0]._ulogprob_vis(samples) - dbn[1]._brf_logz logPx_ = ais.estimate_log_probability(samples)[0] self.assertTrue(np.abs(logPx_.mean() - logPx.mean()) < 0.1)
def test_ais_with_rbm(self): rbm = RBM(5, 20) rbm.W = np.matrix(np.random.randn(5, 20)) rbm.b = np.matrix(np.random.randn(5, 1)) rbm.c = np.matrix(np.random.randn(20, 1)) ais = Estimator(rbm) ais_logz = ais.estimate_log_partition_function(100, np.arange(0, 1, 0.001)) brf_logz = utils.logsumexp(rbm._ulogprob_vis(utils.binary_numbers(rbm.X.shape[0]))) lower = np.log(np.exp(ais_logz) - 4. * np.sqrt(rbm._ais_var)) upper = np.log(np.exp(ais_logz) + 4. * np.sqrt(rbm._ais_var)) self.assertTrue(upper - lower < 1.) self.assertTrue(lower < brf_logz and brf_logz < upper)
def test_ais_with_gaussianrbm(self): rbm = GaussianRBM(30, 10) rbm.c = np.matrix(np.random.randn(10, 1)) rbm.W = np.matrix(np.random.randn(30, 10)) rbm.b = np.matrix(np.random.rand(30, 1)) ais = Estimator(rbm) ais_logz = ais.estimate_log_partition_function(100, np.arange(0., 1., 1E-4)) brf_logz = utils.logsumexp(rbm._ulogprob_hid(utils.binary_numbers(rbm.Y.shape[0]))) lower = np.log(np.exp(ais_logz) - 4 * np.sqrt(rbm._ais_var)) upper = np.log(np.exp(ais_logz) + 4 * np.sqrt(rbm._ais_var)) self.assertTrue(upper - lower < 1.5) self.assertTrue(lower < brf_logz and brf_logz < upper)
def test_ais_with_semirbm(self): rbm = SemiRBM(5, 20) rbm.L = np.matrix(np.random.randn(5, 5)) rbm.L = np.triu(rbm.L) + np.triu(rbm.L).T - 2 * np.diag(np.diag(rbm.L)) rbm.num_lateral_updates = 5 rbm.sampling_method = SemiRBM.GIBBS ais = Estimator(rbm) ais_logz = ais.estimate_log_partition_function(100, np.arange(0, 1, 0.001)) brf_logz = np.log(np.sum(np.exp(rbm._ulogprob_vis(utils.binary_numbers(rbm.X.shape[0]))))) lower = np.log(np.exp(ais_logz) - 4 * np.sqrt(rbm._ais_var)) upper = np.log(np.exp(ais_logz) + 4 * np.sqrt(rbm._ais_var)) self.assertTrue(upper - lower < 1.) self.assertTrue(lower < brf_logz and brf_logz < upper)
def test_ais_with_gaussianrbm(self):
rbm = GaussianRBM(30, 10)
rbm.c = np.matrix(np.random.randn(10, 1))
rbm.W = np.matrix(np.random.randn(30, 10))
rbm.b = np.matrix(np.random.rand(30, 1))
ais = Estimator(rbm)
ais_logz = ais.estimate_log_partition_function(100,
np.arange(0., 1., 1E-4))
brf_logz = utils.logsumexp(
rbm._ulogprob_hid(utils.binary_numbers(rbm.Y.shape[0])))
lower = np.log(np.exp(ais_logz) - 4 * np.sqrt(rbm._ais_var))
upper = np.log(np.exp(ais_logz) + 4 * np.sqrt(rbm._ais_var))
self.assertTrue(upper - lower < 1.5)
self.assertTrue(lower < brf_logz and brf_logz < upper)
def test_ais_with_rbm(self):
rbm = RBM(5, 20)
rbm.W = np.matrix(np.random.randn(5, 20))
rbm.b = np.matrix(np.random.randn(5, 1))
rbm.c = np.matrix(np.random.randn(20, 1))
ais = Estimator(rbm)
ais_logz = ais.estimate_log_partition_function(100,
np.arange(0, 1, 0.001))
brf_logz = utils.logsumexp(
rbm._ulogprob_vis(utils.binary_numbers(rbm.X.shape[0])))
lower = np.log(np.exp(ais_logz) - 4. * np.sqrt(rbm._ais_var))
upper = np.log(np.exp(ais_logz) + 4. * np.sqrt(rbm._ais_var))
self.assertTrue(upper - lower < 1.)
self.assertTrue(lower < brf_logz and brf_logz < upper)
def test_ais_with_semirbm(self):
rbm = SemiRBM(5, 20)
rbm.L = np.matrix(np.random.randn(5, 5))
rbm.L = np.triu(rbm.L) + np.triu(rbm.L).T - 2 * np.diag(np.diag(rbm.L))
rbm.num_lateral_updates = 5
rbm.sampling_method = SemiRBM.GIBBS
ais = Estimator(rbm)
ais_logz = ais.estimate_log_partition_function(100,
np.arange(0, 1, 0.001))
brf_logz = np.log(
np.sum(
np.exp(rbm._ulogprob_vis(utils.binary_numbers(
rbm.X.shape[0])))))
lower = np.log(np.exp(ais_logz) - 4 * np.sqrt(rbm._ais_var))
upper = np.log(np.exp(ais_logz) + 4 * np.sqrt(rbm._ais_var))
self.assertTrue(upper - lower < 1.)
self.assertTrue(lower < brf_logz and brf_logz < upper)