def test_ais_with_dbn_sanity_check(self): dbn = DBN(RBM(5, 20)) dbn.add_layer(RBM(20, 5)) dbn.add_layer(RBM(5, 20)) dbn[0].W = np.matrix(np.random.randn(5, 20)) dbn[0].b = np.matrix(np.random.rand(5, 1) - 0.5) dbn[0].c = np.matrix(np.random.rand(20, 1) - 0.5) dbn[1].W = dbn[0].W.T dbn[1].b = dbn[0].c dbn[1].c = dbn[0].b dbn[2].W = dbn[0].W dbn[2].b = dbn[0].b dbn[2].c = dbn[0].c samples = dbn.sample(100) ais = Estimator(dbn[0]) rbm_logz = ais.estimate_log_partition_function(100, np.sqrt(np.arange(0, 1, 0.001))) rbm_probs = ais.estimate_log_probability(samples) ais = Estimator(dbn) dbn_logz = ais.estimate_log_partition_function(100, np.sqrt(np.arange(0, 1, 0.001))) dbn_probs = ais.estimate_log_probability(samples) self.assertTrue(abs(dbn_logz - rbm_logz) / rbm_logz < 0.02) self.assertTrue((np.exp(dbn_probs) - np.exp(rbm_probs)).mean() < 0.02)
def test_ais_with_dbn(self): dbn = DBN(RBM(10, 10)) dbn.add_layer(RBM(10, 20)) hidden_states = utils.binary_numbers(dbn[0].Y.shape[0]) ais = Estimator(dbn) ais_logz = ais.estimate_log_partition_function(100, np.sqrt(np.arange(0, 1, 0.001))) brf_logz = utils.logsumexp(dbn[1]._ulogprob_vis(hidden_states)) brf_probs = [] dbn_probs = [] dbn_bound = [] for i in range(50): sample = np.matrix(np.random.rand(10, 1)) > 0.5 prob, bound = ais.estimate_log_probability(sample, 200) brf_probs.append(utils.logsumexp(dbn[1]._ulogprob_vis(hidden_states) + dbn[0]._clogprob_vis_hid(sample, hidden_states), 1) - brf_logz) dbn_probs.append(prob) dbn_bound.append(bound) self.assertTrue(np.mean(dbn_bound) < np.mean(dbn_probs)) self.assertTrue(np.abs(np.mean(dbn_probs) - np.mean(brf_probs)) < 0.1)
def test_ais_with_semirbm_dbn(self):
dbn = DBN(RBM(5, 5))
dbn.add_layer(SemiRBM(5, 5))
ais = Estimator(dbn)
ais.estimate_log_partition_function(100,
np.arange(0, 1, 1E-3),
layer=0)
ais.estimate_log_partition_function(10, np.arange(0, 1, 1E-3), layer=1)
dbn[0]._brf_logz = utils.logsumexp(dbn[0]._ulogprob_vis(
utils.binary_numbers(dbn[0].X.shape[0])))
dbn[1]._brf_logz = utils.logsumexp(dbn[1]._ulogprob_vis(
utils.binary_numbers(dbn[1].X.shape[0])))
samples = np.concatenate(
[dbn.sample(25, 100, 20),
np.matrix(np.random.rand(5, 25) > 0.5)], 1)
Y = utils.binary_numbers(dbn[0].Y.shape[0])
X = utils.binary_numbers(dbn[0].X.shape[0])
logRy = dbn[1]._ulogprob_vis(Y)
logQy = utils.logsumexp(dbn[0]._ulogprob(X, Y, all_pairs=True), 0)
log_sum = utils.logsumexp(
dbn[0]._clogprob_hid_vis(samples, Y, all_pairs=True) - logQy +
logRy, 1)
logPx = log_sum + dbn[0]._ulogprob_vis(samples) - dbn[1]._brf_logz
logPx_ = ais.estimate_log_probability(samples)[0]
self.assertTrue(np.abs(logPx_.mean() - logPx.mean()) < 0.1)
def test_ais_with_dbn(self):
dbn = DBN(RBM(10, 10))
dbn.add_layer(RBM(10, 20))
hidden_states = utils.binary_numbers(dbn[0].Y.shape[0])
ais = Estimator(dbn)
ais_logz = ais.estimate_log_partition_function(
100, np.sqrt(np.arange(0, 1, 0.001)))
brf_logz = utils.logsumexp(dbn[1]._ulogprob_vis(hidden_states))
brf_probs = []
dbn_probs = []
dbn_bound = []
for i in range(50):
sample = np.matrix(np.random.rand(10, 1)) > 0.5
prob, bound = ais.estimate_log_probability(sample, 200)
brf_probs.append(
utils.logsumexp(
dbn[1]._ulogprob_vis(hidden_states) +
dbn[0]._clogprob_vis_hid(sample, hidden_states), 1) -
brf_logz)
dbn_probs.append(prob)
dbn_bound.append(bound)
self.assertTrue(np.mean(dbn_bound) < np.mean(dbn_probs))
self.assertTrue(np.abs(np.mean(dbn_probs) - np.mean(brf_probs)) < 0.1)
def test_ais_with_dbn_sanity_check(self):
dbn = DBN(RBM(5, 20))
dbn.add_layer(RBM(20, 5))
dbn.add_layer(RBM(5, 20))
dbn[0].W = np.matrix(np.random.randn(5, 20))
dbn[0].b = np.matrix(np.random.rand(5, 1) - 0.5)
dbn[0].c = np.matrix(np.random.rand(20, 1) - 0.5)
dbn[1].W = dbn[0].W.T
dbn[1].b = dbn[0].c
dbn[1].c = dbn[0].b
dbn[2].W = dbn[0].W
dbn[2].b = dbn[0].b
dbn[2].c = dbn[0].c
samples = dbn.sample(100)
ais = Estimator(dbn[0])
rbm_logz = ais.estimate_log_partition_function(
100, np.sqrt(np.arange(0, 1, 0.001)))
rbm_probs = ais.estimate_log_probability(samples)
ais = Estimator(dbn)
dbn_logz = ais.estimate_log_partition_function(
100, np.sqrt(np.arange(0, 1, 0.001)))
dbn_probs = ais.estimate_log_probability(samples)
self.assertTrue(abs(dbn_logz - rbm_logz) / rbm_logz < 0.02)
self.assertTrue((np.exp(dbn_probs) - np.exp(rbm_probs)).mean() < 0.02)
def main(argv):
num_visibles = 28 * 28
num_hiddens = [1000, 1000]
num_epochs = 50
batch_size = 100
# load data samples
data = load('../data/mnist.npz')['train'] / 255.
# train 1st layer
dbn = DBN(RBM(num_visibles, num_hiddens[0]))
dbn.train(data, num_epochs, batch_size)
# train 2nd layer
dbn.add_layer(RBM(num_hiddens[0], num_hiddens[1]))
dbn.train(data, num_epochs, batch_size, [1E-1, 1E-2])
return 0
def main(argv):
num_visibles = 28 * 28
num_hiddens = [1000, 1000]
num_epochs = 50
batch_size = 100
# load data samples
data = load('../data/mnist.npz')['train'] / 255.
# train 1st layer
dbn = DBN(RBM(num_visibles, num_hiddens[0]))
dbn.train(data, num_epochs, batch_size)
# train 2nd layer
dbn.add_layer(RBM(num_hiddens[0], num_hiddens[1]))
dbn.train(data, num_epochs, batch_size, [1E-1, 1E-2])
return 0
def test_ais_with_semirbm_dbn(self): dbn = DBN(RBM(5, 5)) dbn.add_layer(SemiRBM(5, 5)) ais = Estimator(dbn) ais.estimate_log_partition_function(100, np.arange(0, 1, 1E-3), layer=0) ais.estimate_log_partition_function(10, np.arange(0, 1, 1E-3), layer=1) dbn[0]._brf_logz = utils.logsumexp(dbn[0]._ulogprob_vis(utils.binary_numbers(dbn[0].X.shape[0]))) dbn[1]._brf_logz = utils.logsumexp(dbn[1]._ulogprob_vis(utils.binary_numbers(dbn[1].X.shape[0]))) samples = np.concatenate([dbn.sample(25, 100, 20), np.matrix(np.random.rand(5, 25) > 0.5)], 1) Y = utils.binary_numbers(dbn[0].Y.shape[0]) X = utils.binary_numbers(dbn[0].X.shape[0]) logRy = dbn[1]._ulogprob_vis(Y) logQy = utils.logsumexp(dbn[0]._ulogprob(X, Y, all_pairs=True), 0) log_sum = utils.logsumexp(dbn[0]._clogprob_hid_vis(samples, Y, all_pairs=True) - logQy + logRy, 1) logPx = log_sum + dbn[0]._ulogprob_vis(samples) - dbn[1]._brf_logz logPx_ = ais.estimate_log_probability(samples)[0] self.assertTrue(np.abs(logPx_.mean() - logPx.mean()) < 0.1)
def main(argv):
# load preprocessed data samples
print 'loading data...\t',
data = load('./data/vanhateren.npz')
print '[DONE]'
print
# remove DC component (first component)
data_train = data['train'][1:, :]
data_test = data['test'][1:, :]
# create 1st layer
dbn = DBN(GaussianRBM(num_visibles=data_train.shape[0], num_hiddens=100))
# hyperparameters
dbn[0].learning_rate = 5E-3
dbn[0].weight_decay = 1E-2
dbn[0].momentum = 0.9
dbn[0].sigma = 0.65
dbn[0].cd_steps = 1
dbn[0].persistent = True
# train 1st layer
print 'training...\t',
dbn.train(data_train, num_epochs=100, batch_size=100)
print '[DONE]'
# evaluate 1st layer
print 'evaluating...\t',
logptf = dbn.estimate_log_partition_function(num_ais_samples=100, beta_weights=arange(0, 1, 1E-3))
loglik = dbn.estimate_log_likelihood(data_test)
print '[DONE]'
print
print 'estimated log-partf.:\t', logptf
print 'estimated log-loss:\t', -loglik / data_test.shape[0] / log(2)
print
# create 2nd layer
dbn.add_layer(SemiRBM(num_visibles=100, num_hiddens=100))
# initialize parameters
dbn[1].L = dbn[0].W.T * dbn[0].W
dbn[1].b = dbn[0].W.T * dbn[0].b + dbn[0].c + 0.5 * asmatrix(diag(dbn[1].L)).T
dbn[1].L = dbn[1].L - asmatrix(diag(diag(dbn[1].L)))
# hyperparameters
dbn[1].learning_rate = 5E-3
dbn[1].learning_rate_lateral = 5E-4
dbn[1].weight_decay = 5E-3
dbn[1].weight_decay_lateral = 5E-3
dbn[1].momentum = 0.9
dbn[1].momentum_lateral = 0.9
dbn[1].num_lateral_updates = 20
dbn[1].damping = 0.2
dbn[1].cd_steps = 1
dbn[1].persistent = True
# train 2nd layer
print 'training...\t',
dbn.train(data_train, num_epochs=100, batch_size=100)
print '[DONE]'
# evaluate 2nd layer
print 'evaluating...\t',
logptf = dbn.estimate_log_partition_function(num_ais_samples=100, beta_weights=arange(0, 1, 1E-3))
loglik = dbn.estimate_log_likelihood(data_test, num_samples=100)
print '[DONE]'
print
print 'estimated log-partf.:\t', logptf
print 'estimated log-loss:\t', -loglik / data_test.shape[0] / log(2)
print
# fine-tune with wake-sleep
dbn[0].learning_rate /= 4.
dbn[1].learning_rate /= 4.
print 'fine-tuning...\t',
dbn.train_wake_sleep(data_train, num_epochs=10, batch_size=10)
print '[DONE]'
# reevaluate
print 'evaluating...\t',
logptf = dbn.estimate_log_partition_function(num_ais_samples=100, beta_weights=arange(0, 1, 1E-3))
loglik = dbn.estimate_log_likelihood(data_test, num_samples=100)
print '[DONE]'
print
print 'estimated log-partf.:\t', logptf
print 'estimated log-loss:\t', -loglik / data_test.shape[0] / log(2)
return 0
