def test_load_preprocessed(self):
dataset = m_loader.load_digits(shared=False,
pre={'threshold': 0.5},
n=[10, 10, 10])
for (x, y) in dataset:
# print sp.stats.itemfreq(x)
self.assertTrue(np.all((np.unique(x) == np.array([0, 1]))))
def get_data(shape):
dataset_name = 'sharp_equi{}_{}'.format(shape, shape)
# Load data
train, valid, test = m_loader.load_digits(n=[1000, 10, 100])
# train, valid, test = k_loader.load_kanade(set_name=dataset_name, pre={'scale': True})
train_x, train_y = train
return train_x
def test_full(self):
train, valid, test = mnist_loader.load_digits(n=[100, 0, 100], pre={'scale':True})
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
tr = rbm_config.TrainParam(learning_rate=0.0001,
momentum_type=rbm.CLASSICAL,
momentum=0.5,
weight_decay=0.01,
sparsity_constraint=False,
sparsity_target=0.01,
sparsity_cost=0.01,
sparsity_decay=0.1,
epochs=10)
n_visible = train_x.get_value().shape[1]
n_hidden = 100
gaussian_rbm = rbm.GaussianRBM(n_visible,
n_visible,
n_hidden,
associative=False,
cd_type=rbm.CLASSICAL,
cd_steps=1,
visible_unit=rbm_units.GaussianVisibleUnit,
hidden_unit=rbm_units.RBMUnit,
train_parameters=tr,
progress_logger=rbm_logger.ProgressLogger())
curr_dir = store.move_to('simple_gaussian_rbm_test')
print "... moved to {}".format(curr_dir)
# Train RBM
gaussian_rbm.train(train_x)
def associate_data2label(cache=False):
cache=True
print "Testing ClassRBM with generative target (i.e. AssociativeRBM with picture-label association)"
# Load mnist hand digits, class label is already set to binary
train, valid, test = m_loader.load_digits(n=[50000, 100, 30], pre={'label_vector': True})
train_x, train_y = train
test_x, test_y = test
train_y = T.cast(train_y, dtype=theano.config.floatX)
test_y = T.cast(test_y, dtype=theano.config.floatX)
# Initialise the RBM and training parameters
tr = TrainParam(learning_rate=0.001,
momentum_type=CLASSICAL,
momentum=0.5,
weight_decay=0.0001,
sparsity_constraint=False,
sparsity_target=0.1,
sparsity_cost=0.01,
sparsity_decay=0.9,
dropout=True,
dropout_rate=0.8,
epochs=20)
n_visible = train_x.get_value().shape[1]
n_visible2 = 10
n_hidden = 500
config = RBMConfig(v_n=n_visible,
v2_n=n_visible2,
h_n=n_hidden,
associative=True,
cd_type=CLASSICAL,
cd_steps=1,
train_params=tr,
progress_logger=ProgressLogger())
rbm = RBM(config)
store.move_to('label_test/' + str(rbm))
loaded = store.retrieve_object(str(rbm))
if loaded and cache:
rbm = loaded
print "... loaded precomputed rbm"
else:
rbm.train(train_x, train_y)
rbm.save()
# rbm.associative = False
# rbm.reconstruct(test_x, 10)
# rbm.associative = True
y = map(np.argmax, test_y.eval())
pred = rbm.classify(test_x)
print y
print pred
score = np.sum(y == pred) * 1. / len(y)
print "Classification Rate: {}".format(score)
def associate_data2dataADBN(cache=False, train_further=True):
print "Testing Associative RBM which tries to learn even-oddness of numbers"
f = open('adbnlog.txt', 'a')
# project set-up
data_manager = store.StorageManager('AssociativeDBN_digits', log=False)
shape = 28
train_n = 10000
test_n = 1000
# Load mnist hand digits, class label is already set to binary
dataset = m_loader.load_digits(n=[train_n, 100, test_n],
digits=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
pre={'binary_label': True})
tr_x, tr_y = dataset[0]
te_x, te_y = dataset[2]
tr_x01 = m_loader.sample_image(tr_y)
clf = SimpleClassifier('logistic', te_x.get_value(), te_y.eval())
for dropout in [True, False]:
for sc in [True, False]:
for lr in [0.001, 0.0001, 0.00005, 0.00001]:
for n in [100, 250, 500]:
config = get_brain_model_AssociativeDBNConfig(28, data_manager=data_manager)
config.top_rbm.train_params.learning_rate = lr
config.top_rbm.train_params.sparsity_constraint = sc
config.top_rbm.train_params.dropout = dropout
config.n_association = n
adbn = associative_dbn.AssociativeDBN(config=config, data_manager=data_manager)
brain_c = adbn
f.write(str(brain_c.association_layer) + "\n")
errors = []
for i in xrange(0, 5):
f.write("Epoch %d \n" % (i * 10))
brain_c.train(tr_x, tr_x01,
cache=[[True, True, True], [True, True, True], False],
train_further=[[False, True, True], [False, True, True], True])
if i == 0:
# Reconstruction
recon_right = brain_c.dbn_left.reconstruct(tr_x, k=10, plot_every=1, plot_n=100,
img_name='adbn_left_recon_{}'.format(shape))
recon_left = brain_c.dbn_right.reconstruct(tr_x01, k=10, plot_every=1, plot_n=100,
img_name='adbn_right_recon_{}'.format(shape))
for j in [5, 10]:
recon_x = brain_c.recall(te_x, associate_steps=j, recall_steps=0,
img_name='adbn_child_recon_{}1'.format(shape), y_type='active_h')
error = clf.get_score(recon_x, te_y.eval())
f.write("active_h %f\n" % error)
recon_x = brain_c.recall(te_x, associate_steps=j, recall_steps=0,
img_name='adbn_child_recon_{}2'.format(shape),
y_type='v_noisy_active_h')
error = clf.get_score(recon_x, te_y.eval())
f.write("v_noisy_active_h %f\n" % error)
f.close()
def get_data(shape):
dataset_name = 'sharp_equi{}_{}'.format(shape, shape)
# Load data
train, valid, test = m_loader.load_digits(n=[1000, 10, 100])
# train, valid, test = k_loader.load_kanade(set_name=dataset_name, pre={'scale': True})
train_x, train_y = train
return train_x
def test_sample_image(self):
train, valid, test = m_loader.load_digits(digits=[2, 3], n=[10, 0, 0], pre={'binary_label':True})
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
train_x01 = m_loader.sample_image(train_y, shared=False)
print train_y.eval()
utils.save_images(train_x01, 'test_image/sampled_img.png')
def test_load_vectorised(self):
dataset = m_loader.load_digits(shared=False, pre={'label_vector':True}, n=[10, 10, 10])
train, valid, test = dataset
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
self.assertTrue(train_y.shape[1] == 10)
self.assertTrue(valid_y.shape[1] == 10)
self.assertTrue(test_y.shape[1] == 10)
def test_sample_image(self):
train, valid, test = m_loader.load_digits(digits=[2, 3],
n=[10, 0, 0],
pre={'binary_label': True})
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
train_x01 = m_loader.sample_image(train_y, shared=False)
print train_y.eval()
utils.save_images(train_x01, 'test_image/sampled_img.png')
def associate_data2dataADBN_Finetune(cache=False, train_further=False):
print "Testing Associative RBM which tries to learn even-oddness of numbers"
f = open('adbn_errors.txt', 'w')
# project set-up
proj_name = 'ADBN_digits'
data_manager = store.StorageManager(proj_name, log=True)
shape = 28
train_n = 10000
test_n = 1000
# Load mnist hand digits, class label is already set to binary
dataset = m_loader.load_digits(n=[train_n, 0, test_n],
digits=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
pre={'binary_label': True})
tr_x, tr_y = dataset[0]
te_x, te_y = dataset[2]
tr_x01 = m_loader.sample_image(tr_y)
te_x01 = m_loader.sample_image(te_y)
clf = SimpleClassifier('logistic', te_x01.get_value(), te_y.eval())
configs = get_adbns()
for a in xrange(10):
for n, config in enumerate(configs):
t = 'l{}_r{}_t{}'.format(config.left_dbn.topology, config.right_dbn.topology, config.n_association)
f.write('{}:{}:'.format(a, t))
brain_c = associative_dbn.AssociativeDBN(config=config,
data_manager=store.StorageManager('{}/{}'.format(proj_name, n),
log=False))
brain_c.train(tr_x, tr_x01, cache=True, train_further=True)
recon = brain_c.recall(tr_x,
associate_steps=10,
recall_steps=0,
img_name='{}_{}'.format(a, t))
error = clf.get_score(recon, tr_y.eval())
print error
f.write('{}, '.format(error))
for i in xrange(0, 10):
brain_c.fine_tune(tr_x, tr_x01, epochs=1)
for y_type in ['active_h', 'v_noisy_active_h', 'zero']:
recon = brain_c.recall(tr_x,
associate_steps=10,
recall_steps=0,
img_name='{}_{}_{}_{}'.format(a, t, y_type, i),
y_type=y_type)
error = clf.get_score(recon, tr_y.eval())
print error
f.write('{}, '.format(error))
f.write('\n')
f.close()
def test_load_vectorised(self):
dataset = m_loader.load_digits(shared=False,
pre={'label_vector': True},
n=[10, 10, 10])
train, valid, test = dataset
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
self.assertTrue(train_y.shape[1] == 10)
self.assertTrue(valid_y.shape[1] == 10)
self.assertTrue(test_y.shape[1] == 10)
def test_shared(self):
dataset = m_loader.load_digits(n=[10, 10, 10])
train, valid, test = dataset
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
# print type(train_x)
# print type(train_y)
self.assertTrue(isinstance(train_x, theano.tensor.sharedvar.TensorSharedVariable))
self.assertTrue(isinstance(valid_x, theano.tensor.sharedvar.TensorSharedVariable))
self.assertTrue(isinstance(test_x, theano.tensor.sharedvar.TensorSharedVariable))
def test_load_raw(self):
dataset = m_loader.load_digits(shared=False)
self.assertTrue(len(dataset) == 3)
train, valid, test = dataset
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
self.assertTrue(
len(train_x[0]) == 784 and len(valid_x[0]) == len(train_x[0])
and len(test_x[0]) == len(train_x[0]))
self.assertTrue(len(train_x) == len(train_y) and len(train_y) == 50000)
self.assertTrue(len(valid_x) == len(valid_y) and len(valid_y) == 10000)
self.assertTrue(len(test_x) == len(test_y) and len(test_y) == 10000)
def test_shared(self):
dataset = m_loader.load_digits(n=[10, 10, 10])
train, valid, test = dataset
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
# print type(train_x)
# print type(train_y)
self.assertTrue(
isinstance(train_x, theano.tensor.sharedvar.TensorSharedVariable))
self.assertTrue(
isinstance(valid_x, theano.tensor.sharedvar.TensorSharedVariable))
self.assertTrue(
isinstance(test_x, theano.tensor.sharedvar.TensorSharedVariable))
def test_load_raw(self):
dataset = m_loader.load_digits(shared=False)
self.assertTrue(len(dataset) == 3)
train, valid, test = dataset
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
self.assertTrue(len(train_x[0]) == 784 and
len(valid_x[0]) == len(train_x[0]) and
len(test_x[0]) == len(train_x[0]))
self.assertTrue(len(train_x) == len(train_y) and len(train_y) == 50000)
self.assertTrue(len(valid_x) == len(valid_y) and len(valid_y) == 10000)
self.assertTrue(len(test_x) == len(test_y) and len(test_y) == 10000)
def test_load_fixed_number(self):
# [train_n, valid_n, test_n]
dataset = m_loader.load_digits(shared=False, n=[500, 400, 300])
self.assertTrue(len(dataset) == 3)
train, valid, test = dataset
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
# Statistics
# print sp.stats.itemfreq(train_y)
self.assertTrue(len(train_x[0]) == 784 and
len(valid_x[0]) == len(train_x[0]) and
len(test_x[0]) == len(train_x[0]))
self.assertTrue(len(train_x) == len(train_y) and len(train_y) == 500)
self.assertTrue(len(valid_x) == len(valid_y) and len(valid_y) == 400)
self.assertTrue(len(test_x) == len(test_y) and len(test_y) == 300)
def test_load_fixed_number(self):
# [train_n, valid_n, test_n]
dataset = m_loader.load_digits(shared=False, n=[500, 400, 300])
self.assertTrue(len(dataset) == 3)
train, valid, test = dataset
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
# Statistics
# print sp.stats.itemfreq(train_y)
self.assertTrue(
len(train_x[0]) == 784 and len(valid_x[0]) == len(train_x[0])
and len(test_x[0]) == len(train_x[0]))
self.assertTrue(len(train_x) == len(train_y) and len(train_y) == 500)
self.assertTrue(len(valid_x) == len(valid_y) and len(valid_y) == 400)
self.assertTrue(len(test_x) == len(test_y) and len(test_y) == 300)
def test_rbm():
print "Testing RBM"
# Load mnist hand digits
datasets = mnist_loader.load_digits(n=[100, 0, 100], digits=[1])
train_set_x, train_set_y = datasets[0]
test_set_x, test_set_y = datasets[2]
# Initialise the RBM and training parameters
tr = rbm_config.TrainParam(learning_rate=0.01,
momentum_type=NESTEROV,
momentum=0.5,
weight_decay=0.01,
sparsity_constraint=True,
sparsity_target=0.01,
sparsity_cost=0.01,
sparsity_decay=0.1)
n_visible = train_set_x.get_value().shape[1]
n_hidden = 2
config = rbm_config.RBMConfig(v_n=n_visible,
v2_n=n_visible,
h_n=n_hidden,
associative=False,
cd_type=CLASSICAL,
cd_steps=1,
train_params=tr,
progress_logger=ProgressLogger())
rbm = RBM(config)
print "... initialised RBM"
curr_dir = store.move_to(str(rbm))
print "... moved to {}".format(curr_dir)
# Train RBM
rbm.train(train_set_x)
# Test RBM
rbm.reconstruct(test_set_x, k=1, plot_n=20)
def test_rbm():
print "Testing RBM"
# Load mnist hand digits
datasets = mnist_loader.load_digits(n=[100, 0, 100], digits=[1])
train_set_x, train_set_y = datasets[0]
test_set_x, test_set_y = datasets[2]
# Initialise the RBM and training parameters
tr = rbm_config.TrainParam(learning_rate=0.01,
momentum_type=NESTEROV,
momentum=0.5,
weight_decay=0.01,
sparsity_constraint=True,
sparsity_target=0.01,
sparsity_cost=0.01,
sparsity_decay=0.1)
n_visible = train_set_x.get_value().shape[1]
n_hidden = 2
config = rbm_config.RBMConfig(v_n=n_visible,
v2_n=n_visible,
h_n=n_hidden,
associative=False,
cd_type=CLASSICAL,
cd_steps=1,
train_params=tr,
progress_logger=ProgressLogger())
rbm = RBM(config)
print "... initialised RBM"
curr_dir = store.move_to(str(rbm))
print "... moved to {}".format(curr_dir)
# Train RBM
rbm.train(train_set_x)
# Test RBM
rbm.reconstruct(test_set_x, k=1, plot_n=20)
def test_load_individual_digits(self):
chosen_digits = [0, 1]
dataset = m_loader.load_digits(shared=False, digits=chosen_digits)
self.assertTrue(len(dataset) == 3)
train, valid, test = dataset
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
self.assertTrue(len(train_x) == len(train_y))
self.assertTrue(len(valid_x) == len(valid_y))
self.assertTrue(len(test_x) == len(test_y))
self.assertTrue((np.unique(train_y) == np.array(chosen_digits)).all())
self.assertTrue((np.unique(valid_y) == np.array(chosen_digits)).all())
self.assertTrue((np.unique(test_y) == np.array(chosen_digits)).all())
utils.save_images(train_x[0:100], 'test_image/zero_and_one_train.png')
utils.save_images(valid_x[0:100], 'test_image/zero_and_one_valid.png')
utils.save_images(test_x[0:100], 'test_image/zero_and_one_test.png')
def test_load_individual_digits(self):
chosen_digits = [0, 1]
dataset = m_loader.load_digits(shared=False, digits=chosen_digits)
self.assertTrue(len(dataset) == 3)
train, valid, test = dataset
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
self.assertTrue(len(train_x) == len(train_y))
self.assertTrue(len(valid_x) == len(valid_y))
self.assertTrue(len(test_x) == len(test_y))
self.assertTrue((np.unique(train_y) == np.array(chosen_digits)).all())
self.assertTrue((np.unique(valid_y) == np.array(chosen_digits)).all())
self.assertTrue((np.unique(test_y) == np.array(chosen_digits)).all())
utils.save_images(train_x[0:100], 'test_image/zero_and_one_train.png')
utils.save_images(valid_x[0:100], 'test_image/zero_and_one_valid.png')
utils.save_images(test_x[0:100], 'test_image/zero_and_one_test.png')
def test_full(self):
train, valid, test = mnist_loader.load_digits(n=[100, 0, 100],
pre={'scale': True})
train_x, train_y = train
valid_x, valid_y = valid
test_x, test_y = test
tr = rbm_config.TrainParam(learning_rate=0.0001,
momentum_type=rbm.CLASSICAL,
momentum=0.5,
weight_decay=0.01,
sparsity_constraint=False,
sparsity_target=0.01,
sparsity_cost=0.01,
sparsity_decay=0.1,
epochs=10)
n_visible = train_x.get_value().shape[1]
n_hidden = 100
gaussian_rbm = rbm.GaussianRBM(
n_visible,
n_visible,
n_hidden,
associative=False,
cd_type=rbm.CLASSICAL,
cd_steps=1,
visible_unit=rbm_units.GaussianVisibleUnit,
hidden_unit=rbm_units.RBMUnit,
train_parameters=tr,
progress_logger=rbm_logger.ProgressLogger())
curr_dir = store.move_to('simple_gaussian_rbm_test')
print "... moved to {}".format(curr_dir)
# Train RBM
gaussian_rbm.train(train_x)
def associate_data2dataJDBN(cache=False, train_further=False):
print "Testing Associative RBM which tries to learn even-oddness of numbers"
f = open('errors.txt', 'w')
# project set-up
proj_name = 'JDBN_digits'
data_manager = store.StorageManager(proj_name, log=False)
shape = 28
train_n = 10000
test_n = 1000
# Load mnist hand digits, class label is already set to binary
dataset = m_loader.load_digits(n=[train_n, 0, test_n],
digits=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
pre={'binary_label': True})
tr_x, tr_y = dataset[0]
te_x, te_y = dataset[2]
tr_x01 = m_loader.sample_image(tr_y)
te_x01 = m_loader.sample_image(te_y)
ones = m_loader.load_digits(n=[test_n, 0, 0], digits=[1])[0][0]
zeroes = m_loader.load_digits(n=[test_n, 0, 0], digits=[0])[0][0]
tr_X = theano.shared(np.concatenate([tr_x.get_value(), tr_x01.get_value()], axis=1))
initial_y = np.random.binomial(n=1, p=0.0, size=te_x.get_value(True).shape).astype(t_float_x)
initial_y_uni = np.random.uniform(low=0, high=1, size=te_x.get_value(True).shape).astype(t_float_x)
initial_y_bi001 = np.random.binomial(n=1, p=0.01, size=te_x.get_value(True).shape).astype(t_float_x)
initial_y_bi01 = np.random.binomial(n=1, p=0.1, size=te_x.get_value(True).shape).astype(t_float_x)
te_X = theano.shared(np.concatenate([te_x.get_value(), initial_y], axis=1))
te_X2 = theano.shared(np.concatenate([te_x.get_value(), initial_y_uni], axis=1))
te_X3 = theano.shared(np.concatenate([te_x.get_value(), initial_y_bi01], axis=1))
te_X4 = theano.shared(np.concatenate([te_x.get_value(), initial_y_bi001], axis=1))
clf = SimpleClassifier('logistic', te_x01.get_value(), te_y.eval())
configs = []
for h_n in [100, 250, 500]:
config1 = get_brain_model_JointDBNConfig(shape, data_manager)
config1.topology = [784 * 2, h_n, h_n]
config1.rbm_configs[0].h_n = h_n
config1.rbm_configs[1].v_n = h_n
config1.rbm_configs[1].h_n = h_n
configs.append(config1)
for h_n in [100, 250, 500]:
config1 = get_brain_model_JointDBNConfig(shape, data_manager)
config1.topology = [784 * 2, h_n, h_n * 2]
config1.rbm_configs[0].h_n = h_n
config1.rbm_configs[1].v_n = h_n
config1.rbm_configs[1].h_n = h_n * 2
configs.append(config1)
for h_n in [100, 250, 500]:
config1 = get_brain_model_JointDBNConfig(shape, data_manager)
config1.topology = [784 * 2, h_n * 2, h_n]
config1.rbm_configs[0].h_n = h_n * 2
config1.rbm_configs[1].v_n = h_n * 2
config1.rbm_configs[1].h_n = h_n
configs.append(config1)
for a in xrange(10):
for config in configs:
f.write('{}:{}:'.format(a, config.topology))
brain_c = DBN.DBN(config=config)
brain_c.pretrain(tr_X, cache=[True, True], train_further=[True, True])
recon_x = brain_c.reconstruct(te_X, k=1, plot_n=100, img_name='{}_{}_recon'.format(a, config.topology))
recon = recon_x[:, 784:]
error = clf.get_score(recon, te_y.eval())
print error
f.write('{}, '.format(error))
for i in xrange(0, 5):
brain_c.fine_tune(tr_X)
recon_x = brain_c.reconstruct(te_X, k=1, plot_n=100,
img_name=('{}_{}_recon_fine_tune{}'.format(a, config.topology, i)))
recon = recon_x[:, 784:]
error = clf.get_score(recon, te_y.eval())
print error
f.write('{}, '.format(error))
recon_x = brain_c.reconstruct(te_X4, k=1, plot_n=100,
img_name=('{}_{}_recon_fine_tune_2_{}'.format(a, config.topology, i)))
recon = recon_x[:, 784:]
error = clf.get_score(recon, te_y.eval())
print error
f.write('{}, '.format(error))
f.write('\n')
f.close()
def associate_data2data(cache=False, train_further=True):
print "Testing Associative RBM which tries to learn even-oddness of numbers"
# project set-up
data_manager = store.StorageManager('EvenOddP', log=True)
train_n = 10000
test_n = 1000
# Load mnist hand digits, class label is already set to binary
dataset = m_loader.load_digits(n=[train_n, 100, test_n],
digits=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
pre={'binary_label': True})
tr_x, tr_y = dataset[0]
te_x, te_y = dataset[2]
tr_x01 = m_loader.sample_image(tr_y)
te_x01 = m_loader.sample_image(te_y)
ones = m_loader.load_digits(n=[test_n, 0, 0], digits=[1])[0][0]
zeroes = m_loader.load_digits(n=[test_n, 0, 0], digits=[0])[0][0]
concat1 = theano.function([], T.concatenate([tr_x, tr_x01], axis=1))()
# concat2 = theano.function([], T.concatenate([tr_x01, tr_x], axis=1))()
# c = np.concatenate([concat1, concat2], axis=0)
# np.random.shuffle(c)
# tr_concat_x = theano.shared(c, name='tr_concat_x')
tr_concat_x = theano.shared(concat1, name='tr_concat_x')
tr = TrainParam(learning_rate=0.001,
momentum_type=NESTEROV,
momentum=0.5,
weight_decay=0.1,
sparsity_constraint=True,
sparsity_target=0.1,
sparsity_decay=0.9,
sparsity_cost=0.1,
dropout=True,
dropout_rate=0.5,
epochs=1)
# Even odd test
k = 1
n_visible = 784 * 2
n_visible2 = 0
n_hidden = 300
print "number of hidden nodes: %d" % n_hidden
config = RBMConfig(v_n=n_visible,
v2_n=n_visible2,
h_n=n_hidden,
cd_type=CLASSICAL,
cd_steps=k,
train_params=tr,
progress_logger=ProgressLogger(img_shape=(28 * 2, 28)))
rbm = RBM(config=config)
# Load RBM (test)
loaded = store.retrieve_object(str(rbm))
if loaded and cache:
rbm = loaded
print "... loaded precomputed rbm"
errors = []
for i in xrange(0, 10):
# Train RBM
if not loaded or train_further:
rbm.train(tr_concat_x)
# Save RBM
data_manager.persist(rbm)
# Reconstruct using RBM
recon_x = rbm.reconstruct_association_opt(te_x, k=10, bit_p=0)
clf = SimpleClassifier('logistic', te_x.get_value(), te_y.eval())
orig = te_y.eval()
error = clf.get_score(recon_x, orig)
print error
errors.append(error)
print errors
def test_binary_label(self):
train, valid, test = m_loader.load_digits(digits=[2, 3], n=[10, 0, 0], pre={'binary_label':True})
def associate_data2dataDBN(cache=False):
print "Testing Joint DBN which tries to learn even-oddness of numbers"
# project set-up
data_manager = store.StorageManager('associative_dbn_test', log=True)
# Load mnist hand digits, class label is already set to binary
train, valid, test = m_loader.load_digits(n=[500, 100, 100], digits=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
pre={'binary_label': True})
train_x, train_y = train
test_x, test_y = test
train_x01 = m_loader.sample_image(train_y)
dataset01 = m_loader.load_digits(n=[500, 100, 100], digits=[0, 1])
# Initialise RBM parameters
# fixed base train param
base_tr = RBM.TrainParam(learning_rate=0.01,
momentum_type=RBM.CLASSICAL,
momentum=0.5,
weight_decay=0.0005,
sparsity_constraint=False,
epochs=20)
# top layer parameters
tr = RBM.TrainParam(learning_rate=0.1,
find_learning_rate=True,
momentum_type=RBM.NESTEROV,
momentum=0.5,
weight_decay=0.001,
sparsity_constraint=False,
epochs=20)
tr_top = RBM.TrainParam(learning_rate=0.1,
find_learning_rate=True,
momentum_type=RBM.CLASSICAL,
momentum=0.5,
weight_decay=0.001,
sparsity_constraint=False,
epochs=20)
# Layer 1
# Layer 2
# Layer 3
topology = [784, 500, 500, 100]
config = associative_dbn.DefaultADBNConfig()
config.topology_left = [784, 500, 500, 100]
config.topology_right = [784, 500, 500, 100]
config.reuse_dbn = False
config.top_rbm_params = tr_top
config.base_rbm_params = [base_tr, tr, tr]
for cd_type in [RBM.CLASSICAL, RBM.PERSISTENT]:
for n_ass in [100, 250, 500, 750, 1000]:
config.n_association = n_ass
config.top_cd_type = cd_type
# Construct DBN
assoc_dbn = associative_dbn.AssociativeDBN(config=config, data_manager=data_manager)
# Train
assoc_dbn.train(train_x, train_x01, cache=cache, optimise=True)
for n_recall in [1, 3, 5, 7, 10]:
for n_think in [0, 1, 3, 5, 7, 10]: # 1, 3, 5, 7, 10]:
# Reconstruct
sampled = assoc_dbn.recall(test_x, n_recall, n_think)
# Sample from top layer to generate data
sample_n = 100
utils.save_images(sampled, image_name='reconstruced_{}_{}_{}.png'.format(n_ass, n_recall, n_think),
shape=(sample_n / 10, 10))
dataset01[2] = (theano.shared(sampled), test_y)
def test_load_preprocessed(self):
dataset = m_loader.load_digits(shared=False, pre={'threshold':0.5}, n=[10, 10, 10])
for (x, y) in dataset:
# print sp.stats.itemfreq(x)
self.assertTrue(np.all((np.unique(x) == np.array([0, 1]))))
def test_digits(self):
tr = rbm_config.TrainParam(learning_rate=0.05,
momentum_type=rbm_config.CLASSICAL,
momentum=0.5,
weight_decay=0,
sparsity_constraint=False,
sparsity_target=0.1**9,
sparsity_cost=10**8,
sparsity_decay=0.9,
epochs=5)
config = rbm_config.RBMConfig()
config.v_n = 784
config.h_n = 100
config.v_unit = rbm_units.GaussianVisibleUnit
config.h_unit = rbm_units.ReLUnit
config.progress_logger = rbm_logger.ProgressLogger()
config.train_params = tr
np_rand = np.random.RandomState(123)
# Weights
W = np_rand.uniform(low=-1. / 10, high=1. / 10,
size=(784, 100)).astype(np.float32)
vb = np.zeros(784, dtype=np.float32)
hb = np.array(100, dtype=np.float32)
Wt = theano.shared(W, name='W')
vbt = theano.shared(vb, name='vbias')
hbt = theano.shared(hb, name='hbias')
g_rbm = rbm.RBM(config, W=Wt, h_bias=hbt, v_bias=vbt)
self.assertTrue(g_rbm)
self.assertTrue(isinstance(g_rbm.v_unit,
rbm_units.GaussianVisibleUnit))
self.assertTrue(isinstance(g_rbm.h_unit, rbm_units.RBMUnit))
self.assertTrue(
np.count_nonzero(g_rbm.W.get_value(borrow=True) - W) == 0)
self.assertTrue(
np.count_nonzero(g_rbm.v_bias.get_value(borrow=True) - vb) == 0)
self.assertTrue(
np.count_nonzero(g_rbm.h_bias.get_value(borrow=True) - hb) == 0)
tr, vl, te = mnist_loader.load_digits(n=[5, 10, 10],
pre={'scale': True})
v = tr[0]
# print 'inputs:'
# table = ss.itemfreq(v.get_value(borrow=True))
# x = [pt[0] for pt in table]
# y = [pt[1] for pt in table]
# plt.plot(x, y)
# plt.show()
# v = theano.shared(x)
_, _, h = g_rbm.sample_h_given_v(v)
_, _, vs = g_rbm.sample_v_given_h(h)
_, _, hs = g_rbm.sample_h_given_v(vs)
dw = T.dot(v.T, h) - T.dot(vs.T, hs)
dv = T.sum(v - vs, axis=0)
dh = T.sum(h - hs, axis=0)
gr = g_rbm.get_partial_derivatives(v, None)['gradients']
gdw, gdv, gdh = gr[0], gr[1], gr[2]
print gdw, gdv, gdh
compute_derivative = theano.function([], [dw, dv, dh, gdw, gdv, gdh])
for i in xrange(1):
a, b, c, d, e, f = compute_derivative()
# print a, b, c
print 'unfold'
print a[0], b[1:5], c[1:5]
print 'rbm'
print d[0], e[1:5], f[1:5]
def test_digits(self):
tr = rbm_config.TrainParam(learning_rate=0.05,
momentum_type=rbm_config.CLASSICAL,
momentum=0.5,
weight_decay=0,
sparsity_constraint=False,
sparsity_target=0.1 ** 9,
sparsity_cost=10 ** 8,
sparsity_decay=0.9,
epochs=5)
config = rbm_config.RBMConfig()
config.v_n = 784
config.h_n = 100
config.v_unit = rbm_units.GaussianVisibleUnit
config.h_unit = rbm_units.ReLUnit
config.progress_logger = rbm_logger.ProgressLogger()
config.train_params = tr
np_rand = np.random.RandomState(123)
# Weights
W = np_rand.uniform(low=-1./10, high=1./10, size=(784, 100)).astype(np.float32)
vb = np.zeros(784, dtype=np.float32)
hb = np.array(100, dtype=np.float32)
Wt = theano.shared(W, name='W')
vbt = theano.shared(vb, name='vbias')
hbt = theano.shared(hb, name='hbias')
g_rbm = rbm.RBM(config, W=Wt, h_bias=hbt, v_bias=vbt)
self.assertTrue(g_rbm)
self.assertTrue(isinstance(g_rbm.v_unit, rbm_units.GaussianVisibleUnit))
self.assertTrue(isinstance(g_rbm.h_unit, rbm_units.RBMUnit))
self.assertTrue(np.count_nonzero(g_rbm.W.get_value(borrow=True) - W) == 0)
self.assertTrue(np.count_nonzero(g_rbm.v_bias.get_value(borrow=True) - vb) == 0)
self.assertTrue(np.count_nonzero(g_rbm.h_bias.get_value(borrow=True) - hb) == 0)
tr, vl, te = mnist_loader.load_digits(n=[5, 10, 10], pre={'scale': True})
v = tr[0]
# print 'inputs:'
# table = ss.itemfreq(v.get_value(borrow=True))
# x = [pt[0] for pt in table]
# y = [pt[1] for pt in table]
# plt.plot(x, y)
# plt.show()
# v = theano.shared(x)
_, _, h = g_rbm.sample_h_given_v(v)
_, _, vs = g_rbm.sample_v_given_h(h)
_, _, hs = g_rbm.sample_h_given_v(vs)
dw = T.dot(v.T, h) - T.dot(vs.T, hs)
dv = T.sum(v - vs, axis=0)
dh = T.sum(h - hs, axis=0)
gr = g_rbm.get_partial_derivatives(v, None)['gradients']
gdw, gdv, gdh = gr[0], gr[1], gr[2]
print gdw, gdv, gdh
compute_derivative = theano.function([], [dw, dv, dh, gdw, gdv, gdh])
for i in xrange(1):
a, b, c, d, e, f = compute_derivative()
# print a, b, c
print 'unfold'
print a[0], b[1:5], c[1:5]
print 'rbm'
print d[0], e[1:5], f[1:5]
def test_binary_label(self):
train, valid, test = m_loader.load_digits(digits=[2, 3],
n=[10, 0, 0],
pre={'binary_label': True})
