def recons():
data = Mnist()
data.MODE = "test"
model = RBM()
fig, ax = plt.subplots(nrows=2, ncols=10)
c = 0
for i in range(0, 100, 10):
x = data[i + 2][0].t()[0] * 255
x = x.view(784, 1)
mask = (torch.rand(x.shape).cuda() > 0.2).float()
x = x * mask
x_new = x.clone()
ax[0][c].imshow(x.view(28, 28).detach().cpu().numpy())
for i in range(6):
x_new = model.reconstruction(x_new)
x = x_new.view(28, 28).detach().cpu().numpy()
ax[1][c].imshow(x)
c += 1
plt.show()
def assess_rbm(clf, noisy_data, noisy_label, noisy_levels, tr_x,postfix=''):
f_score = open('report{}.txt'.format(postfix), 'a')
f_metric = open('metric{}.txt'.format(postfix), 'a')
# Initialise architecture
config = get_rbm_config(25, n_hidden=500, epochs=2)
model = RBM(config)
pred_table = {}
for l in xrange(len(noisy_levels)):
pred_table[l] = []
for i in xrange(50):
# Train architecture
model.train(tr_x)
j = 0
for xs, ys in zip(noisy_data, noisy_label):
recon_xs = model.reconstruct(xs, img_name='test_rbm')
pred, metric = clf.get_score(recon_xs, ys, True)
print pred
print metric
f_metric.write('{}25_25, Epoch:{}\n'.format(noisy_levels[j], i))
f_metric.write(metric)
pred_table[j].append(pred)
j += 1
for k in pred_table:
f_score.write('{}:{}\n'.format(noisy_levels[k], pred_table[k]))
f_score.close()
f_metric.close()
def assess_rbm(clf, noisy_data, noisy_label, noisy_levels, tr_x, postfix=''):
f_score = open('report{}.txt'.format(postfix), 'a')
f_metric = open('metric{}.txt'.format(postfix), 'a')
# Initialise architecture
config = get_rbm_config(25, n_hidden=500, epochs=2)
model = RBM(config)
pred_table = {}
for l in xrange(len(noisy_levels)):
pred_table[l] = []
for i in xrange(50):
# Train architecture
model.train(tr_x)
j = 0
for xs, ys in zip(noisy_data, noisy_label):
recon_xs = model.reconstruct(xs, img_name='test_rbm')
pred, metric = clf.get_score(recon_xs, ys, True)
print pred
print metric
f_metric.write('{}25_25, Epoch:{}\n'.format(noisy_levels[j], i))
f_metric.write(metric)
pred_table[j].append(pred)
j += 1
for k in pred_table:
f_score.write('{}:{}\n'.format(noisy_levels[k], pred_table[k]))
f_score.close()
f_metric.close()
def modeler():
data = Mnist()
model = RBM()
ks = [3, 1, 4, 2, 6, 4, 2, 7, 2, 6, 8, 11, 15, 2, 13, 1, 2, 6, 7, 4, 6]
for k in ks:
model.k = k
model.CDk(data)
def recon():
x = torch.load("weights/rbm/test.pt")
model = RBM()
fig, ax = plt.subplots(nrows=1, ncols=2)
x_new = x.clone()
ax[0].imshow(x.view(28, 28).numpy())
x_new = model.reconstruction(x_new.cuda())
ax[1].imshow(x_new.view(28, 28).detach().cpu().numpy())
plt.show()
def test_rbm():
print "Testing RBM"
data_manager = store.StorageManager('TestRBM')
# Load Cohn Kanade dataset
datasets = loader.load_kanade(pre={'scale': True}, n=100, set_name='sharp_equi25_25')
train_set_x, train_set_y = datasets[0]
test_set_x, test_set_y = datasets[2]
# Initilise the RBM and training parameters
tr = TrainParam(learning_rate=0.0001,
momentum_type=NESTEROV,
momentum=0.5,
weight_decay=0.0001,
sparsity_constraint=True,
sparsity_target=0.01,
sparsity_cost=0.1,
sparsity_decay=0.9,
dropout=True,
dropout_rate=0.5,
batch_size=10,
epochs=10)
n_visible = train_set_x.get_value(borrow=True).shape[1]
n_hidden = 10
config = RBMConfig()
config.v_n = n_visible
config.h_n = n_hidden
config.v_unit = rbm_units.GaussianVisibleUnit
# config.h_unit = rbm_units.ReLUnit
config.progress_logger = ProgressLogger(img_shape=(25, 25))
config.train_params = tr
rbm = RBM(config)
print "... initialised RBM"
load = store.retrieve_object(str(rbm))
if load:
rbm = load
for i in xrange(0, 1):
# Train RBM
rbm.train(train_set_x)
data_manager.persist(rbm)
# Test RBM Reconstruction via Linear Classifier
clf = SimpleClassifier(classifier='logistic', train_x=train_set_x, train_y=train_set_y)
recon_te = rbm.reconstruct(test_set_x, k=1, plot_n=100, plot_every=1,img_name='recon_te_{}.png'.format(i))
print 'Original Score: {}'.format(clf.get_score(test_set_x, test_set_y))
print 'Recon Score: {}'.format(clf.get_score(recon_te, test_set_y.eval()))
def untie_weights(self, include_top=False):
# Untie all the weights except for the top layer
self.untied = True
layers = self.rbm_layers if include_top else self.rbm_layers[:-1]
for i, rbm in enumerate(layers):
W = rbm.W.get_value(borrow=False)
h_bias = rbm.h_bias.get_value(borrow=False)
v_bias = rbm.v_bias.get_value(borrow=False)
W1 = theano.shared(W, name=('inference_W_%d' % i))
v1 = theano.shared(v_bias, name=('inference_vbias_%d' % i))
h1 = theano.shared(h_bias, name=('inference_hbias_%d' % i))
W2 = theano.shared(copy.deepcopy(W), name=('generative_W_%d' % i))
v2 = theano.shared(copy.deepcopy(v_bias), name=('generative_vbias_%d' % i))
h2 = theano.shared(copy.deepcopy(h_bias), name=('generative_hbias_%d' % i))
self.inference_layers.append(RBM(config=rbm.config, W=W1, h_bias=h1, v_bias=v1))
self.generative_layers.append(RBM(config=rbm.config, W=W2, h_bias=h2, v_bias=v2))
def get_brain_model_RBM(shape):
# Initialise RBM
tr = TrainParam(learning_rate=0.0001,
momentum_type=NESTEROV,
momentum=0.9,
weight_decay=0.0001,
sparsity_constraint=True,
sparsity_decay=0.9,
sparsity_cost=100,
sparsity_target=0.01,
batch_size=10,
epochs=10)
n_visible = shape * shape * 2
n_hidden = 500
config = RBMConfig()
config.v_n = n_visible
config.h_n = n_hidden
config.v_unit = rbm_units.GaussianVisibleUnit
# config.h_unit = rbm_units.ReLUnit
config.progress_logger = ProgressLogger(img_shape=(shape * 2, shape))
config.train_params = tr
brain_c = RBM(config)
print "... initialised RBM"
return brain_c
def setUpAssociativeRBM(self, v=5, v2=5, h=10):
tr = TrainParam(learning_rate=0.01,
momentum_type=CLASSICAL,
momentum=0.5,
weight_decay=0.01,
sparsity_constraint=False,
batch_size=1,
epochs=5)
config = RBMConfig()
config.associative = True
config.v_n = v
config.v2_n = v2
config.h_n = h
config.train_params = tr
config.progress_logger = AssociationProgressLogger()
self.rbm = RBM(config)
self.x = np.array([[1, 1, 1, 0, 0]], dtype=t_float_x)
self.y = np.array([[0, 0, 0, 0, 1]], dtype=t_float_x)
self.x2 = np.array([[1, 1, 1, 0, 0], [0, 0, 0, 0, 1]], dtype=t_float_x)
self.y2 = np.array([[0, 0, 1, 1, 1], [0, 0, 0, 0, 1]], dtype=t_float_x)
self.yl = np.array([[0, 0], [0, 1]], dtype=t_float_x)
self.tx = theano.shared(np.array([[1, 1, 1, 0, 0]], dtype=t_float_x))
self.ty = theano.shared(np.array([[0, 0, 0, 0, 1]], dtype=t_float_x))
self.tz = theano.shared(
np.array([[0, 0, 0, 0, 1], [0, 0, 0, 0, 1], [0, 0, 0, 0, 1],
[0, 0, 0, 0, 1], [0, 0, 0, 0, 1]],
dtype=t_float_x))
self.tl = theano.shared(
np.array([[0, 1], [0, 1], [0, 1], [0, 1], [1, 0]],
dtype=t_float_x))
def classify():
data = Mnist()
rbm = RBM()
model = RBMClassifier(rbm).to(device)
loss_fn = torch.nn.CrossEntropyLoss(reduction='mean')
optimizer = Adam(model.classifier.parameters(),
lr=0.0001,
weight_decay=0.00001)
for e in range(100):
tots = 0
for i, (x, y) in enumerate(data):
if torch.Size([784, 0]) == x.shape:
break
out = model(x)
optimizer.zero_grad()
loss = loss_fn(out, y.long())
tots += loss.item()
loss.backward()
optimizer.step()
if i % 50 == 0:
print(e, i, tots / (i + 1))
torch.save(model.state_dict(), "weights/rbm/rbmclass.pth")
def train_kanade():
print "Testing RBM"
data_manager = store.StorageManager('Kanade/SimpleRBMTest')
# Load mnist hand digits
datasets = loader.load_kanade(n=500, set_name='25_25', emotions=['happy', 'sadness'], pre={'scale2unit': True})
train_x, train_y = datasets[0]
sparsity_constraint = True
# Initialise the RBM and training parameters
tr = rbm_config.TrainParam(learning_rate=0.0001,
momentum_type=rbm_config.NESTEROV,
momentum=0.9,
weight_decay=0.001,
sparsity_constraint=sparsity_constraint,
sparsity_target=0.01,
sparsity_cost=1,
sparsity_decay=0.9,
dropout=True,
epochs=100)
n_visible = train_x.get_value().shape[1]
n_hidden = 500
config = rbm_config.RBMConfig(v_n=n_visible,
v2_n=n_visible,
h_n=n_hidden,
v_unit=rbm_units.GaussianVisibleUnit,
associative=False,
cd_type=rbm_config.CLASSICAL,
cd_steps=1,
train_params=tr,
progress_logger=rbm_logger.ProgressLogger(img_shape=(25, 25)))
rbm = RBM(config)
print "... initialised RBM"
# Train RBM
rbm.train(train_x)
# Test RBM
rbm.reconstruct(train_x, k=5, plot_n=10, plot_every=1)
# Store Parameters
data_manager.persist(rbm)
def __init__(self,
visible_units=256,
hidden_units=[64, 100],
k=2,
learning_rate=1e-5,
learning_rate_decay=False,
xavier_init=False,
increase_to_cd_k=False,
use_gpu=False):
super(DBN, self).__init__()
self.n_layers = len(hidden_units)
self.rbm_layers = []
self.rbm_nodes = []
# Creating different RBM layers
for i in range(self.n_layers):
input_size = 0
if i == 0:
input_size = visible_units
else:
input_size = hidden_units[i - 1]
rbm = RBM(visible_units=input_size,
hidden_units=hidden_units[i],
k=k,
learning_rate=learning_rate,
learning_rate_decay=learning_rate_decay,
xavier_init=xavier_init,
increase_to_cd_k=increase_to_cd_k,
use_gpu=use_gpu)
self.rbm_layers.append(rbm)
# rbm_layers = [RBM(rbn_nodes[i-1] , rbm_nodes[i],use_gpu=use_cuda) for i in range(1,len(rbm_nodes))]
self.W_rec = [
nn.Parameter(self.rbm_layers[i].W.data.clone())
for i in range(self.n_layers - 1)
]
self.W_gen = [
nn.Parameter(self.rbm_layers[i].W.data)
for i in range(self.n_layers - 1)
]
self.bias_rec = [
nn.Parameter(self.rbm_layers[i].h_bias.data.clone())
for i in range(self.n_layers - 1)
]
self.bias_gen = [
nn.Parameter(self.rbm_layers[i].v_bias.data)
for i in range(self.n_layers - 1)
]
self.W_mem = nn.Parameter(self.rbm_layers[-1].W.data)
self.v_bias_mem = nn.Parameter(self.rbm_layers[-1].v_bias.data)
self.h_bias_mem = nn.Parameter(self.rbm_layers[-1].h_bias.data)
for i in range(self.n_layers - 1):
self.register_parameter('W_rec%i' % i, self.W_rec[i])
self.register_parameter('W_gen%i' % i, self.W_gen[i])
self.register_parameter('bias_rec%i' % i, self.bias_rec[i])
self.register_parameter('bias_gen%i' % i, self.bias_gen[i])
def associate_data2label(cache=False):
cache=True
print "Testing ClassRBM with generative target (i.e. AssociativeRBM with picture-label association)"
# Load mnist hand digits, class label is already set to binary
train, valid, test = m_loader.load_digits(n=[50000, 100, 30], pre={'label_vector': True})
train_x, train_y = train
test_x, test_y = test
train_y = T.cast(train_y, dtype=theano.config.floatX)
test_y = T.cast(test_y, dtype=theano.config.floatX)
# Initialise the RBM and training parameters
tr = TrainParam(learning_rate=0.001,
momentum_type=CLASSICAL,
momentum=0.5,
weight_decay=0.0001,
sparsity_constraint=False,
sparsity_target=0.1,
sparsity_cost=0.01,
sparsity_decay=0.9,
dropout=True,
dropout_rate=0.8,
epochs=20)
n_visible = train_x.get_value().shape[1]
n_visible2 = 10
n_hidden = 500
config = RBMConfig(v_n=n_visible,
v2_n=n_visible2,
h_n=n_hidden,
associative=True,
cd_type=CLASSICAL,
cd_steps=1,
train_params=tr,
progress_logger=ProgressLogger())
rbm = RBM(config)
store.move_to('label_test/' + str(rbm))
loaded = store.retrieve_object(str(rbm))
if loaded and cache:
rbm = loaded
print "... loaded precomputed rbm"
else:
rbm.train(train_x, train_y)
rbm.save()
# rbm.associative = False
# rbm.reconstruct(test_x, 10)
# rbm.associative = True
y = map(np.argmax, test_y.eval())
pred = rbm.classify(test_x)
print y
print pred
score = np.sum(y == pred) * 1. / len(y)
print "Classification Rate: {}".format(score)
def get_instance(model, n_hidden=64):
settings = {
"n_visible": 784,
"n_hidden": n_hidden,
"learning_rate": 0.01,
"momentum": 0.95,
"use_tqdm": True
}
if model == "rbm":
return RBM(**settings)
if model == "frbm":
return FRBM(**settings)
return None
def test():
data = Mnist()
data.MODE = "test"
rbm = RBM()
model = RBMClassifier(rbm).to(device)
model.load_state_dict(torch.load("weights/rbm/rbmclass.pth"))
sm = torch.nn.Softmax()
x, y = data[0]
out = model(x)
val, ind = out.max(1)
def __init__(self, config=DBNConfig()):
numpy_rng = config.numpy_rng
theano_rng = config.theano_rng
topology = config.topology
load_layer = config.load_layer
n_outs = config.n_outs
data_manager = config.data_manager
out_dir = config.out_dir
tr = config.training_parameters
rbm_configs = config.rbm_configs
n_ins = topology[1]
hidden_layers_sizes = topology[1:]
self.sigmoid_layers = []
self.rbm_layers = []
self.untied = False
self.inference_layers = []
self.generative_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
self.topology = topology
self.out_dir = out_dir
self.data_manager = data_manager
assert self.n_layers > 0
if not numpy_rng:
numpy_rng = np.random.RandomState(123)
else:
numpy_rng = np.random.RandomState(123)
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
if type(tr) is not list:
tr = [tr for i in xrange(self.n_layers)]
if type(rbm_configs) is not list:
rbm_configs = [rbm_configs for i in xrange(self.n_layers)]
for i in xrange(self.n_layers):
rbm_config = rbm_configs[i]
rbm_config.v_n = topology[i]
rbm_config.h_n = topology[i + 1]
# rbm_config.training_parameters = tr[i] # Ensure it has parameters
rbm_layer = RBM(rbm_config)
self.rbm_layers.append(rbm_layer)
def train_kanade():
print "Testing RBM"
data_manager = store.StorageManager('Kanade/SimpleRBMTest')
# Load mnist hand digits
datasets = loader.load_kanade(n=500,
set_name='25_25',
emotions=['happy', 'sadness'],
pre={'scale2unit': True})
train_x, train_y = datasets[0]
sparsity_constraint = True
# Initialise the RBM and training parameters
tr = rbm_config.TrainParam(learning_rate=0.0001,
momentum_type=rbm_config.NESTEROV,
momentum=0.9,
weight_decay=0.001,
sparsity_constraint=sparsity_constraint,
sparsity_target=0.01,
sparsity_cost=1,
sparsity_decay=0.9,
dropout=True,
epochs=100)
n_visible = train_x.get_value().shape[1]
n_hidden = 500
config = rbm_config.RBMConfig(
v_n=n_visible,
v2_n=n_visible,
h_n=n_hidden,
v_unit=rbm_units.GaussianVisibleUnit,
associative=False,
cd_type=rbm_config.CLASSICAL,
cd_steps=1,
train_params=tr,
progress_logger=rbm_logger.ProgressLogger(img_shape=(25, 25)))
rbm = RBM(config)
print "... initialised RBM"
# Train RBM
rbm.train(train_x)
# Test RBM
rbm.reconstruct(train_x, k=5, plot_n=10, plot_every=1)
# Store Parameters
data_manager.persist(rbm)
def setUpSimpleRBM(self):
v = 2
v2 = 2
h = 10
tr = TrainParam(learning_rate=0.01,
momentum_type=CLASSICAL,
momentum=0.5,
weight_decay=0.01,
sparsity_constraint=False,
batch_size=1,
epochs=5)
config = RBMConfig()
config.associative = True
config.v_n = v
config.v2_n = v2
config.h_n = h
config.train_params = tr
config.progress_logger = ProgressLogger()
self.rbm = RBM(config)
def setUpRBM(self):
v = 5
v2 = 5
h = 10
tr = TrainParam(learning_rate=0.01,
momentum_type=CLASSICAL,
momentum=0.5,
weight_decay=0.01,
sparsity_constraint=False,
batch_size=1,
epochs=5)
config = RBMConfig()
config.associative = True
config.v_n = v
config.v2_n = v2
config.h_n = h
config.train_params = tr
config.progress_logger = ProgressLogger()
self.rbm = RBM(config)
self.rbmx1 = np.array([[2, 5, 5, 2, 1]], dtype=t_float_x)
self.rbmx2 = np.array([[5, 2, 3, 8, 1]], dtype=t_float_x)
def setUpRBM(self):
v = 5
v2 = 5
h = 10
tr = TrainParam(learning_rate=0.01,
momentum_type=CLASSICAL,
momentum=0.5,
weight_decay=0.01,
sparsity_constraint=False,
batch_size=1,
epochs=15)
config = RBMConfig()
config.v_n = v
config.v2_n = v2
config.h_n = h
config.train_params = tr
config.progress_logger = ProgressLogger()
self.rbm = RBM(config)
self.x = np.array([[1, 1, 1, 0, 0]], dtype=t_float_x)
self.tx = theano.shared(np.array([[1, 1, 1, 0, 0]], dtype=t_float_x))
self.x2 = np.array([[1, 1, 1, 0, 0], [0, 0, 0, 0, 1]], dtype=t_float_x)
self.tx2 = theano.shared(
np.array([
[1, 1, 1, 0, 0],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 1],
[0, 0, 0, 0, 1],
],
dtype=t_float_x))
biases_h, biases_v, weights, likelihoods = rbm1.train(); generate_animation(rbm1, 'rbm1', weights, biases_v, biases_h) generate_noise_animation(rbm1, 'rbm1', weights, biases_v, biases_h) generate_sample_animation(rbm1, 'rbm1', samples, weights, biases_v, biases_h) images, labels = import_mnist(); images = (images - np.min(images, 0)) / (np.max(images, 0) + 0.0001); rbm2 = RBM(images, num_iter=5, num_h=400); biases_h, biases_v, weights, likelihoods = rbm2.train(); generate_animation(rbm2, 'rbm2', weights, biases_v, biases_h) generate_noise_animation(rbm2, 'rbm2', weights, biases_v, biases_h) generate_sample_animation(rbm2, 'rbm2', samples, weights, biases_v, biases_h)""" rbm3 = RBM(images, num_iter=40, num_h=400) biases_h, biases_v, weights, likelihoods = rbm3.train() test_logistic_regression(weights, biases_v, biases_h) test_generator(rbm3, weights, biases_v, biases_h) generate_likelihood_plot(likelihoods) generate_filter_plots(weights) generate_sample_plot(rbm3, samples, weights, biases_v, biases_h) generate_noise_plot(rbm3, weights, biases_v, biases_h) # generate_animation(rbm3, 'rbm3', weights, biases_v, biases_h) # generate_noise_animation(rbm3, 'rbm3', weights, biases_v, biases_h) # generate_sample_animation(rbm3, 'rbm3', samples, weights, biases_v, biases_h) # test_logistic_regression(weights, biases_v, biases_h)
EPOCHS = 1
SIZE_HIDDEN = 500
SIZE_VISIBLE = 784
# load binary mnist sample dataset
dataset = pandas.read_csv(MNIST_TRAIN_BINARY, delimiter=',', dtype=int, header=None)
# leave the first column out since it contains the labels
dataset = dataset.values[:, 1:]
# compute batch set
idx = prepare_batches(len(dataset), SIZE_BATCH)
# load distribution
bernoulli = Bernoulli(SIZE_VISIBLE, SIZE_HIDDEN)
gibbs = BlockGibbsSampler(bernoulli, sampling_steps=1)
sgd = SGD(bernoulli)
rbm = RBM(bernoulli, gibbs, sgd)
y_axis = list()
# pyplot.plot(y_axis)
# pyplot.ylabel('update')
# pyplot.xlabel('Batch')
# pyplot.ion()
# pyplot.show()
for epoch in range(EPOCHS):
for b_idx in idx:
batch = dataset[b_idx[0]:b_idx[1], :]
d_weight_update, _, _ = rbm.train_batch(batch)
rec_error = rbm.compute_reconstruction_error(batch)
def __init__(self,
np_rng,
theano_rng=None,
n_ins=784,
hidden_layers_sizes=[500, 500],
n_outs=10):
"""This class is made to support a variable number of layers.
:type np_rng: np.random.RandomState
:param np_rng: np random number generator used to draw initial
weights
:type theano_rng: theano.tensor.shared_randomstreams.RandomStreams
:param theano_rng: Theano random generator; if None is given one is
generated based on a seed drawn from `rng`
:type n_ins: int
:param n_ins: dimension of the inputs to the DBN
:type hidden_layers_sizes: list of ints
:param hidden_layers_sizes: intermediate layers size, must contain
at least one value
:type n_outs: int
:param n_outs: dimension of the output of the network
"""
self.sigmoid_layers = []
self.rbm_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng:
theano_rng = MRG_RandomStreams(np_rng.randint(2**30))
# allocate symbolic variables for the data
# the data is presented as rasterized images
self.x = self.inputs = T.matrix('x')
# the labels are presented as 1D vector of [int] labels
self.y = T.ivector('y')
# end-snippet-1
# The DBN is an MLP, for which all weights of intermediate
# layers are shared with a different RBM. We will first
# construct the DBN as a deep multilayer perceptron, and when
# constructing each sigmoidal layer we also construct an RBM
# that shares weights with that layer. During pretraining we
# will train these RBMs (which will lead to chainging the
# weights of the MLP as well) During finetuning we will finish
# training the DBN by doing stochastic gradient descent on the
# MLP.
for i in range(self.n_layers):
# construct the sigmoidal layer
# the size of the inputs is either the number of hidden
# units of the layer below or the inputs size if we are on
# the first layer
if i == 0: inputs_size = n_ins
else: inputs_size = hidden_layers_sizes[i - 1]
# the inputs to this layer is either the activation of the
# hidden layer below or the inputs of the DBN if you are on
# the first layer
if i == 0: layer_inputs = self.x
else: layer_inputs = self.sigmoid_layers[-1].output
sigmoid_layer = Hidden_Layer(rng=np_rng,
inputs=layer_inputs,
n_in=inputs_size,
n_out=hidden_layers_sizes[i],
activation=activation)
# add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
# its arguably a philosophical question... but we are
# going to only declare that the parameters of the
# sigmoid_layers are parameters of the DBN. The visible
# biases in the RBM are parameters of those RBMs, but not
# of the DBN.
self.params.extend(sigmoid_layer.params)
# Construct an RBM that shared weights with this layer
rbm_layer = RBM(np_rng=np_rng,
theano_rng=theano_rng,
inputs=layer_inputs,
n_visible=inputs_size,
n_hidden=hidden_layers_sizes[i],
w=sigmoid_layer.w,
hbias=sigmoid_layer.b)
self.rbm_layers.append(rbm_layer)
# We now need to add a logistic layer on top of the MLP
self.logitLayer = Logistic_Regression(
inputs=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
self.params.extend(self.logitLayer.params)
# compute the cost for second phase of training, defined as the
# negative log likelihood of the logistic regression (output) layer
self.finetune_cost = self.logitLayer.negativeLogLikelihood(self.y)
# compute the gradients with respect to the model parameters
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logitLayer.errors(self.y)
# get prediction from logistic sgd
self.y_pred = self.logitLayer.y_pred
self.p_y_given_x = self.logitLayer.p_y_given_x
pass
def KanadeAssociativeRBM(cache=False, train_further=False):
print "Testing Associative RBM which tries to learn the ID map "
# print "Testing Associative RBM which tries to learn the following mapping: {anger, saddness, disgust} -> {sadness}, {contempt, happy, surprise} -> {happy}"
# project set-up
data_manager = store.StorageManager('Kanade/OptMFSparse0.01RBMTest', log=True)
# data_manager = store.StorageManager('Kanade/OptAssociativeRBMTest', log=True)
shape = 25
dataset_name = 'sharp_equi{}_{}'.format(shape, shape)
# Load kanade database
mapping = None # id map
# mapping = {'anger': 'sadness', 'contempt': 'happy', 'disgust': 'sadness', 'fear': 'sadness', 'happy': 'happy',
# 'sadness': 'sadness', 'surprise': 'happy'}
train, valid, test = loader.load_kanade(pre={'scale': True}, set_name=dataset_name)
train_x, train_y = train
test_x, test_y = test
# Sample associated image
train_x_mapped, train_y_mapped = loader.sample_image(train_y, mapping=mapping, pre={'scale': True},
set_name=dataset_name)
test_x_mapped, test_y_mapped = loader.sample_image(test_y, mapping=mapping, pre={'scale': True},
set_name=dataset_name)
# Concatenate images
concat1 = T.concatenate([train_x, train_x_mapped], axis=1)
# concat2 = T.concatenate([train_x_mapped, train_x], axis=1)
# concat = T.concatenate([concat1, concat2], axis=0)
# train_tX = theano.function([], concat)()
train_tX = theano.function([], concat1)()
train_X = theano.shared(train_tX)
# Train classifier to be used for classifying reconstruction associated image layer
# mapped_data = loader.load_kanade(#emotions=['sadness', 'happy'],
# pre={'scale': True},
# set_name=dataset_name) # Target Image
# clf_orig = SimpleClassifier('logistic', mapped_data[0][0], mapped_data[0][1])
clf_orig = SimpleClassifier('logistic', train_x, train_y)
# Initialise RBM
tr = rbm_config.TrainParam(learning_rate=0.0001,
momentum_type=rbm_config.NESTEROV,
momentum=0.9,
weight_decay=0.0001,
sparsity_constraint=True,
sparsity_target=0.01,
sparsity_cost=100,
sparsity_decay=0.9,
batch_size=10,
epochs=10)
n_visible = shape * shape * 2
n_hidden = 500
config = rbm_config.RBMConfig()
config.v_n = n_visible
config.h_n = n_hidden
config.v_unit = rbm_units.GaussianVisibleUnit
# config.h_unit = rbm_units.ReLUnit
config.progress_logger = rbm_logger.ProgressLogger(img_shape=(shape * 2, shape))
config.train_params = tr
rbm = RBM(config)
print "... initialised RBM"
# Load RBM (test)
loaded = data_manager.retrieve(str(rbm))
if loaded:
rbm = loaded
else:
rbm.set_initial_hidden_bias()
rbm.set_hidden_mean_activity(train_X)
# Train RBM - learn joint distribution
# rbm.pretrain_lr(train_x, train_x01)
for i in xrange(0, 10):
if not cache or train_further:
rbm.train(train_X)
data_manager.persist(rbm)
print "... reconstruction of associated images"
# Get reconstruction with train data to get 'mapped' images to train classifiers on
reconstruction = rbm.reconstruct(train_X, 1,
plot_n=100,
plot_every=1,
img_name='recon_train')
reconstruct_assoc_part = reconstruction[:, (shape ** 2):]
# Get associated images of test data
nsamples = np.random.normal(0, 1, test_x.get_value(True).shape).astype(np.float32)
initial_y = theano.shared(nsamples, name='initial_y')
utils.save_images(nsamples[0:100], 'initialisation.png', (10, 10), (25, 25))
test_x_associated = rbm.reconstruct_association_opt(test_x, initial_y,
5,
0.,
plot_n=100,
plot_every=1,
img_name='recon_test_gibbs')
mf_recon = rbm.mean_field_inference_opt(test_x, y=initial_y,
sample=False,
k=10,
img_name='recon_test_mf_raw')
# Concatenate images
test_MFX = theano.function([], T.concatenate([test_x, mf_recon], axis=1))()
test_MF = theano.shared(test_MFX)
reconstruction = rbm.reconstruct(test_MF, 1,
plot_n=100,
plot_every=1,
img_name='recon_test_mf_recon')
mf_recon = reconstruction[:, (shape ** 2):]
print "... reconstructed"
# Classify the reconstructions
# 1. Train classifier on original images
score_orig = clf_orig.get_score(test_x_associated, test_y_mapped.eval())
score_orig_mf = clf_orig.get_score(test_x_associated, test_y_mapped.eval())
# 2. Train classifier on reconstructed images
clf_recon = SimpleClassifier('logistic', reconstruct_assoc_part, train_y_mapped.eval())
score_retrain = clf_recon.get_score(test_x_associated, test_y_mapped.eval())
score_retrain_mf = clf_recon.get_score(mf_recon, test_y_mapped.eval())
out_msg = '{} (orig, retrain):{},{}'.format(rbm, score_orig, score_retrain)
out_msg2 = '{} (orig, retrain):{},{}'.format(rbm, score_orig_mf, score_retrain_mf)
print out_msg
print out_msg2
new_training_params = {'n_epoch': 20,
'lrate': 0.1,
'w_penalty': 0.01,
'verbose': True}
# 'display_every': 1.,
# 'visualize': True,
# 'vis_fun': vis_fun}
# UPDATE TRAINING PARAMETERS
train_params.update(new_training_params)
# print ''
# print '-'*5+'Training parameters'+'-'*5
# train_params.show()
# INITIALIZE RBM MODEL AND TRAINER
model = RBM(model_params)
trainer = RBMTrainer(model, train_params)
# print ''
# print '-'*5+'RBM Object'+'-'*5
# trainer.rbm.show()
# print ''
# print '-'*5+'RBM Trainer Object'+'-'*5
# trainer.show()
import h5py
mnist_file = '/home/dustin/data/featureLearning/MNIST/mnistLarge.mat'
mnist = h5py.File(mnist_file)
data = mnist['testData'].value.T
images, labels = import_mnist()
images = (images - np.min(images, 0)) / (np.max(images, 0) + 0.0001)
labels = labels
# transform labels into one-hot encoding
labels_one_hot = []
for i in range(0, np.size(labels, axis=0)):
one_hot = [0] * 10
one_hot[labels[i]] = 1
labels_one_hot.append(one_hot)
train_data = np.concatenate((images, labels_one_hot), axis=1)
rbm1 = RBM(train_data, num_iter=16, num_h=450)
biases_h, biases_v, weights, likelihoods = rbm1.train()
# generate_likelihood_plot(likelihoods)
images_test, labels_test = import_test()
images_test = (images_test -
np.min(images_test, 0)) / (np.max(images_test, 0) + 0.0001)
num_correct = 0
predictions = []
for i in range(0, np.size(images_test, axis=0)):
model_ans = compute_largest_prob(images_test[i], biases_h, weights)
predictions.append(model_ans)
if (model_ans == labels_test[i]):
def associate_data2data(cache=False, train_further=True):
print "Testing Associative RBM which tries to learn even-oddness of numbers"
# project set-up
data_manager = store.StorageManager('EvenOddP', log=True)
train_n = 10000
test_n = 1000
# Load mnist hand digits, class label is already set to binary
dataset = m_loader.load_digits(n=[train_n, 100, test_n],
digits=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
pre={'binary_label': True})
tr_x, tr_y = dataset[0]
te_x, te_y = dataset[2]
tr_x01 = m_loader.sample_image(tr_y)
te_x01 = m_loader.sample_image(te_y)
ones = m_loader.load_digits(n=[test_n, 0, 0], digits=[1])[0][0]
zeroes = m_loader.load_digits(n=[test_n, 0, 0], digits=[0])[0][0]
concat1 = theano.function([], T.concatenate([tr_x, tr_x01], axis=1))()
# concat2 = theano.function([], T.concatenate([tr_x01, tr_x], axis=1))()
# c = np.concatenate([concat1, concat2], axis=0)
# np.random.shuffle(c)
# tr_concat_x = theano.shared(c, name='tr_concat_x')
tr_concat_x = theano.shared(concat1, name='tr_concat_x')
tr = TrainParam(learning_rate=0.001,
momentum_type=NESTEROV,
momentum=0.5,
weight_decay=0.1,
sparsity_constraint=True,
sparsity_target=0.1,
sparsity_decay=0.9,
sparsity_cost=0.1,
dropout=True,
dropout_rate=0.5,
epochs=1)
# Even odd test
k = 1
n_visible = 784 * 2
n_visible2 = 0
n_hidden = 300
print "number of hidden nodes: %d" % n_hidden
config = RBMConfig(v_n=n_visible,
v2_n=n_visible2,
h_n=n_hidden,
cd_type=CLASSICAL,
cd_steps=k,
train_params=tr,
progress_logger=ProgressLogger(img_shape=(28 * 2, 28)))
rbm = RBM(config=config)
# Load RBM (test)
loaded = store.retrieve_object(str(rbm))
if loaded and cache:
rbm = loaded
print "... loaded precomputed rbm"
errors = []
for i in xrange(0, 10):
# Train RBM
if not loaded or train_further:
rbm.train(tr_concat_x)
# Save RBM
data_manager.persist(rbm)
# Reconstruct using RBM
recon_x = rbm.reconstruct_association_opt(te_x, k=10, bit_p=0)
clf = SimpleClassifier('logistic', te_x.get_value(), te_y.eval())
orig = te_y.eval()
error = clf.get_score(recon_x, orig)
print error
errors.append(error)
print errors
def associate_data2dataDBN(cache=False):
print "Testing Joint DBN which tries to learn even-oddness of numbers"
# project set-up
data_manager = store.StorageManager('associative_dbn_test', log=True)
# Load mnist hand digits, class label is already set to binary
train, valid, test = m_loader.load_digits(n=[500, 100, 100], digits=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
pre={'binary_label': True})
train_x, train_y = train
test_x, test_y = test
train_x01 = m_loader.sample_image(train_y)
dataset01 = m_loader.load_digits(n=[500, 100, 100], digits=[0, 1])
# Initialise RBM parameters
# fixed base train param
base_tr = RBM.TrainParam(learning_rate=0.01,
momentum_type=RBM.CLASSICAL,
momentum=0.5,
weight_decay=0.0005,
sparsity_constraint=False,
epochs=20)
# top layer parameters
tr = RBM.TrainParam(learning_rate=0.1,
find_learning_rate=True,
momentum_type=RBM.NESTEROV,
momentum=0.5,
weight_decay=0.001,
sparsity_constraint=False,
epochs=20)
tr_top = RBM.TrainParam(learning_rate=0.1,
find_learning_rate=True,
momentum_type=RBM.CLASSICAL,
momentum=0.5,
weight_decay=0.001,
sparsity_constraint=False,
epochs=20)
# Layer 1
# Layer 2
# Layer 3
topology = [784, 500, 500, 100]
config = associative_dbn.DefaultADBNConfig()
config.topology_left = [784, 500, 500, 100]
config.topology_right = [784, 500, 500, 100]
config.reuse_dbn = False
config.top_rbm_params = tr_top
config.base_rbm_params = [base_tr, tr, tr]
for cd_type in [RBM.CLASSICAL, RBM.PERSISTENT]:
for n_ass in [100, 250, 500, 750, 1000]:
config.n_association = n_ass
config.top_cd_type = cd_type
# Construct DBN
assoc_dbn = associative_dbn.AssociativeDBN(config=config, data_manager=data_manager)
# Train
assoc_dbn.train(train_x, train_x01, cache=cache, optimise=True)
for n_recall in [1, 3, 5, 7, 10]:
for n_think in [0, 1, 3, 5, 7, 10]: # 1, 3, 5, 7, 10]:
# Reconstruct
sampled = assoc_dbn.recall(test_x, n_recall, n_think)
# Sample from top layer to generate data
sample_n = 100
utils.save_images(sampled, image_name='reconstruced_{}_{}_{}.png'.format(n_ass, n_recall, n_think),
shape=(sample_n / 10, 10))
dataset01[2] = (theano.shared(sampled), test_y)
def _create_prior(self):
logger.debug("_create_prior")
num_rbm_nodes_per_layer = self._config.model.n_latent_hierarchy_lvls * self._latent_dimensions // 2
return RBM(n_visible=num_rbm_nodes_per_layer,
n_hidden=num_rbm_nodes_per_layer)
SIZE_VISIBLE = 784
# load binary mnist sample dataset
dataset = pandas.read_csv(MNIST_TRAIN, delimiter=',', dtype=numpy.float64, header=None)
# leave the first column out since it contains the labels
# dataset must be normalized to have unit variance by column (sigma_i == 1)
dataset = pre.scale(dataset.values[:,1:], axis=0)
# compute batch set
idx = prepare_batches(len(dataset), SIZE_BATCH)
# load distribution
gaussian = GaussianBinary(SIZE_VISIBLE, SIZE_HIDDEN)
gibbs = BlockGibbsSampler(gaussian, sampling_steps=1)
sgd = SGD(gaussian, learning_rate=0.001, weight_decay=0, momentum=0)
rbm = RBM(gaussian, gibbs, sgd)
# pyplot.figure(1)
# pyplot.ion()
# pyplot.show()
# vmin = numpy.min(dataset)
# vmax = numpy.max(dataset)
for epoch in range(EPOCHS):
for b_idx in idx:
batch = dataset[b_idx[0]:b_idx[1], :]
d_weight_update, _, _ = rbm.train_batch(batch)
rec_probs, rec_state = rbm.reconstruct(batch,steps=10)
pyplot.clf()
img = numpy.reshape(rec_state[-1,:], newshape=(28,28))
def KanadeAssociativeRBM(cache=False, train_further=False):
print "Testing Associative RBM which tries to learn the ID map "
# print "Testing Associative RBM which tries to learn the following mapping: {anger, saddness, disgust} -> {sadness}, {contempt, happy, surprise} -> {happy}"
# project set-up
data_manager = store.StorageManager('Kanade/OptMFSparse0.01RBMTest',
log=True)
# data_manager = store.StorageManager('Kanade/OptAssociativeRBMTest', log=True)
shape = 25
dataset_name = 'sharp_equi{}_{}'.format(shape, shape)
# Load kanade database
mapping = None # id map
# mapping = {'anger': 'sadness', 'contempt': 'happy', 'disgust': 'sadness', 'fear': 'sadness', 'happy': 'happy',
# 'sadness': 'sadness', 'surprise': 'happy'}
train, valid, test = loader.load_kanade(pre={'scale': True},
set_name=dataset_name)
train_x, train_y = train
test_x, test_y = test
# Sample associated image
train_x_mapped, train_y_mapped = loader.sample_image(train_y,
mapping=mapping,
pre={'scale': True},
set_name=dataset_name)
test_x_mapped, test_y_mapped = loader.sample_image(test_y,
mapping=mapping,
pre={'scale': True},
set_name=dataset_name)
# Concatenate images
concat1 = T.concatenate([train_x, train_x_mapped], axis=1)
# concat2 = T.concatenate([train_x_mapped, train_x], axis=1)
# concat = T.concatenate([concat1, concat2], axis=0)
# train_tX = theano.function([], concat)()
train_tX = theano.function([], concat1)()
train_X = theano.shared(train_tX)
# Train classifier to be used for classifying reconstruction associated image layer
# mapped_data = loader.load_kanade(#emotions=['sadness', 'happy'],
# pre={'scale': True},
# set_name=dataset_name) # Target Image
# clf_orig = SimpleClassifier('logistic', mapped_data[0][0], mapped_data[0][1])
clf_orig = SimpleClassifier('logistic', train_x, train_y)
# Initialise RBM
tr = rbm_config.TrainParam(learning_rate=0.0001,
momentum_type=rbm_config.NESTEROV,
momentum=0.9,
weight_decay=0.0001,
sparsity_constraint=True,
sparsity_target=0.01,
sparsity_cost=100,
sparsity_decay=0.9,
batch_size=10,
epochs=10)
n_visible = shape * shape * 2
n_hidden = 500
config = rbm_config.RBMConfig()
config.v_n = n_visible
config.h_n = n_hidden
config.v_unit = rbm_units.GaussianVisibleUnit
# config.h_unit = rbm_units.ReLUnit
config.progress_logger = rbm_logger.ProgressLogger(img_shape=(shape * 2,
shape))
config.train_params = tr
rbm = RBM(config)
print "... initialised RBM"
# Load RBM (test)
loaded = data_manager.retrieve(str(rbm))
if loaded:
rbm = loaded
else:
rbm.set_initial_hidden_bias()
rbm.set_hidden_mean_activity(train_X)
# Train RBM - learn joint distribution
# rbm.pretrain_lr(train_x, train_x01)
for i in xrange(0, 10):
if not cache or train_further:
rbm.train(train_X)
data_manager.persist(rbm)
print "... reconstruction of associated images"
# Get reconstruction with train data to get 'mapped' images to train classifiers on
reconstruction = rbm.reconstruct(train_X,
1,
plot_n=100,
plot_every=1,
img_name='recon_train')
reconstruct_assoc_part = reconstruction[:, (shape**2):]
# Get associated images of test data
nsamples = np.random.normal(0, 1,
test_x.get_value(True).shape).astype(
np.float32)
initial_y = theano.shared(nsamples, name='initial_y')
utils.save_images(nsamples[0:100], 'initialisation.png', (10, 10),
(25, 25))
test_x_associated = rbm.reconstruct_association_opt(
test_x,
initial_y,
5,
0.,
plot_n=100,
plot_every=1,
img_name='recon_test_gibbs')
mf_recon = rbm.mean_field_inference_opt(test_x,
y=initial_y,
sample=False,
k=10,
img_name='recon_test_mf_raw')
# Concatenate images
test_MFX = theano.function([], T.concatenate([test_x, mf_recon],
axis=1))()
test_MF = theano.shared(test_MFX)
reconstruction = rbm.reconstruct(test_MF,
1,
plot_n=100,
plot_every=1,
img_name='recon_test_mf_recon')
mf_recon = reconstruction[:, (shape**2):]
print "... reconstructed"
# Classify the reconstructions
# 1. Train classifier on original images
score_orig = clf_orig.get_score(test_x_associated,
test_y_mapped.eval())
score_orig_mf = clf_orig.get_score(test_x_associated,
test_y_mapped.eval())
# 2. Train classifier on reconstructed images
clf_recon = SimpleClassifier('logistic', reconstruct_assoc_part,
train_y_mapped.eval())
score_retrain = clf_recon.get_score(test_x_associated,
test_y_mapped.eval())
score_retrain_mf = clf_recon.get_score(mf_recon, test_y_mapped.eval())
out_msg = '{} (orig, retrain):{},{}'.format(rbm, score_orig,
score_retrain)
out_msg2 = '{} (orig, retrain):{},{}'.format(rbm, score_orig_mf,
score_retrain_mf)
print out_msg
print out_msg2
def associate_data2dataDBN(cache=False):
print "Testing Associative DBN which tries to learn even-oddness of numbers"
# project set-up
data_manager = store.StorageManager('Kanade/associative_dbn_test',
log=True)
# Load mnist hand digits, class label is already set to binary
dataset = loader.load_kanade(n=500,
emotions=['anger', 'sadness', 'happy'],
pre={'scale2unit': True})
train_x, train_y = dataset
train_x01 = loader.sample_image(train_y)
dataset01 = loader.load_kanade(n=500)
# Initialise RBM parameters
# fixed base train param
base_tr = RBM.TrainParam(learning_rate=0.001,
momentum_type=RBM.CLASSICAL,
momentum=0.5,
weight_decay=0.0005,
sparsity_constraint=False,
epochs=20)
# top layer parameters
tr = RBM.TrainParam(
learning_rate=0.001,
# find_learning_rate=True,
momentum_type=RBM.NESTEROV,
momentum=0.5,
weight_decay=0.001,
sparsity_constraint=False,
epochs=20)
tr_top = RBM.TrainParam(
learning_rate=0.001,
# find_learning_rate=True,
momentum_type=RBM.CLASSICAL,
momentum=0.5,
weight_decay=0.001,
sparsity_constraint=False,
epochs=20)
# Layer 1
# Layer 2
# Layer 3
# topology = [784, 500, 500, 100]
config = associative_dbn.DefaultADBNConfig()
config.topology_left = [625, 500, 500, 100]
config.topology_right = [625, 500, 500, 100]
config.reuse_dbn = False
config.top_rbm_params = tr_top
config.base_rbm_params = [base_tr, tr, tr]
count = 0
for cd_type in [RBM.CLASSICAL, RBM.PERSISTENT]:
for n_ass in [100, 250, 500, 750, 1000]:
config.n_association = n_ass
config.top_cd_type = cd_type
# Construct DBN
ass_dbn = associative_dbn.AssociativeDBN(config=config,
data_manager=data_manager)
# Train
for trainN in xrange(0, 5):
ass_dbn.train(train_x, train_x01, cache=cache)
for n_recall in [1, 3, 10]:
for n_think in [0, 1, 3, 5, 10]: # 1, 3, 5, 7, 10]:
# Reconstruct
sampled = ass_dbn.recall(train_x, n_recall, n_think)
# Sample from top layer to generate data
sample_n = 100
utils.save_images(
sampled,
image_name='{}_reconstruced_{}_{}_{}.png'.format(
count, n_ass, n_recall, n_think),
shape=(sample_n / 10, 10),
img_shape=(25, 25))
count += 1
