def run(train_data, test_data):
batch_size = 10
n_samples = np.array(train_data).shape[0]
n_batches = int(np.ceil(float(n_samples) / batch_size))
batch_slices = list(
gen_even_slices(n_batches * batch_size, n_batches, n_samples))
nodes = [50, 75, 100, 150]
for item in nodes:
errors = []
model = BernoulliRBM(n_components=item,
learning_rate=0.1,
batch_size=10,
n_iter=1,
random_state=None,
verbose=1)
for _ in range(20):
for batch_slice in batch_slices:
model.partial_fit(train_data[batch_slice])
errors.append(percent_error(model.gibbs(test_data), test_data))
plot.plot_points(errors)
plot.plot_heatmap(reformat_data(test_data[0]))
plot.plot_heatmap(reformat_data(model.gibbs(test_data)[0]))
if item == 50 or item == 100:
plot.plot_heatmap(model.__dict__['components_'].reshape(item, 784))
def test_gibbs_smoke():
# Check if we don't get NaNs sampling the full digits dataset.
# Also check that sampling again will yield different results.
X = Xdigits
rbm1 = BernoulliRBM(n_components=42, batch_size=40, n_iter=20, random_state=42)
rbm1.fit(X)
X_sampled = rbm1.gibbs(X)
assert_all_finite(X_sampled)
X_sampled2 = rbm1.gibbs(X)
assert np.all((X_sampled != X_sampled2).max(axis=1))
def test_gibbs_smoke():
"""Check if we don't get NaNs sampling the full digits dataset.
Also check that sampling again will yield different results."""
X = Xdigits
rbm1 = BernoulliRBM(n_components=42, batch_size=40, n_iter=20, random_state=42)
rbm1.fit(X)
X_sampled = rbm1.gibbs(X)
assert_all_finite(X_sampled)
X_sampled2 = rbm1.gibbs(X)
assert_true(np.all((X_sampled != X_sampled2).max(axis=1)))
def rbm(epochs, hidden, eta, graph_error_epoch_relation):
print("Running the rbm...")
if(graph_error_epoch_relation):
print("Graphing average error as a function of epoch...")
error_list = []
for i in range(epochs):
rbm = BernoulliRBM(n_components=hidden, learning_rate=eta, batch_size=100, n_iter=i, verbose = True, random_state = 1)
rbm.fit(train_in)
total_error = 0
for image in train_in:
reconstruction = rbm.gibbs(image).astype(int)
error = mean_squared_error(image, reconstruction)
total_error = total_error + error
error_list.append(total_error/8000)
print(error_list)
plt.figure("Epoch-Loss relation in RBM")
plt.plot(error_list)
print("Creating reconstructed images, using test data...")
rbm = BernoulliRBM(n_components=hidden, learning_rate=eta, batch_size=100, n_iter=epochs, verbose = True, random_state = 1)
rbm.fit(train_in)
plt.figure("RBM mnist digits", figsize = (20, 4))
samples = [18, 3, 7, 0, 2, 1,15 , 8, 6 ,5 ]
i = 0
for index in samples:
image = test_in[index]
reconstruction = rbm.gibbs(image).astype(int)
# display original
ax = plt.subplot(2, 10, i + 1)
plt.imshow(image.reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# display reconstruction
ax = plt.subplot(2, 10, i + 1 + 10)
plt.imshow(reconstruction.reshape(28, 28))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
i = i + 1
plt.show()
return rbm
def test_rbm_pcd_gibbs(x_test,
Whv,
bh,
bv,
p_target=0.5,
n_gibbs_steps=5000,
thinning=10,
burnin=20):
rbm = BernoulliRBM(n_components=Whv.shape[0], learning_rate=0.0)
rbm.components_, rbm.intercept_hidden_, rbm.intercept_visible_ = Whv, bh, bv
evidence_mask = np.random.binomial(
1, p_target, x_test.shape) # 0: target node, 1: evidence node,
V = np.random.binomial(1, p_target, x_test.shape)
V = x_test * evidence_mask + V * (1 - evidence_mask)
prob1 = np.zeros_like(V)
count = 0
for it in range(n_gibbs_steps):
V = rbm.gibbs(V)
V = x_test * evidence_mask + V * (1 - evidence_mask)
if (it + 1) % thinning == 0 and it > burnin:
prob1 += V
count += 1
prob1 /= count
prob1_clipped = prob1.clip(1e-15, 1 - 1e-15)
target_mask = 1 - evidence_mask
logp = x_test * np.log(prob1_clipped) + (
1 - x_test) * np.log(1 - prob1_clipped)
logp *= target_mask
return -logp.sum() / target_mask.sum() / np.log(2)
def test_gibbs_smoke():
""" just seek if we don't get NaNs sampling the full digits dataset """
rng = np.random.RandomState(42)
X = Xdigits
rbm1 = BernoulliRBM(n_components=42, batch_size=10,
n_iter=20, random_state=rng)
rbm1.fit(X)
X_sampled = rbm1.gibbs(X)
assert_all_finite(X_sampled)
class RBM:
def __init__(self, images, n_components, learning_rate, batch_size, n_iter, random_state):
"""
:param images: input data for the RBM neural network
:param n_components: number of hidden units for the RBM neural network
:param learning_rate: learning rate for the RBM neural network
:param batch_size: batch size for the RBM neural network
:param n_iter: number of iterations/epochs for the RBM neural network
:param random_state: random state for the RBM neural network
"""
self.images = images
self.batch_size = batch_size
self.epochs = n_iter
self.x = 0
self.rbm = BernoulliRBM(
n_components=n_components,
learning_rate=learning_rate,
batch_size=batch_size,
n_iter=self.epochs,
random_state=random_state,
verbose=1)
def fit(self):
"""
:return: void
"""
self.x, _ = self.images.train.next_batch(self.batch_size)
self.rbm.fit(self.x)
def gibbs_sampling(self, k):
"""
:param k: number of steps of Gibbs sampling
:return: void
"""
for i in range(k):
gibbs_x = self.rbm.gibbs(self.x)
self.x = np.zeros_like(self.x)
self.x[gibbs_x] = 1
def generate_images(self, num_hidden_nodes):
"""
:param num_hidden_nodes: number of hidden nodes/units
:return: void
"""
plt.figure(figsize=(6, 6))
for i, comp in enumerate(self.x):
plt.subplot(10, 10, i + 1)
plt.imshow(comp.reshape((28, 28)), cmap="gray", interpolation='nearest')
plt.xticks(())
plt.yticks(())
plt.suptitle("RBM reconstructed image with " + str(num_hidden_nodes) + " hidden nodes", fontsize=16)
plt.subplots_adjust(wspace=0.1, hspace=0.1)
plt.savefig("RBM reconstructed image with " + str(num_hidden_nodes) + " hidden nodes.png")
def test_gibbs():
rng = np.random.RandomState(42)
X = Xdigits[:100]
rbm1 = BernoulliRBM(n_components=2, batch_size=5,
n_iter=5, random_state=rng)
rbm1.fit(X)
Xt1 = np.mean([rbm1.gibbs(X[0]) for i in range(100)], 0)
Xt2 = np.mean([rbm1._sample_visibles(rbm1._sample_hiddens(X[0], rng), rng)
for i in range(1000)], 0)
assert_almost_equal(Xt1, Xt2, decimal=1)
def test_fit_gibbs():
# Gibbs on the RBM hidden layer should be able to recreate [[0], [1]]
# from the same input
rng = np.random.RandomState(42)
X = np.array([[0.], [1.]])
rbm1 = BernoulliRBM(n_components=2, batch_size=2,
n_iter=42, random_state=rng)
# you need that much iters
rbm1.fit(X)
assert_almost_equal(rbm1.components_,
np.array([[0.02649814], [0.02009084]]), decimal=4)
assert_almost_equal(rbm1.gibbs(X), X)
return rbm1
def test_fit_gibbs_sparse():
# Gibbs on the RBM hidden layer should be able to recreate [[0], [1]] from
# the same input even when the input is sparse, and test against non-sparse
rbm1 = test_fit_gibbs()
rng = np.random.RandomState(42)
from scipy.sparse import csc_matrix
X = csc_matrix([[0.], [1.]])
rbm2 = BernoulliRBM(n_components=2, batch_size=2,
n_iter=42, random_state=rng)
rbm2.fit(X)
assert_almost_equal(rbm2.components_,
np.array([[0.02649814], [0.02009084]]), decimal=4)
assert_almost_equal(rbm2.gibbs(X), X.toarray())
assert_almost_equal(rbm1.components_, rbm2.components_)
class RBMModel(JointModel):
def __init__(self, hyper_params, random=True, name=None):
super(RBMModel, self).__init__(hyper_params, random, name)
self.model = BernoulliRBM()
def set_params(self, params):
self.model = BernoulliRBM(**params)
def evaluate(self, X):
return self.model.score_samples(X).mean()
def generate_samples(self, start, step=1):
for i in range(step):
start = self.model.gibbs(start)
return start
train_target = readData('targetdigit_trn')
test_input = readData('bindigit_tst')
test_target = readData('targetdigit_tst')
#create and train model
rbm = BernoulliRBM(n_components=50,
learning_rate=.2,
batch_size=100,
n_iter=20)
rbm.fit(train_input, y=test_input)
original = []
for image in range(len(first_number_index)):
original += [train_input[first_number_index[image]]]
reconstructed_boolean = rbm.gibbs(original)
reconstructed = np.zeros(np.shape(reconstructed_boolean))
for i in range(np.shape(reconstructed_boolean)[0]):
for j in range(np.shape(reconstructed_boolean)[1]):
if reconstructed_boolean[i][j]:
reconstructed[i][j] = 1
else:
reconstructed[i][j] = 0
for idx in range(10):
plt.imsave('images/rbm_org_' + str(idx) + '.png',
original[idx].reshape(28, 28),
cmap=cm.gray)
plt.imsave('images/rbm_rec_' + str(idx) + '.png',
reconstructed[idx].reshape(28, 28),
cmap=cm.gray)
def make_rbm(data, components=50, n_iter=20):
model = BernoulliRBM(n_components=components, learning_rate=0.1, batch_size=10, n_iter=n_iter,
random_state=None, verbose=0)
model = model.fit(data)
print("error: {}".format(percent_error(model.gibbs(data), data)))
return model
def main():
#Read data
train, train_targets = data_handling.read_train_dataset()
test, test_targets = data_handling.read_test_dataset()
## Models with different number of hidden nodes: 50,75,100,150
logistic_50 = linear_model.LogisticRegression()
logistic_75 = linear_model.LogisticRegression()
logistic_100 = linear_model.LogisticRegression()
logistic_150 = linear_model.LogisticRegression()
rbm_50 = BernoulliRBM(random_state=0, verbose=True)
rbm_75 = BernoulliRBM(random_state=0, verbose=True)
rbm_100 = BernoulliRBM(random_state=0, verbose=True)
rbm_150 = BernoulliRBM(random_state=0, verbose=True)
# Hyper-parameters:
learning_rate = 0.06
n_iter = 20
# More components (hidden nodes) tend to give better prediction performance, but larger fitting time
error_50 = []
error_75 = []
error_100 = []
error_150 = []
#Training:
classifier_50, rbm_50 = train_classifier(rbm_50,
logistic_50,
train,
train_targets,
learning_rate,
n_iter,
n_hnodes=50)
classifier_75, rbm_75 = train_classifier(rbm_75,
logistic_75,
train,
train_targets,
learning_rate,
n_iter,
n_hnodes=75)
classifier_100, rbm_100 = train_classifier(rbm_100,
logistic_100,
train,
train_targets,
learning_rate,
n_iter,
n_hnodes=100)
classifier_150, rbm_150 = train_classifier(rbm_150,
logistic_150,
train,
train_targets,
learning_rate,
n_iter,
n_hnodes=150)
# Evaluation
print("Evaluation:")
print(
"Logistic regression using RBM features with 50 hidden nodes:\n%s\n" %
(metrics.classification_report(test_targets,
classifier_50.predict(test))))
print(
"Logistic regression using RBM features with 75 hidden nodes:\n%s\n" %
(metrics.classification_report(test_targets,
classifier_75.predict(test))))
print(
"Logistic regression using RBM features with 100 hidden nodes:\n%s\n" %
(metrics.classification_report(test_targets,
classifier_100.predict(test))))
print(
"Logistic regression using RBM features with 150 hidden nodes:\n%s\n" %
(metrics.classification_report(test_targets,
classifier_150.predict(test))))
# Plotting
plot_images(rbm_50, rbm_75, rbm_100, rbm_150)
# Predict test set
# image from each digit
example_digits_indexs = [
18, 3, 7, 0, 2, 1, 14, 8, 6, 5
] # indexs in the test partition digits: 0,1,2,3,4,5,6,7,8,9
prediction_50 = rbm_50.gibbs(test).astype(int)
prediction_75 = rbm_75.gibbs(test).astype(int)
prediction_100 = rbm_100.gibbs(test).astype(int)
prediction_150 = rbm_150.gibbs(test).astype(int)
print(calculate_error(prediction_50, test))
print(calculate_error(prediction_75, test))
print(calculate_error(prediction_100, test))
print(calculate_error(prediction_150, test))
plt.figure(figsize=(20, 20))
for index, i in enumerate(example_digits_indexs):
plt.subplot(10, 5, 5 * index + 1)
plt.imshow(test[i].reshape((28, 28)),
cmap=plt.cm.gray_r,
interpolation='nearest')
plt.subplot(10, 5, 5 * index + 2)
plt.imshow(prediction_50[i].reshape((28, 28)),
cmap=plt.cm.gray_r,
interpolation='nearest')
plt.subplot(10, 5, 5 * index + 3)
plt.imshow(prediction_75[i].reshape((28, 28)),
cmap=plt.cm.gray_r,
interpolation='nearest')
plt.subplot(10, 5, 5 * index + 4)
plt.imshow(prediction_100[i].reshape((28, 28)),
cmap=plt.cm.gray_r,
interpolation='nearest')
plt.subplot(10, 5, 5 * index + 5)
plt.imshow(prediction_150[i].reshape((28, 28)),
cmap=plt.cm.gray_r,
interpolation='nearest')
plt.show()
samples.append(np.array(clone).flatten())
print(i)
samples = np.array(samples)
print(samples.shape)
random.shuffle(samples)
samples = np.array(samples)
train, test = samples[:1000], samples[1000:]
# Initializing the RBM and training it
epochs = int(input("How many epochs would you like? "))
rbm = BernoulliRBM(learning_rate = 0.1, n_iter = epochs, n_components = 512)
rbm.fit(train)
print("done")
# Recreating image
recreate = rbm.gibbs(cflaw).reshape(79, 106, 8)
#recreate = rbm.gibbs(train[0]).reshape(79, 106, 8)
accuracy = list(recreate.flatten() - train[0])
print(str(round(100*(accuracy.count(0)/66992), 4 )) + "%")
image = []
for y in range(0, len(recreate)):
col = []
for x in range(0, len(recreate[y])):
row = []
for i in range(0, len(recreate[y][x])):
if recreate[y][x][i]:
row.append('1')
else:
row.append('0')
col.append(int("".join(row), base=2))
image.append(col)
for dim in [50, 75, 100, 150]:
print "[Info] # Hidden Node =", dim
model = BernoulliRBM(n_components=dim)
model.set_params(learning_rate=0.02, random_state=1)
RBM_error = []
AE_error = []
for n_iter in range(6):
train_n_iter = (n_iter + 1) * 10
model.set_params(n_iter=train_n_iter, verbose=False)
print "Training RBM with n_iter =", train_n_iter, "and n_hidden =", dim, "..."
model.fit(train_data)
error_count = 0.
for i in range(n_exp):
error_count += np.sum(
np.sum(np.abs(model.gibbs(test_data) -
test_data))) / 784. / 2000. / n_exp
RBM_error.append(error_count)
print "RBM error with n_iter =", train_n_iter, ":", error_count
print "RMB Error with dim =", dim, ":", RBM_error
plt.plot(x, RBM_error, label="RMB {}".format(dim))
print "Training AE with n_iter =", 10, "and n_hidden =", dim, "..."
model = MLPClassifier(hidden_layer_sizes=(dim, ),
random_state=1,
learning_rate_init=0.003,
verbose=False,
max_iter=10,
warm_start=True)
model.fit(train_data, train_data)
AE_error.append(
