def main():
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
''' restricted boltzmann machine '''
print("\nStarting a Restricted Boltzmann Machine..")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=200,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
# try:
# rbm.loadfromfile_rbm(loc="trained_rbm", name="vis--hid")
# except IOError:
# rbm.cd1(visible_trainset=train_imgs, n_iterations=1)
# rbm.savetofile_rbm("trained_single_rbm", "single")
rbm.cd1(visible_trainset=train_imgs, n_iterations=10, disphist=False)
for im in range(10):
rbm.recall(test_imgs[im], "generations/recall" + str(im))
save_img("generations/data" + str(im), test_imgs[im])
# get_activations(test_imgs, rbm)
rbm.reconstruct_error(test_imgs)
def units_stats():
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
units = np.array([500, 400, 300, 200])
stats = []
for i in range(len(units)):
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=units[i],
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
stats.append(
rbm.cd1(visible_trainset=train_imgs,
n_iterations=5,
verbose=True,
stats_err=True))
# Plot
x = np.arange(len(stats[0]))
for i in range(len(stats)):
plt.plot(x, stats[i], label="h_dim=" + str(units[i]))
plt.title(
"Reconstruction error trough the contrastive divergence algorithm")
plt.xlabel("Reconstruction Error")
plt.xlabel("Epoch")
plt.legend(loc="upper right")
plt.show()
def plot_batch_probabilities():
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=200,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
rbm.cd1(visible_trainset=train_imgs, n_iterations=10000, calc_err=False)
batch_input = test_imgs[:rbm.batch_size]
p_h_given_v, _ = rbm.get_h_given_v(batch_input)
image = p_h_given_v * 255
image = Image.fromarray(image.astype(np.uint8))
image.save('batch_probs_20_batch.png', format='PNG')
def recall_digits():
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=10)
rbm.cd1(visible_trainset=train_imgs, n_iterations=10000, calc_err=False)
for i in range(10):
plt.imshow(np.reshape(test_imgs[i], (28, 28)), cmap='gray')
plt.title("Test image " + str(i + 1))
plt.show()
""" print(test_imgs[i].shape)
image = test_imgs[i] * 255
image = image.reshape((28, 28))
image = Image.fromarray(image.astype(np.uint8))
image.save('img_init_' + str(i), format='PNG') """
recon = rbm.reconstruct_img(test_imgs[i])
""" image = recon * 255
image = image.reshape((28, 28))
image = Image.fromarray(image.astype(np.uint8))
image.save('img_recon_' + str(i), format='PNG') """
plt.imshow(np.reshape(recon, (28, 28)), cmap='gray')
plt.title("Reconstructed image " + str(i + 1))
plt.show()
def train_net_units(units, n_epochs, results):
print("Starting network with " + str((units + 2) * 100) + " units")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=(units + 2) * 100,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
res = rbm.cd1(visible_trainset=train_imgs,
max_epochs=n_epochs,
n_iterations=3000,
bool_print=True)
results[units, :] = res
def plot_recon_err():
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
hidden_units = [200, 300, 400, 500]
#hidden_units = [200]
results = []
iters = 1000
''' restricted boltzmann machine '''
print("\nStarting a Restricted Boltzmann Machine..")
for i in range(len(hidden_units)):
print(i)
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=hidden_units[i],
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=10)
results.append(
rbm.cd1(visible_trainset=train_imgs,
n_iterations=iters,
calc_err=True))
epochs = np.arange(len(results[0]))
for i in range(len(results)):
plt.plot(epochs, results[i], label=str(hidden_units[i]) + " units")
plt.legend(loc="upper right")
plt.xticks(np.arange(0, 11, step=1))
plt.xlabel("Iterations")
plt.ylabel("Average reconstruction error")
plt.show()
# Visualize the first 10 digits
for i in range(10):
plot_digit(train_imgs[i], train_lbls_digits[i])
if RBM:
# Restricted Boltzmann Machine
print ("\nStarting a Restricted Boltzmann Machine...")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * \
image_size[1], ndim_hidden=200, is_bottom=True,
image_size=image_size, is_top=False, n_labels=10, batch_size=20,
learning_rate=0.1)
rbm.cd1(X=train_imgs, n_iterations=30000)
if DBN:
# Deep Belief Net
if TWO_LAYER:
print ("\n>>> Starting a two-layer Deep Belief Net...")
# Initialize the two layer network
dbn = DeepBeliefNetTwoLayer(sizes={"vis": image_size[0] *
image_size[1], "hid": 500, "top": 2000, "lbl": 10},
image_size=image_size, n_labels=10, batch_size=10)
else:
print ("\n>>> Starting a two-layer Deep Belief Net...")
# Initialize the three layer network
dbn = DeepBeliefNet(sizes={"vis": image_size[0] * image_size[1],
"hid": 500, "pen": 500, "top": 2000, "lbl": 10},
train_imgs,train_lbls,test_imgs,test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000)
''' restricted boltzmann machine '''
print ("\nStarting a Restricted Boltzmann Machine..")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20
)
loss_array = rbm.cd1(visible_trainset=train_imgs, n_iterations=63000)
plt.plot(loss_array)
plt.xlabel("epochs")
plt.ylabel("recon_loss")
plt.show()
#np.save("Data/lossarray_500", loss_array)
''' deep- belief net '''
# print ("\nStarting a Deep Belief Net..")
#
# dbn = DeepBeliefNet(sizes={"vis":image_size[0]*image_size[1], "hid":500, "pen":500, "top":2000, "lbl":10},
# image_size=image_size,
# n_labels=10,
# batch_size=10
# )
rbm_errors = []
for hidden in n_hidden:
print('RBM with {} hidden units:\n'.format(hidden))
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=hidden,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
epoch_errors = rbm.cd1(visible_trainset=train_imgs, n_iterations=10)
rbm_errors.append(epoch_errors)
# 7x100
print(np.shape(rbm_errors))
# print(rbm_errors[0])
plt.figure()
plt.title('Comparison of different RBM architectures performance')
plt.ylabel('Mean reconstruction error')
plt.xlabel('Training samples')
for i, n in enumerate(n_hidden):
plt.plot(np.arange(1, 600001, 6000),
rbm_errors[i],
label='N_hidden={}'.format(n))
plt.legend()
plt.show()
batch_size = 20
ndim_hidden = 500
epochs = 10
iterations = int(60000 * epochs / batch_size)
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=ndim_hidden,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=batch_size)
rbm.cd1(visible_trainset=train_imgs, n_iterations=n_iterations)
rbm.cd1(visible_trainset=train_imgs, n_iterations=n_iterations)
''' deep- belief net '''
print("\nStarting a Deep Belief Net..")
dbn = DeepBeliefNet(sizes={
"vis": image_size[0] * image_size[1],
"hid": 500,
"pen": 500,
"top": 2000,
"lbl": 10
},
image_size=image_size,
n_labels=10,
# QUESTIONs 1 and 2 (4.1)
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
loss_err = np.empty(0)
previous_i = 0
for i in range(10, 21):
n_epochs = i
loss_err_aux = rbm.cd1(visible_trainset=train_imgs,
n_iterations=n_epochs * 3000 + 1,
bottom_n_iterations=previous_i * 3000,
n_nod=str(rbm.ndim_hidden),
n_ep=str(i))
loss_err = np.concatenate((loss_err, loss_err_aux))
linspace = np.arange(0, len(loss_err) * rbm.print_period, rbm.print_period)
plt.figure()
plt.plot(linspace, loss_err)
plt.ylabel("Reconstruction Loss")
plt.xlabel('Iteration number')
plt.title('Iteration number vs Reconstruction Loss (' + str(n_epochs) +
' epochs)')
plt.show()
previous_i = i
#%%
''' restricted boltzmann machine '''
print ("\nStarting a Restricted Boltzmann Machine..")
total_err = []
rbm500 = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20
)
#print(train_imgs.shape)
total_err.append(rbm500.cd1(visible_trainset=train_imgs, n_iterations=10))
rbm400 = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1],
ndim_hidden=400,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20
)
#print(train_imgs.shape)
total_err.append(rbm400.cd1(visible_trainset=train_imgs, n_iterations=10))
rbm300 = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1],
ndim_hidden=300,
#
# ''' restricted boltzmann machine '''
#
# print ("\nStarting a Restricted Boltzmann Machine..")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
rbm.cd1(visible_trainset=train_imgs, test=test_imgs, n_iterations=3000)
# ''' deep- belief net '''
#
# print ("\nStarting a Restricted Boltzmann Machine..")
#
# rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1],
# ndim_hidden=500,
# is_bottom=True,
# image_size=image_size,
# is_top=False,
# n_labels=10,
# batch_size=20
# )
#
# rbm.cd1(visible_trainset=train_imgs, n_iterations=3000,epochs=10)
print("\nStarting a Restricted Boltzmann Machine..")
en = np.zeros((10,4))
er = np.zeros((10,4))
cont = 0
for i in np.arange(200,600,100):
rbm = RestrictedBoltzmannMachine(
ndim_visible=image_size[0] * image_size[1],
ndim_hidden=i,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20,
)
result, final_v_prob, energy, error = rbm.cd1(visible_trainset=train_imgs, epochs=10, lr=1e-2, shuffle=True, verbose=True)
en[:,cont] = energy
er[:,cont] = error
cont += 1
#%%
plt.figure(figsize=(13,5))
plt.subplot(121)
plt.plot(range(1,11), en[:,0],'b',label='200 nodes hidden layer')
plt.plot(range(1,11), en[:,1],'orange',label='300 nodes hidden layer')
plt.plot(range(1,11), en[:,2],'g',label='400 nodes hidden layer')
plt.plot(range(1,11), en[:,3],'r',label='500 nodes hidden layer')
plt.xlabel('Epochs')
plt.ylabel('Energy')
plt.title('Energy variation with the number of epochs')
''' restricted boltzmann machine '''
hidden_nodes_list = np.array([200, 300, 400, 500])
recon_errors = np.zeros((len(hidden_nodes_list), 20))
print("\nStarting a Restricted Boltzmann Machine..")
for idx, ndim in enumerate(hidden_nodes_list):
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=ndim,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=10)
recon_errors[idx] = rbm.cd1(visible_trainset=train_imgs,
n_iterations=20)
x = np.arange(0, 20, 1)
for i in range(len(hidden_nodes_list)):
plt.plot(x,
recon_errors[i],
label='Hidden' + str(hidden_nodes_list[i]))
plt.xlabel('Epochs')
plt.ylabel('Reconstruction error')
plt.title('Recon loss vs Hidden layers')
plt.legend()
plt.show()
''' deep- belief net '''
print("\nStarting a Deep Belief Net..")
hidden_node_list = np.array([150, 200, 250, 300, 350, 400, 450, 500])
recon_loss_records = np.zeros((len(hidden_node_list), 10))
for idx, ndim in enumerate(hidden_node_list):
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=ndim,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
#rbm.cd1(visible_trainset=train_imgs, n_iterations=10)
recon_loss_records[idx] = rbm.cd1(visible_trainset=train_imgs,
n_iterations=10)
#plotting reconstruction losses
plt.title(
"Reconstruction losses during each epoch \ndepending on amount of hidden nodes (batch size=20)"
)
plt.xlabel("Epochs")
plt.ylabel("Reconstruction loss")
epochs = np.arange(10)
for i in range(len(hidden_node_list)):
plt.plot(epochs,
recon_loss_records[i],
label=str(hidden_node_list[i]) + " hidden nodes")
plt.legend(loc="upper right")
plt.show()