def units_stats():
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
units = np.array([500, 400, 300, 200])
stats = []
for i in range(len(units)):
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=units[i],
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
stats.append(
rbm.cd1(visible_trainset=train_imgs,
n_iterations=5,
verbose=True,
stats_err=True))
# Plot
x = np.arange(len(stats[0]))
for i in range(len(stats)):
plt.plot(x, stats[i], label="h_dim=" + str(units[i]))
plt.title(
"Reconstruction error trough the contrastive divergence algorithm")
plt.xlabel("Reconstruction Error")
plt.xlabel("Epoch")
plt.legend(loc="upper right")
plt.show()
def __init__(self, sizes, image_size, n_labels, batch_size):
"""
Args:
sizes: Dictionary of layer names and dimensions
image_size: Image dimension of data
n_labels: Number of label categories
batch_size: Size of mini-batch
"""
self.rbm_stack = {
'vis--hid' : RestrictedBoltzmannMachine(ndim_visible=sizes["vis"], ndim_hidden=sizes["hid"],
is_bottom=True, image_size=image_size, batch_size=batch_size, name='vis--hid'),
'hid--pen': RestrictedBoltzmannMachine(ndim_visible=sizes["hid"], ndim_hidden=sizes["pen"], batch_size=batch_size, name='hid--pen'),
'pen+lbl--top' : RestrictedBoltzmannMachine(ndim_visible=sizes["pen"]+sizes["lbl"], ndim_hidden=sizes["top"],
is_top=True, n_labels=n_labels, batch_size=batch_size, name='pen+lbl--top')
}
self.sizes = sizes
self.image_size = image_size
self.batch_size = batch_size
self.n_gibbs_recog = 20
self.n_gibbs_gener = 200
self.n_gibbs_wakesleep = 5
self.print_period = 2000
return
def main():
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
''' restricted boltzmann machine '''
print("\nStarting a Restricted Boltzmann Machine..")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=200,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
# try:
# rbm.loadfromfile_rbm(loc="trained_rbm", name="vis--hid")
# except IOError:
# rbm.cd1(visible_trainset=train_imgs, n_iterations=1)
# rbm.savetofile_rbm("trained_single_rbm", "single")
rbm.cd1(visible_trainset=train_imgs, n_iterations=10, disphist=False)
for im in range(10):
rbm.recall(test_imgs[im], "generations/recall" + str(im))
save_img("generations/data" + str(im), test_imgs[im])
# get_activations(test_imgs, rbm)
rbm.reconstruct_error(test_imgs)
def train_net_units(units, n_epochs, results):
print("Starting network with " + str((units + 2) * 100) + " units")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=(units + 2) * 100,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
res = rbm.cd1(visible_trainset=train_imgs,
max_epochs=n_epochs,
n_iterations=3000,
bool_print=True)
results[units, :] = res
def plot_recon_err():
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
hidden_units = [200, 300, 400, 500]
#hidden_units = [200]
results = []
iters = 1000
''' restricted boltzmann machine '''
print("\nStarting a Restricted Boltzmann Machine..")
for i in range(len(hidden_units)):
print(i)
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=hidden_units[i],
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=10)
results.append(
rbm.cd1(visible_trainset=train_imgs,
n_iterations=iters,
calc_err=True))
epochs = np.arange(len(results[0]))
for i in range(len(results)):
plt.plot(epochs, results[i], label=str(hidden_units[i]) + " units")
plt.legend(loc="upper right")
plt.xticks(np.arange(0, 11, step=1))
plt.xlabel("Iterations")
plt.ylabel("Average reconstruction error")
plt.show()
def plot_batch_probabilities():
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=200,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
rbm.cd1(visible_trainset=train_imgs, n_iterations=10000, calc_err=False)
batch_input = test_imgs[:rbm.batch_size]
p_h_given_v, _ = rbm.get_h_given_v(batch_input)
image = p_h_given_v * 255
image = Image.fromarray(image.astype(np.uint8))
image.save('batch_probs_20_batch.png', format='PNG')
def recall_digits():
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=10)
rbm.cd1(visible_trainset=train_imgs, n_iterations=10000, calc_err=False)
for i in range(10):
plt.imshow(np.reshape(test_imgs[i], (28, 28)), cmap='gray')
plt.title("Test image " + str(i + 1))
plt.show()
""" print(test_imgs[i].shape)
image = test_imgs[i] * 255
image = image.reshape((28, 28))
image = Image.fromarray(image.astype(np.uint8))
image.save('img_init_' + str(i), format='PNG') """
recon = rbm.reconstruct_img(test_imgs[i])
""" image = recon * 255
image = image.reshape((28, 28))
image = Image.fromarray(image.astype(np.uint8))
image.save('img_recon_' + str(i), format='PNG') """
plt.imshow(np.reshape(recon, (28, 28)), cmap='gray')
plt.title("Reconstructed image " + str(i + 1))
plt.show()
from matplotlib import pyplot as plt
from dbn import DeepBeliefNet
if __name__ == "__main__":
image_size = [28,28]
train_imgs,train_lbls,test_imgs,test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000)
''' restricted boltzmann machine '''
print ("\nStarting a Restricted Boltzmann Machine..")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20
)
loss_array = rbm.cd1(visible_trainset=train_imgs, n_iterations=63000)
plt.plot(loss_array)
plt.xlabel("epochs")
plt.ylabel("recon_loss")
plt.show()
#np.save("Data/lossarray_500", loss_array)
''' deep- belief net '''
# print ("\nStarting a Deep Belief Net..")
#
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
''' restricted boltzmann machine '''
print("\nStarting a Restricted Boltzmann Machine..")
rbm_errors = []
for hidden in n_hidden:
print('RBM with {} hidden units:\n'.format(hidden))
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=hidden,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
epoch_errors = rbm.cd1(visible_trainset=train_imgs, n_iterations=10)
rbm_errors.append(epoch_errors)
# 7x100
print(np.shape(rbm_errors))
# print(rbm_errors[0])
plt.figure()
plt.title('Comparison of different RBM architectures performance')
plt.ylabel('Mean reconstruction error')
plt.xlabel('Training samples')
for i, n in enumerate(n_hidden):
from rbm import RestrictedBoltzmannMachine
import matplotlib.pyplot as plt
if __name__ == "__main__":
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
''' restricted boltzmann machine '''
print("\nTesting a Restricted Boltzmann Machine..")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
rbm.weight_vh = np.load("trained_rbm/weights_100_20.npy")
rbm.bias_v = np.load("trained_rbm/bias_v_100_20..npy")
rbm.bias_h = np.load("trained_rbm/bias_h_100_20..npy")
fig, axs = plt.subplots(10, 2, figsize=(12, 12))
plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0)
# print("MIN WEIGHT", weights.min(), "MAX WEIGHT", weights.max())
for i in range(10):
image = train_imgs[i]
v0 = np.random.binomial(1, 0.2, (784, ))
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
''' restricted boltzmann machine '''
print("\nStarting a Restricted Boltzmann Machine..")
hidden_node_list = np.array([150, 200, 250, 300, 350, 400, 450, 500])
recon_loss_records = np.zeros((len(hidden_node_list), 10))
for idx, ndim in enumerate(hidden_node_list):
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=ndim,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
#rbm.cd1(visible_trainset=train_imgs, n_iterations=10)
recon_loss_records[idx] = rbm.cd1(visible_trainset=train_imgs,
n_iterations=10)
#plotting reconstruction losses
plt.title(
"Reconstruction losses during each epoch \ndepending on amount of hidden nodes (batch size=20)"
)
plt.xlabel("Epochs")
plt.ylabel("Reconstruction loss")
epochs = np.arange(10)
''' restricted boltzmann machine '''
print("\nStarting a Restricted Boltzmann Machine..")
### 500 hidden unit test
batch_size = 20
ndim_hidden = 500
n_train = 60000
n_iterations = 60000 / batch_size #math.floor(60000/batch_size)
epochs = 1
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=ndim_hidden,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=batch_size)
recon_err_500 = []
for i in range(epochs):
rbm.cd1(visible_trainset=train_imgs, n_iterations=n_iterations)
recon_err_500 += rbm.get_err_rec()
# reset rbm
ndim_hidden = 200
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=ndim_hidden,
is_bottom=True,
def main():
"""
Task Main
"""
# read in ice cream csv data
ice_cream_df = pd.read_csv('./icecream.csv', sep=',', header=None)
# convert pandas data frame to numpy matrix
ice_cream = ice_cream_df.to_numpy()
# get user input for restricted boltzmann machine initialization parameters
print('Initialize Restricted Boltzmann Machine')
num_hidden_units = int(
input('number of hidden nodes in Restricted Boltzmann Machine: '))
converge_constant = int(
input(
'convergence constant for search and converge learning (enter 0 if constant learning is desired): '
))
# train the restricted boltzmann machine with varying learning rates
learning_rates = [0.001, 0.01, 0.1]
num_epochs = range(150)
abs_mean_errors = np.zeros((len(learning_rates), len(num_epochs)))
for i, learning_rate in enumerate(learning_rates):
for j, epochs in enumerate(num_epochs):
# initialize restricted boltzmann machine
rbm = RestrictedBoltzmannMachine(num_visible_units=10,
num_hidden_units=num_hidden_units)
# train the restricted boltzmann machine
rbm.train(ice_cream,
learning_rate=learning_rate,
max_num_epochs=epochs,
converge_constant=converge_constant)
# calculate the predictions
predictions = rbm.predict_batch(ice_cream)
# calculate absolute mean error
abs_mean_error = rbm.absolute_mean_error(ice_cream, predictions)
# append the error to the list
abs_mean_errors[i, j] = abs_mean_error
# plot the absolute mean error results
plt.plot(num_epochs,
abs_mean_errors[0, :],
color='red',
label=f'learning rate = { learning_rates[0] }')
plt.plot(num_epochs,
abs_mean_errors[1, :],
color='green',
label=f'learning rate = { learning_rates[1] }')
plt.plot(num_epochs,
abs_mean_errors[2, :],
color='blue',
label=f'learning rate = { learning_rates[2] }')
plt.title('Absolute Mean Error of Predicted Labels After Training')
plt.xlabel('number of epochs')
plt.ylabel('absolute mean error')
plt.legend()
plt.text(90, 4, f'number of hidden units = { num_hidden_units }')
plt.text(90, 3.8, f'search and converge constant = { converge_constant }')
plt.show()
from dbn import DeepBeliefNet
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
''' restricted boltzmann machine '''
print("\nStarting a Restricted Boltzmann Machine..")
#%%
# QUESTIONs 1 and 2 (4.1)
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
loss_err = np.empty(0)
previous_i = 0
for i in range(10, 21):
n_epochs = i
loss_err_aux = rbm.cd1(visible_trainset=train_imgs,
n_iterations=n_epochs * 3000 + 1,
bottom_n_iterations=previous_i * 3000,
n_nod=str(rbm.ndim_hidden),
n_ep=str(i))
loss_err = np.concatenate((loss_err, loss_err_aux))
linspace = np.arange(0, len(loss_err) * rbm.print_period, rbm.print_period)
plt.figure()
from dbn import DeepBeliefNet
import matplotlib.pyplot as plt
if __name__ == "__main__":
image_size = [28,28]
train_imgs,train_lbls,test_imgs,test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000)
''' restricted boltzmann machine '''
print ("\nStarting a Restricted Boltzmann Machine..")
total_err = []
rbm500 = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20
)
#print(train_imgs.shape)
total_err.append(rbm500.cd1(visible_trainset=train_imgs, n_iterations=10))
rbm400 = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1],
ndim_hidden=400,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20
)
)
#%%
""" restricted boltzmann machine """
print("\nStarting a Restricted Boltzmann Machine..")
en = np.zeros((10,4))
er = np.zeros((10,4))
cont = 0
for i in np.arange(200,600,100):
rbm = RestrictedBoltzmannMachine(
ndim_visible=image_size[0] * image_size[1],
ndim_hidden=i,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20,
)
result, final_v_prob, energy, error = rbm.cd1(visible_trainset=train_imgs, epochs=10, lr=1e-2, shuffle=True, verbose=True)
en[:,cont] = energy
er[:,cont] = error
cont += 1
#%%
plt.figure(figsize=(13,5))
plt.subplot(121)
plt.plot(range(1,11), en[:,0],'b',label='200 nodes hidden layer')
entradas = np.array([aluno_1, aluno_2, aluno_3, aluno_4, aluno_5, aluno_6])
pesos = np.array(
[[0.0, 0.0, 0.0],
[0.0, 0.1, 0.2],
[0.0, 0.3, 0.4],
[0.0, 0.5, 0.6],
[0.0, 0.7, 0.8],
[0.0, 0.9, 1.0],
[0.0, 0.1, 0.2]])
probabilidade_associacao_neuronios = np.array([[False, False, True],
[False, True, True],
[False, True, True],
[True, True, True],
[True, True, True],
[True, True, True]])
mlp = RestrictedBoltzmannMachine(
entradas=entradas, quantidade_neuronios_ocultos=2, epocas=5000,
taxa_aprendizagem=0.1, precisao=0, pesos=pesos, debug=False, plot=False
)
mlp.treinar()
print(mlp.pesos)
#Pessoa que assistiria somente Gladiador e Titanic
dados_de_teste = np.array([[1,0,1,1,0,0]])
mlp.prever(amostras=dados_de_teste)
mlp.prever(amostras=entradas)
# Test set class histogram
create_histogram(test_lbls_digits, bins=19,
title="Class distribution of the test data",
ylabel="Occurences", xlabel="Class")
# Visualize the first 10 digits
for i in range(10):
plot_digit(train_imgs[i], train_lbls_digits[i])
if RBM:
# Restricted Boltzmann Machine
print ("\nStarting a Restricted Boltzmann Machine...")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * \
image_size[1], ndim_hidden=200, is_bottom=True,
image_size=image_size, is_top=False, n_labels=10, batch_size=20,
learning_rate=0.1)
rbm.cd1(X=train_imgs, n_iterations=30000)
if DBN:
# Deep Belief Net
if TWO_LAYER:
print ("\n>>> Starting a two-layer Deep Belief Net...")
# Initialize the two layer network
dbn = DeepBeliefNetTwoLayer(sizes={"vis": image_size[0] *
image_size[1], "hid": 500, "top": 2000, "lbl": 10},
image_size=image_size, n_labels=10, batch_size=10)
else:
print ("\n>>> Starting a two-layer Deep Belief Net...")
from dbn import DeepBeliefNet
if __name__ == "__main__":
image_size = [28, 28]
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size,
n_train=60000,
n_test=10000)
''' restricted boltzmann machine '''
print("\nStarting a Restricted Boltzmann Machine..")
rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] *
image_size[1],
ndim_hidden=500,
is_bottom=True,
image_size=image_size,
is_top=False,
n_labels=10,
batch_size=20)
#print(train_imgs.shape)
#rbm.cd1(visible_trainset=train_imgs, n_iterations=20)
#print(rbm.delta_weight_vh,rbm.weight_vh)
''' deep- belief net '''
print("\nStarting a Deep Belief Net..")
dbn = DeepBeliefNet(sizes={
"vis": image_size[0] * image_size[1],
"hid": 500,
"pen": 500,
def main():
setupStartTime = time.time()
#load data
trainImages, trainLabels, testImages, testLabels = loadMNIST()
trainImages = binarize(trainImages)
testImages = binarize(testImages)
trainImagesByLabel, testImagesByLabel = \
sortMNISTByLabel(trainImages, trainLabels, testImages, testLabels)
#parameters
numberVisibleUnits = trainImages.shape[-1]
numberHiddenUnits = 200
#numberHiddenUnits = int(numberVisibleUnits * 2./3.)
temperature, nCDSteps = 1., 1
#sigma = 0.01
sigma = 2. / np.sqrt(numberVisibleUnits + numberHiddenUnits)
#gradientWalker = 'sgd'
gradientWalker = 'adam'
iterations, miniBatchSize = int(6e5), 10
internalRngSeed, externalRngSeed = 1337, 1234
rng = RandomState(seed=externalRngSeed)
plotNumber, plotStride = 5, 1
trainingReconstructionErrorOutputStride = 10
trainingOutputStride = iterations // 5
equilibrateFantasyForOutput = 100
#l1Coefficient = 1e-5
l1Coefficient = None
l2Coefficient = 1e-4
gamma = 0.1
adversary =
fileMidfix = f'{gradientWalker}-{iterations}'
parameterFileNameIn, parameterFileNameOut = None, ''.join(('mnistRBM-', fileMidfix, 'step.para'))
#parameterFileNameIn, parameterFileNameOut = f'mnistRBM-{gradientWalker}-1000000step.para', f'mnistRBM-{gradientWalker}-{iterations+1000000}step.para'
runTraining = True
verbose = False
mnistReconProbPlotFilePrefix = ''.join(('mnistReconProb-', fileMidfix, 'steps-'))
mnistReconPlotFilePrefix = ''.join(('mnistRecon-', fileMidfix, 'steps-'))
parameterHistogramFilePrefix = ''.join(('paraHistogram-', fileMidfix, 'steps-'))
gradientHistogramFilePrefix = ''.join(('gradHistogram-', fileMidfix, 'steps-'))
numReceptiveFields, receptiveFieldFilePrefix = 9, ''.join(('receptiveField-', fileMidfix, 'steps-'))
hiddenUnitActivationsSubset = rng.randint(numberHiddenUnits, size=numberHiddenUnits//10)
hiddenUnitActivationFilePrefix = ''.join(('hiddenUnitActivation-', fileMidfix, 'steps-'))
feFileName = ''.join(('fe-', fileMidfix, 'steps-'))
feRatioFileName = ''.join(('feRatio-', fileMidfix, 'steps-'))
if gradientWalker == 'sgd':
learningRate = 1e-4
elif gradientWalker == 'adam' or gradientWalker == 'adamAdversarial':
#learningRate = 1e-4
#learningRate = powerLawGenerator(1e-2, -0.1)
learningRate = powerLawGenerator(1e-3, -0.1)
adams = dict(zip(['visible', 'hidden', 'weights'],
[Adam(stepSize=learningRate) for _ in range(3)]))
else:
exit(1)
#setup RBM
visibleProportionOn = np.sum([images.sum(axis=0) for images in trainImagesByLabel], axis=0) / trainImages.shape[0]
#visibleProportionOn = None
visibleLayer = np.zeros(numberVisibleUnits)
hiddenLayer = np.zeros(numberHiddenUnits)
if parameterFileNameIn is not None:
with open(parameterFileNameIn, 'r') as parameterFile:
rbm = RestrictedBoltzmannMachine(visibleLayer, hiddenLayer,
temperature=temperature, sigma=sigma,
visibleProportionOn=visibleProportionOn,
parameterFile=parameterFile, rngSeed=internalRngSeed)
else:
rbm = RestrictedBoltzmannMachine(visibleLayer, hiddenLayer,
temperature=temperature, sigma=sigma,
visibleProportionOn=visibleProportionOn, rngSeed=internalRngSeed)
if gradientWalker == 'sgd':
updateParameters = lambda miniBatch, \
miniFantasyBatch: \
rbm.updateParametersSGD(miniBatch,
miniFantasyBatch,
learningRate,
nCDSteps=nCDSteps,
l1Coefficient=l1Coefficient,
l2Coefficient=l2Coefficient,
verbose=verbose)
elif gradientWalker == 'adam':
updateParameters = lambda miniBatch, \
miniFantasyBatch: \
rbm.updateParametersAdam(miniBatch,
miniFantasyBatch,
adams,
nCDSteps=nCDSteps,
l1Coefficient=l1Coefficient,
l2Coefficient=l2Coefficient,
verbose=verbose)
elif gradientWalker == 'adamAdversarial':
updateParameters = lambda miniBatch, \
miniFantasyBatch: \
rbm.updateParametersAdamAdversarial(miniBatch,
miniFantasyBatch,
adams,
gamma,
adversary,
nCDSteps=nCDSteps,
l1Coefficient=l1Coefficient,
l2Coefficient=l2Coefficient,
verbose=verbose)
else:
exit(1)
#build dict for parameter histogram output
weightParameterTypes = {'Visible': rbm.visibleBias,
'Hidden': rbm.hiddenBias,
'Weights': rbm.weights}
weightHistogramsByParameterType = {'Visible': [],
'Hidden': [],
'Weights': []}
gradientParameterTypes = {'Visible': rbm.visibleStep,
'Hidden': rbm.hiddenStep,
'Weights': rbm.weightStep}
gradientHistogramsByParameterType = {'Visible': [],
'Hidden': [],
'Weights': []}
historicalRBMs = []
hiddenUnitActivations = []
historicalFEs = []
trainSamplesForFE = getMiniBatchByLabel(trainImagesByLabel, miniBatchSize*10, rng)
testSamplesForFE = getMiniBatchByLabel(testImagesByLabel, miniBatchSize*10, rng)
setupEndTime = time.time()
if runTraining is True:
loopStartTime = time.time()
#build fantasy batch
miniFantasyBatch = np.copy(getMiniBatchByLabel(trainImagesByLabel, miniBatchSize, rng))
for i in range(iterations):
miniBatch = getMiniBatchByLabel(trainImagesByLabel, miniBatchSize, rng)
miniFantasyBatch = updateParameters(miniBatch, miniFantasyBatch)
if (i+1) % trainingReconstructionErrorOutputStride == 0:
print(i, rbm.computeReconstructionError(getMiniBatchByLabel(testImagesByLabel, miniBatchSize, rng)))
if (i+1) % trainingOutputStride == 0:
for parameterType in weightParameterTypes:
xs, ys, _ = diagnostics.computeHistogramArray(weightParameterTypes[parameterType].flatten())
weightHistogramsByParameterType[parameterType].append((i, xs, ys))
xs, ys, _ = diagnostics.computeHistogramArray(gradientParameterTypes[parameterType].flatten())
gradientHistogramsByParameterType[parameterType].append((i, xs, ys))
historicalRBMs.append((i, rbm.copy()))
hiddenUnitActivations.append((i, rbm.storeHiddenActivationsOnMiniBatch(miniBatch,
hiddenUnits=hiddenUnitActivationsSubset)))
historicalFEs.append((i,
rbm.computeMeanFreeEnergy(trainSamplesForFE),
rbm.computeMeanFreeEnergy(testSamplesForFE)))
loopEndTime = time.time()
if parameterFileNameOut is not None:
with open(parameterFileNameOut, 'w') as parameterFile:
rbm.dumpParameterFile(parameterFile)
outputStartTime = time.time()
#plot reconstruction series
visibleStarts = getMiniBatchByLabel(testImagesByLabel, 10, rng)
plotReconstructionSeries(visibleStarts, rbm, mnistReconProbPlotFilePrefix, mnistReconPlotFilePrefix)
#plot fantasy particle series
visibleStarts = miniFantasyBatch
for i, visible in enumerate(visibleStarts):
rbm.visibleLayer = visible
for _ in range(equilibrateFantasyForOutput):
visibleStarts[i], _ = rbm.gibbsSample(hiddenUnitsStochastic=False)
plotReconstructionSeries(visibleStarts, rbm, mnistReconProbPlotFilePrefix+'fantasy', mnistReconPlotFilePrefix+'fantasy')
#plot parameter histograms
plotParameterHistograms(weightHistogramsByParameterType, gradientHistogramsByParameterType, parameterHistogramFilePrefix, gradientHistogramFilePrefix)
#plot receptive fields
hiddenUnitIndices = rng.randint(rbm.hiddenBias.shape[0], size=numReceptiveFields)
for i, historicalRBM in historicalRBMs:
receptiveFieldFileName = ''.join((receptiveFieldFilePrefix,
f'{i}.pdf'))
plotReceptiveFields(historicalRBM, hiddenUnitIndices, fileName=receptiveFieldFileName)
#plot hidden unit activations
for i, hiddenUnitActivation in hiddenUnitActivations:
hiddenUnitActivationFileName = ''.join((hiddenUnitActivationFilePrefix,
f'{i}.pdf'))
diagnostics.plotHiddenActivationsOnMiniBatch(hiddenUnitActivation,
fileName=hiddenUnitActivationFileName)
#plot FE vs time
t = [fe[0] for fe in historicalFEs]
trainFE = np.array([fe[1] for fe in historicalFEs])
testFE = np.array([fe[2] for fe in historicalFEs])
diagnostics.plotTrainingTestAverageFEVsTime(t, trainFE, testFE, fileName=feFileName)
diagnostics.plotTrainingTestAverageFEVsTime(t, trainFE/testFE, None, fileName=feRatioFileName)
outputEndTime = time.time()
print(f'setup time {setupEndTime-setupStartTime}s')
if runTraining is True:
print(f'training loop time {loopEndTime-loopStartTime}s')
print(f'output time {outputEndTime-outputStartTime}s')
print("RMSE: ", score)
scores3.append(score)
matplotlib.pyplot.plot(space, scores3, 'rx-')
matplotlib.pyplot.xlabel('Bias')
matplotlib.pyplot.ylabel('RMSE')
matplotlib.pyplot.title('Singular Value Decomposition')
matplotlib.pyplot.savefig('../plots/singular_value_decomposition3.png')
matplotlib.pyplot.gcf().clear()
print("-------------Singular Value Decomposition Testing Complete-------------")
print("------------------Restricted Boltzmann Machine Testing-----------------")
model = RestrictedBoltzmannMachine()
model.preprocess(X, Y)
model.weight_initialize()
space = (numpy.linspace(0.001, 0.01, 10))
max_error = 100000.0
best_learning_rate = 0.001
scores1 = []
scores2 = []
scores3 = []
for k in space:
print("Epoch: %i", k)
model.set_learning_rate(k)
model.train()
score = model.RMSE(X_val, Y_val)
print("RMSE: ", score)