def units_stats(): image_size = [28, 28] train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) units = np.array([500, 400, 300, 200]) stats = [] for i in range(len(units)): rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=units[i], is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20) stats.append( rbm.cd1(visible_trainset=train_imgs, n_iterations=5, verbose=True, stats_err=True)) # Plot x = np.arange(len(stats[0])) for i in range(len(stats)): plt.plot(x, stats[i], label="h_dim=" + str(units[i])) plt.title( "Reconstruction error trough the contrastive divergence algorithm") plt.xlabel("Reconstruction Error") plt.xlabel("Epoch") plt.legend(loc="upper right") plt.show()
def __init__(self, sizes, image_size, n_labels, batch_size): """ Args: sizes: Dictionary of layer names and dimensions image_size: Image dimension of data n_labels: Number of label categories batch_size: Size of mini-batch """ self.rbm_stack = { 'vis--hid' : RestrictedBoltzmannMachine(ndim_visible=sizes["vis"], ndim_hidden=sizes["hid"], is_bottom=True, image_size=image_size, batch_size=batch_size, name='vis--hid'), 'hid--pen': RestrictedBoltzmannMachine(ndim_visible=sizes["hid"], ndim_hidden=sizes["pen"], batch_size=batch_size, name='hid--pen'), 'pen+lbl--top' : RestrictedBoltzmannMachine(ndim_visible=sizes["pen"]+sizes["lbl"], ndim_hidden=sizes["top"], is_top=True, n_labels=n_labels, batch_size=batch_size, name='pen+lbl--top') } self.sizes = sizes self.image_size = image_size self.batch_size = batch_size self.n_gibbs_recog = 20 self.n_gibbs_gener = 200 self.n_gibbs_wakesleep = 5 self.print_period = 2000 return
def main(): image_size = [28, 28] train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) ''' restricted boltzmann machine ''' print("\nStarting a Restricted Boltzmann Machine..") rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=200, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20) # try: # rbm.loadfromfile_rbm(loc="trained_rbm", name="vis--hid") # except IOError: # rbm.cd1(visible_trainset=train_imgs, n_iterations=1) # rbm.savetofile_rbm("trained_single_rbm", "single") rbm.cd1(visible_trainset=train_imgs, n_iterations=10, disphist=False) for im in range(10): rbm.recall(test_imgs[im], "generations/recall" + str(im)) save_img("generations/data" + str(im), test_imgs[im]) # get_activations(test_imgs, rbm) rbm.reconstruct_error(test_imgs)
def train_net_units(units, n_epochs, results): print("Starting network with " + str((units + 2) * 100) + " units") rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=(units + 2) * 100, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20) res = rbm.cd1(visible_trainset=train_imgs, max_epochs=n_epochs, n_iterations=3000, bool_print=True) results[units, :] = res
def plot_recon_err(): image_size = [28, 28] train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) hidden_units = [200, 300, 400, 500] #hidden_units = [200] results = [] iters = 1000 ''' restricted boltzmann machine ''' print("\nStarting a Restricted Boltzmann Machine..") for i in range(len(hidden_units)): print(i) rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=hidden_units[i], is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=10) results.append( rbm.cd1(visible_trainset=train_imgs, n_iterations=iters, calc_err=True)) epochs = np.arange(len(results[0])) for i in range(len(results)): plt.plot(epochs, results[i], label=str(hidden_units[i]) + " units") plt.legend(loc="upper right") plt.xticks(np.arange(0, 11, step=1)) plt.xlabel("Iterations") plt.ylabel("Average reconstruction error") plt.show()
def plot_batch_probabilities(): image_size = [28, 28] train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=200, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20) rbm.cd1(visible_trainset=train_imgs, n_iterations=10000, calc_err=False) batch_input = test_imgs[:rbm.batch_size] p_h_given_v, _ = rbm.get_h_given_v(batch_input) image = p_h_given_v * 255 image = Image.fromarray(image.astype(np.uint8)) image.save('batch_probs_20_batch.png', format='PNG')
def recall_digits(): image_size = [28, 28] train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=500, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=10) rbm.cd1(visible_trainset=train_imgs, n_iterations=10000, calc_err=False) for i in range(10): plt.imshow(np.reshape(test_imgs[i], (28, 28)), cmap='gray') plt.title("Test image " + str(i + 1)) plt.show() """ print(test_imgs[i].shape) image = test_imgs[i] * 255 image = image.reshape((28, 28)) image = Image.fromarray(image.astype(np.uint8)) image.save('img_init_' + str(i), format='PNG') """ recon = rbm.reconstruct_img(test_imgs[i]) """ image = recon * 255 image = image.reshape((28, 28)) image = Image.fromarray(image.astype(np.uint8)) image.save('img_recon_' + str(i), format='PNG') """ plt.imshow(np.reshape(recon, (28, 28)), cmap='gray') plt.title("Reconstructed image " + str(i + 1)) plt.show()
from matplotlib import pyplot as plt from dbn import DeepBeliefNet if __name__ == "__main__": image_size = [28,28] train_imgs,train_lbls,test_imgs,test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) ''' restricted boltzmann machine ''' print ("\nStarting a Restricted Boltzmann Machine..") rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1], ndim_hidden=500, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20 ) loss_array = rbm.cd1(visible_trainset=train_imgs, n_iterations=63000) plt.plot(loss_array) plt.xlabel("epochs") plt.ylabel("recon_loss") plt.show() #np.save("Data/lossarray_500", loss_array) ''' deep- belief net ''' # print ("\nStarting a Deep Belief Net..") #
train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) ''' restricted boltzmann machine ''' print("\nStarting a Restricted Boltzmann Machine..") rbm_errors = [] for hidden in n_hidden: print('RBM with {} hidden units:\n'.format(hidden)) rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=hidden, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20) epoch_errors = rbm.cd1(visible_trainset=train_imgs, n_iterations=10) rbm_errors.append(epoch_errors) # 7x100 print(np.shape(rbm_errors)) # print(rbm_errors[0]) plt.figure() plt.title('Comparison of different RBM architectures performance') plt.ylabel('Mean reconstruction error') plt.xlabel('Training samples') for i, n in enumerate(n_hidden):
from rbm import RestrictedBoltzmannMachine import matplotlib.pyplot as plt if __name__ == "__main__": image_size = [28, 28] train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) ''' restricted boltzmann machine ''' print("\nTesting a Restricted Boltzmann Machine..") rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=500, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20) rbm.weight_vh = np.load("trained_rbm/weights_100_20.npy") rbm.bias_v = np.load("trained_rbm/bias_v_100_20..npy") rbm.bias_h = np.load("trained_rbm/bias_h_100_20..npy") fig, axs = plt.subplots(10, 2, figsize=(12, 12)) plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) # print("MIN WEIGHT", weights.min(), "MAX WEIGHT", weights.max()) for i in range(10): image = train_imgs[i] v0 = np.random.binomial(1, 0.2, (784, ))
image_size = [28, 28] train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) ''' restricted boltzmann machine ''' print("\nStarting a Restricted Boltzmann Machine..") hidden_node_list = np.array([150, 200, 250, 300, 350, 400, 450, 500]) recon_loss_records = np.zeros((len(hidden_node_list), 10)) for idx, ndim in enumerate(hidden_node_list): rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=ndim, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20) #rbm.cd1(visible_trainset=train_imgs, n_iterations=10) recon_loss_records[idx] = rbm.cd1(visible_trainset=train_imgs, n_iterations=10) #plotting reconstruction losses plt.title( "Reconstruction losses during each epoch \ndepending on amount of hidden nodes (batch size=20)" ) plt.xlabel("Epochs") plt.ylabel("Reconstruction loss") epochs = np.arange(10)
''' restricted boltzmann machine ''' print("\nStarting a Restricted Boltzmann Machine..") ### 500 hidden unit test batch_size = 20 ndim_hidden = 500 n_train = 60000 n_iterations = 60000 / batch_size #math.floor(60000/batch_size) epochs = 1 rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=ndim_hidden, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=batch_size) recon_err_500 = [] for i in range(epochs): rbm.cd1(visible_trainset=train_imgs, n_iterations=n_iterations) recon_err_500 += rbm.get_err_rec() # reset rbm ndim_hidden = 200 rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=ndim_hidden, is_bottom=True,
def main(): """ Task Main """ # read in ice cream csv data ice_cream_df = pd.read_csv('./icecream.csv', sep=',', header=None) # convert pandas data frame to numpy matrix ice_cream = ice_cream_df.to_numpy() # get user input for restricted boltzmann machine initialization parameters print('Initialize Restricted Boltzmann Machine') num_hidden_units = int( input('number of hidden nodes in Restricted Boltzmann Machine: ')) converge_constant = int( input( 'convergence constant for search and converge learning (enter 0 if constant learning is desired): ' )) # train the restricted boltzmann machine with varying learning rates learning_rates = [0.001, 0.01, 0.1] num_epochs = range(150) abs_mean_errors = np.zeros((len(learning_rates), len(num_epochs))) for i, learning_rate in enumerate(learning_rates): for j, epochs in enumerate(num_epochs): # initialize restricted boltzmann machine rbm = RestrictedBoltzmannMachine(num_visible_units=10, num_hidden_units=num_hidden_units) # train the restricted boltzmann machine rbm.train(ice_cream, learning_rate=learning_rate, max_num_epochs=epochs, converge_constant=converge_constant) # calculate the predictions predictions = rbm.predict_batch(ice_cream) # calculate absolute mean error abs_mean_error = rbm.absolute_mean_error(ice_cream, predictions) # append the error to the list abs_mean_errors[i, j] = abs_mean_error # plot the absolute mean error results plt.plot(num_epochs, abs_mean_errors[0, :], color='red', label=f'learning rate = { learning_rates[0] }') plt.plot(num_epochs, abs_mean_errors[1, :], color='green', label=f'learning rate = { learning_rates[1] }') plt.plot(num_epochs, abs_mean_errors[2, :], color='blue', label=f'learning rate = { learning_rates[2] }') plt.title('Absolute Mean Error of Predicted Labels After Training') plt.xlabel('number of epochs') plt.ylabel('absolute mean error') plt.legend() plt.text(90, 4, f'number of hidden units = { num_hidden_units }') plt.text(90, 3.8, f'search and converge constant = { converge_constant }') plt.show()
from dbn import DeepBeliefNet image_size = [28, 28] train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) ''' restricted boltzmann machine ''' print("\nStarting a Restricted Boltzmann Machine..") #%% # QUESTIONs 1 and 2 (4.1) rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=500, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20) loss_err = np.empty(0) previous_i = 0 for i in range(10, 21): n_epochs = i loss_err_aux = rbm.cd1(visible_trainset=train_imgs, n_iterations=n_epochs * 3000 + 1, bottom_n_iterations=previous_i * 3000, n_nod=str(rbm.ndim_hidden), n_ep=str(i)) loss_err = np.concatenate((loss_err, loss_err_aux)) linspace = np.arange(0, len(loss_err) * rbm.print_period, rbm.print_period) plt.figure()
from dbn import DeepBeliefNet import matplotlib.pyplot as plt if __name__ == "__main__": image_size = [28,28] train_imgs,train_lbls,test_imgs,test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) ''' restricted boltzmann machine ''' print ("\nStarting a Restricted Boltzmann Machine..") total_err = [] rbm500 = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1], ndim_hidden=500, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20 ) #print(train_imgs.shape) total_err.append(rbm500.cd1(visible_trainset=train_imgs, n_iterations=10)) rbm400 = RestrictedBoltzmannMachine(ndim_visible=image_size[0]*image_size[1], ndim_hidden=400, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20 )
) #%% """ restricted boltzmann machine """ print("\nStarting a Restricted Boltzmann Machine..") en = np.zeros((10,4)) er = np.zeros((10,4)) cont = 0 for i in np.arange(200,600,100): rbm = RestrictedBoltzmannMachine( ndim_visible=image_size[0] * image_size[1], ndim_hidden=i, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20, ) result, final_v_prob, energy, error = rbm.cd1(visible_trainset=train_imgs, epochs=10, lr=1e-2, shuffle=True, verbose=True) en[:,cont] = energy er[:,cont] = error cont += 1 #%% plt.figure(figsize=(13,5)) plt.subplot(121) plt.plot(range(1,11), en[:,0],'b',label='200 nodes hidden layer')
entradas = np.array([aluno_1, aluno_2, aluno_3, aluno_4, aluno_5, aluno_6]) pesos = np.array( [[0.0, 0.0, 0.0], [0.0, 0.1, 0.2], [0.0, 0.3, 0.4], [0.0, 0.5, 0.6], [0.0, 0.7, 0.8], [0.0, 0.9, 1.0], [0.0, 0.1, 0.2]]) probabilidade_associacao_neuronios = np.array([[False, False, True], [False, True, True], [False, True, True], [True, True, True], [True, True, True], [True, True, True]]) mlp = RestrictedBoltzmannMachine( entradas=entradas, quantidade_neuronios_ocultos=2, epocas=5000, taxa_aprendizagem=0.1, precisao=0, pesos=pesos, debug=False, plot=False ) mlp.treinar() print(mlp.pesos) #Pessoa que assistiria somente Gladiador e Titanic dados_de_teste = np.array([[1,0,1,1,0,0]]) mlp.prever(amostras=dados_de_teste) mlp.prever(amostras=entradas)
# Test set class histogram create_histogram(test_lbls_digits, bins=19, title="Class distribution of the test data", ylabel="Occurences", xlabel="Class") # Visualize the first 10 digits for i in range(10): plot_digit(train_imgs[i], train_lbls_digits[i]) if RBM: # Restricted Boltzmann Machine print ("\nStarting a Restricted Boltzmann Machine...") rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * \ image_size[1], ndim_hidden=200, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20, learning_rate=0.1) rbm.cd1(X=train_imgs, n_iterations=30000) if DBN: # Deep Belief Net if TWO_LAYER: print ("\n>>> Starting a two-layer Deep Belief Net...") # Initialize the two layer network dbn = DeepBeliefNetTwoLayer(sizes={"vis": image_size[0] * image_size[1], "hid": 500, "top": 2000, "lbl": 10}, image_size=image_size, n_labels=10, batch_size=10) else: print ("\n>>> Starting a two-layer Deep Belief Net...")
from dbn import DeepBeliefNet if __name__ == "__main__": image_size = [28, 28] train_imgs, train_lbls, test_imgs, test_lbls = read_mnist(dim=image_size, n_train=60000, n_test=10000) ''' restricted boltzmann machine ''' print("\nStarting a Restricted Boltzmann Machine..") rbm = RestrictedBoltzmannMachine(ndim_visible=image_size[0] * image_size[1], ndim_hidden=500, is_bottom=True, image_size=image_size, is_top=False, n_labels=10, batch_size=20) #print(train_imgs.shape) #rbm.cd1(visible_trainset=train_imgs, n_iterations=20) #print(rbm.delta_weight_vh,rbm.weight_vh) ''' deep- belief net ''' print("\nStarting a Deep Belief Net..") dbn = DeepBeliefNet(sizes={ "vis": image_size[0] * image_size[1], "hid": 500, "pen": 500,
def main(): setupStartTime = time.time() #load data trainImages, trainLabels, testImages, testLabels = loadMNIST() trainImages = binarize(trainImages) testImages = binarize(testImages) trainImagesByLabel, testImagesByLabel = \ sortMNISTByLabel(trainImages, trainLabels, testImages, testLabels) #parameters numberVisibleUnits = trainImages.shape[-1] numberHiddenUnits = 200 #numberHiddenUnits = int(numberVisibleUnits * 2./3.) temperature, nCDSteps = 1., 1 #sigma = 0.01 sigma = 2. / np.sqrt(numberVisibleUnits + numberHiddenUnits) #gradientWalker = 'sgd' gradientWalker = 'adam' iterations, miniBatchSize = int(6e5), 10 internalRngSeed, externalRngSeed = 1337, 1234 rng = RandomState(seed=externalRngSeed) plotNumber, plotStride = 5, 1 trainingReconstructionErrorOutputStride = 10 trainingOutputStride = iterations // 5 equilibrateFantasyForOutput = 100 #l1Coefficient = 1e-5 l1Coefficient = None l2Coefficient = 1e-4 gamma = 0.1 adversary = fileMidfix = f'{gradientWalker}-{iterations}' parameterFileNameIn, parameterFileNameOut = None, ''.join(('mnistRBM-', fileMidfix, 'step.para')) #parameterFileNameIn, parameterFileNameOut = f'mnistRBM-{gradientWalker}-1000000step.para', f'mnistRBM-{gradientWalker}-{iterations+1000000}step.para' runTraining = True verbose = False mnistReconProbPlotFilePrefix = ''.join(('mnistReconProb-', fileMidfix, 'steps-')) mnistReconPlotFilePrefix = ''.join(('mnistRecon-', fileMidfix, 'steps-')) parameterHistogramFilePrefix = ''.join(('paraHistogram-', fileMidfix, 'steps-')) gradientHistogramFilePrefix = ''.join(('gradHistogram-', fileMidfix, 'steps-')) numReceptiveFields, receptiveFieldFilePrefix = 9, ''.join(('receptiveField-', fileMidfix, 'steps-')) hiddenUnitActivationsSubset = rng.randint(numberHiddenUnits, size=numberHiddenUnits//10) hiddenUnitActivationFilePrefix = ''.join(('hiddenUnitActivation-', fileMidfix, 'steps-')) feFileName = ''.join(('fe-', fileMidfix, 'steps-')) feRatioFileName = ''.join(('feRatio-', fileMidfix, 'steps-')) if gradientWalker == 'sgd': learningRate = 1e-4 elif gradientWalker == 'adam' or gradientWalker == 'adamAdversarial': #learningRate = 1e-4 #learningRate = powerLawGenerator(1e-2, -0.1) learningRate = powerLawGenerator(1e-3, -0.1) adams = dict(zip(['visible', 'hidden', 'weights'], [Adam(stepSize=learningRate) for _ in range(3)])) else: exit(1) #setup RBM visibleProportionOn = np.sum([images.sum(axis=0) for images in trainImagesByLabel], axis=0) / trainImages.shape[0] #visibleProportionOn = None visibleLayer = np.zeros(numberVisibleUnits) hiddenLayer = np.zeros(numberHiddenUnits) if parameterFileNameIn is not None: with open(parameterFileNameIn, 'r') as parameterFile: rbm = RestrictedBoltzmannMachine(visibleLayer, hiddenLayer, temperature=temperature, sigma=sigma, visibleProportionOn=visibleProportionOn, parameterFile=parameterFile, rngSeed=internalRngSeed) else: rbm = RestrictedBoltzmannMachine(visibleLayer, hiddenLayer, temperature=temperature, sigma=sigma, visibleProportionOn=visibleProportionOn, rngSeed=internalRngSeed) if gradientWalker == 'sgd': updateParameters = lambda miniBatch, \ miniFantasyBatch: \ rbm.updateParametersSGD(miniBatch, miniFantasyBatch, learningRate, nCDSteps=nCDSteps, l1Coefficient=l1Coefficient, l2Coefficient=l2Coefficient, verbose=verbose) elif gradientWalker == 'adam': updateParameters = lambda miniBatch, \ miniFantasyBatch: \ rbm.updateParametersAdam(miniBatch, miniFantasyBatch, adams, nCDSteps=nCDSteps, l1Coefficient=l1Coefficient, l2Coefficient=l2Coefficient, verbose=verbose) elif gradientWalker == 'adamAdversarial': updateParameters = lambda miniBatch, \ miniFantasyBatch: \ rbm.updateParametersAdamAdversarial(miniBatch, miniFantasyBatch, adams, gamma, adversary, nCDSteps=nCDSteps, l1Coefficient=l1Coefficient, l2Coefficient=l2Coefficient, verbose=verbose) else: exit(1) #build dict for parameter histogram output weightParameterTypes = {'Visible': rbm.visibleBias, 'Hidden': rbm.hiddenBias, 'Weights': rbm.weights} weightHistogramsByParameterType = {'Visible': [], 'Hidden': [], 'Weights': []} gradientParameterTypes = {'Visible': rbm.visibleStep, 'Hidden': rbm.hiddenStep, 'Weights': rbm.weightStep} gradientHistogramsByParameterType = {'Visible': [], 'Hidden': [], 'Weights': []} historicalRBMs = [] hiddenUnitActivations = [] historicalFEs = [] trainSamplesForFE = getMiniBatchByLabel(trainImagesByLabel, miniBatchSize*10, rng) testSamplesForFE = getMiniBatchByLabel(testImagesByLabel, miniBatchSize*10, rng) setupEndTime = time.time() if runTraining is True: loopStartTime = time.time() #build fantasy batch miniFantasyBatch = np.copy(getMiniBatchByLabel(trainImagesByLabel, miniBatchSize, rng)) for i in range(iterations): miniBatch = getMiniBatchByLabel(trainImagesByLabel, miniBatchSize, rng) miniFantasyBatch = updateParameters(miniBatch, miniFantasyBatch) if (i+1) % trainingReconstructionErrorOutputStride == 0: print(i, rbm.computeReconstructionError(getMiniBatchByLabel(testImagesByLabel, miniBatchSize, rng))) if (i+1) % trainingOutputStride == 0: for parameterType in weightParameterTypes: xs, ys, _ = diagnostics.computeHistogramArray(weightParameterTypes[parameterType].flatten()) weightHistogramsByParameterType[parameterType].append((i, xs, ys)) xs, ys, _ = diagnostics.computeHistogramArray(gradientParameterTypes[parameterType].flatten()) gradientHistogramsByParameterType[parameterType].append((i, xs, ys)) historicalRBMs.append((i, rbm.copy())) hiddenUnitActivations.append((i, rbm.storeHiddenActivationsOnMiniBatch(miniBatch, hiddenUnits=hiddenUnitActivationsSubset))) historicalFEs.append((i, rbm.computeMeanFreeEnergy(trainSamplesForFE), rbm.computeMeanFreeEnergy(testSamplesForFE))) loopEndTime = time.time() if parameterFileNameOut is not None: with open(parameterFileNameOut, 'w') as parameterFile: rbm.dumpParameterFile(parameterFile) outputStartTime = time.time() #plot reconstruction series visibleStarts = getMiniBatchByLabel(testImagesByLabel, 10, rng) plotReconstructionSeries(visibleStarts, rbm, mnistReconProbPlotFilePrefix, mnistReconPlotFilePrefix) #plot fantasy particle series visibleStarts = miniFantasyBatch for i, visible in enumerate(visibleStarts): rbm.visibleLayer = visible for _ in range(equilibrateFantasyForOutput): visibleStarts[i], _ = rbm.gibbsSample(hiddenUnitsStochastic=False) plotReconstructionSeries(visibleStarts, rbm, mnistReconProbPlotFilePrefix+'fantasy', mnistReconPlotFilePrefix+'fantasy') #plot parameter histograms plotParameterHistograms(weightHistogramsByParameterType, gradientHistogramsByParameterType, parameterHistogramFilePrefix, gradientHistogramFilePrefix) #plot receptive fields hiddenUnitIndices = rng.randint(rbm.hiddenBias.shape[0], size=numReceptiveFields) for i, historicalRBM in historicalRBMs: receptiveFieldFileName = ''.join((receptiveFieldFilePrefix, f'{i}.pdf')) plotReceptiveFields(historicalRBM, hiddenUnitIndices, fileName=receptiveFieldFileName) #plot hidden unit activations for i, hiddenUnitActivation in hiddenUnitActivations: hiddenUnitActivationFileName = ''.join((hiddenUnitActivationFilePrefix, f'{i}.pdf')) diagnostics.plotHiddenActivationsOnMiniBatch(hiddenUnitActivation, fileName=hiddenUnitActivationFileName) #plot FE vs time t = [fe[0] for fe in historicalFEs] trainFE = np.array([fe[1] for fe in historicalFEs]) testFE = np.array([fe[2] for fe in historicalFEs]) diagnostics.plotTrainingTestAverageFEVsTime(t, trainFE, testFE, fileName=feFileName) diagnostics.plotTrainingTestAverageFEVsTime(t, trainFE/testFE, None, fileName=feRatioFileName) outputEndTime = time.time() print(f'setup time {setupEndTime-setupStartTime}s') if runTraining is True: print(f'training loop time {loopEndTime-loopStartTime}s') print(f'output time {outputEndTime-outputStartTime}s')
print("RMSE: ", score) scores3.append(score) matplotlib.pyplot.plot(space, scores3, 'rx-') matplotlib.pyplot.xlabel('Bias') matplotlib.pyplot.ylabel('RMSE') matplotlib.pyplot.title('Singular Value Decomposition') matplotlib.pyplot.savefig('../plots/singular_value_decomposition3.png') matplotlib.pyplot.gcf().clear() print("-------------Singular Value Decomposition Testing Complete-------------") print("------------------Restricted Boltzmann Machine Testing-----------------") model = RestrictedBoltzmannMachine() model.preprocess(X, Y) model.weight_initialize() space = (numpy.linspace(0.001, 0.01, 10)) max_error = 100000.0 best_learning_rate = 0.001 scores1 = [] scores2 = [] scores3 = [] for k in space: print("Epoch: %i", k) model.set_learning_rate(k) model.train() score = model.RMSE(X_val, Y_val) print("RMSE: ", score)