def example_mnist_tap_machine(paysage_path=None, num_epochs=10, show_plot=False): num_hidden_units = 256 batch_size = 100 learning_rate = schedules.power_law_decay(initial=0.1, coefficient=0.1) (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches data = batch.HDFBatch(shuffled_filepath, 'train/images', batch_size, transform=batch.binarize_color, train_fraction=0.95) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = model.Model([vis_layer, hid_layer]) rbm.initialize(data, 'glorot_normal') perf = fit.ProgressMonitor( data, metrics=['ReconstructionError', 'EnergyDistance', 'HeatCapacity']) opt = optimizers.Gradient(stepsize=learning_rate, tolerance=1e-4, ascent=True) sampler = fit.DrivenSequentialMC.from_batch(rbm, data) sgd = fit.SGD(rbm, data, opt, num_epochs, sampler=sampler, method=fit.tap, monitor=perf) # fit the model print('Training with stochastic gradient ascent using TAP expansion') sgd.train() util.show_metrics(rbm, perf) valid = data.get('validate') util.show_reconstructions(rbm, valid, fit, show_plot, n_recon=10, vertical=False) util.show_fantasy_particles(rbm, valid, fit, show_plot, n_fantasy=25) util.show_weights(rbm, show_plot, n_weights=25) # close the HDF5 store data.close() print("Done")
def example_mnist_deep_rbm(paysage_path=None, num_epochs=10, show_plot=False): num_hidden_units = 500 batch_size = 100 learning_rate = schedules.power_law_decay(initial=0.01, coefficient=0.1) mc_steps = 1 (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches data = batch.HDFBatch(shuffled_filepath, 'train/images', batch_size, transform=batch.binarize_color, train_fraction=0.99) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_1_layer = layers.BernoulliLayer(num_hidden_units) hid_2_layer = layers.BernoulliLayer(num_hidden_units) rbm = model.Model([vis_layer, hid_1_layer, hid_2_layer]) rbm.initialize(data) metrics = [ 'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore', 'HeatCapacity' ] perf = fit.ProgressMonitor(data, metrics=metrics) # set up the optimizer and the fit method opt = optimizers.ADAM(stepsize=learning_rate) sampler = fit.SequentialMC.from_batch(rbm, data) cd = fit.SGD(rbm, data, opt, num_epochs, method=fit.pcd, sampler=sampler, mcsteps=mc_steps, monitor=perf) # fit the model print('training with contrastive divergence') cd.train_layerwise() # evaluate the model util.show_metrics(rbm, perf) valid = data.get('validate') util.show_reconstructions(rbm, valid, fit, show_plot) util.show_fantasy_particles(rbm, valid, fit, show_plot) util.show_weights(rbm, show_plot) # close the HDF5 store data.close() print("Done")
def example_mnist_rbm(paysage_path=None, show_plot = False): num_hidden_units = 500 batch_size = 50 num_epochs = 10 learning_rate = 0.01 mc_steps = 1 (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches data = batch.Batch(shuffled_filepath, 'train/images', batch_size, transform=batch.binarize_color, train_fraction=0.99) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = hidden.Model([vis_layer, hid_layer]) rbm.initialize(data) # set up the optimizer and the fit method opt = optimizers.ADAM(rbm, stepsize=learning_rate, scheduler=optimizers.PowerLawDecay(0.1)) sampler = fit.DrivenSequentialMC.from_batch(rbm, data, method='stochastic') cd = fit.PCD(rbm, data, opt, sampler, num_epochs, mcsteps=mc_steps, skip=200, metrics=[M.ReconstructionError(), M.EnergyDistance(), M.EnergyGap(), M.EnergyZscore()]) # fit the model print('training with contrastive divergence') cd.train() # evaluate the model # this will be the same as the final epoch results # it is repeated here to be consistent with the sklearn rbm example metrics = [M.ReconstructionError(), M.EnergyDistance(), M.EnergyGap(), M.EnergyZscore()] performance = fit.ProgressMonitor(0, data, metrics=metrics) util.show_metrics(rbm, performance) util.show_reconstructions(rbm, data.get('validate'), fit, show_plot) util.show_fantasy_particles(rbm, data.get('validate'), fit, show_plot) util.show_weights(rbm, show_plot) # close the HDF5 store data.close() print("Done")
def example_mnist_grbm(paysage_path=None, num_epochs=10, show_plot=False): num_hidden_units = 500 batch_size = 50 learning_rate = 0.001 # gaussian rbm usually requires smaller learnign rate mc_steps = 1 (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches data = batch.Batch(shuffled_filepath, 'train/images', batch_size, transform=transform, train_fraction=0.99) # set up the model and initialize the parameters vis_layer = layers.GaussianLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = model.Model([vis_layer, hid_layer]) rbm.initialize(data) metrics = [ 'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore' ] perf = fit.ProgressMonitor(data, metrics=metrics) opt = optimizers.ADAM(stepsize=learning_rate, scheduler=optimizers.PowerLawDecay(0.1)) sampler = fit.DrivenSequentialMC.from_batch(rbm, data, method='stochastic') cd = fit.SGD(rbm, data, opt, num_epochs, method=fit.pcd, sampler=sampler, mcsteps=mc_steps, monitor=perf) # fit the model print('training with contrastive divergence') cd.train() # evaluate the model util.show_metrics(rbm, perf) util.show_reconstructions(rbm, data.get('validate'), fit, show_plot) util.show_fantasy_particles(rbm, data.get('validate'), fit, show_plot) util.show_weights(rbm, show_plot) # close the HDF5 store data.close() print("Done")
def run(paysage_path=None, num_epochs=10, show_plot=False): num_hidden_units = 500 batch_size = 100 learning_rate = schedules.PowerLawDecay(initial=0.001, coefficient=0.1) mc_steps = 1 (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches data = batch.HDFBatch(shuffled_filepath, 'train/images', batch_size, transform=pre.binarize_color, train_fraction=0.99) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.GaussianLayer(num_hidden_units) hid_layer.set_fixed_params(["loc", "log_var"]) rbm = model.Model([vis_layer, hid_layer]) rbm.initialize(data, method="glorot_normal") metrics = ['ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore', 'HeatCapacity', 'WeightSparsity', 'WeightSquare'] perf = fit.ProgressMonitor(data, metrics=metrics) # set up the optimizer and the fit method opt = optimizers.ADAM(stepsize=learning_rate) sampler = fit.DrivenSequentialMC.from_batch(rbm, data) cd = fit.SGD(rbm, data, opt, num_epochs, sampler, method=fit.pcd, mcsteps=mc_steps, monitor=perf) # fit the model print('training with contrastive divergence') cd.train() # evaluate the model util.show_metrics(rbm, perf) valid = data.get('validate') util.show_reconstructions(rbm, valid, fit, show_plot, n_recon=10, vertical=False, num_to_avg=10) util.show_fantasy_particles(rbm, valid, fit, show_plot, n_fantasy=25) util.show_weights(rbm, show_plot, n_weights=25) # close the HDF5 store data.close() print("Done")
def example_mnist_tap_machine(paysage_path=None, num_epochs = 10, show_plot=True): num_hidden_units = 256 batch_size = 100 learning_rate = 0.1 (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches data = batch.Batch(shuffled_filepath, 'train/images', batch_size, transform=batch.binarize_color, train_fraction=0.95) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = tap_machine.TAP_rbm([vis_layer, hid_layer], num_persistent_samples=0, tolerance_EMF=1e-4, max_iters_EMF=25, terms=2) rbm.initialize(data, 'glorot_normal') perf = fit.ProgressMonitor(data, metrics=['ReconstructionError', 'EnergyDistance']) opt = optimizers.Gradient(stepsize=learning_rate, scheduler=optimizers.PowerLawDecay(0.1), tolerance=1e-4, ascent=True) sgd = fit.SGD(rbm, data, opt, num_epochs, method=fit.tap, monitor=perf) # fit the model print('training with stochastic gradient ascent ') sgd.train() util.show_metrics(rbm, perf) util.show_reconstructions(rbm, data.get('validate'), fit, show_plot) util.show_fantasy_particles(rbm, data.get('validate'), fit, show_plot) util.show_weights(rbm, show_plot) # close the HDF5 store data.close() print("Done")
def run(paysage_path=None, num_epochs=10, show_plot=False): num_hidden_units = 256 batch_size = 100 learning_rate = schedules.PowerLawDecay(initial=0.01, coefficient=0.1) mc_steps = 1 (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches import pandas data = batch.InMemoryBatch(pre.binarize_color( be.float_tensor( pandas.read_hdf(shuffled_filepath, key='train/images').as_matrix())), batch_size, train_fraction=0.95) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = model.Model([vis_layer, hid_layer]) rbm.weights[0].add_penalty({'matrix': pen.l2_penalty(0.001)}) rbm.initialize(data, method='glorot_normal') metrics = [ 'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore', 'HeatCapacity', 'WeightSparsity', 'WeightSquare' ] perf = fit.ProgressMonitor(data, metrics=metrics) # set up the optimizer and the fit method opt = optimizers.ADAM(stepsize=learning_rate) sampler = fit.DrivenSequentialMC.from_batch(rbm, data) cd = fit.SGD(rbm, data, opt, num_epochs, sampler, method=fit.pcd, mcsteps=mc_steps, monitor=perf) # fit the model print('training with contrastive divergence') cd.train() # evaluate the model util.show_metrics(rbm, perf) valid = data.get('validate') util.show_reconstructions(rbm, valid, fit, show_plot, n_recon=10, vertical=False, num_to_avg=10) util.show_fantasy_particles(rbm, valid, fit, show_plot, n_fantasy=25) util.show_weights(rbm, show_plot, n_weights=25) # close the HDF5 store data.close() print("Done")
def run(paysage_path=None, num_epochs=10, show_plot=False): num_hidden_units = 100 batch_size = 100 learning_rate = schedules.PowerLawDecay(initial=0.012, coefficient=0.1) mc_steps = 1 (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches data = batch.HDFBatch(shuffled_filepath, 'train/images', batch_size, transform=pre.binarize_color, train_fraction=0.95) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_1_layer = layers.BernoulliLayer(num_hidden_units) hid_2_layer = layers.BernoulliLayer(num_hidden_units) hid_3_layer = layers.BernoulliLayer(num_hidden_units) rbm = model.Model([vis_layer, hid_1_layer, hid_2_layer, hid_3_layer]) rbm.initialize(data, method='glorot_normal') print("Norms of the weights before training") util.weight_norm_histogram(rbm, show_plot=show_plot) # small penalties prevent the weights from consolidating rbm.weights[1].add_penalty({'matrix': pen.logdet_penalty(0.001)}) rbm.weights[2].add_penalty({'matrix': pen.logdet_penalty(0.001)}) metrics = [ 'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore', 'HeatCapacity', 'WeightSparsity', 'WeightSquare' ] perf = fit.ProgressMonitor(data, metrics=metrics) # set up the optimizer and the fit method opt = optimizers.ADAM(stepsize=learning_rate) cd = fit.LayerwisePretrain(rbm, data, opt, num_epochs, method=fit.pcd, mcsteps=mc_steps, metrics=metrics) # fit the model print('training with persistent contrastive divergence') cd.train() # evaluate the model util.show_metrics(rbm, perf) valid = data.get('validate') util.show_reconstructions(rbm, valid, fit, show_plot, num_to_avg=10) util.show_fantasy_particles(rbm, valid, fit, show_plot) from math import sqrt dim = tuple([28] + [int(sqrt(num_hidden_units)) for _ in range(rbm.num_weights)]) util.show_weights(rbm, show_plot, dim=dim, n_weights=16) util.show_one_hot_reconstructions(rbm, fit, dim=28, n_recon=16, num_to_avg=1) print("Norms of the weights after training") util.weight_norm_histogram(rbm, show_plot=show_plot) # close the HDF5 store data.close() print("Done")