def test_pca_compare_var(): # create some random data num_samples = 10000 dim = 10 batch_size = 100 num_components = 3 # generate some data mean = np.random.random(dim) cov_factor = np.random.random((dim, dim)) cov = np.dot(cov_factor, cov_factor.T) samples = be.float_tensor( np.random.multivariate_normal(mean, cov, size=num_samples)) samples_train, samples_validate = batch.split_tensor(samples, 0.9) data = batch.Batch({ 'train': batch.InMemoryTable(samples_train, batch_size), 'validate': batch.InMemoryTable(samples_validate, batch_size) }) # find the principal directions pca_sgd = factorization.PCA.from_batch(data, num_components, epochs=10, grad_steps_per_minibatch=1, stepsize=0.01) pca_svd = factorization.PCA.from_svd(samples_train, num_components) assert be.norm(pca_sgd.var - pca_svd.var) / be.norm(pca_sgd.var) < 1e-1
def test_grbm_reload(): vis_layer = layers.BernoulliLayer(num_vis, center=True) hid_layer = layers.GaussianLayer(num_hid, center=True) # create some extrinsics grbm = BoltzmannMachine([vis_layer, hid_layer]) data = batch.Batch({ 'train': batch.InMemoryTable(be.randn((10 * num_samples, num_vis)), num_samples) }) grbm.initialize(data) with tempfile.NamedTemporaryFile() as file: # save the model store = pandas.HDFStore(file.name, mode='w') grbm.save(store) store.close() # reload store = pandas.HDFStore(file.name, mode='r') grbm_reload = BoltzmannMachine.from_saved(store) store.close() # check the two models are consistent vis_data = vis_layer.random((num_samples, num_vis)) data_state = State.from_visible(vis_data, grbm) vis_orig = grbm.deterministic_iteration(1, data_state)[0] vis_reload = grbm_reload.deterministic_iteration(1, data_state)[0] assert be.allclose(vis_orig, vis_reload) assert be.allclose(grbm.layers[0].moments.mean, grbm_reload.layers[0].moments.mean) assert be.allclose(grbm.layers[0].moments.var, grbm_reload.layers[0].moments.var) assert be.allclose(grbm.layers[1].moments.mean, grbm_reload.layers[1].moments.mean) assert be.allclose(grbm.layers[1].moments.var, grbm_reload.layers[1].moments.var)
def example_mnist_rbm(paysage_path=None, show_plot = False): num_hidden_units = 500 batch_size = 50 num_epochs = 10 learning_rate = 0.01 mc_steps = 1 (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches data = batch.Batch(shuffled_filepath, 'train/images', batch_size, transform=batch.binarize_color, train_fraction=0.99) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = hidden.Model([vis_layer, hid_layer]) rbm.initialize(data) # set up the optimizer and the fit method opt = optimizers.ADAM(rbm, stepsize=learning_rate, scheduler=optimizers.PowerLawDecay(0.1)) sampler = fit.DrivenSequentialMC.from_batch(rbm, data, method='stochastic') cd = fit.PCD(rbm, data, opt, sampler, num_epochs, mcsteps=mc_steps, skip=200, metrics=[M.ReconstructionError(), M.EnergyDistance(), M.EnergyGap(), M.EnergyZscore()]) # fit the model print('training with contrastive divergence') cd.train() # evaluate the model # this will be the same as the final epoch results # it is repeated here to be consistent with the sklearn rbm example metrics = [M.ReconstructionError(), M.EnergyDistance(), M.EnergyGap(), M.EnergyZscore()] performance = fit.ProgressMonitor(0, data, metrics=metrics) util.show_metrics(rbm, performance) util.show_reconstructions(rbm, data.get('validate'), fit, show_plot) util.show_fantasy_particles(rbm, data.get('validate'), fit, show_plot) util.show_weights(rbm, show_plot) # close the HDF5 store data.close() print("Done")
def example_mnist_hopfield(paysage_path=None, num_epochs=10, show_plot=False): num_hidden_units = 500 batch_size = 50 learning_rate = 0.001 mc_steps = 1 (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches data = batch.Batch(shuffled_filepath, 'train/images', batch_size, transform=batch.binarize_color, train_fraction=0.99) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.GaussianLayer(num_hidden_units) rbm = model.Model([vis_layer, hid_layer]) rbm.initialize(data) metrics = [ 'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore' ] perf = fit.ProgressMonitor(data, metrics=metrics) # set up the optimizer and the fit method opt = optimizers.ADAM(stepsize=learning_rate, scheduler=optimizers.PowerLawDecay(0.1)) sampler = fit.DrivenSequentialMC.from_batch(rbm, data, method='stochastic') cd = fit.SGD(rbm, data, opt, num_epochs, method=fit.pcd, sampler=sampler, mcsteps=mc_steps, monitor=perf) # fit the model print('training with contrastive divergence') cd.train() # evaluate the model util.show_metrics(rbm, perf) util.show_reconstructions(rbm, data.get('validate'), fit, show_plot) util.show_fantasy_particles(rbm, data.get('validate'), fit, show_plot) util.show_weights(rbm, show_plot) # close the HDF5 store data.close() print("Done")
def test_gaussian_1D_1mode_train(): # create some example data num = 10000 mu = 3 sigma = 1 samples = be.randn((num, 1)) * sigma + mu # set up the reader to get minibatches batch_size = 100 samples_train, samples_validate = batch.split_tensor(samples, 0.9) data = batch.Batch({ 'train': batch.InMemoryTable(samples_train, batch_size), 'validate': batch.InMemoryTable(samples_validate, batch_size) }) # parameters learning_rate = schedules.PowerLawDecay(initial=0.1, coefficient=0.1) mc_steps = 1 num_epochs = 10 num_sample_steps = 100 # set up the model and initialize the parameters vis_layer = layers.GaussianLayer(1) hid_layer = layers.OneHotLayer(1) rbm = BoltzmannMachine([vis_layer, hid_layer]) rbm.initialize(data, method='hinton') # modify the parameters to shift the initialized model from the data # this forces it to train rbm.layers[0].params = layers.ParamsGaussian( rbm.layers[0].params.loc - 3, rbm.layers[0].params.log_var - 1) # set up the optimizer and the fit method opt = optimizers.ADAM(stepsize=learning_rate) cd = fit.SGD(rbm, data) # fit the model print('training with persistent contrastive divergence') cd.train(opt, num_epochs, method=fit.pcd, mcsteps=mc_steps) # sample data from the trained model model_state = \ samplers.SequentialMC.generate_fantasy_state(rbm, num, num_sample_steps) pts_trained = model_state[0] percent_error = 10 mu_trained = be.mean(pts_trained) assert numpy.abs(mu_trained / mu - 1) < (percent_error / 100) sigma_trained = numpy.sqrt(be.var(pts_trained)) assert numpy.abs(sigma_trained / sigma - 1) < (percent_error / 100)
def test_grbm_save(): vis_layer = layers.BernoulliLayer(num_vis, center=True) hid_layer = layers.GaussianLayer(num_hid, center=True) grbm = BoltzmannMachine([vis_layer, hid_layer]) data = batch.Batch({ 'train': batch.InMemoryTable(be.randn((10 * num_samples, num_vis)), num_samples) }) grbm.initialize(data) with tempfile.NamedTemporaryFile() as file: store = pandas.HDFStore(file.name, mode='w') grbm.save(store) store.close()
def example_mnist_tap_machine(paysage_path=None, num_epochs = 10, show_plot=True): num_hidden_units = 256 batch_size = 100 learning_rate = 0.1 (_, _, shuffled_filepath) = \ util.default_paths(paysage_path) # set up the reader to get minibatches data = batch.Batch(shuffled_filepath, 'train/images', batch_size, transform=batch.binarize_color, train_fraction=0.95) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = tap_machine.TAP_rbm([vis_layer, hid_layer], num_persistent_samples=0, tolerance_EMF=1e-4, max_iters_EMF=25, terms=2) rbm.initialize(data, 'glorot_normal') perf = fit.ProgressMonitor(data, metrics=['ReconstructionError', 'EnergyDistance']) opt = optimizers.Gradient(stepsize=learning_rate, scheduler=optimizers.PowerLawDecay(0.1), tolerance=1e-4, ascent=True) sgd = fit.SGD(rbm, data, opt, num_epochs, method=fit.tap, monitor=perf) # fit the model print('training with stochastic gradient ascent ') sgd.train() util.show_metrics(rbm, perf) util.show_reconstructions(rbm, data.get('validate'), fit, show_plot) util.show_fantasy_particles(rbm, data.get('validate'), fit, show_plot) util.show_weights(rbm, show_plot) # close the HDF5 store data.close() print("Done")
def test_pca_save_read_num_components(): # create some random data num_samples = 10000 dim = 10 batch_size = 100 num_components = 3 num_components_save = 2 # generate some data mean = np.random.random(dim) cov_factor = np.random.random((dim, dim)) cov = np.dot(cov_factor, cov_factor.T) samples = be.float_tensor( np.random.multivariate_normal(mean, cov, size=num_samples)) samples_train, samples_validate = batch.split_tensor(samples, 0.9) data = batch.Batch({ 'train': batch.InMemoryTable(samples_train, batch_size), 'validate': batch.InMemoryTable(samples_validate, batch_size) }) # find the principal directions pca = factorization.PCA.from_batch(data, num_components, epochs=10, grad_steps_per_minibatch=1, stepsize=0.01) # save it pca_file = tempfile.NamedTemporaryFile() store = pd.HDFStore(pca_file.name, mode="w") pca.save(store, num_components_save=num_components_save) # read it pca_read = factorization.PCA.from_saved(store) store.close() # check it assert be.allclose(pca.W[:, :num_components_save], pca_read.W) assert be.allclose(pca.var[:num_components_save], pca_read.var) assert pca.stepsize == pca_read.stepsize assert pca_read.num_components == num_components_save
def test_hdf_batch(): # the temporary storage file store_file = tempfile.NamedTemporaryFile() # create data num_rows = 10000 num_cols = 10 df_A = pd.DataFrame(np.arange(num_rows*num_cols).reshape(num_rows, num_cols)) df_B = df_A + num_rows*num_cols # save it with pd.HDFStore(store_file.name, mode="w", format="table") as store: store.append("train", df_A) store.append("validate", df_B) # read it back with the HDFtable batch_size = 1000 num_train_batches = num_rows // batch_size data = batch.Batch( {"train": batch.HDFtable(store_file.name, "train", batch_size), "validate": batch.HDFtable(store_file.name, "validate", batch_size)}) # loop through thrice, checking the data for i_loop in range(3): i_batch = 0 while True: # get the data try: batch_data_train = data.get("train") batch_data_validate = data.get("validate") except StopIteration: assert i_batch == num_train_batches i_batch = 0 data.reset_generator("all") break # check it assert np.all(be.to_numpy_array(batch_data_train) == \ df_A.values[i_batch * batch_size: (i_batch + 1) * batch_size]) assert np.all(be.to_numpy_array(batch_data_validate) == \ df_B.values[i_batch * batch_size: (i_batch + 1) * batch_size]) i_batch += 1
def test_in_memory_batch(): # create data num_rows = 10000 num_cols = 10 tensor = be.rand((num_rows, num_cols)) # read it back with Batch batch_size = 1000 num_train_batches = num_rows // batch_size with batch.Batch({ 'train': batch.InMemoryTable(tensor, batch_size), 'validate': batch.InMemoryTable(tensor, batch_size) }) as data: # loop through thrice, checking the data i_batch = 0 while True: # get the data try: batch_data_train = data.get("train") batch_data_validate = data.get("validate") except StopIteration: assert i_batch == num_train_batches i_batch = 0 data.reset_generator("all") break # check it assert be.allclose( batch_data_train, tensor[i_batch * batch_size:(i_batch + 1) * batch_size]) assert be.allclose( batch_data_validate, tensor[i_batch * batch_size:(i_batch + 1) * batch_size]) i_batch += 1
def test_rbm(paysage_path=None): num_hidden_units = 50 batch_size = 50 num_epochs = 1 learning_rate = schedules.PowerLawDecay(initial=0.01, coefficient=0.1) mc_steps = 1 if not paysage_path: paysage_path = os.path.dirname( os.path.dirname(os.path.abspath(__file__))) filepath = os.path.join(paysage_path, 'examples', 'mnist', 'mnist.h5') if not os.path.exists(filepath): raise IOError( "{} does not exist. run mnist/download_mnist.py to fetch from the web" .format(filepath)) shuffled_filepath = os.path.join(paysage_path, 'examples', 'mnist', 'shuffled_mnist.h5') # shuffle the data if not os.path.exists(shuffled_filepath): shuffler = batch.DataShuffler(filepath, shuffled_filepath, complevel=0) shuffler.shuffle() # set a seed for the random number generator be.set_seed() import pandas samples = pre.binarize_color( be.float_tensor( pandas.read_hdf(shuffled_filepath, key='train/images').values[:10000])) samples_train, samples_validate = batch.split_tensor(samples, 0.95) data = batch.Batch({ 'train': batch.InMemoryTable(samples_train, batch_size), 'validate': batch.InMemoryTable(samples_validate, batch_size) }) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = BoltzmannMachine([vis_layer, hid_layer]) rbm.initialize(data) # obtain initial estimate of the reconstruction error perf = ProgressMonitor() untrained_performance = perf.epoch_update(data, rbm, store=True, show=False) # set up the optimizer and the fit method opt = optimizers.RMSProp(stepsize=learning_rate) cd = fit.SGD(rbm, data) # fit the model print('training with contrastive divergence') cd.train(opt, num_epochs, method=fit.pcd, mcsteps=mc_steps) # obtain an estimate of the reconstruction error after 1 epoch trained_performance = cd.monitor.memory[-1] assert (trained_performance['ReconstructionError'] < untrained_performance['ReconstructionError']), \ "Reconstruction error did not decrease" # close the HDF5 store data.close()
def test_tap_machine(paysage_path=None): num_hidden_units = 10 batch_size = 50 num_epochs = 1 learning_rate = 0.01 if not paysage_path: paysage_path = os.path.dirname( os.path.dirname(os.path.abspath(__file__))) filepath = os.path.join(paysage_path, 'mnist', 'mnist.h5') if not os.path.exists(filepath): raise IOError( "{} does not exist. run mnist/download_mnist.py to fetch from the web" .format(filepath)) shuffled_filepath = os.path.join(paysage_path, 'mnist', 'shuffled_mnist.h5') # shuffle the data if not os.path.exists(shuffled_filepath): shuffler = batch.DataShuffler(filepath, shuffled_filepath, complevel=0) shuffler.shuffle() # set a seed for the random number generator be.set_seed() # set up the reader to get minibatches data = batch.Batch(shuffled_filepath, 'train/images', batch_size, transform=batch.binarize_color, train_fraction=0.1) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = tap_machine.TAP_rbm([vis_layer, hid_layer], tolerance_EMF=1e-2, max_iters_EMF=50) rbm.initialize(data) # obtain initial estimate of the reconstruction error perf = fit.ProgressMonitor(data, metrics=['ReconstructionError']) untrained_performance = perf.check_progress(rbm) # set up the optimizer and the fit method opt = optimizers.Gradient(stepsize=learning_rate, scheduler=optimizers.PowerLawDecay(0.1), tolerance=1e-3, ascent=True) solver = fit.SGD(rbm, data, opt, num_epochs, method=fit.tap, monitor=perf) # fit the model print('training with stochastic gradient ascent') solver.train() # obtain an estimate of the reconstruction error after 1 epoch trained_performance = perf.check_progress(rbm) assert (trained_performance['ReconstructionError'] < untrained_performance['ReconstructionError']), \ "Reconstruction error did not decrease" # close the HDF5 store data.close()
def test_tap_machine(paysage_path=None): num_hidden_units = 10 batch_size = 100 num_epochs = 5 learning_rate = schedules.PowerLawDecay(initial=0.1, coefficient=1.0) if not paysage_path: paysage_path = os.path.dirname( os.path.dirname(os.path.abspath(__file__))) filepath = os.path.join(paysage_path, 'examples', 'mnist', 'mnist.h5') if not os.path.exists(filepath): raise IOError( "{} does not exist. run mnist/download_mnist.py to fetch from the web" .format(filepath)) shuffled_filepath = os.path.join(paysage_path, 'examples', 'mnist', 'shuffled_mnist.h5') # shuffle the data if not os.path.exists(shuffled_filepath): shuffler = batch.DataShuffler(filepath, shuffled_filepath, complevel=0) shuffler.shuffle() # set a seed for the random number generator be.set_seed() # set up the reader to get minibatches samples = pre.binarize_color( be.float_tensor( pandas.read_hdf(shuffled_filepath, key='train/images').as_matrix()[:10000])) samples_train, samples_validate = batch.split_tensor(samples, 0.95) data = batch.Batch({ 'train': batch.InMemoryTable(samples_train, batch_size), 'validate': batch.InMemoryTable(samples_validate, batch_size) }) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = BoltzmannMachine([vis_layer, hid_layer]) rbm.initialize(data) # obtain initial estimate of the reconstruction error perf = ProgressMonitor(generator_metrics = \ [ReconstructionError(), TAPLogLikelihood(10), TAPFreeEnergy(10)]) untrained_performance = perf.epoch_update(data, rbm, store=True, show=False) # set up the optimizer and the fit method opt = optimizers.Gradient(stepsize=learning_rate, tolerance=1e-5) tap = fit.TAP(True, 0.1, 0.01, 25, True, 0.5, 0.001, 0.0) solver = fit.SGD(rbm, data) solver.monitor.generator_metrics.append(TAPLogLikelihood(10)) solver.monitor.generator_metrics.append(TAPFreeEnergy(10)) # fit the model print('training with stochastic gradient ascent') solver.train(opt, num_epochs, method=tap.tap_update) # obtain an estimate of the reconstruction error after 1 epoch trained_performance = solver.monitor.memory[-1] assert (trained_performance['TAPLogLikelihood'] > untrained_performance['TAPLogLikelihood']), \ "TAP log-likelihood did not increase" assert (trained_performance['ReconstructionError'] < untrained_performance['ReconstructionError']), \ "Reconstruction error did not decrease" # close the HDF5 store data.close()
def test_rbm(paysage_path=None): """TODO : this is just a placeholder, need to clean up & simplifiy setup. Also need to figure how to deal with consistent random seeding throughout the codebase to obtain deterministic checkable results. """ num_hidden_units = 50 batch_size = 50 num_epochs = 1 learning_rate = 0.01 mc_steps = 1 if not paysage_path: paysage_path = os.path.dirname( os.path.dirname(os.path.abspath(__file__))) filepath = os.path.join(paysage_path, 'mnist', 'mnist.h5') if not os.path.exists(filepath): raise IOError( "{} does not exist. run mnist/download_mnist.py to fetch from the web" .format(filepath)) shuffled_filepath = os.path.join(paysage_path, 'mnist', 'shuffled_mnist.h5') # shuffle the data if not os.path.exists(shuffled_filepath): shuffler = batch.DataShuffler(filepath, shuffled_filepath, complevel=0) shuffler.shuffle() # set a seed for the random number generator be.set_seed() # set up the reader to get minibatches data = batch.Batch(shuffled_filepath, 'train/images', batch_size, transform=batch.binarize_color, train_fraction=0.99) # set up the model and initialize the parameters vis_layer = layers.BernoulliLayer(data.ncols) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = hidden.Model([vis_layer, hid_layer]) rbm.initialize(data) # obtain initial estimate of the reconstruction error perf = fit.ProgressMonitor(0, data, metrics=[M.ReconstructionError()]) untrained_performance = perf.check_progress(rbm, 0) # set up the optimizer and the fit method opt = optimizers.RMSProp(rbm, stepsize=learning_rate, scheduler=optimizers.PowerLawDecay(0.1)) sampler = fit.DrivenSequentialMC.from_batch(rbm, data, method='stochastic') cd = fit.PCD(rbm, data, opt, sampler, num_epochs, mcsteps=mc_steps, skip=200, metrics=[M.ReconstructionError()]) # fit the model print('training with contrastive divergence') cd.train() # obtain an estimate of the reconstruction error after 1 epoch trained_performance = perf.check_progress(rbm, 0) assert (trained_performance['ReconstructionError'] < untrained_performance['ReconstructionError']), \ "Reconstruction error did not decrease" # close the HDF5 store data.close()