def run(pretrain_epochs=5, finetune_epochs=5, fit_method=fit.LayerwisePretrain,
show_plot=False):
num_hidden_units = [20**2, 15**2, 10**2]
batch_size = 100
mc_steps = 5
beta_std = 0.6
# set up the reader to get minibatches
data = util.create_batch(batch_size, train_fraction=0.95, transform=transform)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_layer = [layers.BernoulliLayer(n) for n in num_hidden_units]
rbm = BoltzmannMachine([vis_layer] + hid_layer)
# add some penalties
for c in rbm.connections:
c.weights.add_penalty({"matrix": pen.l1_adaptive_decay_penalty_2(1e-4)})
print("Norms of the weights before training")
util.weight_norm_histogram(rbm, show_plot=show_plot)
print('pre-training with persistent contrastive divergence')
cd = fit_method(rbm, data)
learning_rate = schedules.PowerLawDecay(initial=5e-3, coefficient=1)
opt = optimizers.ADAM(stepsize=learning_rate)
cd.train(opt, pretrain_epochs, method=fit.pcd, mcsteps=mc_steps,
init_method="glorot_normal")
util.show_weights(rbm, show_plot, n_weights=16)
print('fine tuning')
cd = fit.StochasticGradientDescent(rbm, data)
cd.monitor.generator_metrics.append(M.JensenShannonDivergence())
learning_rate = schedules.PowerLawDecay(initial=1e-3, coefficient=1)
opt = optimizers.ADAM(stepsize=learning_rate)
cd.train(opt, finetune_epochs, mcsteps=mc_steps, beta_std=beta_std)
util.show_metrics(rbm, cd.monitor)
# evaluate the model
valid = data.get('validate')
util.show_reconstructions(rbm, valid, show_plot, num_to_avg=10)
util.show_fantasy_particles(rbm, valid, show_plot, n_fantasy=10,
beta_std=beta_std, fantasy_steps=100)
util.show_weights(rbm, show_plot, n_weights=16)
print("Norms of the weights after training")
util.weight_norm_histogram(rbm, show_plot=show_plot)
# close the HDF5 store
data.close()
print("Done")
return rbm
def example_mnist_deep_rbm(paysage_path=None, num_epochs=10, show_plot=False):
num_hidden_units = 500
batch_size = 100
learning_rate = schedules.power_law_decay(initial=0.01, coefficient=0.1)
mc_steps = 1
(_, _, shuffled_filepath) = \
util.default_paths(paysage_path)
# set up the reader to get minibatches
data = batch.HDFBatch(shuffled_filepath,
'train/images',
batch_size,
transform=batch.binarize_color,
train_fraction=0.99)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_1_layer = layers.BernoulliLayer(num_hidden_units)
hid_2_layer = layers.BernoulliLayer(num_hidden_units)
rbm = model.Model([vis_layer, hid_1_layer, hid_2_layer])
rbm.initialize(data)
metrics = [
'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore',
'HeatCapacity'
]
perf = fit.ProgressMonitor(data, metrics=metrics)
# set up the optimizer and the fit method
opt = optimizers.ADAM(stepsize=learning_rate)
sampler = fit.SequentialMC.from_batch(rbm, data)
cd = fit.SGD(rbm,
data,
opt,
num_epochs,
method=fit.pcd,
sampler=sampler,
mcsteps=mc_steps,
monitor=perf)
# fit the model
print('training with contrastive divergence')
cd.train_layerwise()
# evaluate the model
util.show_metrics(rbm, perf)
valid = data.get('validate')
util.show_reconstructions(rbm, valid, fit, show_plot)
util.show_fantasy_particles(rbm, valid, fit, show_plot)
util.show_weights(rbm, show_plot)
# close the HDF5 store
data.close()
print("Done")
def example_mnist_rbm(paysage_path=None, show_plot = False):
num_hidden_units = 500
batch_size = 50
num_epochs = 10
learning_rate = 0.01
mc_steps = 1
(_, _, shuffled_filepath) = \
util.default_paths(paysage_path)
# set up the reader to get minibatches
data = batch.Batch(shuffled_filepath,
'train/images',
batch_size,
transform=batch.binarize_color,
train_fraction=0.99)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_layer = layers.BernoulliLayer(num_hidden_units)
rbm = hidden.Model([vis_layer, hid_layer])
rbm.initialize(data)
# set up the optimizer and the fit method
opt = optimizers.ADAM(rbm,
stepsize=learning_rate,
scheduler=optimizers.PowerLawDecay(0.1))
sampler = fit.DrivenSequentialMC.from_batch(rbm, data,
method='stochastic')
cd = fit.PCD(rbm, data, opt, sampler,
num_epochs, mcsteps=mc_steps, skip=200,
metrics=[M.ReconstructionError(),
M.EnergyDistance(),
M.EnergyGap(),
M.EnergyZscore()])
# fit the model
print('training with contrastive divergence')
cd.train()
# evaluate the model
# this will be the same as the final epoch results
# it is repeated here to be consistent with the sklearn rbm example
metrics = [M.ReconstructionError(), M.EnergyDistance(),
M.EnergyGap(), M.EnergyZscore()]
performance = fit.ProgressMonitor(0, data, metrics=metrics)
util.show_metrics(rbm, performance)
util.show_reconstructions(rbm, data.get('validate'), fit, show_plot)
util.show_fantasy_particles(rbm, data.get('validate'), fit, show_plot)
util.show_weights(rbm, show_plot)
# close the HDF5 store
data.close()
print("Done")
def example_mnist_grbm(paysage_path=None, num_epochs=10, show_plot=False):
num_hidden_units = 500
batch_size = 50
learning_rate = 0.001 # gaussian rbm usually requires smaller learnign rate
mc_steps = 1
(_, _, shuffled_filepath) = \
util.default_paths(paysage_path)
# set up the reader to get minibatches
data = batch.Batch(shuffled_filepath,
'train/images',
batch_size,
transform=transform,
train_fraction=0.99)
# set up the model and initialize the parameters
vis_layer = layers.GaussianLayer(data.ncols)
hid_layer = layers.BernoulliLayer(num_hidden_units)
rbm = model.Model([vis_layer, hid_layer])
rbm.initialize(data)
metrics = [
'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore'
]
perf = fit.ProgressMonitor(data, metrics=metrics)
opt = optimizers.ADAM(stepsize=learning_rate,
scheduler=optimizers.PowerLawDecay(0.1))
sampler = fit.DrivenSequentialMC.from_batch(rbm, data, method='stochastic')
cd = fit.SGD(rbm,
data,
opt,
num_epochs,
method=fit.pcd,
sampler=sampler,
mcsteps=mc_steps,
monitor=perf)
# fit the model
print('training with contrastive divergence')
cd.train()
# evaluate the model
util.show_metrics(rbm, perf)
util.show_reconstructions(rbm, data.get('validate'), fit, show_plot)
util.show_fantasy_particles(rbm, data.get('validate'), fit, show_plot)
util.show_weights(rbm, show_plot)
# close the HDF5 store
data.close()
print("Done")
def test_gaussian_1D_1mode_train():
# create some example data
num = 10000
mu = 3
sigma = 1
samples = be.randn((num, 1)) * sigma + mu
# set up the reader to get minibatches
batch_size = 100
samples_train, samples_validate = batch.split_tensor(samples, 0.9)
data = batch.Batch({
'train':
batch.InMemoryTable(samples_train, batch_size),
'validate':
batch.InMemoryTable(samples_validate, batch_size)
})
# parameters
learning_rate = schedules.PowerLawDecay(initial=0.1, coefficient=0.1)
mc_steps = 1
num_epochs = 10
num_sample_steps = 100
# set up the model and initialize the parameters
vis_layer = layers.GaussianLayer(1)
hid_layer = layers.OneHotLayer(1)
rbm = BoltzmannMachine([vis_layer, hid_layer])
rbm.initialize(data, method='hinton')
# modify the parameters to shift the initialized model from the data
# this forces it to train
rbm.layers[0].params = layers.ParamsGaussian(
rbm.layers[0].params.loc - 3, rbm.layers[0].params.log_var - 1)
# set up the optimizer and the fit method
opt = optimizers.ADAM(stepsize=learning_rate)
cd = fit.SGD(rbm, data)
# fit the model
print('training with persistent contrastive divergence')
cd.train(opt, num_epochs, method=fit.pcd, mcsteps=mc_steps)
# sample data from the trained model
model_state = \
samplers.SequentialMC.generate_fantasy_state(rbm, num, num_sample_steps)
pts_trained = model_state[0]
percent_error = 10
mu_trained = be.mean(pts_trained)
assert numpy.abs(mu_trained / mu - 1) < (percent_error / 100)
sigma_trained = numpy.sqrt(be.var(pts_trained))
assert numpy.abs(sigma_trained / sigma - 1) < (percent_error / 100)
def run(paysage_path=None, num_epochs=10, show_plot=False):
num_hidden_units = 500
batch_size = 100
learning_rate = schedules.PowerLawDecay(initial=0.001, coefficient=0.1)
mc_steps = 1
(_, _, shuffled_filepath) = \
util.default_paths(paysage_path)
# set up the reader to get minibatches
data = batch.HDFBatch(shuffled_filepath,
'train/images',
batch_size,
transform=pre.binarize_color,
train_fraction=0.99)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_layer = layers.GaussianLayer(num_hidden_units)
hid_layer.set_fixed_params(["loc", "log_var"])
rbm = model.Model([vis_layer, hid_layer])
rbm.initialize(data, method="glorot_normal")
metrics = ['ReconstructionError', 'EnergyDistance', 'EnergyGap',
'EnergyZscore', 'HeatCapacity', 'WeightSparsity', 'WeightSquare']
perf = fit.ProgressMonitor(data, metrics=metrics)
# set up the optimizer and the fit method
opt = optimizers.ADAM(stepsize=learning_rate)
sampler = fit.DrivenSequentialMC.from_batch(rbm, data)
cd = fit.SGD(rbm, data, opt, num_epochs, sampler, method=fit.pcd,
mcsteps=mc_steps, monitor=perf)
# fit the model
print('training with contrastive divergence')
cd.train()
# evaluate the model
util.show_metrics(rbm, perf)
valid = data.get('validate')
util.show_reconstructions(rbm, valid, fit, show_plot,
n_recon=10, vertical=False, num_to_avg=10)
util.show_fantasy_particles(rbm, valid, fit, show_plot, n_fantasy=25)
util.show_weights(rbm, show_plot, n_weights=25)
# close the HDF5 store
data.close()
print("Done")
def run(num_epochs=10, show_plot=False):
num_hidden_units = 256
batch_size = 100
learning_rate = schedules.PowerLawDecay(initial=0.001, coefficient=0.1)
mc_steps = 1
# set up the reader to get minibatches
data = util.create_batch(batch_size,
train_fraction=0.95,
transform=transform)
# set up the model and initialize the parameters
vis_layer = layers.GaussianLayer(data.ncols)
hid_layer = layers.BernoulliLayer(num_hidden_units)
rbm = BoltzmannMachine([vis_layer, hid_layer])
rbm.initialize(data, 'stddev')
rbm.layers[0].params.log_var[:] = \
be.log(0.05*be.ones_like(rbm.layers[0].params.log_var))
opt = optimizers.ADAM(stepsize=learning_rate)
# This example parameter set for TAP uses gradient descent to optimize the
# Gibbs free energy:
tap = fit.TAP(True, 1.0, 0.01, 100, False, 0.9, 0.001, 0.5)
# This example parameter set for TAP uses self-consistent iteration to
# optimize the Gibbs free energy:
#tap = fit.TAP(False, tolerance=0.001, max_iters=100)
sgd = fit.SGD(rbm, data)
sgd.monitor.generator_metrics.append(TAPFreeEnergy())
sgd.monitor.generator_metrics.append(TAPLogLikelihood())
# fit the model
print('Training with stochastic gradient ascent using TAP expansion')
sgd.train(opt, num_epochs, method=tap.tap_update, mcsteps=mc_steps)
util.show_metrics(rbm, sgd.monitor)
valid = data.get('validate')
util.show_reconstructions(rbm,
valid,
show_plot,
n_recon=10,
vertical=False,
num_to_avg=10)
util.show_fantasy_particles(rbm, valid, show_plot, n_fantasy=5)
util.show_weights(rbm, show_plot, n_weights=25)
# close the HDF5 store
data.close()
print("Done")
def run(num_epochs=10, show_plot=False):
num_hidden_units = 100
batch_size = 100
mc_steps = 10
beta_std = 0.6
# set up the reader to get minibatches
with util.create_batch(batch_size,
train_fraction=0.95,
transform=transform) as data:
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_layer = layers.BernoulliLayer(num_hidden_units, center=False)
rbm = BoltzmannMachine([vis_layer, hid_layer])
rbm.connections[0].weights.add_penalty(
{'matrix': pen.l2_penalty(0.001)})
rbm.initialize(data, method='pca')
print('training with persistent contrastive divergence')
cd = fit.SGD(rbm, data)
learning_rate = schedules.PowerLawDecay(initial=0.01, coefficient=0.1)
opt = optimizers.ADAM(stepsize=learning_rate)
cd.train(opt, num_epochs, mcsteps=mc_steps, method=fit.pcd)
util.show_metrics(rbm, cd.monitor)
# evaluate the model
valid = data.get('validate')
util.show_reconstructions(rbm,
valid,
show_plot,
n_recon=10,
vertical=False,
num_to_avg=10)
util.show_fantasy_particles(rbm,
valid,
show_plot,
n_fantasy=5,
beta_std=beta_std,
fantasy_steps=100)
util.show_weights(rbm, show_plot, n_weights=100)
print("Done")
return rbm
def run(num_epochs=1, show_plot=False):
num_hidden_units = 1
batch_size = 100
mc_steps = 10
beta_std = 0.6
# set up the reader to get minibatches
with batch.in_memory_batch(samples, batch_size,
train_fraction=0.95) as data:
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_layer = layers.BernoulliLayer(num_hidden_units, center=False)
rbm = BoltzmannMachine([vis_layer, hid_layer])
rbm.connections[0].weights.add_penalty(
{'matrix': pen.l2_penalty(0.001)}) # Add regularization term
rbm.initialize(data, method='hinton') # Initialize weights
cd = fit.SGD(rbm, data)
learning_rate = schedules.PowerLawDecay(initial=0.01,
coefficient=0.1)
opt = optimizers.ADAM(stepsize=learning_rate)
print("Train the model...")
cd.train(opt,
num_epochs,
mcsteps=mc_steps,
method=fit.pcd,
verbose=False)
'''
# write on file KL divergences
reverse_KL_div = [ cd.monitor.memory[i]['ReverseKLDivergence'] for i in range(0,len(cd.monitor.memory)) ]
KL_div = [ cd.monitor.memory[i]['KLDivergence'] for i in range(0,len(cd.monitor.memory)) ]
for i in range(0,len(cd.monitor.memory)):
out_file1.write(str(KL_div[i])+" "+str(reverse_KL_div[i])+"\n")
out_file1.close()
# save weights on file
filename = "results/weights/weights-"+temperature[:-4]+".jpg"
Gprotein_util.show_weights(rbm, show_plot=False, n_weights=8, Filename=filename, random=False)
'''
return rbm
def run(num_epochs=10, show_plot=False):
num_hidden_units = 256
batch_size = 100
learning_rate = schedules.PowerLawDecay(initial=0.001, coefficient=0.1)
mc_steps = 1
# set up the reader to get minibatches
data = util.create_batch(batch_size,
train_fraction=0.95,
transform=transform)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_layer = layers.GaussianLayer(num_hidden_units)
rbm = BoltzmannMachine([vis_layer, hid_layer])
rbm.initialize(data)
# set up the optimizer and the fit method
opt = optimizers.ADAM(stepsize=learning_rate)
cd = fit.SGD(rbm, data)
# fit the model
print('training with contrastive divergence')
cd.train(opt, num_epochs, method=fit.pcd, mcsteps=mc_steps)
# evaluate the model
util.show_metrics(rbm, cd.monitor)
valid = data.get('validate')
util.show_reconstructions(rbm,
valid,
show_plot,
n_recon=10,
vertical=False,
num_to_avg=10)
util.show_fantasy_particles(rbm, valid, show_plot, n_fantasy=5)
util.show_weights(rbm, show_plot, n_weights=25)
# close the HDF5 store
data.close()
print("Done")
def run(num_epochs=20, show_plot=False):
num_hidden_units = 200
batch_size = 100
mc_steps = 10
beta_std = 0.95
# set up the reader to get minibatches
data = util.create_batch(batch_size, train_fraction=0.95, transform=transform)
# set up the model and initialize the parameters
vis_layer = layers.GaussianLayer(data.ncols, center=False)
hid_layer = layers.BernoulliLayer(num_hidden_units, center=True)
hid_layer.set_fixed_params(hid_layer.get_param_names())
rbm = BoltzmannMachine([vis_layer, hid_layer])
rbm.initialize(data, 'pca', epochs = 500, verbose=True)
print('training with persistent contrastive divergence')
cd = fit.SGD(rbm, data, fantasy_steps=10)
cd.monitor.generator_metrics.append(M.JensenShannonDivergence())
learning_rate = schedules.PowerLawDecay(initial=1e-3, coefficient=5)
opt = optimizers.ADAM(stepsize=learning_rate)
cd.train(opt, num_epochs, method=fit.pcd, mcsteps=mc_steps,
beta_std=beta_std, burn_in=1)
# evaluate the model
util.show_metrics(rbm, cd.monitor)
valid = data.get('validate')
util.show_reconstructions(rbm, valid, show_plot, n_recon=10, vertical=False)
util.show_fantasy_particles(rbm, valid, show_plot, n_fantasy=5)
util.show_weights(rbm, show_plot, n_weights=100)
# close the HDF5 store
data.close()
print("Done")
return rbm
def run(paysage_path=None, num_epochs=10, show_plot=False):
num_hidden_units = 256
batch_size = 100
learning_rate = schedules.PowerLawDecay(initial=0.01, coefficient=0.1)
mc_steps = 1
(_, _, shuffled_filepath) = \
util.default_paths(paysage_path)
# set up the reader to get minibatches
import pandas
data = batch.InMemoryBatch(pre.binarize_color(
be.float_tensor(
pandas.read_hdf(shuffled_filepath,
key='train/images').as_matrix())),
batch_size,
train_fraction=0.95)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_layer = layers.BernoulliLayer(num_hidden_units)
rbm = model.Model([vis_layer, hid_layer])
rbm.weights[0].add_penalty({'matrix': pen.l2_penalty(0.001)})
rbm.initialize(data, method='glorot_normal')
metrics = [
'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore',
'HeatCapacity', 'WeightSparsity', 'WeightSquare'
]
perf = fit.ProgressMonitor(data, metrics=metrics)
# set up the optimizer and the fit method
opt = optimizers.ADAM(stepsize=learning_rate)
sampler = fit.DrivenSequentialMC.from_batch(rbm, data)
cd = fit.SGD(rbm,
data,
opt,
num_epochs,
sampler,
method=fit.pcd,
mcsteps=mc_steps,
monitor=perf)
# fit the model
print('training with contrastive divergence')
cd.train()
# evaluate the model
util.show_metrics(rbm, perf)
valid = data.get('validate')
util.show_reconstructions(rbm,
valid,
fit,
show_plot,
n_recon=10,
vertical=False,
num_to_avg=10)
util.show_fantasy_particles(rbm, valid, fit, show_plot, n_fantasy=25)
util.show_weights(rbm, show_plot, n_weights=25)
# close the HDF5 store
data.close()
print("Done")
def run(paysage_path=None, num_epochs=10, show_plot=False):
num_hidden_units = 100
batch_size = 100
learning_rate = schedules.PowerLawDecay(initial=0.012, coefficient=0.1)
mc_steps = 1
(_, _, shuffled_filepath) = \
util.default_paths(paysage_path)
# set up the reader to get minibatches
data = batch.HDFBatch(shuffled_filepath,
'train/images',
batch_size,
transform=pre.binarize_color,
train_fraction=0.95)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_1_layer = layers.BernoulliLayer(num_hidden_units)
hid_2_layer = layers.BernoulliLayer(num_hidden_units)
hid_3_layer = layers.BernoulliLayer(num_hidden_units)
rbm = model.Model([vis_layer, hid_1_layer, hid_2_layer, hid_3_layer])
rbm.initialize(data, method='glorot_normal')
print("Norms of the weights before training")
util.weight_norm_histogram(rbm, show_plot=show_plot)
# small penalties prevent the weights from consolidating
rbm.weights[1].add_penalty({'matrix': pen.logdet_penalty(0.001)})
rbm.weights[2].add_penalty({'matrix': pen.logdet_penalty(0.001)})
metrics = [
'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore',
'HeatCapacity', 'WeightSparsity', 'WeightSquare'
]
perf = fit.ProgressMonitor(data, metrics=metrics)
# set up the optimizer and the fit method
opt = optimizers.ADAM(stepsize=learning_rate)
cd = fit.LayerwisePretrain(rbm,
data,
opt,
num_epochs,
method=fit.pcd,
mcsteps=mc_steps,
metrics=metrics)
# fit the model
print('training with persistent contrastive divergence')
cd.train()
# evaluate the model
util.show_metrics(rbm, perf)
valid = data.get('validate')
util.show_reconstructions(rbm, valid, fit, show_plot, num_to_avg=10)
util.show_fantasy_particles(rbm, valid, fit, show_plot)
from math import sqrt
dim = tuple([28] +
[int(sqrt(num_hidden_units)) for _ in range(rbm.num_weights)])
util.show_weights(rbm, show_plot, dim=dim, n_weights=16)
util.show_one_hot_reconstructions(rbm,
fit,
dim=28,
n_recon=16,
num_to_avg=1)
print("Norms of the weights after training")
util.weight_norm_histogram(rbm, show_plot=show_plot)
# close the HDF5 store
data.close()
print("Done")
