def example_mnist_tap_machine(paysage_path=None,
num_epochs=10,
show_plot=False):
num_hidden_units = 256
batch_size = 100
learning_rate = schedules.power_law_decay(initial=0.1, coefficient=0.1)
(_, _, shuffled_filepath) = \
util.default_paths(paysage_path)
# set up the reader to get minibatches
data = batch.HDFBatch(shuffled_filepath,
'train/images',
batch_size,
transform=batch.binarize_color,
train_fraction=0.95)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_layer = layers.BernoulliLayer(num_hidden_units)
rbm = model.Model([vis_layer, hid_layer])
rbm.initialize(data, 'glorot_normal')
perf = fit.ProgressMonitor(
data,
metrics=['ReconstructionError', 'EnergyDistance', 'HeatCapacity'])
opt = optimizers.Gradient(stepsize=learning_rate,
tolerance=1e-4,
ascent=True)
sampler = fit.DrivenSequentialMC.from_batch(rbm, data)
sgd = fit.SGD(rbm,
data,
opt,
num_epochs,
sampler=sampler,
method=fit.tap,
monitor=perf)
# fit the model
print('Training with stochastic gradient ascent using TAP expansion')
sgd.train()
util.show_metrics(rbm, perf)
valid = data.get('validate')
util.show_reconstructions(rbm,
valid,
fit,
show_plot,
n_recon=10,
vertical=False)
util.show_fantasy_particles(rbm, valid, fit, show_plot, n_fantasy=25)
util.show_weights(rbm, show_plot, n_weights=25)
# close the HDF5 store
data.close()
print("Done")
def example_mnist_deep_rbm(paysage_path=None, num_epochs=10, show_plot=False):
num_hidden_units = 500
batch_size = 100
learning_rate = schedules.power_law_decay(initial=0.01, coefficient=0.1)
mc_steps = 1
(_, _, shuffled_filepath) = \
util.default_paths(paysage_path)
# set up the reader to get minibatches
data = batch.HDFBatch(shuffled_filepath,
'train/images',
batch_size,
transform=batch.binarize_color,
train_fraction=0.99)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_1_layer = layers.BernoulliLayer(num_hidden_units)
hid_2_layer = layers.BernoulliLayer(num_hidden_units)
rbm = model.Model([vis_layer, hid_1_layer, hid_2_layer])
rbm.initialize(data)
metrics = [
'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore',
'HeatCapacity'
]
perf = fit.ProgressMonitor(data, metrics=metrics)
# set up the optimizer and the fit method
opt = optimizers.ADAM(stepsize=learning_rate)
sampler = fit.SequentialMC.from_batch(rbm, data)
cd = fit.SGD(rbm,
data,
opt,
num_epochs,
method=fit.pcd,
sampler=sampler,
mcsteps=mc_steps,
monitor=perf)
# fit the model
print('training with contrastive divergence')
cd.train_layerwise()
# evaluate the model
util.show_metrics(rbm, perf)
valid = data.get('validate')
util.show_reconstructions(rbm, valid, fit, show_plot)
util.show_fantasy_particles(rbm, valid, fit, show_plot)
util.show_weights(rbm, show_plot)
# close the HDF5 store
data.close()
print("Done")
def run(paysage_path=None, num_epochs=10, show_plot=False):
num_hidden_units = 500
batch_size = 100
learning_rate = schedules.PowerLawDecay(initial=0.001, coefficient=0.1)
mc_steps = 1
(_, _, shuffled_filepath) = \
util.default_paths(paysage_path)
# set up the reader to get minibatches
data = batch.HDFBatch(shuffled_filepath,
'train/images',
batch_size,
transform=pre.binarize_color,
train_fraction=0.99)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_layer = layers.GaussianLayer(num_hidden_units)
hid_layer.set_fixed_params(["loc", "log_var"])
rbm = model.Model([vis_layer, hid_layer])
rbm.initialize(data, method="glorot_normal")
metrics = ['ReconstructionError', 'EnergyDistance', 'EnergyGap',
'EnergyZscore', 'HeatCapacity', 'WeightSparsity', 'WeightSquare']
perf = fit.ProgressMonitor(data, metrics=metrics)
# set up the optimizer and the fit method
opt = optimizers.ADAM(stepsize=learning_rate)
sampler = fit.DrivenSequentialMC.from_batch(rbm, data)
cd = fit.SGD(rbm, data, opt, num_epochs, sampler, method=fit.pcd,
mcsteps=mc_steps, monitor=perf)
# fit the model
print('training with contrastive divergence')
cd.train()
# evaluate the model
util.show_metrics(rbm, perf)
valid = data.get('validate')
util.show_reconstructions(rbm, valid, fit, show_plot,
n_recon=10, vertical=False, num_to_avg=10)
util.show_fantasy_particles(rbm, valid, fit, show_plot, n_fantasy=25)
util.show_weights(rbm, show_plot, n_weights=25)
# close the HDF5 store
data.close()
print("Done")
def test_rbm(paysage_path=None):
num_hidden_units = 50
batch_size = 50
num_epochs = 1
learning_rate = schedules.PowerLawDecay(initial=0.01, coefficient=0.1)
mc_steps = 1
if not paysage_path:
paysage_path = os.path.dirname(
os.path.dirname(os.path.abspath(__file__)))
filepath = os.path.join(paysage_path, 'mnist', 'mnist.h5')
if not os.path.exists(filepath):
raise IOError(
"{} does not exist. run mnist/download_mnist.py to fetch from the web"
.format(filepath))
shuffled_filepath = os.path.join(paysage_path, 'mnist',
'shuffled_mnist.h5')
# shuffle the data
if not os.path.exists(shuffled_filepath):
shuffler = batch.DataShuffler(filepath, shuffled_filepath, complevel=0)
shuffler.shuffle()
# set a seed for the random number generator
be.set_seed()
# set up the reader to get minibatches
data = batch.HDFBatch(shuffled_filepath,
'train/images',
batch_size,
transform=pre.binarize_color,
train_fraction=0.99)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_layer = layers.BernoulliLayer(num_hidden_units)
rbm = model.Model([vis_layer, hid_layer])
rbm.initialize(data)
# obtain initial estimate of the reconstruction error
perf = fit.ProgressMonitor(data, metrics=['ReconstructionError'])
untrained_performance = perf.check_progress(rbm)
# set up the optimizer and the fit method
opt = optimizers.RMSProp(stepsize=learning_rate)
sampler = fit.DrivenSequentialMC.from_batch(rbm, data)
cd = fit.SGD(rbm,
data,
opt,
num_epochs,
sampler,
method=fit.pcd,
mcsteps=mc_steps,
monitor=perf)
# fit the model
print('training with contrastive divergence')
cd.train()
# obtain an estimate of the reconstruction error after 1 epoch
trained_performance = perf.check_progress(rbm)
assert (trained_performance['ReconstructionError'] <
untrained_performance['ReconstructionError']), \
"Reconstruction error did not decrease"
# close the HDF5 store
data.close()
def run(paysage_path=None, num_epochs=10, show_plot=False):
num_hidden_units = 100
batch_size = 100
learning_rate = schedules.PowerLawDecay(initial=0.012, coefficient=0.1)
mc_steps = 1
(_, _, shuffled_filepath) = \
util.default_paths(paysage_path)
# set up the reader to get minibatches
data = batch.HDFBatch(shuffled_filepath,
'train/images',
batch_size,
transform=pre.binarize_color,
train_fraction=0.95)
# set up the model and initialize the parameters
vis_layer = layers.BernoulliLayer(data.ncols)
hid_1_layer = layers.BernoulliLayer(num_hidden_units)
hid_2_layer = layers.BernoulliLayer(num_hidden_units)
hid_3_layer = layers.BernoulliLayer(num_hidden_units)
rbm = model.Model([vis_layer, hid_1_layer, hid_2_layer, hid_3_layer])
rbm.initialize(data, method='glorot_normal')
print("Norms of the weights before training")
util.weight_norm_histogram(rbm, show_plot=show_plot)
# small penalties prevent the weights from consolidating
rbm.weights[1].add_penalty({'matrix': pen.logdet_penalty(0.001)})
rbm.weights[2].add_penalty({'matrix': pen.logdet_penalty(0.001)})
metrics = [
'ReconstructionError', 'EnergyDistance', 'EnergyGap', 'EnergyZscore',
'HeatCapacity', 'WeightSparsity', 'WeightSquare'
]
perf = fit.ProgressMonitor(data, metrics=metrics)
# set up the optimizer and the fit method
opt = optimizers.ADAM(stepsize=learning_rate)
cd = fit.LayerwisePretrain(rbm,
data,
opt,
num_epochs,
method=fit.pcd,
mcsteps=mc_steps,
metrics=metrics)
# fit the model
print('training with persistent contrastive divergence')
cd.train()
# evaluate the model
util.show_metrics(rbm, perf)
valid = data.get('validate')
util.show_reconstructions(rbm, valid, fit, show_plot, num_to_avg=10)
util.show_fantasy_particles(rbm, valid, fit, show_plot)
from math import sqrt
dim = tuple([28] +
[int(sqrt(num_hidden_units)) for _ in range(rbm.num_weights)])
util.show_weights(rbm, show_plot, dim=dim, n_weights=16)
util.show_one_hot_reconstructions(rbm,
fit,
dim=28,
n_recon=16,
num_to_avg=1)
print("Norms of the weights after training")
util.weight_norm_histogram(rbm, show_plot=show_plot)
# close the HDF5 store
data.close()
print("Done")