def __init__(self):
# Define some model hyperparameters to work with MNIST images!
input_size = 28*28 # dimensions of image
hidden_size = 1000 # number of hidden units - generally bigger than input size for DAE
# Now, define the symbolic input to the model (Theano)
# We use a matrix rather than a vector so that minibatch processing can be done in parallel.
x = T.matrix("X")
self.inputs = [x]
# Build the model's parameters - a weight matrix and two bias vectors
W = get_weights_uniform(shape=(input_size, hidden_size), name="W")
b0 = get_bias(shape=input_size, name="b0")
b1 = get_bias(shape=hidden_size, name="b1")
self.params = [W, b0, b1]
# Perform the computation for a denoising autoencoder!
# first, add noise (corrupt) the input
corrupted_input = salt_and_pepper(input=x, noise_level=0.4)
# next, run the hidden layer given the inputs (the encoding function)
hiddens = tanh(T.dot(corrupted_input, W) + b1)
# finally, create the reconstruction from the hidden layer (we tie the weights with W.T)
reconstruction = sigmoid(T.dot(hiddens, W.T) + b0)
# the training cost is reconstruction error - with MNIST this is binary cross-entropy
self.train_cost = binary_crossentropy(output=reconstruction, target=x)
# Compile everything into a Theano function for prediction!
# When using real-world data in predictions, we wouldn't corrupt the input first.
# Therefore, create another version of the hiddens and reconstruction without adding the noise
hiddens_predict = tanh(T.dot(x, W) + b1)
self.recon_predict = sigmoid(T.dot(hiddens_predict, W.T) + b0)
def __init__(self):
# Define some model hyperparameters to work with MNIST images!
input_size = 28 * 28 # dimensions of image
hidden_size = 1000 # number of hidden units - generally bigger than input size for DAE
# Now, define the symbolic input to the model (Theano)
# We use a matrix rather than a vector so that minibatch processing can be done in parallel.
x = T.matrix("X")
self.inputs = [x]
# Build the model's parameters - a weight matrix and two bias vectors
W = get_weights_uniform(shape=(input_size, hidden_size), name="W")
b0 = get_bias(shape=input_size, name="b0")
b1 = get_bias(shape=hidden_size, name="b1")
self.params = [W, b0, b1]
# Perform the computation for a denoising autoencoder!
# first, add noise (corrupt) the input
corrupted_input = salt_and_pepper(input=x, noise_level=0.4)
# next, run the hidden layer given the inputs (the encoding function)
hiddens = tanh(T.dot(corrupted_input, W) + b1)
# finally, create the reconstruction from the hidden layer (we tie the weights with W.T)
reconstruction = sigmoid(T.dot(hiddens, W.T) + b0)
# the training cost is reconstruction error - with MNIST this is binary cross-entropy
self.train_cost = binary_crossentropy(output=reconstruction, target=x)
# Compile everything into a Theano function for prediction!
# When using real-world data in predictions, we wouldn't corrupt the input first.
# Therefore, create another version of the hiddens and reconstruction without adding the noise
hiddens_predict = tanh(T.dot(x, W) + b1)
self.recon_predict = sigmoid(T.dot(hiddens_predict, W.T) + b0)
# grab the MNIST dataset
mnist = MNIST()
# create your shiny new DAE
dae = DenoisingAutoencoder()
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=dae, dataset=mnist)
# perform training!
optimizer.train()
# test it on some images!
test_data, _ = mnist.getSubset(TEST)
test_data = test_data[:25].eval()
corrupted_test = salt_and_pepper(test_data, 0.4).eval()
# use the run function!
reconstructed_images = dae.run(corrupted_test)
# create an image from this reconstruction!
# imports for working with tiling outputs into one image
from opendeep.utils.image import tile_raster_images
import numpy
import PIL
# stack the image matrices together in three 5x5 grids next to each other using numpy
stacked = numpy.vstack([
numpy.vstack([
test_data[i * 5:(i + 1) * 5], corrupted_test[i * 5:(i + 1) * 5],
reconstructed_images[i * 5:(i + 1) * 5]
]) for i in range(5)
])
def __init__(self,
config=None,
defaults=_defaults,
inputs_hook=None,
hiddens_hook=None,
params_hook=None,
input_size=None,
hidden_size=None,
corruption_level=None,
hidden_activation=None,
visible_activation=None,
cost_function=None):
# Now, initialize with Model class to combine config and defaults!
# Here, defaults is defined via a dictionary. However, you could also
# pass a filename to a JSON or YAML file with the same format.
super(DenoisingAutoencoder, self).__init__(config=config,
defaults=defaults)
# Any parameter from the 'config' will overwrite the 'defaults' dictionary.
# These parameters are now accessible from the 'self.args' variable!
# When accessing model parameters, it is best practice to try to find the parameters
# explicitly passed first, and then go to the 'self.args' configuration.
# Define model hyperparameters
# deal with the inputs_hook and hiddens_hook for the size parameters!
# if the hook exists, grab the size from the first element of the tuple.
if inputs_hook:
input_size = inputs_hook[0]
# otherwise, grab the size from the configurations.
else:
input_size = input_size or self.args.get('input_size')
if hiddens_hook:
hidden_size = hiddens_hook[0]
else:
hidden_size = hidden_size or self.args.get('hidden_size')
corruption_level = corruption_level or self.args.get(
'corruption_level')
# use the helper methods to grab appropriate activation functions from names!
hidden_act_name = hidden_activation or self.args.get(
'hidden_activation')
hidden_activation = get_activation_function(hidden_act_name)
visible_act_name = visible_activation or self.args.get(
'visible_activation')
visible_activation = get_activation_function(visible_act_name)
# do the same for the cost function
cost_func_name = cost_function or self.args.get('cost_function')
cost_function = get_cost_function(cost_func_name)
# Now, define the symbolic input to the model (Theano)
# We use a matrix rather than a vector so that minibatch processing can be done in parallel.
# Make sure to deal with 'inputs_hook' if it exists!
if inputs_hook:
# grab the new input variable from the inputs_hook tuple
x = inputs_hook[1]
else:
x = T.fmatrix("X")
self.inputs = [x]
# Build the model's parameters - a weight matrix and two bias vectors
# Make sure to deal with 'params_hook' if it exists!
if params_hook:
# check to see if it contains the three necessary variables
assert len(params_hook
) == 3, "Not correct number of params to DAE, needs 3!"
W, b0, b1 = params_hook
else:
W = get_weights_uniform(shape=(input_size, hidden_size), name="W")
b0 = get_bias(shape=input_size, name="b0")
b1 = get_bias(shape=hidden_size, name="b1")
self.params = [W, b0, b1]
# Perform the computation for a denoising autoencoder!
# first, add noise (corrupt) the input
corrupted_input = salt_and_pepper(input=x,
corruption_level=corruption_level)
# next, compute the hidden layer given the inputs (the encoding function)
# We don't need to worry about hiddens_hook during training, because we can't
# compute a cost without having the input!
# hiddens_hook is more for the predict function and linking methods below.
hiddens = hidden_activation(T.dot(corrupted_input, W) + b1)
# finally, create the reconstruction from the hidden layer (we tie the weights with W.T)
reconstruction = visible_activation(T.dot(hiddens, W.T) + b0)
# the training cost is reconstruction error
self.train_cost = cost_function(output=reconstruction, target=x)
# Compile everything into a Theano function for prediction!
# When using real-world data in predictions, we wouldn't corrupt the input first.
# Therefore, create another version of the hiddens and reconstruction without adding the noise.
# Here is where we would handle hiddens_hook because this is a generative model!
# For the predict function, it would take in the hiddens instead of the input variable x.
if hiddens_hook:
self.hiddens = hiddens_hook[1]
else:
self.hiddens = hidden_activation(T.dot(x, W) + b1)
# make the reconstruction (generated) from the hiddens
self.recon_predict = visible_activation(T.dot(self.hiddens, W.T) + b0)
# now compile the predict function accordingly - if it used x or hiddens as the input.
if hiddens_hook:
self.f_predict = function(inputs=[self.hiddens],
outputs=self.recon_predict)
else:
self.f_predict = function(inputs=[x], outputs=self.recon_predict)
def __init__(self, inputs_hook=None, hiddens_hook=None, params_hook=None,
input_size=28*28, hidden_size=1000, noise_level=0.4,
hidden_activation='tanh', visible_activation='sigmoid', cost_function='binary_crossentropy'):
# initialize the Model superclass
super(DenoisingAutoencoder, self).__init__(
**{arg: val for (arg, val) in locals().iteritems() if arg is not 'self'}
)
# Define model hyperparameters
# deal with the inputs_hook and hiddens_hook for the size parameters!
# if the hook exists, grab the size from the first element of the tuple.
if self.inputs_hook is not None:
assert len(self.inputs_hook) == 2, "Was expecting inputs_hook to be a tuple."
self.input_size = inputs_hook[0]
if self.hiddens_hook is not None:
assert len(self.hiddens_hook) == 2, "was expecting hiddens_hook to be a tuple."
hidden_size = hiddens_hook[0]
# use the helper methods to grab appropriate activation functions from names!
hidden_activation = get_activation_function(hidden_activation)
visible_activation = get_activation_function(visible_activation)
# do the same for the cost function
cost_function = get_cost_function(cost_function)
# Now, define the symbolic input to the model (Theano)
# We use a matrix rather than a vector so that minibatch processing can be done in parallel.
# Make sure to deal with 'inputs_hook' if it exists!
if self.inputs_hook is not None:
# grab the new input variable from the inputs_hook tuple
x = self.inputs_hook[1]
else:
x = T.matrix("X")
self.inputs = [x]
# Build the model's parameters - a weight matrix and two bias vectors
# Make sure to deal with 'params_hook' if it exists!
if self.params_hook:
# check to see if it contains the three necessary variables
assert len(self.params_hook) == 3, "Not correct number of params to DAE, needs 3!"
W, b0, b1 = self.params_hook
else:
W = get_weights_uniform(shape=(self.input_size, hidden_size), name="W")
b0 = get_bias(shape=self.input_size, name="b0")
b1 = get_bias(shape=hidden_size, name="b1")
self.params = [W, b0, b1]
# Perform the computation for a denoising autoencoder!
# first, add noise (corrupt) the input
corrupted_input = salt_and_pepper(input=x, noise_level=noise_level)
# next, run the hidden layer given the inputs (the encoding function)
# We don't need to worry about hiddens_hook during training, because we can't
# run a cost without having the input!
# hiddens_hook is more for the run function and linking methods below.
hiddens = hidden_activation(T.dot(corrupted_input, W) + b1)
# finally, create the reconstruction from the hidden layer (we tie the weights with W.T)
reconstruction = visible_activation(T.dot(hiddens, W.T) + b0)
# the training cost is reconstruction error
self.train_cost = cost_function(output=reconstruction, target=x)
# Compile everything into a Theano function for prediction!
# When using real-world data in predictions, we wouldn't corrupt the input first.
# Therefore, create another version of the hiddens and reconstruction without adding the noise.
# Here is where we would handle hiddens_hook because this is a generative model!
# For the run function, it would take in the hiddens instead of the input variable x.
if self.hiddens_hook is not None:
self.hiddens = self.hiddens_hook[1]
else:
self.hiddens = hidden_activation(T.dot(x, W) + b1)
# make the reconstruction (generated) from the hiddens
self.recon_predict = visible_activation(T.dot(self.hiddens, W.T) + b0)
# now compile the run function accordingly - if it used x or hiddens as the input.
if self.hiddens_hook is not None:
self.f_run = function(inputs=[self.hiddens], outputs=self.recon_predict)
else:
self.f_run = function(inputs=[x], outputs=self.recon_predict)
from opendeep.optimization.adadelta import AdaDelta
# grab the MNIST dataset
mnist = MNIST()
# create your shiny new DAE
dae = DenoisingAutoencoder()
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=dae, dataset=mnist, epochs=100)
# perform training!
optimizer.train()
# test it on some images!
test_data = mnist.test_inputs[:25]
corrupted_test = salt_and_pepper(test_data, 0.4).eval()
# use the run function!
reconstructed_images = dae.run(corrupted_test)
# create an image from this reconstruction!
# imports for working with tiling outputs into one image
from opendeep.utils.image import tile_raster_images
import numpy
import PIL
# stack the image matrices together in three 5x5 grids next to each other using numpy
stacked = numpy.vstack(
[numpy.vstack([test_data[i*5 : (i+1)*5],
corrupted_test[i*5 : (i+1)*5],
reconstructed_images[i*5 : (i+1)*5]])
for i in range(5)])
def __init__(self,
inputs_hook=None,
hiddens_hook=None,
params_hook=None,
input_size=28 * 28,
hidden_size=1000,
noise_level=0.4,
hidden_activation='tanh',
visible_activation='sigmoid',
cost_function='binary_crossentropy'):
# initialize the Model superclass
super(DenoisingAutoencoder, self).__init__(**{
arg: val
for (arg, val) in locals().iteritems() if arg is not 'self'
})
# Define model hyperparameters
# deal with the inputs_hook and hiddens_hook for the size parameters!
# if the hook exists, grab the size from the first element of the tuple.
if self.inputs_hook is not None:
assert len(self.inputs_hook
) == 2, "Was expecting inputs_hook to be a tuple."
self.input_size = inputs_hook[0]
if self.hiddens_hook is not None:
assert len(self.hiddens_hook
) == 2, "was expecting hiddens_hook to be a tuple."
hidden_size = hiddens_hook[0]
# use the helper methods to grab appropriate activation functions from names!
hidden_activation = get_activation_function(hidden_activation)
visible_activation = get_activation_function(visible_activation)
# do the same for the cost function
cost_function = get_cost_function(cost_function)
# Now, define the symbolic input to the model (Theano)
# We use a matrix rather than a vector so that minibatch processing can be done in parallel.
# Make sure to deal with 'inputs_hook' if it exists!
if self.inputs_hook is not None:
# grab the new input variable from the inputs_hook tuple
x = self.inputs_hook[1]
else:
x = T.matrix("X")
self.inputs = [x]
# Build the model's parameters - a weight matrix and two bias vectors
# Make sure to deal with 'params_hook' if it exists!
if self.params_hook:
# check to see if it contains the three necessary variables
assert len(self.params_hook
) == 3, "Not correct number of params to DAE, needs 3!"
W, b0, b1 = self.params_hook
else:
W = get_weights_uniform(shape=(self.input_size, hidden_size),
name="W")
b0 = get_bias(shape=self.input_size, name="b0")
b1 = get_bias(shape=hidden_size, name="b1")
self.params = [W, b0, b1]
# Perform the computation for a denoising autoencoder!
# first, add noise (corrupt) the input
corrupted_input = salt_and_pepper(input=x, noise_level=noise_level)
# next, run the hidden layer given the inputs (the encoding function)
# We don't need to worry about hiddens_hook during training, because we can't
# run a cost without having the input!
# hiddens_hook is more for the run function and linking methods below.
hiddens = hidden_activation(T.dot(corrupted_input, W) + b1)
# finally, create the reconstruction from the hidden layer (we tie the weights with W.T)
reconstruction = visible_activation(T.dot(hiddens, W.T) + b0)
# the training cost is reconstruction error
self.train_cost = cost_function(output=reconstruction, target=x)
# Compile everything into a Theano function for prediction!
# When using real-world data in predictions, we wouldn't corrupt the input first.
# Therefore, create another version of the hiddens and reconstruction without adding the noise.
# Here is where we would handle hiddens_hook because this is a generative model!
# For the run function, it would take in the hiddens instead of the input variable x.
if self.hiddens_hook is not None:
self.hiddens = self.hiddens_hook[1]
else:
self.hiddens = hidden_activation(T.dot(x, W) + b1)
# make the reconstruction (generated) from the hiddens
self.recon_predict = visible_activation(T.dot(self.hiddens, W.T) + b0)
# now compile the run function accordingly - if it used x or hiddens as the input.
if self.hiddens_hook is not None:
self.f_run = function(inputs=[self.hiddens],
outputs=self.recon_predict)
else:
self.f_run = function(inputs=[x], outputs=self.recon_predict)
