def run_dae():
########################################
# Initialization things with arguments #
########################################
config_root_logger()
log.info("Creating a new DAE")
mnist = MNIST()
config = {
"outdir": 'outputs/dae/mnist/',
"input_size": 28*28,
"tied_weights": True
}
dae = DenoisingAutoencoder(**config)
# # Load initial weights and biases from file
# params_to_load = 'dae_params.pkl'
# dae.load_params(params_to_load)
optimizer = AdaDelta(dae, mnist)
optimizer.train()
# Save some reconstruction output images
import opendeep.data.dataset as datasets
n_examples = 100
test_xs, _ = mnist.getSubset(subset=datasets.TEST)
test_xs = test_xs[:n_examples].eval()
dae.create_reconstruction_image(test_xs)
def run_mlp():
# # define the model layers
# layer1 = BasicLayer(input_size=784, output_size=1000, activation='rectifier')
# layer2 = BasicLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=1000, activation='rectifier')
# classlayer3 = SoftmaxLayer(inputs_hook=(1000, layer2.get_outputs()), output_size=10, out_as_probs=False)
# # add the layers to the prototype
# mlp = Prototype(layers=[layer1, layer2, classlayer3])
# test the new way to automatically fill in inputs_hook for models
mlp = Prototype()
mlp.add(BasicLayer(input_size=784, output_size=1000, activation="rectifier", noise="dropout"))
mlp.add(BasicLayer(output_size=1500, activation="tanh", noise="dropout"))
mlp.add(SoftmaxLayer(output_size=10))
mnist = MNIST()
optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=10)
optimizer.train()
test_data, test_labels = mnist.getSubset(subset=TEST)
test_data = test_data[:25].eval()
test_labels = test_labels[:25].eval()
# use the run function!
yhat = mlp.run(test_data)
print("-------")
print("Prediction: %s" % str(yhat))
print("Actual: %s" % str(test_labels.astype("int32")))
def setUp(self):
# configure the root logger
logger.config_root_logger()
# get a logger for this session
self.log = logging.getLogger(__name__)
# get the mnist dataset
self.mnist = MNIST(binary=False, concat_train_valid=True)
def run_dae():
########################################
# Initialization things with arguments #
########################################
config_root_logger()
log.info("Creating a new DAE")
mnist = MNIST()
config = {
"outdir": 'outputs/dae/mnist/',
"input_size": 28 * 28,
"tied_weights": True
}
dae = DenoisingAutoencoder(**config)
# # Load initial weights and biases from file
# params_to_load = 'dae_params.pkl'
# dae.load_params(params_to_load)
optimizer = AdaDelta(dae, mnist)
optimizer.train()
# Save some reconstruction output images
import opendeep.data.dataset as datasets
n_examples = 100
test_xs, _ = mnist.getSubset(subset=datasets.TEST)
test_xs = test_xs[:n_examples].eval()
dae.create_reconstruction_image(test_xs)
def run_mlp():
# # define the model layers
# layer1 = BasicLayer(input_size=784, output_size=1000, activation='rectifier')
# layer2 = BasicLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=1000, activation='rectifier')
# classlayer3 = SoftmaxLayer(inputs_hook=(1000, layer2.get_outputs()), output_size=10, out_as_probs=False)
# # add the layers to the prototype
# mlp = Prototype(layers=[layer1, layer2, classlayer3])
# test the new way to automatically fill in inputs_hook for models
mlp = Prototype()
mlp.add(
BasicLayer(input_size=784,
output_size=1000,
activation='rectifier',
noise='dropout'))
mlp.add(BasicLayer(output_size=1500, activation='tanh', noise='dropout'))
mlp.add(SoftmaxLayer(output_size=10))
mnist = MNIST()
optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=10)
optimizer.train()
test_data, test_labels = mnist.getSubset(subset=TEST)
test_data = test_data[:25].eval()
test_labels = test_labels[:25].eval()
# use the run function!
yhat = mlp.run(test_data)
print('-------')
print('Prediction: %s' % str(yhat))
print('Actual: %s' % str(test_labels.astype('int32')))
def bernoulli_csl(switch=0):
mnist = MNIST()
train_x = mnist.train_inputs[0]
valid_x = mnist.valid_inputs[0]
test_x = mnist.test_inputs[0]
mnist_b = MNIST(binary=True)
train_x_b = mnist_b.train_inputs[0]
valid_x_b = mnist_b.valid_inputs[0]
test_x_b = mnist_b.test_inputs[0]
means = as_floatX(test_x).eval()
means = numpy.clip(a=means, a_min=1e-10, a_max=(1 - (1e-5)))
#means = as_floatX(numpy.random.uniform(size=(10000,784))) * 0 + 0.5
minibatches = as_floatX(test_x_b.reshape((1000, 10, 784))).eval()
if switch:
# when means is a matrix of (N,D), representing only 1 chain
csl_fn = _compile_csl_fn_v2(means)
compute_CSL_with_minibatches_one_chain(csl_fn, minibatches)
else:
# when means is a 3D tensor (N, K, D)
# When there are N chains, each chain having K samples of dimension D
chains = means.reshape(10, 100, 10, 784)
csl_fn = _compile_csl_fn()
compute_CSL_with_minibatches(csl_fn, minibatches, chains)
del mnist
del mnist_b
def setUp(self):
# configure the root logger
logger.config_root_logger()
# get a logger for this session
self.log = logging.getLogger(__name__)
# get the mnist dataset
self.mnist = MNIST(binary=False)
# instantiate the sequential iterator
self.sequentialIterator = SequentialIterator(self.mnist, dataset.TRAIN,
255, 255)
# instantiate the random iterator
self.randomIterator = RandomIterator(self.mnist, dataset.TRAIN, 255,
255)
def setUp(self):
# configure the root logger
logger.config_root_logger()
# get a logger for this session
self.log = logging.getLogger(__name__)
# get the mnist dataset
self.mnist = MNIST(binary=False, concat_train_valid=True)
class TestMNIST(unittest.TestCase):
def setUp(self):
# configure the root logger
logger.config_root_logger()
# get a logger for this session
self.log = logging.getLogger(__name__)
# get the mnist dataset
self.mnist = MNIST(binary=False, concat_train_valid=True)
def testSizes(self):
assert self.mnist.getDataShape(dataset.TRAIN) == (60000, 784)
assert self.mnist.getDataShape(dataset.VALID) == (10000, 784)
assert self.mnist.getDataShape(dataset.TEST) == (10000, 784)
def tearDown(self):
del self.mnist
def run_dae():
########################################
# Initialization things with arguments #
########################################
config_root_logger()
log.info("Creating a new DAE")
mnist = MNIST()
config = {
"outdir": 'outputs/dae/mnist/',
"input_size": 28 * 28,
"tied_weights": True
}
dae = DenoisingAutoencoder(**config)
# # Load initial weights and biases from file
# params_to_load = 'dae_params.pkl'
# dae.load_params(params_to_load)
optimizer = AdaDelta(model=dae, dataset=mnist, epochs=100)
optimizer.train()
# Save some reconstruction output images
n_examples = 100
test_xs = mnist.test_inputs[:n_examples]
dae.create_reconstruction_image(test_xs)
del dae, mnist
class TestMNIST(unittest.TestCase):
def setUp(self):
# configure the root logger
logger.config_root_logger()
# get a logger for this session
self.log = logging.getLogger(__name__)
# get the mnist dataset
self.mnist = MNIST(binary=False, concat_train_valid=True)
def testSizes(self):
assert self.mnist.getDataShape(dataset.TRAIN) == (60000, 784)
assert self.mnist.getDataShape(dataset.VALID) == (10000, 784)
assert self.mnist.getDataShape(dataset.TEST) == (10000, 784)
def tearDown(self):
del self.mnist
def main():
# First, let's create a simple feedforward MLP with one hidden layer as a Prototype.
mlp = Prototype()
mlp.add(
BasicLayer(input_size=28 * 28,
output_size=1000,
activation='rectifier',
noise='dropout'))
mlp.add(SoftmaxLayer(output_size=10))
# Now, we get to choose what values we want to monitor, and what datasets we would like to monitor on!
# Each Model (in our case, the Prototype), has a get_monitors method that will return a useful
# dictionary of {string_name: monitor_theano_expression} for various computations of the model we might
# care about. By default, this method returns an empty dictionary - it was the model creator's job to
# include potential monitor values.
mlp_monitors = mlp.get_monitors()
mlp_channel = MonitorsChannel(name="error")
for name, expression in mlp_monitors.items():
mlp_channel.add(
Monitor(name=name,
expression=expression,
train=True,
valid=True,
test=True))
# create some monitors for statistics about the hidden and output weights!
# let's look at the mean, variance, and standard deviation of the weights matrices.
weights_channel = MonitorsChannel(name="weights")
hiddens_1 = mlp[0].get_params()[0]
hiddens1_mean = T.mean(hiddens_1)
weights_channel.add(
Monitor(name="hiddens_mean", expression=hiddens1_mean, train=True))
hiddens_2 = mlp[1].get_params()[0]
hiddens2_mean = T.mean(hiddens_2)
weights_channel.add(
Monitor(name="out_mean", expression=hiddens2_mean, train=True))
# create our plot object to do live plotting!
plot = Plot(bokeh_doc_name="Monitor Tutorial",
monitor_channels=[mlp_channel, weights_channel],
open_browser=True)
# use SGD optimizer
optimizer = SGD(model=mlp,
dataset=MNIST(concat_train_valid=False),
n_epoch=500,
save_frequency=100,
batch_size=600,
learning_rate=.01,
lr_decay=False,
momentum=.9,
nesterov_momentum=True)
# train, with the plot!
optimizer.train(plot=plot)
class TestMNIST(unittest.TestCase):
def setUp(self):
# configure the root logger
logger.config_root_logger()
# get a logger for this session
self.log = logging.getLogger(__name__)
# get the mnist dataset
self.mnist = MNIST(binary=False)
# instantiate the sequential iterator
self.sequentialIterator = SequentialIterator(self.mnist, dataset.TRAIN,
255, 255)
# instantiate the random iterator
self.randomIterator = RandomIterator(self.mnist, dataset.TRAIN, 255,
255)
def testSizes(self):
assert self.mnist.getDataShape(dataset.TRAIN) == (60000, 784)
assert self.mnist.getDataShape(dataset.VALID) == (10000, 784)
assert self.mnist.getDataShape(dataset.TEST) == (10000, 784)
def testSequentialIterator(self):
self.log.debug('TESTING SEQUENTIAL ITERATOR')
i = 0
for _, y in self.sequentialIterator:
if i < 2:
self.log.debug(y)
i += 1
assert i == 235
def testRandomIterator(self):
self.log.debug('TESTING RANDOM ITERATOR')
i = 0
for x, y in self.randomIterator:
if i < 2:
self.log.debug(y)
i += 1
assert i == 235
def tearDown(self):
del self.mnist
del self.sequentialIterator
del self.randomIterator
def testDefaultSizes(self):
mnist = MNIST(path="../../datasets/mnist.pkl.gz")
self.assertEquals(mnist.train_inputs.shape, (50000, 1, 28, 28))
self.assertEquals(mnist.train_targets.shape, (50000, ))
self.assertEquals(mnist.valid_inputs.shape, (10000, 1, 28, 28))
self.assertEquals(mnist.valid_targets.shape, (10000, ))
self.assertEquals(mnist.test_inputs.shape, (10000, 1, 28, 28))
self.assertEquals(mnist.test_targets.shape, (10000, ))
del mnist
def bernoulli_csl(switch=0):
mnist = MNIST()
train_x, _ = mnist.getSubset(TRAIN)
valid_x, _ = mnist.getSubset(VALID)
test_x, _ = mnist.getSubset(TEST)
mnist_b = MNIST(binary=True)
train_x_b, _ = mnist_b.getSubset(TRAIN)
valid_x_b, _ = mnist_b.getSubset(VALID)
test_x_b, _ = mnist_b.getSubset(TEST)
means = as_floatX(test_x).eval()
means = numpy.clip(a=means, a_min=1e-10, a_max=(1 - (1e-5)))
# means = as_floatX(numpy.random.uniform(size=(10000,784))) * 0 + 0.5
minibatches = as_floatX(test_x_b.reshape((1000, 10, 784))).eval()
if switch:
# when means is a matrix of (N,D), representing only 1 chain
csl_fn = _compile_csl_fn_v2(means)
compute_CSL_with_minibatches_one_chain(csl_fn, minibatches)
else:
# when means is a 3D tensor (N, K, D)
# When there are N chains, each chain having K samples of dimension D
chains = means.reshape(10, 100, 10, 784)
csl_fn = _compile_csl_fn()
compute_CSL_with_minibatches(csl_fn, minibatches, chains)
del mnist
del mnist_b
def create_mlp():
# define the model layers
relu_layer1 = BasicLayer(input_size=784, output_size=1000, activation='rectifier')
relu_layer2 = BasicLayer(inputs_hook=(1000, relu_layer1.get_outputs()), output_size=1000, activation='rectifier')
class_layer3 = SoftmaxLayer(inputs_hook=(1000, relu_layer2.get_outputs()), output_size=10, out_as_probs=False)
# add the layers as a Prototype
mlp = Prototype(layers=[relu_layer1, relu_layer2, class_layer3])
mnist = MNIST()
optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20)
optimizer.train()
test_data, test_labels = mnist.getSubset(TEST)
test_data = test_data[:25].eval()
test_labels = test_labels[:25].eval()
# use the run function!
preds = mlp.run(test_data)
log.info('-------')
log.info("predicted: %s",str(preds))
log.info("actual: %s",str(test_labels.astype('int32')))
def testConcatSizes(self):
mnist = MNIST(path="../../datasets/mnist.pkl.gz",
concat_train_valid=True,
flatten=False)
self.assertEquals(mnist.train_inputs.shape, (60000, 1, 28, 28))
self.assertEquals(mnist.train_targets.shape, (60000, ))
self.assertEquals(mnist.valid_inputs.shape, (10000, 1, 28, 28))
self.assertEquals(mnist.valid_targets.shape, (10000, ))
self.assertEquals(mnist.test_inputs.shape, (10000, 1, 28, 28))
self.assertEquals(mnist.test_targets.shape, (10000, ))
del mnist
def run_dae():
########################################
# Initialization things with arguments #
########################################
config_root_logger()
log.info("Creating a new DAE")
mnist = MNIST()
config = {"output_path": '../../../../outputs/dae/mnist/'}
dae = DenoisingAutoencoder(config=config, dataset=mnist)
# # Load initial weights and biases from file
# params_to_load = 'dae_params.pkl'
# dae.load_params(params_to_load)
optimizer = AdaDelta(dae, mnist)
optimizer.train()
# Save some reconstruction output images
import opendeep.data.dataset as datasets
n_examples = 100
test_xs = mnist.getDataByIndices(indices=range(n_examples), subset=datasets.TEST)
dae.create_reconstruction_image(test_xs)
def main():
########################################
# Initialization things with arguments #
########################################
logger.config_root_logger()
log.info("Creating a new DAE")
mnist = MNIST()
config = {"output_path": '../../../outputs/dae/mnist/'}
dae = DenoisingAutoencoder(config=config, dataset=mnist)
# # Load initial weights and biases from file
# params_to_load = 'dae_params.pkl'
# dae.load_params(params_to_load)
optimizer = AdaDelta(dae, mnist)
optimizer.train()
# Save some reconstruction output images
import opendeep.data.dataset as datasets
n_examples = 100
test_xs = mnist.getDataByIndices(indices=range(n_examples), subset=datasets.TEST)
dae.create_reconstruction_image(test_xs)
def main(plot=None, n_epoch=10):
print('... loading and seting-up data')
# don't concatenate together train and valid sets
mnist_dataset = MNIST(concat_train_valid=False)
print('... building the model structure')
# create the mlp model from a Prototype
model = build_model()
optimizer, error = setup_optimization(model, n_epoch, mnist_dataset)
print('... training the model')
# [optional] use keyboardInterrupt to save the latest parameters.
if plot:
plot = Plot("OpenDeep MLP Example", monitor_channels=error, open_browser=True)
optimizer.train(monitor_channels=error, plot=plot)
print('... evaluating model')
test_data, test_labels = split_data(mnist_dataset)
evaluate(test_data, test_labels, model)
def main(sequence):
rnn_gsn = RNN_GSN()
# data! (needs to be 3d for rnn).
mnist = MNIST(sequence_number=sequence, seq_3d=True, seq_length=50)
# optimizer!
optimizer = RMSProp(model=rnn_gsn,
dataset=mnist,
epochs=500,
batch_size=50,
save_freq=10,
stop_patience=30,
stop_threshold=.9995,
learning_rate=1e-6,
decay=.95,
max_scaling=1e5,
grad_clip=5.,
hard_clip=False)
# train!
optimizer.train()
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
from opendeep.log.logger import config_root_logger
config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating a Denoising Autoencoder!")
# import the dataset and optimizer to use
from opendeep.data.dataset import TEST
from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
# grab the MNIST dataset
mnist = MNIST()
# create your shiny new DAE
dae = DenoisingAutoencoder()
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=dae, dataset=mnist)
# perform training!
optimizer.train()
# test it on some images!
test_data = mnist.getDataByIndices(indices=range(25), subset=TEST)
corrupted_test = salt_and_pepper(test_data, 0.4).eval()
# use the predict function!
reconstructed_images = dae.predict(corrupted_test)
def run_sequence(sequence=0):
log.info("Creating RNN-RBM for sequence %d!" % sequence)
# grab the MNIST dataset
mnist = MNIST(sequence_number=sequence, concat_train_valid=True)
outdir = "outputs/rnnrbm/mnist_%d/" % sequence
# create the RNN-RBM
rng = numpy.random.RandomState(1234)
mrg = RandomStreams(rng.randint(2**30))
rnnrbm = RNN_RBM(input_size=28 * 28,
hidden_size=1000,
rnn_hidden_size=100,
k=15,
weights_init='uniform',
weights_interval=4 * numpy.sqrt(6. / (28 * 28 + 500)),
rnn_weights_init='identity',
rnn_hidden_activation='relu',
rnn_weights_std=1e-4,
mrg=mrg,
outdir=outdir)
# load pretrained rbm on mnist
# rnnrbm.load_params(outdir + 'trained_epoch_200.pkl')
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=rnnrbm,
dataset=mnist,
n_epoch=200,
batch_size=100,
minimum_batch_size=2,
learning_rate=1e-8,
save_frequency=10,
early_stop_length=200)
crossentropy = Monitor('crossentropy',
rnnrbm.get_monitors()['crossentropy'],
test=True)
error = Monitor('error', rnnrbm.get_monitors()['mse'], test=True)
plot = Plot(bokeh_doc_name='rnnrbm_mnist_%d' % sequence,
monitor_channels=[crossentropy, error],
open_browser=True)
# perform training!
optimizer.train(plot=plot)
# use the generate function!
log.debug("generating images...")
generated, ut = rnnrbm.generate(initial=None, n_steps=400)
# Construct image
image = Image.fromarray(
tile_raster_images(X=generated,
img_shape=(28, 28),
tile_shape=(20, 20),
tile_spacing=(1, 1)))
image.save(outdir + "rnnrbm_mnist_generated.png")
log.debug('saved generated.png')
# Construct image from the weight matrix
image = Image.fromarray(
tile_raster_images(X=rnnrbm.W.get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=closest_to_square_factors(
rnnrbm.hidden_size),
tile_spacing=(1, 1)))
image.save(outdir + "rnnrbm_mnist_weights.png")
log.debug("done!")
del mnist
del rnnrbm
del optimizer
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
from opendeep.log.logger import config_root_logger
config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating a Denoising Autoencoder!")
# import the dataset and optimizer to use
from opendeep.data.dataset import TEST
from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
# grab the MNIST dataset
mnist = MNIST()
# create your shiny new DAE
dae = DenoisingAutoencoder()
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=dae, dataset=mnist, n_epoch=1)
# perform training!
optimizer.train()
# test it on some images!
test_data_m, _ = mnist.getSubset(TEST)
test_data_u, _ = mnist.getSubset(TEST)
test_data = test_data_u[:25].eval()
test_data1 = test_data_m[0:5].eval()
test_data2 = test_data_m[5:10].eval()
from opendeep.models.single_layer.basic import Dense, Softmax
from opendeep.models.utils import Activation
from opendeep.models.container import Prototype
from opendeep.optimization.loss import Neg_LL
from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
from opendeep.log import config_root_logger
config_root_logger()
# grab the MNIST dataset
mnist = MNIST()
# create the basic layer
layer1 = Dense(inputs=((None, 28 * 28), matrix("x")),
outputs=1000,
activation='linear')
layer1_act = Activation(inputs=((None, 1000), layer1.get_outputs()),
activation='relu')
# create the softmax classifier
layer2 = Softmax(inputs=((None, 1000), layer1_act.get_outputs()),
outputs=10,
out_as_probs=True)
# create the mlp from the two layers
mlp = Prototype(layers=[layer1, layer1_act, layer2])
# define the loss function
loss = Neg_LL(inputs=mlp.get_outputs(),
targets=vector("y", dtype="int64"),
def add_list_layers():
# You can also add lists of layers at a time (or as initialization) to a Prototype! This lets you specify
# more complex interactions between layers!
hidden1 = BasicLayer(input_size=28 * 28,
output_size=512,
activation='rectifier',
noise='dropout')
hidden2 = BasicLayer(inputs_hook=(512, hidden1.get_outputs()),
output_size=512,
activation='rectifier',
noise='dropout')
class_layer = SoftmaxLayer(inputs_hook=(512, hidden2.get_outputs()),
output_size=10)
mlp = Prototype([hidden1, hidden2, class_layer])
return mlp
if __name__ == '__main__':
mlp = sequential_add_layers()
optimizer = SGD(model=mlp,
dataset=MNIST(concat_train_valid=True),
n_epoch=500,
batch_size=600,
learning_rate=.01,
momentum=.9,
nesterov_momentum=True)
optimizer.train()
def run_sequence(sequence=0):
log.info("Creating RNN-GSN for sequence %d!" % sequence)
# grab the MNIST dataset
mnist = MNIST(sequence_number=sequence, concat_train_valid=True)
outdir = "outputs/rnngsn/mnist_%d/" % sequence
rng = numpy.random.RandomState(1234)
mrg = RandomStreams(rng.randint(2**30))
rnngsn = RNN_GSN(layers=2,
walkbacks=4,
input_size=28 * 28,
hidden_size=1000,
tied_weights=True,
rnn_hidden_size=100,
weights_init='uniform',
weights_interval='montreal',
rnn_weights_init='identity',
mrg=mrg,
outdir=outdir)
# load pretrained rbm on mnist
# rnngsn.load_gsn_params('outputs/trained_gsn_epoch_1000.pkl')
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=rnngsn,
dataset=mnist,
n_epoch=200,
batch_size=100,
minimum_batch_size=2,
learning_rate=1e-6,
save_frequency=1,
early_stop_length=200)
# optimizer = SGD(model=rnngsn,
# dataset=mnist,
# n_epoch=300,
# batch_size=100,
# minimum_batch_size=2,
# learning_rate=.25,
# lr_decay='exponential',
# lr_factor=.995,
# momentum=0.5,
# nesterov_momentum=True,
# momentum_decay=False,
# save_frequency=20,
# early_stop_length=100)
crossentropy = Monitor('crossentropy',
rnngsn.get_monitors()['noisy_recon_cost'],
test=True)
error = Monitor('error', rnngsn.get_monitors()['mse'], test=True)
# perform training!
optimizer.train(monitor_channels=[crossentropy, error])
# use the generate function!
log.debug("generating images...")
generated, ut = rnngsn.generate(initial=None, n_steps=400)
# Construct image
image = Image.fromarray(
tile_raster_images(X=generated,
img_shape=(28, 28),
tile_shape=(20, 20),
tile_spacing=(1, 1)))
image.save(outdir + "rnngsn_mnist_generated.png")
log.debug('saved generated.png')
# Construct image from the weight matrix
image = Image.fromarray(
tile_raster_images(X=rnngsn.weights_list[0].get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=closest_to_square_factors(
rnngsn.hidden_size),
tile_spacing=(1, 1)))
image.save(outdir + "rnngsn_mnist_weights.png")
log.debug("done!")
del mnist
del rnngsn
del optimizer
from opendeep.data.dataset import TEST
from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
import opendeep.log.logger as logger
logger.config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating softmax!")
# grab the MNIST dataset
mnist = MNIST()
# create the softmax classifier
s = SoftmaxLayer(input_size=28 * 28, output_size=10, out_as_probs=False)
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=s, dataset=mnist, n_epoch=20)
# perform training!
optimizer.train()
# test it on some images!
test_data, test_labels = mnist.getSubset(subset=TEST)
test_data = test_data[:25].eval()
test_labels = test_labels[:25].eval()
# use the run function!
preds = s.run(test_data)
print('-------')
print(preds)
print(test_labels.astype('int32'))
from opendeep.data.dataset import TEST
from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
import opendeep.log.logger as logger
logger.config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating MLP!")
# grab the MNIST dataset
mnist = MNIST()
# create the basic layer
layer1 = BasicLayer(input_size=28*28, output_size=1000, activation='relu')
# create the softmax classifier
layer2 = SoftmaxLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=10, out_as_probs=False)
# create the mlp from the two layers
mlp = Prototype(layers=[layer1, layer2])
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20)
# perform training!
optimizer.train()
# test it on some images!
test_data = mnist.getDataByIndices(indices=range(25), subset=TEST)
# use the predict function!
preds = mlp.predict(test_data)
print '-------'
from opendeep.models.single_layer.basic import SoftmaxLayer
# import the dataset and optimizer to use
from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
from opendeep.log import config_root_logger
config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating softmax!")
# grab the MNIST dataset
mnist = MNIST()
# create the softmax classifier
s = SoftmaxLayer(input_size=28 * 28, output_size=10, out_as_probs=False)
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=s, dataset=mnist, epochs=20)
# perform training!
optimizer.train()
# test it on some images!
test_data, test_labels = mnist.test_inputs[:25], mnist.test_targets[:25]
# use the run function!
preds = s.run(test_data)
print('-------')
print(preds)
print(test_labels.astype('int32'))
print()
print()
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
import opendeep.log.logger as logger
logger.config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating a Denoising Autoencoder!")
# import the dataset and optimizer to use
from opendeep.data.dataset import TEST
from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
# grab the MNIST dataset
mnist = MNIST()
# create your shiny new DAE
dae = DenoisingAutoencoder()
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=dae, dataset=mnist)
# perform training!
optimizer.train()
# test it on some images!
test_data = mnist.getDataByIndices(indices=range(25), subset=TEST)
corrupted_test = salt_and_pepper(test_data, 0.4)
# use the predict function!
reconstructed_images = dae.predict(test_data)
# import the dataset and optimizer to use
from opendeep.data.dataset import TEST
from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
import opendeep.log.logger as logger
logger.config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating MLP!")
# grab the MNIST dataset
mnist = MNIST()
# create the basic layer
layer1 = BasicLayer(input_size=28 * 28,
output_size=1000,
activation='relu')
# create the softmax classifier
layer2 = SoftmaxLayer(inputs_hook=(1000, layer1.get_outputs()),
output_size=10,
out_as_probs=False)
# create the mlp from the two layers
mlp = Prototype(layers=[layer1, layer2])
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20)
# perform training!
optimizer.train()
# test it on some images!
from opendeep.data.dataset import TEST
from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
import opendeep.log.logger as logger
logger.config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating softmax!")
# grab the MNIST dataset
mnist = MNIST()
# create the softmax classifier
s = SoftmaxLayer(input_size=28 * 28, output_size=10, out_as_probs=False)
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=s, dataset=mnist, n_epoch=20)
# perform training!
optimizer.train()
# test it on some images!
test_data = mnist.getDataByIndices(indices=range(25), subset=TEST)
# use the predict function!
preds = s.predict(test_data)
print '-------'
print preds
print mnist.getLabelsByIndices(indices=range(25), subset=TEST)
print
print
from opendeep.utils.image import tile_raster_images
from opendeep.utils.misc import closest_to_square_factors
import PIL.Image as Image
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
import opendeep.log.logger as logger
logger.config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating RBM!")
# grab the MNIST dataset
mnist = MNIST(concat_train_valid=False)
# create the RBM
rng = numpy.random.RandomState(1234)
mrg = theano.tensor.shared_randomstreams.RandomStreams(rng.randint(2**30))
rbm = RBM(input_size=28*28, hidden_size=500, k=15, weights_init='uniform', weights_interval=4*numpy.sqrt(6./(28*28+500)), mrg=mrg)
# rbm.load_params('rbm_trained.pkl')
# make an optimizer to train it (AdaDelta is a good default)
# optimizer = SGD(model=rbm, dataset=mnist, batch_size=20, learning_rate=0.1, lr_decay=False, nesterov_momentum=False, momentum=False)
optimizer = Optimizer(lr_decay=False, learning_rate=0.1, model=rbm, dataset=mnist, batch_size=20, save_frequency=1)
ll = Monitor('pseudo-log', rbm.get_monitors()['pseudo-log'])
# perform training!
optimizer.train(monitor_channels=ll)
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
from opendeep.log.logger import config_root_logger
config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating a Denoising Autoencoder!")
# import the dataset and optimizer to use
from opendeep.data.dataset import TEST
from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
# grab the MNIST dataset
mnist = MNIST()
# create your shiny new DAE
dae = DenoisingAutoencoder()
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=dae, dataset=mnist)
# perform training!
optimizer.train()
# test it on some images!
test_data, _ = mnist.getSubset(TEST)
test_data = test_data[:25].eval()
corrupted_test = salt_and_pepper(test_data, 0.4).eval()
# use the run function!
reconstructed_images = dae.run(corrupted_test)
def main():
########################################
# Initialization things with arguments #
########################################
# use these arguments to get results from paper referenced above
_train_args = {"n_epoch": 1000, # maximum number of times to run through the dataset
"batch_size": 100, # number of examples to process in parallel (minibatch)
"minimum_batch_size": 1, # the minimum number of examples for a batch to be considered
"save_frequency": 1, # how many epochs between saving parameters
"early_stop_threshold": .9995, # multiplier for how much the train cost to improve to not stop early
"early_stop_length": 500, # how many epochs to wait to see if the threshold has been reached
"learning_rate": .25, # initial learning rate for SGD
"lr_decay": 'exponential', # the decay function to use for the learning rate parameter
"lr_factor": .995, # by how much to decay the learning rate each epoch
"momentum": 0.5, # the parameter momentum amount
'momentum_decay': False, # how to decay the momentum each epoch (if applicable)
'momentum_factor': 0, # by how much to decay the momentum (in this case not at all)
'nesterov_momentum': False, # whether to use nesterov momentum update (accelerated momentum)
}
config_root_logger()
log.info("Creating a new GSN")
mnist = MNIST(concat_train_valid=True)
gsn = GSN(layers=2,
walkbacks=4,
hidden_size=1500,
visible_activation='sigmoid',
hidden_activation='tanh',
input_size=28*28,
tied_weights=True,
hidden_add_noise_sigma=2,
input_salt_and_pepper=0.4,
outdir='outputs/test_gsn/',
vis_init=False,
noiseless_h1=True,
input_sampling=True,
weights_init='uniform',
weights_interval='montreal',
bias_init=0,
cost_function='binary_crossentropy')
recon_cost_channel = MonitorsChannel(name='cost')
recon_cost_channel.add(Monitor('recon_cost', gsn.get_monitors()['recon_cost'], test=True))
recon_cost_channel.add(Monitor('noisy_recon_cost', gsn.get_monitors()['noisy_recon_cost'], test=True))
# Load initial weights and biases from file
# params_to_load = '../../../outputs/gsn/mnist/trained_epoch_395.pkl'
# gsn.load_params(params_to_load)
optimizer = SGD(model=gsn, dataset=mnist, **_train_args)
# optimizer = AdaDelta(model=gsn, dataset=mnist, n_epoch=200, batch_size=100, learning_rate=1e-6)
optimizer.train(monitor_channels=recon_cost_channel)
# Save some reconstruction output images
import opendeep.data.dataset as datasets
n_examples = 100
xs_test, _ = mnist.getSubset(datasets.TEST)
xs_test = xs_test[:n_examples].eval()
noisy_xs_test = gsn.f_noise(xs_test)
reconstructed = gsn.run(noisy_xs_test)
# Concatenate stuff
stacked = numpy.vstack(
[numpy.vstack([xs_test[i * 10: (i + 1) * 10],
noisy_xs_test[i * 10: (i + 1) * 10],
reconstructed[i * 10: (i + 1) * 10]])
for i in range(10)])
number_reconstruction = PIL.Image.fromarray(
tile_raster_images(stacked, (gsn.image_height, gsn.image_width), (10, 30))
)
number_reconstruction.save(gsn.outdir + 'reconstruction.png')
log.info("saved output image!")
# Construct image from the weight matrix
image = PIL.Image.fromarray(
tile_raster_images(
X=gsn.weights_list[0].get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=closest_to_square_factors(gsn.hidden_size),
tile_spacing=(1, 1)
)
)
image.save(gsn.outdir + "gsn_mnist_weights.png")
