def run_dae(): ######################################## # Initialization things with arguments # ######################################## config_root_logger() log.info("Creating a new DAE") mnist = MNIST() config = { "outdir": 'outputs/dae/mnist/', "input_size": 28*28, "tied_weights": True } dae = DenoisingAutoencoder(**config) # # Load initial weights and biases from file # params_to_load = 'dae_params.pkl' # dae.load_params(params_to_load) optimizer = AdaDelta(dae, mnist) optimizer.train() # Save some reconstruction output images import opendeep.data.dataset as datasets n_examples = 100 test_xs, _ = mnist.getSubset(subset=datasets.TEST) test_xs = test_xs[:n_examples].eval() dae.create_reconstruction_image(test_xs)
def run_mlp(): # # define the model layers # layer1 = BasicLayer(input_size=784, output_size=1000, activation='rectifier') # layer2 = BasicLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=1000, activation='rectifier') # classlayer3 = SoftmaxLayer(inputs_hook=(1000, layer2.get_outputs()), output_size=10, out_as_probs=False) # # add the layers to the prototype # mlp = Prototype(layers=[layer1, layer2, classlayer3]) # test the new way to automatically fill in inputs_hook for models mlp = Prototype() mlp.add(BasicLayer(input_size=784, output_size=1000, activation="rectifier", noise="dropout")) mlp.add(BasicLayer(output_size=1500, activation="tanh", noise="dropout")) mlp.add(SoftmaxLayer(output_size=10)) mnist = MNIST() optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=10) optimizer.train() test_data, test_labels = mnist.getSubset(subset=TEST) test_data = test_data[:25].eval() test_labels = test_labels[:25].eval() # use the run function! yhat = mlp.run(test_data) print("-------") print("Prediction: %s" % str(yhat)) print("Actual: %s" % str(test_labels.astype("int32")))
def setUp(self): # configure the root logger logger.config_root_logger() # get a logger for this session self.log = logging.getLogger(__name__) # get the mnist dataset self.mnist = MNIST(binary=False, concat_train_valid=True)
def run_dae(): ######################################## # Initialization things with arguments # ######################################## config_root_logger() log.info("Creating a new DAE") mnist = MNIST() config = { "outdir": 'outputs/dae/mnist/', "input_size": 28 * 28, "tied_weights": True } dae = DenoisingAutoencoder(**config) # # Load initial weights and biases from file # params_to_load = 'dae_params.pkl' # dae.load_params(params_to_load) optimizer = AdaDelta(dae, mnist) optimizer.train() # Save some reconstruction output images import opendeep.data.dataset as datasets n_examples = 100 test_xs, _ = mnist.getSubset(subset=datasets.TEST) test_xs = test_xs[:n_examples].eval() dae.create_reconstruction_image(test_xs)
def run_mlp(): # # define the model layers # layer1 = BasicLayer(input_size=784, output_size=1000, activation='rectifier') # layer2 = BasicLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=1000, activation='rectifier') # classlayer3 = SoftmaxLayer(inputs_hook=(1000, layer2.get_outputs()), output_size=10, out_as_probs=False) # # add the layers to the prototype # mlp = Prototype(layers=[layer1, layer2, classlayer3]) # test the new way to automatically fill in inputs_hook for models mlp = Prototype() mlp.add( BasicLayer(input_size=784, output_size=1000, activation='rectifier', noise='dropout')) mlp.add(BasicLayer(output_size=1500, activation='tanh', noise='dropout')) mlp.add(SoftmaxLayer(output_size=10)) mnist = MNIST() optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=10) optimizer.train() test_data, test_labels = mnist.getSubset(subset=TEST) test_data = test_data[:25].eval() test_labels = test_labels[:25].eval() # use the run function! yhat = mlp.run(test_data) print('-------') print('Prediction: %s' % str(yhat)) print('Actual: %s' % str(test_labels.astype('int32')))
def bernoulli_csl(switch=0): mnist = MNIST() train_x = mnist.train_inputs[0] valid_x = mnist.valid_inputs[0] test_x = mnist.test_inputs[0] mnist_b = MNIST(binary=True) train_x_b = mnist_b.train_inputs[0] valid_x_b = mnist_b.valid_inputs[0] test_x_b = mnist_b.test_inputs[0] means = as_floatX(test_x).eval() means = numpy.clip(a=means, a_min=1e-10, a_max=(1 - (1e-5))) #means = as_floatX(numpy.random.uniform(size=(10000,784))) * 0 + 0.5 minibatches = as_floatX(test_x_b.reshape((1000, 10, 784))).eval() if switch: # when means is a matrix of (N,D), representing only 1 chain csl_fn = _compile_csl_fn_v2(means) compute_CSL_with_minibatches_one_chain(csl_fn, minibatches) else: # when means is a 3D tensor (N, K, D) # When there are N chains, each chain having K samples of dimension D chains = means.reshape(10, 100, 10, 784) csl_fn = _compile_csl_fn() compute_CSL_with_minibatches(csl_fn, minibatches, chains) del mnist del mnist_b
def setUp(self): # configure the root logger logger.config_root_logger() # get a logger for this session self.log = logging.getLogger(__name__) # get the mnist dataset self.mnist = MNIST(binary=False) # instantiate the sequential iterator self.sequentialIterator = SequentialIterator(self.mnist, dataset.TRAIN, 255, 255) # instantiate the random iterator self.randomIterator = RandomIterator(self.mnist, dataset.TRAIN, 255, 255)
class TestMNIST(unittest.TestCase): def setUp(self): # configure the root logger logger.config_root_logger() # get a logger for this session self.log = logging.getLogger(__name__) # get the mnist dataset self.mnist = MNIST(binary=False, concat_train_valid=True) def testSizes(self): assert self.mnist.getDataShape(dataset.TRAIN) == (60000, 784) assert self.mnist.getDataShape(dataset.VALID) == (10000, 784) assert self.mnist.getDataShape(dataset.TEST) == (10000, 784) def tearDown(self): del self.mnist
def run_dae(): ######################################## # Initialization things with arguments # ######################################## config_root_logger() log.info("Creating a new DAE") mnist = MNIST() config = { "outdir": 'outputs/dae/mnist/', "input_size": 28 * 28, "tied_weights": True } dae = DenoisingAutoencoder(**config) # # Load initial weights and biases from file # params_to_load = 'dae_params.pkl' # dae.load_params(params_to_load) optimizer = AdaDelta(model=dae, dataset=mnist, epochs=100) optimizer.train() # Save some reconstruction output images n_examples = 100 test_xs = mnist.test_inputs[:n_examples] dae.create_reconstruction_image(test_xs) del dae, mnist
def main(): # First, let's create a simple feedforward MLP with one hidden layer as a Prototype. mlp = Prototype() mlp.add( BasicLayer(input_size=28 * 28, output_size=1000, activation='rectifier', noise='dropout')) mlp.add(SoftmaxLayer(output_size=10)) # Now, we get to choose what values we want to monitor, and what datasets we would like to monitor on! # Each Model (in our case, the Prototype), has a get_monitors method that will return a useful # dictionary of {string_name: monitor_theano_expression} for various computations of the model we might # care about. By default, this method returns an empty dictionary - it was the model creator's job to # include potential monitor values. mlp_monitors = mlp.get_monitors() mlp_channel = MonitorsChannel(name="error") for name, expression in mlp_monitors.items(): mlp_channel.add( Monitor(name=name, expression=expression, train=True, valid=True, test=True)) # create some monitors for statistics about the hidden and output weights! # let's look at the mean, variance, and standard deviation of the weights matrices. weights_channel = MonitorsChannel(name="weights") hiddens_1 = mlp[0].get_params()[0] hiddens1_mean = T.mean(hiddens_1) weights_channel.add( Monitor(name="hiddens_mean", expression=hiddens1_mean, train=True)) hiddens_2 = mlp[1].get_params()[0] hiddens2_mean = T.mean(hiddens_2) weights_channel.add( Monitor(name="out_mean", expression=hiddens2_mean, train=True)) # create our plot object to do live plotting! plot = Plot(bokeh_doc_name="Monitor Tutorial", monitor_channels=[mlp_channel, weights_channel], open_browser=True) # use SGD optimizer optimizer = SGD(model=mlp, dataset=MNIST(concat_train_valid=False), n_epoch=500, save_frequency=100, batch_size=600, learning_rate=.01, lr_decay=False, momentum=.9, nesterov_momentum=True) # train, with the plot! optimizer.train(plot=plot)
class TestMNIST(unittest.TestCase): def setUp(self): # configure the root logger logger.config_root_logger() # get a logger for this session self.log = logging.getLogger(__name__) # get the mnist dataset self.mnist = MNIST(binary=False) # instantiate the sequential iterator self.sequentialIterator = SequentialIterator(self.mnist, dataset.TRAIN, 255, 255) # instantiate the random iterator self.randomIterator = RandomIterator(self.mnist, dataset.TRAIN, 255, 255) def testSizes(self): assert self.mnist.getDataShape(dataset.TRAIN) == (60000, 784) assert self.mnist.getDataShape(dataset.VALID) == (10000, 784) assert self.mnist.getDataShape(dataset.TEST) == (10000, 784) def testSequentialIterator(self): self.log.debug('TESTING SEQUENTIAL ITERATOR') i = 0 for _, y in self.sequentialIterator: if i < 2: self.log.debug(y) i += 1 assert i == 235 def testRandomIterator(self): self.log.debug('TESTING RANDOM ITERATOR') i = 0 for x, y in self.randomIterator: if i < 2: self.log.debug(y) i += 1 assert i == 235 def tearDown(self): del self.mnist del self.sequentialIterator del self.randomIterator
def testDefaultSizes(self): mnist = MNIST(path="../../datasets/mnist.pkl.gz") self.assertEquals(mnist.train_inputs.shape, (50000, 1, 28, 28)) self.assertEquals(mnist.train_targets.shape, (50000, )) self.assertEquals(mnist.valid_inputs.shape, (10000, 1, 28, 28)) self.assertEquals(mnist.valid_targets.shape, (10000, )) self.assertEquals(mnist.test_inputs.shape, (10000, 1, 28, 28)) self.assertEquals(mnist.test_targets.shape, (10000, )) del mnist
def bernoulli_csl(switch=0): mnist = MNIST() train_x, _ = mnist.getSubset(TRAIN) valid_x, _ = mnist.getSubset(VALID) test_x, _ = mnist.getSubset(TEST) mnist_b = MNIST(binary=True) train_x_b, _ = mnist_b.getSubset(TRAIN) valid_x_b, _ = mnist_b.getSubset(VALID) test_x_b, _ = mnist_b.getSubset(TEST) means = as_floatX(test_x).eval() means = numpy.clip(a=means, a_min=1e-10, a_max=(1 - (1e-5))) # means = as_floatX(numpy.random.uniform(size=(10000,784))) * 0 + 0.5 minibatches = as_floatX(test_x_b.reshape((1000, 10, 784))).eval() if switch: # when means is a matrix of (N,D), representing only 1 chain csl_fn = _compile_csl_fn_v2(means) compute_CSL_with_minibatches_one_chain(csl_fn, minibatches) else: # when means is a 3D tensor (N, K, D) # When there are N chains, each chain having K samples of dimension D chains = means.reshape(10, 100, 10, 784) csl_fn = _compile_csl_fn() compute_CSL_with_minibatches(csl_fn, minibatches, chains) del mnist del mnist_b
def create_mlp(): # define the model layers relu_layer1 = BasicLayer(input_size=784, output_size=1000, activation='rectifier') relu_layer2 = BasicLayer(inputs_hook=(1000, relu_layer1.get_outputs()), output_size=1000, activation='rectifier') class_layer3 = SoftmaxLayer(inputs_hook=(1000, relu_layer2.get_outputs()), output_size=10, out_as_probs=False) # add the layers as a Prototype mlp = Prototype(layers=[relu_layer1, relu_layer2, class_layer3]) mnist = MNIST() optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20) optimizer.train() test_data, test_labels = mnist.getSubset(TEST) test_data = test_data[:25].eval() test_labels = test_labels[:25].eval() # use the run function! preds = mlp.run(test_data) log.info('-------') log.info("predicted: %s",str(preds)) log.info("actual: %s",str(test_labels.astype('int32')))
def testConcatSizes(self): mnist = MNIST(path="../../datasets/mnist.pkl.gz", concat_train_valid=True, flatten=False) self.assertEquals(mnist.train_inputs.shape, (60000, 1, 28, 28)) self.assertEquals(mnist.train_targets.shape, (60000, )) self.assertEquals(mnist.valid_inputs.shape, (10000, 1, 28, 28)) self.assertEquals(mnist.valid_targets.shape, (10000, )) self.assertEquals(mnist.test_inputs.shape, (10000, 1, 28, 28)) self.assertEquals(mnist.test_targets.shape, (10000, )) del mnist
def run_dae(): ######################################## # Initialization things with arguments # ######################################## config_root_logger() log.info("Creating a new DAE") mnist = MNIST() config = {"output_path": '../../../../outputs/dae/mnist/'} dae = DenoisingAutoencoder(config=config, dataset=mnist) # # Load initial weights and biases from file # params_to_load = 'dae_params.pkl' # dae.load_params(params_to_load) optimizer = AdaDelta(dae, mnist) optimizer.train() # Save some reconstruction output images import opendeep.data.dataset as datasets n_examples = 100 test_xs = mnist.getDataByIndices(indices=range(n_examples), subset=datasets.TEST) dae.create_reconstruction_image(test_xs)
def main(): ######################################## # Initialization things with arguments # ######################################## logger.config_root_logger() log.info("Creating a new DAE") mnist = MNIST() config = {"output_path": '../../../outputs/dae/mnist/'} dae = DenoisingAutoencoder(config=config, dataset=mnist) # # Load initial weights and biases from file # params_to_load = 'dae_params.pkl' # dae.load_params(params_to_load) optimizer = AdaDelta(dae, mnist) optimizer.train() # Save some reconstruction output images import opendeep.data.dataset as datasets n_examples = 100 test_xs = mnist.getDataByIndices(indices=range(n_examples), subset=datasets.TEST) dae.create_reconstruction_image(test_xs)
def main(plot=None, n_epoch=10): print('... loading and seting-up data') # don't concatenate together train and valid sets mnist_dataset = MNIST(concat_train_valid=False) print('... building the model structure') # create the mlp model from a Prototype model = build_model() optimizer, error = setup_optimization(model, n_epoch, mnist_dataset) print('... training the model') # [optional] use keyboardInterrupt to save the latest parameters. if plot: plot = Plot("OpenDeep MLP Example", monitor_channels=error, open_browser=True) optimizer.train(monitor_channels=error, plot=plot) print('... evaluating model') test_data, test_labels = split_data(mnist_dataset) evaluate(test_data, test_labels, model)
def main(sequence): rnn_gsn = RNN_GSN() # data! (needs to be 3d for rnn). mnist = MNIST(sequence_number=sequence, seq_3d=True, seq_length=50) # optimizer! optimizer = RMSProp(model=rnn_gsn, dataset=mnist, epochs=500, batch_size=50, save_freq=10, stop_patience=30, stop_threshold=.9995, learning_rate=1e-6, decay=.95, max_scaling=1e5, grad_clip=5., hard_clip=False) # train! optimizer.train()
if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere import logging from opendeep.log.logger import config_root_logger config_root_logger() log = logging.getLogger(__name__) log.info("Creating a Denoising Autoencoder!") # import the dataset and optimizer to use from opendeep.data.dataset import TEST from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta # grab the MNIST dataset mnist = MNIST() # create your shiny new DAE dae = DenoisingAutoencoder() # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=dae, dataset=mnist) # perform training! optimizer.train() # test it on some images! test_data = mnist.getDataByIndices(indices=range(25), subset=TEST) corrupted_test = salt_and_pepper(test_data, 0.4).eval() # use the predict function! reconstructed_images = dae.predict(corrupted_test)
def run_sequence(sequence=0): log.info("Creating RNN-RBM for sequence %d!" % sequence) # grab the MNIST dataset mnist = MNIST(sequence_number=sequence, concat_train_valid=True) outdir = "outputs/rnnrbm/mnist_%d/" % sequence # create the RNN-RBM rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2**30)) rnnrbm = RNN_RBM(input_size=28 * 28, hidden_size=1000, rnn_hidden_size=100, k=15, weights_init='uniform', weights_interval=4 * numpy.sqrt(6. / (28 * 28 + 500)), rnn_weights_init='identity', rnn_hidden_activation='relu', rnn_weights_std=1e-4, mrg=mrg, outdir=outdir) # load pretrained rbm on mnist # rnnrbm.load_params(outdir + 'trained_epoch_200.pkl') # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=rnnrbm, dataset=mnist, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=1e-8, save_frequency=10, early_stop_length=200) crossentropy = Monitor('crossentropy', rnnrbm.get_monitors()['crossentropy'], test=True) error = Monitor('error', rnnrbm.get_monitors()['mse'], test=True) plot = Plot(bokeh_doc_name='rnnrbm_mnist_%d' % sequence, monitor_channels=[crossentropy, error], open_browser=True) # perform training! optimizer.train(plot=plot) # use the generate function! log.debug("generating images...") generated, ut = rnnrbm.generate(initial=None, n_steps=400) # Construct image image = Image.fromarray( tile_raster_images(X=generated, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1))) image.save(outdir + "rnnrbm_mnist_generated.png") log.debug('saved generated.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images(X=rnnrbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors( rnnrbm.hidden_size), tile_spacing=(1, 1))) image.save(outdir + "rnnrbm_mnist_weights.png") log.debug("done!") del mnist del rnnrbm del optimizer
if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere import logging from opendeep.log.logger import config_root_logger config_root_logger() log = logging.getLogger(__name__) log.info("Creating a Denoising Autoencoder!") # import the dataset and optimizer to use from opendeep.data.dataset import TEST from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta # grab the MNIST dataset mnist = MNIST() # create your shiny new DAE dae = DenoisingAutoencoder() # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=dae, dataset=mnist, n_epoch=1) # perform training! optimizer.train() # test it on some images! test_data_m, _ = mnist.getSubset(TEST) test_data_u, _ = mnist.getSubset(TEST) test_data = test_data_u[:25].eval() test_data1 = test_data_m[0:5].eval() test_data2 = test_data_m[5:10].eval()
from opendeep.models.single_layer.basic import Dense, Softmax from opendeep.models.utils import Activation from opendeep.models.container import Prototype from opendeep.optimization.loss import Neg_LL from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere from opendeep.log import config_root_logger config_root_logger() # grab the MNIST dataset mnist = MNIST() # create the basic layer layer1 = Dense(inputs=((None, 28 * 28), matrix("x")), outputs=1000, activation='linear') layer1_act = Activation(inputs=((None, 1000), layer1.get_outputs()), activation='relu') # create the softmax classifier layer2 = Softmax(inputs=((None, 1000), layer1_act.get_outputs()), outputs=10, out_as_probs=True) # create the mlp from the two layers mlp = Prototype(layers=[layer1, layer1_act, layer2]) # define the loss function loss = Neg_LL(inputs=mlp.get_outputs(), targets=vector("y", dtype="int64"),
def add_list_layers(): # You can also add lists of layers at a time (or as initialization) to a Prototype! This lets you specify # more complex interactions between layers! hidden1 = BasicLayer(input_size=28 * 28, output_size=512, activation='rectifier', noise='dropout') hidden2 = BasicLayer(inputs_hook=(512, hidden1.get_outputs()), output_size=512, activation='rectifier', noise='dropout') class_layer = SoftmaxLayer(inputs_hook=(512, hidden2.get_outputs()), output_size=10) mlp = Prototype([hidden1, hidden2, class_layer]) return mlp if __name__ == '__main__': mlp = sequential_add_layers() optimizer = SGD(model=mlp, dataset=MNIST(concat_train_valid=True), n_epoch=500, batch_size=600, learning_rate=.01, momentum=.9, nesterov_momentum=True) optimizer.train()
def run_sequence(sequence=0): log.info("Creating RNN-GSN for sequence %d!" % sequence) # grab the MNIST dataset mnist = MNIST(sequence_number=sequence, concat_train_valid=True) outdir = "outputs/rnngsn/mnist_%d/" % sequence rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2**30)) rnngsn = RNN_GSN(layers=2, walkbacks=4, input_size=28 * 28, hidden_size=1000, tied_weights=True, rnn_hidden_size=100, weights_init='uniform', weights_interval='montreal', rnn_weights_init='identity', mrg=mrg, outdir=outdir) # load pretrained rbm on mnist # rnngsn.load_gsn_params('outputs/trained_gsn_epoch_1000.pkl') # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=rnngsn, dataset=mnist, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=1e-6, save_frequency=1, early_stop_length=200) # optimizer = SGD(model=rnngsn, # dataset=mnist, # n_epoch=300, # batch_size=100, # minimum_batch_size=2, # learning_rate=.25, # lr_decay='exponential', # lr_factor=.995, # momentum=0.5, # nesterov_momentum=True, # momentum_decay=False, # save_frequency=20, # early_stop_length=100) crossentropy = Monitor('crossentropy', rnngsn.get_monitors()['noisy_recon_cost'], test=True) error = Monitor('error', rnngsn.get_monitors()['mse'], test=True) # perform training! optimizer.train(monitor_channels=[crossentropy, error]) # use the generate function! log.debug("generating images...") generated, ut = rnngsn.generate(initial=None, n_steps=400) # Construct image image = Image.fromarray( tile_raster_images(X=generated, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1))) image.save(outdir + "rnngsn_mnist_generated.png") log.debug('saved generated.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images(X=rnngsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors( rnngsn.hidden_size), tile_spacing=(1, 1))) image.save(outdir + "rnngsn_mnist_weights.png") log.debug("done!") del mnist del rnngsn del optimizer
from opendeep.data.dataset import TEST from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere import logging import opendeep.log.logger as logger logger.config_root_logger() log = logging.getLogger(__name__) log.info("Creating softmax!") # grab the MNIST dataset mnist = MNIST() # create the softmax classifier s = SoftmaxLayer(input_size=28 * 28, output_size=10, out_as_probs=False) # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=s, dataset=mnist, n_epoch=20) # perform training! optimizer.train() # test it on some images! test_data, test_labels = mnist.getSubset(subset=TEST) test_data = test_data[:25].eval() test_labels = test_labels[:25].eval() # use the run function! preds = s.run(test_data) print('-------') print(preds) print(test_labels.astype('int32'))
from opendeep.data.dataset import TEST from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere import logging import opendeep.log.logger as logger logger.config_root_logger() log = logging.getLogger(__name__) log.info("Creating MLP!") # grab the MNIST dataset mnist = MNIST() # create the basic layer layer1 = BasicLayer(input_size=28*28, output_size=1000, activation='relu') # create the softmax classifier layer2 = SoftmaxLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=10, out_as_probs=False) # create the mlp from the two layers mlp = Prototype(layers=[layer1, layer2]) # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20) # perform training! optimizer.train() # test it on some images! test_data = mnist.getDataByIndices(indices=range(25), subset=TEST) # use the predict function! preds = mlp.predict(test_data) print '-------'
from opendeep.models.single_layer.basic import SoftmaxLayer # import the dataset and optimizer to use from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere import logging from opendeep.log import config_root_logger config_root_logger() log = logging.getLogger(__name__) log.info("Creating softmax!") # grab the MNIST dataset mnist = MNIST() # create the softmax classifier s = SoftmaxLayer(input_size=28 * 28, output_size=10, out_as_probs=False) # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=s, dataset=mnist, epochs=20) # perform training! optimizer.train() # test it on some images! test_data, test_labels = mnist.test_inputs[:25], mnist.test_targets[:25] # use the run function! preds = s.run(test_data) print('-------') print(preds) print(test_labels.astype('int32')) print() print()
if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere import logging import opendeep.log.logger as logger logger.config_root_logger() log = logging.getLogger(__name__) log.info("Creating a Denoising Autoencoder!") # import the dataset and optimizer to use from opendeep.data.dataset import TEST from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta # grab the MNIST dataset mnist = MNIST() # create your shiny new DAE dae = DenoisingAutoencoder() # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=dae, dataset=mnist) # perform training! optimizer.train() # test it on some images! test_data = mnist.getDataByIndices(indices=range(25), subset=TEST) corrupted_test = salt_and_pepper(test_data, 0.4) # use the predict function! reconstructed_images = dae.predict(test_data)
# import the dataset and optimizer to use from opendeep.data.dataset import TEST from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere import logging import opendeep.log.logger as logger logger.config_root_logger() log = logging.getLogger(__name__) log.info("Creating MLP!") # grab the MNIST dataset mnist = MNIST() # create the basic layer layer1 = BasicLayer(input_size=28 * 28, output_size=1000, activation='relu') # create the softmax classifier layer2 = SoftmaxLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=10, out_as_probs=False) # create the mlp from the two layers mlp = Prototype(layers=[layer1, layer2]) # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20) # perform training! optimizer.train() # test it on some images!
from opendeep.data.dataset import TEST from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere import logging import opendeep.log.logger as logger logger.config_root_logger() log = logging.getLogger(__name__) log.info("Creating softmax!") # grab the MNIST dataset mnist = MNIST() # create the softmax classifier s = SoftmaxLayer(input_size=28 * 28, output_size=10, out_as_probs=False) # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=s, dataset=mnist, n_epoch=20) # perform training! optimizer.train() # test it on some images! test_data = mnist.getDataByIndices(indices=range(25), subset=TEST) # use the predict function! preds = s.predict(test_data) print '-------' print preds print mnist.getLabelsByIndices(indices=range(25), subset=TEST) print print
from opendeep.utils.image import tile_raster_images from opendeep.utils.misc import closest_to_square_factors import PIL.Image as Image if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere import logging import opendeep.log.logger as logger logger.config_root_logger() log = logging.getLogger(__name__) log.info("Creating RBM!") # grab the MNIST dataset mnist = MNIST(concat_train_valid=False) # create the RBM rng = numpy.random.RandomState(1234) mrg = theano.tensor.shared_randomstreams.RandomStreams(rng.randint(2**30)) rbm = RBM(input_size=28*28, hidden_size=500, k=15, weights_init='uniform', weights_interval=4*numpy.sqrt(6./(28*28+500)), mrg=mrg) # rbm.load_params('rbm_trained.pkl') # make an optimizer to train it (AdaDelta is a good default) # optimizer = SGD(model=rbm, dataset=mnist, batch_size=20, learning_rate=0.1, lr_decay=False, nesterov_momentum=False, momentum=False) optimizer = Optimizer(lr_decay=False, learning_rate=0.1, model=rbm, dataset=mnist, batch_size=20, save_frequency=1) ll = Monitor('pseudo-log', rbm.get_monitors()['pseudo-log']) # perform training! optimizer.train(monitor_channels=ll)
if __name__ == '__main__': # set up the logging environment to display outputs (optional) # although this is recommended over print statements everywhere import logging from opendeep.log.logger import config_root_logger config_root_logger() log = logging.getLogger(__name__) log.info("Creating a Denoising Autoencoder!") # import the dataset and optimizer to use from opendeep.data.dataset import TEST from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta # grab the MNIST dataset mnist = MNIST() # create your shiny new DAE dae = DenoisingAutoencoder() # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=dae, dataset=mnist) # perform training! optimizer.train() # test it on some images! test_data, _ = mnist.getSubset(TEST) test_data = test_data[:25].eval() corrupted_test = salt_and_pepper(test_data, 0.4).eval() # use the run function! reconstructed_images = dae.run(corrupted_test)
def main(): ######################################## # Initialization things with arguments # ######################################## # use these arguments to get results from paper referenced above _train_args = {"n_epoch": 1000, # maximum number of times to run through the dataset "batch_size": 100, # number of examples to process in parallel (minibatch) "minimum_batch_size": 1, # the minimum number of examples for a batch to be considered "save_frequency": 1, # how many epochs between saving parameters "early_stop_threshold": .9995, # multiplier for how much the train cost to improve to not stop early "early_stop_length": 500, # how many epochs to wait to see if the threshold has been reached "learning_rate": .25, # initial learning rate for SGD "lr_decay": 'exponential', # the decay function to use for the learning rate parameter "lr_factor": .995, # by how much to decay the learning rate each epoch "momentum": 0.5, # the parameter momentum amount 'momentum_decay': False, # how to decay the momentum each epoch (if applicable) 'momentum_factor': 0, # by how much to decay the momentum (in this case not at all) 'nesterov_momentum': False, # whether to use nesterov momentum update (accelerated momentum) } config_root_logger() log.info("Creating a new GSN") mnist = MNIST(concat_train_valid=True) gsn = GSN(layers=2, walkbacks=4, hidden_size=1500, visible_activation='sigmoid', hidden_activation='tanh', input_size=28*28, tied_weights=True, hidden_add_noise_sigma=2, input_salt_and_pepper=0.4, outdir='outputs/test_gsn/', vis_init=False, noiseless_h1=True, input_sampling=True, weights_init='uniform', weights_interval='montreal', bias_init=0, cost_function='binary_crossentropy') recon_cost_channel = MonitorsChannel(name='cost') recon_cost_channel.add(Monitor('recon_cost', gsn.get_monitors()['recon_cost'], test=True)) recon_cost_channel.add(Monitor('noisy_recon_cost', gsn.get_monitors()['noisy_recon_cost'], test=True)) # Load initial weights and biases from file # params_to_load = '../../../outputs/gsn/mnist/trained_epoch_395.pkl' # gsn.load_params(params_to_load) optimizer = SGD(model=gsn, dataset=mnist, **_train_args) # optimizer = AdaDelta(model=gsn, dataset=mnist, n_epoch=200, batch_size=100, learning_rate=1e-6) optimizer.train(monitor_channels=recon_cost_channel) # Save some reconstruction output images import opendeep.data.dataset as datasets n_examples = 100 xs_test, _ = mnist.getSubset(datasets.TEST) xs_test = xs_test[:n_examples].eval() noisy_xs_test = gsn.f_noise(xs_test) reconstructed = gsn.run(noisy_xs_test) # Concatenate stuff stacked = numpy.vstack( [numpy.vstack([xs_test[i * 10: (i + 1) * 10], noisy_xs_test[i * 10: (i + 1) * 10], reconstructed[i * 10: (i + 1) * 10]]) for i in range(10)]) number_reconstruction = PIL.Image.fromarray( tile_raster_images(stacked, (gsn.image_height, gsn.image_width), (10, 30)) ) number_reconstruction.save(gsn.outdir + 'reconstruction.png') log.info("saved output image!") # Construct image from the weight matrix image = PIL.Image.fromarray( tile_raster_images( X=gsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(gsn.hidden_size), tile_spacing=(1, 1) ) ) image.save(gsn.outdir + "gsn_mnist_weights.png")