def bernoulli_csl(switch=0): mnist = MNIST() train_x, _ = mnist.getSubset(TRAIN) valid_x, _ = mnist.getSubset(VALID) test_x, _ = mnist.getSubset(TEST) mnist_b = MNIST(binary=True) train_x_b, _ = mnist_b.getSubset(TRAIN) valid_x_b, _ = mnist_b.getSubset(VALID) test_x_b, _ = mnist_b.getSubset(TEST) means = as_floatX(test_x).eval() means = numpy.clip(a=means, a_min=1e-10, a_max=(1 - (1e-5))) # means = as_floatX(numpy.random.uniform(size=(10000,784))) * 0 + 0.5 minibatches = as_floatX(test_x_b.reshape((1000, 10, 784))).eval() if switch: # when means is a matrix of (N,D), representing only 1 chain csl_fn = _compile_csl_fn_v2(means) compute_CSL_with_minibatches_one_chain(csl_fn, minibatches) else: # when means is a 3D tensor (N, K, D) # When there are N chains, each chain having K samples of dimension D chains = means.reshape(10, 100, 10, 784) csl_fn = _compile_csl_fn() compute_CSL_with_minibatches(csl_fn, minibatches, chains) del mnist del mnist_b
def bernoulli_csl(switch=0): mnist = MNIST() train_x, _ = mnist.getSubset(TRAIN) valid_x, _ = mnist.getSubset(VALID) test_x, _ = mnist.getSubset(TEST) mnist_b = MNIST(binary=True) train_x_b, _ = mnist_b.getSubset(TRAIN) valid_x_b, _ = mnist_b.getSubset(VALID) test_x_b, _ = mnist_b.getSubset(TEST) means = as_floatX(test_x).eval() means = numpy.clip(a=means, a_min=1e-10, a_max=(1 - (1e-5))) #means = as_floatX(numpy.random.uniform(size=(10000,784))) * 0 + 0.5 minibatches = as_floatX(test_x_b.reshape((1000, 10, 784))).eval() if switch: # when means is a matrix of (N,D), representing only 1 chain csl_fn = _compile_csl_fn_v2(means) compute_CSL_with_minibatches_one_chain(csl_fn, minibatches) else: # when means is a 3D tensor (N, K, D) # When there are N chains, each chain having K samples of dimension D chains = means.reshape(10, 100, 10, 784) csl_fn = _compile_csl_fn() compute_CSL_with_minibatches(csl_fn, minibatches, chains) del mnist del mnist_b
def run_dae(): ######################################## # Initialization things with arguments # ######################################## config_root_logger() log.info("Creating a new DAE") mnist = MNIST() config = { "outdir": 'outputs/dae/mnist/', "input_size": 28 * 28, "tied_weights": True } dae = DenoisingAutoencoder(**config) # # Load initial weights and biases from file # params_to_load = 'dae_params.pkl' # dae.load_params(params_to_load) optimizer = AdaDelta(dae, mnist) optimizer.train() # Save some reconstruction output images import opendeep.data.dataset as datasets n_examples = 100 test_xs, _ = mnist.getSubset(subset=datasets.TEST) test_xs = test_xs[:n_examples].eval() dae.create_reconstruction_image(test_xs)
def run_dae(): ######################################## # Initialization things with arguments # ######################################## config_root_logger() log.info("Creating a new DAE") mnist = MNIST() config = { "outdir": 'outputs/dae/mnist/', "input_size": 28*28, "tied_weights": True } dae = DenoisingAutoencoder(**config) # # Load initial weights and biases from file # params_to_load = 'dae_params.pkl' # dae.load_params(params_to_load) optimizer = AdaDelta(dae, mnist) optimizer.train() # Save some reconstruction output images import opendeep.data.dataset as datasets n_examples = 100 test_xs, _ = mnist.getSubset(subset=datasets.TEST) test_xs = test_xs[:n_examples].eval() dae.create_reconstruction_image(test_xs)
def run_mlp(): # # define the model layers # layer1 = BasicLayer(input_size=784, output_size=1000, activation='rectifier') # layer2 = BasicLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=1000, activation='rectifier') # classlayer3 = SoftmaxLayer(inputs_hook=(1000, layer2.get_outputs()), output_size=10, out_as_probs=False) # # add the layers to the prototype # mlp = Prototype(layers=[layer1, layer2, classlayer3]) # test the new way to automatically fill in inputs_hook for models mlp = Prototype() mlp.add(BasicLayer(input_size=784, output_size=1000, activation="rectifier", noise="dropout")) mlp.add(BasicLayer(output_size=1500, activation="tanh", noise="dropout")) mlp.add(SoftmaxLayer(output_size=10)) mnist = MNIST() optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=10) optimizer.train() test_data, test_labels = mnist.getSubset(subset=TEST) test_data = test_data[:25].eval() test_labels = test_labels[:25].eval() # use the run function! yhat = mlp.run(test_data) print("-------") print("Prediction: %s" % str(yhat)) print("Actual: %s" % str(test_labels.astype("int32")))
def run_mlp(): # # define the model layers # layer1 = BasicLayer(input_size=784, output_size=1000, activation='rectifier') # layer2 = BasicLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=1000, activation='rectifier') # classlayer3 = SoftmaxLayer(inputs_hook=(1000, layer2.get_outputs()), output_size=10, out_as_probs=False) # # add the layers to the prototype # mlp = Prototype(layers=[layer1, layer2, classlayer3]) # test the new way to automatically fill in inputs_hook for models mlp = Prototype() mlp.add( BasicLayer(input_size=784, output_size=1000, activation='rectifier', noise='dropout')) mlp.add(BasicLayer(output_size=1500, activation='tanh', noise='dropout')) mlp.add(SoftmaxLayer(output_size=10)) mnist = MNIST() optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=10) optimizer.train() test_data, test_labels = mnist.getSubset(subset=TEST) test_data = test_data[:25].eval() test_labels = test_labels[:25].eval() # use the run function! yhat = mlp.run(test_data) print('-------') print('Prediction: %s' % str(yhat)) print('Actual: %s' % str(test_labels.astype('int32')))
def create_mlp(): # define the model layers relu_layer1 = BasicLayer(input_size=784, output_size=1000, activation='rectifier') relu_layer2 = BasicLayer(inputs_hook=(1000, relu_layer1.get_outputs()), output_size=1000, activation='rectifier') class_layer3 = SoftmaxLayer(inputs_hook=(1000, relu_layer2.get_outputs()), output_size=10, out_as_probs=False) # add the layers as a Prototype mlp = Prototype(layers=[relu_layer1, relu_layer2, class_layer3]) mnist = MNIST() optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20) optimizer.train() test_data, test_labels = mnist.getSubset(TEST) test_data = test_data[:25].eval() test_labels = test_labels[:25].eval() # use the run function! preds = mlp.run(test_data) log.info('-------') log.info("predicted: %s",str(preds)) log.info("actual: %s",str(test_labels.astype('int32')))
from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta # grab the MNIST dataset mnist = MNIST() # create your shiny new DAE dae = DenoisingAutoencoder() # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=dae, dataset=mnist) # perform training! optimizer.train() # test it on some images! test_data, _ = mnist.getSubset(TEST) test_data = test_data[:25].eval() corrupted_test = salt_and_pepper(test_data, 0.4).eval() # use the run function! reconstructed_images = dae.run(corrupted_test) # create an image from this reconstruction! # imports for working with tiling outputs into one image from opendeep.utils.image import tile_raster_images import numpy import PIL # stack the image matrices together in three 5x5 grids next to each other using numpy stacked = numpy.vstack([ numpy.vstack([ test_data[i * 5:(i + 1) * 5], corrupted_test[i * 5:(i + 1) * 5], reconstructed_images[i * 5:(i + 1) * 5]
import logging import opendeep.log.logger as logger logger.config_root_logger() log = logging.getLogger(__name__) log.info("Creating softmax!") # grab the MNIST dataset mnist = MNIST() # create the softmax classifier s = SoftmaxLayer(input_size=28 * 28, output_size=10, out_as_probs=False) # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=s, dataset=mnist, n_epoch=20) # perform training! optimizer.train() # test it on some images! test_data, test_labels = mnist.getSubset(subset=TEST) test_data = test_data[:25].eval() test_labels = test_labels[:25].eval() # use the run function! preds = s.run(test_data) print('-------') print(preds) print(test_labels.astype('int32')) print() print() del mnist del s del optimizer log.info("Creating softmax with categorical cross-entropy!")
rng = numpy.random.RandomState(1234) mrg = theano.tensor.shared_randomstreams.RandomStreams(rng.randint(2**30)) rbm = RBM(input_size=28*28, hidden_size=500, k=15, weights_init='uniform', weights_interval=4*numpy.sqrt(6./(28*28+500)), mrg=mrg) # rbm.load_params('rbm_trained.pkl') # make an optimizer to train it (AdaDelta is a good default) # optimizer = SGD(model=rbm, dataset=mnist, batch_size=20, learning_rate=0.1, lr_decay=False, nesterov_momentum=False, momentum=False) optimizer = Optimizer(lr_decay=False, learning_rate=0.1, model=rbm, dataset=mnist, batch_size=20, save_frequency=1) ll = Monitor('pseudo-log', rbm.get_monitors()['pseudo-log']) # perform training! optimizer.train(monitor_channels=ll) # test it on some images! test_data = mnist.getSubset(TEST)[0] test_data = test_data[:25].eval() # use the run function! preds = rbm.run(test_data) # Construct image from the test matrix image = Image.fromarray( tile_raster_images( X=test_data, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1) ) ) image.save('rbm_test.png')
# grab the MNIST dataset mnist = MNIST() # create the basic layer layer1 = BasicLayer(input_size=28 * 28, output_size=1000, activation='relu') # create the softmax classifier layer2 = SoftmaxLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=10, out_as_probs=False) # create the mlp from the two layers mlp = Prototype(layers=[layer1, layer2]) # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20) # perform training! optimizer.train() # test it on some images! test_data, test_labels = mnist.getSubset(subset=TEST) test_data = test_data[:25].eval() test_labels = test_labels[:25].eval() # use the run function! preds = mlp.run(test_data) print('-------') print(preds) print(test_labels.astype('int32')) print() print() del mnist del mlp del optimizer
from opendeep.data.standard_datasets.image.mnist import MNIST from opendeep.optimization.adadelta import AdaDelta # grab the MNIST dataset mnist = MNIST() # create your shiny new DAE dae = DenoisingAutoencoder() # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=dae, dataset=mnist, n_epoch=1) # perform training! optimizer.train() # test it on some images! test_data_m, _ = mnist.getSubset(TEST) test_data_u, _ = mnist.getSubset(TEST) test_data = test_data_u[:25].eval() test_data1 = test_data_m[0:5].eval() test_data2 = test_data_m[5:10].eval() test_data3 = test_data_m[10:15].eval() test_data4 = test_data_m[15:20].eval() test_data5 = test_data_m[20:25].eval() #git test corrupted_test1 = salt_and_pepper_custom(test_data1).eval() corrupted_test2 = salt_and_pepper_custom(test_data2).eval() corrupted_test3 = salt_and_pepper_custom(test_data3).eval() corrupted_test4 = salt_and_pepper_custom(test_data4).eval() corrupted_test5 = salt_and_pepper_custom(test_data5).eval() corrupted_test = np.row_stack((corrupted_test1, corrupted_test2,
def main(): ######################################## # Initialization things with arguments # ######################################## # use these arguments to get results from paper referenced above _train_args = {"n_epoch": 1000, # maximum number of times to run through the dataset "batch_size": 100, # number of examples to process in parallel (minibatch) "minimum_batch_size": 1, # the minimum number of examples for a batch to be considered "save_frequency": 1, # how many epochs between saving parameters "early_stop_threshold": .9995, # multiplier for how much the train cost to improve to not stop early "early_stop_length": 500, # how many epochs to wait to see if the threshold has been reached "learning_rate": .25, # initial learning rate for SGD "lr_decay": 'exponential', # the decay function to use for the learning rate parameter "lr_factor": .995, # by how much to decay the learning rate each epoch "momentum": 0.5, # the parameter momentum amount 'momentum_decay': False, # how to decay the momentum each epoch (if applicable) 'momentum_factor': 0, # by how much to decay the momentum (in this case not at all) 'nesterov_momentum': False, # whether to use nesterov momentum update (accelerated momentum) } config_root_logger() log.info("Creating a new GSN") mnist = MNIST(concat_train_valid=True) gsn = GSN(layers=2, walkbacks=4, hidden_size=1500, visible_activation='sigmoid', hidden_activation='tanh', input_size=28*28, tied_weights=True, hidden_add_noise_sigma=2, input_salt_and_pepper=0.4, outdir='outputs/test_gsn/', vis_init=False, noiseless_h1=True, input_sampling=True, weights_init='uniform', weights_interval='montreal', bias_init=0, cost_function='binary_crossentropy') recon_cost_channel = MonitorsChannel(name='cost') recon_cost_channel.add(Monitor('recon_cost', gsn.get_monitors()['recon_cost'], test=True)) recon_cost_channel.add(Monitor('noisy_recon_cost', gsn.get_monitors()['noisy_recon_cost'], test=True)) # Load initial weights and biases from file # params_to_load = '../../../outputs/gsn/mnist/trained_epoch_395.pkl' # gsn.load_params(params_to_load) optimizer = SGD(model=gsn, dataset=mnist, **_train_args) # optimizer = AdaDelta(model=gsn, dataset=mnist, n_epoch=200, batch_size=100, learning_rate=1e-6) optimizer.train(monitor_channels=recon_cost_channel) # Save some reconstruction output images import opendeep.data.dataset as datasets n_examples = 100 xs_test, _ = mnist.getSubset(datasets.TEST) xs_test = xs_test[:n_examples].eval() noisy_xs_test = gsn.f_noise(xs_test) reconstructed = gsn.run(noisy_xs_test) # Concatenate stuff stacked = numpy.vstack( [numpy.vstack([xs_test[i * 10: (i + 1) * 10], noisy_xs_test[i * 10: (i + 1) * 10], reconstructed[i * 10: (i + 1) * 10]]) for i in range(10)]) number_reconstruction = PIL.Image.fromarray( tile_raster_images(stacked, (gsn.image_height, gsn.image_width), (10, 30)) ) number_reconstruction.save(gsn.outdir + 'reconstruction.png') log.info("saved output image!") # Construct image from the weight matrix image = PIL.Image.fromarray( tile_raster_images( X=gsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(gsn.hidden_size), tile_spacing=(1, 1) ) ) image.save(gsn.outdir + "gsn_mnist_weights.png")