def create_reconstruction_image(self, input_data): """ Adds noise to an input and saves an image from the reconstruction running the input through the computation graph. """ n_examples = len(input_data) xs_test = input_data noisy_xs_test = self.f_noise(input_data) reconstructed = self.run(noisy_xs_test) # Concatenate stuff width, height = closest_to_square_factors(n_examples) stacked = numpy.vstack([ numpy.vstack([ xs_test[i * width:(i + 1) * width], noisy_xs_test[i * width:(i + 1) * width], reconstructed[i * width:(i + 1) * width] ]) for i in range(height) ]) number_reconstruction = PIL.Image.fromarray( tile_raster_images(stacked, (self.image_height, self.image_width), (height, 3 * width))) save_path = os.path.join(self.outdir, 'gsn_reconstruction.png') save_path = os.path.realpath(save_path) number_reconstruction.save(save_path) log.info("saved output image to %s", save_path)
def create_reconstruction_image(self, input_data): """ Adds noise to an input and saves an image from the reconstruction running the input through the computation graph. """ n_examples = len(input_data) xs_test = input_data noisy_xs_test = self.f_noise(input_data) reconstructed = self.run(noisy_xs_test) # Concatenate stuff width, height = closest_to_square_factors(n_examples) stacked = numpy.vstack( [numpy.vstack([xs_test[i * width: (i + 1) * width], noisy_xs_test[i * width: (i + 1) * width], reconstructed[i * width: (i + 1) * width]]) for i in range(height)]) number_reconstruction = PIL.Image.fromarray( tile_raster_images(stacked, (self.image_height, self.image_width), (height, 3*width)) ) save_path = os.path.join(self.outdir, 'gsn_reconstruction.png') save_path = os.path.realpath(save_path) number_reconstruction.save(save_path) log.info("saved output image to %s", save_path)
def run_sequence(sequence=0): log.info("Creating RNN-GSN for sequence %d!" % sequence) # grab the MNIST dataset mnist = MNIST(sequence_number=sequence, concat_train_valid=True) outdir = "outputs/rnngsn/mnist_%d/" % sequence rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2**30)) rnngsn = RNN_GSN(layers=2, walkbacks=4, input_size=28 * 28, hidden_size=1000, tied_weights=True, rnn_hidden_size=100, weights_init='uniform', weights_interval='montreal', rnn_weights_init='identity', mrg=mrg, outdir=outdir) # load pretrained rbm on mnist # rnngsn.load_gsn_params('outputs/trained_gsn_epoch_1000.pkl') # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=rnngsn, dataset=mnist, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=1e-6, save_frequency=1, early_stop_length=200) # optimizer = SGD(model=rnngsn, # dataset=mnist, # n_epoch=300, # batch_size=100, # minimum_batch_size=2, # learning_rate=.25, # lr_decay='exponential', # lr_factor=.995, # momentum=0.5, # nesterov_momentum=True, # momentum_decay=False, # save_frequency=20, # early_stop_length=100) crossentropy = Monitor('crossentropy', rnngsn.get_monitors()['noisy_recon_cost'], test=True) error = Monitor('error', rnngsn.get_monitors()['mse'], test=True) # perform training! optimizer.train(monitor_channels=[crossentropy, error]) # use the generate function! log.debug("generating images...") generated, ut = rnngsn.generate(initial=None, n_steps=400) # Construct image image = Image.fromarray( tile_raster_images(X=generated, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1))) image.save(outdir + "rnngsn_mnist_generated.png") log.debug('saved generated.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images(X=rnngsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors( rnngsn.hidden_size), tile_spacing=(1, 1))) image.save(outdir + "rnngsn_mnist_weights.png") log.debug("done!") del mnist del rnngsn del optimizer
def run_sequence(sequence=0): log.info("Creating RNN-GSN for sequence %d!" % sequence) # grab the MNIST dataset mnist = MNIST(sequence_number=sequence, concat_train_valid=True) outdir = "outputs/rnngsn/mnist_%d/" % sequence rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2 ** 30)) rnngsn = RNN_GSN(layers=2, walkbacks=4, input_size=28 * 28, hidden_size=1000, tied_weights=True, rnn_hidden_size=100, weights_init='uniform', weights_interval='montreal', rnn_weights_init='identity', mrg=mrg, outdir=outdir) # load pretrained rbm on mnist # rnngsn.load_gsn_params('outputs/trained_gsn_epoch_1000.pkl') # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=rnngsn, dataset=mnist, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=1e-6, save_frequency=1, early_stop_length=200) # optimizer = SGD(model=rnngsn, # dataset=mnist, # n_epoch=300, # batch_size=100, # minimum_batch_size=2, # learning_rate=.25, # lr_decay='exponential', # lr_factor=.995, # momentum=0.5, # nesterov_momentum=True, # momentum_decay=False, # save_frequency=20, # early_stop_length=100) crossentropy = Monitor('crossentropy', rnngsn.get_monitors()['noisy_recon_cost'], test=True) error = Monitor('error', rnngsn.get_monitors()['mse'], test=True) # perform training! optimizer.train(monitor_channels=[crossentropy, error]) # use the generate function! log.debug("generating images...") generated, ut = rnngsn.generate(initial=None, n_steps=400) # Construct image image = Image.fromarray( tile_raster_images( X=generated, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1) ) ) image.save(outdir + "rnngsn_mnist_generated.png") log.debug('saved generated.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rnngsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(rnngsn.hidden_size), tile_spacing=(1, 1) ) ) image.save(outdir + "rnngsn_mnist_weights.png") log.debug("done!") del mnist del rnngsn del optimizer
def __init__(self, inputs_hook=None, hiddens_hook=None, params_hook=None, outdir='outputs/gsn/', input_size=None, hidden_size=1000, layers=2, walkbacks=4, visible_activation='sigmoid', hidden_activation='tanh', input_sampling=True, mrg=RNG_MRG.MRG_RandomStreams(1), tied_weights=True, weights_init='uniform', weights_interval='montreal', weights_mean=0, weights_std=5e-3, bias_init=0.0, cost_function='binary_crossentropy', cost_args=None, add_noise=True, noiseless_h1=True, hidden_noise='gaussian', hidden_noise_level=2, input_noise='salt_and_pepper', input_noise_level=0.4, noise_decay='exponential', noise_annealing=1, image_width=None, image_height=None, **kwargs): """ Initialize a GSN. Parameters ---------- inputs_hook : Tuple of (shape, variable) Routing information for the model to accept inputs from elsewhere. This is used for linking different models together (e.g. setting the Softmax model's input layer to the DAE's hidden layer gives a newly supervised classification model). For now, it needs to include the shape information (normally the dimensionality of the input i.e. n_in). hiddens_hook : Tuple of (shape, variable) Routing information for the model to accept its hidden representation from elsewhere. This is used for linking different models together (e.g. setting the DAE model's hidden layers to the RNN's output layer gives a generative recurrent model.) For now, it needs to include the shape information (normally the dimensionality of the hiddens i.e. n_hidden). params_hook : List(theano shared variable) A list of model parameters (shared theano variables) that you should use when constructing this model (instead of initializing your own shared variables). This parameter is useful when you want to have two versions of the model that use the same parameters - such as a training model with dropout applied to layers and one without for testing, where the parameters are shared between the two. outdir : str The directory you want outputs (parameters, images, etc.) to save to. If None, nothing will be saved. input_size : int The size (dimensionality) of the input to the DAE. If shape is provided in `inputs_hook`, this is optional. The :class:`Model` requires an `output_size`, which gets set to this value because the DAE is an unsupervised model. The output is a reconstruction of the input. hidden_size : int The size (dimensionality) of the hidden layer for the DAE. Generally, you want it to be larger than `input_size`, which is known as *overcomplete*. visible_activation : str or callable The nonlinear (or linear) visible activation to perform after the dot product from hiddens -> visible layer. This activation function should be appropriate for the input unit types, i.e. 'sigmoid' for binary inputs. See opendeep.utils.activation for a list of available activation functions. Alternatively, you can pass your own function to be used as long as it is callable. hidden_activation : str or callable The nonlinear (or linear) hidden activation to perform after the dot product from visible -> hiddens layer. See opendeep.utils.activation for a list of available activation functions. Alternatively, you can pass your own function to be used as long as it is callable. layers : int The number of hidden layers to use. walkbacks : int The number of walkbacks to perform (the variable K in Bengio's paper above). A walkback is a Gibbs sample from the DAE, which means the model generates inputs in sequence, where each generated input is compared to the original input to create the reconstruction cost for training. For running the model, the very last generated input in the Gibbs chain is used as the output. input_sampling : bool During walkbacks, whether to sample from the generated input to create a new starting point for the next walkback (next step in the Gibbs chain). This generally makes walkbacks more effective by making the process more stochastic - more likely to find spurious modes in the model's representation. mrg : random A random number generator that is used when adding noise into the network and for sampling from the input. I recommend using Theano's sandbox.rng_mrg.MRG_RandomStreams. tied_weights : bool DAE has two weight matrices - W from input -> hiddens and V from hiddens -> input. This boolean determines if V = W.T, which 'ties' V to W and reduces the number of parameters necessary during training. weights_init : str Determines the method for initializing model weights. See opendeep.utils.nnet for options. weights_interval : str or float If Uniform `weights_init`, the +- interval to use. See opendeep.utils.nnet for options. weights_mean : float If Gaussian `weights_init`, the mean value to use. weights_std : float If Gaussian `weights_init`, the standard deviation to use. bias_init : float The initial value to use for the bias parameter. Most often, the default of 0.0 is preferred. cost_function : str or callable The function to use when calculating the reconstruction cost of the model. This should be appropriate for the type of input, i.e. use 'binary_crossentropy' for binary inputs, or 'mse' for real-valued inputs. See opendeep.utils.cost for options. You can also specify your own function, which needs to be callable. cost_args : dict Any additional named keyword arguments to pass to the specified `cost_function`. add_noise : bool Whether to add noise (corrupt) the input before passing it through the computation graph during training. This should most likely be set to the default of True, because this is a *denoising* autoencoder after all. noiseless_h1 : bool Whether to not add noise (corrupt) the hidden layer during computation. hidden_noise : str What type of noise to use for corrupting the hidden layer (if not `noiseless_h1`). See opendeep.utils.noise for options. This should be appropriate for the hidden unit activation, i.e. Gaussian for tanh or other real-valued activations, etc. hidden_noise_level : float The amount of noise to use for the noise function specified by `hidden_noise`. This could be the standard deviation for gaussian noise, the interval for uniform noise, the dropout amount, etc. input_noise : str What type of noise to use for corrupting the input before computation (if `add_noise`). See opendeep.utils.noise for options. This should be appropriate for the input units, i.e. salt-and-pepper for binary units, etc. input_noise_level : float The amount of noise used to corrupt the input. This could be the masking probability for salt-and-pepper, standard deviation for Gaussian, interval for Uniform, etc. noise_decay : str or False Whether to use `input_noise` scheduling (decay `input_noise_level` during the course of training), and if so, the string input specifies what type of decay to use. See opendeep.utils.decay for options. Noise decay (known as noise scheduling) effectively helps the DAE learn larger variance features first, and then smaller ones later (almost as a kind of curriculum learning). May help it converge faster. noise_annealing : float The amount to reduce the `input_noise_level` after each training epoch based on the decay function specified in `noise_decay`. image_width : int If the input should be represented as an image, the width of the input image. If not specified, it will be close to the square factor of the `input_size`. image_height : int If the input should be represented as an image, the height of the input image. If not specified, it will be close to the square factor of the `input_size`. """ # init Model to combine the defaults and config dictionaries with the initial parameters. initial_parameters = locals().copy() initial_parameters.pop('self') super(GSN, self).__init__(**initial_parameters) # when the input should be thought of as an image, either use the specified width and height, # or try to make as square as possible. if image_height is None and image_width is None: (_h, _w) = closest_to_square_factors(self.input_size) self.image_width = _w self.image_height = _h else: self.image_height = image_height self.image_width = image_width ############################ # Theano variables and RNG # ############################ if self.inputs_hook is None: self.X = T.matrix('X') else: # inputs_hook is a (shape, input) tuple self.X = self.inputs_hook[1] ########################## # Network specifications # ########################## # generally, walkbacks should be at least 2*layers if layers % 2 == 0: if walkbacks < 2*layers: log.warning('Not enough walkbacks for the layers! Layers is %s and walkbacks is %s. ' 'Generaly want 2X walkbacks to layers', str(layers), str(walkbacks)) else: if walkbacks < 2*layers-1: log.warning('Not enough walkbacks for the layers! Layers is %s and walkbacks is %s. ' 'Generaly want 2X walkbacks to layers', str(layers), str(walkbacks)) self.add_noise = add_noise self.noise_annealing = as_floatX(noise_annealing) # noise schedule parameter self.hidden_noise_level = sharedX(hidden_noise_level, dtype=theano.config.floatX) self.hidden_noise = get_noise(name=hidden_noise, noise_level=self.hidden_noise_level, mrg=mrg) self.input_noise_level = sharedX(input_noise_level, dtype=theano.config.floatX) self.input_noise = get_noise(name=input_noise, noise_level=self.input_noise_level, mrg=mrg) self.walkbacks = walkbacks self.tied_weights = tied_weights self.layers = layers self.noiseless_h1 = noiseless_h1 self.input_sampling = input_sampling self.noise_decay = noise_decay # if there was a hiddens_hook, unpack the hidden layers in the tensor if self.hiddens_hook is not None: hidden_size = self.hiddens_hook[0] self.hiddens_flag = True else: self.hiddens_flag = False # determine the sizes of each layer in a list. # layer sizes, from h0 to hK (h0 is the visible layer) hidden_size = list(raise_to_list(hidden_size)) if len(hidden_size) == 1: self.layer_sizes = [self.input_size] + hidden_size * self.layers else: assert len(hidden_size) == self.layers, "Hiddens sizes and number of hidden layers mismatch." + \ "Hiddens %d and layers %d" % (len(hidden_size), self.layers) self.layer_sizes = [self.input_size] + hidden_size if self.hiddens_hook is not None: self.hiddens = self.unpack_hiddens(self.hiddens_hook[1]) ######################### # Activation functions! # ######################### # hidden unit activation self.hidden_activation = get_activation_function(hidden_activation) # Visible layer activation self.visible_activation = get_activation_function(visible_activation) # make sure the sampling functions are appropriate for the activation functions. if is_binary(self.visible_activation): self.visible_sampling = mrg.binomial else: # TODO: implement non-binary activation log.error("Non-binary visible activation not supported yet!") raise NotImplementedError("Non-binary visible activation not supported yet!") # Cost function self.cost_function = get_cost_function(cost_function) self.cost_args = cost_args or dict() ############### # Parameters! # ############### # make sure to deal with params_hook! if self.params_hook is not None: # if tied weights, expect layers*2 + 1 params if self.tied_weights: assert len(self.params_hook) == 2*layers + 1, \ "Tied weights: expected {0!s} params, found {1!s}!".format(2*layers+1, len(self.params_hook)) self.weights_list = self.params_hook[:layers] self.bias_list = self.params_hook[layers:] # if untied weights, expect layers*3 + 1 params else: assert len(self.params_hook) == 3*layers + 1, \ "Untied weights: expected {0!s} params, found {1!s}!".format(3*layers+1, len(self.params_hook)) self.weights_list = self.params_hook[:2*layers] self.bias_list = self.params_hook[2*layers:] # otherwise, construct our params else: # initialize a list of weights and biases based on layer_sizes for the GSN self.weights_list = [get_weights(weights_init=weights_init, shape=(self.layer_sizes[i], self.layer_sizes[i+1]), name="W_{0!s}_{1!s}".format(i, i+1), rng=mrg, # if gaussian mean=weights_mean, std=weights_std, # if uniform interval=weights_interval) for i in range(layers)] # add more weights if we aren't tying weights between layers (need to add for higher-lower layers now) if not tied_weights: self.weights_list.extend( [get_weights(weights_init=weights_init, shape=(self.layer_sizes[i+1], self.layer_sizes[i]), name="W_{0!s}_{1!s}".format(i+1, i), rng=mrg, # if gaussian mean=weights_mean, std=weights_std, # if uniform interval=weights_interval) for i in reversed(range(layers))] ) # initialize each layer bias to 0's. self.bias_list = [get_bias(shape=(self.layer_sizes[i],), name='b_' + str(i), init_values=bias_init) for i in range(layers+1)] # build the params of the model into a list self.params = self.weights_list + self.bias_list log.debug("gsn params: %s", str(self.params)) # using the properties, build the computational graph self.cost, self.monitors, self.output, self.hiddens = self.build_computation_graph()
def run_midi(dataset): log.info("Creating RNN-RBM for dataset %s!", dataset) outdir = "outputs/rnnrbm/%s/" % dataset # grab the MIDI dataset if dataset == 'nottingham': midi = Nottingham() elif dataset == 'jsb': midi = JSBChorales() elif dataset == 'muse': midi = MuseData() elif dataset == 'piano_de': midi = PianoMidiDe() else: raise AssertionError("dataset %s not recognized." % dataset) # create the RNN-RBM # rng = numpy.random # rng.seed(0xbeef) # mrg = RandomStreams(seed=rng.randint(1 << 30)) rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2 ** 30)) # rnnrbm = RNN_RBM(input_size=88, # hidden_size=150, # rnn_hidden_size=100, # k=15, # weights_init='gaussian', # weights_std=0.01, # rnn_weights_init='gaussian', # rnn_weights_std=0.0001, # rng=rng, # outdir=outdir) rnnrbm = RNN_RBM(input_size=88, hidden_size=150, rnn_hidden_size=100, k=15, weights_init='gaussian', weights_std=0.01, rnn_weights_init='identity', rnn_hidden_activation='relu', # rnn_weights_init='gaussian', # rnn_hidden_activation='tanh', rnn_weights_std=0.0001, mrg=mrg, outdir=outdir) # make an optimizer to train it optimizer = SGD(model=rnnrbm, dataset=midi, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=.001, save_frequency=10, early_stop_length=200, momentum=False, momentum_decay=False, nesterov_momentum=False) optimizer = AdaDelta(model=rnnrbm, dataset=midi, n_epoch=200, batch_size=100, minimum_batch_size=2, # learning_rate=1e-4, learning_rate=1e-6, save_frequency=10, early_stop_length=200) ll = Monitor('pseudo-log', rnnrbm.get_monitors()['pseudo-log'], test=True) mse = Monitor('frame-error', rnnrbm.get_monitors()['mse'], valid=True, test=True) plot = Plot(bokeh_doc_name='rnnrbm_midi_%s' % dataset, monitor_channels=[ll, mse], open_browser=True) # perform training! optimizer.train(plot=plot) # use the generate function! generated, _ = rnnrbm.generate(initial=None, n_steps=200) dt = 0.3 r = (21, 109) midiwrite(outdir + 'rnnrbm_generated_midi.mid', generated, r=r, dt=dt) if has_pylab: extent = (0, dt * len(generated)) + r pylab.figure() pylab.imshow(generated.T, origin='lower', aspect='auto', interpolation='nearest', cmap=pylab.cm.gray_r, extent=extent) pylab.xlabel('time (s)') pylab.ylabel('MIDI note number') pylab.title('generated piano-roll') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rnnrbm.W.get_value(borrow=True).T, img_shape=closest_to_square_factors(rnnrbm.input_size), tile_shape=closest_to_square_factors(rnnrbm.hidden_size), tile_spacing=(1, 1) ) ) image.save(outdir + 'rnnrbm_midi_weights.png') log.debug("done!") del midi del rnnrbm del optimizer
tile_spacing=(1, 1) ) ) image.save('rbm_test.png') # Construct image from the preds matrix image = Image.fromarray( tile_raster_images( X=preds, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1) ) ) image.save('rbm_preds.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(rbm.hidden_size), tile_spacing=(1, 1) ) ) image.save('rbm_weights.png') del mnist del rbm del optimizer
# perform training! optimizer.train(monitor_channels=ll) # test it on some images! test_data = mnist.test_inputs[:25] # use the run function! preds = rbm.run(test_data) # Construct image from the test matrix image = Image.fromarray(tile_raster_images(X=test_data, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1))) image.save("rbm_test.png") # Construct image from the preds matrix image = Image.fromarray(tile_raster_images(X=preds, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1))) image.save("rbm_preds.png") # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(config_args["hidden_size"]), tile_spacing=(1, 1), ) ) image.save("rbm_weights.png") del mnist del rbm del optimizer
tile_spacing=(1, 1) ) ) image.save('rbm_test.png') # Construct image from the preds matrix image = Image.fromarray( tile_raster_images( X=preds, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1) ) ) image.save('rbm_preds.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(config_args['hiddens']), tile_spacing=(1, 1) ) ) image.save('rbm_weights.png') del mnist del rbm del optimizer
def run_midi(dataset): log.info("Creating RNN-RBM for dataset %s!", dataset) outdir = "outputs/rnnrbm/%s/" % dataset # grab the MIDI dataset if dataset == 'nottingham': midi = Nottingham() elif dataset == 'jsb': midi = JSBChorales() elif dataset == 'muse': midi = MuseData() elif dataset == 'piano_de': midi = PianoMidiDe() else: raise AssertionError("dataset %s not recognized." % dataset) # create the RNN-RBM # rng = numpy.random # rng.seed(0xbeef) # mrg = RandomStreams(seed=rng.randint(1 << 30)) rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2**30)) # rnnrbm = RNN_RBM(input_size=88, # hidden_size=150, # rnn_hidden_size=100, # k=15, # weights_init='gaussian', # weights_std=0.01, # rnn_weights_init='gaussian', # rnn_weights_std=0.0001, # rng=rng, # outdir=outdir) rnnrbm = RNN_RBM( input_size=88, hidden_size=150, rnn_hidden_size=100, k=15, weights_init='gaussian', weights_std=0.01, rnn_weights_init='identity', rnn_hidden_activation='relu', # rnn_weights_init='gaussian', # rnn_hidden_activation='tanh', rnn_weights_std=0.0001, mrg=mrg, outdir=outdir) # make an optimizer to train it optimizer = SGD(model=rnnrbm, dataset=midi, epochs=200, batch_size=100, min_batch_size=2, learning_rate=.001, save_freq=10, stop_patience=200, momentum=False, momentum_decay=False, nesterov_momentum=False) optimizer = AdaDelta( model=rnnrbm, dataset=midi, epochs=200, batch_size=100, min_batch_size=2, # learning_rate=1e-4, learning_rate=1e-6, save_freq=10, stop_patience=200) ll = Monitor('pseudo-log', rnnrbm.get_monitors()['pseudo-log'], test=True) mse = Monitor('frame-error', rnnrbm.get_monitors()['mse'], valid=True, test=True) plot = Plot(bokeh_doc_name='rnnrbm_midi_%s' % dataset, monitor_channels=[ll, mse], open_browser=True) # perform training! optimizer.train(plot=plot) # use the generate function! generated, _ = rnnrbm.generate(initial=None, n_steps=200) dt = 0.3 r = (21, 109) midiwrite(outdir + 'rnnrbm_generated_midi.mid', generated, r=r, dt=dt) if has_pylab: extent = (0, dt * len(generated)) + r pylab.figure() pylab.imshow(generated.T, origin='lower', aspect='auto', interpolation='nearest', cmap=pylab.cm.gray_r, extent=extent) pylab.xlabel('time (s)') pylab.ylabel('MIDI note number') pylab.title('generated piano-roll') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rnnrbm.W.get_value(borrow=True).T, img_shape=closest_to_square_factors(rnnrbm.input_size), tile_shape=closest_to_square_factors(rnnrbm.hidden_size), tile_spacing=(1, 1))) image.save(outdir + 'rnnrbm_midi_weights.png') log.debug("done!") del midi del rnnrbm del optimizer
def run_sequence(sequence=0): log.info("Creating RNN-RBM for sequence %d!" % sequence) # grab the MNIST dataset mnist = MNIST(sequence_number=sequence, concat_train_valid=True) outdir = "outputs/rnnrbm/mnist_%d/" % sequence # create the RNN-RBM rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2 ** 30)) rnnrbm = RNN_RBM(input_size=28 * 28, hidden_size=1000, rnn_hidden_size=100, k=15, weights_init='uniform', weights_interval=4 * numpy.sqrt(6. / (28 * 28 + 500)), rnn_weights_init='identity', rnn_hidden_activation='relu', rnn_weights_std=1e-4, mrg=mrg, outdir=outdir) # load pretrained rbm on mnist # rnnrbm.load_params(outdir + 'trained_epoch_200.pkl') # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=rnnrbm, dataset=mnist, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=1e-8, save_frequency=10, early_stop_length=200) crossentropy = Monitor('crossentropy', rnnrbm.get_monitors()['crossentropy'], test=True) error = Monitor('error', rnnrbm.get_monitors()['mse'], test=True) plot = Plot(bokeh_doc_name='rnnrbm_mnist_%d' % sequence, monitor_channels=[crossentropy, error], open_browser=True) # perform training! optimizer.train(plot=plot) # use the generate function! log.debug("generating images...") generated, ut = rnnrbm.generate(initial=None, n_steps=400) # Construct image image = Image.fromarray( tile_raster_images( X=generated, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1) ) ) image.save(outdir + "rnnrbm_mnist_generated.png") log.debug('saved generated.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rnnrbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(rnnrbm.hidden_size), tile_spacing=(1, 1) ) ) image.save(outdir + "rnnrbm_mnist_weights.png") log.debug("done!") del mnist del rnnrbm del optimizer
def run_sequence(sequence=0): log.info("Creating RNN-RBM for sequence %d!" % sequence) # grab the MNIST dataset mnist = MNIST(sequence_number=sequence, concat_train_valid=True) outdir = "outputs/rnnrbm/mnist_%d/" % sequence # create the RNN-RBM rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2**30)) rnnrbm = RNN_RBM(input_size=28 * 28, hidden_size=1000, rnn_hidden_size=100, k=15, weights_init='uniform', weights_interval=4 * numpy.sqrt(6. / (28 * 28 + 500)), rnn_weights_init='identity', rnn_hidden_activation='relu', rnn_weights_std=1e-4, mrg=mrg, outdir=outdir) # load pretrained rbm on mnist # rnnrbm.load_params(outdir + 'trained_epoch_200.pkl') # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=rnnrbm, dataset=mnist, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=1e-8, save_frequency=10, early_stop_length=200) crossentropy = Monitor('crossentropy', rnnrbm.get_monitors()['crossentropy'], test=True) error = Monitor('error', rnnrbm.get_monitors()['mse'], test=True) plot = Plot(bokeh_doc_name='rnnrbm_mnist_%d' % sequence, monitor_channels=[crossentropy, error], open_browser=True) # perform training! optimizer.train(plot=plot) # use the generate function! log.debug("generating images...") generated, ut = rnnrbm.generate(initial=None, n_steps=400) # Construct image image = Image.fromarray( tile_raster_images(X=generated, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1))) image.save(outdir + "rnnrbm_mnist_generated.png") log.debug('saved generated.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images(X=rnnrbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors( rnnrbm.hidden_size), tile_spacing=(1, 1))) image.save(outdir + "rnnrbm_mnist_weights.png") log.debug("done!") del mnist del rnnrbm del optimizer
preds = rbm.run(test_data) # Construct image from the test matrix image = Image.fromarray( tile_raster_images(X=test_data, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1))) image.save('rbm_test.png') # Construct image from the preds matrix image = Image.fromarray( tile_raster_images(X=preds, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1))) image.save('rbm_preds.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images(X=rbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors( config_args['hidden_size']), tile_spacing=(1, 1))) image.save('rbm_weights.png') del mnist del rbm del optimizer
def run_audio(dataset): log.info("Creating RNN-GSN for dataset %s!", dataset) outdir = "outputs/rnngsn/%s/" % dataset # grab the dataset if dataset == 'tedlium': dataset = tedlium.TEDLIUMDataset(max_speeches=3) extra_args = { } elif dataset == 'codegolf': dataset = codegolf.CodeGolfDataset() extra_args = { } else: raise ValueError("dataset %s not recognized." % dataset) rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2 ** 30)) assert dataset.window_size == 256, dataset.window_size rnngsn = RNN_GSN(layers=2, walkbacks=4, input_size=dataset.window_size, image_height = 1, image_width = 256, hidden_size=128, rnn_hidden_size=128, weights_init='gaussian', weights_std=0.01, rnn_weights_init='identity', rnn_hidden_activation='relu', rnn_weights_std=0.0001, mrg=mrg, outdir=outdir, **extra_args) # make an optimizer to train it optimizer = AdaDelta(model=rnngsn, dataset=dataset, epochs=200, batch_size=128, min_batch_size=2, learning_rate=1e-6, save_freq=1, stop_patience=100) ll = Monitor('crossentropy', rnngsn.get_monitors()['noisy_recon_cost'],test=True) mse = Monitor('frame-error', rnngsn.get_monitors()['mse'],train=True,test=True,valid=True) plot = Plot( bokeh_doc_name='rnngsn_tedlium_%s'%dataset, monitor_channels=[ll,mse],open_browser=True ) # perform training! optimizer.train(plot=plot) # use the generate function! generated, _ = rnngsn.generate(initial=None, n_steps=200) # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rnngsn.weights_list[0].get_value(borrow=True).T, img_shape=closest_to_square_factors(rnngsn.input_size), tile_shape=closest_to_square_factors(rnngsn.hidden_size), tile_spacing=(1, 1) ) ) image.save(outdir + 'rnngsn_%s_weights.png'%(dataset,)) log.debug("done!") del rnngsn del optimizer if has_pylab: pylab.show()
def __init__(self, inputs_hook=None, hiddens_hook=None, params_hook=None, outdir='outputs/gsn/', input_size=None, hidden_size=1000, layers=2, walkbacks=4, visible_activation='sigmoid', hidden_activation='tanh', input_sampling=True, mrg=RNG_MRG.MRG_RandomStreams(1), tied_weights=True, weights_init='uniform', weights_interval='montreal', weights_mean=0, weights_std=5e-3, bias_init=0.0, cost_function='binary_crossentropy', cost_args=None, add_noise=True, noiseless_h1=True, hidden_noise='gaussian', hidden_noise_level=2, input_noise='salt_and_pepper', input_noise_level=0.4, noise_decay='exponential', noise_annealing=1, image_width=None, image_height=None, **kwargs): """ Initialize a GSN. Parameters ---------- inputs_hook : Tuple of (shape, variable) Routing information for the model to accept inputs from elsewhere. This is used for linking different models together (e.g. setting the Softmax model's input layer to the DAE's hidden layer gives a newly supervised classification model). For now, it needs to include the shape information (normally the dimensionality of the input i.e. n_in). hiddens_hook : Tuple of (shape, variable) Routing information for the model to accept its hidden representation from elsewhere. This is used for linking different models together (e.g. setting the DAE model's hidden layers to the RNN's output layer gives a generative recurrent model.) For now, it needs to include the shape information (normally the dimensionality of the hiddens i.e. n_hidden). params_hook : List(theano shared variable) A list of model parameters (shared theano variables) that you should use when constructing this model (instead of initializing your own shared variables). This parameter is useful when you want to have two versions of the model that use the same parameters - such as a training model with dropout applied to layers and one without for testing, where the parameters are shared between the two. outdir : str The directory you want outputs (parameters, images, etc.) to save to. If None, nothing will be saved. input_size : int The size (dimensionality) of the input to the DAE. If shape is provided in `inputs_hook`, this is optional. The :class:`Model` requires an `output_size`, which gets set to this value because the DAE is an unsupervised model. The output is a reconstruction of the input. hidden_size : int The size (dimensionality) of the hidden layer for the DAE. Generally, you want it to be larger than `input_size`, which is known as *overcomplete*. visible_activation : str or callable The nonlinear (or linear) visible activation to perform after the dot product from hiddens -> visible layer. This activation function should be appropriate for the input unit types, i.e. 'sigmoid' for binary inputs. See opendeep.utils.activation for a list of available activation functions. Alternatively, you can pass your own function to be used as long as it is callable. hidden_activation : str or callable The nonlinear (or linear) hidden activation to perform after the dot product from visible -> hiddens layer. See opendeep.utils.activation for a list of available activation functions. Alternatively, you can pass your own function to be used as long as it is callable. layers : int The number of hidden layers to use. walkbacks : int The number of walkbacks to perform (the variable K in Bengio's paper above). A walkback is a Gibbs sample from the DAE, which means the model generates inputs in sequence, where each generated input is compared to the original input to create the reconstruction cost for training. For running the model, the very last generated input in the Gibbs chain is used as the output. input_sampling : bool During walkbacks, whether to sample from the generated input to create a new starting point for the next walkback (next step in the Gibbs chain). This generally makes walkbacks more effective by making the process more stochastic - more likely to find spurious modes in the model's representation. mrg : random A random number generator that is used when adding noise into the network and for sampling from the input. I recommend using Theano's sandbox.rng_mrg.MRG_RandomStreams. tied_weights : bool DAE has two weight matrices - W from input -> hiddens and V from hiddens -> input. This boolean determines if V = W.T, which 'ties' V to W and reduces the number of parameters necessary during training. weights_init : str Determines the method for initializing model weights. See opendeep.utils.nnet for options. weights_interval : str or float If Uniform `weights_init`, the +- interval to use. See opendeep.utils.nnet for options. weights_mean : float If Gaussian `weights_init`, the mean value to use. weights_std : float If Gaussian `weights_init`, the standard deviation to use. bias_init : float The initial value to use for the bias parameter. Most often, the default of 0.0 is preferred. cost_function : str or callable The function to use when calculating the reconstruction cost of the model. This should be appropriate for the type of input, i.e. use 'binary_crossentropy' for binary inputs, or 'mse' for real-valued inputs. See opendeep.utils.cost for options. You can also specify your own function, which needs to be callable. cost_args : dict Any additional named keyword arguments to pass to the specified `cost_function`. add_noise : bool Whether to add noise (corrupt) the input before passing it through the computation graph during training. This should most likely be set to the default of True, because this is a *denoising* autoencoder after all. noiseless_h1 : bool Whether to not add noise (corrupt) the hidden layer during computation. hidden_noise : str What type of noise to use for corrupting the hidden layer (if not `noiseless_h1`). See opendeep.utils.noise for options. This should be appropriate for the hidden unit activation, i.e. Gaussian for tanh or other real-valued activations, etc. hidden_noise_level : float The amount of noise to use for the noise function specified by `hidden_noise`. This could be the standard deviation for gaussian noise, the interval for uniform noise, the dropout amount, etc. input_noise : str What type of noise to use for corrupting the input before computation (if `add_noise`). See opendeep.utils.noise for options. This should be appropriate for the input units, i.e. salt-and-pepper for binary units, etc. input_noise_level : float The amount of noise used to corrupt the input. This could be the masking probability for salt-and-pepper, standard deviation for Gaussian, interval for Uniform, etc. noise_decay : str or False Whether to use `input_noise` scheduling (decay `input_noise_level` during the course of training), and if so, the string input specifies what type of decay to use. See opendeep.utils.decay for options. Noise decay (known as noise scheduling) effectively helps the DAE learn larger variance features first, and then smaller ones later (almost as a kind of curriculum learning). May help it converge faster. noise_annealing : float The amount to reduce the `input_noise_level` after each training epoch based on the decay function specified in `noise_decay`. image_width : int If the input should be represented as an image, the width of the input image. If not specified, it will be close to the square factor of the `input_size`. image_height : int If the input should be represented as an image, the height of the input image. If not specified, it will be close to the square factor of the `input_size`. """ # init Model to combine the defaults and config dictionaries with the initial parameters. initial_parameters = locals().copy() initial_parameters.pop('self') super(GSN, self).__init__(**initial_parameters) # when the input should be thought of as an image, either use the specified width and height, # or try to make as square as possible. if image_height is None and image_width is None: (_h, _w) = closest_to_square_factors(self.input_size) self.image_width = _w self.image_height = _h else: self.image_height = image_height self.image_width = image_width ############################ # Theano variables and RNG # ############################ if self.inputs_hook is None: self.X = T.matrix('X') else: # inputs_hook is a (shape, input) tuple self.X = self.inputs_hook[1] ########################## # Network specifications # ########################## # generally, walkbacks should be at least 2*layers if layers % 2 == 0: if walkbacks < 2 * layers: log.warning( 'Not enough walkbacks for the layers! Layers is %s and walkbacks is %s. ' 'Generaly want 2X walkbacks to layers', str(layers), str(walkbacks)) else: if walkbacks < 2 * layers - 1: log.warning( 'Not enough walkbacks for the layers! Layers is %s and walkbacks is %s. ' 'Generaly want 2X walkbacks to layers', str(layers), str(walkbacks)) self.add_noise = add_noise self.noise_annealing = as_floatX( noise_annealing) # noise schedule parameter self.hidden_noise_level = sharedX(hidden_noise_level, dtype=theano.config.floatX) self.hidden_noise = get_noise(name=hidden_noise, noise_level=self.hidden_noise_level, mrg=mrg) self.input_noise_level = sharedX(input_noise_level, dtype=theano.config.floatX) self.input_noise = get_noise(name=input_noise, noise_level=self.input_noise_level, mrg=mrg) self.walkbacks = walkbacks self.tied_weights = tied_weights self.layers = layers self.noiseless_h1 = noiseless_h1 self.input_sampling = input_sampling self.noise_decay = noise_decay # if there was a hiddens_hook, unpack the hidden layers in the tensor if self.hiddens_hook is not None: hidden_size = self.hiddens_hook[0] self.hiddens_flag = True else: self.hiddens_flag = False # determine the sizes of each layer in a list. # layer sizes, from h0 to hK (h0 is the visible layer) hidden_size = list(raise_to_list(hidden_size)) if len(hidden_size) == 1: self.layer_sizes = [self.input_size] + hidden_size * self.layers else: assert len(hidden_size) == self.layers, "Hiddens sizes and number of hidden layers mismatch." + \ "Hiddens %d and layers %d" % (len(hidden_size), self.layers) self.layer_sizes = [self.input_size] + hidden_size if self.hiddens_hook is not None: self.hiddens = self.unpack_hiddens(self.hiddens_hook[1]) ######################### # Activation functions! # ######################### # hidden unit activation self.hidden_activation = get_activation_function(hidden_activation) # Visible layer activation self.visible_activation = get_activation_function(visible_activation) # make sure the sampling functions are appropriate for the activation functions. if is_binary(self.visible_activation): self.visible_sampling = mrg.binomial else: # TODO: implement non-binary activation log.error("Non-binary visible activation not supported yet!") raise NotImplementedError( "Non-binary visible activation not supported yet!") # Cost function self.cost_function = get_cost_function(cost_function) self.cost_args = cost_args or dict() ############### # Parameters! # ############### # make sure to deal with params_hook! if self.params_hook is not None: # if tied weights, expect layers*2 + 1 params if self.tied_weights: assert len(self.params_hook) == 2*layers + 1, \ "Tied weights: expected {0!s} params, found {1!s}!".format(2*layers+1, len(self.params_hook)) self.weights_list = self.params_hook[:layers] self.bias_list = self.params_hook[layers:] # if untied weights, expect layers*3 + 1 params else: assert len(self.params_hook) == 3*layers + 1, \ "Untied weights: expected {0!s} params, found {1!s}!".format(3*layers+1, len(self.params_hook)) self.weights_list = self.params_hook[:2 * layers] self.bias_list = self.params_hook[2 * layers:] # otherwise, construct our params else: # initialize a list of weights and biases based on layer_sizes for the GSN self.weights_list = [ get_weights( weights_init=weights_init, shape=(self.layer_sizes[i], self.layer_sizes[i + 1]), name="W_{0!s}_{1!s}".format(i, i + 1), rng=mrg, # if gaussian mean=weights_mean, std=weights_std, # if uniform interval=weights_interval) for i in range(layers) ] # add more weights if we aren't tying weights between layers (need to add for higher-lower layers now) if not tied_weights: self.weights_list.extend([ get_weights( weights_init=weights_init, shape=(self.layer_sizes[i + 1], self.layer_sizes[i]), name="W_{0!s}_{1!s}".format(i + 1, i), rng=mrg, # if gaussian mean=weights_mean, std=weights_std, # if uniform interval=weights_interval) for i in reversed(range(layers)) ]) # initialize each layer bias to 0's. self.bias_list = [ get_bias(shape=(self.layer_sizes[i], ), name='b_' + str(i), init_values=bias_init) for i in range(layers + 1) ] # build the params of the model into a list self.params = self.weights_list + self.bias_list log.debug("gsn params: %s", str(self.params)) # using the properties, build the computational graph self.cost, self.monitors, self.output, self.hiddens = self.build_computation_graph( )
def main(): ######################################## # Initialization things with arguments # ######################################## # use these arguments to get results from paper referenced above _train_args = {"epochs": 1000, # maximum number of times to run through the dataset "batch_size": 100, # number of examples to process in parallel (minibatch) "min_batch_size": 1, # the minimum number of examples for a batch to be considered "save_freq": 1, # how many epochs between saving parameters "stop_threshold": .9995, # multiplier for how much the train cost to improve to not stop early "stop_patience": 500, # how many epochs to wait to see if the threshold has been reached "learning_rate": .25, # initial learning rate for SGD "lr_decay": 'exponential', # the decay function to use for the learning rate parameter "lr_decay_factor": .995, # by how much to decay the learning rate each epoch "momentum": 0.5, # the parameter momentum amount 'momentum_decay': False, # how to decay the momentum each epoch (if applicable) 'momentum_factor': 0, # by how much to decay the momentum (in this case not at all) 'nesterov_momentum': False, # whether to use nesterov momentum update (accelerated momentum) } config_root_logger() log.info("Creating a new GSN") mnist = MNIST(concat_train_valid=True) gsn = GSN(layers=2, walkbacks=4, hidden_size=1500, visible_activation='sigmoid', hidden_activation='tanh', input_size=28*28, tied_weights=True, hidden_add_noise_sigma=2, input_salt_and_pepper=0.4, outdir='outputs/test_gsn/', vis_init=False, noiseless_h1=True, input_sampling=True, weights_init='uniform', weights_interval='montreal', bias_init=0, cost_function='binary_crossentropy') recon_cost_channel = MonitorsChannel(name='cost') recon_cost_channel.add(Monitor('recon_cost', gsn.get_monitors()['recon_cost'], test=True)) recon_cost_channel.add(Monitor('noisy_recon_cost', gsn.get_monitors()['noisy_recon_cost'], test=True)) # Load initial weights and biases from file # params_to_load = '../../../outputs/gsn/mnist/trained_epoch_395.pkl' # gsn.load_params(params_to_load) optimizer = SGD(model=gsn, dataset=mnist, **_train_args) # optimizer = AdaDelta(model=gsn, dataset=mnist, epochs=200, batch_size=100, learning_rate=1e-6) optimizer.train(monitor_channels=recon_cost_channel) # Save some reconstruction output images n_examples = 100 xs_test = mnist.test_inputs[:n_examples] noisy_xs_test = gsn.f_noise(xs_test) reconstructed = gsn.run(noisy_xs_test) # Concatenate stuff stacked = numpy.vstack( [numpy.vstack([xs_test[i * 10: (i + 1) * 10], noisy_xs_test[i * 10: (i + 1) * 10], reconstructed[i * 10: (i + 1) * 10]]) for i in range(10)]) number_reconstruction = PIL.Image.fromarray( tile_raster_images(stacked, (gsn.image_height, gsn.image_width), (10, 30)) ) number_reconstruction.save(gsn.outdir + 'reconstruction.png') log.info("saved output image!") # Construct image from the weight matrix image = PIL.Image.fromarray( tile_raster_images( X=gsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(gsn.hidden_size), tile_spacing=(1, 1) ) ) image.save(gsn.outdir + "gsn_mnist_weights.png")
def main(): ######################################## # Initialization things with arguments # ######################################## # use these arguments to get results from paper referenced above _train_args = {"n_epoch": 1000, # maximum number of times to run through the dataset "batch_size": 100, # number of examples to process in parallel (minibatch) "minimum_batch_size": 1, # the minimum number of examples for a batch to be considered "save_frequency": 1, # how many epochs between saving parameters "early_stop_threshold": .9995, # multiplier for how much the train cost to improve to not stop early "early_stop_length": 500, # how many epochs to wait to see if the threshold has been reached "learning_rate": .25, # initial learning rate for SGD "lr_decay": 'exponential', # the decay function to use for the learning rate parameter "lr_factor": .995, # by how much to decay the learning rate each epoch "momentum": 0.5, # the parameter momentum amount 'momentum_decay': False, # how to decay the momentum each epoch (if applicable) 'momentum_factor': 0, # by how much to decay the momentum (in this case not at all) 'nesterov_momentum': False, # whether to use nesterov momentum update (accelerated momentum) } config_root_logger() log.info("Creating a new GSN") mnist = MNIST(concat_train_valid=True) gsn = GSN(layers=2, walkbacks=4, hidden_size=1500, visible_activation='sigmoid', hidden_activation='tanh', input_size=28*28, tied_weights=True, hidden_add_noise_sigma=2, input_salt_and_pepper=0.4, outdir='outputs/test_gsn/', vis_init=False, noiseless_h1=True, input_sampling=True, weights_init='uniform', weights_interval='montreal', bias_init=0, cost_function='binary_crossentropy') recon_cost_channel = MonitorsChannel(name='cost') recon_cost_channel.add(Monitor('recon_cost', gsn.get_monitors()['recon_cost'], test=True)) recon_cost_channel.add(Monitor('noisy_recon_cost', gsn.get_monitors()['noisy_recon_cost'], test=True)) # Load initial weights and biases from file # params_to_load = '../../../outputs/gsn/mnist/trained_epoch_395.pkl' # gsn.load_params(params_to_load) optimizer = SGD(model=gsn, dataset=mnist, **_train_args) # optimizer = AdaDelta(model=gsn, dataset=mnist, n_epoch=200, batch_size=100, learning_rate=1e-6) optimizer.train(monitor_channels=recon_cost_channel) # Save some reconstruction output images import opendeep.data.dataset as datasets n_examples = 100 xs_test, _ = mnist.getSubset(datasets.TEST) xs_test = xs_test[:n_examples].eval() noisy_xs_test = gsn.f_noise(xs_test) reconstructed = gsn.run(noisy_xs_test) # Concatenate stuff stacked = numpy.vstack( [numpy.vstack([xs_test[i * 10: (i + 1) * 10], noisy_xs_test[i * 10: (i + 1) * 10], reconstructed[i * 10: (i + 1) * 10]]) for i in range(10)]) number_reconstruction = PIL.Image.fromarray( tile_raster_images(stacked, (gsn.image_height, gsn.image_width), (10, 30)) ) number_reconstruction.save(gsn.outdir + 'reconstruction.png') log.info("saved output image!") # Construct image from the weight matrix image = PIL.Image.fromarray( tile_raster_images( X=gsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(gsn.hidden_size), tile_spacing=(1, 1) ) ) image.save(gsn.outdir + "gsn_mnist_weights.png")