def create_reconstruction_image(self, input_data): """ Adds noise to an input and saves an image from the reconstruction running the input through the computation graph. """ n_examples = len(input_data) xs_test = input_data noisy_xs_test = self.f_noise(input_data) reconstructed = self.run(noisy_xs_test) # Concatenate stuff width, height = closest_to_square_factors(n_examples) stacked = numpy.vstack([ numpy.vstack([ xs_test[i * width:(i + 1) * width], noisy_xs_test[i * width:(i + 1) * width], reconstructed[i * width:(i + 1) * width] ]) for i in range(height) ]) number_reconstruction = PIL.Image.fromarray( tile_raster_images(stacked, (self.image_height, self.image_width), (height, 3 * width))) save_path = os.path.join(self.outdir, 'gsn_reconstruction.png') save_path = os.path.realpath(save_path) number_reconstruction.save(save_path) log.info("saved output image to %s", save_path)
def create_reconstruction_image(self, input_data): """ Adds noise to an input and saves an image from the reconstruction running the input through the computation graph. """ n_examples = len(input_data) xs_test = input_data noisy_xs_test = self.f_noise(input_data) reconstructed = self.run(noisy_xs_test) # Concatenate stuff width, height = closest_to_square_factors(n_examples) stacked = numpy.vstack( [numpy.vstack([xs_test[i * width: (i + 1) * width], noisy_xs_test[i * width: (i + 1) * width], reconstructed[i * width: (i + 1) * width]]) for i in range(height)]) number_reconstruction = PIL.Image.fromarray( tile_raster_images(stacked, (self.image_height, self.image_width), (height, 3*width)) ) save_path = os.path.join(self.outdir, 'gsn_reconstruction.png') save_path = os.path.realpath(save_path) number_reconstruction.save(save_path) log.info("saved output image to %s", save_path)
dae = DenoisingAutoencoder() # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=dae, dataset=mnist) # perform training! optimizer.train() # test it on some images! test_data, _ = mnist.getSubset(TEST) test_data = test_data[:25].eval() corrupted_test = salt_and_pepper(test_data, 0.4).eval() # use the run function! reconstructed_images = dae.run(corrupted_test) # create an image from this reconstruction! # imports for working with tiling outputs into one image from opendeep.utils.image import tile_raster_images import numpy import PIL # stack the image matrices together in three 5x5 grids next to each other using numpy stacked = numpy.vstack([ numpy.vstack([ test_data[i * 5:(i + 1) * 5], corrupted_test[i * 5:(i + 1) * 5], reconstructed_images[i * 5:(i + 1) * 5] ]) for i in range(5) ]) # convert the combined matrix into an image image = PIL.Image.fromarray( tile_raster_images(stacked, (28, 28), (5, 3 * 5))) # save it! image.save("dae_reconstruction_test.png")
def run_audio(dataset): log.info("Creating RNN-GSN for dataset %s!", dataset) outdir = "outputs/rnngsn/%s/" % dataset # grab the dataset if dataset == 'tedlium': dataset = tedlium.TEDLIUMDataset(max_speeches=3) extra_args = { } elif dataset == 'codegolf': dataset = codegolf.CodeGolfDataset() extra_args = { } else: raise ValueError("dataset %s not recognized." % dataset) rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2 ** 30)) assert dataset.window_size == 256, dataset.window_size rnngsn = RNN_GSN(layers=2, walkbacks=4, input_size=dataset.window_size, image_height = 1, image_width = 256, hidden_size=128, rnn_hidden_size=128, weights_init='gaussian', weights_std=0.01, rnn_weights_init='identity', rnn_hidden_activation='relu', rnn_weights_std=0.0001, mrg=mrg, outdir=outdir, **extra_args) # make an optimizer to train it optimizer = AdaDelta(model=rnngsn, dataset=dataset, epochs=200, batch_size=128, min_batch_size=2, learning_rate=1e-6, save_freq=1, stop_patience=100) ll = Monitor('crossentropy', rnngsn.get_monitors()['noisy_recon_cost'],test=True) mse = Monitor('frame-error', rnngsn.get_monitors()['mse'],train=True,test=True,valid=True) plot = Plot( bokeh_doc_name='rnngsn_tedlium_%s'%dataset, monitor_channels=[ll,mse],open_browser=True ) # perform training! optimizer.train(plot=plot) # use the generate function! generated, _ = rnngsn.generate(initial=None, n_steps=200) # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rnngsn.weights_list[0].get_value(borrow=True).T, img_shape=closest_to_square_factors(rnngsn.input_size), tile_shape=closest_to_square_factors(rnngsn.hidden_size), tile_spacing=(1, 1) ) ) image.save(outdir + 'rnngsn_%s_weights.png'%(dataset,)) log.debug("done!") del rnngsn del optimizer if has_pylab: pylab.show()
# make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=dae, dataset=mnist, epochs=100) # perform training! optimizer.train() # test it on some images! test_data = mnist.test_inputs[:25] corrupted_test = salt_and_pepper(test_data, 0.4).eval() # use the run function! reconstructed_images = dae.run(corrupted_test) # create an image from this reconstruction! # imports for working with tiling outputs into one image from opendeep.utils.image import tile_raster_images import numpy import PIL # stack the image matrices together in three 5x5 grids next to each other using numpy stacked = numpy.vstack( [numpy.vstack([test_data[i*5 : (i+1)*5], corrupted_test[i*5 : (i+1)*5], reconstructed_images[i*5 : (i+1)*5]]) for i in range(5)]) # convert the combined matrix into an image image = PIL.Image.fromarray( tile_raster_images(stacked, (28, 28), (5, 3*5)) ) # save it! image.save("dae_reconstruction_test.png")
def run_sequence(sequence=0): log.info("Creating RNN-GSN for sequence %d!" % sequence) # grab the MNIST dataset mnist = MNIST(sequence_number=sequence, concat_train_valid=True) outdir = "outputs/rnngsn/mnist_%d/" % sequence rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2**30)) rnngsn = RNN_GSN(layers=2, walkbacks=4, input_size=28 * 28, hidden_size=1000, tied_weights=True, rnn_hidden_size=100, weights_init='uniform', weights_interval='montreal', rnn_weights_init='identity', mrg=mrg, outdir=outdir) # load pretrained rbm on mnist # rnngsn.load_gsn_params('outputs/trained_gsn_epoch_1000.pkl') # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=rnngsn, dataset=mnist, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=1e-6, save_frequency=1, early_stop_length=200) # optimizer = SGD(model=rnngsn, # dataset=mnist, # n_epoch=300, # batch_size=100, # minimum_batch_size=2, # learning_rate=.25, # lr_decay='exponential', # lr_factor=.995, # momentum=0.5, # nesterov_momentum=True, # momentum_decay=False, # save_frequency=20, # early_stop_length=100) crossentropy = Monitor('crossentropy', rnngsn.get_monitors()['noisy_recon_cost'], test=True) error = Monitor('error', rnngsn.get_monitors()['mse'], test=True) # perform training! optimizer.train(monitor_channels=[crossentropy, error]) # use the generate function! log.debug("generating images...") generated, ut = rnngsn.generate(initial=None, n_steps=400) # Construct image image = Image.fromarray( tile_raster_images(X=generated, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1))) image.save(outdir + "rnngsn_mnist_generated.png") log.debug('saved generated.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images(X=rnngsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors( rnngsn.hidden_size), tile_spacing=(1, 1))) image.save(outdir + "rnngsn_mnist_weights.png") log.debug("done!") del mnist del rnngsn del optimizer
def run_sequence(sequence=0): log.info("Creating RNN-GSN for sequence %d!" % sequence) # grab the MNIST dataset mnist = MNIST(sequence_number=sequence, concat_train_valid=True) outdir = "outputs/rnngsn/mnist_%d/" % sequence rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2 ** 30)) rnngsn = RNN_GSN(layers=2, walkbacks=4, input_size=28 * 28, hidden_size=1000, tied_weights=True, rnn_hidden_size=100, weights_init='uniform', weights_interval='montreal', rnn_weights_init='identity', mrg=mrg, outdir=outdir) # load pretrained rbm on mnist # rnngsn.load_gsn_params('outputs/trained_gsn_epoch_1000.pkl') # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=rnngsn, dataset=mnist, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=1e-6, save_frequency=1, early_stop_length=200) # optimizer = SGD(model=rnngsn, # dataset=mnist, # n_epoch=300, # batch_size=100, # minimum_batch_size=2, # learning_rate=.25, # lr_decay='exponential', # lr_factor=.995, # momentum=0.5, # nesterov_momentum=True, # momentum_decay=False, # save_frequency=20, # early_stop_length=100) crossentropy = Monitor('crossentropy', rnngsn.get_monitors()['noisy_recon_cost'], test=True) error = Monitor('error', rnngsn.get_monitors()['mse'], test=True) # perform training! optimizer.train(monitor_channels=[crossentropy, error]) # use the generate function! log.debug("generating images...") generated, ut = rnngsn.generate(initial=None, n_steps=400) # Construct image image = Image.fromarray( tile_raster_images( X=generated, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1) ) ) image.save(outdir + "rnngsn_mnist_generated.png") log.debug('saved generated.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rnngsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(rnngsn.hidden_size), tile_spacing=(1, 1) ) ) image.save(outdir + "rnngsn_mnist_weights.png") log.debug("done!") del mnist del rnngsn del optimizer
rbm = RBM(**config_args) # rbm.load_params('rbm_trained.pkl') optimizer = Optimizer(learning_rate=0.1, model=rbm, dataset=mnist, batch_size=20, epochs=15) ll = Monitor("pseudo-log", rbm.get_monitors()["pseudo-log"]) # perform training! optimizer.train(monitor_channels=ll) # test it on some images! test_data = mnist.test_inputs[:25] # use the run function! preds = rbm.run(test_data) # Construct image from the test matrix image = Image.fromarray(tile_raster_images(X=test_data, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1))) image.save("rbm_test.png") # Construct image from the preds matrix image = Image.fromarray(tile_raster_images(X=preds, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1))) image.save("rbm_preds.png") # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(config_args["hidden_size"]), tile_spacing=(1, 1), ) )
ll = Monitor('pseudo-log', rbm.get_monitors()['pseudo-log']) # perform training! optimizer.train(monitor_channels=ll) # test it on some images! test_data = mnist.getSubset(TEST)[0] test_data = test_data[:25].eval() # use the run function! preds = rbm.run(test_data) # Construct image from the test matrix image = Image.fromarray( tile_raster_images( X=test_data, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1) ) ) image.save('rbm_test.png') # Construct image from the preds matrix image = Image.fromarray( tile_raster_images( X=preds, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1) ) ) image.save('rbm_preds.png')
def run_midi(dataset): log.info("Creating RNN-RBM for dataset %s!", dataset) outdir = "outputs/rnnrbm/%s/" % dataset # grab the MIDI dataset if dataset == 'nottingham': midi = Nottingham() elif dataset == 'jsb': midi = JSBChorales() elif dataset == 'muse': midi = MuseData() elif dataset == 'piano_de': midi = PianoMidiDe() else: raise AssertionError("dataset %s not recognized." % dataset) # create the RNN-RBM # rng = numpy.random # rng.seed(0xbeef) # mrg = RandomStreams(seed=rng.randint(1 << 30)) rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2**30)) # rnnrbm = RNN_RBM(input_size=88, # hidden_size=150, # rnn_hidden_size=100, # k=15, # weights_init='gaussian', # weights_std=0.01, # rnn_weights_init='gaussian', # rnn_weights_std=0.0001, # rng=rng, # outdir=outdir) rnnrbm = RNN_RBM( input_size=88, hidden_size=150, rnn_hidden_size=100, k=15, weights_init='gaussian', weights_std=0.01, rnn_weights_init='identity', rnn_hidden_activation='relu', # rnn_weights_init='gaussian', # rnn_hidden_activation='tanh', rnn_weights_std=0.0001, mrg=mrg, outdir=outdir) # make an optimizer to train it optimizer = SGD(model=rnnrbm, dataset=midi, epochs=200, batch_size=100, min_batch_size=2, learning_rate=.001, save_freq=10, stop_patience=200, momentum=False, momentum_decay=False, nesterov_momentum=False) optimizer = AdaDelta( model=rnnrbm, dataset=midi, epochs=200, batch_size=100, min_batch_size=2, # learning_rate=1e-4, learning_rate=1e-6, save_freq=10, stop_patience=200) ll = Monitor('pseudo-log', rnnrbm.get_monitors()['pseudo-log'], test=True) mse = Monitor('frame-error', rnnrbm.get_monitors()['mse'], valid=True, test=True) plot = Plot(bokeh_doc_name='rnnrbm_midi_%s' % dataset, monitor_channels=[ll, mse], open_browser=True) # perform training! optimizer.train(plot=plot) # use the generate function! generated, _ = rnnrbm.generate(initial=None, n_steps=200) dt = 0.3 r = (21, 109) midiwrite(outdir + 'rnnrbm_generated_midi.mid', generated, r=r, dt=dt) if has_pylab: extent = (0, dt * len(generated)) + r pylab.figure() pylab.imshow(generated.T, origin='lower', aspect='auto', interpolation='nearest', cmap=pylab.cm.gray_r, extent=extent) pylab.xlabel('time (s)') pylab.ylabel('MIDI note number') pylab.title('generated piano-roll') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rnnrbm.W.get_value(borrow=True).T, img_shape=closest_to_square_factors(rnnrbm.input_size), tile_shape=closest_to_square_factors(rnnrbm.hidden_size), tile_spacing=(1, 1))) image.save(outdir + 'rnnrbm_midi_weights.png') log.debug("done!") del midi del rnnrbm del optimizer
print "Shapes:" print train_x.shape, train_y.shape print valid_x.shape, valid_y.shape print test_x.shape, test_y.shape print "--------------" print "Example input:" print train_x[0] print "Example label:" print train_y[0] # Show example images - using tile_raster_images helper function from OpenDeep to get 28x28 image from 784 array. input_images = train_x[:25] im = Image.fromarray( tile_raster_images(input_images, img_shape=(28, 28), tile_shape=(1, 25), tile_spacing=(1, 1))) im.save("example_mnist_numbers.png") ######### # Model # ######### # Cool, now we know a little about the input data, let's design the MLP to work with it! # An MLP looks like this: input -> hiddens -> output classification # Each stage is just a matrix multiplication with a nonlinear function applied after. # Inputs are matrices where rows are examples and columns are pixels - so create a symbolic Theano matrix. x = T.matrix('xs') # Now let's start building the equation for our MLP! # The first transformation is the input x -> hidden layer h. # We defined this transformation with h = tanh(x.dot(W_x) + b_h)
def run_sequence(sequence=0): log.info("Creating RNN-RBM for sequence %d!" % sequence) # grab the MNIST dataset mnist = MNIST(sequence_number=sequence, concat_train_valid=True) outdir = "outputs/rnnrbm/mnist_%d/" % sequence # create the RNN-RBM rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2 ** 30)) rnnrbm = RNN_RBM(input_size=28 * 28, hidden_size=1000, rnn_hidden_size=100, k=15, weights_init='uniform', weights_interval=4 * numpy.sqrt(6. / (28 * 28 + 500)), rnn_weights_init='identity', rnn_hidden_activation='relu', rnn_weights_std=1e-4, mrg=mrg, outdir=outdir) # load pretrained rbm on mnist # rnnrbm.load_params(outdir + 'trained_epoch_200.pkl') # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=rnnrbm, dataset=mnist, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=1e-8, save_frequency=10, early_stop_length=200) crossentropy = Monitor('crossentropy', rnnrbm.get_monitors()['crossentropy'], test=True) error = Monitor('error', rnnrbm.get_monitors()['mse'], test=True) plot = Plot(bokeh_doc_name='rnnrbm_mnist_%d' % sequence, monitor_channels=[crossentropy, error], open_browser=True) # perform training! optimizer.train(plot=plot) # use the generate function! log.debug("generating images...") generated, ut = rnnrbm.generate(initial=None, n_steps=400) # Construct image image = Image.fromarray( tile_raster_images( X=generated, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1) ) ) image.save(outdir + "rnnrbm_mnist_generated.png") log.debug('saved generated.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rnnrbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(rnnrbm.hidden_size), tile_spacing=(1, 1) ) ) image.save(outdir + "rnnrbm_mnist_weights.png") log.debug("done!") del mnist del rnnrbm del optimizer
def run_sequence(sequence=0): log.info("Creating RNN-RBM for sequence %d!" % sequence) # grab the MNIST dataset mnist = MNIST(sequence_number=sequence, concat_train_valid=True) outdir = "outputs/rnnrbm/mnist_%d/" % sequence # create the RNN-RBM rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2**30)) rnnrbm = RNN_RBM(input_size=28 * 28, hidden_size=1000, rnn_hidden_size=100, k=15, weights_init='uniform', weights_interval=4 * numpy.sqrt(6. / (28 * 28 + 500)), rnn_weights_init='identity', rnn_hidden_activation='relu', rnn_weights_std=1e-4, mrg=mrg, outdir=outdir) # load pretrained rbm on mnist # rnnrbm.load_params(outdir + 'trained_epoch_200.pkl') # make an optimizer to train it (AdaDelta is a good default) optimizer = AdaDelta(model=rnnrbm, dataset=mnist, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=1e-8, save_frequency=10, early_stop_length=200) crossentropy = Monitor('crossentropy', rnnrbm.get_monitors()['crossentropy'], test=True) error = Monitor('error', rnnrbm.get_monitors()['mse'], test=True) plot = Plot(bokeh_doc_name='rnnrbm_mnist_%d' % sequence, monitor_channels=[crossentropy, error], open_browser=True) # perform training! optimizer.train(plot=plot) # use the generate function! log.debug("generating images...") generated, ut = rnnrbm.generate(initial=None, n_steps=400) # Construct image image = Image.fromarray( tile_raster_images(X=generated, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1))) image.save(outdir + "rnnrbm_mnist_generated.png") log.debug('saved generated.png') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images(X=rnnrbm.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors( rnnrbm.hidden_size), tile_spacing=(1, 1))) image.save(outdir + "rnnrbm_mnist_weights.png") log.debug("done!") del mnist del rnnrbm del optimizer
# test it on some images! test_data, _ = mnist.getSubset(TEST) test_data = test_data[:25].eval() corrupted_test = salt_and_pepper(test_data, 0.4).eval() # use the run function! reconstructed_images = dae.run(corrupted_test) # create an image from this reconstruction! # imports for working with tiling outputs into one image from opendeep.utils.image import tile_raster_images import numpy import PIL # stack the image matrices together in three 5x5 grids next to each other using numpy stacked = numpy.vstack( [ numpy.vstack( [ test_data[i * 5 : (i + 1) * 5], corrupted_test[i * 5 : (i + 1) * 5], reconstructed_images[i * 5 : (i + 1) * 5], ] ) for i in range(5) ] ) # convert the combined matrix into an image image = PIL.Image.fromarray(tile_raster_images(stacked, (28, 28), (5, 3 * 5))) # save it! image.save("dae_reconstruction_test.png")
def main(): ######################################## # Initialization things with arguments # ######################################## # use these arguments to get results from paper referenced above _train_args = {"epochs": 1000, # maximum number of times to run through the dataset "batch_size": 100, # number of examples to process in parallel (minibatch) "min_batch_size": 1, # the minimum number of examples for a batch to be considered "save_freq": 1, # how many epochs between saving parameters "stop_threshold": .9995, # multiplier for how much the train cost to improve to not stop early "stop_patience": 500, # how many epochs to wait to see if the threshold has been reached "learning_rate": .25, # initial learning rate for SGD "lr_decay": 'exponential', # the decay function to use for the learning rate parameter "lr_decay_factor": .995, # by how much to decay the learning rate each epoch "momentum": 0.5, # the parameter momentum amount 'momentum_decay': False, # how to decay the momentum each epoch (if applicable) 'momentum_factor': 0, # by how much to decay the momentum (in this case not at all) 'nesterov_momentum': False, # whether to use nesterov momentum update (accelerated momentum) } config_root_logger() log.info("Creating a new GSN") mnist = MNIST(concat_train_valid=True) gsn = GSN(layers=2, walkbacks=4, hidden_size=1500, visible_activation='sigmoid', hidden_activation='tanh', input_size=28*28, tied_weights=True, hidden_add_noise_sigma=2, input_salt_and_pepper=0.4, outdir='outputs/test_gsn/', vis_init=False, noiseless_h1=True, input_sampling=True, weights_init='uniform', weights_interval='montreal', bias_init=0, cost_function='binary_crossentropy') recon_cost_channel = MonitorsChannel(name='cost') recon_cost_channel.add(Monitor('recon_cost', gsn.get_monitors()['recon_cost'], test=True)) recon_cost_channel.add(Monitor('noisy_recon_cost', gsn.get_monitors()['noisy_recon_cost'], test=True)) # Load initial weights and biases from file # params_to_load = '../../../outputs/gsn/mnist/trained_epoch_395.pkl' # gsn.load_params(params_to_load) optimizer = SGD(model=gsn, dataset=mnist, **_train_args) # optimizer = AdaDelta(model=gsn, dataset=mnist, epochs=200, batch_size=100, learning_rate=1e-6) optimizer.train(monitor_channels=recon_cost_channel) # Save some reconstruction output images n_examples = 100 xs_test = mnist.test_inputs[:n_examples] noisy_xs_test = gsn.f_noise(xs_test) reconstructed = gsn.run(noisy_xs_test) # Concatenate stuff stacked = numpy.vstack( [numpy.vstack([xs_test[i * 10: (i + 1) * 10], noisy_xs_test[i * 10: (i + 1) * 10], reconstructed[i * 10: (i + 1) * 10]]) for i in range(10)]) number_reconstruction = PIL.Image.fromarray( tile_raster_images(stacked, (gsn.image_height, gsn.image_width), (10, 30)) ) number_reconstruction.save(gsn.outdir + 'reconstruction.png') log.info("saved output image!") # Construct image from the weight matrix image = PIL.Image.fromarray( tile_raster_images( X=gsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(gsn.hidden_size), tile_spacing=(1, 1) ) ) image.save(gsn.outdir + "gsn_mnist_weights.png")
def run_midi(dataset): log.info("Creating RNN-RBM for dataset %s!", dataset) outdir = "outputs/rnnrbm/%s/" % dataset # grab the MIDI dataset if dataset == 'nottingham': midi = Nottingham() elif dataset == 'jsb': midi = JSBChorales() elif dataset == 'muse': midi = MuseData() elif dataset == 'piano_de': midi = PianoMidiDe() else: raise AssertionError("dataset %s not recognized." % dataset) # create the RNN-RBM # rng = numpy.random # rng.seed(0xbeef) # mrg = RandomStreams(seed=rng.randint(1 << 30)) rng = numpy.random.RandomState(1234) mrg = RandomStreams(rng.randint(2 ** 30)) # rnnrbm = RNN_RBM(input_size=88, # hidden_size=150, # rnn_hidden_size=100, # k=15, # weights_init='gaussian', # weights_std=0.01, # rnn_weights_init='gaussian', # rnn_weights_std=0.0001, # rng=rng, # outdir=outdir) rnnrbm = RNN_RBM(input_size=88, hidden_size=150, rnn_hidden_size=100, k=15, weights_init='gaussian', weights_std=0.01, rnn_weights_init='identity', rnn_hidden_activation='relu', # rnn_weights_init='gaussian', # rnn_hidden_activation='tanh', rnn_weights_std=0.0001, mrg=mrg, outdir=outdir) # make an optimizer to train it optimizer = SGD(model=rnnrbm, dataset=midi, n_epoch=200, batch_size=100, minimum_batch_size=2, learning_rate=.001, save_frequency=10, early_stop_length=200, momentum=False, momentum_decay=False, nesterov_momentum=False) optimizer = AdaDelta(model=rnnrbm, dataset=midi, n_epoch=200, batch_size=100, minimum_batch_size=2, # learning_rate=1e-4, learning_rate=1e-6, save_frequency=10, early_stop_length=200) ll = Monitor('pseudo-log', rnnrbm.get_monitors()['pseudo-log'], test=True) mse = Monitor('frame-error', rnnrbm.get_monitors()['mse'], valid=True, test=True) plot = Plot(bokeh_doc_name='rnnrbm_midi_%s' % dataset, monitor_channels=[ll, mse], open_browser=True) # perform training! optimizer.train(plot=plot) # use the generate function! generated, _ = rnnrbm.generate(initial=None, n_steps=200) dt = 0.3 r = (21, 109) midiwrite(outdir + 'rnnrbm_generated_midi.mid', generated, r=r, dt=dt) if has_pylab: extent = (0, dt * len(generated)) + r pylab.figure() pylab.imshow(generated.T, origin='lower', aspect='auto', interpolation='nearest', cmap=pylab.cm.gray_r, extent=extent) pylab.xlabel('time (s)') pylab.ylabel('MIDI note number') pylab.title('generated piano-roll') # Construct image from the weight matrix image = Image.fromarray( tile_raster_images( X=rnnrbm.W.get_value(borrow=True).T, img_shape=closest_to_square_factors(rnnrbm.input_size), tile_shape=closest_to_square_factors(rnnrbm.hidden_size), tile_spacing=(1, 1) ) ) image.save(outdir + 'rnnrbm_midi_weights.png') log.debug("done!") del midi del rnnrbm del optimizer
print "Shapes:" print train_x.shape, train_y.shape print valid_x.shape, valid_y.shape print test_x.shape, test_y.shape print "--------------" print "Example input:" print train_x[0] print "Example label:" print train_y[0] # Show example images - using tile_raster_images helper function from OpenDeep to get 28x28 image from 784 array. input_images = train_x[:25] im = Image.fromarray( tile_raster_images(input_images, img_shape=(28, 28), tile_shape=(1, 25), tile_spacing=(1, 1)) ) im.save("example_mnist_numbers.png") ######### # Model # ######### # Cool, now we know a little about the input data, let's design the MLP to work with it! # An MLP looks like this: input -> hiddens -> output classification # Each stage is just a matrix multiplication with a nonlinear function applied after. # Inputs are matrices where rows are examples and columns are pixels - so create a symbolic Theano matrix. x = T.matrix('xs') # Now let's start building the equation for our MLP! # The first transformation is the input x -> hidden layer h.
def main(): ######################################## # Initialization things with arguments # ######################################## # use these arguments to get results from paper referenced above _train_args = {"n_epoch": 1000, # maximum number of times to run through the dataset "batch_size": 100, # number of examples to process in parallel (minibatch) "minimum_batch_size": 1, # the minimum number of examples for a batch to be considered "save_frequency": 1, # how many epochs between saving parameters "early_stop_threshold": .9995, # multiplier for how much the train cost to improve to not stop early "early_stop_length": 500, # how many epochs to wait to see if the threshold has been reached "learning_rate": .25, # initial learning rate for SGD "lr_decay": 'exponential', # the decay function to use for the learning rate parameter "lr_factor": .995, # by how much to decay the learning rate each epoch "momentum": 0.5, # the parameter momentum amount 'momentum_decay': False, # how to decay the momentum each epoch (if applicable) 'momentum_factor': 0, # by how much to decay the momentum (in this case not at all) 'nesterov_momentum': False, # whether to use nesterov momentum update (accelerated momentum) } config_root_logger() log.info("Creating a new GSN") mnist = MNIST(concat_train_valid=True) gsn = GSN(layers=2, walkbacks=4, hidden_size=1500, visible_activation='sigmoid', hidden_activation='tanh', input_size=28*28, tied_weights=True, hidden_add_noise_sigma=2, input_salt_and_pepper=0.4, outdir='outputs/test_gsn/', vis_init=False, noiseless_h1=True, input_sampling=True, weights_init='uniform', weights_interval='montreal', bias_init=0, cost_function='binary_crossentropy') recon_cost_channel = MonitorsChannel(name='cost') recon_cost_channel.add(Monitor('recon_cost', gsn.get_monitors()['recon_cost'], test=True)) recon_cost_channel.add(Monitor('noisy_recon_cost', gsn.get_monitors()['noisy_recon_cost'], test=True)) # Load initial weights and biases from file # params_to_load = '../../../outputs/gsn/mnist/trained_epoch_395.pkl' # gsn.load_params(params_to_load) optimizer = SGD(model=gsn, dataset=mnist, **_train_args) # optimizer = AdaDelta(model=gsn, dataset=mnist, n_epoch=200, batch_size=100, learning_rate=1e-6) optimizer.train(monitor_channels=recon_cost_channel) # Save some reconstruction output images import opendeep.data.dataset as datasets n_examples = 100 xs_test, _ = mnist.getSubset(datasets.TEST) xs_test = xs_test[:n_examples].eval() noisy_xs_test = gsn.f_noise(xs_test) reconstructed = gsn.run(noisy_xs_test) # Concatenate stuff stacked = numpy.vstack( [numpy.vstack([xs_test[i * 10: (i + 1) * 10], noisy_xs_test[i * 10: (i + 1) * 10], reconstructed[i * 10: (i + 1) * 10]]) for i in range(10)]) number_reconstruction = PIL.Image.fromarray( tile_raster_images(stacked, (gsn.image_height, gsn.image_width), (10, 30)) ) number_reconstruction.save(gsn.outdir + 'reconstruction.png') log.info("saved output image!") # Construct image from the weight matrix image = PIL.Image.fromarray( tile_raster_images( X=gsn.weights_list[0].get_value(borrow=True).T, img_shape=(28, 28), tile_shape=closest_to_square_factors(gsn.hidden_size), tile_spacing=(1, 1) ) ) image.save(gsn.outdir + "gsn_mnist_weights.png")