def create_reconstruction_image(self, input_data):
"""
Adds noise to an input and saves an image from the reconstruction running the input through the computation
graph.
"""
n_examples = len(input_data)
xs_test = input_data
noisy_xs_test = self.f_noise(input_data)
reconstructed = self.run(noisy_xs_test)
# Concatenate stuff
width, height = closest_to_square_factors(n_examples)
stacked = numpy.vstack([
numpy.vstack([
xs_test[i * width:(i + 1) * width],
noisy_xs_test[i * width:(i + 1) * width],
reconstructed[i * width:(i + 1) * width]
]) for i in range(height)
])
number_reconstruction = PIL.Image.fromarray(
tile_raster_images(stacked, (self.image_height, self.image_width),
(height, 3 * width)))
save_path = os.path.join(self.outdir, 'gsn_reconstruction.png')
save_path = os.path.realpath(save_path)
number_reconstruction.save(save_path)
log.info("saved output image to %s", save_path)
def create_reconstruction_image(self, input_data):
"""
Adds noise to an input and saves an image from the reconstruction running the input through the computation
graph.
"""
n_examples = len(input_data)
xs_test = input_data
noisy_xs_test = self.f_noise(input_data)
reconstructed = self.run(noisy_xs_test)
# Concatenate stuff
width, height = closest_to_square_factors(n_examples)
stacked = numpy.vstack(
[numpy.vstack([xs_test[i * width: (i + 1) * width],
noisy_xs_test[i * width: (i + 1) * width],
reconstructed[i * width: (i + 1) * width]])
for i in range(height)])
number_reconstruction = PIL.Image.fromarray(
tile_raster_images(stacked, (self.image_height, self.image_width), (height, 3*width))
)
save_path = os.path.join(self.outdir, 'gsn_reconstruction.png')
save_path = os.path.realpath(save_path)
number_reconstruction.save(save_path)
log.info("saved output image to %s", save_path)
dae = DenoisingAutoencoder()
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=dae, dataset=mnist)
# perform training!
optimizer.train()
# test it on some images!
test_data, _ = mnist.getSubset(TEST)
test_data = test_data[:25].eval()
corrupted_test = salt_and_pepper(test_data, 0.4).eval()
# use the run function!
reconstructed_images = dae.run(corrupted_test)
# create an image from this reconstruction!
# imports for working with tiling outputs into one image
from opendeep.utils.image import tile_raster_images
import numpy
import PIL
# stack the image matrices together in three 5x5 grids next to each other using numpy
stacked = numpy.vstack([
numpy.vstack([
test_data[i * 5:(i + 1) * 5], corrupted_test[i * 5:(i + 1) * 5],
reconstructed_images[i * 5:(i + 1) * 5]
]) for i in range(5)
])
# convert the combined matrix into an image
image = PIL.Image.fromarray(
tile_raster_images(stacked, (28, 28), (5, 3 * 5)))
# save it!
image.save("dae_reconstruction_test.png")
def run_audio(dataset):
log.info("Creating RNN-GSN for dataset %s!", dataset)
outdir = "outputs/rnngsn/%s/" % dataset
# grab the dataset
if dataset == 'tedlium':
dataset = tedlium.TEDLIUMDataset(max_speeches=3)
extra_args = {
}
elif dataset == 'codegolf':
dataset = codegolf.CodeGolfDataset()
extra_args = {
}
else:
raise ValueError("dataset %s not recognized." % dataset)
rng = numpy.random.RandomState(1234)
mrg = RandomStreams(rng.randint(2 ** 30))
assert dataset.window_size == 256, dataset.window_size
rnngsn = RNN_GSN(layers=2,
walkbacks=4,
input_size=dataset.window_size,
image_height = 1,
image_width = 256,
hidden_size=128,
rnn_hidden_size=128,
weights_init='gaussian',
weights_std=0.01,
rnn_weights_init='identity',
rnn_hidden_activation='relu',
rnn_weights_std=0.0001,
mrg=mrg,
outdir=outdir, **extra_args)
# make an optimizer to train it
optimizer = AdaDelta(model=rnngsn,
dataset=dataset,
epochs=200,
batch_size=128,
min_batch_size=2,
learning_rate=1e-6,
save_freq=1,
stop_patience=100)
ll = Monitor('crossentropy', rnngsn.get_monitors()['noisy_recon_cost'],test=True)
mse = Monitor('frame-error', rnngsn.get_monitors()['mse'],train=True,test=True,valid=True)
plot = Plot(
bokeh_doc_name='rnngsn_tedlium_%s'%dataset, monitor_channels=[ll,mse],open_browser=True
)
# perform training!
optimizer.train(plot=plot)
# use the generate function!
generated, _ = rnngsn.generate(initial=None, n_steps=200)
# Construct image from the weight matrix
image = Image.fromarray(
tile_raster_images(
X=rnngsn.weights_list[0].get_value(borrow=True).T,
img_shape=closest_to_square_factors(rnngsn.input_size),
tile_shape=closest_to_square_factors(rnngsn.hidden_size),
tile_spacing=(1, 1)
)
)
image.save(outdir + 'rnngsn_%s_weights.png'%(dataset,))
log.debug("done!")
del rnngsn
del optimizer
if has_pylab:
pylab.show()
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=dae, dataset=mnist, epochs=100)
# perform training!
optimizer.train()
# test it on some images!
test_data = mnist.test_inputs[:25]
corrupted_test = salt_and_pepper(test_data, 0.4).eval()
# use the run function!
reconstructed_images = dae.run(corrupted_test)
# create an image from this reconstruction!
# imports for working with tiling outputs into one image
from opendeep.utils.image import tile_raster_images
import numpy
import PIL
# stack the image matrices together in three 5x5 grids next to each other using numpy
stacked = numpy.vstack(
[numpy.vstack([test_data[i*5 : (i+1)*5],
corrupted_test[i*5 : (i+1)*5],
reconstructed_images[i*5 : (i+1)*5]])
for i in range(5)])
# convert the combined matrix into an image
image = PIL.Image.fromarray(
tile_raster_images(stacked, (28, 28), (5, 3*5))
)
# save it!
image.save("dae_reconstruction_test.png")
def run_sequence(sequence=0):
log.info("Creating RNN-GSN for sequence %d!" % sequence)
# grab the MNIST dataset
mnist = MNIST(sequence_number=sequence, concat_train_valid=True)
outdir = "outputs/rnngsn/mnist_%d/" % sequence
rng = numpy.random.RandomState(1234)
mrg = RandomStreams(rng.randint(2**30))
rnngsn = RNN_GSN(layers=2,
walkbacks=4,
input_size=28 * 28,
hidden_size=1000,
tied_weights=True,
rnn_hidden_size=100,
weights_init='uniform',
weights_interval='montreal',
rnn_weights_init='identity',
mrg=mrg,
outdir=outdir)
# load pretrained rbm on mnist
# rnngsn.load_gsn_params('outputs/trained_gsn_epoch_1000.pkl')
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=rnngsn,
dataset=mnist,
n_epoch=200,
batch_size=100,
minimum_batch_size=2,
learning_rate=1e-6,
save_frequency=1,
early_stop_length=200)
# optimizer = SGD(model=rnngsn,
# dataset=mnist,
# n_epoch=300,
# batch_size=100,
# minimum_batch_size=2,
# learning_rate=.25,
# lr_decay='exponential',
# lr_factor=.995,
# momentum=0.5,
# nesterov_momentum=True,
# momentum_decay=False,
# save_frequency=20,
# early_stop_length=100)
crossentropy = Monitor('crossentropy',
rnngsn.get_monitors()['noisy_recon_cost'],
test=True)
error = Monitor('error', rnngsn.get_monitors()['mse'], test=True)
# perform training!
optimizer.train(monitor_channels=[crossentropy, error])
# use the generate function!
log.debug("generating images...")
generated, ut = rnngsn.generate(initial=None, n_steps=400)
# Construct image
image = Image.fromarray(
tile_raster_images(X=generated,
img_shape=(28, 28),
tile_shape=(20, 20),
tile_spacing=(1, 1)))
image.save(outdir + "rnngsn_mnist_generated.png")
log.debug('saved generated.png')
# Construct image from the weight matrix
image = Image.fromarray(
tile_raster_images(X=rnngsn.weights_list[0].get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=closest_to_square_factors(
rnngsn.hidden_size),
tile_spacing=(1, 1)))
image.save(outdir + "rnngsn_mnist_weights.png")
log.debug("done!")
del mnist
del rnngsn
del optimizer
def run_sequence(sequence=0):
log.info("Creating RNN-GSN for sequence %d!" % sequence)
# grab the MNIST dataset
mnist = MNIST(sequence_number=sequence, concat_train_valid=True)
outdir = "outputs/rnngsn/mnist_%d/" % sequence
rng = numpy.random.RandomState(1234)
mrg = RandomStreams(rng.randint(2 ** 30))
rnngsn = RNN_GSN(layers=2,
walkbacks=4,
input_size=28 * 28,
hidden_size=1000,
tied_weights=True,
rnn_hidden_size=100,
weights_init='uniform',
weights_interval='montreal',
rnn_weights_init='identity',
mrg=mrg,
outdir=outdir)
# load pretrained rbm on mnist
# rnngsn.load_gsn_params('outputs/trained_gsn_epoch_1000.pkl')
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=rnngsn,
dataset=mnist,
n_epoch=200,
batch_size=100,
minimum_batch_size=2,
learning_rate=1e-6,
save_frequency=1,
early_stop_length=200)
# optimizer = SGD(model=rnngsn,
# dataset=mnist,
# n_epoch=300,
# batch_size=100,
# minimum_batch_size=2,
# learning_rate=.25,
# lr_decay='exponential',
# lr_factor=.995,
# momentum=0.5,
# nesterov_momentum=True,
# momentum_decay=False,
# save_frequency=20,
# early_stop_length=100)
crossentropy = Monitor('crossentropy', rnngsn.get_monitors()['noisy_recon_cost'], test=True)
error = Monitor('error', rnngsn.get_monitors()['mse'], test=True)
# perform training!
optimizer.train(monitor_channels=[crossentropy, error])
# use the generate function!
log.debug("generating images...")
generated, ut = rnngsn.generate(initial=None, n_steps=400)
# Construct image
image = Image.fromarray(
tile_raster_images(
X=generated,
img_shape=(28, 28),
tile_shape=(20, 20),
tile_spacing=(1, 1)
)
)
image.save(outdir + "rnngsn_mnist_generated.png")
log.debug('saved generated.png')
# Construct image from the weight matrix
image = Image.fromarray(
tile_raster_images(
X=rnngsn.weights_list[0].get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=closest_to_square_factors(rnngsn.hidden_size),
tile_spacing=(1, 1)
)
)
image.save(outdir + "rnngsn_mnist_weights.png")
log.debug("done!")
del mnist
del rnngsn
del optimizer
rbm = RBM(**config_args)
# rbm.load_params('rbm_trained.pkl')
optimizer = Optimizer(learning_rate=0.1, model=rbm, dataset=mnist, batch_size=20, epochs=15)
ll = Monitor("pseudo-log", rbm.get_monitors()["pseudo-log"])
# perform training!
optimizer.train(monitor_channels=ll)
# test it on some images!
test_data = mnist.test_inputs[:25]
# use the run function!
preds = rbm.run(test_data)
# Construct image from the test matrix
image = Image.fromarray(tile_raster_images(X=test_data, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1)))
image.save("rbm_test.png")
# Construct image from the preds matrix
image = Image.fromarray(tile_raster_images(X=preds, img_shape=(28, 28), tile_shape=(5, 5), tile_spacing=(1, 1)))
image.save("rbm_preds.png")
# Construct image from the weight matrix
image = Image.fromarray(
tile_raster_images(
X=rbm.W.get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=closest_to_square_factors(config_args["hidden_size"]),
tile_spacing=(1, 1),
)
)
ll = Monitor('pseudo-log', rbm.get_monitors()['pseudo-log'])
# perform training!
optimizer.train(monitor_channels=ll)
# test it on some images!
test_data = mnist.getSubset(TEST)[0]
test_data = test_data[:25].eval()
# use the run function!
preds = rbm.run(test_data)
# Construct image from the test matrix
image = Image.fromarray(
tile_raster_images(
X=test_data,
img_shape=(28, 28),
tile_shape=(5, 5),
tile_spacing=(1, 1)
)
)
image.save('rbm_test.png')
# Construct image from the preds matrix
image = Image.fromarray(
tile_raster_images(
X=preds,
img_shape=(28, 28),
tile_shape=(5, 5),
tile_spacing=(1, 1)
)
)
image.save('rbm_preds.png')
def run_midi(dataset):
log.info("Creating RNN-RBM for dataset %s!", dataset)
outdir = "outputs/rnnrbm/%s/" % dataset
# grab the MIDI dataset
if dataset == 'nottingham':
midi = Nottingham()
elif dataset == 'jsb':
midi = JSBChorales()
elif dataset == 'muse':
midi = MuseData()
elif dataset == 'piano_de':
midi = PianoMidiDe()
else:
raise AssertionError("dataset %s not recognized." % dataset)
# create the RNN-RBM
# rng = numpy.random
# rng.seed(0xbeef)
# mrg = RandomStreams(seed=rng.randint(1 << 30))
rng = numpy.random.RandomState(1234)
mrg = RandomStreams(rng.randint(2**30))
# rnnrbm = RNN_RBM(input_size=88,
# hidden_size=150,
# rnn_hidden_size=100,
# k=15,
# weights_init='gaussian',
# weights_std=0.01,
# rnn_weights_init='gaussian',
# rnn_weights_std=0.0001,
# rng=rng,
# outdir=outdir)
rnnrbm = RNN_RBM(
input_size=88,
hidden_size=150,
rnn_hidden_size=100,
k=15,
weights_init='gaussian',
weights_std=0.01,
rnn_weights_init='identity',
rnn_hidden_activation='relu',
# rnn_weights_init='gaussian',
# rnn_hidden_activation='tanh',
rnn_weights_std=0.0001,
mrg=mrg,
outdir=outdir)
# make an optimizer to train it
optimizer = SGD(model=rnnrbm,
dataset=midi,
epochs=200,
batch_size=100,
min_batch_size=2,
learning_rate=.001,
save_freq=10,
stop_patience=200,
momentum=False,
momentum_decay=False,
nesterov_momentum=False)
optimizer = AdaDelta(
model=rnnrbm,
dataset=midi,
epochs=200,
batch_size=100,
min_batch_size=2,
# learning_rate=1e-4,
learning_rate=1e-6,
save_freq=10,
stop_patience=200)
ll = Monitor('pseudo-log', rnnrbm.get_monitors()['pseudo-log'], test=True)
mse = Monitor('frame-error',
rnnrbm.get_monitors()['mse'],
valid=True,
test=True)
plot = Plot(bokeh_doc_name='rnnrbm_midi_%s' % dataset,
monitor_channels=[ll, mse],
open_browser=True)
# perform training!
optimizer.train(plot=plot)
# use the generate function!
generated, _ = rnnrbm.generate(initial=None, n_steps=200)
dt = 0.3
r = (21, 109)
midiwrite(outdir + 'rnnrbm_generated_midi.mid', generated, r=r, dt=dt)
if has_pylab:
extent = (0, dt * len(generated)) + r
pylab.figure()
pylab.imshow(generated.T,
origin='lower',
aspect='auto',
interpolation='nearest',
cmap=pylab.cm.gray_r,
extent=extent)
pylab.xlabel('time (s)')
pylab.ylabel('MIDI note number')
pylab.title('generated piano-roll')
# Construct image from the weight matrix
image = Image.fromarray(
tile_raster_images(
X=rnnrbm.W.get_value(borrow=True).T,
img_shape=closest_to_square_factors(rnnrbm.input_size),
tile_shape=closest_to_square_factors(rnnrbm.hidden_size),
tile_spacing=(1, 1)))
image.save(outdir + 'rnnrbm_midi_weights.png')
log.debug("done!")
del midi
del rnnrbm
del optimizer
print "Shapes:"
print train_x.shape, train_y.shape
print valid_x.shape, valid_y.shape
print test_x.shape, test_y.shape
print "--------------"
print "Example input:"
print train_x[0]
print "Example label:"
print train_y[0]
# Show example images - using tile_raster_images helper function from OpenDeep to get 28x28 image from 784 array.
input_images = train_x[:25]
im = Image.fromarray(
tile_raster_images(input_images,
img_shape=(28, 28),
tile_shape=(1, 25),
tile_spacing=(1, 1)))
im.save("example_mnist_numbers.png")
#########
# Model #
#########
# Cool, now we know a little about the input data, let's design the MLP to work with it!
# An MLP looks like this: input -> hiddens -> output classification
# Each stage is just a matrix multiplication with a nonlinear function applied after.
# Inputs are matrices where rows are examples and columns are pixels - so create a symbolic Theano matrix.
x = T.matrix('xs')
# Now let's start building the equation for our MLP!
# The first transformation is the input x -> hidden layer h.
# We defined this transformation with h = tanh(x.dot(W_x) + b_h)
def run_sequence(sequence=0):
log.info("Creating RNN-RBM for sequence %d!" % sequence)
# grab the MNIST dataset
mnist = MNIST(sequence_number=sequence, concat_train_valid=True)
outdir = "outputs/rnnrbm/mnist_%d/" % sequence
# create the RNN-RBM
rng = numpy.random.RandomState(1234)
mrg = RandomStreams(rng.randint(2 ** 30))
rnnrbm = RNN_RBM(input_size=28 * 28,
hidden_size=1000,
rnn_hidden_size=100,
k=15,
weights_init='uniform',
weights_interval=4 * numpy.sqrt(6. / (28 * 28 + 500)),
rnn_weights_init='identity',
rnn_hidden_activation='relu',
rnn_weights_std=1e-4,
mrg=mrg,
outdir=outdir)
# load pretrained rbm on mnist
# rnnrbm.load_params(outdir + 'trained_epoch_200.pkl')
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=rnnrbm,
dataset=mnist,
n_epoch=200,
batch_size=100,
minimum_batch_size=2,
learning_rate=1e-8,
save_frequency=10,
early_stop_length=200)
crossentropy = Monitor('crossentropy', rnnrbm.get_monitors()['crossentropy'], test=True)
error = Monitor('error', rnnrbm.get_monitors()['mse'], test=True)
plot = Plot(bokeh_doc_name='rnnrbm_mnist_%d' % sequence, monitor_channels=[crossentropy, error], open_browser=True)
# perform training!
optimizer.train(plot=plot)
# use the generate function!
log.debug("generating images...")
generated, ut = rnnrbm.generate(initial=None, n_steps=400)
# Construct image
image = Image.fromarray(
tile_raster_images(
X=generated,
img_shape=(28, 28),
tile_shape=(20, 20),
tile_spacing=(1, 1)
)
)
image.save(outdir + "rnnrbm_mnist_generated.png")
log.debug('saved generated.png')
# Construct image from the weight matrix
image = Image.fromarray(
tile_raster_images(
X=rnnrbm.W.get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=closest_to_square_factors(rnnrbm.hidden_size),
tile_spacing=(1, 1)
)
)
image.save(outdir + "rnnrbm_mnist_weights.png")
log.debug("done!")
del mnist
del rnnrbm
del optimizer
def run_sequence(sequence=0):
log.info("Creating RNN-RBM for sequence %d!" % sequence)
# grab the MNIST dataset
mnist = MNIST(sequence_number=sequence, concat_train_valid=True)
outdir = "outputs/rnnrbm/mnist_%d/" % sequence
# create the RNN-RBM
rng = numpy.random.RandomState(1234)
mrg = RandomStreams(rng.randint(2**30))
rnnrbm = RNN_RBM(input_size=28 * 28,
hidden_size=1000,
rnn_hidden_size=100,
k=15,
weights_init='uniform',
weights_interval=4 * numpy.sqrt(6. / (28 * 28 + 500)),
rnn_weights_init='identity',
rnn_hidden_activation='relu',
rnn_weights_std=1e-4,
mrg=mrg,
outdir=outdir)
# load pretrained rbm on mnist
# rnnrbm.load_params(outdir + 'trained_epoch_200.pkl')
# make an optimizer to train it (AdaDelta is a good default)
optimizer = AdaDelta(model=rnnrbm,
dataset=mnist,
n_epoch=200,
batch_size=100,
minimum_batch_size=2,
learning_rate=1e-8,
save_frequency=10,
early_stop_length=200)
crossentropy = Monitor('crossentropy',
rnnrbm.get_monitors()['crossentropy'],
test=True)
error = Monitor('error', rnnrbm.get_monitors()['mse'], test=True)
plot = Plot(bokeh_doc_name='rnnrbm_mnist_%d' % sequence,
monitor_channels=[crossentropy, error],
open_browser=True)
# perform training!
optimizer.train(plot=plot)
# use the generate function!
log.debug("generating images...")
generated, ut = rnnrbm.generate(initial=None, n_steps=400)
# Construct image
image = Image.fromarray(
tile_raster_images(X=generated,
img_shape=(28, 28),
tile_shape=(20, 20),
tile_spacing=(1, 1)))
image.save(outdir + "rnnrbm_mnist_generated.png")
log.debug('saved generated.png')
# Construct image from the weight matrix
image = Image.fromarray(
tile_raster_images(X=rnnrbm.W.get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=closest_to_square_factors(
rnnrbm.hidden_size),
tile_spacing=(1, 1)))
image.save(outdir + "rnnrbm_mnist_weights.png")
log.debug("done!")
del mnist
del rnnrbm
del optimizer
# test it on some images!
test_data, _ = mnist.getSubset(TEST)
test_data = test_data[:25].eval()
corrupted_test = salt_and_pepper(test_data, 0.4).eval()
# use the run function!
reconstructed_images = dae.run(corrupted_test)
# create an image from this reconstruction!
# imports for working with tiling outputs into one image
from opendeep.utils.image import tile_raster_images
import numpy
import PIL
# stack the image matrices together in three 5x5 grids next to each other using numpy
stacked = numpy.vstack(
[
numpy.vstack(
[
test_data[i * 5 : (i + 1) * 5],
corrupted_test[i * 5 : (i + 1) * 5],
reconstructed_images[i * 5 : (i + 1) * 5],
]
)
for i in range(5)
]
)
# convert the combined matrix into an image
image = PIL.Image.fromarray(tile_raster_images(stacked, (28, 28), (5, 3 * 5)))
# save it!
image.save("dae_reconstruction_test.png")
def main():
########################################
# Initialization things with arguments #
########################################
# use these arguments to get results from paper referenced above
_train_args = {"epochs": 1000, # maximum number of times to run through the dataset
"batch_size": 100, # number of examples to process in parallel (minibatch)
"min_batch_size": 1, # the minimum number of examples for a batch to be considered
"save_freq": 1, # how many epochs between saving parameters
"stop_threshold": .9995, # multiplier for how much the train cost to improve to not stop early
"stop_patience": 500, # how many epochs to wait to see if the threshold has been reached
"learning_rate": .25, # initial learning rate for SGD
"lr_decay": 'exponential', # the decay function to use for the learning rate parameter
"lr_decay_factor": .995, # by how much to decay the learning rate each epoch
"momentum": 0.5, # the parameter momentum amount
'momentum_decay': False, # how to decay the momentum each epoch (if applicable)
'momentum_factor': 0, # by how much to decay the momentum (in this case not at all)
'nesterov_momentum': False, # whether to use nesterov momentum update (accelerated momentum)
}
config_root_logger()
log.info("Creating a new GSN")
mnist = MNIST(concat_train_valid=True)
gsn = GSN(layers=2,
walkbacks=4,
hidden_size=1500,
visible_activation='sigmoid',
hidden_activation='tanh',
input_size=28*28,
tied_weights=True,
hidden_add_noise_sigma=2,
input_salt_and_pepper=0.4,
outdir='outputs/test_gsn/',
vis_init=False,
noiseless_h1=True,
input_sampling=True,
weights_init='uniform',
weights_interval='montreal',
bias_init=0,
cost_function='binary_crossentropy')
recon_cost_channel = MonitorsChannel(name='cost')
recon_cost_channel.add(Monitor('recon_cost', gsn.get_monitors()['recon_cost'], test=True))
recon_cost_channel.add(Monitor('noisy_recon_cost', gsn.get_monitors()['noisy_recon_cost'], test=True))
# Load initial weights and biases from file
# params_to_load = '../../../outputs/gsn/mnist/trained_epoch_395.pkl'
# gsn.load_params(params_to_load)
optimizer = SGD(model=gsn, dataset=mnist, **_train_args)
# optimizer = AdaDelta(model=gsn, dataset=mnist, epochs=200, batch_size=100, learning_rate=1e-6)
optimizer.train(monitor_channels=recon_cost_channel)
# Save some reconstruction output images
n_examples = 100
xs_test = mnist.test_inputs[:n_examples]
noisy_xs_test = gsn.f_noise(xs_test)
reconstructed = gsn.run(noisy_xs_test)
# Concatenate stuff
stacked = numpy.vstack(
[numpy.vstack([xs_test[i * 10: (i + 1) * 10],
noisy_xs_test[i * 10: (i + 1) * 10],
reconstructed[i * 10: (i + 1) * 10]])
for i in range(10)])
number_reconstruction = PIL.Image.fromarray(
tile_raster_images(stacked, (gsn.image_height, gsn.image_width), (10, 30))
)
number_reconstruction.save(gsn.outdir + 'reconstruction.png')
log.info("saved output image!")
# Construct image from the weight matrix
image = PIL.Image.fromarray(
tile_raster_images(
X=gsn.weights_list[0].get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=closest_to_square_factors(gsn.hidden_size),
tile_spacing=(1, 1)
)
)
image.save(gsn.outdir + "gsn_mnist_weights.png")
def run_midi(dataset):
log.info("Creating RNN-RBM for dataset %s!", dataset)
outdir = "outputs/rnnrbm/%s/" % dataset
# grab the MIDI dataset
if dataset == 'nottingham':
midi = Nottingham()
elif dataset == 'jsb':
midi = JSBChorales()
elif dataset == 'muse':
midi = MuseData()
elif dataset == 'piano_de':
midi = PianoMidiDe()
else:
raise AssertionError("dataset %s not recognized." % dataset)
# create the RNN-RBM
# rng = numpy.random
# rng.seed(0xbeef)
# mrg = RandomStreams(seed=rng.randint(1 << 30))
rng = numpy.random.RandomState(1234)
mrg = RandomStreams(rng.randint(2 ** 30))
# rnnrbm = RNN_RBM(input_size=88,
# hidden_size=150,
# rnn_hidden_size=100,
# k=15,
# weights_init='gaussian',
# weights_std=0.01,
# rnn_weights_init='gaussian',
# rnn_weights_std=0.0001,
# rng=rng,
# outdir=outdir)
rnnrbm = RNN_RBM(input_size=88,
hidden_size=150,
rnn_hidden_size=100,
k=15,
weights_init='gaussian',
weights_std=0.01,
rnn_weights_init='identity',
rnn_hidden_activation='relu',
# rnn_weights_init='gaussian',
# rnn_hidden_activation='tanh',
rnn_weights_std=0.0001,
mrg=mrg,
outdir=outdir)
# make an optimizer to train it
optimizer = SGD(model=rnnrbm,
dataset=midi,
n_epoch=200,
batch_size=100,
minimum_batch_size=2,
learning_rate=.001,
save_frequency=10,
early_stop_length=200,
momentum=False,
momentum_decay=False,
nesterov_momentum=False)
optimizer = AdaDelta(model=rnnrbm,
dataset=midi,
n_epoch=200,
batch_size=100,
minimum_batch_size=2,
# learning_rate=1e-4,
learning_rate=1e-6,
save_frequency=10,
early_stop_length=200)
ll = Monitor('pseudo-log', rnnrbm.get_monitors()['pseudo-log'], test=True)
mse = Monitor('frame-error', rnnrbm.get_monitors()['mse'], valid=True, test=True)
plot = Plot(bokeh_doc_name='rnnrbm_midi_%s' % dataset, monitor_channels=[ll, mse], open_browser=True)
# perform training!
optimizer.train(plot=plot)
# use the generate function!
generated, _ = rnnrbm.generate(initial=None, n_steps=200)
dt = 0.3
r = (21, 109)
midiwrite(outdir + 'rnnrbm_generated_midi.mid', generated, r=r, dt=dt)
if has_pylab:
extent = (0, dt * len(generated)) + r
pylab.figure()
pylab.imshow(generated.T, origin='lower', aspect='auto',
interpolation='nearest', cmap=pylab.cm.gray_r,
extent=extent)
pylab.xlabel('time (s)')
pylab.ylabel('MIDI note number')
pylab.title('generated piano-roll')
# Construct image from the weight matrix
image = Image.fromarray(
tile_raster_images(
X=rnnrbm.W.get_value(borrow=True).T,
img_shape=closest_to_square_factors(rnnrbm.input_size),
tile_shape=closest_to_square_factors(rnnrbm.hidden_size),
tile_spacing=(1, 1)
)
)
image.save(outdir + 'rnnrbm_midi_weights.png')
log.debug("done!")
del midi
del rnnrbm
del optimizer
print "Shapes:"
print train_x.shape, train_y.shape
print valid_x.shape, valid_y.shape
print test_x.shape, test_y.shape
print "--------------"
print "Example input:"
print train_x[0]
print "Example label:"
print train_y[0]
# Show example images - using tile_raster_images helper function from OpenDeep to get 28x28 image from 784 array.
input_images = train_x[:25]
im = Image.fromarray(
tile_raster_images(input_images,
img_shape=(28, 28),
tile_shape=(1, 25),
tile_spacing=(1, 1))
)
im.save("example_mnist_numbers.png")
#########
# Model #
#########
# Cool, now we know a little about the input data, let's design the MLP to work with it!
# An MLP looks like this: input -> hiddens -> output classification
# Each stage is just a matrix multiplication with a nonlinear function applied after.
# Inputs are matrices where rows are examples and columns are pixels - so create a symbolic Theano matrix.
x = T.matrix('xs')
# Now let's start building the equation for our MLP!
# The first transformation is the input x -> hidden layer h.
def main():
########################################
# Initialization things with arguments #
########################################
# use these arguments to get results from paper referenced above
_train_args = {"n_epoch": 1000, # maximum number of times to run through the dataset
"batch_size": 100, # number of examples to process in parallel (minibatch)
"minimum_batch_size": 1, # the minimum number of examples for a batch to be considered
"save_frequency": 1, # how many epochs between saving parameters
"early_stop_threshold": .9995, # multiplier for how much the train cost to improve to not stop early
"early_stop_length": 500, # how many epochs to wait to see if the threshold has been reached
"learning_rate": .25, # initial learning rate for SGD
"lr_decay": 'exponential', # the decay function to use for the learning rate parameter
"lr_factor": .995, # by how much to decay the learning rate each epoch
"momentum": 0.5, # the parameter momentum amount
'momentum_decay': False, # how to decay the momentum each epoch (if applicable)
'momentum_factor': 0, # by how much to decay the momentum (in this case not at all)
'nesterov_momentum': False, # whether to use nesterov momentum update (accelerated momentum)
}
config_root_logger()
log.info("Creating a new GSN")
mnist = MNIST(concat_train_valid=True)
gsn = GSN(layers=2,
walkbacks=4,
hidden_size=1500,
visible_activation='sigmoid',
hidden_activation='tanh',
input_size=28*28,
tied_weights=True,
hidden_add_noise_sigma=2,
input_salt_and_pepper=0.4,
outdir='outputs/test_gsn/',
vis_init=False,
noiseless_h1=True,
input_sampling=True,
weights_init='uniform',
weights_interval='montreal',
bias_init=0,
cost_function='binary_crossentropy')
recon_cost_channel = MonitorsChannel(name='cost')
recon_cost_channel.add(Monitor('recon_cost', gsn.get_monitors()['recon_cost'], test=True))
recon_cost_channel.add(Monitor('noisy_recon_cost', gsn.get_monitors()['noisy_recon_cost'], test=True))
# Load initial weights and biases from file
# params_to_load = '../../../outputs/gsn/mnist/trained_epoch_395.pkl'
# gsn.load_params(params_to_load)
optimizer = SGD(model=gsn, dataset=mnist, **_train_args)
# optimizer = AdaDelta(model=gsn, dataset=mnist, n_epoch=200, batch_size=100, learning_rate=1e-6)
optimizer.train(monitor_channels=recon_cost_channel)
# Save some reconstruction output images
import opendeep.data.dataset as datasets
n_examples = 100
xs_test, _ = mnist.getSubset(datasets.TEST)
xs_test = xs_test[:n_examples].eval()
noisy_xs_test = gsn.f_noise(xs_test)
reconstructed = gsn.run(noisy_xs_test)
# Concatenate stuff
stacked = numpy.vstack(
[numpy.vstack([xs_test[i * 10: (i + 1) * 10],
noisy_xs_test[i * 10: (i + 1) * 10],
reconstructed[i * 10: (i + 1) * 10]])
for i in range(10)])
number_reconstruction = PIL.Image.fromarray(
tile_raster_images(stacked, (gsn.image_height, gsn.image_width), (10, 30))
)
number_reconstruction.save(gsn.outdir + 'reconstruction.png')
log.info("saved output image!")
# Construct image from the weight matrix
image = PIL.Image.fromarray(
tile_raster_images(
X=gsn.weights_list[0].get_value(borrow=True).T,
img_shape=(28, 28),
tile_shape=closest_to_square_factors(gsn.hidden_size),
tile_spacing=(1, 1)
)
)
image.save(gsn.outdir + "gsn_mnist_weights.png")
