def __init__(self, dimensions, noise=0.1, reconstruction_penalty=1.0,
sparsity_penalty=1.0, embedding_penalty=1.0,
learning_rate=0.01, seed=None):
"""Initialize a stack of autoencoders with sparsity penalty
dimensions is a python sequence of the input, hidden and output
activation unit of the stacked architecture.
TODO: implement fine-tuning by applying SGD on the encoder using a
divergence measure on the pairwise similarities in input and output
space as object function to minimize. E.g.: (t-)SNE or Elastic
Embedding.
"""
assert len(dimensions) >= 2
self.rng = np.random.RandomState(seed)
self.noise_rng = RandomStreams(seed)
# build a stack of autoencoders for the requested dimensions
self.autoencoders = []
previous_output = T.matrix('ae_in')
for in_dim, out_dim in zip(dimensions[:-1], dimensions[1:]):
ae = Autoencoder(in_dim, out_dim, tied=True, noise=noise,
rng=self.rng, noise_rng=self.noise_rng)
ae.build(previous_output)
previous_output = ae.output
self.autoencoders.append(ae)
# chain the encoding parts as a feed forward network
self.encoder = NNet(self.autoencoders, errors.mse)
self.encoder.build(T.matrix('enc_in'), T.vector('enc_target'))
# compile the training functions
self.reconstruction_penalty = reconstruction_penalty
self.sparsity_penalty = sparsity_penalty
self.embedding_penalty = embedding_penalty
self.pre_trainers = []
for ae in self.autoencoders:
cost = self.get_ae_cost(ae)
pre_train = theano.function(
[self.autoencoders[0].input],
cost,
updates=get_updates(ae.pre_params, cost, learning_rate)
)
self.pre_trainers.append(pre_train)
# compile the enconding function
self.encode = theano.function([self.encoder.input], self.encoder.output)
def fine_tune(self, data, batch_size=50, epochs=100, learning_rate=0.1,
checkpoint=10, patience=20, tolerance=1e-5):
"""Use SGD to optimize the embedding computed by the encoder stack"""
data = np.atleast_2d(data)
data = np.asanyarray(data, dtype=theano.config.floatX)
n_samples, n_features = data.shape
best_error = None
best_epoch = 0
n_batches = n_samples / batch_size
cost = self.get_embedding_cost(self.autoencoders[-1])
tuner = theano.function(
[self.autoencoders[0].input],
cost,
updates=get_updates(self.encoder.params, cost, learning_rate)
)
shuffled = data.copy()
for e in xrange(epochs):
# reshuffling data to enforce I.I.D. assumption
self.rng.shuffle(shuffled)
err = np.zeros(n_batches)
for b in xrange(n_batches):
batch_input = shuffled[b * batch_size:(b + 1) * batch_size]
err[b] = tuner(batch_input).mean()
error = err.mean()
if e % checkpoint == 0:
print "fine tune: epoch [%03d/%03d]: err: %0.5f" % (
e + 1, epochs, error)
if best_error is None or error < best_error - tolerance:
best_error = error
best_epoch = e
else:
if e - best_epoch > patience:
print "fine tune: early stopping at epoch %d" % (e + 1)
break
