def pretrain(self, batches, test_images, n_epochs=10, **train_params):
data = tt.matrix('data', dtype=self.dtype)
cost, updates = self.get_cost_updates(data, **train_params)
train_rbm = theano.function([data], cost, updates=updates)
for epoch in range(n_epochs):
# train on each mini-batch
costs = []
for batch in batches:
costs.append(train_rbm(batch))
print "Epoch %d: %0.3f" % (epoch, np.mean(costs))
# plot reconstructions on test set
plt.figure(2)
plt.clf()
x = test_images
y = rbm.encode(test_images)
z = rbm.decode(y)
plotting.compare([x.reshape(-1, 28, 28),
z.reshape(-1, 28, 28)],
rows=5,
cols=20,
vlims=(-1, 2) if GAUSSIAN else (0, 1))
plt.draw()
print "Test error:", rms(x - z, axis=1).mean()
# plot filters for first layer only
plt.figure(3)
plt.clf()
plotting.filters(self.filters, rows=10, cols=20)
plt.draw()
def pretrain(self, batches, test_images, n_epochs=10, **train_params):
data = tt.matrix("data", dtype=self.dtype)
cost, updates = self.get_cost_updates(data, **train_params)
train_rbm = theano.function([data], cost, updates=updates)
for epoch in range(n_epochs):
# train on each mini-batch
costs = []
for batch in batches:
costs.append(train_rbm(batch))
print "Epoch %d: %0.3f" % (epoch, np.mean(costs))
# plot reconstructions on test set
plt.figure(2)
plt.clf()
x = test_images
y = rbm.encode(test_images)
z = rbm.decode(y)
plotting.compare(
[x.reshape(-1, 28, 28), z.reshape(-1, 28, 28)], rows=5, cols=20, vlims=(-1, 2) if GAUSSIAN else (0, 1)
)
plt.draw()
print "Test error:", rms(x - z, axis=1).mean()
# plot filters for first layer only
plt.figure(3)
plt.clf()
plotting.filters(self.filters, rows=10, cols=20)
plt.draw()
def pretrain(self, batches, dbn=None, test_images=None,
n_epochs=10, **train_params):
data = tt.matrix('data', dtype=self.dtype)
cost, updates = self.get_cost_updates(data, **train_params)
train_rbm = theano.function([data], cost, updates=updates)
for epoch in range(n_epochs):
# train on each mini-batch
costs = []
for batch in batches:
costs.append(train_rbm(batch))
print "Epoch %d: %0.3f" % (epoch, np.mean(costs))
if dbn is not None and test_images is not None:
# plot reconstructions on test set
plt.figure(2)
plt.clf()
recons = dbn.reconstruct(test_images)
plotting.compare([test_images.reshape(-1, 28, 28),
recons.reshape(-1, 28, 28)],
rows=5, cols=20)
plt.draw()
# plot filters for first layer only
if dbn is not None and self is dbn.rbms[0]:
plt.figure(3)
plt.clf()
plotting.filters(self.filters, rows=10, cols=20)
plt.draw()
def sgd_backprop(self, images, test_images, batch_size=100, rate=0.1, n_epochs=10):
dtype = theano.config.floatX
params = [self.W, self.c, self.b]
# --- compute backprop function
x = tt.matrix('images')
xn = x + self.theano_rng.normal(size=x.shape, std=1, dtype=dtype)
# compute coding error
y = self.propup(xn)
z = self.propdown(y)
rmses = tt.sqrt(tt.mean((x - z)**2, axis=1))
error = tt.mean(rmses)
# compute gradients
grads = tt.grad(error, params)
updates = collections.OrderedDict()
for param, grad in zip(params, grads):
updates[param] = param - tt.cast(rate, dtype) * grad
if self.mask is not None:
updates[self.W] = updates[self.W] * self.mask
train_dbn = theano.function([x], error, updates=updates)
# --- perform SGD
batches = images.reshape(-1, batch_size, images.shape[1])
assert np.isfinite(batches).all()
for epoch in range(n_epochs):
costs = []
for batch in batches:
costs.append(train_dbn(batch))
self.check_params()
print "Epoch %d: %0.3f" % (epoch, np.mean(costs))
# plot reconstructions on test set
plt.figure(2)
plt.clf()
x = test_images
y = self.encode(test_images)
z = self.decode(y)
plotting.compare(
[x.reshape(-1, 28, 28), z.reshape(-1, 28, 28)],
rows=5, cols=20, vlims=(-1, 2))
plt.draw()
print "Test error:", rms(x - z, axis=1).mean()
# plot filters for first layer only
plt.figure(3)
plt.clf()
plotting.filters(self.filters, rows=10, cols=20)
plt.draw()
def test(x):
# test error
# x = test_images[:200]
a = encode(x)
xhat = np.dot(a, decoders)
plt.figure(99)
plt.clf()
plotting.compare(
[x.reshape(-1, 28, 28), xhat.reshape(-1, 28, 28)],
rows=5, cols=20, vlims=(-1, 1))
plt.draw()
print "error", rms(xhat - x, axis=1).mean()
def test(x):
# test error
# x = test_images[:200]
a = encode(x)
xhat = np.dot(a, decoders)
plt.figure(99)
plt.clf()
plotting.compare([x.reshape(-1, 28, 28),
xhat.reshape(-1, 28, 28)],
rows=5,
cols=20,
vlims=(-1, 1))
plt.draw()
print "error", rms(xhat - x, axis=1).mean()
def pretrain(self,
batches,
dbn=None,
test_images=None,
n_epochs=10,
**train_params):
data = tt.matrix('data', dtype=self.dtype)
cost, updates = self.get_cost_updates(data, **train_params)
train_rbm = theano.function([data], cost, updates=updates)
for epoch in range(n_epochs):
# train on each mini-batch
costs = []
for batch in batches:
costs.append(train_rbm(batch))
print "Epoch %d: %0.3f" % (epoch, np.mean(costs))
if dbn is not None and test_images is not None:
# plot reconstructions on test set
plt.figure(2)
plt.clf()
recons = dbn.reconstruct(test_images)
plotting.compare([
test_images.reshape(-1, 28, 28),
recons.reshape(-1, 28, 28)
],
rows=5,
cols=20)
plt.draw()
# plot filters for first layer only
if dbn is not None and self is dbn.rbms[0]:
plt.figure(3)
plt.clf()
plotting.filters(self.filters, rows=10, cols=20)
plt.draw()
cost, updates = rbm.get_cost_updates(lr=0.02, persistent=persistent, k=1)
train_rbm = theano.function([rbm.input], cost,
updates=updates)
ha, hp = rbm.propup(rbm.input)
va, vp = rbm.propdown(hp)
reconstruct_rbm = theano.function([rbm.input], vp)
for epoch in range(n_epochs):
costs = []
for batch in batches:
costs.append(train_rbm(batch))
print "Epoch %d: %0.3f" % (epoch, np.mean(costs))
weights = rbm.W.get_value()
plt.figure(2)
plt.clf()
plotting.filters(weights.T.reshape(-1, 28, 28), rows=5, cols=10)
test_batch = batches[0]
recons = reconstruct_rbm(test_batch)
plt.figure(3)
plt.clf()
plotting.compare([test_batch.reshape(-1, 28, 28), recons.reshape(-1, 28, 28)], rows=5, cols=20)
plt.draw()
def show_recons(x, z):
plotting.compare([x.reshape(-1, 28, 28), z.reshape(-1, 28, 28)],
rows=5, cols=20, vlims=(-1, 2))
def show_recons(x, z):
plotting.compare(
[x.reshape(-1, 28, 28), z.reshape(-1, 28, 28)],
rows=5,
cols=20,
vlims=(-1, 2))
rbm = RBM(shapes[i], shapes[i+1], rf_shape=rf_shapes[i])
rbm.statistical_encoders(data)
rbm.pretrain(data)
# rbm.backprop(data)
data = rbm.encode(data)
dbn.rbms.append(rbm)
rmses = dbn.test_reconstruction(valid_images)
print "RBM %d error: %0.3f (%0.3f)" % (i, rmses.mean(), rmses.std())
plt.figure(99)
plt.clf()
recons = dbn.reconstruct(test_images)
plotting.compare(
[test_images.reshape(-1, 28, 28), recons.reshape(-1, 28, 28)],
rows=5, cols=20, vlims=(-1, 1))
plt.show()
# plt.savefig('nef.png')
dbn.backprop(train_images[:10000], n_epochs=10)
plt.figure(199)
plt.clf()
recons = dbn.reconstruct(test_images)
plotting.compare(
[test_images.reshape(-1, 28, 28), recons.reshape(-1, 28, 28)],
rows=5, cols=20, vlims=(-1, 1))
plt.show()
rbm.statistical_encoders(data)
rbm.pretrain(data)
# rbm.backprop(data)
data = rbm.encode(data)
dbn.rbms.append(rbm)
rmses = dbn.test_reconstruction(valid_images)
print "RBM %d error: %0.3f (%0.3f)" % (i, rmses.mean(), rmses.std())
plt.figure(99)
plt.clf()
recons = dbn.reconstruct(test_images)
plotting.compare([test_images.reshape(-1, 28, 28),
recons.reshape(-1, 28, 28)],
rows=5,
cols=20,
vlims=(-1, 1))
plt.show()
# plt.savefig('nef.png')
dbn.backprop(train_images[:10000], n_epochs=10)
plt.figure(199)
plt.clf()
recons = dbn.reconstruct(test_images)
plotting.compare([test_images.reshape(-1, 28, 28),
recons.reshape(-1, 28, 28)],
rows=5,
cols=20,
vlims=(-1, 1))
def sgd_backprop(self,
images,
test_images,
batch_size=100,
rate=0.1,
n_epochs=10):
dtype = theano.config.floatX
params = [self.W, self.c, self.b]
# --- compute backprop function
x = tt.matrix('images')
xn = x + self.theano_rng.normal(size=x.shape, std=1, dtype=dtype)
# compute coding error
y = self.propup(xn)
z = self.propdown(y)
rmses = tt.sqrt(tt.mean((x - z)**2, axis=1))
error = tt.mean(rmses)
# compute gradients
grads = tt.grad(error, params)
updates = collections.OrderedDict()
for param, grad in zip(params, grads):
updates[param] = param - tt.cast(rate, dtype) * grad
if self.mask is not None:
updates[self.W] = updates[self.W] * self.mask
train_dbn = theano.function([x], error, updates=updates)
# --- perform SGD
batches = images.reshape(-1, batch_size, images.shape[1])
assert np.isfinite(batches).all()
for epoch in range(n_epochs):
costs = []
for batch in batches:
costs.append(train_dbn(batch))
self.check_params()
print "Epoch %d: %0.3f" % (epoch, np.mean(costs))
# plot reconstructions on test set
plt.figure(2)
plt.clf()
x = test_images
y = self.encode(test_images)
z = self.decode(y)
plotting.compare([x.reshape(-1, 28, 28),
z.reshape(-1, 28, 28)],
rows=5,
cols=20,
vlims=(-1, 2))
plt.draw()
print "Test error:", rms(x - z, axis=1).mean()
# plot filters for first layer only
plt.figure(3)
plt.clf()
plotting.filters(self.filters, rows=10, cols=20)
plt.draw()