def pca(dataset):
mean = dataset[:3000].mean(axis=0)
theanets.log('computing whitening transform')
x = dataset[:3000] - mean
vals, vecs = np.linalg.eigh(np.dot(x.T, x) / len(x))
vals = vals[::-1]
vecs = vecs[:, ::-1]
vals = np.sqrt(vals[:K])
vecs = vecs[:, :K]
def whiten(x):
return np.dot(x, np.dot(vecs, np.diag(1. / vals)))
def color(z):
return np.dot(z, np.dot(np.diag(vals), vecs.T))
return whiten, color
def pca(dataset):
mean = dataset[:3000].mean(axis=0)
theanets.log('computing whitening transform')
x = dataset[:3000] - mean
vals, vecs = np.linalg.eigh(np.dot(x.T, x) / len(x))
vals = vals[::-1]
vecs = vecs[:, ::-1]
vals = np.sqrt(vals[:K])
vecs = vecs[:, :K]
def whiten(x):
return np.dot(x, np.dot(vecs, np.diag(1. / vals)))
def color(z):
return np.dot(z, np.dot(np.diag(vals), vecs.T))
return whiten, color
#!/usr/bin/env python
import numpy.random as rng
import theanets
TIME = 10
BATCH_SIZE = 32
e = theanets.Experiment(theanets.recurrent.Autoencoder,
layers=(3, ('rnn', 10), 3),
batch_size=BATCH_SIZE)
def generate():
return [rng.randn(TIME, BATCH_SIZE, 3).astype('f')]
batch = generate()
theanets.log('data batches: {}', batch[0].shape)
e.train(generate)
import numpy.random as rng
import theanets
BATCH_SIZE = 32
STEPS = 20
weight = np.zeros((STEPS, BATCH_SIZE, 1), 'f')
weight[-1:] = 1
def examples():
x, z = rng.uniform(0, 1, size=(2, STEPS, BATCH_SIZE, 1))
y = np.zeros((STEPS, BATCH_SIZE, 1))
idx = list(range(STEPS - 1))
for b in range(BATCH_SIZE):
rng.shuffle(idx)
y[idx[0], b] = 1
y[idx[1], b] = 1
z[-1, b] = x[idx[0], b] + x[idx[1], b]
return np.concatenate([x, y], axis=2).astype('f'), z.astype('f'), weight
src, tgt, wgt = examples()
theanets.log('data batches: {} -> {} @ {}', src.shape, tgt.shape, wgt.shape)
e = theanets.Experiment(
theanets.recurrent.Regressor,
layers=(2, dict(form='rnn', activation='relu', size=100, radius=1), 1),
weighted=True)
e.train(examples)
prd = e.network.transform(src)
wave_ax.plot(T, SIN, ':', label='Target', alpha=0.7, color='#111111')
# For each layer type, train a model containing that layer, and plot its
# predicted output.
for i, layer in enumerate((
dict(form='rnn', activation='linear', diagonal=0.5),
dict(form='rnn', activation='relu', diagonal=0.5),
dict(form='rrnn', activation='relu', rate='vector', diagonal=0.5),
dict(form='scrn', activation='elu'),
dict(form='gru', activation='relu'),
dict(form='lstm', activation='tanh'),
dict(form='clockwork', activation='linear', periods=(1, 4, 16, 64)))):
name = '{form}+{activation}'.format(**layer)
layer['size'] = 64
theanets.log('training {} model', name)
net = theanets.recurrent.Regressor([1, layer, 1])
losses = []
for tm, _ in net.itertrain([ZERO, WAVES],
monitor_gradients=True,
batch_size=BATCH_SIZE,
algorithm='rmsprop',
learning_rate=0.0001,
momentum=0.9,
min_improvement=0.01):
losses.append(tm['loss'])
prd = net.predict(ZERO)
wave_ax.plot(T, prd[0, :, 0].flatten(), label=name, alpha=0.7, color=COLORS[i])
learn_ax.plot(losses, label=name, alpha=0.7, color=COLORS[i])
#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''Example using the theanets package for learning the XOR relation.'''
import numpy as np
import theanets
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]], dtype='f')
Y = np.array([[0], [1], [1], [0]], dtype='f')
net = theanets.Regressor([dict(size=2, input_noise=0.3), 2, 1])
net.train([X, Y], algo='rmsprop', patience=10, batch_size=4)
theanets.log('Input: {}', [list(x) for x in X])
theanets.log('XOR output: {}', Y.T)
theanets.log('NN XOR predictions: {}', net.predict(X.astype('f')).T.round(2))
class WeightInverse(theanets.Regularizer):
def loss(self, layers, outputs):
return sum((1 / (w * w).sum(axis=0)).sum()
for l in layers for w in l.params
if w.ndim > 1)
(train, ), (valid, ), _ = load_mnist()
# mean-center the digits and compute a pca whitening transform.
m = train.mean(axis=0)
train -= m
valid -= m
theanets.log('computing whitening transform')
vals, vecs = np.linalg.eigh(np.dot(train.T, train) / len(train))
vals = vals[::-1]
vecs = vecs[:, ::-1]
K = 197 # this retains 99% of the variance in the digit data.
vals = np.sqrt(vals[:K])
vecs = vecs[:, :K]
def whiten(x):
return np.dot(x, np.dot(vecs, np.diag(1. / vals)))
def color(z):
return np.dot(z, np.dot(np.diag(vals), vecs.T))
)
weights = np.ones_like(labels)
weights[labels.nonzero()] *= 10
def split(a, b):
return [samples[a:b].astype('float32'),
labels[a:b].astype('int32'),
weights[a:b].astype('float32')]
train = split(0, 9000)
valid = split(9000, 10000)
net = theanets.Classifier(
layers=(100, 10, 2),
weighted=True,
)
net.train(train, valid)
truth = valid[1]
theanets.log('# of true 1s: {}', truth.sum())
guess = net.predict(valid[0])
theanets.log('# of predicted 1s: {}', guess.sum())
cm = sklearn.metrics.confusion_matrix(truth, guess)
theanets.log('confusion matrix (true class = rows, predicted class = cols):')
theanets.log(str(cm))
weights = np.ones_like(labels)
weights[labels.nonzero()] *= 10
def split(a, b):
return [
samples[a:b].astype('float32'), labels[a:b].astype('int32'),
weights[a:b].astype('float32')
]
train = split(0, 9000)
valid = split(9000, 10000)
net = theanets.Classifier(
layers=(100, 10, 2),
weighted=True,
)
net.train(train, valid)
truth = valid[1]
theanets.log('# of true 1s: {}', truth.sum())
guess = net.predict(valid[0])
theanets.log('# of predicted 1s: {}', guess.sum())
cm = sklearn.metrics.confusion_matrix(truth, guess)
theanets.log('confusion matrix (true class = rows, predicted class = cols):')
theanets.log(str(cm))
import numpy as np
import theanets
import scipy.io
import os
import tempfile
import urllib
import zipfile
BATCH_SIZE = 32
TRAIN_NC = os.path.join(tempfile.gettempdir(), 'chime1_train.nc')
VALID_NC = os.path.join(tempfile.gettempdir(), 'chime1_valid.nc')
ZIPURL = 'https://github.com/craffel/lstm_benchmarks/archive/master.zip'
# If needed, get the data files from https://github.com/craffel/lstm_benchmarks.
if not os.path.isfile(TRAIN_NC) or not os.path.isfile(VALID_NC):
theanets.log('attempting data copy from url: {}', ZIPURL)
z = zipfile.ZipFile(io.BytesIO(urllib.urlopen(ZIPURL).read()))
with open(TRAIN_NC, 'wb') as savefile:
savefile.write(z.read('lstm_benchmarks-master/data/train_1_speaker.nc'))
with open(VALID_NC, 'wb') as savefile:
savefile.write(z.read('lstm_benchmarks-master/data/val_1_speaker.nc'))
z.close()
def batch_at(features, labels, seq_begins, seq_lengths):
'''Extract a single batch of data to pass to the model being trained.
Parameters
----------
features, labels : ndarray
Arrays of the input features and target labels.
STEPS = 20
weight = np.zeros((STEPS, BATCH_SIZE, 1), 'f')
weight[-1:] = 1
def examples():
x, z = rng.uniform(0, 1, size=(2, STEPS, BATCH_SIZE, 1))
y = np.zeros((STEPS, BATCH_SIZE, 1))
idx = list(range(STEPS - 1))
for b in range(BATCH_SIZE):
rng.shuffle(idx)
y[idx[0], b] = 1
y[idx[1], b] = 1
z[-1, b] = x[idx[0], b] + x[idx[1], b]
return np.concatenate([x, y], axis=2).astype('f'), z.astype('f'), weight
src, tgt, wgt = examples()
theanets.log('data batches: {} -> {} @ {}', src.shape, tgt.shape, wgt.shape)
e = theanets.Experiment(theanets.recurrent.Regressor,
layers=(2,
dict(form='rnn',
activation='relu',
size=100,
radius=1), 1),
weighted=True)
e.train(examples)
prd = e.network.transform(src)
#!/usr/bin/env python
import numpy.random as rng
import theanets
TIME = 10
BATCH_SIZE = 32
e = theanets.Experiment(
theanets.recurrent.Autoencoder,
layers=(3, ('rnn', 10), 3),
batch_size=BATCH_SIZE)
def generate():
return [rng.randn(TIME, BATCH_SIZE, 3).astype('f')]
batch = generate()
theanets.log('data batches: {}', batch[0].shape)
e.train(generate)