def test_load_data():
data = load_data()
assert len(data) == 3 # 3 data sets
train_data = list(data[0])
validation_data = list(data[1])
test_data = list(data[2])
assert len(train_data) == 50000 # 50,000 training images
assert len(validation_data) == 10000 # 10,000 testing images
assert len(test_data) == 10000 # 10,000 testing images
for data_set in [train_data, validation_data, test_data]:
assert len(
data_set[0][0]
) == 784 # feature is a vector of size 784, 28x28 pixels image flattened
assert type(data_set[0][0]) == np.ndarray
assert data_set[0][0].shape == (784, 1)
assert len(data_set[0]
[1]) == 10 # label is a one-hot encoding vector of size 10
assert type(data_set[0][1]) == np.ndarray
assert data_set[0][1].shape == (10, 1)
assert sum(
data_set[0]
[1]) == 1 # only one bit in the label vector is 1, other 9 are 0
def main():
# can use sigmoid by inputting T.nnet.sigmoid
rng = np.random.RandomState(1234)
training_data, valid_data, test_data = load_data('mnist.pkl.gz')
trainer = MLP(rng, [28 * 28, 500, 10], T.tanh)
m_b_s = 100 #mini_batch_size
n_epoch = 1000
eta = 0.01
L1_reg = 0.00
L2_reg = 0.0001
trainer.SGD(training_data, test_data, L1_reg, L2_reg, eta, n_epoch, m_b_s)
def train_cvae_mnist():
# load data
dataset = load_mnist.load_data()
X_train, y_train = dataset[0]
X_valid, y_valid = dataset[1]
y_train = onehot(y_train).astype(np.float32)
y_valid = onehot(y_valid).astype(np.float32)
y = onehot((np.arange(100)/10) % 10).astype(np.float32)
# set params
batch_size = 100
epoch = 200
rng1 = np.random.RandomState(1)
z_dim = 50
# make encoder, decoder
encoder = Sequential(n_in=X_train.shape[1]+y_train.shape[1], rng=rng1)
encoder.add(Dense(600))
encoder.add(BatchNormalization())
encoder.add(Activation('softplus'))
encoder.add(Dense(600))
encoder.add(BatchNormalization())
encoder.add(Activation('softplus'))
encoder = Encoder(network=encoder, mean_layer=Dense(z_dim), logvar_layer=Dense(z_dim), rng=rng1)
decoder = Sequential(n_in=z_dim+y_train.shape[1], rng=rng1)
decoder.add(BatchNormalization())
decoder.add(Activation('softplus'))
decoder.add(Dense(600))
decoder.add(BatchNormalization())
decoder.add(Activation('softplus'))
decoder = Decoder(rng=rng1, network=decoder, mean_layer=Dense(X_train.shape[1]))
# concat encoder and decoder, and define loss
cvae = CVAE(rng1, encoder=encoder, decoder=decoder)
opt = Adam(lr=3e-4)
cvae.compile(opt=opt)
f_encode = encoder.function(variable(X_train), variable(y), mode='pred')
f_decode = decoder.function(variable(X_train), variable(y), mode='pred')
def binarize(x):
return rng1.binomial(1, x).astype(np.float32)
train_batches = BatchIterator([X_train, y_train], batch_size, aug=[binarize, None])
# train
for i in xrange(epoch/10):
print('epoch:{0}-{1}'.format(i*10, (i+1)*10))
cvae.fit(train_batches, epoch=10)
z = f_encode(np.concatenate((X_valid[:100], y_valid[:100]), axis=1))[0]
z = np.tile(z[:10, :z_dim], (10, 1)).reshape(100, z_dim)
reconstract = f_decode(np.concatenate((z, y), axis=1))[0]
plot = 255 * np.vstack((X_valid[:10], reconstract))
saveimg(plot.reshape(110, 28, 28).astype(np.uint8), (11, 10),
'imgs/CVAE/CVAE_MNIST_analogy_epoch' + str((i+1)*10) + '.png')
def main():
dataset = load_mnist.load_data()
X_train, y_train = dataset[0]
X_valid, _ = dataset[1]
batch_size = 64
z_dim = 50
n_hidden = 500
epoch = 200
opt_gen = Adam(lr=1e-4, beta1=0.5)
opt_dis = Adam(lr=1e-4, beta1=0.5)
train_began_mnist(X_train, X_valid, z_dim, n_hidden, opt_gen, opt_dis, 'elu', epoch, batch_size)
def main():
dataset = load_mnist.load_data()
X_train, y_train = dataset[0]
batch_size = 64
z_dim = 100
n_hidden = 500
k = 5
epoch = 500
opt_gen = RMSprop(lr=0.00005, rho=0.5)
opt_dis = RMSprop(lr=0.00005, rho=0.5)
train_wgan_mnist(X_train, z_dim, n_hidden, opt_gen, opt_dis, 'relu', epoch, batch_size, k)
def main():
# can use sigmoid by inputting T.nnet.sigmoid
rng = np.random.RandomState(1234)
training_data, valid_data, test_data = load_data('mnist.pkl.gz')
trainer = MLP(rng,[28*28,500,10],T.tanh)
m_b_s = 100 #mini_batch_size
n_epoch = 1000
eta = 0.01
L1_reg = 0.00
L2_reg = 0.0001
trainer.SGD(training_data,test_data,L1_reg, L2_reg,eta,n_epoch,m_b_s)
def main():
dataset = load_mnist.load_data()
X_train, y_train = dataset[0]
batch_size = 100
z_dim = 50
n_hidden = 600
k = 1
epoch = 200*k
opt_gen = Adam(lr=2e-4, beta1=0.5)
opt_dis = Adam(lr=2e-4, beta1=0.5)
train_ebgan_mnist(X_train, z_dim, n_hidden, opt_gen, opt_dis, 'elu', epoch, batch_size, k,
use_PT=True)
def svm_baseline():
train_data, _, test_data = load_data()
train_features = [e[0].reshape(784, ) for e in train_data]
train_labels = [np.argmax(e[1].reshape(10, )) for e in train_data]
# train
clf = svm.SVC()
print("DEBUG 01 -> before fit")
clf.fit(train_features, train_labels)
print("DEBUG 02 -> after fit")
# test
test_features = [e[0].reshape(784, ) for e in test_data]
test_labels = [np.argmax(e[1].reshape(10, )) for e in test_data]
predictions = [a for a in clf.predict(test_features)]
num_correct = sum(int(a == y) for a, y in zip(predictions, test_labels))
print("Baseline classifier using an SVM.")
print("%s of %s values correct." % (num_correct, len(test_labels)))
def main():
dataset = load_mnist.load_data()
X_train, y_train = dataset[0]
X_valid, y_valid = dataset[1]
print('training M1 model...')
vae = train_vae_mnist(X_train, X_valid, z_dim=50, n_hidden=600, lr=3e-4,
activation='softplus', epoch=1000, batch_size=100)
n_l = 100
X_l = np.zeros((n_l, X_train.shape[1]), dtype=np.float32)
y_l = np.zeros((n_l, 10), dtype=np.float32)
idx = np.zeros(49900, dtype=np.int32)
start = 0
for i in range(10):
idx_i = np.where(y_train == i)[0]
X_l[i * n_l/10:(i + 1) * n_l/10] = X_train[idx_i[:n_l/10]]
idx[start:start+len(idx_i[n_l/10:])] = idx_i[n_l/10:]
y_l[i * n_l/10:(i + 1) * n_l/10, i] = 1.
start += len(idx_i[n_l/10:])
X_u = X_train[idx]
epoch = 2000
n_hidden = 500
z_dim = 50
dim = 50
alpha = 1.
mode = 'mc'
filename = ('results/SSVAE/m1m2_valid_acc_' + str(mode) +'_alpha_'
+ str(alpha) +'_.txt')
print('training M2 model...')
encoder_function = vae.encoder.function(variable(X_l), mode='sampling')
def sampler(x):
return encoder_function(x)[0]
X_valid = vae.encoder.predict(BatchIterator([X_valid], 100))
train_ssvae_mnist(X_l, y_l, X_u, X_valid, y_valid, dim, z_dim, n_hidden,
alpha, 3e-4, 'softplus', epoch,
'gaussian', sampler, mode, filename)
import numpy as np
from ml.bo import bo
import load_mnist
from ml.deeplearning.layers import Dense, Activation
from ml.deeplearning.optimizers import SGD
from ml.deeplearning.objectives import MulticlassLogLoss, L2Regularization
from ml.deeplearning.models import Sequential
if __name__ == '__main__':
dataset = load_mnist.load_data()
x_train, y_train = dataset[0]
x_test, y_test = dataset[1]
def make(params):
rng = np.random.RandomState(123)
opt = SGD(lr=params[0], momentum=params[1])
loss = [MulticlassLogLoss(), L2Regularization(params[2])]
clf = Sequential(784, rng=rng, iprint=False)
clf.add(Dense(10))
clf.add(Activation('softmax'))
clf.compile(loss=loss, opt=opt, batch_size=100, nb_epoch=100)
return clf
def eval(clf, train_x, train_y, valid_x, valid_y):
clf.fit(train_x, train_y)
score = clf.accuracy(valid_x, valid_y)
return score
intervals = [[0.001, 1.], [0.01, 1.], [0.001, 0.01]]
from load_mnist import load_data
from utils import tile_raster_images
from PIL import Image
CORRUPTION_LEVEL = 0.3
LEARNING_RATE = 0.1
TRAINING_EPOCHS = 5
BATCH_SIZE = 20
DATASET = '../Datasets/mnist.pkl.gz'
import theano.tensor as T
import timeit
if __name__ == '__main__':
datasets = load_data(DATASET)
train_set_x, train_set_y = datasets[0]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / BATCH_SIZE
np_data = train_set_x.get_value()
####################################
# BUILDING THE MODEL NO CORRUPTION #
####################################
da = DenoisingAutoencoder(n_visible=28 * 28, n_hidden=500, batch_size=BATCH_SIZE)
start_time = timeit.default_timer()
# go through training epochs
for epoch in range(TRAINING_EPOCHS):
def train_dcgan_mnist():
# load data
dataset = load_mnist.load_data()
X_train, y_train = dataset[0]
X_valid, y_valid = dataset[1]
X_train = X_train.reshape((50000, 1, 28, 28)) * 2. - 1
X_valid = X_valid.reshape((10000, 1, 28, 28)) * 2 - 1.
batch_size = 100
k = 1
n_epoch = 200
filename = 'imgs/DCGAN/DCGAN_MNIST_'
# make discriminator
rng1 = np.random.RandomState(1)
d = discriminator(rng1)
# make generator
rng2 = np.random.RandomState(1234)
g = generator(rng2)
# concat models for training generator
concat_g = Sequential(100, rng2)
concat_g.add(g)
concat_g.add(d, add_params=False)
concat_g.compile(train_loss=[CrossEntropy(),
L2Regularization(weight=1e-5)],
opt=Adam(lr=0.0002, beta1=0.5))
# make label
ones = np.ones(batch_size).astype(np.int8)
zeros = np.zeros(batch_size).astype(np.int8)
# generate first fake
for i, layer in enumerate(g.layers):
if hasattr(layer, 'moving'):
g.layers[i].moving = False
z = np.random.uniform(low=-1, high=1, size=batch_size*100)
z = z.reshape(batch_size, 100).astype(np.float32)
fake = g.predict(z)
for i, layer in enumerate(g.layers):
if hasattr(layer, 'moving'):
g.layers[i].moving = True
g.pred_function = None
z_plot = np.random.uniform(low=-1, high=1, size=100*100)
z_plot = z_plot.reshape(batch_size, 100).astype(np.float32)
# training
for i in xrange(n_epoch):
start = 0
print('epoch:{0}'.format(i+1))
X_train, y_train = utils.shuffle(X_train, y_train)
s = timeit.default_timer()
for j in xrange(50000/batch_size):
# train discriminator
d.fit_on_batch(X_train[start:start+batch_size], ones)
d.fit_on_batch(fake, zeros)
# train generator
if j % k == 0:
z = np.random.uniform(low=-1, high=1, size=batch_size*100)
z = z.reshape(batch_size, 100).astype(np.float32)
concat_g.fit_on_batch(z, ones)
# generate fake
z = np.random.uniform(low=-1, high=1, size=batch_size*100)
z = z.reshape(batch_size, 100).astype(np.float32)
fake = g.predict(z)
start += batch_size
e1 = timeit.default_timer()
utils.progbar(j+1, 50000/batch_size, e1-s)
# validation
z = np.random.uniform(low=-1, high=1, size=10000*100)
z = z.reshape(10000, 100).astype(np.float32)
fake_valid = g.predict(z)
acc_real = d.accuracy(X_valid, np.ones(10000).astype(np.int8))
sys.stdout.write(' Real ACC:{0:.3f}'.format(acc_real))
acc_fake = d.accuracy(fake_valid, np.zeros(10000).astype(np.int8))
sys.stdout.write(' Gene ACC:{0:.3f}'.format(acc_fake))
e = timeit.default_timer()
sys.stdout.write(', {0:.2f}s'.format(e-s))
sys.stdout.write('\n')
print g.layers[1].var_inf.get_value()
if (i+1) % 10 == 0:
print('generate fake...')
generation = 255.0 * (g.predict(z_plot)+1) / 2.
generation = generation.reshape(100, 28, 28)
utils.saveimg(generation, (10, 10),
filename + 'epoch' + str(i+1) + '.png')
if (i+1) % 10 == 0:
z1 = np.random.uniform(low=-1, high=1, size=100)
z2 = np.random.uniform(low=-1, high=1, size=100)
z = np.zeros((100, 100))
for j in xrange(100):
z[j] = z1 + (-z1 + z2) * j / 99.
generation = 255.0 * (g.predict(z.astype(np.float32))+1) / 2.
generation = generation.reshape(100, 28, 28)
utils.saveimg(generation, (10, 10),
filename + 'Analogy_epoch' + str(i+1) + '.png')
# tag::test_setup[] from load_mnist import load_data from network import SequentialNetwork from layers import DenseLayer, ActivationLayer training_data, test_data = load_data() # <1> """ print(type(training_data)) # list print(len(training_data)) # 60 000 print(len(test_data)) # 10 000 """ len_train_dat = len(training_data) len_test_dat = len(test_data) kilo = 1000 new_len1 = int(0.5 * kilo) # to speed up on slow PC's ;-) new_len2 = int(0.2 * kilo) training_data, test_data = training_data[:new_len1], test_data[:new_len2] net = SequentialNetwork() # <2> net.add(DenseLayer(784, 392)) # <3> net.add(ActivationLayer(392)) net.add(DenseLayer(392, 196)) net.add(ActivationLayer(196)) net.add(DenseLayer(196, 10)) net.add(ActivationLayer(10)) # <4> # <1> First, load training and test data.
import network
from load_mnist import load_data
# net = network.Network([784, 196, 49, 10])
net = network.Network([784, 30, 10])
train_data, _, test_data = load_data()
net.sgd(train_data,
epochs=30,
mini_batch_size=10,
eta=3.0,
test_data=test_data)
import numpy as np
from load_mnist import load_data
from layers import sigmoid_double
def average_digit(data, digit):
filtered_data = [x[0] for x in data if np.argmax(x[1]) == digit]
filtered_array = np.asarray(filtered_data)
return np.average(filtered_array, axis=0)
train, test = load_data()
avg_eight = average_digit(train, 8)
from matplotlib import pyplot as plt
img = (np.reshape(avg_eight, (28, 28)))
plt.imshow(img)
# plt.show()
x_3 = train[2][0] # 4
x_18 = train[17][0] # 8
W = np.transpose(avg_eight)
np.dot(W, x_3)
np.dot(W, x_18)
def predict(x, W, b):
return sigmoid_double(np.dot(W, x) + b)
__author__ = 'sorensonderby'
import numpy as np
from load_mnist import load_data
from PIL import Image
import scipy.io as sio
import matplotlib
import matplotlib.pyplot as plt
import os
ORG_SHP = [28, 28]
OUT_SHP = [100, 100]
NUM_DISTORTIONS = 8
dist_size = (9, 9) # should be odd?
NUM_DISTORTIONS_DB = 100000
mnist_data = load_data('MNIST/mnist.pkl.gz')
outfile = "mnist_sequence3_sample_8distortions9x9"
np.random.seed(1234)
### create list with distortions
all_digits = np.concatenate([mnist_data['X_train'], mnist_data['X_valid']],
axis=0)
all_digits = all_digits.reshape([-1] + ORG_SHP)
num_digits = all_digits.shape[0]
distortions = []
for i in range(NUM_DISTORTIONS_DB):
rand_digit = np.random.randint(num_digits)
rand_x = np.random.randint(ORG_SHP[1] - dist_size[1])
rand_y = np.random.randint(ORG_SHP[0] - dist_size[0])
__author__ = 'sorensonderby'
import numpy as np
from load_mnist import load_data
from PIL import Image
import scipy.io as sio
import matplotlib
import matplotlib.pyplot as plt
import os
ORG_SHP = [28, 28]
OUT_SHP = [100, 100]
NUM_DISTORTIONS = 8
dist_size = (9, 9) # should be odd?
NUM_DISTORTIONS_DB = 100000
mnist_data = load_data('../MNIST/mnist.pkl.gz')
outfile = "mnist_sequence3_sample_8dsistortions9x9"
np.random.seed(1234)
### create list with distortions
all_digits = np.concatenate([mnist_data['X_train'],
mnist_data['X_valid'],
mnist_data['X_train']], axis=0)
all_digits = all_digits.reshape([-1] + ORG_SHP)
num_digits = all_digits.shape[0]
distortions = []
for i in range(NUM_DISTORTIONS_DB):
rand_digit = np.random.randint(num_digits)
import numpy as np
from load_mnist import load_data
from PIL import Image
import scipy.io as sio
import matplotlib
import matplotlib.pyplot as plt
import os
ORG_SHP = [28, 28]
OUT_SHP = [40, 40]
NUM_DISTORTIONS = 5
dist_size = (5, 5) # should be odd?
NUM_DISTORTIONS_DB = 100000
mnist_data = load_data(
'./mnist.pkl.gz', "https://s3.amazonaws.com/img-datasets/mnist.pkl.gz")
outfile = "mnist_sequence3_sample_5dsistortions5x5"
np.random.seed(1234)
# create list with distortions
all_digits = np.concatenate([mnist_data['X_train'],
mnist_data['X_valid']], axis=0)
all_digits = all_digits.reshape([-1] + ORG_SHP)
num_digits = all_digits.shape[0]
distortions = []
for i in range(NUM_DISTORTIONS_DB):
rand_digit = np.random.randint(num_digits)
import numpy as np
from scipy.optimize import minimize
from initial_params import initial_params
from sparse_autoencoder_cost import sparse_autoencoder_cost, sigmoid, der_sigmoid
from load_mnist import generate_patch, load_data
from display_network import display_network
visible_size = 28 * 28
hidden_size = 196
sparsity_param = 0.1
lamda = 0.003
beta = 3
images = np.transpose(load_data())[:, 0:10000]
patches = generate_patch()
theta = initial_params(visible_size, hidden_size)
J = lambda th: sparse_autoencoder_cost(
visible_size, hidden_size, th, lambda x: sigmoid(x), lambda x: der_sigmoid(x), lamda, beta, sparsity_param, images
)
options_ = {"maxiter": 800, "disp": True}
result = minimize(J, theta, method="L-BFGS-B", jac=True, options=options_)
opt_theta = result.x
print result
W1 = opt_theta[0 : hidden_size * visible_size].reshape(hidden_size, visible_size).transpose()
h = dropout(h, p_drop_hidden)
h2 = rectify(T.dot(h, w_h2))
h2 = dropout(h2, p_drop_hidden)
h3 = rectify(T.dot(h2, w_h3))
h3 = dropout(h3, p_drop_hidden)
h4 = rectify(T.dot(h3, w_h4))
h4 = dropout(h4, p_drop_hidden)
py_x = softmax(T.dot(h4, w_o))
return h, h2, h3, h4, py_x
(trX, trY), (teX, teY), num_of_class = load_mnist.load_data()
row, col = np.shape(trX)
# col = c * x * y
rank = num_of_class / 2
hidden = (col + num_of_class) * 2 / 3
print "Num of hidden neurons:", hidden
print "Num of classes:", num_of_class
print "Rank:", rank
X = T.fmatrix()
Y = T.fmatrix()
w_h = init_weights((col, hidden)) # 784 <- input
w_h2 = init_weights((hidden, hidden))
w_h3 = init_weights((hidden, hidden))
w_h4 = init_weights((hidden, hidden))
w_o = init_weights((hidden, num_of_class)) # 10 <- output
import network2
from load_mnist import load_data
# net = network2.Network([784, 196, 49, 10])
net = network2.Network([784, 30, 10])
train_data, validation_data, test_data = load_data()
net.sgd(train_data,
epochs=30,
mini_batch_size=10,
eta=0.5,
lmbda=1.0E-4,
evaluation_data=test_data,
monitor_evaluation_cost=True,
monitor_evaluation_accuracy=True,
monitor_training_cost=True,
monitor_training_accuracy=True,
monitor_weight_stats=True)
# tag::test_setup[]
import load_mnist
import network
from layers import DenseLayer, ActivationLayer
training_data, test_data = load_mnist.load_data() # <1>
net = network.SequentialNetwork() # <2>
net.add(DenseLayer(784, 392)) # <3>
net.add(ActivationLayer(392))
net.add(DenseLayer(392, 196))
net.add(ActivationLayer(196))
net.add(DenseLayer(196, 10))
net.add(ActivationLayer(10)) # <4>
# <1> First, load training and test data.
# <2> Next, initialize a sequential neural network.
# <3> You can then add dense and activation layers one by one.
# <4> The final layer has size 10, the number of classes to predict.
# end::test_setup[]
# tag::test_run[]
net.train(training_data,
epochs=10,
mini_batch_size=10,
learning_rate=3.0,
test_data=test_data) # <1>
# <1> You can now easily train the model by specifying train and test data, the number of epochs, the mini-batch size and the learning rate.
# end::test_run[]
from ml.deeplearning.models import Sequential
from ml.utils import visualize, BatchIterator
def autoencoder(n_in, n_hidden=500, rng=np.random.RandomState(1234),
activations=['relu', 'sigmoid']):
clf = Sequential(n_in, rng)
clf.add(Dense(n_hidden))
clf.add(Activation(activations[0]))
clf.add(Decoder(clf.layers[0]))
clf.add(Activation(activations[1]))
return clf
if __name__ == '__main__':
dataset = load_data()
X_train, y_train = dataset[0]
X_valid, y_valid = dataset[1]
X_test, y_test = dataset[2]
opt = SGD(lr=0.01, momentum=0.9)
loss = SquaredError()
clf = autoencoder(28*28)
clf.compile(opt=opt, train_loss=loss)
train_batches = BatchIterator([X_train, X_train], batch_size=128,
shuffle=True)
valid_batches = BatchIterator([X_valid, X_valid], batch_size=128,
shuffle=False)
clf.fit(train_batches, 100, valid_batches)